diff options
726 files changed, 20336 insertions, 7811 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index f5874744b5..2a3cb1c6d9 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,12 +1,9 @@ env: # Global defaults CIRRUS_CLONE_DEPTH: 1 - PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" + CIRRUS_LOG_TIMESTAMP: true MAKEJOBS: "-j10" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache CI_FAILFAST_TEST_LEAVE_DANGLING: "1" # Cirrus CI does not care about dangling processes and setting this variable avoids killing the CI script itself on error - CCACHE_MAXSIZE: "200M" - CCACHE_DIR: "/tmp/ccache_dir" - CCACHE_NOHASHDIR: "1" # Debug info might contain a stale path if the build dir changes, but this is fine # A self-hosted machine(s) can be used via Cirrus CI. It can be configured with # multiple users to run tasks in parallel. No sudo permission is required. @@ -16,9 +13,9 @@ env: # Global defaults # Generally, a persistent worker must run Ubuntu 23.04+ or Debian 12+. # # The following specific types should exist, with the following requirements: -# - small: For an x86_64 machine, recommended to have 2 CPUs and 8 GB of memory. -# - medium: For an x86_64 machine, recommended to have 4 CPUs and 16 GB of memory. -# - arm64: For an aarch64 machine, recommended to have 2 CPUs and 8 GB of memory. +# - small: For an x86_64 machine, with at least 2 vCPUs and 8 GB of memory. +# - medium: For an x86_64 machine, with at least 4 vCPUs and 16 GB of memory. +# - arm64: For an aarch64 machine, with at least 2 vCPUs and 8 GB of memory. # # CI jobs for the latter configuration can be run on x86_64 hardware # by installing qemu-user-static, which works out of the box with @@ -39,14 +36,13 @@ env: # Global defaults # This requires installing Podman instead of Docker. # # Futhermore: -# - apt-get is required due to PACKAGE_MANAGER_INSTALL # - podman-docker-4.1+ is required due to the bugfix in 4.1 # (https://github.com/bitcoin/bitcoin/pull/21652#issuecomment-1657098200) # - The ./ci/ dependencies (with cirrus-cli) should be installed. One-liner example # for a single user setup with sudo permission: # # ``` -# apt update && apt install git screen python3 bash podman-docker curl -y && curl -L -o cirrus "https://github.com/cirruslabs/cirrus-cli/releases/latest/download/cirrus-linux-$(dpkg --print-architecture)" && mv cirrus /usr/local/bin/cirrus && chmod +x /usr/local/bin/cirrus +# apt update && apt install git screen python3 bash podman-docker uidmap slirp4netns curl -y && curl -L -o cirrus "https://github.com/cirruslabs/cirrus-cli/releases/latest/download/cirrus-linux-$(dpkg --print-architecture)" && mv cirrus /usr/local/bin/cirrus && chmod +x /usr/local/bin/cirrus # ``` # # - There are no strict requirements on the hardware. Having fewer CPU threads @@ -75,8 +71,8 @@ filter_template: &FILTER_TEMPLATE base_template: &BASE_TEMPLATE << : *FILTER_TEMPLATE merge_base_script: - # Unconditionally install git (used in fingerprint_script). - - git --version || bash -c "$PACKAGE_MANAGER_INSTALL git" + # Require git (used in fingerprint_script). + - git --version || ( apt-get update && apt-get install -y git ) - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi - git fetch --depth=1 $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge" - git checkout FETCH_HEAD # Use merged changes to detect silent merge conflicts @@ -135,7 +131,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_arm.sh" task: - name: 'Win64, unit tests, no gui tests, no functional tests' + name: 'Win64-cross' << : *GLOBAL_TASK_TEMPLATE persistent_worker: labels: diff --git a/.editorconfig b/.editorconfig index ae7e92d1c8..c5f3028c50 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,17 +10,17 @@ insert_final_newline = true trim_trailing_whitespace = true # Source code files -[*.{h,cpp,py,sh}] +[*.{h,cpp,rs,py,sh}] indent_size = 4 -# .cirrus.yml, .fuzzbuzz.yml, etc. +# .cirrus.yml, etc. [*.yml] indent_size = 2 -# Makefiles -[{*.am,Makefile.*.include}] +# Makefiles (only relevant for depends build) +[Makefile] indent_style = tab -# Autoconf scripts -[configure.ac] +# CMake files +[{CMakeLists.txt,*.cmake,*.cmake.in}] indent_size = 2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa4b04b5e6..740d31ae56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,27 +67,42 @@ jobs: echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD $EXCLUDE_MERGE_BASE_ANCESTORS | head -1)" >> "$GITHUB_ENV" - run: | sudo apt-get update - sudo apt-get install clang ccache build-essential cmake pkg-config python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y + sudo apt-get install clang ccache build-essential cmake pkg-config python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libzmq3-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y - name: Compile and run tests run: | # Run tests on commits after the last merge commit and before the PR head commit # Use clang++, because it is a bit faster and uses less memory than g++ - git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && CC=clang CXX=clang++ cmake -B build -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWITH_BDB=ON -DWITH_NATPMP=ON -DWITH_MINIUPNPC=ON -DWITH_USDT=ON && cmake --build build -j $(nproc) && ctest --test-dir build -j $(nproc) && ./build/test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.TEST_BASE }} + git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && CC=clang CXX=clang++ cmake -B build -DWERROR=ON -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWITH_BDB=ON -DWITH_USDT=ON -DCMAKE_CXX_FLAGS='-Wno-error=unused-member-function' && cmake --build build -j $(nproc) && ctest --output-on-failure --stop-on-failure --test-dir build -j $(nproc) && ./build/test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.TEST_BASE }} - macos-native-x86_64: - name: 'macOS 13 native, x86_64, no depends, sqlite only, gui' + macos-native-arm64: + name: ${{ matrix.job-name }} # Use latest image, but hardcode version to avoid silent upgrades (and breaks). # See: https://github.com/actions/runner-images#available-images. - runs-on: macos-13 + runs-on: macos-14 - # No need to run on the read-only mirror, unless it is a PR. - if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' + # When a contributor maintains a fork of the repo, any pull request they make + # to their own fork, or to the main repository, will trigger two CI runs: + # one for the branch push and one for the pull request. + # This can be avoided by setting SKIP_BRANCH_PUSH=true as a custom env variable + # in Github repository settings. + if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + job-type: [standard, fuzz] + include: + - job-type: standard + file-env: './ci/test/00_setup_env_mac_native.sh' + job-name: 'macOS 14 native, arm64, no depends, sqlite only, gui' + - job-type: fuzz + file-env: './ci/test/00_setup_env_mac_native_fuzz.sh' + job-name: 'macOS 14 native, arm64, fuzz' + env: DANGER_RUN_CI_ON_HOST: 1 - FILE_ENV: './ci/test/00_setup_env_mac_native.sh' BASE_ROOT_DIR: ${{ github.workspace }} steps: @@ -105,7 +120,7 @@ jobs: run: | # A workaround for "The `brew link` step did not complete successfully" error. brew install --quiet python@3 || brew link --overwrite python@3 - brew install --quiet automake libtool pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq qt@5 qrencode + brew install --quiet coreutils ninja pkgconf gnu-getopt ccache boost libevent zeromq qt@5 qrencode - name: Set Ccache directory run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" @@ -115,11 +130,13 @@ jobs: uses: actions/cache/restore@v4 with: path: ${{ env.CCACHE_DIR }} - key: ${{ github.job }}-ccache-${{ github.run_id }} - restore-keys: ${{ github.job }}-ccache- + key: ${{ github.job }}-${{ matrix.job-type }}-ccache-${{ github.run_id }} + restore-keys: ${{ github.job }}-${{ matrix.job-type }}-ccache- - name: CI script run: ./ci/test_run_all.sh + env: + FILE_ENV: ${{ matrix.file-env }} - name: Save Ccache cache uses: actions/cache/save@v4 @@ -127,21 +144,32 @@ jobs: with: path: ${{ env.CCACHE_DIR }} # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache - key: ${{ github.job }}-ccache-${{ github.run_id }} + key: ${{ github.job }}-${{ matrix.job-type }}-ccache-${{ github.run_id }} win64-native: - name: 'Win64 native, VS 2022' + name: ${{ matrix.job-name }} # Use latest image, but hardcode version to avoid silent upgrades (and breaks). # See: https://github.com/actions/runner-images#available-images. runs-on: windows-2022 - # No need to run on the read-only mirror, unless it is a PR. - if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' + if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} env: PYTHONUTF8: 1 TEST_RUNNER_TIMEOUT_FACTOR: 40 + strategy: + fail-fast: false + matrix: + job-type: [standard, fuzz] + include: + - job-type: standard + generate-options: '-DBUILD_GUI=ON -DWITH_BDB=ON -DWITH_ZMQ=ON -DBUILD_BENCH=ON -DWERROR=ON' + job-name: 'Win64 native, VS 2022' + - job-type: fuzz + generate-options: '-DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON -DVCPKG_MANIFEST_FEATURES="sqlite" -DBUILD_GUI=OFF -DBUILD_FOR_FUZZING=ON -DWERROR=ON' + job-name: 'Win64 native fuzz, VS 2022' + steps: - name: Checkout uses: actions/checkout@v4 @@ -182,11 +210,11 @@ jobs: - name: Generate build system run: | - cmake -B build --preset vs2022-static -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_INSTALLATION_ROOT\scripts\buildsystems\vcpkg.cmake" -DBUILD_GUI=ON -DWITH_BDB=ON -DWITH_MINIUPNPC=ON -DWITH_ZMQ=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWERROR=ON + cmake -B build --preset vs2022-static -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_INSTALLATION_ROOT\scripts\buildsystems\vcpkg.cmake" ${{ matrix.generate-options }} - name: Save vcpkg binary cache uses: actions/cache/save@v4 - if: github.event_name != 'pull_request' && steps.vcpkg-binary-cache.outputs.cache-hit != 'true' + if: github.event_name != 'pull_request' && steps.vcpkg-binary-cache.outputs.cache-hit != 'true' && matrix.job-type == 'standard' with: path: ~/AppData/Local/vcpkg/archives key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }} @@ -197,11 +225,13 @@ jobs: cmake --build . -j $env:NUMBER_OF_PROCESSORS --config Release - name: Run test suite + if: matrix.job-type == 'standard' working-directory: build run: | - ctest -j $env:NUMBER_OF_PROCESSORS -C Release + ctest --output-on-failure --stop-on-failure -j $env:NUMBER_OF_PROCESSORS -C Release - name: Run functional tests + if: matrix.job-type == 'standard' working-directory: build env: BITCOIND: '${{ github.workspace }}\build\src\Release\bitcoind.exe' @@ -212,25 +242,27 @@ jobs: shell: cmd run: py -3 test\functional\test_runner.py --jobs %NUMBER_OF_PROCESSORS% --ci --quiet --tmpdirprefix=%RUNNER_TEMP% --combinedlogslen=99999999 --timeout-factor=%TEST_RUNNER_TIMEOUT_FACTOR% %TEST_RUNNER_EXTRA% - - name: Clone fuzz corpus + - name: Clone corpora + if: matrix.job-type == 'fuzz' run: | git clone --depth=1 https://github.com/bitcoin-core/qa-assets "$env:RUNNER_TEMP\qa-assets" Set-Location "$env:RUNNER_TEMP\qa-assets" Write-Host "Using qa-assets repo from commit ..." git log -1 - - name: Run fuzz binaries + - name: Run fuzz tests + if: matrix.job-type == 'fuzz' working-directory: build env: BITCOINFUZZ: '${{ github.workspace }}\build\src\test\fuzz\Release\fuzz.exe' shell: cmd - run: py -3 test\fuzz\test_runner.py --par %NUMBER_OF_PROCESSORS% --loglevel DEBUG %RUNNER_TEMP%\qa-assets\fuzz_corpora + run: | + py -3 test\fuzz\test_runner.py --par %NUMBER_OF_PROCESSORS% --loglevel DEBUG %RUNNER_TEMP%\qa-assets\fuzz_corpora asan-lsan-ubsan-integer-no-depends-usdt: name: 'ASan + LSan + UBSan + integer, no depends, USDT' runs-on: ubuntu-24.04 # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools - # No need to run on the read-only mirror, unless it is a PR. - if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' + if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: 120 env: FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" diff --git a/.python-version b/.python-version index 43077b2460..1445aee866 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.9.18 +3.10.14 diff --git a/CMakeLists.txt b/CMakeLists.txt index 555e5b99b7..2dba6f255d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ endif() #============================= # Project / Package metadata #============================= -set(PACKAGE_NAME "Bitcoin Core") +set(CLIENT_NAME "Bitcoin Core") set(CLIENT_VERSION_MAJOR 28) set(CLIENT_VERSION_MINOR 99) set(CLIENT_VERSION_BUILD 0) @@ -47,14 +47,14 @@ project(BitcoinCore LANGUAGES NONE ) -set(PACKAGE_VERSION ${PROJECT_VERSION}) +set(CLIENT_VERSION_STRING ${PROJECT_VERSION}) if(CLIENT_VERSION_RC GREATER 0) - string(APPEND PACKAGE_VERSION "rc${CLIENT_VERSION_RC}") + string(APPEND CLIENT_VERSION_STRING "rc${CLIENT_VERSION_RC}") endif() set(COPYRIGHT_HOLDERS "The %s developers") -set(COPYRIGHT_HOLDERS_FINAL "The ${PACKAGE_NAME} developers") -set(PACKAGE_BUGREPORT "https://github.com/bitcoin/bitcoin/issues") +set(COPYRIGHT_HOLDERS_FINAL "The ${CLIENT_NAME} developers") +set(CLIENT_BUGREPORT "https://github.com/bitcoin/bitcoin/issues") #============================= # Language setup @@ -121,28 +121,9 @@ option(REDUCE_EXPORTS "Attempt to reduce exported symbols in the resulting execu option(WERROR "Treat compiler warnings as errors." OFF) option(WITH_CCACHE "Attempt to use ccache for compiling." ON) -option(WITH_NATPMP "Enable NAT-PMP." OFF) -if(WITH_NATPMP) - find_package(NATPMP MODULE REQUIRED) -endif() - -option(WITH_MINIUPNPC "Enable UPnP." OFF) -if(WITH_MINIUPNPC) - find_package(MiniUPnPc MODULE REQUIRED) -endif() - option(WITH_ZMQ "Enable ZMQ notifications." OFF) if(WITH_ZMQ) - if(VCPKG_TARGET_TRIPLET) - find_package(ZeroMQ CONFIG REQUIRED) - else() - # The ZeroMQ project has provided config files since v4.2.2. - # However, mainstream distributions do not yet provide CMake - # config files for ZeroMQ packages. If they do in the future, - # find_package(ZeroMQ) may be used instead. - find_package(PkgConfig REQUIRED) - pkg_check_modules(libzmq REQUIRED IMPORTED_TARGET libzmq>=4) - endif() + find_package(ZeroMQ 4.0.0 MODULE REQUIRED) endif() option(WITH_USDT "Enable tracepoints for Userspace, Statically Defined Tracing." OFF) @@ -154,8 +135,7 @@ cmake_dependent_option(ENABLE_EXTERNAL_SIGNER "Enable external signer support." cmake_dependent_option(WITH_QRENCODE "Enable QR code support." ON "BUILD_GUI" OFF) if(WITH_QRENCODE) - find_package(PkgConfig REQUIRED) - pkg_check_modules(libqrencode REQUIRED IMPORTED_TARGET libqrencode) + find_package(QRencode MODULE REQUIRED) set(USE_QRCODE TRUE) endif() @@ -182,7 +162,7 @@ if(BUILD_GUI) if(BUILD_GUI_TESTS) list(APPEND qt_components Test) endif() - find_package(Qt5 5.11.3 MODULE REQUIRED + find_package(Qt 5.11.3 MODULE REQUIRED COMPONENTS ${qt_components} ) unset(qt_components) @@ -239,8 +219,6 @@ if(BUILD_FOR_FUZZING) set(BUILD_WALLET_TOOL OFF) set(BUILD_GUI OFF) set(ENABLE_EXTERNAL_SIGNER OFF) - set(WITH_NATPMP OFF) - set(WITH_MINIUPNPC OFF) set(WITH_ZMQ OFF) set(BUILD_TESTS OFF) set(BUILD_GUI_TESTS OFF) @@ -248,7 +226,7 @@ if(BUILD_FOR_FUZZING) set(BUILD_FUZZ_BINARY ON) target_compile_definitions(core_interface INTERFACE - ABORT_ON_FAILED_ASSUME + FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION ) endif() @@ -275,8 +253,8 @@ if(WIN32) ]=] target_compile_definitions(core_interface INTERFACE - _WIN32_WINNT=0x0601 - _WIN32_IE=0x0501 + _WIN32_WINNT=0x0A00 + _WIN32_IE=0x0A00 WIN32_LEAN_AND_MEAN NOMINMAX ) @@ -314,9 +292,20 @@ if(WIN32) # See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412. try_append_cxx_flags("-Wa,-muse-unaligned-vector-move" TARGET core_interface SKIP_LINK) try_append_linker_flag("-static" TARGET core_interface) - # We require Windows 7 (NT 6.1) or later. + # We support Windows 10+, however it's not possible to set these values accordingly, + # due to a bug in mingw-w64. See https://sourceforge.net/p/mingw-w64/bugs/968/. + # As a best effort, target Windows 8. try_append_linker_flag("-Wl,--major-subsystem-version,6" TARGET core_interface) - try_append_linker_flag("-Wl,--minor-subsystem-version,1" TARGET core_interface) + try_append_linker_flag("-Wl,--minor-subsystem-version,2" TARGET core_interface) + endif() + + # Workaround producing large object files, which cannot be handled by the assembler. + # More likely to happen with no, or lower levels of optimisation. + # See discussion in https://github.com/bitcoin/bitcoin/issues/28109. + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + try_append_cxx_flags("/bigobj" TARGET core_interface_debug SKIP_LINK) + else() + try_append_cxx_flags("-Wa,-mbig-obj" TARGET core_interface_debug SKIP_LINK) endif() endif() @@ -329,10 +318,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SIZEOF_VOID_P EQUAL 4) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - target_compile_definitions(core_interface INTERFACE - MAC_OSX - OBJC_OLD_DISPATCH_PROTOTYPES=0 - ) + target_compile_definitions(core_interface INTERFACE OBJC_OLD_DISPATCH_PROTOTYPES=0) # These flags are specific to ld64, and may cause issues with other linkers. # For example: GNU ld will interpret -dead_strip as -de and then try and use # "ad_strip" as the symbol for the entry point. @@ -442,6 +428,7 @@ else() try_append_cxx_flags("-Wunreachable-code" TARGET warn_interface SKIP_LINK) try_append_cxx_flags("-Wdocumentation" TARGET warn_interface SKIP_LINK) try_append_cxx_flags("-Wself-assign" TARGET warn_interface SKIP_LINK) + try_append_cxx_flags("-Wbidi-chars=any" TARGET warn_interface SKIP_LINK) try_append_cxx_flags("-Wundef" TARGET warn_interface SKIP_LINK) # Some compilers (gcc) ignore unknown -Wno-* options, but warn about all @@ -460,6 +447,18 @@ configure_file(contrib/filter-lcov.py filter-lcov.py USE_SOURCE_PERMISSIONS COPY # Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. try_append_cxx_flags("-fno-extended-identifiers" TARGET core_interface SKIP_LINK) +# Avoiding the `-ffile-prefix-map` compiler option because it implies +# `-fcoverage-prefix-map` on Clang or `-fprofile-prefix-map` on GCC, +# which can cause issues with coverage builds, particularly when using +# Clang in the OSS-Fuzz environment due to its use of other options +# and a third party script, or with GCC. +try_append_cxx_flags("-fdebug-prefix-map=A=B" TARGET core_interface SKIP_LINK + IF_CHECK_PASSED "-fdebug-prefix-map=${PROJECT_SOURCE_DIR}/src=." +) +try_append_cxx_flags("-fmacro-prefix-map=A=B" TARGET core_interface SKIP_LINK + IF_CHECK_PASSED "-fmacro-prefix-map=${PROJECT_SOURCE_DIR}/src=." +) + # Currently all versions of gcc are subject to a class of bugs, see the # gccbug_90348 test case (only reproduces on GCC 11 and earlier) and # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111843. To work around that, set @@ -498,15 +497,18 @@ if(ENABLE_HARDENING) try_append_cxx_flags("-fcf-protection=full" TARGET hardening_interface) if(MINGW) - # stack-clash-protection doesn't compile with GCC 10 and earlier. - # In any case, it is a no-op for Windows. + # stack-clash-protection is a no-op for Windows. # See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90458 for more details. else() try_append_cxx_flags("-fstack-clash-protection" TARGET hardening_interface) endif() if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") - try_append_cxx_flags("-mbranch-protection=bti" TARGET hardening_interface SKIP_LINK) + if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + try_append_cxx_flags("-mbranch-protection=bti" TARGET hardening_interface SKIP_LINK) + else() + try_append_cxx_flags("-mbranch-protection=standard" TARGET hardening_interface SKIP_LINK) + endif() endif() try_append_linker_flag("-Wl,--enable-reloc-section" TARGET hardening_interface) @@ -541,7 +543,7 @@ if(WERROR) unset(werror_flag) endif() -find_package(Python3 3.9 COMPONENTS Interpreter) +find_package(Python3 3.10 COMPONENTS Interpreter) if(Python3_EXECUTABLE) set(PYTHON_COMMAND ${Python3_EXECUTABLE}) else() @@ -566,6 +568,12 @@ endif() if(BUILD_TESTS) enable_testing() endif() + +if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.29) + # have "make test" depend on "make all" + set(CMAKE_SKIP_TEST_ALL_DEPENDENCY FALSE) +endif() + # TODO: The `CMAKE_SKIP_BUILD_RPATH` variable setting can be deleted # in the future after reordering Guix script commands to # perform binary checks after the installation step. @@ -616,9 +624,6 @@ if(ENABLE_WALLET) message(" - legacy wallets (Berkeley DB) ..... ${WITH_BDB}") endif() message(" external signer ..................... ${ENABLE_EXTERNAL_SIGNER}") -message(" port mapping:") -message(" - using NAT-PMP .................... ${WITH_NATPMP}") -message(" - using UPnP ....................... ${WITH_MINIUPNPC}") message(" ZeroMQ .............................. ${WITH_ZMQ}") message(" USDT tracing ........................ ${WITH_USDT}") message(" QR code (GUI) ....................... ${WITH_QRENCODE}") diff --git a/CMakePresets.json b/CMakePresets.json index 041e3c1cbf..da838f2b0e 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -15,8 +15,7 @@ "toolchainFile": "$env{VCPKG_ROOT}\\scripts\\buildsystems\\vcpkg.cmake", "cacheVariables": { "VCPKG_TARGET_TRIPLET": "x64-windows", - "BUILD_GUI": "ON", - "WITH_QRENCODE": "OFF" + "BUILD_GUI": "ON" } }, { @@ -32,8 +31,7 @@ "toolchainFile": "$env{VCPKG_ROOT}\\scripts\\buildsystems\\vcpkg.cmake", "cacheVariables": { "VCPKG_TARGET_TRIPLET": "x64-windows-static", - "BUILD_GUI": "ON", - "WITH_QRENCODE": "OFF" + "BUILD_GUI": "ON" } }, { @@ -73,7 +71,6 @@ "BUILD_GUI_TESTS": "ON", "BUILD_KERNEL_LIB": "ON", "BUILD_SHARED_LIBS": "ON", - "BUILD_TESTING": "ON", "BUILD_TESTS": "ON", "BUILD_TX": "ON", "BUILD_UTIL": "ON", @@ -84,9 +81,7 @@ "ENABLE_WALLET": "ON", "WARN_INCOMPATIBLE_BDB": "OFF", "WITH_BDB": "ON", - "WITH_MINIUPNPC": "ON", "WITH_MULTIPROCESS": "ON", - "WITH_NATPMP": "ON", "WITH_QRENCODE": "ON", "WITH_SQLITE": "ON", "WITH_USDT": "ON", diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e86ac5cb7a..25637b40aa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,8 +36,7 @@ list or changes that are Some of them might no longer be applicable. So if you are interested, but unsure, you might want to leave a comment on the issue first. -You may also participate in the weekly -[Bitcoin Core PR Review Club](https://bitcoincore.reviews/) meeting. +You may also participate in the [Bitcoin Core PR Review Club](https://bitcoincore.reviews/). ### Good First Issue Label @@ -53,7 +53,8 @@ and extending unit tests can be found in [/src/test/README.md](/src/test/README. There are also [regression and integration tests](/test), written in Python. -These tests can be run (if the [test dependencies](/test) are installed) with: `test/functional/test_runner.py` +These tests can be run (if the [test dependencies](/test) are installed) with: `build/test/functional/test_runner.py` +(assuming `build` is your build directory). The CI (Continuous Integration) systems make sure that every pull request is built for Windows, Linux, and macOS, and that unit/sanity tests are run automatically. diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index d899c0c67a..9ef1f37c73 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -59,7 +59,7 @@ curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_ tar --xz -xf - --directory /tmp/ mv "/tmp/shellcheck-${SHELLCHECK_VERSION}/shellcheck" /usr/bin/ -MLC_VERSION=v0.18.0 +MLC_VERSION=v0.19.0 MLC_BIN=mlc-x86_64-linux curl -sL "https://github.com/becheran/mlc/releases/download/${MLC_VERSION}/${MLC_BIN}" -o "/usr/bin/mlc" chmod +x /usr/bin/mlc diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 944655d8ad..021d5e1597 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -53,18 +53,18 @@ export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} # See man 7 debconf export DEBIAN_FRONTEND=noninteractive -export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:-100M} +export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:-500M} export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} # The cache dir. # This folder exists only on the ci guest, and on the ci host as a volume. -export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache} +export CCACHE_DIR="${CCACHE_DIR:-$BASE_SCRATCH_DIR/ccache}" # Folder where the build result is put (bin and lib). export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out} # The folder for previous release binaries. # This folder exists only on the ci guest, and on the ci host as a volume. export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases} -export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config curl ca-certificates ccache python3 rsync git procps bison e2fsprogs cmake} +export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential pkg-config curl ca-certificates ccache python3 rsync git procps bison e2fsprogs cmake} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"} diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 881f006732..22657742d7 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2022 The Bitcoin Core developers +# Copyright (c) 2020-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,9 +9,9 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_centos export CI_IMAGE_NAME_TAG="quay.io/centos/amd64:stream9" -export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison util-linux e2fsprogs cmake" +export STREAM_GCC_V="12" +export CI_BASE_PACKAGES="gcc-toolset-${STREAM_GCC_V}-gcc-c++ glibc-devel.x86_64 gcc-toolset-${STREAM_GCC_V}-libstdc++-devel.x86_64 glibc-devel.i686 gcc-toolset-${STREAM_GCC_V}-libstdc++-devel.i686 ccache make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison e2fsprogs cmake" export PIP_PACKAGES="pyzmq" export GOAL="install" -export NO_WERROR=1 # Suppress error: #warning _FORTIFY_SOURCE > 2 is treated like 2 on this platform [-Werror=cpp] export BITCOIN_CONFIG="-DWITH_ZMQ=ON -DBUILD_GUI=ON -DREDUCE_EXPORTS=ON" export CONFIG_SHELL="/bin/dash" diff --git a/ci/test/00_setup_env_mac_native.sh b/ci/test/00_setup_env_mac_native.sh index 76668d97f2..e01a56895b 100755 --- a/ci/test/00_setup_env_mac_native.sh +++ b/ci/test/00_setup_env_mac_native.sh @@ -6,14 +6,12 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-apple-darwin # Homebrew's python@3.12 is marked as externally managed (PEP 668). # Therefore, `--break-system-packages` is needed. export PIP_PACKAGES="--break-system-packages zmq" export GOAL="install" -export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DWITH_MINIUPNPC=ON -DWITH_NATPMP=ON -DREDUCE_EXPORTS=ON" +export CMAKE_GENERATOR="Ninja" +export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DREDUCE_EXPORTS=ON" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" -export CCACHE_MAXSIZE=400M -export RUN_FUZZ_TESTS=true diff --git a/ci/test/00_setup_env_mac_native_fuzz.sh b/ci/test/00_setup_env_mac_native_fuzz.sh new file mode 100755 index 0000000000..1a453a4353 --- /dev/null +++ b/ci/test/00_setup_env_mac_native_fuzz.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CMAKE_GENERATOR="Ninja" +export BITCOIN_CONFIG="-DBUILD_FOR_FUZZING=ON" +export CI_OS_NAME="macos" +export NO_DEPENDS=1 +export OSX_SDK="" +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export RUN_FUZZ_TESTS=true diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 0ec30f23af..d7ff7c2c13 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -19,17 +19,17 @@ else fi export CONTAINER_NAME=ci_native_asan -export PACKAGES="systemtap-sdt-dev clang-18 llvm-18 libclang-rt-18-dev python3-zmq qtbase5-dev qttools5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" +export APT_LLVM_V="19" +export PACKAGES="systemtap-sdt-dev clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev python3-zmq qtbase5-dev qttools5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" export NO_DEPENDS=1 export GOAL="install" export BITCOIN_CONFIG="\ -DWITH_USDT=ON -DWITH_ZMQ=ON -DWITH_BDB=ON -DWARN_INCOMPATIBLE_BDB=OFF -DBUILD_GUI=ON \ -DSANITIZERS=address,float-divide-by-zero,integer,undefined \ - -DCMAKE_C_COMPILER=clang-18 \ - -DCMAKE_CXX_COMPILER=clang++-18 \ + -DCMAKE_C_COMPILER=clang-${APT_LLVM_V} \ + -DCMAKE_CXX_COMPILER=clang++-${APT_LLVM_V} \ -DCMAKE_C_FLAGS='-ftrivial-auto-var-init=pattern' \ -DCMAKE_CXX_FLAGS='-ftrivial-auto-var-init=pattern -Wno-error=deprecated-declarations' \ -DAPPEND_CXXFLAGS='-std=c++23' \ -DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \ " -export CCACHE_MAXSIZE=300M diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index e1a353056d..597e2401a2 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -8,7 +8,8 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_fuzz -export PACKAGES="clang-18 llvm-18 libclang-rt-18-dev libevent-dev libboost-dev libsqlite3-dev" +export APT_LLVM_V="19" +export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev libevent-dev libboost-dev libsqlite3-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false @@ -18,10 +19,9 @@ export CI_CONTAINER_CAP="--cap-add SYS_PTRACE" # If run with (ASan + LSan), the export BITCOIN_CONFIG="\ -DBUILD_FOR_FUZZING=ON \ -DSANITIZERS=fuzzer,address,undefined,float-divide-by-zero,integer \ - -DCMAKE_C_COMPILER=clang-18 \ - -DCMAKE_CXX_COMPILER=clang++-18 \ + -DCMAKE_C_COMPILER=clang-${APT_LLVM_V} \ + -DCMAKE_CXX_COMPILER=clang++-${APT_LLVM_V} \ -DCMAKE_C_FLAGS='-ftrivial-auto-var-init=pattern' \ -DCMAKE_CXX_FLAGS='-ftrivial-auto-var-init=pattern' \ " -export CCACHE_MAXSIZE=200M -export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-18" +export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-${APT_LLVM_V}" diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 7cea4d73af..cfdbc8c014 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -31,4 +31,3 @@ export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" export RUN_FUZZ_TESTS=true -export CCACHE_MAXSIZE=250M diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 02903b5199..c65c05bff9 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -21,4 +21,4 @@ export BITCOIN_CONFIG="\ -DCMAKE_C_COMPILER=clang-16 \ -DCMAKE_CXX_COMPILER=clang++-16 \ " -export CCACHE_MAXSIZE=200M +export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-16" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 2c85ba31d1..cd4ac99942 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -27,5 +27,3 @@ export BITCOIN_CONFIG="\ -DAPPEND_CPPFLAGS='-U_FORTIFY_SOURCE' \ " export USE_MEMORY_SANITIZER="true" -export RUN_FUNCTIONAL_TESTS="false" -export CCACHE_MAXSIZE=250M diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index 479628d3e8..3d5d1b7745 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel export CI_IMAGE_NAME_TAG="docker.io/debian:bookworm" -# Use minimum supported python3.9 (or best-effort 3.11) and clang-16, see doc/dependencies.md +# Use minimum supported python3.10 (or best-effort 3.11) and clang-16, see doc/dependencies.md export PACKAGES="python3-zmq clang-16 llvm-16 libc++abi-16-dev libc++-16-dev" export DEP_OPTS="NO_WALLET=1 CC=clang-16 CXX='clang++-16 -stdlib=libc++'" export GOAL="install" diff --git a/ci/test/00_setup_env_native_previous_releases.sh b/ci/test/00_setup_env_native_previous_releases.sh index 2482e545e1..19a33f14dd 100755 --- a/ci/test/00_setup_env_native_previous_releases.sh +++ b/ci/test/00_setup_env_native_previous_releases.sh @@ -8,9 +8,9 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_previous_releases export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" -# Use minimum supported python3.9 (or best effort 3.10) and gcc-11, see doc/dependencies.md +# Use minimum supported python3.10 and gcc-11, see doc/dependencies.md export PACKAGES="gcc-11 g++-11 python3-zmq" -export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 CC=gcc-11 CXX=g++-11" +export DEP_OPTS="DEBUG=1 CC=gcc-11 CXX=g++-11" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index 5105455df8..b85fae859c 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -8,8 +8,9 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_tidy -export TIDY_LLVM_V="18" -export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev qtbase5-dev qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" +export TIDY_LLVM_V="19" +export APT_LLVM_V="${TIDY_LLVM_V}" +export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq libevent-dev libboost-dev libzmq3-dev systemtap-sdt-dev qtbase5-dev qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false @@ -18,11 +19,10 @@ export RUN_CHECK_DEPS=true export RUN_TIDY=true export GOAL="install" export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DWITH_NATPMP=ON -DWITH_MINIUPNPC=ON -DWITH_USDT=ON -DWITH_BDB=ON -DWARN_INCOMPATIBLE_BDB=OFF \ + -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DWITH_USDT=ON -DWITH_BDB=ON -DWARN_INCOMPATIBLE_BDB=OFF \ -DENABLE_HARDENING=OFF \ -DCMAKE_C_COMPILER=clang-${TIDY_LLVM_V} \ -DCMAKE_CXX_COMPILER=clang++-${TIDY_LLVM_V} \ -DCMAKE_C_FLAGS_RELWITHDEBINFO='-O0 -g0' \ -DCMAKE_CXX_FLAGS_RELWITHDEBINFO='-O0 -g0' \ " -export CCACHE_MAXSIZE=200M diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 9c2da778b4..d82dda344f 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -8,8 +8,9 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" -export PACKAGES="clang-18 llvm-18 libclang-rt-18-dev libc++abi-18-dev libc++-18-dev python3-zmq" -export DEP_OPTS="CC=clang-18 CXX='clang++-18 -stdlib=libc++'" +export APT_LLVM_V="19" +export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev libc++abi-${APT_LLVM_V}-dev libc++-${APT_LLVM_V}-dev python3-zmq" +export DEP_OPTS="CC=clang-${APT_LLVM_V} CXX='clang++-${APT_LLVM_V} -stdlib=libc++'" export GOAL="install" export BITCOIN_CONFIG="-DWITH_ZMQ=ON -DSANITIZERS=thread \ -DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES'" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 60bbe83119..3acb5842a4 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -8,14 +8,14 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang-16 llvm-16 libclang-rt-16-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" +export PACKAGES="valgrind clang-16 llvm-16 libclang-rt-16-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # feature_init excluded for now, see https://github.com/bitcoin/bitcoin/issues/30011 ; bind tests excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" # TODO enable GUI export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DWITH_BDB=ON -DWITH_NATPMP=ON -DWITH_MINIUPNPC=ON -DWARN_INCOMPATIBLE_BDB=OFF -DBUILD_GUI=OFF \ + -DWITH_ZMQ=ON -DWITH_BDB=ON -DWARN_INCOMPATIBLE_BDB=OFF -DBUILD_GUI=OFF \ -DCMAKE_C_COMPILER=clang-16 \ -DCMAKE_CXX_COMPILER=clang++-16 \ " diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 15b9e407b6..25da64c524 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -11,6 +11,9 @@ export CI_IMAGE_NAME_TAG="docker.io/amd64/debian:bookworm" # Check that https:/ export HOST=x86_64-w64-mingw32 export DPKG_ADD_ARCH="i386" export PACKAGES="nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file" +# Install wine, but do not run unit tests, as they surface frequent +# false-positives. +export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-false} export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" # Prior to 11.0.0, the mingw-w64 headers were missing noreturn attributes, causing warnings when diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index bb99fc30e9..0130e820c2 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -15,10 +15,23 @@ if [ "$(git config --global ${CFG_DONE})" == "true" ]; then exit 0 fi +MAKEJOBS="-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds. + if [ -n "$DPKG_ADD_ARCH" ]; then dpkg --add-architecture "$DPKG_ADD_ARCH" fi +if [ -n "${APT_LLVM_V}" ]; then + ${CI_RETRY_EXE} apt-get update + ${CI_RETRY_EXE} apt-get install curl -y + curl "https://apt.llvm.org/llvm-snapshot.gpg.key" | tee "/etc/apt/trusted.gpg.d/apt.llvm.org.asc" + ( + # shellcheck disable=SC2034 + source /etc/os-release + echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-${APT_LLVM_V} main" > "/etc/apt/sources.list.d/llvm-toolchain-${VERSION_CODENAME}-${APT_LLVM_V}.list" + ) +fi + if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then bash -c "dnf -y install epel-release" bash -c "dnf -y --allowerasing install $CI_BASE_PACKAGES $PACKAGES" @@ -36,7 +49,7 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-18.1.3" /msan/llvm-project + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-19.1.6" /msan/llvm-project cmake -G Ninja -B /msan/clang_build/ \ -DLLVM_ENABLE_PROJECTS="clang" \ @@ -45,7 +58,7 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ -S /msan/llvm-project/llvm - ninja -C /msan/clang_build/ "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds + ninja -C /msan/clang_build/ "$MAKEJOBS" ninja -C /msan/clang_build/ install-runtimes update-alternatives --install /usr/bin/clang++ clang++ /msan/clang_build/bin/clang++ 100 @@ -64,7 +77,7 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then -DLIBCXX_HARDENING_MODE=debug \ -S /msan/llvm-project/runtimes - ninja -C /msan/cxx_build/ "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds + ninja -C /msan/cxx_build/ "$MAKEJOBS" # Clear no longer needed source folder du -sh /msan/llvm-project @@ -74,7 +87,7 @@ fi if [[ "${RUN_TIDY}" == "true" ]]; then ${CI_RETRY_EXE} git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_"${TIDY_LLVM_V}" /include-what-you-use cmake -B /iwyu-build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-"${TIDY_LLVM_V}" -S /include-what-you-use - make -C /iwyu-build/ install "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds + make -C /iwyu-build/ install "$MAKEJOBS" fi mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" diff --git a/ci/test/02_run_container.sh b/ci/test/02_run_container.sh index afd447c347..ce01db325c 100755 --- a/ci/test/02_run_container.sh +++ b/ci/test/02_run_container.sh @@ -15,12 +15,22 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value and "HOME" != key and "PATH" != key and "USER" != key]' | tee "/tmp/env-$USER-$CONTAINER_NAME" # System-dependent env vars must be kept as is. So read them from the container. docker run --rm "${CI_IMAGE_NAME_TAG}" bash -c "env | grep --extended-regexp '^(HOME|PATH|USER)='" | tee --append "/tmp/env-$USER-$CONTAINER_NAME" + + # Env vars during the build can not be changed. For example, a modified + # $MAKEJOBS is ignored in the build process. Use --cpuset-cpus as an + # approximation to respect $MAKEJOBS somewhat, if cpuset is available. + MAYBE_CPUSET="" + if [ "$HAVE_CGROUP_CPUSET" ]; then + MAYBE_CPUSET="--cpuset-cpus=$( python3 -c "import random;P=$( nproc );M=min(P,int('$MAKEJOBS'.lstrip('-j')));print(','.join(map(str,sorted(random.sample(range(P),M)))))" )" + fi echo "Creating $CI_IMAGE_NAME_TAG container to run in" + # shellcheck disable=SC2086 DOCKER_BUILDKIT=1 docker build \ --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \ --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ --build-arg "FILE_ENV=${FILE_ENV}" \ + $MAYBE_CPUSET \ --label="${CI_IMAGE_LABEL}" \ --tag="${CONTAINER_NAME}" \ "${BASE_READ_ONLY_DIR}" @@ -48,6 +58,19 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR" fi + if [ "$DANGER_CI_ON_HOST_CCACHE_FOLDER" ]; then + # Temporary exclusion for https://github.com/bitcoin/bitcoin/issues/31108 + # to allow CI configs and envs generated in the past to work for a bit longer. + # Can be removed in March 2025. + if [ "${CCACHE_DIR}" != "/tmp/ccache_dir" ]; then + if [ ! -d "${CCACHE_DIR}" ]; then + echo "Error: Directory '${CCACHE_DIR}' must be created in advance." + exit 1 + fi + CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=${CCACHE_DIR}" + fi # End temporary exclusion + fi + docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index 7ee424ac9b..6e77a8927c 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -10,15 +10,14 @@ set -ex export ASAN_OPTIONS="detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" -export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1" +export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:second_deadlock_stack=1" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" +echo "Number of available processing units: $(nproc)" if [ "$CI_OS_NAME" == "macos" ]; then top -l 1 -s 0 | awk ' /PhysMem/ {print}' - echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" else free -m -h - echo "Number of CPUs (nproc): $(nproc)" echo "System info: $(uname --kernel-name --kernel-release)" lscpu fi @@ -30,6 +29,10 @@ df -h # Tests that run natively guess the host export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} +echo "=== BEGIN env ===" +env +echo "=== END env ===" + ( # compact->outputs[i].file_size is uninitialized memory, so reading it is UB. # The statistic bytes_written is only used for logging, which is disabled in @@ -90,6 +93,8 @@ fi if [ -z "$NO_DEPENDS" ]; then if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then SHELL_OPTS="CONFIG_SHELL=/bin/dash" + # shellcheck disable=SC1090 + source "/opt/rh/gcc-toolset-${STREAM_GCC_V}/enable" else SHELL_OPTS="CONFIG_SHELL=" fi @@ -142,7 +147,7 @@ if [ "$RUN_CHECK_DEPS" = "true" ]; then fi if [ "$RUN_UNIT_TESTS" = "true" ]; then - DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" CTEST_OUTPUT_ON_FAILURE=ON ctest "${MAKEJOBS}" + DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" CTEST_OUTPUT_ON_FAILURE=ON ctest --stop-on-failure "${MAKEJOBS}" --timeout $(( TEST_RUNNER_TIMEOUT_FACTOR * 60 )) fi if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then diff --git a/cmake/bitcoin-config.h.in b/cmake/bitcoin-build-config.h.in index 094eb8040a..56e0519fac 100644 --- a/cmake/bitcoin-config.h.in +++ b/cmake/bitcoin-build-config.h.in @@ -24,7 +24,7 @@ #define COPYRIGHT_HOLDERS_FINAL "@COPYRIGHT_HOLDERS_FINAL@" /* Replacement for %s in copyright holders string */ -#define COPYRIGHT_HOLDERS_SUBSTITUTION "@PACKAGE_NAME@" +#define COPYRIGHT_HOLDERS_SUBSTITUTION "@CLIENT_NAME@" /* Copyright year */ #define COPYRIGHT_YEAR @COPYRIGHT_YEAR@ @@ -71,9 +71,6 @@ */ #cmakedefine01 HAVE_DECL_SETSID -/* Define this symbol if evhttp_connection_get_peer expects const char** */ -#cmakedefine HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR 1 - /* Define to 1 if fdatasync is available. */ #cmakedefine HAVE_FDATASYNC 1 @@ -124,16 +121,16 @@ #cmakedefine HAVE_VM_VM_PARAM_H 1 /* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "@PACKAGE_BUGREPORT@" +#define CLIENT_BUGREPORT "@CLIENT_BUGREPORT@" /* Define to the full name of this package. */ -#define PACKAGE_NAME "@PACKAGE_NAME@" +#define CLIENT_NAME "@CLIENT_NAME@" /* Define to the home page for this package. */ -#define PACKAGE_URL "@PROJECT_HOMEPAGE_URL@" +#define CLIENT_URL "@PROJECT_HOMEPAGE_URL@" /* Define to the version of this package. */ -#define PACKAGE_VERSION "@PACKAGE_VERSION@" +#define CLIENT_VERSION_STRING "@CLIENT_VERSION_STRING@" /* Define to 1 if strerror_r returns char *. */ #cmakedefine STRERROR_R_CHAR_P 1 diff --git a/cmake/ccache.cmake b/cmake/ccache.cmake index 099aa66411..d1cc204c8e 100644 --- a/cmake/ccache.cmake +++ b/cmake/ccache.cmake @@ -23,14 +23,6 @@ if(NOT MSVC) else() set(WITH_CCACHE OFF) endif() - if(WITH_CCACHE) - try_append_cxx_flags("-fdebug-prefix-map=A=B" TARGET core_interface SKIP_LINK - IF_CHECK_PASSED "-fdebug-prefix-map=${PROJECT_SOURCE_DIR}=." - ) - try_append_cxx_flags("-fmacro-prefix-map=A=B" TARGET core_interface SKIP_LINK - IF_CHECK_PASSED "-fmacro-prefix-map=${PROJECT_SOURCE_DIR}=." - ) - endif() endif() mark_as_advanced(CCACHE_EXECUTABLE) diff --git a/cmake/introspection.cmake b/cmake/introspection.cmake index 5435a109d4..29c93869a7 100644 --- a/cmake/introspection.cmake +++ b/cmake/introspection.cmake @@ -6,7 +6,7 @@ include(CheckCXXSourceCompiles) include(CheckCXXSymbolExists) include(CheckIncludeFileCXX) -# The following HAVE_{HEADER}_H variables go to the bitcoin-config.h header. +# The following HAVE_{HEADER}_H variables go to the bitcoin-build-config.h header. check_include_file_cxx(sys/prctl.h HAVE_SYS_PRCTL_H) check_include_file_cxx(sys/resources.h HAVE_SYS_RESOURCES_H) check_include_file_cxx(sys/vmmeter.h HAVE_SYS_VMMETER_H) diff --git a/cmake/module/FindLibevent.cmake b/cmake/module/FindLibevent.cmake index 901a4f3bd4..c006b43d60 100644 --- a/cmake/module/FindLibevent.cmake +++ b/cmake/module/FindLibevent.cmake @@ -35,46 +35,52 @@ function(check_evhttp_connection_get_peer target) " HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR ) cmake_pop_check_state() - set(HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR ${HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR} PARENT_SCOPE) + target_compile_definitions(${target} INTERFACE + $<$<BOOL:${HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR}>:HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR> + ) endfunction() +set(_libevent_components core extra) +if(NOT WIN32) + list(APPEND _libevent_components pthreads) +endif() + +find_package(Libevent ${Libevent_FIND_VERSION} QUIET + NO_MODULE +) include(FindPackageHandleStandardArgs) -if(VCPKG_TARGET_TRIPLET) - find_package(Libevent ${Libevent_FIND_VERSION} NO_MODULE QUIET - COMPONENTS extra +if(Libevent_FOUND) + find_package(Libevent ${Libevent_FIND_VERSION} QUIET + REQUIRED COMPONENTS ${_libevent_components} + NO_MODULE ) find_package_handle_standard_args(Libevent REQUIRED_VARS Libevent_DIR VERSION_VAR Libevent_VERSION ) check_evhttp_connection_get_peer(libevent::extra) - add_library(libevent::libevent ALIAS libevent::extra) - mark_as_advanced(Libevent_DIR) - mark_as_advanced(_event_h) - mark_as_advanced(_event_lib) else() find_package(PkgConfig REQUIRED) - pkg_check_modules(libevent QUIET - IMPORTED_TARGET - libevent>=${Libevent_FIND_VERSION} - ) - set(_libevent_required_vars libevent_LIBRARY_DIRS libevent_FOUND) - if(NOT WIN32) - pkg_check_modules(libevent_pthreads QUIET - IMPORTED_TARGET - libevent_pthreads>=${Libevent_FIND_VERSION} + foreach(component IN LISTS _libevent_components) + pkg_check_modules(libevent_${component} + REQUIRED QUIET + IMPORTED_TARGET GLOBAL + libevent_${component}>=${Libevent_FIND_VERSION} ) - list(APPEND _libevent_required_vars libevent_pthreads_FOUND) - endif() + if(TARGET PkgConfig::libevent_${component} AND NOT TARGET libevent::${component}) + add_library(libevent::${component} ALIAS PkgConfig::libevent_${component}) + endif() + endforeach() find_package_handle_standard_args(Libevent - REQUIRED_VARS ${_libevent_required_vars} - VERSION_VAR libevent_VERSION + REQUIRED_VARS libevent_core_LIBRARY_DIRS + VERSION_VAR libevent_core_VERSION ) - unset(_libevent_required_vars) - check_evhttp_connection_get_peer(PkgConfig::libevent) - add_library(libevent::libevent ALIAS PkgConfig::libevent) - if(NOT WIN32) - add_library(libevent::pthreads ALIAS PkgConfig::libevent_pthreads) - endif() + check_evhttp_connection_get_peer(PkgConfig::libevent_extra) endif() + +unset(_libevent_components) + +mark_as_advanced(Libevent_DIR) +mark_as_advanced(_event_h) +mark_as_advanced(_event_lib) diff --git a/cmake/module/FindMiniUPnPc.cmake b/cmake/module/FindMiniUPnPc.cmake deleted file mode 100644 index 34b94f05f1..0000000000 --- a/cmake/module/FindMiniUPnPc.cmake +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2023-present The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or https://opensource.org/license/mit/. - -if(NOT MSVC) - find_package(PkgConfig REQUIRED) - pkg_check_modules(PC_MiniUPnPc QUIET miniupnpc) -endif() - -find_path(MiniUPnPc_INCLUDE_DIR - NAMES miniupnpc/miniupnpc.h - PATHS ${PC_MiniUPnPc_INCLUDE_DIRS} -) - -if(MiniUPnPc_INCLUDE_DIR) - file( - STRINGS "${MiniUPnPc_INCLUDE_DIR}/miniupnpc/miniupnpc.h" version_strings - REGEX "^#define[\t ]+MINIUPNPC_API_VERSION[\t ]+[0-9]+" - ) - string(REGEX REPLACE "^#define[\t ]+MINIUPNPC_API_VERSION[\t ]+([0-9]+)" "\\1" MiniUPnPc_API_VERSION "${version_strings}") - - # The minimum supported miniUPnPc API version is set to 17. This excludes - # versions with known vulnerabilities. - if(MiniUPnPc_API_VERSION GREATER_EQUAL 17) - set(MiniUPnPc_API_VERSION_OK TRUE) - endif() -endif() - -if(MSVC) - cmake_path(GET MiniUPnPc_INCLUDE_DIR PARENT_PATH MiniUPnPc_IMPORTED_PATH) - find_library(MiniUPnPc_LIBRARY_DEBUG - NAMES miniupnpc PATHS ${MiniUPnPc_IMPORTED_PATH}/debug/lib - NO_DEFAULT_PATH - ) - find_library(MiniUPnPc_LIBRARY_RELEASE - NAMES miniupnpc PATHS ${MiniUPnPc_IMPORTED_PATH}/lib - NO_DEFAULT_PATH - ) - set(MiniUPnPc_required MiniUPnPc_IMPORTED_PATH) -else() - find_library(MiniUPnPc_LIBRARY - NAMES miniupnpc - PATHS ${PC_MiniUPnPc_LIBRARY_DIRS} - ) - set(MiniUPnPc_required MiniUPnPc_LIBRARY) -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(MiniUPnPc - REQUIRED_VARS ${MiniUPnPc_required} MiniUPnPc_INCLUDE_DIR MiniUPnPc_API_VERSION_OK -) - -if(MiniUPnPc_FOUND AND NOT TARGET MiniUPnPc::MiniUPnPc) - add_library(MiniUPnPc::MiniUPnPc UNKNOWN IMPORTED) - set_target_properties(MiniUPnPc::MiniUPnPc PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${MiniUPnPc_INCLUDE_DIR}" - ) - if(MSVC) - if(MiniUPnPc_LIBRARY_DEBUG) - set_property(TARGET MiniUPnPc::MiniUPnPc APPEND PROPERTY IMPORTED_CONFIGURATIONS DEBUG) - set_target_properties(MiniUPnPc::MiniUPnPc PROPERTIES - IMPORTED_LOCATION_DEBUG "${MiniUPnPc_LIBRARY_DEBUG}" - ) - endif() - if(MiniUPnPc_LIBRARY_RELEASE) - set_property(TARGET MiniUPnPc::MiniUPnPc APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) - set_target_properties(MiniUPnPc::MiniUPnPc PROPERTIES - IMPORTED_LOCATION_RELEASE "${MiniUPnPc_LIBRARY_RELEASE}" - ) - endif() - else() - set_target_properties(MiniUPnPc::MiniUPnPc PROPERTIES - IMPORTED_LOCATION "${MiniUPnPc_LIBRARY}" - ) - endif() - set_property(TARGET MiniUPnPc::MiniUPnPc PROPERTY - INTERFACE_COMPILE_DEFINITIONS USE_UPNP=1 $<$<PLATFORM_ID:Windows>:MINIUPNP_STATICLIB> - ) -endif() - -mark_as_advanced( - MiniUPnPc_INCLUDE_DIR - MiniUPnPc_LIBRARY -) diff --git a/cmake/module/FindNATPMP.cmake b/cmake/module/FindNATPMP.cmake deleted file mode 100644 index 930555232b..0000000000 --- a/cmake/module/FindNATPMP.cmake +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2023-present The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or https://opensource.org/license/mit/. - -find_path(NATPMP_INCLUDE_DIR - NAMES natpmp.h -) - -find_library(NATPMP_LIBRARY - NAMES natpmp -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(NATPMP - REQUIRED_VARS NATPMP_LIBRARY NATPMP_INCLUDE_DIR -) - -if(NATPMP_FOUND AND NOT TARGET NATPMP::NATPMP) - add_library(NATPMP::NATPMP UNKNOWN IMPORTED) - set_target_properties(NATPMP::NATPMP PROPERTIES - IMPORTED_LOCATION "${NATPMP_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${NATPMP_INCLUDE_DIR}" - ) - set_property(TARGET NATPMP::NATPMP PROPERTY - INTERFACE_COMPILE_DEFINITIONS USE_NATPMP=1 $<$<PLATFORM_ID:Windows>:NATPMP_STATICLIB> - ) -endif() - -mark_as_advanced( - NATPMP_INCLUDE_DIR - NATPMP_LIBRARY -) diff --git a/cmake/module/FindQRencode.cmake b/cmake/module/FindQRencode.cmake new file mode 100644 index 0000000000..39e3b8b679 --- /dev/null +++ b/cmake/module/FindQRencode.cmake @@ -0,0 +1,71 @@ +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +#[=======================================================================[ +FindQRencode +------------ + +Finds the QRencode header and library. + +This is a wrapper around find_package()/pkg_check_modules() commands that: + - facilitates searching in various build environments + - prints a standard log message + +#]=======================================================================] + +find_package(PkgConfig QUIET) +if(PKG_CONFIG_FOUND) + pkg_check_modules(PC_QRencode QUIET libqrencode) +endif() + +find_path(QRencode_INCLUDE_DIR + NAMES qrencode.h + PATHS ${PC_QRencode_INCLUDE_DIRS} +) + +find_library(QRencode_LIBRARY_RELEASE + NAMES qrencode + PATHS ${PC_QRencode_LIBRARY_DIRS} +) +find_library(QRencode_LIBRARY_DEBUG + NAMES qrencoded qrencode + PATHS ${PC_QRencode_LIBRARY_DIRS} +) +include(SelectLibraryConfigurations) +select_library_configurations(QRencode) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(QRencode + REQUIRED_VARS QRencode_LIBRARY QRencode_INCLUDE_DIR + VERSION_VAR PC_QRencode_VERSION +) + +if(QRencode_FOUND) + if(NOT TARGET QRencode::QRencode) + add_library(QRencode::QRencode UNKNOWN IMPORTED) + endif() + if(QRencode_LIBRARY_RELEASE) + set_property(TARGET QRencode::QRencode APPEND PROPERTY + IMPORTED_CONFIGURATIONS RELEASE + ) + set_target_properties(QRencode::QRencode PROPERTIES + IMPORTED_LOCATION_RELEASE "${QRencode_LIBRARY_RELEASE}" + ) + endif() + if(QRencode_LIBRARY_DEBUG) + set_property(TARGET QRencode::QRencode APPEND PROPERTY + IMPORTED_CONFIGURATIONS DEBUG + ) + set_target_properties(QRencode::QRencode PROPERTIES + IMPORTED_LOCATION_DEBUG "${QRencode_LIBRARY_DEBUG}" + ) + endif() + set_target_properties(QRencode::QRencode PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${QRencode_INCLUDE_DIR}" + ) +endif() + +mark_as_advanced( + QRencode_INCLUDE_DIR +) diff --git a/cmake/module/FindQt5.cmake b/cmake/module/FindQt.cmake index f39ee53d5b..2e43294a99 100644 --- a/cmake/module/FindQt5.cmake +++ b/cmake/module/FindQt.cmake @@ -3,10 +3,10 @@ # file COPYING or https://opensource.org/license/mit/. #[=======================================================================[ -FindQt5 -------- +FindQt +------ -Finds the Qt 5 headers and libraries. +Finds the Qt headers and libraries. This is a wrapper around find_package() command that: - facilitates searching in various build environments @@ -19,7 +19,7 @@ if(CMAKE_HOST_APPLE) find_program(HOMEBREW_EXECUTABLE brew) if(HOMEBREW_EXECUTABLE) execute_process( - COMMAND ${HOMEBREW_EXECUTABLE} --prefix qt@5 + COMMAND ${HOMEBREW_EXECUTABLE} --prefix qt@${Qt_FIND_VERSION_MAJOR} OUTPUT_VARIABLE _qt_homebrew_prefix ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE @@ -40,10 +40,10 @@ endif() # /usr/x86_64-w64-mingw32/lib/libm.a or /usr/arm-linux-gnueabihf/lib/libm.a. set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH) -find_package(Qt5 ${Qt5_FIND_VERSION} - COMPONENTS ${Qt5_FIND_COMPONENTS} +find_package(Qt${Qt_FIND_VERSION_MAJOR} ${Qt_FIND_VERSION} + COMPONENTS ${Qt_FIND_COMPONENTS} HINTS ${_qt_homebrew_prefix} - PATH_SUFFIXES Qt5 # Required on OpenBSD systems. + PATH_SUFFIXES Qt${Qt_FIND_VERSION_MAJOR} # Required on OpenBSD systems. ) unset(_qt_homebrew_prefix) @@ -56,11 +56,11 @@ else() endif() include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Qt5 - REQUIRED_VARS Qt5_DIR - VERSION_VAR Qt5_VERSION +find_package_handle_standard_args(Qt + REQUIRED_VARS Qt${Qt_FIND_VERSION_MAJOR}_DIR + VERSION_VAR Qt${Qt_FIND_VERSION_MAJOR}_VERSION ) -foreach(component IN LISTS Qt5_FIND_COMPONENTS ITEMS "") - mark_as_advanced(Qt5${component}_DIR) +foreach(component IN LISTS Qt_FIND_COMPONENTS ITEMS "") + mark_as_advanced(Qt${Qt_FIND_VERSION_MAJOR}${component}_DIR) endforeach() diff --git a/cmake/module/FindUSDT.cmake b/cmake/module/FindUSDT.cmake index 0ba9a58fc1..0be7c28ff5 100644 --- a/cmake/module/FindUSDT.cmake +++ b/cmake/module/FindUSDT.cmake @@ -36,13 +36,16 @@ if(USDT_INCLUDE_DIR) include(CheckCXXSourceCompiles) set(CMAKE_REQUIRED_INCLUDES ${USDT_INCLUDE_DIR}) check_cxx_source_compiles(" + // Setting SDT_USE_VARIADIC lets systemtap (sys/sdt.h) know that we want to use + // the optional variadic macros to define tracepoints. + #define SDT_USE_VARIADIC 1 #include <sys/sdt.h> int main() { - DTRACE_PROBE(context, event); + STAP_PROBEV(context, event); int a, b, c, d, e, f, g; - DTRACE_PROBE7(context, event, a, b, c, d, e, f, g); + STAP_PROBEV(context, event, a, b, c, d, e, f, g); } " HAVE_USDT_H ) diff --git a/cmake/module/FindZeroMQ.cmake b/cmake/module/FindZeroMQ.cmake new file mode 100644 index 0000000000..eecd9b2453 --- /dev/null +++ b/cmake/module/FindZeroMQ.cmake @@ -0,0 +1,41 @@ +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +#[=======================================================================[ +FindZeroMQ +---------- + +Finds the ZeroMQ headers and library. + +This is a wrapper around find_package()/pkg_check_modules() commands that: + - facilitates searching in various build environments + - prints a standard log message + +#]=======================================================================] + +include(FindPackageHandleStandardArgs) +find_package(ZeroMQ ${ZeroMQ_FIND_VERSION} NO_MODULE QUIET) +if(ZeroMQ_FOUND) + find_package_handle_standard_args(ZeroMQ + REQUIRED_VARS ZeroMQ_DIR + VERSION_VAR ZeroMQ_VERSION + ) + if(TARGET libzmq) + add_library(zeromq ALIAS libzmq) + elseif(TARGET libzmq-static) + add_library(zeromq ALIAS libzmq-static) + endif() + mark_as_advanced(ZeroMQ_DIR) +else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(libzmq QUIET + IMPORTED_TARGET + libzmq>=${ZeroMQ_FIND_VERSION} + ) + find_package_handle_standard_args(ZeroMQ + REQUIRED_VARS libzmq_LIBRARY_DIRS + VERSION_VAR libzmq_VERSION + ) + add_library(zeromq ALIAS PkgConfig::libzmq) +endif() diff --git a/cmake/module/GenerateSetupNsi.cmake b/cmake/module/GenerateSetupNsi.cmake index 3c358c5495..97a53b071d 100644 --- a/cmake/module/GenerateSetupNsi.cmake +++ b/cmake/module/GenerateSetupNsi.cmake @@ -5,8 +5,8 @@ function(generate_setup_nsi) set(abs_top_srcdir ${PROJECT_SOURCE_DIR}) set(abs_top_builddir ${PROJECT_BINARY_DIR}) - set(PACKAGE_URL ${PROJECT_HOMEPAGE_URL}) - set(PACKAGE_TARNAME "bitcoin") + set(CLIENT_URL ${PROJECT_HOMEPAGE_URL}) + set(CLIENT_TARNAME "bitcoin") set(BITCOIN_GUI_NAME "bitcoin-qt") set(BITCOIN_DAEMON_NAME "bitcoind") set(BITCOIN_CLI_NAME "bitcoin-cli") diff --git a/cmake/module/Maintenance.cmake b/cmake/module/Maintenance.cmake index 456419b722..61251d2439 100644 --- a/cmake/module/Maintenance.cmake +++ b/cmake/module/Maintenance.cmake @@ -98,7 +98,7 @@ function(add_macos_deploy_target) file(CONFIGURE OUTPUT ${macos_app}/Contents/Resources/empty.lproj CONTENT "") configure_file(${PROJECT_SOURCE_DIR}/src/qt/res/icons/bitcoin.icns ${macos_app}/Contents/Resources/bitcoin.icns NO_SOURCE_PERMISSIONS COPYONLY) file(CONFIGURE OUTPUT ${macos_app}/Contents/Resources/Base.lproj/InfoPlist.strings - CONTENT "{ CFBundleDisplayName = \"@PACKAGE_NAME@\"; CFBundleName = \"@PACKAGE_NAME@\"; }" + CONTENT "{ CFBundleDisplayName = \"@CLIENT_NAME@\"; CFBundleName = \"@CLIENT_NAME@\"; }" ) add_custom_command( @@ -109,7 +109,7 @@ function(add_macos_deploy_target) VERBATIM ) - string(REPLACE " " "-" osx_volname ${PACKAGE_NAME}) + string(REPLACE " " "-" osx_volname ${CLIENT_NAME}) if(CMAKE_HOST_APPLE) add_custom_command( OUTPUT ${PROJECT_BINARY_DIR}/${osx_volname}.zip diff --git a/cmake/module/TryAppendCXXFlags.cmake b/cmake/module/TryAppendCXXFlags.cmake index c07455e89e..dc0a9b7f8f 100644 --- a/cmake/module/TryAppendCXXFlags.cmake +++ b/cmake/module/TryAppendCXXFlags.cmake @@ -72,7 +72,8 @@ function(try_append_cxx_flags flags) target_compile_options(${TACXXF_TARGET} INTERFACE ${TACXXF_IF_CHECK_PASSED}) endif() if(DEFINED TACXXF_VAR) - string(STRIP "${${TACXXF_VAR}} ${TACXXF_IF_CHECK_PASSED}" ${TACXXF_VAR}) + list(JOIN TACXXF_IF_CHECK_PASSED " " flags_if_check_passed_as_string) + string(STRIP "${${TACXXF_VAR}} ${flags_if_check_passed_as_string}" ${TACXXF_VAR}) endif() else() if(DEFINED TACXXF_TARGET) diff --git a/cmake/module/TryAppendLinkerFlag.cmake b/cmake/module/TryAppendLinkerFlag.cmake index 8cbd83678d..be41a2e1cc 100644 --- a/cmake/module/TryAppendLinkerFlag.cmake +++ b/cmake/module/TryAppendLinkerFlag.cmake @@ -48,7 +48,8 @@ function(try_append_linker_flag flag) target_link_options(${TALF_TARGET} INTERFACE ${TALF_IF_CHECK_PASSED}) endif() if(DEFINED TALF_VAR) - string(STRIP "${${TALF_VAR}} ${TALF_IF_CHECK_PASSED}" ${TALF_VAR}) + list(JOIN TALF_IF_CHECK_PASSED " " flags_if_check_passed_as_string) + string(STRIP "${${TALF_VAR}} ${flags_if_check_passed_as_string}" ${TALF_VAR}) endif() else() if(DEFINED TALF_TARGET) diff --git a/cmake/script/GenerateHeaderFromJson.cmake b/cmake/script/GenerateHeaderFromJson.cmake index 53d1165272..384ac20d10 100644 --- a/cmake/script/GenerateHeaderFromJson.cmake +++ b/cmake/script/GenerateHeaderFromJson.cmake @@ -2,25 +2,21 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or https://opensource.org/license/mit/. +cmake_path(GET JSON_SOURCE_PATH STEM json_source_basename) + file(READ ${JSON_SOURCE_PATH} hex_content HEX) -string(REGEX MATCHALL "([A-Za-z0-9][A-Za-z0-9])" bytes "${hex_content}") +string(REGEX REPLACE "................" "\\0\n" formatted_bytes "${hex_content}") +string(REGEX REPLACE "[^\n][^\n]" "'\\\\x\\0'," formatted_bytes "${formatted_bytes}") -file(WRITE ${HEADER_PATH} "#include <string_view>\n") -file(APPEND ${HEADER_PATH} "namespace json_tests{\n") -get_filename_component(json_source_basename ${JSON_SOURCE_PATH} NAME_WE) -file(APPEND ${HEADER_PATH} "inline constexpr char detail_${json_source_basename}_bytes[]{\n") +set(header_content +"#include <string_view> -set(i 0) -foreach(byte ${bytes}) - math(EXPR i "${i} + 1") - math(EXPR remainder "${i} % 8") - if(remainder EQUAL 0) - file(APPEND ${HEADER_PATH} "0x${byte},\n") - else() - file(APPEND ${HEADER_PATH} "0x${byte}, ") - endif() -endforeach() +namespace json_tests { +inline constexpr char detail_${json_source_basename}_bytes[] { +${formatted_bytes} +}; -file(APPEND ${HEADER_PATH} "\n};\n") -file(APPEND ${HEADER_PATH} "inline constexpr std::string_view ${json_source_basename}{std::begin(detail_${json_source_basename}_bytes), std::end(detail_${json_source_basename}_bytes)};") -file(APPEND ${HEADER_PATH} "\n}") +inline constexpr std::string_view ${json_source_basename}{std::begin(detail_${json_source_basename}_bytes), std::end(detail_${json_source_basename}_bytes)}; +} +") +file(WRITE ${HEADER_PATH} "${header_content}") diff --git a/cmake/script/GenerateHeaderFromRaw.cmake b/cmake/script/GenerateHeaderFromRaw.cmake index 18a6c2b407..d373d1c4f8 100644 --- a/cmake/script/GenerateHeaderFromRaw.cmake +++ b/cmake/script/GenerateHeaderFromRaw.cmake @@ -2,26 +2,22 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or https://opensource.org/license/mit/. +cmake_path(GET RAW_SOURCE_PATH STEM raw_source_basename) + file(READ ${RAW_SOURCE_PATH} hex_content HEX) -string(REGEX MATCHALL "([A-Za-z0-9][A-Za-z0-9])" bytes "${hex_content}") +string(REGEX REPLACE "................" "\\0\n" formatted_bytes "${hex_content}") +string(REGEX REPLACE "[^\n][^\n]" "std::byte{0x\\0}," formatted_bytes "${formatted_bytes}") -file(WRITE ${HEADER_PATH} "#include <cstddef>\n") -file(APPEND ${HEADER_PATH} "#include <span>\n") -file(APPEND ${HEADER_PATH} "namespace ${RAW_NAMESPACE} {\n") -get_filename_component(raw_source_basename ${RAW_SOURCE_PATH} NAME_WE) -file(APPEND ${HEADER_PATH} "inline constexpr std::byte detail_${raw_source_basename}_raw[]{\n") +set(header_content +"#include <cstddef> +#include <span> -set(i 0) -foreach(byte ${bytes}) - math(EXPR i "${i} + 1") - math(EXPR remainder "${i} % 8") - if(remainder EQUAL 0) - file(APPEND ${HEADER_PATH} "std::byte{0x${byte}},\n") - else() - file(APPEND ${HEADER_PATH} "std::byte{0x${byte}}, ") - endif() -endforeach() +namespace ${RAW_NAMESPACE} { +inline constexpr std::byte detail_${raw_source_basename}_raw[] { +${formatted_bytes} +}; -file(APPEND ${HEADER_PATH} "\n};\n") -file(APPEND ${HEADER_PATH} "inline constexpr std::span ${raw_source_basename}{detail_${raw_source_basename}_raw};\n") -file(APPEND ${HEADER_PATH} "}") +inline constexpr std::span ${raw_source_basename}{detail_${raw_source_basename}_raw}; +} +") +file(WRITE ${HEADER_PATH} "${header_content}") diff --git a/contrib/devtools/bitcoin-tidy/CMakeLists.txt b/contrib/devtools/bitcoin-tidy/CMakeLists.txt index 95345b4782..c6f683f7ab 100644 --- a/contrib/devtools/bitcoin-tidy/CMakeLists.txt +++ b/contrib/devtools/bitcoin-tidy/CMakeLists.txt @@ -25,7 +25,7 @@ find_program(CLANG_TIDY_EXE NAMES "clang-tidy-${LLVM_VERSION_MAJOR}" "clang-tidy message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Found clang-tidy: ${CLANG_TIDY_EXE}") -add_library(bitcoin-tidy MODULE bitcoin-tidy.cpp logprintf.cpp nontrivial-threadlocal.cpp) +add_library(bitcoin-tidy MODULE bitcoin-tidy.cpp nontrivial-threadlocal.cpp) target_include_directories(bitcoin-tidy SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) # Disable RTTI and exceptions as necessary @@ -58,7 +58,7 @@ else() endif() # Create a dummy library that runs clang-tidy tests as a side-effect of building -add_library(bitcoin-tidy-tests OBJECT EXCLUDE_FROM_ALL example_logprintf.cpp example_nontrivial-threadlocal.cpp) +add_library(bitcoin-tidy-tests OBJECT EXCLUDE_FROM_ALL example_nontrivial-threadlocal.cpp) add_dependencies(bitcoin-tidy-tests bitcoin-tidy) set_target_properties(bitcoin-tidy-tests PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_COMMAND}") diff --git a/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp b/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp index 1ef4494973..f2658b5a58 100644 --- a/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp +++ b/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp @@ -2,7 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "logprintf.h" #include "nontrivial-threadlocal.h" #include <clang-tidy/ClangTidyModule.h> @@ -13,7 +12,6 @@ class BitcoinModule final : public clang::tidy::ClangTidyModule public: void addCheckFactories(clang::tidy::ClangTidyCheckFactories& CheckFactories) override { - CheckFactories.registerCheck<bitcoin::LogPrintfCheck>("bitcoin-unterminated-logprintf"); CheckFactories.registerCheck<bitcoin::NonTrivialThreadLocal>("bitcoin-nontrivial-threadlocal"); } }; diff --git a/contrib/devtools/bitcoin-tidy/example_logprintf.cpp b/contrib/devtools/bitcoin-tidy/example_logprintf.cpp deleted file mode 100644 index dc77f668e3..0000000000 --- a/contrib/devtools/bitcoin-tidy/example_logprintf.cpp +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (c) 2023 Bitcoin Developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <string> - -// Test for bitcoin-unterminated-logprintf - -enum LogFlags { - NONE -}; - -enum Level { - None -}; - -template <typename... Args> -static inline void LogPrintf_(const std::string& logging_function, const std::string& source_file, const int source_line, const LogFlags flag, const Level level, const char* fmt, const Args&... args) -{ -} - -#define LogPrintLevel_(category, level, ...) LogPrintf_(__func__, __FILE__, __LINE__, category, level, __VA_ARGS__) -#define LogPrintf(...) LogPrintLevel_(LogFlags::NONE, Level::None, __VA_ARGS__) - -#define LogDebug(category, ...) \ - do { \ - LogPrintf(__VA_ARGS__); \ - } while (0) - - -class CWallet -{ - std::string GetDisplayName() const - { - return "default wallet"; - } - -public: - template <typename... Params> - void WalletLogPrintf(const char* fmt, Params... parameters) const - { - LogPrintf(("%s " + std::string{fmt}).c_str(), GetDisplayName(), parameters...); - }; -}; - -struct ScriptPubKeyMan -{ - std::string GetDisplayName() const - { - return "default wallet"; - } - - template <typename... Params> - void WalletLogPrintf(const char* fmt, Params... parameters) const - { - LogPrintf(("%s " + std::string{fmt}).c_str(), GetDisplayName(), parameters...); - }; -}; - -void good_func() -{ - LogPrintf("hello world!\n"); -} -void good_func2() -{ - CWallet wallet; - wallet.WalletLogPrintf("hi\n"); - ScriptPubKeyMan spkm; - spkm.WalletLogPrintf("hi\n"); - - const CWallet& walletref = wallet; - walletref.WalletLogPrintf("hi\n"); - - auto* walletptr = new CWallet(); - walletptr->WalletLogPrintf("hi\n"); - delete walletptr; -} -void bad_func() -{ - LogPrintf("hello world!"); -} -void bad_func2() -{ - LogPrintf(""); -} -void bad_func3() -{ - // Ending in "..." has no special meaning. - LogPrintf("hello world!..."); -} -void bad_func4_ignored() -{ - LogPrintf("hello world!"); // NOLINT(bitcoin-unterminated-logprintf) -} -void bad_func5() -{ - CWallet wallet; - wallet.WalletLogPrintf("hi"); - ScriptPubKeyMan spkm; - spkm.WalletLogPrintf("hi"); - - const CWallet& walletref = wallet; - walletref.WalletLogPrintf("hi"); - - auto* walletptr = new CWallet(); - walletptr->WalletLogPrintf("hi"); - delete walletptr; -} diff --git a/contrib/devtools/bitcoin-tidy/logprintf.cpp b/contrib/devtools/bitcoin-tidy/logprintf.cpp deleted file mode 100644 index 36beac28c8..0000000000 --- a/contrib/devtools/bitcoin-tidy/logprintf.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2023 Bitcoin Developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "logprintf.h" - -#include <clang/AST/ASTContext.h> -#include <clang/ASTMatchers/ASTMatchFinder.h> - - -namespace { -AST_MATCHER(clang::StringLiteral, unterminated) -{ - size_t len = Node.getLength(); - if (len > 0 && Node.getCodeUnit(len - 1) == '\n') { - return false; - } - return true; -} -} // namespace - -namespace bitcoin { - -void LogPrintfCheck::registerMatchers(clang::ast_matchers::MatchFinder* finder) -{ - using namespace clang::ast_matchers; - - /* - Logprintf(..., ..., ..., ..., ..., "foo", ...) - */ - - finder->addMatcher( - callExpr( - callee(functionDecl(hasName("LogPrintf_"))), - hasArgument(5, stringLiteral(unterminated()).bind("logstring"))), - this); - - /* - auto walletptr = &wallet; - wallet.WalletLogPrintf("foo"); - wallet->WalletLogPrintf("foo"); - */ - finder->addMatcher( - cxxMemberCallExpr( - callee(cxxMethodDecl(hasName("WalletLogPrintf"))), - hasArgument(0, stringLiteral(unterminated()).bind("logstring"))), - this); -} - -void LogPrintfCheck::check(const clang::ast_matchers::MatchFinder::MatchResult& Result) -{ - if (const clang::StringLiteral* lit = Result.Nodes.getNodeAs<clang::StringLiteral>("logstring")) { - const clang::ASTContext& ctx = *Result.Context; - const auto user_diag = diag(lit->getEndLoc(), "Unterminated format string used with LogPrintf"); - const auto& loc = lit->getLocationOfByte(lit->getByteLength(), *Result.SourceManager, ctx.getLangOpts(), ctx.getTargetInfo()); - user_diag << clang::FixItHint::CreateInsertion(loc, "\\n"); - } -} - -} // namespace bitcoin diff --git a/contrib/devtools/bitcoin-tidy/logprintf.h b/contrib/devtools/bitcoin-tidy/logprintf.h deleted file mode 100644 index db95dfe143..0000000000 --- a/contrib/devtools/bitcoin-tidy/logprintf.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2023 Bitcoin Developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef LOGPRINTF_CHECK_H -#define LOGPRINTF_CHECK_H - -#include <clang-tidy/ClangTidyCheck.h> - -namespace bitcoin { - -// Warn about any use of LogPrintf that does not end with a newline. -class LogPrintfCheck final : public clang::tidy::ClangTidyCheck -{ -public: - LogPrintfCheck(clang::StringRef Name, clang::tidy::ClangTidyContext* Context) - : clang::tidy::ClangTidyCheck(Name, Context) {} - - bool isLanguageVersionSupported(const clang::LangOptions& LangOpts) const override - { - return LangOpts.CPlusPlus; - } - void registerMatchers(clang::ast_matchers::MatchFinder* Finder) override; - void check(const clang::ast_matchers::MatchFinder::MatchResult& Result) override; -}; - -} // namespace bitcoin - -#endif // LOGPRINTF_CHECK_H diff --git a/contrib/devtools/check-deps.sh b/contrib/devtools/check-deps.sh index 3bd48b444a..cdfc4e7533 100755 --- a/contrib/devtools/check-deps.sh +++ b/contrib/devtools/check-deps.sh @@ -58,7 +58,7 @@ usage() { echo "Usage: $(basename "${BASH_SOURCE[0]}") [BUILD_DIR]" } -# Output makefile targets, converting library .a paths to libtool .la targets +# Output makefile targets, converting library .a paths to CMake targets lib_targets() { for lib in "${!LIBS[@]}"; do for lib_path in ${LIBS[$lib]}; do @@ -133,7 +133,7 @@ check_disallowed() { dst_obj=$(obj_names "$symbol" "${temp_dir}/${dst}_exports.txt") while read src_obj; do if ! check_suppress "$src_obj" "$dst_obj" "$symbol"; then - echo "Error: $src_obj depends on $dst_obj symbol '$(c++filt "$symbol")', can suppess with:" + echo "Error: $src_obj depends on $dst_obj symbol '$(c++filt "$symbol")', can suppress with:" echo " SUPPRESS[\"$src_obj $dst_obj $symbol\"]=1" result=1 fi @@ -145,7 +145,7 @@ check_disallowed() { # Declare array to track errors which were suppressed. declare -A SUPPRESSED -# Return whether error should be suppressed and record suppresssion in +# Return whether error should be suppressed and record suppression in # SUPPRESSED array. check_suppress() { local src_obj="$1" @@ -161,7 +161,7 @@ check_suppress() { return 1 } -# Warn about error which were supposed to be suppress, but were not encountered. +# Warn about error which were supposed to be suppressed, but were not encountered. check_not_suppressed() { for suppress in "${!SUPPRESS[@]}"; do if [[ ! -v SUPPRESSED[$suppress] ]]; then @@ -194,7 +194,10 @@ cd "$BUILD_DIR/src" extract_symbols "$TEMP_DIR" if check_libraries "$TEMP_DIR"; then echo "Success! No unexpected dependencies were detected." + RET=0 else echo >&2 "Error: Unexpected dependencies were detected. Check previous output." + RET=1 fi rm -r "$TEMP_DIR" +exit $RET diff --git a/contrib/devtools/gen-bitcoin-conf.sh b/contrib/devtools/gen-bitcoin-conf.sh index 2ebbd42022..d830852c9e 100755 --- a/contrib/devtools/gen-bitcoin-conf.sh +++ b/contrib/devtools/gen-bitcoin-conf.sh @@ -72,9 +72,12 @@ cat >> "${EXAMPLE_CONF_FILE}" << 'EOF' # Options for mainnet [main] -# Options for testnet +# Options for testnet3 [test] +# Options for testnet4 +[testnet4] + # Options for signet [signet] diff --git a/contrib/devtools/gen-manpages.py b/contrib/devtools/gen-manpages.py index 92acd9a403..14c8c408e8 100755 --- a/contrib/devtools/gen-manpages.py +++ b/contrib/devtools/gen-manpages.py @@ -6,6 +6,7 @@ import os import subprocess import sys import tempfile +import argparse BINARIES = [ 'src/bitcoind', @@ -16,6 +17,18 @@ BINARIES = [ 'src/qt/bitcoin-qt', ] +parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, +) +parser.add_argument( + "-s", + "--skip-missing-binaries", + action="store_true", + default=False, + help="skip generation for binaries that are not found in the build path", +) +args = parser.parse_args() + # Paths to external utilities. git = os.getenv('GIT', 'git') help2man = os.getenv('HELP2MAN', 'help2man') @@ -38,8 +51,12 @@ for relpath in BINARIES: try: r = subprocess.run([abspath, "--version"], stdout=subprocess.PIPE, check=True, text=True) except IOError: - print(f'{abspath} not found or not an executable', file=sys.stderr) - sys.exit(1) + if(args.skip_missing_binaries): + print(f'{abspath} not found or not an executable. Skipping...', file=sys.stderr) + continue + else: + print(f'{abspath} not found or not an executable', file=sys.stderr) + sys.exit(1) # take first line (which must contain version) verstr = r.stdout.splitlines()[0] # last word of line is the actual version e.g. v22.99.0-5c6b3d5b3508 @@ -51,6 +68,10 @@ for relpath in BINARIES: versions.append((abspath, verstr, copyright)) +if not versions: + print(f'No binaries found in {builddir}. Please ensure the binaries are present in {builddir}, or set another build path using the BUILDDIR env variable.') + sys.exit(1) + if any(verstr.endswith('-dirty') for (_, verstr, _) in versions): print("WARNING: Binaries were built from a dirty tree.") print('man pages generated from dirty binaries should NOT be committed.') diff --git a/contrib/devtools/iwyu/bitcoin.core.imp b/contrib/devtools/iwyu/bitcoin.core.imp index befc949f18..c4c4ba4ceb 100644 --- a/contrib/devtools/iwyu/bitcoin.core.imp +++ b/contrib/devtools/iwyu/bitcoin.core.imp @@ -1,4 +1,3 @@ -# Fixups / upstreamed changes +# Nothing for now. [ - { include: [ "<bits/chrono.h>", private, "<chrono>", public ] }, ] diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 1722c7d290..564f1db5ac 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -235,7 +235,7 @@ def check_MACHO_libraries(binary) -> bool: return ok def check_MACHO_min_os(binary) -> bool: - if binary.build_version.minos == [11,0,0]: + if binary.build_version.minos == [13,0,0]: return True return False @@ -260,7 +260,7 @@ def check_PE_libraries(binary) -> bool: def check_PE_subsystem_version(binary) -> bool: major: int = binary.optional_header.major_subsystem_version minor: int = binary.optional_header.minor_subsystem_version - if major == 6 and minor == 1: + if major == 6 and minor == 2: return True return False diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index 454dbc6bd2..b4b524dac0 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -116,7 +116,7 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cxx, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,11.0', '-Wl,11.4']), + self.assertEqual(call_symbol_check(cxx, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,13.0', '-Wl,11.4']), (1, f'{executable}: failed SDK')) def test_PE(self): @@ -135,7 +135,7 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cxx, source, executable, ['-lpdh', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,1']), + self.assertEqual(call_symbol_check(cxx, source, executable, ['-lpdh', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,2']), (1, 'pdh.dll is not in ALLOWED_LIBRARIES!\n' + executable + ': failed DYNAMIC_LIBRARIES')) @@ -166,7 +166,7 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cxx, source, executable, ['-lole32', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,1']), + self.assertEqual(call_symbol_check(cxx, source, executable, ['-lole32', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,2']), (0, '')) diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh index 23c260b529..885396bb25 100755 --- a/contrib/devtools/test_deterministic_coverage.sh +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -81,7 +81,7 @@ if ! command -v gcovr > /dev/null; then fi if [[ ! -e ${TEST_BITCOIN_BINARY} ]]; then - echo "Error: Executable ${TEST_BITCOIN_BINARY} not found. Run \"./configure --enable-lcov\" and compile." + echo "Error: Executable ${TEST_BITCOIN_BINARY} not found. Run \"cmake -B build -DCMAKE_BUILD_TYPE=Coverage\" and compile." exit 1 fi @@ -90,7 +90,7 @@ get_file_suffix_count() { } if [[ $(get_file_suffix_count gcno) == 0 ]]; then - echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"./configure --enable-lcov\" and re-compile." + echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"cmake -B build -DCMAKE_BUILD_TYPE=Coverage\" and re-compile." exit 1 fi @@ -115,7 +115,7 @@ while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do fi rm "${TEST_OUTPUT_TEMPFILE}" if [[ $(get_file_suffix_count gcda) == 0 ]]; then - echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"./configure --enable-lcov\" and re-compile." + echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"cmake -B build -DCMAKE_BUILD_TYPE=Coverage\" and re-compile." exit 1 fi GCOVR_TEMPFILE=$(mktemp) diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 5e05e1016e..2c9056ce9c 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -31,7 +31,7 @@ section](#choosing-your-security-model) before proceeding to perform a build. In order to perform a build for macOS (which is included in the default set of platform triples to build), you'll need to extract the macOS SDK tarball using -tools found in the [`macdeploy` directory](../macdeploy/README.md). +tools found in the [`macdeploy` directory](../macdeploy/README.md#sdk-extraction). You can then either point to the SDK using the `SDK_PATH` environment variable: @@ -68,7 +68,7 @@ following from the top of a clean repository: The `guix-codesign` command attaches codesignatures (produced by codesigners) to existing non-codesigned outputs. Please see the [release process -documentation](/doc/release-process.md) for more context. +documentation](/doc/release-process.md#codesigning) for more context. It respects many of the same environment variable flags as `guix-build`, with 2 crucial differences: diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 1ffc22a76b..6c252e7870 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -212,7 +212,6 @@ CONFIGFLAGS="-DREDUCE_EXPORTS=ON -DBUILD_BENCH=OFF -DBUILD_GUI_TESTS=OFF -DBUILD HOST_CFLAGS="-O2 -g" HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) case "$HOST" in - *linux*) HOST_CFLAGS+=" -ffile-prefix-map=${DISTSRC}/src=." ;; *mingw*) HOST_CFLAGS+=" -fno-ident" ;; *darwin*) unset HOST_CFLAGS ;; esac @@ -230,8 +229,6 @@ case "$HOST" in *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac -# Make $HOST-specific native binaries from depends available in $PATH -export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" mkdir -p "$DISTSRC" ( cd "$DISTSRC" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 5f62765a65..3244a8dbdc 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -10,10 +10,9 @@ (gnu packages gawk) (gnu packages gcc) ((gnu packages installers) #:select (nsis-x86_64)) - ((gnu packages linux) #:select (linux-libre-headers-6.1 util-linux)) + ((gnu packages linux) #:select (linux-libre-headers-6.1)) (gnu packages llvm) (gnu packages mingw) - (gnu packages moreutils) (gnu packages pkg-config) ((gnu packages python) #:select (python-minimal)) ((gnu packages python-build) #:select (python-tomli)) @@ -21,6 +20,7 @@ ((gnu packages tls) #:select (openssl)) ((gnu packages version-control) #:select (git-minimal)) (guix build-system cmake) + (guix build-system gnu) (guix build-system python) (guix build-system trivial) (guix download) @@ -28,7 +28,7 @@ (guix git-download) ((guix licenses) #:prefix license:) (guix packages) - ((guix utils) #:select (substitute-keyword-arguments))) + ((guix utils) #:select (cc-for-target substitute-keyword-arguments))) (define-syntax-rule (search-our-patches file-name ...) "Return the list of absolute file names corresponding to each @@ -420,6 +420,7 @@ inspecting signatures in Mach-O binaries.") ;; https://gcc.gnu.org/install/configure.html (list "--enable-threads=posix", "--enable-default-ssp=yes", + "--disable-gcov", building-on))))))) (define-public linux-base-gcc @@ -434,6 +435,8 @@ inspecting signatures in Mach-O binaries.") "--enable-default-ssp=yes", "--enable-default-pie=yes", "--enable-standard-branch-protection=yes", + "--enable-cet=yes", + "--disable-gcov", building-on))) ((#:phases phases) `(modify-phases ,phases @@ -448,7 +451,7 @@ inspecting signatures in Mach-O binaries.") #t)))))))) (define-public glibc-2.31 - (let ((commit "8e30f03744837a85e33d84ccd34ed3abe30d37c3")) + (let ((commit "7b27c450c34563a28e634cccb399cd415e71ebfe")) (package (inherit glibc) ;; 2.35 (version "2.31") @@ -460,7 +463,7 @@ inspecting signatures in Mach-O binaries.") (file-name (git-file-name "glibc" commit)) (sha256 (base32 - "1zi0s9yy5zkisw823vivn7zlj8w6g9p3mm7lmlqiixcxdkz4dbn6")) + "017qdpr5id7ddb4lpkzj2li1abvw916m3fc6n7nw28z4h5qbv2n0")) (patches (search-our-patches "glibc-guix-prefix.patch")))) (arguments (substitute-keyword-arguments (package-arguments glibc) @@ -468,8 +471,11 @@ inspecting signatures in Mach-O binaries.") `(append ,flags ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html (list "--enable-stack-protector=all", + "--enable-cet", "--enable-bind-now", "--disable-werror", + "--disable-timezone-tools", + "--disable-profile", building-on))) ((#:phases phases) `(modify-phases ,phases @@ -485,13 +491,42 @@ inspecting signatures in Mach-O binaries.") (("^install-others =.*$") (string-append "install-others = " out "/etc/rpc\n"))))))))))))) +;; The sponge tool from moreutils. +(define-public sponge + (package + (name "sponge") + (version "0.69") + (source (origin + (method url-fetch) + (uri (string-append + "https://git.joeyh.name/index.cgi/moreutils.git/snapshot/ + moreutils-" version ".tar.gz")) + (file-name (string-append "moreutils-" version ".tar.gz")) + (sha256 + (base32 + "1l859qnzccslvxlh5ghn863bkq2vgmqgnik6jr21b9kc6ljmsy8g")))) + (build-system gnu-build-system) + (arguments + (list #:phases + #~(modify-phases %standard-phases + (delete 'configure) + (replace 'install + (lambda* (#:key outputs #:allow-other-keys) + (let ((bin (string-append (assoc-ref outputs "out") "/bin"))) + (install-file "sponge" bin))))) + #:make-flags + #~(list "sponge" (string-append "CC=" #$(cc-for-target))))) + (home-page "https://joeyh.name/code/moreutils/") + (synopsis "Miscellaneous general-purpose command-line tools") + (description "Just sponge") + (license license:gpl2+))) + (packages->manifest (append (list ;; The Basics bash-minimal which coreutils-minimal - util-linux ;; File(system) inspection file grep @@ -501,7 +536,7 @@ inspecting signatures in Mach-O binaries.") patch gawk sed - moreutils + sponge ;; Compression and archiving tar gzip @@ -510,7 +545,6 @@ inspecting signatures in Mach-O binaries.") gcc-toolchain-12 cmake-minimal gnu-make - pkg-config ;; Scripting python-minimal ;; (3.10) ;; Git @@ -526,6 +560,7 @@ inspecting signatures in Mach-O binaries.") osslsigncode)) ((string-contains target "-linux-") (list bison + pkg-config (list gcc-toolchain-12 "static") (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") diff --git a/contrib/seeds/README.md b/contrib/seeds/README.md index fe469aee9e..10945e5b68 100644 --- a/contrib/seeds/README.md +++ b/contrib/seeds/README.md @@ -9,7 +9,7 @@ changes its default return value, as those are the services which seeds are adde to addrman with). The seeds compiled into the release are created from sipa's, achow101's and luke-jr's -DNS seed, virtu's crawler, and fjahr's community AS map data. Run the following commands +DNS seed, virtu's crawler, and asmap community AS map data. Run the following commands from the `/contrib/seeds` directory: ``` @@ -18,7 +18,7 @@ curl https://mainnet.achownodes.xyz/seeds.txt.gz | gzip -dc >> seeds_main.txt curl https://21.ninja/seeds.txt.gz | gzip -dc >> seeds_main.txt curl https://luke.dashjr.org/programs/bitcoin/files/charts/seeds.txt >> seeds_main.txt curl https://testnet.achownodes.xyz/seeds.txt.gz | gzip -dc > seeds_test.txt -curl https://raw.githubusercontent.com/fjahr/asmap-data/main/latest_asmap.dat > asmap-filled.dat +curl https://raw.githubusercontent.com/asmap/asmap-data/main/latest_asmap.dat > asmap-filled.dat python3 makeseeds.py -a asmap-filled.dat -s seeds_main.txt > nodes_main.txt python3 makeseeds.py -a asmap-filled.dat -s seeds_test.txt > nodes_test.txt # TODO: Uncomment when a seeder publishes seeds.txt.gz for testnet4 diff --git a/contrib/signet/README.md b/contrib/signet/README.md index 706b296c54..5fcd8944e6 100644 --- a/contrib/signet/README.md +++ b/contrib/signet/README.md @@ -23,9 +23,8 @@ miner You will first need to pick a difficulty target. Since signet chains are primarily protected by a signature rather than proof of work, there is no need to spend as much energy as possible mining, however you may wish to choose to spend more time than the absolute minimum. The calibrate subcommand can be used to pick a target appropriate for your hardware, eg: - cd src/ - MINER="../contrib/signet/miner" - GRIND="./bitcoin-util grind" + MINER="./contrib/signet/miner" + GRIND="./build/src/bitcoin-util grind" $MINER calibrate --grind-cmd="$GRIND" nbits=1e00f403 for 25s average mining time @@ -33,7 +32,7 @@ It defaults to estimating an nbits value resulting in 25s average time to find a To mine the first block in your custom chain, you can run: - CLI="./bitcoin-cli -conf=mysignet.conf" + CLI="./build/src/bitcoin-cli -conf=mysignet.conf" ADDR=$($CLI -signet getnewaddress) NBITS=1e00f403 $MINER --cli="$CLI" generate --grind-cmd="$GRIND" --address="$ADDR" --nbits=$NBITS diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md index c471770a7d..cf59d7e2bb 100644 --- a/contrib/tracing/README.md +++ b/contrib/tracing/README.md @@ -82,7 +82,7 @@ about the connection. Peers can be selected individually to view recent P2P messages. ``` -$ python3 contrib/tracing/p2p_monitor.py ./build/src/bitcoind +$ python3 contrib/tracing/p2p_monitor.py $(pidof bitcoind) ``` Lists selectable peers and traffic and connection information. @@ -150,7 +150,7 @@ lost. BCC prints: `Possibly lost 2 samples` on lost messages. ``` -$ python3 contrib/tracing/log_raw_p2p_msgs.py ./build/src/bitcoind +$ python3 contrib/tracing/log_raw_p2p_msgs.py $(pidof bitcoind) ``` ``` @@ -241,7 +241,7 @@ A BCC Python script to log the UTXO cache flushes. Based on the `utxocache:flush` tracepoint. ```bash -$ python3 contrib/tracing/log_utxocache_flush.py ./build/src/bitcoind +$ python3 contrib/tracing/log_utxocache_flush.py $(pidof bitcoind) ``` ``` @@ -300,7 +300,7 @@ comprising a timestamp along with all event data available via the event's tracepoint. ```console -$ python3 contrib/tracing/mempool_monitor.py ./build/src/bitcoind +$ python3 contrib/tracing/mempool_monitor.py $(pidof bitcoind) ``` ``` diff --git a/contrib/tracing/connectblock_benchmark.bt b/contrib/tracing/connectblock_benchmark.bt index 4aa4742103..de112af639 100755 --- a/contrib/tracing/connectblock_benchmark.bt +++ b/contrib/tracing/connectblock_benchmark.bt @@ -82,7 +82,7 @@ usdt:./build/src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 @inputs = @inputs + $inputs; @sigops = @sigops + $sigops; - @durations = hist($duration / 1000); + @durations = hist($duration / 1e6); if ($height == $1 && $height != 0) { @start = nsecs; @@ -92,7 +92,7 @@ usdt:./build/src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 if ($2 > 0 && $height >= $2) { @end = nsecs; $duration = @end - @start; - printf("\nTook %d ms to connect the blocks between height %d and %d.\n", $duration / 1000000, $1, $2); + printf("\nTook %d ms to connect the blocks between height %d and %d.\n", $duration / 1e9, $1, $2); exit(); } } @@ -102,7 +102,7 @@ usdt:./build/src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 blocks where the time it took to connect the block is above the <logging threshold in ms>. */ -usdt:./build/src/bitcoind:validation:block_connected / (uint64) arg5 / 1000> $3 / +usdt:./build/src/bitcoind:validation:block_connected / (uint64) arg5 / 1e6 > $3 / { $hash = arg0; $height = (int32) arg1; @@ -120,7 +120,7 @@ usdt:./build/src/bitcoind:validation:block_connected / (uint64) arg5 / 1000> $3 printf("%02x", $b); $p -= 1; } - printf(") %4d tx %5d ins %5d sigops took %4d ms\n", $transactions, $inputs, $sigops, (uint64) $duration / 1000); + printf(") %4d tx %5d ins %5d sigops took %4d ms\n", $transactions, $inputs, $sigops, (uint64) $duration / 1e6); } diff --git a/contrib/tracing/log_raw_p2p_msgs.py b/contrib/tracing/log_raw_p2p_msgs.py index c0ab704106..9cda7fd08d 100755 --- a/contrib/tracing/log_raw_p2p_msgs.py +++ b/contrib/tracing/log_raw_p2p_msgs.py @@ -117,9 +117,9 @@ int trace_outbound_message(struct pt_regs *ctx) { def print_message(event, inbound): - print(f"%s %s msg '%s' from peer %d (%s, %s) with %d bytes: %s" % - ( - f"Warning: incomplete message (only %d out of %d bytes)!" % ( + print("{} {} msg '{}' from peer {} ({}, {}) with {} bytes: {}".format( + + "Warning: incomplete message (only {} out of {} bytes)!".format( len(event.msg), event.msg_size) if len(event.msg) < event.msg_size else "", "inbound" if inbound else "outbound", event.msg_type.decode("utf-8"), @@ -132,8 +132,9 @@ def print_message(event, inbound): ) -def main(bitcoind_path): - bitcoind_with_usdts = USDT(path=str(bitcoind_path)) +def main(pid): + print(f"Hooking into bitcoind with pid {pid}") + bitcoind_with_usdts = USDT(pid=int(pid)) # attaching the trace functions defined in the BPF program to the tracepoints bitcoind_with_usdts.enable_probe( @@ -176,8 +177,8 @@ def main(bitcoind_path): if __name__ == "__main__": - if len(sys.argv) < 2: - print("USAGE:", sys.argv[0], "path/to/bitcoind") + if len(sys.argv) != 2: + print("USAGE:", sys.argv[0], "<pid of bitcoind>") exit() - path = sys.argv[1] - main(path) + pid = sys.argv[1] + main(pid) diff --git a/contrib/tracing/log_utxocache_flush.py b/contrib/tracing/log_utxocache_flush.py index 6c568998e9..8ff9cd5e03 100755 --- a/contrib/tracing/log_utxocache_flush.py +++ b/contrib/tracing/log_utxocache_flush.py @@ -70,8 +70,9 @@ def print_event(event): )) -def main(bitcoind_path): - bitcoind_with_usdts = USDT(path=str(bitcoind_path)) +def main(pid): + print(f"Hooking into bitcoind with pid {pid}") + bitcoind_with_usdts = USDT(pid=int(pid)) # attaching the trace functions defined in the BPF program # to the tracepoints @@ -99,9 +100,9 @@ def main(bitcoind_path): if __name__ == "__main__": - if len(sys.argv) < 2: - print("USAGE: ", sys.argv[0], "path/to/bitcoind") + if len(sys.argv) != 2: + print("USAGE: ", sys.argv[0], "<pid of bitcoind>") exit(1) - path = sys.argv[1] - main(path) + pid = sys.argv[1] + main(pid) diff --git a/contrib/tracing/mempool_monitor.py b/contrib/tracing/mempool_monitor.py index afb5e60372..4492cb7570 100755 --- a/contrib/tracing/mempool_monitor.py +++ b/contrib/tracing/mempool_monitor.py @@ -114,8 +114,9 @@ int trace_replaced(struct pt_regs *ctx) { """ -def main(bitcoind_path): - bitcoind_with_usdts = USDT(path=str(bitcoind_path)) +def main(pid): + print(f"Hooking into bitcoind with pid {pid}") + bitcoind_with_usdts = USDT(pid=int(pid)) # attaching the trace functions defined in the BPF program # to the tracepoints @@ -365,8 +366,8 @@ class Dashboard: if __name__ == "__main__": if len(sys.argv) < 2: - print("USAGE: ", sys.argv[0], "path/to/bitcoind") + print("USAGE: ", sys.argv[0], "<pid of bitcoind>") exit(1) - path = sys.argv[1] - main(path) + pid = sys.argv[1] + main(pid) diff --git a/contrib/tracing/p2p_monitor.py b/contrib/tracing/p2p_monitor.py index 4ff701cac3..63a27fc17d 100755 --- a/contrib/tracing/p2p_monitor.py +++ b/contrib/tracing/p2p_monitor.py @@ -14,8 +14,9 @@ # outbound P2P messages. The eBPF program submits the P2P messages to # this script via a BPF ring buffer. -import sys import curses +import os +import sys from curses import wrapper, panel from bcc import BPF, USDT @@ -115,10 +116,10 @@ class Peer: self.total_outbound_msgs += 1 -def main(bitcoind_path): +def main(pid): peers = dict() - - bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + print(f"Hooking into bitcoind with pid {pid}") + bitcoind_with_usdts = USDT(pid=int(pid)) # attaching the trace functions defined in the BPF program to the tracepoints bitcoind_with_usdts.enable_probe( @@ -245,9 +246,14 @@ def render(screen, peers, cur_list_pos, scroll, ROWS_AVALIABLE_FOR_LIST, info_pa (msg.msg_type, msg.size), curses.A_NORMAL) +def running_as_root(): + return os.getuid() == 0 + if __name__ == "__main__": - if len(sys.argv) < 2: - print("USAGE:", sys.argv[0], "path/to/bitcoind") + if len(sys.argv) != 2: + print("USAGE:", sys.argv[0], "<pid of bitcoind>") exit() - path = sys.argv[1] - main(path) + if not running_as_root(): + print("You might not have the privileges required to hook into the tracepoints!") + pid = sys.argv[1] + main(pid) diff --git a/depends/.gitignore b/depends/.gitignore index be6280f599..c53f7f610e 100644 --- a/depends/.gitignore +++ b/depends/.gitignore @@ -3,6 +3,7 @@ work/ built/ sources/ x86_64* +amd64* i686* mips* arm* diff --git a/depends/Makefile b/depends/Makefile index ad1fb2b049..a2a9f6823e 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -40,9 +40,7 @@ NO_BDB ?= NO_SQLITE ?= NO_WALLET ?= NO_ZMQ ?= -NO_UPNP ?= NO_USDT ?= -NO_NATPMP ?= MULTIPROCESS ?= LTO ?= NO_HARDEN ?= @@ -158,14 +156,11 @@ bdb_packages_$(NO_BDB) = $(bdb_packages) sqlite_packages_$(NO_SQLITE) = $(sqlite_packages) wallet_packages_$(NO_WALLET) = $(bdb_packages_) $(sqlite_packages_) -upnp_packages_$(NO_UPNP) = $(upnp_packages) -natpmp_packages_$(NO_NATPMP) = $(natpmp_packages) - zmq_packages_$(NO_ZMQ) = $(zmq_packages) multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) usdt_packages_$(NO_USDT) = $(usdt_$(host_os)_packages) -packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) +packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(usdt_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) ifneq ($(zmq_packages_),) @@ -191,6 +186,7 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) echo copying packages: $^ echo to: $(@D) cd $(@D); $(foreach package,$^, $(build_TAR) xf $($(package)_cached); ) + echo To build Bitcoin Core with these packages, pass \'--toolchain $(@D)/toolchain.cmake\' to the first CMake invocation. touch $@ ifeq ($(host),$(build)) @@ -232,8 +228,6 @@ $(host_prefix)/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_$(fina -e 's|@wallet_packages@|$(wallet_packages_)|' \ -e 's|@bdb_packages@|$(bdb_packages_)|' \ -e 's|@sqlite_packages@|$(sqlite_packages_)|' \ - -e 's|@upnp_packages@|$(upnp_packages_)|' \ - -e 's|@natpmp_packages@|$(natpmp_packages_)|' \ -e 's|@usdt_packages@|$(usdt_packages_)|' \ -e 's|@no_harden@|$(NO_HARDEN)|' \ -e 's|@multiprocess@|$(MULTIPROCESS)|' \ diff --git a/depends/README.md b/depends/README.md index 19d704a50c..5ecf16e3c4 100644 --- a/depends/README.md +++ b/depends/README.md @@ -22,15 +22,15 @@ created. To use it during configuring Bitcoin Core: Common `host-platform-triplet`s for cross compilation are: -- `i686-pc-linux-gnu` for Linux 32 bit -- `x86_64-pc-linux-gnu` for x86 Linux +- `i686-pc-linux-gnu` for Linux x86 32 bit +- `x86_64-pc-linux-gnu` for Linux x86 64 bit - `x86_64-w64-mingw32` for Win64 - `x86_64-apple-darwin` for macOS - `arm64-apple-darwin` for ARM macOS - `arm-linux-gnueabihf` for Linux ARM 32 bit - `aarch64-linux-gnu` for Linux ARM 64 bit -- `powerpc64-linux-gnu` for Linux POWER 64-bit (big endian) -- `powerpc64le-linux-gnu` for Linux POWER 64-bit (little endian) +- `powerpc64-linux-gnu` for Linux POWER 64 bit (big endian) +- `powerpc64le-linux-gnu` for Linux POWER 64 bit (little endian) - `riscv32-linux-gnu` for Linux RISC-V 32 bit - `riscv64-linux-gnu` for Linux RISC-V 64 bit - `s390x-linux-gnu` for Linux S390X @@ -41,11 +41,17 @@ The paths are automatically configured and no other options are needed. #### Common - apt install automake bison cmake curl libtool make patch pkg-config python3 xz-utils + apt install cmake curl make patch + +#### GUI + +Skip the following packages if you don't intend to use the GUI and will build with [`NO_QT=1`](#dependency-options): + + apt install bison g++ pkg-config python3 xz-utils #### For macOS cross compilation - apt install clang lld llvm g++ zip + apt install clang lld llvm zip Clang 18 or later is required. You must also obtain the macOS SDK before proceeding with a cross-compile. Under the depends directory, create a @@ -54,7 +60,7 @@ For more information, see [SDK Extraction](../contrib/macdeploy/README.md#sdk-ex #### For Win64 cross compilation -- see [build-windows.md](../doc/build-windows.md#cross-compilation-for-ubuntu-and-windows-subsystem-for-linux) + apt install g++-mingw-w64-x86-64-posix #### For linux (including i386, ARM) cross compilation @@ -112,10 +118,8 @@ The following can be set when running make: `make FOO=bar` - `NO_WALLET`: Don't download/build/cache libs needed to enable the wallet - `NO_BDB`: Don't download/build/cache BerkeleyDB - `NO_SQLITE`: Don't download/build/cache SQLite -- `NO_UPNP`: Don't download/build/cache packages needed for enabling UPnP -- `NO_NATPMP`: Don't download/build/cache packages needed for enabling NAT-PMP - `NO_USDT`: Don't download/build/cache packages needed for enabling USDT tracepoints -- `MULTIPROCESS`: Build libmultiprocess (experimental, requires CMake) +- `MULTIPROCESS`: Build libmultiprocess (experimental) - `DEBUG`: Disable some optimizations and enable more runtime checking - `HOST_ID_SALT`: Optional salt to use when generating host package ids - `BUILD_ID_SALT`: Optional salt to use when generating build package ids diff --git a/depends/funcs.mk b/depends/funcs.mk index a2f760bd0e..b07432adec 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -179,7 +179,8 @@ $(1)_cmake=env CC="$$($(1)_cc)" \ CXX="$$($(1)_cxx)" \ CXXFLAGS="$$($(1)_cppflags) $$($(1)_cxxflags)" \ LDFLAGS="$$($(1)_ldflags)" \ - cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" \ + cmake -G "Unix Makefiles" \ + -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" \ -DCMAKE_AR=`which $$($(1)_ar)` \ -DCMAKE_NM=`which $$($(1)_nm)` \ -DCMAKE_RANLIB=`which $$($(1)_ranlib)` \ diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index a27d8b323b..4659d52912 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,4 +1,4 @@ -OSX_MIN_VERSION=11.0 +OSX_MIN_VERSION=13.0 OSX_SDK_VERSION=14.0 XCODE_VERSION=15.0 XCODE_BUILD_ID=15A240d diff --git a/depends/hosts/freebsd.mk b/depends/hosts/freebsd.mk index 8cef32e231..009d215f82 100644 --- a/depends/hosts/freebsd.mk +++ b/depends/hosts/freebsd.mk @@ -4,7 +4,7 @@ freebsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD) freebsd_release_CFLAGS=-O2 freebsd_release_CXXFLAGS=$(freebsd_release_CFLAGS) -freebsd_debug_CFLAGS=-O1 +freebsd_debug_CFLAGS=-O1 -g freebsd_debug_CXXFLAGS=$(freebsd_debug_CFLAGS) ifeq (86,$(findstring 86,$(build_arch))) diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index c09f7b1e3a..4628f7255d 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -1,3 +1,6 @@ +ifneq ($(shell $(SHELL) $(.SHELLFLAGS) "command -v $(host)-gcc-posix"),) +mingw32_CC := $(host)-gcc-posix +endif ifneq ($(shell $(SHELL) $(.SHELLFLAGS) "command -v $(host)-g++-posix"),) mingw32_CXX := $(host)-g++-posix endif @@ -20,5 +23,5 @@ mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS) mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC mingw32_cmake_system_name=Windows -# Windows 7 (NT 6.1). -mingw32_cmake_system_version=6.1 +# Windows 10 +mingw32_cmake_system_version=10.0 diff --git a/depends/hosts/netbsd.mk b/depends/hosts/netbsd.mk index 16dff92d42..d2b79f9d5b 100644 --- a/depends/hosts/netbsd.mk +++ b/depends/hosts/netbsd.mk @@ -7,12 +7,10 @@ netbsd_NM = $(host_toolchain)gcc-nm netbsd_RANLIB = $(host_toolchain)gcc-ranlib endif -netbsd_CXXFLAGS=$(netbsd_CFLAGS) - netbsd_release_CFLAGS=-O2 netbsd_release_CXXFLAGS=$(netbsd_release_CFLAGS) -netbsd_debug_CFLAGS=-O1 +netbsd_debug_CFLAGS=-O1 -g netbsd_debug_CXXFLAGS=$(netbsd_debug_CFLAGS) ifeq (86,$(findstring 86,$(build_arch))) diff --git a/depends/hosts/openbsd.mk b/depends/hosts/openbsd.mk index 63f6d73d55..53595689b6 100644 --- a/depends/hosts/openbsd.mk +++ b/depends/hosts/openbsd.mk @@ -4,7 +4,7 @@ openbsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD) openbsd_release_CFLAGS=-O2 openbsd_release_CXXFLAGS=$(openbsd_release_CFLAGS) -openbsd_debug_CFLAGS=-O1 +openbsd_debug_CFLAGS=-O1 -g openbsd_debug_CXXFLAGS=$(openbsd_debug_CFLAGS) ifeq (86,$(findstring 86,$(build_arch))) diff --git a/depends/packages.md b/depends/packages.md index 7a7a42afa1..6b458f22dd 100644 --- a/depends/packages.md +++ b/depends/packages.md @@ -153,8 +153,8 @@ Most autotools projects can be properly staged using: ## Build outputs: In general, the output of a depends package should not contain any libtool -archives. Instead, the package should output `.pc` (`pkg-config`) files where -possible. +archives or `.pc` (`pkg-config`) files. Instead, the package should output +`.cmake` (CMake) files where possible. From the [Gentoo Wiki entry](https://wiki.gentoo.org/wiki/Project:Quality_Assurance/Handling_Libtool_Archives): diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index 6d792db711..0c211cbc45 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -9,6 +9,7 @@ define $(package)_set_vars := $(package)_config_opts := -DBUILD_TESTING=OFF $(package)_config_opts += -DWITH_OPENSSL=OFF $(package)_config_opts += -DWITH_ZLIB=OFF + $(package)_cxxflags += -ffile-prefix-map=$$($(package)_extract_dir)=/usr endef define $(package)_config_cmds diff --git a/depends/packages/fontconfig.mk b/depends/packages/fontconfig.mk index 6baaecc55a..bd20d7c9ff 100644 --- a/depends/packages/fontconfig.mk +++ b/depends/packages/fontconfig.mk @@ -29,5 +29,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm -rf var lib/*.la + rm -rf bin etc share var lib/*.la endef diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 4c05e8a0a7..9507fe9ddf 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -4,18 +4,17 @@ $(package)_download_path=https://github.com/libevent/libevent/releases/download/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb $(package)_patches=cmake_fixups.patch -$(package)_patches+=fix_mingw_link.patch $(package)_build_subdir=build # When building for Windows, we set _WIN32_WINNT to target the same Windows -# version as we do in configure. Due to quirks in libevents build system, this +# version as we do in releases. Due to quirks in libevents build system, this # is also required to enable support for ipv6. See #19375. define $(package)_set_vars $(package)_config_opts=-DEVENT__DISABLE_BENCHMARK=ON -DEVENT__DISABLE_OPENSSL=ON $(package)_config_opts+=-DEVENT__DISABLE_SAMPLES=ON -DEVENT__DISABLE_REGRESS=ON $(package)_config_opts+=-DEVENT__DISABLE_TESTS=ON -DEVENT__LIBRARY_TYPE=STATIC $(package)_cppflags += -D_GNU_SOURCE - $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601 + $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0A00 ifeq ($(NO_HARDEN),) $(package)_cppflags+=-D_FORTIFY_SOURCE=3 @@ -23,8 +22,7 @@ define $(package)_set_vars endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/cmake_fixups.patch && \ - patch -p1 < $($(package)_patch_dir)/fix_mingw_link.patch + patch -p1 < $($(package)_patch_dir)/cmake_fixups.patch endef define $(package)_config_cmds @@ -40,7 +38,8 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm -rf bin && \ + rm -rf bin lib/pkgconfig && \ rm include/ev*.h && \ - rm include/event2/*_compat.h + rm include/event2/*_compat.h && \ + rm lib/libevent.a endef diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk index c292c49bfb..a181e05100 100644 --- a/depends/packages/libmultiprocess.mk +++ b/depends/packages/libmultiprocess.mk @@ -13,6 +13,7 @@ ifneq ($(host),$(build)) $(package)_config_opts := -DCAPNP_EXECUTABLE="$$(native_capnp_prefixbin)/capnp" $(package)_config_opts += -DCAPNPC_CXX_EXECUTABLE="$$(native_capnp_prefixbin)/capnpc-c++" endif +$(package)_cxxflags += -ffile-prefix-map=$$($(package)_extract_dir)=/usr endef define $(package)_config_cmds diff --git a/depends/packages/libnatpmp.mk b/depends/packages/libnatpmp.mk deleted file mode 100644 index 5a573a18e7..0000000000 --- a/depends/packages/libnatpmp.mk +++ /dev/null @@ -1,20 +0,0 @@ -package=libnatpmp -$(package)_version=f2433bec24ca3d3f22a8a7840728a3ac177f94ba -$(package)_download_path=https://github.com/miniupnp/libnatpmp/archive -$(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=ef84979950dfb3556705b63c9cd6c95501b75e887fba466234b187f3c9029669 -$(package)_build_subdir=build - -define $(package)_config_cmds - $($(package)_cmake) -S .. -B . -endef - -define $(package)_build_cmds - $(MAKE) natpmp -endef - -define $(package)_stage_cmds - mkdir -p $($(package)_staging_prefix_dir)/include $($(package)_staging_prefix_dir)/lib && \ - install ../natpmp.h ../natpmp_declspec.h $($(package)_staging_prefix_dir)/include && \ - install libnatpmp.a $($(package)_staging_prefix_dir)/lib -endef diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk deleted file mode 100644 index f9e114b495..0000000000 --- a/depends/packages/miniupnpc.mk +++ /dev/null @@ -1,36 +0,0 @@ -package=miniupnpc -$(package)_version=2.2.7 -$(package)_download_path=https://miniupnp.tuxfamily.org/files/ -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=b0c3a27056840fd0ec9328a5a9bac3dc5e0ec6d2e8733349cf577b0aa1e70ac1 -$(package)_patches=dont_leak_info.patch cmake_get_src_addr.patch fix_windows_snprintf.patch -$(package)_build_subdir=build - -define $(package)_set_vars -$(package)_config_opts = -DUPNPC_BUILD_SAMPLE=OFF -DUPNPC_BUILD_SHARED=OFF -$(package)_config_opts += -DUPNPC_BUILD_STATIC=ON -DUPNPC_BUILD_TESTS=OFF -$(package)_config_opts_mingw32 += -DMINIUPNPC_TARGET_WINDOWS_VERSION=0x0601 -endef - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/dont_leak_info.patch && \ - patch -p1 < $($(package)_patch_dir)/cmake_get_src_addr.patch && \ - patch -p1 < $($(package)_patch_dir)/fix_windows_snprintf.patch -endef - -define $(package)_config_cmds - $($(package)_cmake) -S .. -B . -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - cmake --install . --prefix $($(package)_staging_prefix_dir) -endef - -define $(package)_postprocess_cmds - rm -rf bin && \ - rm -rf share -endef diff --git a/depends/packages/native_capnp.mk b/depends/packages/native_capnp.mk index 484e78d5d9..e67b103716 100644 --- a/depends/packages/native_capnp.mk +++ b/depends/packages/native_capnp.mk @@ -1,9 +1,9 @@ package=native_capnp -$(package)_version=1.0.1 +$(package)_version=1.1.0 $(package)_download_path=https://capnproto.org/ $(package)_download_file=capnproto-c++-$($(package)_version).tar.gz $(package)_file_name=capnproto-cxx-$($(package)_version).tar.gz -$(package)_sha256_hash=0f7f4b8a76a2cdb284fddef20de8306450df6dd031a47a15ac95bc43c3358e09 +$(package)_sha256_hash=07167580e563f5e821e3b2af1c238c16ec7181612650c5901330fa9a0da50939 define $(package)_set_vars $(package)_config_opts := -DBUILD_TESTING=OFF diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index 185ef9489e..7c69e0f0c6 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -1,8 +1,8 @@ package=native_libmultiprocess -$(package)_version=c1b4ab4eb897d3af09bc9b3cc30e2e6fff87f3e2 +$(package)_version=abe254b9734f2e2b220d1456de195532d6e6ac1e $(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=6edf5ad239ca9963c78f7878486fb41411efc9927c6073928a7d6edf947cac4a +$(package)_sha256_hash=85777073259fdc75d24ac5777a19991ec1156c5f12db50b252b861c95dcb4f46 $(package)_dependencies=native_capnp define $(package)_config_cmds diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 01ed0d7a92..61cf66230c 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -17,9 +17,6 @@ sqlite_packages=sqlite zmq_packages=zeromq -upnp_packages=miniupnpc -natpmp_packages=libnatpmp - multiprocess_packages = libmultiprocess capnp multiprocess_native_packages = native_libmultiprocess native_capnp diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 917e179932..727dcfc9ef 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -10,7 +10,6 @@ $(package)_linguist_tools = lrelease lupdate lconvert $(package)_patches = qt.pro $(package)_patches += qttools_src.pro $(package)_patches += mac-qmake.conf -$(package)_patches += fix_qt_pkgconfig.patch $(package)_patches += no-xlib.patch $(package)_patches += dont_hardcode_pwd.patch $(package)_patches += qtbase-moc-ignore-gcc-macro.patch @@ -64,6 +63,7 @@ $(package)_config_opts += -no-mimetype-database $(package)_config_opts += -no-mtdev $(package)_config_opts += -no-openssl $(package)_config_opts += -no-openvg +$(package)_config_opts += -no-pkg-config $(package)_config_opts += -no-reduce-relocations $(package)_config_opts += -no-schannel $(package)_config_opts += -no-sctp @@ -84,7 +84,6 @@ $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests $(package)_config_opts += -nomake tools $(package)_config_opts += -opensource -$(package)_config_opts += -pkg-config $(package)_config_opts += -prefix $(host_prefix) $(package)_config_opts += -qt-libpng $(package)_config_opts += -qt-pcre @@ -226,7 +225,6 @@ define $(package)_preprocess_cmds cp $($(package)_patch_dir)/qttools_src.pro qttools/src/src.pro && \ patch -p1 -i $($(package)_patch_dir)/fix-macos-linker.patch && \ patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ patch -p1 -i $($(package)_patch_dir)/memory_resource.patch && \ @@ -273,6 +271,6 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm -rf doc/ native/lib/ && \ - rm -f lib/lib*.la + rm -rf doc/ native/lib/ lib/pkgconfig/ && \ + rm -f lib/lib*.la lib/Qt5*.la endef diff --git a/depends/packages/sqlite.mk b/depends/packages/sqlite.mk index 15bc0f4d7a..1cb2e299b9 100644 --- a/depends/packages/sqlite.mk +++ b/depends/packages/sqlite.mk @@ -1,8 +1,8 @@ package=sqlite -$(package)_version=3380500 -$(package)_download_path=https://sqlite.org/2022/ +$(package)_version=3460100 +$(package)_download_path=https://sqlite.org/2024/ $(package)_file_name=sqlite-autoconf-$($(package)_version).tar.gz -$(package)_sha256_hash=5af07de982ba658fd91a03170c945f99c971f6955bc79df3266544373e39869c +$(package)_sha256_hash=67d3fe6d268e6eaddcae3727fce58fcc8e9c53869bdd07a0c61e38ddf2965071 define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-readline --disable-dynamic-extensions --enable-option-checking @@ -31,5 +31,6 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds + rm -rf lib/pkgconfig && \ rm lib/*.la endef diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index 67a0dd88e5..89e10d15ef 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -4,15 +4,13 @@ $(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=6653ef5910f17954861fe72332e68b03ca6e4d9c7160eb3a8de5a5a913bfab43 $(package)_build_subdir=build -$(package)_patches = remove_libstd_link.patch -$(package)_patches += macos_mktemp_check.patch +$(package)_patches = macos_mktemp_check.patch $(package)_patches += builtin_sha1.patch $(package)_patches += fix_have_windows.patch $(package)_patches += openbsd_kqueue_headers.patch $(package)_patches += cmake_minimum.patch $(package)_patches += cacheline_undefined.patch $(package)_patches += no_librt.patch -$(package)_patches += fix_mingw_link.patch define $(package)_set_vars $(package)_config_opts := -DCMAKE_BUILD_TYPE=None -DWITH_DOCS=OFF -DWITH_LIBSODIUM=OFF @@ -20,19 +18,17 @@ define $(package)_set_vars $(package)_config_opts += -DBUILD_SHARED=OFF -DBUILD_TESTS=OFF -DZMQ_BUILD_TESTS=OFF $(package)_config_opts += -DENABLE_DRAFTS=OFF -DZMQ_BUILD_TESTS=OFF $(package)_cxxflags += -ffile-prefix-map=$($(package)_extract_dir)=/usr - $(package)_config_opts_mingw32 += -DZMQ_WIN32_WINNT=0x0601 -DZMQ_HAVE_IPC=OFF + $(package)_config_opts_mingw32 += -DZMQ_WIN32_WINNT=0x0A00 -DZMQ_HAVE_IPC=OFF endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch && \ patch -p1 < $($(package)_patch_dir)/macos_mktemp_check.patch && \ patch -p1 < $($(package)_patch_dir)/builtin_sha1.patch && \ patch -p1 < $($(package)_patch_dir)/cacheline_undefined.patch && \ patch -p1 < $($(package)_patch_dir)/fix_have_windows.patch && \ patch -p1 < $($(package)_patch_dir)/openbsd_kqueue_headers.patch && \ patch -p1 < $($(package)_patch_dir)/cmake_minimum.patch && \ - patch -p1 < $($(package)_patch_dir)/no_librt.patch && \ - patch -p1 < $($(package)_patch_dir)/fix_mingw_link.patch + patch -p1 < $($(package)_patch_dir)/no_librt.patch endef define $(package)_config_cmds @@ -48,5 +44,6 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm -rf share + rm -rf share && \ + rm -rf lib/pkgconfig endef diff --git a/depends/patches/libevent/cmake_fixups.patch b/depends/patches/libevent/cmake_fixups.patch index d80c1a9489..a8812afd1e 100644 --- a/depends/patches/libevent/cmake_fixups.patch +++ b/depends/patches/libevent/cmake_fixups.patch @@ -1,8 +1,5 @@ cmake: set minimum version to 3.5 -Fix generated pkg-config files, see -https://github.com/libevent/libevent/pull/1165. - --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,7 @@ @@ -14,22 +11,3 @@ https://github.com/libevent/libevent/pull/1165. if (POLICY CMP0054) cmake_policy(SET CMP0054 NEW) -diff --git a/cmake/AddEventLibrary.cmake b/cmake/AddEventLibrary.cmake -index 04f5837e..d8ea42c4 100644 ---- a/cmake/AddEventLibrary.cmake -+++ b/cmake/AddEventLibrary.cmake -@@ -20,12 +20,12 @@ macro(generate_pkgconfig LIB_NAME) - - set(LIBS "") - foreach (LIB ${LIB_PLATFORM}) -- set(LIBS "${LIBS} -L${LIB}") -+ set(LIBS "${LIBS} -l${LIB}") - endforeach() - - set(OPENSSL_LIBS "") - foreach(LIB ${OPENSSL_LIBRARIES}) -- set(OPENSSL_LIBS "${OPENSSL_LIBS} -L${LIB}") -+ set(OPENSSL_LIBS "${OPENSSL_LIBS} -l${LIB}") - endforeach() - - configure_file("lib${LIB_NAME}.pc.in" "lib${LIB_NAME}.pc" @ONLY) diff --git a/depends/patches/libevent/fix_mingw_link.patch b/depends/patches/libevent/fix_mingw_link.patch deleted file mode 100644 index 41cbd463c9..0000000000 --- a/depends/patches/libevent/fix_mingw_link.patch +++ /dev/null @@ -1,25 +0,0 @@ -commit d108099913c5fdbe518f3f4d711f248f8522bd10 -Author: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> -Date: Mon Apr 22 06:39:35 2024 +0100 - - build: Add `Iphlpapi` to `Libs.private` in `*.pc` files on Windows - - It has been required since https://github.com/libevent/libevent/pull/923 - at least for the `if_nametoindex` call. - - See https://github.com/libevent/libevent/pull/1622. - - -diff --git a/configure.ac b/configure.ac -index d00e063a..cd1fce37 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -906,7 +906,7 @@ if(WIN32) - list(APPEND HDR_PRIVATE WIN32-Code/getopt.h) - - set(EVENT__DNS_USE_FTIME_FOR_ID 1) -- set(LIB_PLATFORM ws2_32 shell32 advapi32) -+ set(LIB_PLATFORM ws2_32 shell32 advapi32 iphlpapi) - add_definitions( - -D_CRT_SECURE_NO_WARNINGS - -D_CRT_NONSTDC_NO_DEPRECATE) diff --git a/depends/patches/miniupnpc/cmake_get_src_addr.patch b/depends/patches/miniupnpc/cmake_get_src_addr.patch deleted file mode 100644 index bae1b738f3..0000000000 --- a/depends/patches/miniupnpc/cmake_get_src_addr.patch +++ /dev/null @@ -1,22 +0,0 @@ -commit cb2026239c2a3aff393952ccb0ee1c448189402d -Author: fanquake <fanquake@gmail.com> -Date: Fri Mar 22 14:03:54 2024 +0000 - - build: add MINIUPNPC_GET_SRC_ADDR to CMake build - - This mirrors the autotools build. - - See https://github.com/miniupnp/miniupnp/pull/721. - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 1aa95a8..0cacf3e 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -31,6 +31,7 @@ endif () - if (NOT WIN32) - target_compile_definitions(miniupnpc-private INTERFACE - MINIUPNPC_SET_SOCKET_TIMEOUT -+ MINIUPNPC_GET_SRC_ADDR - _BSD_SOURCE _DEFAULT_SOURCE) - if (NOT APPLE AND NOT CMAKE_SYSTEM_NAME MATCHES ".*BSD" AND NOT CMAKE_SYSTEM_NAME STREQUAL "SunOS") - # add_definitions (-D_POSIX_C_SOURCE=200112L) diff --git a/depends/patches/miniupnpc/dont_leak_info.patch b/depends/patches/miniupnpc/dont_leak_info.patch deleted file mode 100644 index 95a09a26dc..0000000000 --- a/depends/patches/miniupnpc/dont_leak_info.patch +++ /dev/null @@ -1,32 +0,0 @@ -commit 51f6dd991c29af66fb4f64c6feb2787cce23a1a7 -Author: fanquake <fanquake@gmail.com> -Date: Mon Jan 8 11:21:40 2024 +0000 - - Don't leak OS and miniupnpc version info in User-Agent - -diff --git a/src/minisoap.c b/src/minisoap.c -index 903ac5f..046e0ea 100644 ---- a/src/minisoap.c -+++ b/src/minisoap.c -@@ -90,7 +90,7 @@ int soapPostSubmit(SOCKET fd, - headerssize = snprintf(headerbuf, sizeof(headerbuf), - "POST %s HTTP/%s\r\n" - "Host: %s%s\r\n" -- "User-Agent: " OS_STRING " " UPNP_VERSION_STRING " MiniUPnPc/" MINIUPNPC_VERSION_STRING "\r\n" -+ "User-Agent: " UPNP_VERSION_STRING "\r\n" - "Content-Length: %d\r\n" - #if (UPNP_VERSION_MAJOR == 1) && (UPNP_VERSION_MINOR == 0) - "Content-Type: text/xml\r\n" -diff --git a/src/miniwget.c b/src/miniwget.c -index e76a5e5..0cc36fe 100644 ---- a/src/miniwget.c -+++ b/src/miniwget.c -@@ -444,7 +444,7 @@ miniwget3(const char * host, - "GET %s HTTP/%s\r\n" - "Host: %s:%d\r\n" - "Connection: Close\r\n" -- "User-Agent: " OS_STRING " " UPNP_VERSION_STRING " MiniUPnPc/" MINIUPNPC_VERSION_STRING "\r\n" -+ "User-Agent: " UPNP_VERSION_STRING "\r\n" - - "\r\n", - path, httpversion, host, port); diff --git a/depends/patches/miniupnpc/fix_windows_snprintf.patch b/depends/patches/miniupnpc/fix_windows_snprintf.patch deleted file mode 100644 index ff9e26231e..0000000000 --- a/depends/patches/miniupnpc/fix_windows_snprintf.patch +++ /dev/null @@ -1,25 +0,0 @@ -commit a1e9de80ab99b4c956a6a4e21d3e0de6f7a1014d -Author: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> -Date: Sat Apr 20 15:14:47 2024 +0100 - - Fix macro expression that guards `snprintf` for Windows - - Otherwise, the `snprintf` is still wrongly emulated for the following - cases: - - mingw-w64 6.0.0 or new with ucrt - - mingw-w64 8.0.0 or new with iso c ext - ---- a/src/win32_snprintf.h -+++ b/src/win32_snprintf.h -@@ -23,9 +23,9 @@ - (defined(_MSC_VER) && _MSC_VER < 1900) /* Visual Studio older than 2015 */ || \ - (defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR) && defined(__NO_ISOCEXT)) /* mingw32 without iso c ext */ || \ - (defined(__MINGW64_VERSION_MAJOR) && /* mingw-w64 not ... */ !( \ -- (defined (__USE_MINGW_ANSI_STDIO) && __USE_MINGW_ANSI_STDIO != 0)) /* ... with ansi stdio */ || \ -+ (defined (__USE_MINGW_ANSI_STDIO) && __USE_MINGW_ANSI_STDIO != 0) /* ... with ansi stdio */ || \ - (__MINGW64_VERSION_MAJOR >= 6 && defined(_UCRT)) /* ... at least 6.0.0 with ucrt */ || \ -- (__MINGW64_VERSION_MAJOR >= 8 && !defined(__NO_ISOCEXT)) /* ... at least 8.0.0 with iso c ext */ || \ -+ (__MINGW64_VERSION_MAJOR >= 8 && !defined(__NO_ISOCEXT))) /* ... at least 8.0.0 with iso c ext */ || \ - 0) || \ - 0) - diff --git a/depends/patches/qt/fix_qt_pkgconfig.patch b/depends/patches/qt/fix_qt_pkgconfig.patch deleted file mode 100644 index 73f4d89f73..0000000000 --- a/depends/patches/qt/fix_qt_pkgconfig.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- old/qtbase/mkspecs/features/qt_module.prf -+++ new/qtbase/mkspecs/features/qt_module.prf -@@ -269,7 +269,7 @@ load(qt_installs) - load(qt_targets) - - # this builds on top of qt_common --!internal_module:if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { -+if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { - CONFIG += create_pc - QMAKE_PKGCONFIG_DESTDIR = pkgconfig - host_build: \ diff --git a/depends/patches/zeromq/fix_mingw_link.patch b/depends/patches/zeromq/fix_mingw_link.patch deleted file mode 100644 index 1434557dc7..0000000000 --- a/depends/patches/zeromq/fix_mingw_link.patch +++ /dev/null @@ -1,31 +0,0 @@ -Fix CMake-generated `libzmq.pc` file - -This change mirrors the Autotools-based build system behavior for -cross-compiling for Windows with static linking. - -See https://github.com/zeromq/libzmq/pull/4706. - - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 03462271..0315e606 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -546,12 +546,18 @@ if(ZMQ_HAVE_WINDOWS) - # Cannot use check_library_exists because the symbol is always declared as char(*)(void) - set(CMAKE_REQUIRED_LIBRARIES "ws2_32.lib") - check_cxx_symbol_exists(WSAStartup "winsock2.h" HAVE_WS2_32) -+ if(HAVE_WS2_32) -+ set(pkg_config_libs_private "${pkg_config_libs_private} -lws2_32") -+ endif() - - set(CMAKE_REQUIRED_LIBRARIES "rpcrt4.lib") - check_cxx_symbol_exists(UuidCreateSequential "rpc.h" HAVE_RPCRT4) - - set(CMAKE_REQUIRED_LIBRARIES "iphlpapi.lib") - check_cxx_symbol_exists(GetAdaptersAddresses "winsock2.h;iphlpapi.h" HAVE_IPHLAPI) -+ if(HAVE_IPHLAPI) -+ set(pkg_config_libs_private "${pkg_config_libs_private} -liphlpapi") -+ endif() - check_cxx_symbol_exists(if_nametoindex "iphlpapi.h" HAVE_IF_NAMETOINDEX) - - set(CMAKE_REQUIRED_LIBRARIES "") diff --git a/depends/patches/zeromq/remove_libstd_link.patch b/depends/patches/zeromq/remove_libstd_link.patch deleted file mode 100644 index ddf91e6abf..0000000000 --- a/depends/patches/zeromq/remove_libstd_link.patch +++ /dev/null @@ -1,25 +0,0 @@ -commit 47d4cd12a2c051815ddda78adebdb3923b260d8a -Author: fanquake <fanquake@gmail.com> -Date: Tue Aug 18 14:45:40 2020 +0800 - - Remove needless linking against libstdc++ - - This is broken for a number of reasons, including: - - g++ understands "static-libstdc++ -lstdc++" to mean "link against - whatever libstdc++ exists, probably shared", which in itself is buggy. - - another stdlib (libc++ for example) may be in use - - See #11981. - -diff --git a/src/libzmq.pc.in b/src/libzmq.pc.in -index 233bc3a..3c2bf0d 100644 ---- a/src/libzmq.pc.in -+++ b/src/libzmq.pc.in -@@ -7,6 +7,6 @@ Name: libzmq - Description: 0MQ c++ library - Version: @VERSION@ - Libs: -L${libdir} -lzmq --Libs.private: -lstdc++ @pkg_config_libs_private@ -+Libs.private: @pkg_config_libs_private@ - Requires.private: @pkg_config_names_private@ - Cflags: -I${includedir} @pkg_config_defines@ diff --git a/depends/toolchain.cmake.in b/depends/toolchain.cmake.in index c733c81edf..e72e6e29e2 100644 --- a/depends/toolchain.cmake.in +++ b/depends/toolchain.cmake.in @@ -55,12 +55,21 @@ set(DEPENDS_COMPILE_DEFINITIONS_DEBUG @CPPFLAGS_DEBUG@) if(NOT DEFINED CMAKE_EXE_LINKER_FLAGS_INIT) set(CMAKE_EXE_LINKER_FLAGS_INIT "@LDFLAGS@") endif() +if(NOT DEFINED CMAKE_SHARED_LINKER_FLAGS_INIT) + set(CMAKE_SHARED_LINKER_FLAGS_INIT "@LDFLAGS@") +endif() if(NOT DEFINED CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO_INIT) set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO_INIT "@LDFLAGS_RELEASE@") endif() +if(NOT DEFINED CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO_INIT) + set(CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO_INIT "@LDFLAGS_RELEASE@") +endif() if(NOT DEFINED CMAKE_EXE_LINKER_FLAGS_DEBUG_INIT) set(CMAKE_EXE_LINKER_FLAGS_DEBUG_INIT "@LDFLAGS_DEBUG@") endif() +if(NOT DEFINED CMAKE_SHARED_LINKER_FLAGS_DEBUG_INIT) + set(CMAKE_SHARED_LINKER_FLAGS_DEBUG_INIT "@LDFLAGS_DEBUG@") +endif() set(CMAKE_AR "@AR@") set(CMAKE_RANLIB "@RANLIB@") @@ -87,15 +96,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT CMAKE_HOST_APPLE) set(CMAKE_FRAMEWORK_PATH "@OSX_SDK@/System/Library/Frameworks") endif() - -# Customize pkg-config behaviour. -cmake_path(APPEND CMAKE_FIND_ROOT_PATH "lib" "pkgconfig" OUTPUT_VARIABLE pkg_config_path) -set(ENV{PKG_CONFIG_PATH} ${pkg_config_path}) -set(ENV{PKG_CONFIG_LIBDIR} ${pkg_config_path}) -unset(pkg_config_path) -set(PKG_CONFIG_ARGN --static) - - # Set configuration options for the main build system. set(qt_packages @qt_packages@) if("${qt_packages}" STREQUAL "") @@ -139,20 +139,6 @@ else() set(WITH_SQLITE ON CACHE BOOL "") endif() -set(upnp_packages @upnp_packages@) -if("${upnp_packages}" STREQUAL "") - set(WITH_MINIUPNPC OFF CACHE BOOL "") -else() - set(WITH_MINIUPNPC ON CACHE BOOL "") -endif() - -set(natpmp_packages @natpmp_packages@) -if("${natpmp_packages}" STREQUAL "") - set(WITH_NATPMP OFF CACHE BOOL "") -else() - set(WITH_NATPMP ON CACHE BOOL "") -endif() - set(usdt_packages @usdt_packages@) if("${usdt_packages}" STREQUAL "") set(WITH_USDT OFF CACHE BOOL "") @@ -168,7 +154,8 @@ endif() if("@multiprocess@" STREQUAL "1") set(WITH_MULTIPROCESS ON CACHE BOOL "") - set(LibmultiprocessNative_DIR "${CMAKE_FIND_ROOT_PATH}/native/lib/cmake/Libmultiprocess" CACHE PATH "") + set(Libmultiprocess_ROOT "${CMAKE_CURRENT_LIST_DIR}" CACHE PATH "") + set(LibmultiprocessNative_ROOT "${CMAKE_CURRENT_LIST_DIR}/native" CACHE PATH "") else() set(WITH_MULTIPROCESS OFF CACHE BOOL "") endif() diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index ccaf31170a..cbbb6551f1 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -38,7 +38,7 @@ PROJECT_NAME = "Bitcoin Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @PACKAGE_VERSION@ +PROJECT_NUMBER = @CLIENT_VERSION_STRING@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/doc/README.md b/doc/README.md index 7f5db1b5bf..79ca53ce76 100644 --- a/doc/README.md +++ b/doc/README.md @@ -3,7 +3,7 @@ Bitcoin Core Setup --------------------- -Bitcoin Core is the original Bitcoin client and it builds the backbone of the network. It downloads and, by default, stores the entire history of Bitcoin transactions, which requires a few hundred gigabytes of disk space. Depending on the speed of your computer and network connection, the synchronization process can take anywhere from a few hours to a day or more. +Bitcoin Core is the original Bitcoin client and it builds the backbone of the network. It downloads and, by default, stores the entire history of Bitcoin transactions, which requires several hundred gigabytes or more of disk space. Depending on the speed of your computer and network connection, the synchronization process can take anywhere from a few hours to several days or more. To download Bitcoin Core, visit [bitcoincore.org](https://bitcoincore.org/en/download/). diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 6664bc2a3a..fbed4a36b2 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -117,7 +117,7 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 "value" : 8.8687, "scriptPubKey" : { "asm" : "OP_DUP OP_HASH160 1c7cebb529b86a04c683dfa87be49de35bcf589e OP_EQUALVERIFY OP_CHECKSIG", - "desc" : "addr(mi7as51dvLJsizWnTMurtRmrP8hG2m1XvD)#gj9tznmy" + "desc" : "addr(mi7as51dvLJsizWnTMurtRmrP8hG2m1XvD)#gj9tznmy", "hex" : "76a9141c7cebb529b86a04c683dfa87be49de35bcf589e88ac", "type" : "pubkeyhash", "address" : "mi7as51dvLJsizWnTMurtRmrP8hG2m1XvD" diff --git a/doc/benchmarking.md b/doc/benchmarking.md index c6dc75dc5c..8f836219c9 100644 --- a/doc/benchmarking.md +++ b/doc/benchmarking.md @@ -40,7 +40,7 @@ The output will look similar to: Help --------------------- - build/src/bench/bench_bitcoin -? + build/src/bench/bench_bitcoin -h To print the various options, like listing the benchmarks without running them or using a regex filter to only run certain benchmarks. diff --git a/doc/bips.md b/doc/bips.md index d544ff822b..a95b3159ec 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -34,7 +34,6 @@ BIPs that are implemented by Bitcoin Core: * [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, and enforced for all peer versions as of **v0.13.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579) and [PR #6641](https://github.com/bitcoin/bitcoin/pull/6641)). * [`BIP 112`](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki): The CHECKSEQUENCEVERIFY opcode has been implemented since **v0.12.1** ([PR #7524](https://github.com/bitcoin/bitcoin/pull/7524)), and has been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). * [`BIP 113`](https://github.com/bitcoin/bips/blob/master/bip-0113.mediawiki): Median time past lock-time calculations have been implemented since **v0.12.1** ([PR #6566](https://github.com/bitcoin/bitcoin/pull/6566)), and has been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). -* [`BIP 125`](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki): Opt-in full replace-by-fee partially implemented: signaling is enforced if configured. For other replacement rules, see doc/policy/mempool-replacements.md. * [`BIP 130`](https://github.com/bitcoin/bips/blob/master/bip-0130.mediawiki): direct headers announcement is negotiated with peer versions `>=70012` as of **v0.12.0** ([PR 6494](https://github.com/bitcoin/bitcoin/pull/6494)). * [`BIP 133`](https://github.com/bitcoin/bips/blob/master/bip-0133.mediawiki): feefilter messages are respected and sent for peer versions `>=70013` as of **v0.13.0** ([PR 7542](https://github.com/bitcoin/bitcoin/pull/7542)). * [`BIP 141`](https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki): Segregated Witness (Consensus Layer) as of **v0.13.0** ([PR 8149](https://github.com/bitcoin/bitcoin/pull/8149)), defined for mainnet as of **v0.13.1** ([PR 8937](https://github.com/bitcoin/bitcoin/pull/8937)), and *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). diff --git a/doc/bitcoin-conf.md b/doc/bitcoin-conf.md index 76711d0e7d..9b31879790 100644 --- a/doc/bitcoin-conf.md +++ b/doc/bitcoin-conf.md @@ -31,7 +31,7 @@ Comments may appear in two ways: ### Network specific options Network specific options can be: -- placed into sections with headers `[main]` (not `[mainnet]`), `[test]` (not `[testnet]`), `[signet]` or `[regtest]`; +- placed into sections with headers `[main]` (not `[mainnet]`), `[test]` (not `[testnet]`, for testnet3), `[testnet4]`, `[signet]` or `[regtest]`; - prefixed with a chain name; e.g., `regtest.maxmempool=100`. Network specific options take precedence over non-network specific options. diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index 6a25e9a834..694224621e 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -42,7 +42,7 @@ from ports. However, you can build DB 4.8 yourself [using depends](/depends). ```bash pkg install gmake -gmake -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 +gmake -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_ZMQ=1 NO_USDT=1 ``` When the build is complete, the Berkeley DB installation location will be displayed: @@ -96,7 +96,7 @@ There is an included test suite that is useful for testing code changes when dev To run the test suite (recommended), you will need to have Python 3 installed: ```bash -pkg install python3 databases/py-sqlite3 +pkg install python3 databases/py-sqlite3 net/py-pyzmq ``` --- diff --git a/doc/build-netbsd.md b/doc/build-netbsd.md index 63bfbd61db..988f3b93a7 100644 --- a/doc/build-netbsd.md +++ b/doc/build-netbsd.md @@ -1,6 +1,6 @@ # NetBSD Build Guide -**Updated for NetBSD [10.0](https://netbsd.org/releases/formal-10/NetBSD-10.0.html)** +**Updated for NetBSD [10.1](https://netbsd.org/releases/formal-10/NetBSD-10.1.html)** This guide describes how to build bitcoind, command-line utilities, and GUI on NetBSD. @@ -83,6 +83,13 @@ pkgin install qrencode Otherwise, if you don't need QR encoding support, use the `-DWITH_QRENCODE=OFF` option to disable this feature in order to compile the GUI. +#### Notifications +###### ZeroMQ + +Bitcoin Core can provide notifications via ZeroMQ. If the package is installed, support will be compiled in. +```bash +pkgin zeromq +``` #### Test Suite Dependencies @@ -90,10 +97,10 @@ There is an included test suite that is useful for testing code changes when dev To run the test suite (recommended), you will need to have Python 3 installed: ```bash -pkgin install python39 +pkgin install python310 py310-zmq ``` -### Building Bitcoin Core +## Building Bitcoin Core ### 1. Configuration diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index fafc91fc5f..1ad90f23bc 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -1,6 +1,6 @@ # OpenBSD Build Guide -**Updated for OpenBSD [7.5](https://www.openbsd.org/75.html)** +**Updated for OpenBSD [7.6](https://www.openbsd.org/76.html)** This guide describes how to build bitcoind, command-line utilities, and GUI on OpenBSD. @@ -44,7 +44,7 @@ from ports. However you can build it yourself, [using depends](/depends). Refer to [depends/README.md](/depends/README.md) for detailed instructions. ```bash -gmake -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 +gmake -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_ZMQ=1 NO_USDT=1 ... to: /path/to/bitcoin/depends/*-unknown-openbsd* ``` @@ -90,7 +90,7 @@ There is an included test suite that is useful for testing code changes when dev To run the test suite (recommended), you will need to have Python 3 installed: ```bash -pkg_add python # Select the newest version of the package. +pkg_add python py3-zmq # Select the newest version of the python package if necessary. ``` ## Building Bitcoin Core diff --git a/doc/build-osx.md b/doc/build-osx.md index 600eebb6ff..80039d9e6a 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -1,15 +1,15 @@ # macOS Build Guide -**Updated for MacOS [14](https://www.apple.com/macos/sonoma/)** +**Updated for MacOS [15](https://www.apple.com/macos/macos-sequoia/)** -This guide describes how to build bitcoind, command-line utilities, and GUI on macOS +This guide describes how to build bitcoind, command-line utilities, and GUI on macOS. ## Preparation The commands in this guide should be executed in a Terminal application. macOS comes with a built-in Terminal located in: -``` +```bash /Applications/Utilities/Terminal.app ``` @@ -48,23 +48,9 @@ See [dependencies.md](dependencies.md) for a complete overview. To install, run the following from your terminal: ``` bash -brew install cmake boost pkg-config libevent +brew install cmake boost pkgconf libevent ``` -For macOS 11 (Big Sur) and 12 (Monterey) you need to install a more recent version of llvm. - -``` bash -brew install llvm -``` - -And append the following to the configure commands below: - -``` bash --DCMAKE_C_COMPILER="$(brew --prefix llvm)/bin/clang" -DCMAKE_CXX_COMPILER="$(brew --prefix llvm)/bin/clang++" -``` - -Try `llvm@17` if compilation fails with the default version of llvm. - ### 4. Clone Bitcoin repository `git` should already be installed by default on your system. @@ -124,30 +110,6 @@ Otherwise, if you don't need QR encoding support, you can pass `-DWITH_QRENCODE= --- -#### Port Mapping Dependencies - -###### miniupnpc - -miniupnpc may be used for UPnP port mapping. -Skip if you do not need this functionality. - -``` bash -brew install miniupnpc -``` - -###### libnatpmp - -libnatpmp may be used for NAT-PMP port mapping. -Skip if you do not need this functionality. - -``` bash -brew install libnatpmp -``` - -Check out the [further configuration](#further-configuration) section for more information. - ---- - #### ZMQ Dependencies Support for ZMQ notifications requires the following dependency. diff --git a/doc/build-unix.md b/doc/build-unix.md index 4c3c659bff..4f04b4fd9f 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -60,10 +60,6 @@ executables, which are based on BerkeleyDB 4.8. Otherwise, you can build Berkele To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) -Optional port mapping libraries (see: `-DWITH_MINIUPNPC=ON` and `-DWITH_NATPMP=ON`): - - sudo apt install libminiupnpc-dev libnatpmp-dev - ZMQ dependencies (provides ZMQ API): sudo apt-get install libzmq3-dev @@ -112,10 +108,6 @@ are based on Berkeley DB 4.8. Otherwise, you can build Berkeley DB [yourself](#b To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) -Optional port mapping libraries (see: `-DWITH_MINIUPNPC=ON` and `-DWITH_NATPMP=ON`): - - sudo dnf install miniupnpc-devel libnatpmp-devel - ZMQ dependencies (provides ZMQ API): sudo dnf install zeromq-devel @@ -153,7 +145,7 @@ The legacy wallet uses Berkeley DB. To ensure backwards compatibility it is recommended to use Berkeley DB 4.8. If you have to build it yourself, and don't want to use any other libraries built in depends, you can do: ```bash -make -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 +make -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_ZMQ=1 NO_USDT=1 ... to: /path/to/bitcoin/depends/x86_64-pc-linux-gnu ``` @@ -190,7 +182,7 @@ Setup and Build Example: Arch Linux ----------------------------------- This example lists the steps necessary to setup and build a command line only distribution of the latest changes on Arch Linux: - pacman --sync --needed cmake boost gcc git libevent make pkgconf python sqlite + pacman --sync --needed cmake boost gcc git libevent make python sqlite git clone https://github.com/bitcoin/bitcoin.git cd bitcoin/ cmake -B build diff --git a/doc/build-windows-msvc.md b/doc/build-windows-msvc.md index 80c2b77f1e..482b7a7d1f 100644 --- a/doc/build-windows-msvc.md +++ b/doc/build-windows-msvc.md @@ -42,9 +42,7 @@ Available presets can be listed as follows: cmake --list-presets ``` -By default, all presets: -- Set `BUILD_GUI` to `ON`. -- Set `WITH_QRENCODE` to `OFF`, due to known build issues when using vcpkg's `libqrencode` package. +By default, all presets set `BUILD_GUI` to `ON`. ## Building diff --git a/doc/build-windows.md b/doc/build-windows.md index 2e7b93da35..0c1418bff9 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -28,36 +28,18 @@ The steps below can be performed on Ubuntu or WSL. The depends system will also work on other Linux distributions, however the commands for installing the toolchain will be different. -First, install the general dependencies: - - sudo apt update - sudo apt upgrade - sudo apt install cmake curl g++ git make pkg-config - -A host toolchain (`g++`) is necessary because some dependency -packages need to build host utilities that are used in the build process. - -See [dependencies.md](dependencies.md) for a complete overview. +See [README.md](../depends/README.md) in the depends directory for which +dependencies to install and [dependencies.md](dependencies.md) for a complete overview. If you want to build the Windows installer using the `deploy` build target, you will need [NSIS](https://nsis.sourceforge.io/Main_Page): - sudo apt install nsis + apt install nsis Acquire the source in the usual way: git clone https://github.com/bitcoin/bitcoin.git cd bitcoin -## Building for 64-bit Windows - -The first step is to install the mingw-w64 cross-compilation toolchain: - -```sh -sudo apt install g++-mingw-w64-x86-64-posix -``` - -Once the toolchain is installed the build steps are common: - Note that for WSL the Bitcoin Core source path MUST be somewhere in the default mount file system, for example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. This means you cannot use a directory that is located directly on the host Windows file system to perform the build. diff --git a/doc/dependencies.md b/doc/dependencies.md index 4c620cf876..ec068801a8 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -9,7 +9,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Clang](https://clang.llvm.org) | [16.0](https://github.com/bitcoin/bitcoin/pull/30263) | | [CMake](https://cmake.org/) | [3.22](https://github.com/bitcoin/bitcoin/pull/30454) | | [GCC](https://gcc.gnu.org) | [11.1](https://github.com/bitcoin/bitcoin/pull/29091) | -| [Python](https://www.python.org) (scripts, tests) | [3.9](https://github.com/bitcoin/bitcoin/pull/28211) | +| [Python](https://www.python.org) (scripts, tests) | [3.10](https://github.com/bitcoin/bitcoin/pull/30527) | | [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | ## Required @@ -31,12 +31,6 @@ You can find installation instructions in the `build-*.md` file for your platfor | [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [4.1.1](https://github.com/bitcoin/bitcoin/pull/27312) | | No | | [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.14](https://github.com/bitcoin/bitcoin/pull/30198) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | -### Networking -| Dependency | Releases | Version used | Minimum required | Runtime | -| --- | --- | --- | --- | --- | -| [libnatpmp](../depends/packages/libnatpmp.mk) | [link](https://github.com/miniupnp/libnatpmp/) | commit [f2433be...](https://github.com/bitcoin/bitcoin/pull/29708) | | No | -| [MiniUPnPc](../depends/packages/miniupnpc.mk) | [link](https://miniupnp.tuxfamily.org/) | [2.2.7](https://github.com/bitcoin/bitcoin/pull/29707) | 2.1 | No | - ### Notifications | Dependency | Releases | Version used | Minimum required | Runtime | | --- | --- | --- | --- | --- | diff --git a/doc/descriptors.md b/doc/descriptors.md index 8c55d4ae37..115ca5e61a 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -23,6 +23,9 @@ Supporting RPCs are: - `listdescriptors` outputs descriptors imported into a descriptor wallet (since v22). - `scanblocks` takes as input descriptors to scan for in blocks and returns the relevant blockhashes (since v25). +- `getdescriptoractivity` takes as input descriptors and blockhashes (as output + by `scanblocks`) and returns rich event data related to spends or receives associated + with the given descriptors. This document describes the language. For the specifics on usage, see the RPC documentation for the functions mentioned above. diff --git a/doc/design/libraries.md b/doc/design/libraries.md index 335d3957d4..24185bf477 100644 --- a/doc/design/libraries.md +++ b/doc/design/libraries.md @@ -8,7 +8,7 @@ | *libbitcoin_crypto* | Hardware-optimized functions for data encryption, hashing, message authentication, and key derivation. | | *libbitcoin_kernel* | Consensus engine and support library used for validation by *libbitcoin_node*. | | *libbitcoinqt* | GUI functionality used by *bitcoin-qt* and *bitcoin-gui* executables. | -| *libbitcoin_ipc* | IPC functionality used by *bitcoin-node*, *bitcoin-wallet*, *bitcoin-gui* executables to communicate when [`--enable-multiprocess`](multiprocess.md) is used. | +| *libbitcoin_ipc* | IPC functionality used by *bitcoin-node*, *bitcoin-wallet*, *bitcoin-gui* executables to communicate when [`-DWITH_MULTIPROCESS=ON`](multiprocess.md) is used. | | *libbitcoin_node* | P2P and RPC server functionality used by *bitcoind* and *bitcoin-qt* executables. | | *libbitcoin_util* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_common*, but lower-level (see [Dependencies](#dependencies)). | | *libbitcoin_wallet* | Wallet functionality used by *bitcoind* and *bitcoin-wallet* executables. | diff --git a/doc/design/multiprocess.md b/doc/design/multiprocess.md index 49410a4213..a781da8d1b 100644 --- a/doc/design/multiprocess.md +++ b/doc/design/multiprocess.md @@ -81,7 +81,7 @@ This section describes the major components of the Inter-Process Communication ( - In the generated code, we have C++ client subclasses that inherit from the abstract classes in [`src/interfaces/`](../../src/interfaces/). These subclasses are the workhorses of the IPC mechanism. - They implement all the methods of the interface, marshalling arguments into a structured format, sending them as requests to the IPC server via a UNIX socket, and handling the responses. - These subclasses effectively mask the complexity of IPC, presenting a familiar C++ interface to developers. -- Internally, the client subclasses generated by the `mpgen` tool wrap [client classes generated by Cap'n Proto](https://capnproto.org/cxxrpc.html#clients), and use them to send IPC requests. +- Internally, the client subclasses generated by the `mpgen` tool wrap [client classes generated by Cap'n Proto](https://capnproto.org/cxxrpc.html#clients), and use them to send IPC requests. The Cap'n Proto client classes are low-level, with non-blocking methods that use asynchronous I/O and pass request and response objects, while mpgen client subclasses provide normal C++ methods that block while executing and convert between request/response objects and arguments/return values. ### C++ Server Classes in Generated Code - On the server side, corresponding generated C++ classes receive IPC requests. These server classes are responsible for unmarshalling method arguments, invoking the corresponding methods in the local [`src/interfaces/`](../../src/interfaces/) objects, and creating the IPC response. @@ -94,7 +94,7 @@ This section describes the major components of the Inter-Process Communication ( - **Asynchronous I/O and Thread Management**: The library is also responsible for managing I/O and threading. Particularly, it ensures that IPC requests never block each other and that new threads on either side of a connection can always make client calls. It also manages worker threads on the server side of calls, ensuring that calls from the same client thread always execute on the same server thread (to avoid locking issues and support nested callbacks). ### Type Hooks in [`src/ipc/capnp/*-types.h`](../../src/ipc/capnp/) -- **Custom Type Conversions**: In [`src/ipc/capnp/*-types.h`](../../src/ipc/capnp/), function overloads of two `libmultiprocess` C++ functions, `mp::CustomReadField` and `mp::CustomBuildFields`, are defined. These overloads are used for customizing the conversion of specific C++ types to and from Cap’n Proto types. +- **Custom Type Conversions**: In [`src/ipc/capnp/*-types.h`](../../src/ipc/capnp/), function overloads of `libmultiprocess` C++ functions, `mp::CustomReadField`, `mp::CustomBuildField`, `mp::CustomReadMessage` and `mp::CustomBuildMessage`, are defined. These overloads are used for customizing the conversion of specific C++ types to and from Cap’n Proto types. - **Handling Special Cases**: The `mpgen` tool and `libmultiprocess` library can convert most C++ types to and from Cap’n Proto types automatically, including interface types, primitive C++ types, standard C++ types like `std::vector`, `std::set`, `std::map`, `std::tuple`, and `std::function`, as well as simple C++ structs that consist of aforementioned types and whose fields correspond 1:1 with Cap’n Proto struct fields. For other types, `*-types.h` files provide custom code to convert between C++ and Cap’n Proto data representations. ### Protocol-Agnostic IPC Code in [`src/ipc/`](../../src/ipc/) @@ -197,7 +197,7 @@ sequenceDiagram - Upon receiving the request, the Cap'n Proto dispatching code in the `bitcoin-node` process calls the `getBlockHash` method of the `Chain` [server class](#c-server-classes-in-generated-code). - The server class is automatically generated by the `mpgen` tool from the [`chain.capnp`](https://github.com/ryanofsky/bitcoin/blob/pr/ipc/src/ipc/capnp/chain.capnp) file in [`src/ipc/capnp/`](../../src/ipc/capnp/). - The `getBlockHash` method of the generated `Chain` server subclass in `bitcoin-wallet` receives a Cap’n Proto request object with the `height` parameter, and calls the `getBlockHash` method on its local `Chain` object with the provided `height`. - - When the call returns, it encapsulates the return value in a Cap’n Proto response, which it sends back to the `bitcoin-wallet` process, + - When the call returns, it encapsulates the return value in a Cap’n Proto response, which it sends back to the `bitcoin-wallet` process. 5. **Response and Return** - The `getBlockHash` method of the generated `Chain` client subclass in `bitcoin-wallet` which sent the request now receives the response. @@ -232,7 +232,7 @@ This modularization represents an advancement in Bitcoin Core's architecture, of - **Cap’n Proto struct**: A structured data format used in Cap’n Proto, similar to structs in C++, for organizing and transporting data across different processes. -- **client class (in generated code)**: A C++ class generated from a Cap’n Proto interface which inherits from a Bitcoin core abstract class, and implements each virtual method to send IPC requests to another process. (see also [components section](#c-client-subclasses-in-generated-code)) +- **client class (in generated code)**: A C++ class generated from a Cap’n Proto interface which inherits from a Bitcoin Core abstract class, and implements each virtual method to send IPC requests to another process. (see also [components section](#c-client-subclasses-in-generated-code)) - **IPC (inter-process communication)**: Mechanisms that enable processes to exchange requests and data. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index a630957f41..37e594e762 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -369,18 +369,18 @@ If you have ccache enabled, absolute paths are stripped from debug information with the `-fdebug-prefix-map` and `-fmacro-prefix-map` options (if supported by the compiler). This might break source file detection in case you move binaries after compilation, debug from the directory other than the project root or use -an IDE that only supports absolute paths for debugging. +an IDE that only supports absolute paths for debugging (e.g. it won't stop at breakpoints). There are a few possible fixes: 1. Configure source file mapping. -For `gdb` create or append to `.gdbinit` file: +For `gdb` create or append to [`.gdbinit` file](https://sourceware.org/gdb/current/onlinedocs/gdb#gdbinit-man): ``` set substitute-path ./src /path/to/project/root/src ``` -For `lldb` create or append to `.lldbinit` file: +For `lldb` create or append to [`.lldbinit` file](https://lldb.llvm.org/man/lldb.html#configuration-files): ``` settings set target.source-map ./src /path/to/project/root/src ``` @@ -392,6 +392,8 @@ ln -s /path/to/project/root/src src 3. Use `debugedit` to modify debug information in the binary. +4. If your IDE has an option for this, change your breakpoints to use the file name only. + ### `debug.log` If the code is behaving strangely, take a look in the `debug.log` file in the data directory; @@ -418,8 +420,8 @@ see [test/functional/](/test/functional) for tests that run in `-regtest` mode. ### DEBUG_LOCKORDER Bitcoin Core is a multi-threaded application, and deadlocks or other -multi-threading bugs can be very difficult to track down. The `--enable-debug` -configure option adds `-DDEBUG_LOCKORDER` to the compiler flags. This inserts +multi-threading bugs can be very difficult to track down. The `-DCMAKE_BUILD_TYPE=Debug` +build option adds `-DDEBUG_LOCKORDER` to the compiler flags. This inserts run-time checks to keep track of which locks are held and adds warnings to the `debug.log` file if inconsistencies are detected. @@ -429,9 +431,8 @@ Defining `DEBUG_LOCKCONTENTION` adds a "lock" logging category to the logging RPC that, when enabled, logs the location and duration of each lock contention to the `debug.log` file. -The `--enable-debug` configure option adds `-DDEBUG_LOCKCONTENTION` to the -compiler flags. You may also enable it manually for a non-debug build by running -configure with `-DDEBUG_LOCKCONTENTION` added to your CPPFLAGS, +The `-DCMAKE_BUILD_TYPE=Debug` build option adds `-DDEBUG_LOCKCONTENTION` to the +compiler flags. You may also enable it manually by building with `-DDEBUG_LOCKCONTENTION` added to your CPPFLAGS, i.e. `CPPFLAGS="-DDEBUG_LOCKCONTENTION"`, then build and run bitcoind. You can then use the `-debug=lock` configuration option at bitcoind startup or @@ -579,7 +580,7 @@ cmake -B build -DSANITIZERS=thread If you are compiling with GCC you will typically need to install corresponding "san" libraries to actually compile with these flags, e.g. libasan for the address sanitizer, libtsan for the thread sanitizer, and libubsan for the -undefined sanitizer. If you are missing required libraries, the configure script +undefined sanitizer. If you are missing required libraries, the build will fail with a linker error when testing the sanitizer flags. The test suite should pass cleanly with the `thread` and `undefined` sanitizers. You @@ -595,7 +596,7 @@ See the CI config for more examples, and upstream documentation for more informa about any additional options. Not all sanitizer options can be enabled at the same time, e.g. trying to build -with `-DSANITIZERS=address,thread` will fail in the configure script as +with `-DSANITIZERS=address,thread` will fail in the build as these sanitizers are mutually incompatible. Refer to your compiler manual to learn more about these options and which sanitizers are supported by your compiler. @@ -619,7 +620,7 @@ The code is multi-threaded and uses mutexes and the Deadlocks due to inconsistent lock ordering (thread 1 locks `cs_main` and then `cs_wallet`, while thread 2 locks them in the opposite order: result, deadlock as each waits for the other to release its lock) are a problem. Compile with -`-DDEBUG_LOCKORDER` (or use `--enable-debug`) to get lock order inconsistencies +`-DDEBUG_LOCKORDER` (or use `-DCMAKE_BUILD_TYPE=Debug`) to get lock order inconsistencies reported in the `debug.log` file. Re-architecting the core code so there are better-defined interfaces @@ -958,7 +959,7 @@ Strings and formatting - *Rationale*: Qt has built-in functionality for converting their string type from/to C++. No need to roll your own. - - In cases where do you call `.c_str()`, you might want to additionally check that the string does not contain embedded '\0' characters, because + - In cases where you do call `.c_str()`, you might want to additionally check that the string does not contain embedded '\0' characters, because it will (necessarily) truncate the string. This might be used to hide parts of the string from logging or to circumvent checks. If a use of strings is sensitive to this, take care to check the string for embedded NULL characters first and reject it if there are any (see `ParsePrechecks` in `strencodings.cpp` for an example). @@ -1062,8 +1063,8 @@ bool Chainstate::PreciousBlock(BlockValidationState& state, CBlockIndex* pindex) ``` - Build and run tests with `-DDEBUG_LOCKORDER` to verify that no potential - deadlocks are introduced. As of 0.12, this is defined by default when - configuring with `--enable-debug`. + deadlocks are introduced. This is defined by default when + building with `-DCMAKE_BUILD_TYPE=Debug`. - When using `LOCK`/`TRY_LOCK` be aware that the lock exists in the context of the current scope, so surround the statement and the code that needs the lock @@ -1398,6 +1399,12 @@ A few guidelines for introducing and reviewing new RPC interfaces: to a multi-value, or due to other historical reasons. **Always** have false map to 0 and true to 1 in this case. +- For new RPC methods, if implementing a `verbosity` argument, use integer verbosity rather than boolean. + Disallow usage of boolean verbosity (see `ParseVerbosity()` in [util.h](/src/rpc/util.h)). + + - *Rationale*: Integer verbosity allows for multiple values. Undocumented boolean verbosity is deprecated + and new RPC methods should prevent its use. + - Don't forget to fill in the argument names correctly in the RPC command table. - *Rationale*: If not, the call cannot be used with name-based arguments. diff --git a/doc/fuzzing.md b/doc/fuzzing.md index f3647c0840..365314a592 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -15,7 +15,7 @@ $ FUZZ=process_message build_fuzz/src/test/fuzz/fuzz # abort fuzzing using ctrl-c ``` -One can use `--prefix=libfuzzer-nosan` to do the same without common sanitizers enabled. +One can use `--preset=libfuzzer-nosan` to do the same without common sanitizers enabled. See [further](#run-without-sanitizers-for-increased-throughput) for more information. There is also a runner script to execute all fuzz targets. Refer to @@ -206,157 +206,14 @@ $ FUZZ=process_message ./honggfuzz/honggfuzz -i inputs/ -- build_fuzz/src/test/f Read the [Honggfuzz documentation](https://github.com/google/honggfuzz/blob/master/docs/USAGE.md) for more information. -## Fuzzing the Bitcoin Core P2P layer using Honggfuzz NetDriver - -Honggfuzz NetDriver allows for very easy fuzzing of TCP servers such as Bitcoin -Core without having to write any custom fuzzing harness. The `bitcoind` server -process is largely fuzzed without modification. - -This makes the fuzzing highly realistic: a bug reachable by the fuzzer is likely -also remotely triggerable by an untrusted peer. - -To quickly get started fuzzing the P2P layer using Honggfuzz NetDriver: - -```sh -$ mkdir bitcoin-honggfuzz-p2p/ -$ cd bitcoin-honggfuzz-p2p/ -$ git clone https://github.com/bitcoin/bitcoin -$ cd bitcoin/ -$ git clone https://github.com/google/honggfuzz -$ cd honggfuzz/ -$ make -$ cd .. -$ git apply << "EOF" -diff --git a/src/compat/compat.h b/src/compat/compat.h -index 8195bceaec..cce2b31ff0 100644 ---- a/src/compat/compat.h -+++ b/src/compat/compat.h -@@ -90,8 +90,12 @@ typedef char* sockopt_arg_type; - // building with a binutils < 2.36 is subject to this ld bug. - #define MAIN_FUNCTION __declspec(dllexport) int main(int argc, char* argv[]) - #else -+#ifdef HFND_FUZZING_ENTRY_FUNCTION_CXX -+#define MAIN_FUNCTION HFND_FUZZING_ENTRY_FUNCTION_CXX(int argc, char* argv[]) -+#else - #define MAIN_FUNCTION int main(int argc, char* argv[]) - #endif -+#endif - - // Note these both should work with the current usage of poll, but best to be safe - // WIN32 poll is broken https://daniel.haxx.se/blog/2012/10/10/wsapoll-is-broken/ -diff --git a/src/net.cpp b/src/net.cpp -index 7601a6ea84..702d0f56ce 100644 ---- a/src/net.cpp -+++ b/src/net.cpp -@@ -727,7 +727,7 @@ int V1TransportDeserializer::readHeader(Span<const uint8_t> msg_bytes) - } - - // Check start string, network magic -- if (memcmp(hdr.pchMessageStart, m_chain_params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { -+ if (false && memcmp(hdr.pchMessageStart, m_chain_params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { // skip network magic checking - LogDebug(BCLog::NET, "Header error: Wrong MessageStart %s received, peer=%d\n", HexStr(hdr.pchMessageStart), m_node_id); - return -1; - } -@@ -788,7 +788,7 @@ CNetMessage V1TransportDeserializer::GetMessage(const std::chrono::microseconds - RandAddEvent(ReadLE32(hash.begin())); - - // Check checksum and header message type string -- if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { -+ if (false && memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { // skip checksum checking - LogDebug(BCLog::NET, "Header error: Wrong checksum (%s, %u bytes), expected %s was %s, peer=%d\n", - SanitizeString(msg.m_type), msg.m_message_size, - HexStr(Span{hash}.first(CMessageHeader::CHECKSUM_SIZE)), -EOF -$ cmake -B build_fuzz \ - -DCMAKE_C_COMPILER="$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang" \ - -DCMAKE_CXX_COMPILER="$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang++" \ - -DENABLE_WALLET=OFF \ - -DBUILD_GUI=OFF \ - -DSANITIZERS=address,undefined -$ cmake --build build_fuzz --target bitcoind -$ mkdir -p inputs/ -$ ./honggfuzz/honggfuzz --exit_upon_crash --quiet --timeout 4 -n 1 -Q \ - -E HFND_TCP_PORT=18444 -f inputs/ -- \ - build_fuzz/src/bitcoind -regtest -discover=0 -dns=0 -dnsseed=0 -listenonion=0 \ - -nodebuglogfile -bind=127.0.0.1:18444 -logthreadnames \ - -debug -``` - -# Fuzzing Bitcoin Core using Eclipser (v1.x) - -## Quickstart guide - -To quickly get started fuzzing Bitcoin Core using [Eclipser v1.x](https://github.com/SoftSec-KAIST/Eclipser/tree/v1.x): - -```sh -$ git clone https://github.com/bitcoin/bitcoin -$ cd bitcoin/ -$ sudo vim /etc/apt/sources.list # Uncomment the lines starting with 'deb-src'. -$ sudo apt-get update -$ sudo apt-get build-dep qemu -$ sudo apt-get install libtool libtool-bin wget automake autoconf bison gdb -``` - -At this point, you must install the .NET core. The process differs, depending on your Linux distribution. -See [this link](https://learn.microsoft.com/en-us/dotnet/core/install/linux) for details. -On Ubuntu 20.04, the following should work: - -```sh -$ wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -$ sudo dpkg -i packages-microsoft-prod.deb -$ rm packages-microsoft-prod.deb -$ sudo apt-get update -$ sudo apt-get install -y dotnet-sdk-2.1 -``` - -You will also want to make sure Python is installed as `python` for the Eclipser install to succeed. - -```sh -$ git clone https://github.com/SoftSec-KAIST/Eclipser.git -$ cd Eclipser -$ git checkout v1.x -$ make -$ cd .. -$ cmake -B build_fuzz -DBUILD_FOR_FUZZING=ON -$ mkdir -p outputs/ -$ FUZZ=bech32 dotnet ./Eclipser/build/Eclipser.dll fuzz -p build_fuzz/src/test/fuzz/fuzz -t 36000 -o outputs --src stdin -``` - -This will perform 10 hours of fuzzing. - -To make further use of the inputs generated by Eclipser, you -must first decode them: - -```sh -$ dotnet Eclipser/build/Eclipser.dll decode -i outputs/testcase -o decoded_outputs -``` -This will place raw inputs in the directory `decoded_outputs/decoded_stdins`. Crashes are in the `outputs/crashes` directory, and must -be decoded in the same way. - -Fuzzing with Eclipser will likely be much more effective if using an existing corpus: - -```sh -$ git clone https://github.com/bitcoin-core/qa-assets -$ FUZZ=bech32 dotnet Eclipser/build/Eclipser.dll fuzz -p build_fuzz/src/test/fuzz/fuzz -t 36000 -i qa-assets/fuzz_corpora/bech32 outputs --src stdin -``` - -Note that fuzzing with Eclipser on certain targets (those that create 'full nodes', e.g. `process_message*`) will, -for now, slowly fill `/tmp/` with improperly cleaned-up files, which will cause spurious crashes. -See [this proposed patch](https://github.com/bitcoin/bitcoin/pull/22472) for more information. - -Read the [Eclipser documentation for v1.x](https://github.com/SoftSec-KAIST/Eclipser/tree/v1.x) for more details on using Eclipser. - - # OSS-Fuzz Bitcoin Core participates in Google's [OSS-Fuzz](https://github.com/google/oss-fuzz/tree/master/projects/bitcoin-core) -program, which includes a dashboard of [publicly disclosed vulnerabilities](https://bugs.chromium.org/p/oss-fuzz/issues/list?q=bitcoin-core). -Generally, we try to disclose vulnerabilities as soon as possible after they -are fixed to give users the knowledge they need to be protected. However, -because Bitcoin is a live P2P network, and not just standalone local software, -we might not fully disclose every issue within Google's standard +program, which includes a dashboard of [publicly disclosed vulnerabilities](https://issues.oss-fuzz.com/issues?q=bitcoin-core%20status:open). + +Bitcoin Core follows its [security disclosure policy](https://bitcoincore.org/en/security-advisories/), +which may differ from Google's standard [90-day disclosure window](https://google.github.io/oss-fuzz/getting-started/bug-disclosure-guidelines/) -if a partial or delayed disclosure is important to protect users or the -function of the network. +. OSS-Fuzz also produces [a fuzzing coverage report](https://oss-fuzz.com/coverage-report/job/libfuzzer_asan_bitcoin-core/latest). diff --git a/doc/multiprocess.md b/doc/multiprocess.md index 7ba89b3ff5..1757296eed 100644 --- a/doc/multiprocess.md +++ b/doc/multiprocess.md @@ -4,7 +4,7 @@ _This document describes usage of the multiprocess feature. For design informati ## Build Option -On unix systems, the `--enable-multiprocess` build option can be passed to `./configure` to build new `bitcoin-node`, `bitcoin-wallet`, and `bitcoin-gui` executables alongside existing `bitcoind` and `bitcoin-qt` executables. +On Unix systems, the `-DWITH_MULTIPROCESS=ON` build option can be passed to build the supplemental `bitcoin-node` and `bitcoin-gui` multiprocess executables. ## Debugging @@ -17,15 +17,17 @@ The multiprocess feature requires [Cap'n Proto](https://capnproto.org/) and [lib ``` cd <BITCOIN_SOURCE_DIRECTORY> make -C depends NO_QT=1 MULTIPROCESS=1 -CONFIG_SITE=$PWD/depends/x86_64-pc-linux-gnu/share/config.site ./configure -make -src/bitcoin-node -regtest -printtoconsole -debug=ipc -BITCOIND=bitcoin-node test/functional/test_runner.py +# Set host platform to output of gcc -dumpmachine or clang -dumpmachine or check the depends/ directory for the generated subdirectory name +HOST_PLATFORM="x86_64-pc-linux-gnu" +cmake -B build --toolchain=depends/$HOST_PLATFORM/toolchain.cmake +cmake --build build +build/src/bitcoin-node -regtest -printtoconsole -debug=ipc +BITCOIND=$(pwd)/build/src/bitcoin-node build/test/functional/test_runner.py ``` -The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option). +The `cmake` build will pick up settings and library locations from the depends directory, so there is no need to pass `-DWITH_MULTIPROCESS=ON` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option). -Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess/blob/master/doc/install.md) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general. +Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `cmake -B build -DWITH_MULTIPROCESS=ON` without using the depends system. The `cmake` build will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess/blob/master/doc/install.md) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general. ## Usage diff --git a/doc/multisig-tutorial.md b/doc/multisig-tutorial.md index 1f21acc2be..28b5a70b2c 100644 --- a/doc/multisig-tutorial.md +++ b/doc/multisig-tutorial.md @@ -82,7 +82,7 @@ multisig_desc="[$multisig_ext_desc, $multisig_int_desc]" `external_desc` and `internal_desc` specify the output type (`wsh`, in this case) and the xpubs involved. They also use BIP 67 (`sortedmulti`), so the wallet can be recreated without worrying about the order of xpubs. Conceptually, descriptors describe a list of scriptPubKey (along with information for spending from it) [[source](https://github.com/bitcoin/bitcoin/issues/21199#issuecomment-780772418)]. -Note that at least two descriptors are usually used, one for internal derivation paths and external ones. There are discussions about eliminating this redundancy, as can been seen in the issue [#17190](https://github.com/bitcoin/bitcoin/issues/17190). +Note that at least two descriptors are usually used, one for internal derivation paths and one for external ones. There are discussions about eliminating this redundancy, as can be seen in the issue [#17190](https://github.com/bitcoin/bitcoin/issues/17190). After creating the descriptors, it is necessary to add the checksum, which is required by the `importdescriptors` RPC. diff --git a/doc/offline-signing-tutorial.md b/doc/offline-signing-tutorial.md index e03a0003a1..8ec4355bc7 100644 --- a/doc/offline-signing-tutorial.md +++ b/doc/offline-signing-tutorial.md @@ -60,7 +60,8 @@ The `watch_only_wallet` wallet will be used to track and validate incoming trans ```sh [online]$ ./build/src/bitcoin-cli -signet -named createwallet \ wallet_name="watch_only_wallet" \ - disable_private_keys=true + disable_private_keys=true \ + blank=true { "name": "watch_only_wallet" diff --git a/doc/policy/mempool-replacements.md b/doc/policy/mempool-replacements.md index f044a0f8ad..eb370672e4 100644 --- a/doc/policy/mempool-replacements.md +++ b/doc/policy/mempool-replacements.md @@ -10,12 +10,7 @@ A transaction ("replacement transaction") may replace its directly conflicting t their in-mempool descendants (together, "original transactions") if, in addition to passing all other consensus and policy rules, each of the following conditions are met: -1. If `-mempoolfullrbf=0` (the value is 1 by default), the directly conflicting transactions all signal replaceability explicitly. A transaction is - signaling BIP125 replaceability if any of its inputs have an nSequence number less than (0xffffffff - 1). - A transaction also signals replaceability if its version field is set to 3. - - *Rationale*: See [BIP125 - explanation](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki#motivation). +1. (Removed) 2. The replacement transaction only include an unconfirmed input if that input was included in one of the directly conflicting transactions. An unconfirmed input spends an output from a @@ -80,3 +75,5 @@ This set of rules is similar but distinct from BIP125. #25353](https://github.com/bitcoin/bitcoin/pull/25353)). * Full replace-by-fee is the default policy as of **v28.0** ([PR #30493](https://github.com/bitcoin/bitcoin/pull/30493)). + +* Signaling for replace-by-fee is no longer required as of [PR 30592](https://github.com/bitcoin/bitcoin/pull/30592). diff --git a/doc/policy/packages.md b/doc/policy/packages.md index b432008d7b..9bd9becfdc 100644 --- a/doc/policy/packages.md +++ b/doc/policy/packages.md @@ -38,8 +38,6 @@ The following rules are enforced for all packages: * Only limited package replacements are currently considered. (#28984) - - If `-mempoolfullrbf=0` (the value is 1 by default), all direct conflicts must signal replacement. - - Packages are 1-parent-1-child, with no in-mempool ancestors of the package. - All conflicting clusters (connected components of mempool transactions) must be clusters of up to size 2. @@ -76,7 +74,7 @@ The following rules are only enforced for packages to be submitted to the mempoo enforced for test accepts): * Packages must be child-with-unconfirmed-parents packages. This also means packages must contain at - least 2 transactions. (#22674) + least 1 transaction. (#31096) - *Rationale*: This allows for fee-bumping by CPFP. Allowing multiple parents makes it possible to fee-bump a batch of transactions. Restricting packages to a defined topology is easier to diff --git a/doc/psbt.md b/doc/psbt.md index 0f31cb8eba..e555718349 100644 --- a/doc/psbt.md +++ b/doc/psbt.md @@ -67,6 +67,10 @@ hardware implementations will typically implement multiple roles simultaneously. input a PSBT, adds UTXO, key, and script data to inputs and outputs that miss it, and optionally signs inputs. Where possible it also finalizes the partial signatures. +- **`descriptorprocesspsbt` (Updater, Signer, Finalizer)** is a node RPC that takes + as input a PSBT and a list of descriptors. It updates SegWit inputs with + information available from the UTXO set and the mempool and signs the inputs using + the provided descriptors. Where possible it also finalizes the partial signatures. - **`utxoupdatepsbt` (Updater)** is a node RPC that takes a PSBT and updates it to include information available from the UTXO set (works only for SegWit inputs). diff --git a/doc/release-notes-28121.md b/doc/release-notes-28121.md new file mode 100644 index 0000000000..911b7c5620 --- /dev/null +++ b/doc/release-notes-28121.md @@ -0,0 +1,2 @@ +The RPC `testmempoolaccept` response now includes a "reject-details" field in some cases, +similar to the complete error messages returned by `sendrawtransaction` (#28121)
\ No newline at end of file diff --git a/doc/release-notes-28358.md b/doc/release-notes-28358.md new file mode 100644 index 0000000000..336aaa59ed --- /dev/null +++ b/doc/release-notes-28358.md @@ -0,0 +1,6 @@ +Updated settings +------ + +- The maximum allowed value for the `-dbcache` configuration option has been + dropped due to recent UTXO set growth. Note that before this change, large `-dbcache` + values were automatically reduced to 16 GiB (1 GiB on 32 bit systems). (#28358) diff --git a/doc/release-notes-30239.md b/doc/release-notes-30239.md new file mode 100644 index 0000000000..9380bddb1c --- /dev/null +++ b/doc/release-notes-30239.md @@ -0,0 +1,12 @@ +P2P and network changes +----------------------- + +Ephemeral dust is a new concept that allows a single +dust output in a transaction, provided the transaction +is zero fee. In order to spend any unconfirmed outputs +from this transaction, the spender must also spend +this dust in addition to any other outputs. + +In other words, this type of transaction +should be created in a transaction package where +the dust is both created and spent simultaneously. diff --git a/doc/release-notes-30592.md b/doc/release-notes-30592.md new file mode 100644 index 0000000000..a3c1a808dc --- /dev/null +++ b/doc/release-notes-30592.md @@ -0,0 +1,7 @@ +Full Replace-By-Fee +=================== + +Starting with v28.0, the `mempoolfullrbf` startup option was set to +default to `1`. With widespread adoption of this policy, users no longer +benefit from disabling it, so the option has been removed, making full +replace-by-fee the standard behavior. (#30592) diff --git a/doc/release-notes-30708.md b/doc/release-notes-30708.md new file mode 100644 index 0000000000..5cf17c7b65 --- /dev/null +++ b/doc/release-notes-30708.md @@ -0,0 +1,6 @@ +New RPCs +-------- + +- `getdescriptoractivity` can be used to find all spend/receive activity relevant to + a given set of descriptors within a set of specified blocks. This call can be used with + `scanblocks` to lessen the need for additional indexing programs. diff --git a/doc/release-notes-31130.md b/doc/release-notes-31130.md new file mode 100644 index 0000000000..d8cb46cb58 --- /dev/null +++ b/doc/release-notes-31130.md @@ -0,0 +1,10 @@ +P2P and network changes +----------------------- + +Support for UPnP was dropped. If you want to open a port automatically, consider using the `-natpmp` +option instead, which uses PCP or NAT-PMP depending on router support. + +Updated settings +------ + +- Setting `-upnp` will now return an error. Consider using `-natpmp` instead. diff --git a/doc/release-notes-31156.md b/doc/release-notes-31156.md new file mode 100644 index 0000000000..c61cfc08b9 --- /dev/null +++ b/doc/release-notes-31156.md @@ -0,0 +1,4 @@ +Test +------ + +The BIP94 timewarp attack mitigation (designed for testnet4) is no longer active on the regtest network. (#31156) diff --git a/doc/release-notes-31175.md b/doc/release-notes-31175.md new file mode 100644 index 0000000000..84a04a7426 --- /dev/null +++ b/doc/release-notes-31175.md @@ -0,0 +1,8 @@ +RPC +--- + +Duplicate blocks submitted with `submitblock` will now persist their block data +even if it was previously pruned. If pruning is activated, the data will be +pruned again eventually once the block file it is persisted in is selected for +pruning. This is consistent with the behaviour of `getblockfrompeer` where the +block is persisted as well even when pruning. diff --git a/doc/release-notes-31223.md b/doc/release-notes-31223.md new file mode 100644 index 0000000000..44f0552fd9 --- /dev/null +++ b/doc/release-notes-31223.md @@ -0,0 +1,15 @@ +P2P and network changes +----------------------- +When the `-port` configuration option is used, the default onion listening port will now +be derived to be that port + 1 instead of being set to a fixed value (8334 on mainnet). +This re-allows setups with multiple local nodes using different `-port` and not using `-bind`, +which would lead to a startup failure in v28.0 due to a port collision. + +Note that a `HiddenServicePort` manually configured in `torrc` may need adjustment if used in +connection with the `-port` option. +For example, if you are using `-port=5555` with a non-standard value and not using `-bind=...=onion`, +previously Bitcoin Core would listen for incoming Tor connections on `127.0.0.1:8334`. +Now it would listen on `127.0.0.1:5556` (`-port` plus one). If you configured the hidden service manually +in torrc now you have to change it from `HiddenServicePort 8333 127.0.0.1:8334` to `HiddenServicePort 8333 +127.0.0.1:5556`, or configure bitcoind with `-bind=127.0.0.1:8334=onion` to get the previous behavior. +(#31223) diff --git a/doc/release-notes-empty-template.md b/doc/release-notes-empty-template.md index 96e28c3763..bd4600b395 100644 --- a/doc/release-notes-empty-template.md +++ b/doc/release-notes-empty-template.md @@ -32,11 +32,18 @@ Upgrading directly from a version of Bitcoin Core that has reached its EOL is possible, but it might take some time if the data directory needs to be migrated. Old wallet versions of Bitcoin Core are generally supported. +Running Bitcoin Core binaries on macOS requires self signing. +``` +cd /path/to/bitcoin-core/bin +xattr -d com.apple.quarantine bitcoin-cli bitcoin-qt bitcoin-tx bitcoin-util bitcoin-wallet bitcoind test_bitcoin +codesign -s - bitcoin-cli bitcoin-qt bitcoin-tx bitcoin-util bitcoin-wallet bitcoind test_bitcoin +``` + Compatibility ============== -Bitcoin Core is supported and extensively tested on operating systems -using the Linux Kernel 3.17+, macOS 11.0+, and Windows 7 and newer. Bitcoin +Bitcoin Core is supported and tested on operating systems using the +Linux Kernel 3.17+, macOS 13.0+, and Windows 10 and newer. Bitcoin Core should also work on most other Unix-like systems but is not as frequently tested on them. It is not recommended to use Bitcoin Core on unsupported systems. diff --git a/doc/release-notes/release-notes-27.2.md b/doc/release-notes/release-notes-27.2.md new file mode 100644 index 0000000000..85bf57c500 --- /dev/null +++ b/doc/release-notes/release-notes-27.2.md @@ -0,0 +1,92 @@ +27.2 Release Notes +===================== + +Bitcoin Core version 27.2 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-27.2/> + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux Kernel 3.17+, macOS 11.0+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +### P2P + +- #30394 net: fix race condition in self-connect detection + +### Init + +- #30435 init: change shutdown order of load block thread and scheduler + +### RPC + +- #30357 Fix cases of calls to FillPSBT errantly returning complete=true + +### PSBT + +- #29855 psbt: Check non witness utxo outpoint early + +### Test + +- #30552 test: fix constructor of msg_tx + +### Doc + +- #30504 doc: use proper doxygen formatting for CTxMemPool::cs + +### Build + +- #30283 upnp: fix build with miniupnpc 2.2.8 +- #30633 Fixes for GCC 15 compatibility + +### CI + +- #30193 ci: move ASan job to GitHub Actions from Cirrus CI +- #30299 ci: remove unused bcc variable from workflow + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Ava Chow +- Cory Fields +- Martin Zumsande +- Matt Whitlock +- Max Edwards +- Sebastian Falbesoner +- Vasil Dimov +- willcl-ark + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-28.0.md b/doc/release-notes/release-notes-28.0.md new file mode 100644 index 0000000000..d9e6a34d0f --- /dev/null +++ b/doc/release-notes/release-notes-28.0.md @@ -0,0 +1,371 @@ +Bitcoin Core version 28.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-28.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Running Bitcoin Core binaries on macOS requires self signing. +``` +cd /path/to/bitcoin-28.0/bin +xattr -d com.apple.quarantine bitcoin-cli bitcoin-qt bitcoin-tx bitcoin-util bitcoin-wallet bitcoind test_bitcoin +codesign -s - bitcoin-cli bitcoin-qt bitcoin-tx bitcoin-util bitcoin-wallet bitcoind test_bitcoin +``` + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux Kernel 3.17+, macOS 11.0+, and Windows 7 and newer. Bitcoin +Core should also work on most other UNIX-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +Testnet4/BIP94 support +----- + +Support for Testnet4 as specified in [BIP94](https://github.com/bitcoin/bips/blob/master/bip-0094.mediawiki) +has been added. The network can be selected with the `-testnet4` option and +the section header is also named `[testnet4]`. + +While the intention is to phase out support for Testnet3 in an upcoming +version, support for it is still available via the known options in this +release. (#29775) + +Windows Data Directory +---------------------- + +The default data directory on Windows has been moved from `C:\Users\Username\AppData\Roaming\Bitcoin` +to `C:\Users\Username\AppData\Local\Bitcoin`. Bitcoin Core will check the existence +of the old directory first and continue to use that directory for backwards +compatibility if it is present. (#27064) + +JSON-RPC 2.0 Support +-------------------- + +The JSON-RPC server now recognizes JSON-RPC 2.0 requests and responds with +strict adherence to the [specification](https://www.jsonrpc.org/specification). +See [JSON-RPC-interface.md](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md#json-rpc-11-vs-20) for details. (#27101) + +JSON-RPC clients may need to be updated to be compatible with the JSON-RPC server. +Please open an issue on GitHub if any compatibility issues are found. + +libbitcoinconsensus Removal +--------------------------- + +The libbitcoin-consensus library was deprecated in 27.0 and is now completely removed. (#29648) + +P2P and Network Changes +----------------------- + +- Previously if Bitcoin Core was listening for P2P connections, either using + default settings or via `bind=addr:port` it would always also bind to + `127.0.0.1:8334` to listen for Tor connections. It was not possible to switch + this off, even if the node didn't use Tor. This has been changed and now + `bind=addr:port` results in binding on `addr:port` only. The default behavior + of binding to `0.0.0.0:8333` and `127.0.0.1:8334` has not been changed. + + If you are using a `bind=...` configuration without `bind=...=onion` and rely + on the previous implied behavior to accept incoming Tor connections at + `127.0.0.1:8334`, you need to now make this explicit by using + `bind=... bind=127.0.0.1:8334=onion`. (#22729) + +- Bitcoin Core will now fail to start up if any of its P2P binds fail, rather + than the previous behaviour where it would only abort startup if all P2P + binds had failed. (#22729) + +- UNIX domain sockets can now be used for proxy connections. Set `-onion` or `-proxy` + to the local socket path with the prefix `unix:` (e.g. `-onion=unix:/home/me/torsocket`). + (#27375) + +- UNIX socket paths are now accepted for `-zmqpubrawblock` and `-zmqpubrawtx` with + the format `-zmqpubrawtx=unix:/path/to/file` (#27679) + +- Additional "in" and "out" flags have been added to `-whitelist` to control whether + permissions apply to inbound connections and/or manual ones (default: inbound only). (#27114) + +- Transactions having a feerate that is too low will be opportunistically paired with + their child transactions and submitted as a package, thus enabling the node to download + 1-parent-1-child packages using the existing transaction relay protocol. Combined with + other mempool policies, this change allows limited "package relay" when a parent transaction + is below the mempool minimum feerate. Topologically Restricted Until Confirmation (TRUC) + parents are additionally allowed to be below the minimum relay feerate (i.e., pay 0 fees). + Use the `submitpackage` RPC to submit packages directly to the node. Warning: this P2P + feature is limited (unlike the `submitpackage` interface, a child with multiple unconfirmed + parents is not supported) and not yet reliable under adversarial conditions. (#28970) + +Mempool Policy Changes +---------------------- + +- Transactions with version number set to 3 are now treated as standard on all networks (#29496), + subject to opt-in Topologically Restricted Until Confirmation (TRUC) transaction policy as + described in [BIP 431](https://github.com/bitcoin/bips/blob/master/bip-0431.mediawiki). The + policy includes limits on spending unconfirmed outputs (#28948), eviction of a previous descendant + if a more incentive-compatible one is submitted (#29306), and a maximum transaction size of 10,000vB + (#29873). These restrictions simplify the assessment of incentive compatibility of accepting or + replacing TRUC transactions, thus ensuring any replacements are more profitable for the node and + making fee-bumping more reliable. + +- Pay To Anchor (P2A) is a new standard witness output type for spending, + a newly recognised output template. This allows for key-less anchor + outputs, with compact spending conditions for additional efficiencies on + top of an equivalent `sh(OP_TRUE)` output, in addition to the txid stability + of the spending transaction. + N.B. propagation of this output spending on the network will be limited + until a sufficient number of nodes on the network adopt this upgrade. (#30352) + +- Limited package RBF is now enabled, where the proposed conflicting package would result in + a connected component, aka cluster, of size 2 in the mempool. All clusters being conflicted + against must be of size 2 or lower. (#28984) + +- The default value of the `-mempoolfullrbf` configuration option has been changed from 0 to 1, + i.e. `mempoolfullrbf=1`. (#30493) + +Updated RPCs +------------ + +- The `dumptxoutset` RPC now returns the UTXO set dump in a new and + improved format. Correspondingly, the `loadtxoutset` RPC now expects + this new format in the dumps it tries to load. Dumps with the old + format are no longer supported and need to be recreated using the + new format to be usable. (#29612) + +- AssumeUTXO mainnet parameters have been added for height 840,000. + This means the `loadtxoutset` RPC can now be used on mainnet with + the matching UTXO set from that height. (#28553) + +- The `warnings` field in `getblockchaininfo`, `getmininginfo` and + `getnetworkinfo` now returns all the active node warnings as an array + of strings, instead of a single warning. The current behaviour + can be temporarily restored by running Bitcoin Core with the configuration + option `-deprecatedrpc=warnings`. (#29845) + +- Previously when using the `sendrawtransaction` RPC and specifying outputs + that are already in the UTXO set, an RPC error code of `-27` with the + message "Transaction already in block chain" was returned in response. + The error message has been changed to "Transaction outputs already in utxo set" + to more accurately describe the source of the issue. (#30212) + +- The default mode for the `estimatesmartfee` RPC has been updated from `conservative` to `economical`, + which is expected to reduce over-estimation for many users, particularly if Replace-by-Fee is an option. + For users that require high confidence in their fee estimates at the cost of potentially over-estimating, + the `conservative` mode remains available. (#30275) + +- RPC `scantxoutset` now returns 2 new fields in the "unspents" JSON array: `blockhash` and `confirmations`. + See the scantxoutset help for details. (#30515) + +- RPC `submitpackage` now allows 2 new arguments to be passed: `maxfeerate` and `maxburnamount`. See the + subtmitpackage help for details. (#28950) + +Changes to wallet-related RPCs can be found in the Wallet section below. + +Updated REST APIs +----------------- +- Parameter validation for `/rest/getutxos` has been improved by rejecting + truncated or overly large txids and malformed outpoint indices via raising + an HTTP_BAD_REQUEST "Parse error". These requests were previously handled + silently. (#30482, #30444) + +Build System +------------ + +- GCC 11.1 or later, or Clang 16.0 or later, +are now required to compile Bitcoin Core. (#29091, #30263) + +- The minimum required glibc to run Bitcoin Core is now +2.31. This means that RHEL 8 and Ubuntu 18.04 (Bionic) +are no-longer supported. (#29987) + +- `--enable-lcov-branch-coverage` has been removed, given +incompatibilities between lcov version 1 & 2. `LCOV_OPTS` +should be used to set any options instead. (#30192) + +Updated Settings +---------------- + +- When running with `-alertnotify`, an alert can now be raised multiple +times instead of just once. Previously, it was only raised when unknown +new consensus rules were activated. Its scope has now been increased to +include all kernel warnings. Specifically, alerts will now also be raised +when an invalid chain with a large amount of work has been detected. +Additional warnings may be added in the future. (#30058) + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +Wallet +------ + +- The wallet now detects when wallet transactions conflict with the mempool. Mempool-conflicting + transactions can be seen in the `"mempoolconflicts"` field of `gettransaction`. The inputs + of mempool-conflicted transactions can now be respent without manually abandoning the + transactions when the parent transaction is dropped from the mempool, which can cause wallet + balances to appear higher. (#27307) + +- A new `max_tx_weight` option has been added to the RPCs `fundrawtransaction`, `walletcreatefundedpsbt`, and `send`. +It specifies the maximum transaction weight. If the limit is exceeded during funding, the transaction will not be built. +The default value is 4,000,000 WU. (#29523) + +- A new `createwalletdescriptor` RPC allows users to add new automatically generated + descriptors to their wallet. This can be used to upgrade wallets created prior to the + introduction of a new standard descriptor, such as taproot. (#29130) + +- A new RPC `gethdkeys` lists all of the BIP32 HD keys in use by all of the descriptors in the wallet. + These keys can be used in conjunction with `createwalletdescriptor` to create and add single key + descriptors to the wallet for a particular key that the wallet already knows. (#29130) + +- The `sendall` RPC can now spend unconfirmed change and will include additional fees as necessary + for the resulting transaction to bump the unconfirmed transactions' feerates to the specified feerate. (#28979) + +- In RPC `bumpfee`, if a `fee_rate` is specified, the feerate is no longer restricted + to following the wallet's incremental feerate of 5 sat/vb. The feerate must still be + at least the sum of the original fee and the mempool's incremental feerate. (#27969) + +GUI Changes +----------- + +- The "Migrate Wallet" menu allows users to migrate any legacy wallet in their wallet +directory, regardless of the wallets loaded. (gui#824) + +- The "Information" window now displays the maximum mempool size along with the +mempool usage. (gui#825) + +Low-level Changes +================= + +Tests +----- + +- The BIP94 timewarp attack mitigation is now active on the `regtest` network. (#30681) + +- A new `-testdatadir` option has been added to `test_bitcoin` to allow specifying the + location of unit test data directories. (#26564) + +Blockstorage +------------ + +- Block files are now XOR'd by default with a key stored in the blocksdir. +Previous releases of Bitcoin Core or previous external software will not be able to read the blocksdir with a non-zero XOR-key. +Refer to the `-blocksxor` help for more details. (#28052) + +Chainstate +---------- + +- The chainstate database flushes that occur when blocks are pruned will no longer +empty the database cache. The cache will remain populated longer, which significantly +reduces the time for initial block download to complete. (#28280) + +Dependencies +------------ + +- The dependency on Boost.Process has been replaced with cpp-subprocess, which is contained in source. +Builders will no longer need Boost.Process to build with external signer support. (#28981) + +Credits +======= + +Thanks to everyone who directly contributed to this release: +- 0xb10c +- Alfonso Roman Zubeldia +- Andrew Toth +- AngusP +- Anthony Towns +- Antoine Poinsot +- Anton A +- Ava Chow +- Ayush Singh +- Ben Westgate +- Brandon Odiwuor +- brunoerg +- bstin +- Charlie +- Christopher Bergqvist +- Cory Fields +- crazeteam +- Daniela Brozzoni +- David Gumberg +- dergoegge +- Edil Medeiros +- Epic Curious +- Fabian Jahr +- fanquake +- furszy +- glozow +- Greg Sanders +- hanmz +- Hennadii Stepanov +- Hernan Marino +- Hodlinator +- ishaanam +- ismaelsadeeq +- Jadi +- Jon Atack +- josibake +- jrakibi +- kevkevin +- kevkevinpal +- Konstantin Akimov +- laanwj +- Larry Ruane +- Lőrinc +- Luis Schwab +- Luke Dashjr +- MarcoFalke +- marcofleon +- Marnix +- Martin Saposnic +- Martin Zumsande +- Matt Corallo +- Matthew Zipkin +- Matt Whitlock +- Max Edwards +- Michael Dietz +- Murch +- nanlour +- pablomartin4btc +- Peter Todd +- Pieter Wuille +- @RandyMcMillan +- RoboSchmied +- Roman Zeyde +- Ryan Ofsky +- Sebastian Falbesoner +- Sergi Delgado Segura +- Sjors Provoost +- spicyzboss +- StevenMia +- stickies-v +- stratospher +- Suhas Daftuar +- sunerok +- tdb3 +- TheCharlatan +- umiumi +- Vasil Dimov +- virtu +- willcl-ark + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-process.md b/doc/release-process.md index aeaed8d68c..197ed8c0a4 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -6,8 +6,8 @@ Release Process ### Before every release candidate * Update release candidate version in `CMakeLists.txt` (`CLIENT_VERSION_RC`). -* Update manpages (after rebuilding the binaries), see [gen-manpages.py](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagespy). -* Update bitcoin.conf and commit changes if they exist, see [gen-bitcoin-conf.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-bitcoin-confsh). +* Update manpages (after rebuilding the binaries), see [gen-manpages.py](/contrib/devtools/README.md#gen-manpagespy). +* Update bitcoin.conf and commit changes if they exist, see [gen-bitcoin-conf.sh](/contrib/devtools/README.md#gen-bitcoin-confsh). ### Before every major and minor release @@ -28,7 +28,7 @@ Release Process #### Before branch-off -* Update translations see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). +* Update translations see [translation_process.md](/doc/translation_process.md#synchronising-translations). * Update hardcoded [seeds](/contrib/seeds/README.md), see [this pull request](https://github.com/bitcoin/bitcoin/pull/27488) for an example. * Update the following variables in [`src/kernel/chainparams.cpp`](/src/kernel/chainparams.cpp) for mainnet, testnet, and signet: - `m_assumed_blockchain_size` and `m_assumed_chain_state_size` with the current size plus some overhead (see @@ -134,7 +134,7 @@ git -C ./guix.sigs pull ### Create the macOS SDK tarball (first time, or when SDK version changes) Create the macOS SDK tarball, see the [macdeploy -instructions](/contrib/macdeploy/README.md#deterministic-macos-app-notes) for +instructions](/contrib/macdeploy/README.md#sdk-extraction) for details. ### Build and attest to build outputs diff --git a/doc/tor.md b/doc/tor.md index 9560dc29eb..30c2381049 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -200,7 +200,7 @@ as well, use `discover` instead: ./bitcoind ... -discover -and open port 8333 on your firewall (or use port mapping, i.e., `-upnp` or `-natpmp`). +and open port 8333 on your firewall (or use port mapping, i.e., `-natpmp`). If you only want to use Tor to reach .onion addresses, but not use it as a proxy for normal IPv4/IPv6 communication, use: diff --git a/doc/tracing.md b/doc/tracing.md index 3948b1ab49..184f6b0100 100644 --- a/doc/tracing.md +++ b/doc/tracing.md @@ -106,7 +106,7 @@ Arguments passed: 3. Transactions in the Block as `uint64` 4. Inputs spend in the Block as `int32` 5. SigOps in the Block (excluding coinbase SigOps) `uint64` -6. Time it took to connect the Block in microseconds (µs) as `uint64` +6. Time it took to connect the Block in nanoseconds (ns) as `uint64` ### Context `utxocache` @@ -245,14 +245,15 @@ Arguments passed: 2. Replaced transaction virtual size as `int32` 3. Replaced transaction fee as `int64` 4. Replaced transaction mempool entry time (epoch) as `uint64` -5. Replacement transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +5. Replacement transaction ID or package hash as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) 6. Replacement transaction virtual size as `int32` 7. Replacement transaction fee as `int64` +8. `bool` indicating if the argument 5. is a transaction ID or package hash (true if it's a transaction ID) -Note: In cases where a single replacement transaction replaces multiple +Note: In cases where a replacement transaction or package replaces multiple existing transactions in the mempool, the tracepoint is called once for each -replaced transaction, with data of the replacement transaction being the same -in each call. +replaced transaction, with data of the replacement transaction or package +being the same in each call. #### Tracepoint `mempool:rejected` @@ -265,42 +266,52 @@ Arguments passed: ## Adding tracepoints to Bitcoin Core -To add a new tracepoint, `#include <util/trace.h>` in the compilation unit where -the tracepoint is inserted. Use one of the `TRACEx` macros listed below -depending on the number of arguments passed to the tracepoint. Up to 12 -arguments can be provided. The `context` and `event` specify the names by which -the tracepoint is referred to. Please use `snake_case` and try to make sure that -the tracepoint names make sense even without detailed knowledge of the -implementation details. Do not forget to update the tracepoint list in this -document. - -```c -#define TRACE(context, event) -#define TRACE1(context, event, a) -#define TRACE2(context, event, a, b) -#define TRACE3(context, event, a, b, c) -#define TRACE4(context, event, a, b, c, d) -#define TRACE5(context, event, a, b, c, d, e) -#define TRACE6(context, event, a, b, c, d, e, f) -#define TRACE7(context, event, a, b, c, d, e, f, g) -#define TRACE8(context, event, a, b, c, d, e, f, g, h) -#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) -#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) -#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) -#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) -``` +Use the `TRACEPOINT` macro to add a new tracepoint. If not yet included, include +`util/trace.h` (defines the tracepoint macros) with `#include <util/trace.h>`. +Each tracepoint needs a `context` and an `event`. Please use `snake_case` and +try to make sure that the tracepoint names make sense even without detailed +knowledge of the implementation details. You can pass zero to twelve arguments +to the tracepoint. Each tracepoint also needs a global semaphore. The semaphore +gates the tracepoint arguments from being processed if we are not attached to +the tracepoint. Add a `TRACEPOINT_SEMAPHORE(context, event)` with the `context` +and `event` of your tracepoint in the top-level namespace at the beginning of +the file. Do not forget to update the tracepoint list in this document. + +For example, the `net:outbound_message` tracepoint in `src/net.cpp` with six +arguments. -For example: +```C++ +// src/net.cpp +TRACEPOINT_SEMAPHORE(net, outbound_message); +… +void CConnman::PushMessage(…) { + … + TRACEPOINT(net, outbound_message, + pnode->GetId(), + pnode->m_addr_name.c_str(), + pnode->ConnectionTypeAsString().c_str(), + sanitizedType.c_str(), + msg.data.size(), + msg.data.data() + ); + … +} +``` +If needed, an extra `if (TRACEPOINT_ACTIVE(context, event)) {...}` check can be +used to prepare somewhat expensive arguments right before the tracepoint. While +the tracepoint arguments are only prepared when we attach something to the +tracepoint, an argument preparation should never hang the process. Hashing and +serialization of data structures is probably fine, a `sleep(10s)` not. ```C++ -TRACE6(net, inbound_message, - pnode->GetId(), - pnode->m_addr_name.c_str(), - pnode->ConnectionTypeAsString().c_str(), - sanitizedType.c_str(), - msg.data.size(), - msg.data.data() -); +// An example tracepoint with an expensive argument. + +TRACEPOINT_SEMAPHORE(example, gated_expensive_argument); +… +if (TRACEPOINT_ACTIVE(example, gated_expensive_argument)) { + expensive_argument = expensive_calulation(); + TRACEPOINT(example, gated_expensive_argument, expensive_argument); +} ``` ### Guidelines and best practices @@ -318,12 +329,6 @@ the tracepoint. See existing examples in [contrib/tracing/]. [contrib/tracing/]: ../contrib/tracing/ -#### No expensive computations for tracepoints -Data passed to the tracepoint should be inexpensive to compute. Although the -tracepoint itself only has overhead when enabled, the code to compute arguments -is always run - even if the tracepoint is not used. For example, avoid -serialization and parsing. - #### Semi-stable API Tracepoints should have a semi-stable API. Users should be able to rely on the tracepoints for scripting. This means tracepoints need to be documented, and the @@ -347,7 +352,7 @@ first six argument fields. Binary data can be placed in later arguments. The BCC supports reading from all 12 arguments. #### Strings as C-style String -Generally, strings should be passed into the `TRACEx` macros as pointers to +Generally, strings should be passed into the `TRACEPOINT` macros as pointers to C-style strings (a null-terminated sequence of characters). For C++ `std::strings`, [`c_str()`] can be used. It's recommended to document the maximum expected string size if known. @@ -370,9 +375,9 @@ $ gdb ./build/src/bitcoind … (gdb) info probes Type Provider Name Where Semaphore Object -stap net inbound_message 0x000000000014419e /build/src/bitcoind -stap net outbound_message 0x0000000000107c05 /build/src/bitcoind -stap validation block_connected 0x00000000002fb10c /build/src/bitcoind +stap net inbound_message 0x000000000014419e 0x0000000000d29bd2 /build/src/bitcoind +stap net outbound_message 0x0000000000107c05 0x0000000000d29bd0 /build/src/bitcoind +stap validation block_connected 0x00000000002fb10c 0x0000000000d29bd8 /build/src/bitcoind … ``` @@ -388,7 +393,7 @@ Displaying notes found in: .note.stapsdt stapsdt 0x0000005d NT_STAPSDT (SystemTap probe descriptors) Provider: net Name: outbound_message - Location: 0x0000000000107c05, Base: 0x0000000000579c90, Semaphore: 0x0000000000000000 + Location: 0x0000000000107c05, Base: 0x0000000000579c90, Semaphore: 0x0000000000d29bd0 Arguments: -8@%r12 8@%rbx 8@%rdi 8@192(%rsp) 8@%rax 8@%rdx … ``` @@ -407,7 +412,7 @@ between distributions. For example, on ``` $ tplist -l ./build/src/bitcoind -v -b'net':b'outbound_message' [sema 0x0] +b'net':b'outbound_message' [sema 0xd29bd0] 1 location(s) 6 argument(s) … diff --git a/doc/translation_process.md b/doc/translation_process.md index e5ed7f4e0a..f4f0add54f 100644 --- a/doc/translation_process.md +++ b/doc/translation_process.md @@ -18,8 +18,8 @@ We use automated scripts to help extract translations in both Qt, and non-Qt sou To automatically regenerate the `bitcoin_en.ts` file, run the following commands: ```sh -cd src/ -make translate +cmake -B build --preset dev-mode -DWITH_BDB=ON -DBUILD_GUI=ON +cmake --build build --target translate ``` **Example Qt translation** diff --git a/doc/translation_strings_policy.md b/doc/translation_strings_policy.md index 1931302dda..4aa4969209 100644 --- a/doc/translation_strings_policy.md +++ b/doc/translation_strings_policy.md @@ -1,10 +1,8 @@ -Translation Strings Policy -=========================== +# Translation Strings Policy This document provides guidelines for internationalization of the Bitcoin Core software. -How to translate? ------------------- +## How to translate? To mark a message as translatable @@ -14,8 +12,7 @@ To mark a message as translatable No internationalization is used for e.g. developer scripts outside `src`. -Strings to be translated -------------------------- +## Strings to be translated On a high level, these strings are to be translated: @@ -27,8 +24,7 @@ Do not translate technical or extremely rare errors. Anything else that appears to the user in the GUI is to be translated. This includes labels, menu items, button texts, tooltips and window titles. This includes messages passed to the GUI through the UI interface through `InitMessage`, `ThreadSafeMessageBox` or `ShowProgress`. -General recommendations ------------------------- +## General recommendations ### Avoid unnecessary translation strings @@ -97,4 +93,4 @@ The second example reduces the number of pluralized words that translators have During a string freeze (often before a major release), no translation strings are to be added, modified or removed. -This can be checked by executing `make translate` in the `src` directory, then verifying that `bitcoin_en.ts` remains unchanged. +This can be checked by building the `translate` target with `cmake` ([instructions](translation_process.md)), then verifying that `bitcoin_en.ts` remains unchanged. diff --git a/doc/zmq.md b/doc/zmq.md index 07c340fb99..0a74d6eef9 100644 --- a/doc/zmq.md +++ b/doc/zmq.md @@ -46,11 +46,10 @@ operation. ## Enabling -By default, the ZeroMQ feature is automatically compiled in if the -necessary prerequisites are found. To disable, use --disable-zmq -during the *configure* step of building bitcoind: +By default, the ZeroMQ feature is not automatically compiled. +To enable, use `-DWITH_ZMQ=ON` when configuring the build system: - $ ./configure --disable-zmq (other options) + $ cmake -B build -DWITH_ZMQ=ON To actually enable operation, one must set the appropriate options on the command line or in the configuration file. @@ -163,7 +162,7 @@ Note that for `*block` topics, when the block chain tip changes, a reorganisation may occur and just the tip will be notified. It is up to the subscriber to retrieve the chain from the last known block to the new tip. Also note that no notification will occur if the tip -was in the active chain--as would be the case after calling invalidateblock RPC. +was in the active chain, as would be the case after calling the `invalidateblock` RPC. In contrast, the `sequence` topic publishes all block connections and disconnections. diff --git a/libbitcoinkernel.pc.in b/libbitcoinkernel.pc.in index a2cb7d3692..b8f9331587 100644 --- a/libbitcoinkernel.pc.in +++ b/libbitcoinkernel.pc.in @@ -3,9 +3,9 @@ exec_prefix=${prefix} libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ -Name: @PACKAGE_NAME@ kernel library +Name: @CLIENT_NAME@ kernel library Description: Experimental library for the Bitcoin Core validation engine. -Version: @PACKAGE_VERSION@ +Version: @CLIENT_VERSION_STRING@ Libs: -L${libdir} -lbitcoinkernel Libs.private: -L${libdir} @LIBS_PRIVATE@ Cflags: -I${includedir} diff --git a/share/qt/Info.plist.in b/share/qt/Info.plist.in index b4e6f6a150..5ff736152f 100644 --- a/share/qt/Info.plist.in +++ b/share/qt/Info.plist.in @@ -3,7 +3,7 @@ <plist version="0.9"> <dict> <key>LSMinimumSystemVersion</key> - <string>11</string> + <string>13</string> <key>LSArchitecturePriority</key> <array> diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py index 39acec8942..4297143023 100755 --- a/share/qt/extract_strings_qt.py +++ b/share/qt/extract_strings_qt.py @@ -56,7 +56,7 @@ files = sys.argv[1:] XGETTEXT=os.getenv('XGETTEXT', 'xgettext') if not XGETTEXT: print('Cannot extract strings: xgettext utility is not installed or not configured.',file=sys.stderr) - print('Please install package "gettext" and re-run \'./configure\'.',file=sys.stderr) + print('Please install package "gettext" and re-run \'cmake -B build\'.',file=sys.stderr) sys.exit(1) child = Popen([XGETTEXT,'--output=-','--from-code=utf-8','-n','--keyword=_'] + files, stdout=PIPE) (out, err) = child.communicate() diff --git a/share/setup.nsi.in b/share/setup.nsi.in index 2ce798bd2d..d1a85cdb7f 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -1,4 +1,4 @@ -Name "@PACKAGE_NAME@ (64-bit)" +Name "@CLIENT_NAME@ (64-bit)" RequestExecutionLevel highest SetCompressor /SOLID lzma @@ -11,8 +11,8 @@ Unicode true # General Symbol Definitions !define REGKEY "SOFTWARE\$(^Name)" -!define COMPANY "@PACKAGE_NAME@ project" -!define URL @PACKAGE_URL@ +!define COMPANY "@CLIENT_NAME@ project" +!define URL @CLIENT_URL@ # MUI Symbol Definitions !define MUI_ICON "@abs_top_srcdir@/share/pixmaps/bitcoin.ico" @@ -24,7 +24,7 @@ Unicode true !define MUI_STARTMENUPAGE_REGISTRY_ROOT HKLM !define MUI_STARTMENUPAGE_REGISTRY_KEY ${REGKEY} !define MUI_STARTMENUPAGE_REGISTRY_VALUENAME StartMenuGroup -!define MUI_STARTMENUPAGE_DEFAULTFOLDER "@PACKAGE_NAME@" +!define MUI_STARTMENUPAGE_DEFAULTFOLDER "@CLIENT_NAME@" !define MUI_FINISHPAGE_RUN "$WINDIR\explorer.exe" !define MUI_FINISHPAGE_RUN_PARAMETERS $INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@ !define MUI_UNICON "${NSISDIR}\Contrib\Graphics\Icons\modern-uninstall.ico" @@ -58,12 +58,12 @@ XPStyle on BrandingText " " ShowInstDetails show VIProductVersion @CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_BUILD@.0 -VIAddVersionKey ProductName "@PACKAGE_NAME@" -VIAddVersionKey ProductVersion "@PACKAGE_VERSION@" +VIAddVersionKey ProductName "@CLIENT_NAME@" +VIAddVersionKey ProductVersion "@CLIENT_VERSION_STRING@" VIAddVersionKey CompanyName "${COMPANY}" VIAddVersionKey CompanyWebsite "${URL}" -VIAddVersionKey FileVersion "@PACKAGE_VERSION@" -VIAddVersionKey FileDescription "Installer for @PACKAGE_NAME@" +VIAddVersionKey FileVersion "@CLIENT_VERSION_STRING@" +VIAddVersionKey FileDescription "Installer for @CLIENT_NAME@" VIAddVersionKey LegalCopyright "Copyright (C) 2009-@COPYRIGHT_YEAR@ @COPYRIGHT_HOLDERS_FINAL@" InstallDirRegKey HKCU "${REGKEY}" Path ShowUninstDetails show @@ -95,21 +95,23 @@ Section -post SEC0001 !insertmacro MUI_STARTMENU_WRITE_BEGIN Application CreateDirectory $SMPROGRAMS\$StartMenuGroup CreateShortcut "$SMPROGRAMS\$StartMenuGroup\$(^Name).lnk" $INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@ - CreateShortcut "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (testnet, 64-bit).lnk" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" "-testnet" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" 1 + CreateShortcut "$SMPROGRAMS\$StartMenuGroup\@CLIENT_NAME@ (testnet).lnk" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" "-testnet" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" 1 + CreateShortcut "$SMPROGRAMS\$StartMenuGroup\@CLIENT_NAME@ (test signet).lnk" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" "-signet" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" 2 + CreateShortcut "$SMPROGRAMS\$StartMenuGroup\@CLIENT_NAME@ (testnet4).lnk" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" "-testnet4" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" 3 CreateShortcut "$SMPROGRAMS\$StartMenuGroup\Uninstall $(^Name).lnk" $INSTDIR\uninstall.exe !insertmacro MUI_STARTMENU_WRITE_END WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayName "$(^Name)" - WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayVersion "@PACKAGE_VERSION@" + WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayVersion "@CLIENT_VERSION_STRING@" WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" Publisher "${COMPANY}" WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" URLInfoAbout "${URL}" WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayIcon $INSTDIR\bitcoin-qt.exe WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" UninstallString $INSTDIR\uninstall.exe WriteRegDWORD HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" NoModify 1 WriteRegDWORD HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" NoRepair 1 - WriteRegStr HKCR "@PACKAGE_TARNAME@" "URL Protocol" "" - WriteRegStr HKCR "@PACKAGE_TARNAME@" "" "URL:Bitcoin" - WriteRegStr HKCR "@PACKAGE_TARNAME@\DefaultIcon" "" $INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@ - WriteRegStr HKCR "@PACKAGE_TARNAME@\shell\open\command" "" '"$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" "%1"' + WriteRegStr HKCR "@CLIENT_TARNAME@" "URL Protocol" "" + WriteRegStr HKCR "@CLIENT_TARNAME@" "" "URL:Bitcoin" + WriteRegStr HKCR "@CLIENT_TARNAME@\DefaultIcon" "" $INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@ + WriteRegStr HKCR "@CLIENT_TARNAME@\shell\open\command" "" '"$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" "%1"' SectionEnd # Macro for selecting uninstaller sections @@ -140,7 +142,9 @@ Section -un.post UNSEC0001 DeleteRegKey HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\Uninstall $(^Name).lnk" Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\$(^Name).lnk" - Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (testnet, 64-bit).lnk" + Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\@CLIENT_NAME@ (testnet).lnk" + Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\@CLIENT_NAME@ (testnet4).lnk" + Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\@CLIENT_NAME@ (test signet).lnk" Delete /REBOOTOK "$SMSTARTUP\Bitcoin.lnk" Delete /REBOOTOK $INSTDIR\uninstall.exe Delete /REBOOTOK $INSTDIR\debug.log @@ -149,7 +153,7 @@ Section -un.post UNSEC0001 DeleteRegValue HKCU "${REGKEY}" Path DeleteRegKey /IfEmpty HKCU "${REGKEY}\Components" DeleteRegKey /IfEmpty HKCU "${REGKEY}" - DeleteRegKey HKCR "@PACKAGE_TARNAME@" + DeleteRegKey HKCR "@CLIENT_TARNAME@" RmDir /REBOOTOK $SMPROGRAMS\$StartMenuGroup RmDir /REBOOTOK $INSTDIR Push $R0 diff --git a/src/.clang-tidy b/src/.clang-tidy index 3569dd04b1..1cf270833a 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -14,6 +14,7 @@ modernize-use-emplace, modernize-use-equals-default, modernize-use-noexcept, modernize-use-nullptr, +modernize-use-starts-ends-with, performance-*, -performance-avoid-endl, -performance-enum-size, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 895c17541f..889c00c783 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,26 +5,18 @@ include(GNUInstallDirs) include(AddWindowsResources) -configure_file(${PROJECT_SOURCE_DIR}/cmake/bitcoin-config.h.in config/bitcoin-config.h USE_SOURCE_PERMISSIONS @ONLY) +configure_file(${PROJECT_SOURCE_DIR}/cmake/bitcoin-build-config.h.in bitcoin-build-config.h USE_SOURCE_PERMISSIONS @ONLY) include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) -# TODO: After the transition from Autotools to CMake, the obj/ subdirectory -# could be dropped as its only purpose was to separate a generated header -# from source files. add_custom_target(generate_build_info - BYPRODUCTS ${PROJECT_BINARY_DIR}/src/obj/build.h - COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/src/obj - COMMAND ${CMAKE_COMMAND} -DBUILD_INFO_HEADER_PATH=${PROJECT_BINARY_DIR}/src/obj/build.h -DSOURCE_DIR=${PROJECT_SOURCE_DIR} -P ${PROJECT_SOURCE_DIR}/cmake/script/GenerateBuildInfo.cmake - COMMENT "Generating obj/build.h" + BYPRODUCTS ${PROJECT_BINARY_DIR}/src/bitcoin-build-info.h + COMMAND ${CMAKE_COMMAND} -DBUILD_INFO_HEADER_PATH=${PROJECT_BINARY_DIR}/src/bitcoin-build-info.h -DSOURCE_DIR=${PROJECT_SOURCE_DIR} -P ${PROJECT_SOURCE_DIR}/cmake/script/GenerateBuildInfo.cmake + COMMENT "Generating bitcoin-build-info.h" VERBATIM ) add_library(bitcoin_clientversion OBJECT EXCLUDE_FROM_ALL clientversion.cpp ) -target_compile_definitions(bitcoin_clientversion - PRIVATE - HAVE_BUILD_INFO -) target_link_libraries(bitcoin_clientversion PRIVATE core_interface @@ -46,6 +38,7 @@ message("Configuring secp256k1 subtree...") set(SECP256K1_DISABLE_SHARED ON CACHE BOOL "" FORCE) set(SECP256K1_ENABLE_MODULE_ECDH OFF CACHE BOOL "" FORCE) set(SECP256K1_ENABLE_MODULE_RECOVERY ON CACHE BOOL "" FORCE) +set(SECP256K1_ENABLE_MODULE_MUSIG OFF CACHE BOOL "" FORCE) set(SECP256K1_BUILD_BENCHMARK OFF CACHE BOOL "" FORCE) set(SECP256K1_BUILD_TESTS ${BUILD_TESTS} CACHE BOOL "" FORCE) set(SECP256K1_BUILD_EXHAUSTIVE_TESTS ${BUILD_TESTS} CACHE BOOL "" FORCE) @@ -80,6 +73,9 @@ if(DEFINED ENV{CFLAGS}) endif() set(CMAKE_EXPORT_COMPILE_COMMANDS OFF) add_subdirectory(secp256k1) +set_target_properties(secp256k1 PROPERTIES + EXCLUDE_FROM_ALL TRUE +) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) string(APPEND CMAKE_C_COMPILE_OBJECT " ${APPEND_CPPFLAGS} ${APPEND_CFLAGS}") @@ -122,6 +118,8 @@ add_library(bitcoin_common STATIC EXCLUDE_FROM_ALL common/init.cpp common/interfaces.cpp common/messages.cpp + common/netif.cpp + common/pcp.cpp common/run_command.cpp common/settings.cpp common/signmessage.cpp @@ -252,10 +250,12 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL node/psbt.cpp node/timeoffsets.cpp node/transaction.cpp + node/txdownloadman_impl.cpp node/txreconciliation.cpp node/utxo_snapshot.cpp node/warnings.cpp noui.cpp + policy/ephemeral_policy.cpp policy/fees.cpp policy/fees_args.cpp policy/packages.cpp @@ -294,15 +294,14 @@ target_link_libraries(bitcoin_node core_interface bitcoin_common bitcoin_util + $<TARGET_NAME_IF_EXISTS:bitcoin_zmq> leveldb minisketch univalue Boost::headers - $<TARGET_NAME_IF_EXISTS:libevent::libevent> + $<TARGET_NAME_IF_EXISTS:libevent::core> + $<TARGET_NAME_IF_EXISTS:libevent::extra> $<TARGET_NAME_IF_EXISTS:libevent::pthreads> - $<TARGET_NAME_IF_EXISTS:NATPMP::NATPMP> - $<TARGET_NAME_IF_EXISTS:MiniUPnPc::MiniUPnPc> - $<TARGET_NAME_IF_EXISTS:bitcoin_zmq> $<TARGET_NAME_IF_EXISTS:USDT::headers> ) @@ -333,6 +332,22 @@ if(WITH_MULTIPROCESS) $<TARGET_NAME_IF_EXISTS:bitcoin_wallet> ) list(APPEND installable_targets bitcoin-node) + + if(BUILD_TESTS) + # bitcoin_ipc_test library target is defined here in src/CMakeLists.txt + # instead of src/test/CMakeLists.txt so capnp files in src/test/ are able to + # reference capnp files in src/ipc/capnp/ by relative path. The Cap'n Proto + # compiler only allows importing by relative path when the importing and + # imported files are underneath the same compilation source prefix, so the + # source prefix must be src/, not src/test/ + add_library(bitcoin_ipc_test STATIC EXCLUDE_FROM_ALL + test/ipc_test.cpp + ) + target_capnp_sources(bitcoin_ipc_test ${PROJECT_SOURCE_DIR} + test/ipc_test.capnp + ) + add_dependencies(bitcoin_ipc_test bitcoin_ipc_headers) + endif() endif() @@ -356,7 +371,8 @@ if(BUILD_CLI) bitcoin_cli bitcoin_common bitcoin_util - $<TARGET_NAME_IF_EXISTS:libevent::libevent> + libevent::core + libevent::extra ) list(APPEND installable_targets bitcoin-cli) endif() diff --git a/src/addrdb.cpp b/src/addrdb.cpp index e9838d7222..889f7b3859 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <addrdb.h> @@ -73,7 +73,7 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data remove(pathTmp); return false; } - if (!FileCommit(fileout.Get())) { + if (!fileout.Commit()) { fileout.fclose(); remove(pathTmp); LogError("%s: Failed to flush file %s\n", __func__, fs::PathToString(pathTmp)); @@ -150,7 +150,7 @@ bool CBanDB::Write(const banmap_t& banSet) bool CBanDB::Read(banmap_t& banSet) { if (fs::exists(m_banlist_dat)) { - LogPrintf("banlist.dat ignored because it can only be read by " PACKAGE_NAME " version 22.x. Remove %s to silence this warning.\n", fs::quoted(fs::PathToString(m_banlist_dat))); + LogPrintf("banlist.dat ignored because it can only be read by " CLIENT_NAME " version 22.x. Remove %s to silence this warning.\n", fs::quoted(fs::PathToString(m_banlist_dat))); } // If the JSON banlist does not exist, then recreate it if (!fs::exists(m_banlist_json)) { @@ -215,7 +215,7 @@ util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgro DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { return util::Error{strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), - e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr)))}; + e.what(), CLIENT_BUGREPORT, fs::quoted(fs::PathToString(path_addr)))}; } return addrman; } diff --git a/src/addrman.cpp b/src/addrman.cpp index 673d25efcf..9c3a24db90 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <addrman.h> #include <addrman_impl.h> @@ -188,7 +188,7 @@ void AddrManImpl::Serialize(Stream& s_) const int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); s << nUBuckets; - std::unordered_map<int, int> mapUnkIds; + std::unordered_map<nid_type, int> mapUnkIds; int nIds = 0; for (const auto& entry : mapInfo) { mapUnkIds[entry.first] = nIds; @@ -253,7 +253,7 @@ void AddrManImpl::Unserialize(Stream& s_) throw InvalidAddrManVersionError(strprintf( "Unsupported format of addrman database: %u. It is compatible with formats >=%u, " "but the maximum supported by this version of %s is %u.", - uint8_t{format}, lowest_compatible, PACKAGE_NAME, uint8_t{FILE_FORMAT})); + uint8_t{format}, lowest_compatible, CLIENT_NAME, uint8_t{FILE_FORMAT})); } s >> nKey; @@ -398,7 +398,7 @@ void AddrManImpl::Unserialize(Stream& s_) } } -AddrInfo* AddrManImpl::Find(const CService& addr, int* pnId) +AddrInfo* AddrManImpl::Find(const CService& addr, nid_type* pnId) { AssertLockHeld(cs); @@ -413,11 +413,11 @@ AddrInfo* AddrManImpl::Find(const CService& addr, int* pnId) return nullptr; } -AddrInfo* AddrManImpl::Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) +AddrInfo* AddrManImpl::Create(const CAddress& addr, const CNetAddr& addrSource, nid_type* pnId) { AssertLockHeld(cs); - int nId = nIdCount++; + nid_type nId = nIdCount++; mapInfo[nId] = AddrInfo(addr, addrSource); mapAddr[addr] = nId; mapInfo[nId].nRandomPos = vRandom.size(); @@ -438,8 +438,8 @@ void AddrManImpl::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) const assert(nRndPos1 < vRandom.size() && nRndPos2 < vRandom.size()); - int nId1 = vRandom[nRndPos1]; - int nId2 = vRandom[nRndPos2]; + nid_type nId1 = vRandom[nRndPos1]; + nid_type nId2 = vRandom[nRndPos2]; const auto it_1{mapInfo.find(nId1)}; const auto it_2{mapInfo.find(nId2)}; @@ -453,7 +453,7 @@ void AddrManImpl::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) const vRandom[nRndPos2] = nId1; } -void AddrManImpl::Delete(int nId) +void AddrManImpl::Delete(nid_type nId) { AssertLockHeld(cs); @@ -476,7 +476,7 @@ void AddrManImpl::ClearNew(int nUBucket, int nUBucketPos) // if there is an entry in the specified bucket, delete it. if (vvNew[nUBucket][nUBucketPos] != -1) { - int nIdDelete = vvNew[nUBucket][nUBucketPos]; + nid_type nIdDelete = vvNew[nUBucket][nUBucketPos]; AddrInfo& infoDelete = mapInfo[nIdDelete]; assert(infoDelete.nRefCount > 0); infoDelete.nRefCount--; @@ -488,7 +488,7 @@ void AddrManImpl::ClearNew(int nUBucket, int nUBucketPos) } } -void AddrManImpl::MakeTried(AddrInfo& info, int nId) +void AddrManImpl::MakeTried(AddrInfo& info, nid_type nId) { AssertLockHeld(cs); @@ -515,7 +515,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) // first make space to add it (the existing tried entry there is moved to new, deleting whatever is there). if (vvTried[nKBucket][nKBucketPos] != -1) { // find an item to evict - int nIdEvict = vvTried[nKBucket][nKBucketPos]; + nid_type nIdEvict = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nIdEvict) == 1); AddrInfo& infoOld = mapInfo[nIdEvict]; @@ -554,7 +554,7 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, std::c if (!addr.IsRoutable()) return false; - int nId; + nid_type nId; AddrInfo* pinfo = Find(addr, &nId); // Do not set a penalty for a source's self-announcement @@ -627,7 +627,7 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, NodeSecond { AssertLockHeld(cs); - int nId; + nid_type nId; m_last_good = time; @@ -710,7 +710,7 @@ void AddrManImpl::Attempt_(const CService& addr, bool fCountFailure, NodeSeconds } } -std::pair<CAddress, NodeSeconds> AddrManImpl::Select_(bool new_only, std::optional<Network> network) const +std::pair<CAddress, NodeSeconds> AddrManImpl::Select_(bool new_only, const std::unordered_set<Network>& networks) const { AssertLockHeld(cs); @@ -719,13 +719,18 @@ std::pair<CAddress, NodeSeconds> AddrManImpl::Select_(bool new_only, std::option size_t new_count = nNew; size_t tried_count = nTried; - if (network.has_value()) { - auto it = m_network_counts.find(*network); - if (it == m_network_counts.end()) return {}; - - auto counts = it->second; - new_count = counts.n_new; - tried_count = counts.n_tried; + if (!networks.empty()) { + new_count = 0; + tried_count = 0; + for (auto& network : networks) { + auto it = m_network_counts.find(network); + if (it == m_network_counts.end()) { + continue; + } + auto counts = it->second; + new_count += counts.n_new; + tried_count += counts.n_tried; + } } if (new_only && new_count == 0) return {}; @@ -753,14 +758,15 @@ std::pair<CAddress, NodeSeconds> AddrManImpl::Select_(bool new_only, std::option // Iterate over the positions of that bucket, starting at the initial one, // and looping around. - int i, position, node_id; + int i, position; + nid_type node_id; for (i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { position = (initial_position + i) % ADDRMAN_BUCKET_SIZE; node_id = GetEntry(search_tried, bucket, position); if (node_id != -1) { - if (network.has_value()) { + if (!networks.empty()) { const auto it{mapInfo.find(node_id)}; - if (Assume(it != mapInfo.end()) && it->second.GetNetwork() == *network) break; + if (Assume(it != mapInfo.end()) && networks.contains(it->second.GetNetwork())) break; } else { break; } @@ -786,7 +792,7 @@ std::pair<CAddress, NodeSeconds> AddrManImpl::Select_(bool new_only, std::option } } -int AddrManImpl::GetEntry(bool use_tried, size_t bucket, size_t position) const +nid_type AddrManImpl::GetEntry(bool use_tried, size_t bucket, size_t position) const { AssertLockHeld(cs); @@ -806,9 +812,11 @@ int AddrManImpl::GetEntry(bool use_tried, size_t bucket, size_t position) const std::vector<CAddress> AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered) const { AssertLockHeld(cs); + Assume(max_pct <= 100); size_t nNodes = vRandom.size(); if (max_pct != 0) { + max_pct = std::min(max_pct, size_t{100}); nNodes = max_pct * nNodes / 100; } if (max_addresses != 0) { @@ -818,6 +826,7 @@ std::vector<CAddress> AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct // gather a list of random nodes, skipping those of low quality const auto now{Now<NodeSeconds>()}; std::vector<CAddress> addresses; + addresses.reserve(nNodes); for (unsigned int n = 0; n < vRandom.size(); n++) { if (addresses.size() >= nNodes) break; @@ -849,7 +858,7 @@ std::vector<std::pair<AddrInfo, AddressPosition>> AddrManImpl::GetEntries_(bool std::vector<std::pair<AddrInfo, AddressPosition>> infos; for (int bucket = 0; bucket < bucket_count; ++bucket) { for (int position = 0; position < ADDRMAN_BUCKET_SIZE; ++position) { - int id = GetEntry(from_tried, bucket, position); + nid_type id = GetEntry(from_tried, bucket, position); if (id >= 0) { AddrInfo info = mapInfo.at(id); AddressPosition location = AddressPosition( @@ -904,8 +913,8 @@ void AddrManImpl::ResolveCollisions_() { AssertLockHeld(cs); - for (std::set<int>::iterator it = m_tried_collisions.begin(); it != m_tried_collisions.end();) { - int id_new = *it; + for (std::set<nid_type>::iterator it = m_tried_collisions.begin(); it != m_tried_collisions.end();) { + nid_type id_new = *it; bool erase_collision = false; @@ -923,7 +932,7 @@ void AddrManImpl::ResolveCollisions_() } else if (vvTried[tried_bucket][tried_bucket_pos] != -1) { // The position in the tried bucket is not empty // Get the to-be-evicted address that is being tested - int id_old = vvTried[tried_bucket][tried_bucket_pos]; + nid_type id_old = vvTried[tried_bucket][tried_bucket_pos]; AddrInfo& info_old = mapInfo[id_old]; const auto current_time{Now<NodeSeconds>()}; @@ -969,11 +978,11 @@ std::pair<CAddress, NodeSeconds> AddrManImpl::SelectTriedCollision_() if (m_tried_collisions.size() == 0) return {}; - std::set<int>::iterator it = m_tried_collisions.begin(); + std::set<nid_type>::iterator it = m_tried_collisions.begin(); // Selects a random element from m_tried_collisions std::advance(it, insecure_rand.randrange(m_tried_collisions.size())); - int id_new = *it; + nid_type id_new = *it; // If id_new not found in mapInfo remove it from m_tried_collisions if (mapInfo.count(id_new) != 1) { @@ -1058,15 +1067,15 @@ int AddrManImpl::CheckAddrman() const LOG_TIME_MILLIS_WITH_CATEGORY_MSG_ONCE( strprintf("new %i, tried %i, total %u", nNew, nTried, vRandom.size()), BCLog::ADDRMAN); - std::unordered_set<int> setTried; - std::unordered_map<int, int> mapNew; + std::unordered_set<nid_type> setTried; + std::unordered_map<nid_type, int> mapNew; std::unordered_map<Network, NewTriedCount> local_counts; if (vRandom.size() != (size_t)(nTried + nNew)) return -7; for (const auto& entry : mapInfo) { - int n = entry.first; + nid_type n = entry.first; const AddrInfo& info = entry.second; if (info.fInTried) { if (!TicksSinceEpoch<std::chrono::seconds>(info.m_last_success)) { @@ -1208,11 +1217,11 @@ std::pair<CAddress, NodeSeconds> AddrManImpl::SelectTriedCollision() return ret; } -std::pair<CAddress, NodeSeconds> AddrManImpl::Select(bool new_only, std::optional<Network> network) const +std::pair<CAddress, NodeSeconds> AddrManImpl::Select(bool new_only, const std::unordered_set<Network>& networks) const { LOCK(cs); Check(); - auto addrRet = Select_(new_only, network); + auto addrRet = Select_(new_only, networks); Check(); return addrRet; } @@ -1315,9 +1324,9 @@ std::pair<CAddress, NodeSeconds> AddrMan::SelectTriedCollision() return m_impl->SelectTriedCollision(); } -std::pair<CAddress, NodeSeconds> AddrMan::Select(bool new_only, std::optional<Network> network) const +std::pair<CAddress, NodeSeconds> AddrMan::Select(bool new_only, const std::unordered_set<Network>& networks) const { - return m_impl->Select(new_only, network); + return m_impl->Select(new_only, networks); } std::vector<CAddress> AddrMan::GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered) const diff --git a/src/addrman.h b/src/addrman.h index be2ee8c2cb..2ddf146862 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -15,6 +15,7 @@ #include <cstdint> #include <memory> #include <optional> +#include <unordered_set> #include <utility> #include <vector> @@ -154,18 +155,18 @@ public: * an address from the new table or an empty pair. Passing `false` will return an * empty pair or an address from either the new or tried table (it does not * guarantee a tried entry). - * @param[in] network Select only addresses of this network (nullopt = all). Passing a network may + * @param[in] networks Select only addresses of these networks (empty = all). Passing networks may * slow down the search. * @return CAddress The record for the selected peer. * seconds The last time we attempted to connect to that peer. */ - std::pair<CAddress, NodeSeconds> Select(bool new_only = false, std::optional<Network> network = std::nullopt) const; + std::pair<CAddress, NodeSeconds> Select(bool new_only = false, const std::unordered_set<Network>& networks = {}) const; /** * Return all or many randomly selected addresses, optionally by network. * * @param[in] max_addresses Maximum number of addresses to return (0 = all). - * @param[in] max_pct Maximum percentage of addresses to return (0 = all). + * @param[in] max_pct Maximum percentage of addresses to return (0 = all). Value must be from 0 to 100. * @param[in] network Select only addresses of this network (nullopt = all). * @param[in] filtered Select only addresses that are considered good quality (false = all). * diff --git a/src/addrman_impl.h b/src/addrman_impl.h index dd7f7b318f..a0390b7154 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -33,6 +33,13 @@ static constexpr int32_t ADDRMAN_BUCKET_SIZE_LOG2{6}; static constexpr int ADDRMAN_BUCKET_SIZE{1 << ADDRMAN_BUCKET_SIZE_LOG2}; /** + * User-defined type for the internally used nIds + * This used to be int, making it feasible for attackers to cause an overflow, + * see https://bitcoincore.org/en/2024/07/31/disclose-addrman-int-overflow/ + */ +using nid_type = int64_t; + +/** * Extended statistics about a CAddress */ class AddrInfo : public CAddress @@ -125,7 +132,7 @@ public: std::pair<CAddress, NodeSeconds> SelectTriedCollision() EXCLUSIVE_LOCKS_REQUIRED(!cs); - std::pair<CAddress, NodeSeconds> Select(bool new_only, std::optional<Network> network) const + std::pair<CAddress, NodeSeconds> Select(bool new_only, const std::unordered_set<Network>& networks) const EXCLUSIVE_LOCKS_REQUIRED(!cs); std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered = true) const @@ -179,36 +186,36 @@ private: static constexpr uint8_t INCOMPATIBILITY_BASE = 32; //! last used nId - int nIdCount GUARDED_BY(cs){0}; + nid_type nIdCount GUARDED_BY(cs){0}; //! table with information about all nIds - std::unordered_map<int, AddrInfo> mapInfo GUARDED_BY(cs); + std::unordered_map<nid_type, AddrInfo> mapInfo GUARDED_BY(cs); //! find an nId based on its network address and port. - std::unordered_map<CService, int, CServiceHash> mapAddr GUARDED_BY(cs); + std::unordered_map<CService, nid_type, CServiceHash> mapAddr GUARDED_BY(cs); //! randomly-ordered vector of all nIds //! This is mutable because it is unobservable outside the class, so any //! changes to it (even in const methods) are also unobservable. - mutable std::vector<int> vRandom GUARDED_BY(cs); + mutable std::vector<nid_type> vRandom GUARDED_BY(cs); // number of "tried" entries int nTried GUARDED_BY(cs){0}; //! list of "tried" buckets - int vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); + nid_type vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); //! number of (unique) "new" entries int nNew GUARDED_BY(cs){0}; //! list of "new" buckets - int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); + nid_type vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); //! last time Good was called (memory only). Initially set to 1 so that "never" is strictly worse. NodeSeconds m_last_good GUARDED_BY(cs){1s}; //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. - std::set<int> m_tried_collisions; + std::set<nid_type> m_tried_collisions; /** Perform consistency checks every m_consistency_check_ratio operations (if non-zero). */ const int32_t m_consistency_check_ratio; @@ -225,22 +232,22 @@ private: std::unordered_map<Network, NewTriedCount> m_network_counts GUARDED_BY(cs); //! Find an entry. - AddrInfo* Find(const CService& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); + AddrInfo* Find(const CService& addr, nid_type* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Create a new entry and add it to the internal data structures mapInfo, mapAddr and vRandom. - AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); + AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, nid_type* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Swap two elements in vRandom. void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2) const EXCLUSIVE_LOCKS_REQUIRED(cs); //! Delete an entry. It must not be in tried, and have refcount 0. - void Delete(int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); + void Delete(nid_type nId) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Clear a position in a "new" table. This is the only place where entries are actually deleted. void ClearNew(int nUBucket, int nUBucketPos) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Move an entry from the "new" table(s) to the "tried" table - void MakeTried(AddrInfo& info, int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); + void MakeTried(AddrInfo& info, nid_type nId) EXCLUSIVE_LOCKS_REQUIRED(cs); /** Attempt to add a single address to addrman's new table. * @see AddrMan::Add() for parameters. */ @@ -252,13 +259,13 @@ private: void Attempt_(const CService& addr, bool fCountFailure, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(cs); - std::pair<CAddress, NodeSeconds> Select_(bool new_only, std::optional<Network> network) const EXCLUSIVE_LOCKS_REQUIRED(cs); + std::pair<CAddress, NodeSeconds> Select_(bool new_only, const std::unordered_set<Network>& networks) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** Helper to generalize looking up an addrman entry from either table. * - * @return int The nid of the entry. If the addrman position is empty or not found, returns -1. + * @return nid_type The nid of the entry. If the addrman position is empty or not found, returns -1. * */ - int GetEntry(bool use_tried, size_t bucket, size_t position) const EXCLUSIVE_LOCKS_REQUIRED(cs); + nid_type GetEntry(bool use_tried, size_t bucket, size_t position) const EXCLUSIVE_LOCKS_REQUIRED(cs); std::vector<CAddress> GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered = true) const EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/arith_uint256.h b/src/arith_uint256.h index 38b7453034..60b371f6d3 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -26,6 +26,7 @@ class base_uint protected: static_assert(BITS / 32 > 0 && BITS % 32 == 0, "Template parameter BITS must be a positive multiple of 32."); static constexpr int WIDTH = BITS / 32; + /** Big integer represented with 32-bit digits, least-significant first. */ uint32_t pn[WIDTH]; public: diff --git a/src/base58.cpp b/src/base58.cpp index f9165ed55f..cab99f0cb5 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -139,7 +139,7 @@ std::string EncodeBase58Check(Span<const unsigned char> input) // add 4-byte hash check to the end std::vector<unsigned char> vch(input.begin(), input.end()); uint256 hash = Hash(vch); - vch.insert(vch.end(), (unsigned char*)&hash, (unsigned char*)&hash + 4); + vch.insert(vch.end(), hash.data(), hash.data() + 4); return EncodeBase58(vch); } diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt index 8a52980e07..4589ef177c 100644 --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -34,6 +34,7 @@ add_executable(bench_bitcoin load_external.cpp lockedpool.cpp logging.cpp + mempool_ephemeral_spends.cpp mempool_eviction.cpp mempool_stress.cpp merkle_root.cpp @@ -71,6 +72,7 @@ if(ENABLE_WALLET) wallet_create_tx.cpp wallet_loading.cpp wallet_ismine.cpp + wallet_migration.cpp ) target_link_libraries(bench_bitcoin bitcoin_wallet) endif() diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index c0ef7b2279..ceef6c29ab 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -133,7 +133,7 @@ static void AddrManSelectByNetwork(benchmark::Bench& bench) FillAddrMan(addrman); bench.run([&] { - (void)addrman.Select(/*new_only=*/false, NET_I2P); + (void)addrman.Select(/*new_only=*/false, {NET_I2P}); }); } diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index a2dbb11888..26daff5070 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -27,9 +27,26 @@ using util::Join; const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; -const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS{}; +/** + * Retrieves the available test setup command line arguments that may be used + * in the benchmark. They will be used only if the benchmark utilizes a + * 'BasicTestingSetup' or any child of it. + */ +static std::function<std::vector<const char*>()> g_bench_command_line_args{}; +const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS = []() { + return g_bench_command_line_args(); +}; -const std::function<std::string()> G_TEST_GET_FULL_NAME{}; +/** + * Retrieve the name of the currently in-use benchmark. + * This is applicable only to benchmarks that utilize the unit test + * framework context setup (e.g. ones using 'MakeNoLogFileContext<TestingSetup>()'). + * It places the datadir of each benchmark run within their respective benchmark name. + */ +static std::string g_running_benchmark_name; +const std::function<std::string()> G_TEST_GET_FULL_NAME = []() { + return g_running_benchmark_name; +}; namespace { @@ -94,6 +111,14 @@ void BenchRunner::RunAll(const Args& args) std::cout << "Running with -sanity-check option, output is being suppressed as benchmark results will be useless." << std::endl; } + // Load inner test setup args + g_bench_command_line_args = [&args]() { + std::vector<const char*> ret; + ret.reserve(args.setup_args.size()); + for (const auto& arg : args.setup_args) ret.emplace_back(arg.c_str()); + return ret; + }; + std::vector<ankerl::nanobench::Result> benchmarkResults; for (const auto& [name, bench_func] : benchmarks()) { const auto& [func, priority_level] = bench_func; @@ -117,6 +142,7 @@ void BenchRunner::RunAll(const Args& args) bench.output(nullptr); } bench.name(name); + g_running_benchmark_name = name; if (args.min_time > 0ms) { // convert to nanos before dividing to reduce rounding errors std::chrono::nanoseconds min_time_ns = args.min_time; diff --git a/src/bench/bench.h b/src/bench/bench.h index f0705f4fed..2df203ce23 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -61,6 +61,7 @@ struct Args { fs::path output_json; std::string regex_filter; uint8_t priority; + std::vector<std::string> setup_args; }; class BenchRunner diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 555dca7d59..88afe68a1a 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -8,6 +8,7 @@ #include <tinyformat.h> #include <util/fs.h> #include <util/string.h> +#include <test/util/setup_common.h> #include <chrono> #include <cstdint> @@ -27,6 +28,7 @@ static const std::string DEFAULT_PRIORITY{"all"}; static void SetupBenchArgs(ArgsManager& argsman) { SetupHelpOptions(argsman); + SetupCommonTestArgs(argsman); argsman.AddArg("-asymptote=<n1,n2,n3,...>", "Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-filter=<regex>", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -60,6 +62,18 @@ static uint8_t parsePriorityLevel(const std::string& str) { return levels; } +static std::vector<std::string> parseTestSetupArgs(const ArgsManager& argsman) +{ + // Parses unit test framework arguments supported by the benchmark framework. + std::vector<std::string> args; + static std::vector<std::string> AVAILABLE_ARGS = {"-testdatadir"}; + for (const std::string& arg_name : AVAILABLE_ARGS) { + auto op_arg = argsman.GetArg(arg_name); + if (op_arg) args.emplace_back(strprintf("%s=%s", arg_name, *op_arg)); + } + return args; +} + int main(int argc, char** argv) { ArgsManager argsman; @@ -131,6 +145,7 @@ int main(int argc, char** argv) args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); args.sanity_check = argsman.GetBoolArg("-sanity-check", false); args.priority = parsePriorityLevel(argsman.GetArg("-priority-level", DEFAULT_PRIORITY)); + args.setup_args = parseTestSetupArgs(argsman); benchmark::BenchRunner::RunAll(args); diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 4005701cae..00102aefe2 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -20,19 +20,23 @@ #include <memory> #include <vector> +using node::BlockAssembler; + static void AssembleBlock(benchmark::Bench& bench) { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); CScriptWitness witness; witness.stack.push_back(WITNESS_STACK_ELEM_OP_TRUE); + BlockAssembler::Options options; + options.coinbase_output_script = P2WSH_OP_TRUE; // Collect some loose transactions that spend the coinbases of our mined blocks constexpr size_t NUM_BLOCKS{200}; std::array<CTransactionRef, NUM_BLOCKS - COINBASE_MATURITY + 1> txs; for (size_t b{0}; b < NUM_BLOCKS; ++b) { CMutableTransaction tx; - tx.vin.emplace_back(MineBlock(test_setup->m_node, P2WSH_OP_TRUE)); + tx.vin.emplace_back(MineBlock(test_setup->m_node, options)); tx.vin.back().scriptWitness = witness; tx.vout.emplace_back(1337, P2WSH_OP_TRUE); if (NUM_BLOCKS - b >= COINBASE_MATURITY) @@ -48,7 +52,7 @@ static void AssembleBlock(benchmark::Bench& bench) } bench.run([&] { - PrepareBlock(test_setup->m_node, P2WSH_OP_TRUE); + PrepareBlock(test_setup->m_node, options); }); } static void BlockAssemblerAddPackageTxns(benchmark::Bench& bench) @@ -56,11 +60,12 @@ static void BlockAssemblerAddPackageTxns(benchmark::Bench& bench) FastRandomContext det_rand{true}; auto testing_setup{MakeNoLogFileContext<TestChain100Setup>()}; testing_setup->PopulateMempool(det_rand, /*num_transactions=*/1000, /*submit=*/true); - node::BlockAssembler::Options assembler_options; + BlockAssembler::Options assembler_options; assembler_options.test_block_validity = false; + assembler_options.coinbase_output_script = P2WSH_OP_TRUE; bench.run([&] { - PrepareBlock(testing_setup->m_node, P2WSH_OP_TRUE, assembler_options); + PrepareBlock(testing_setup->m_node, assembler_options); }); } diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index d1454f3634..8134154eb1 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -34,9 +34,9 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) explicit PrevectorJob(FastRandomContext& insecure_rand){ p.resize(insecure_rand.randrange(PREVECTOR_SIZE*2)); } - bool operator()() + std::optional<int> operator()() { - return true; + return std::nullopt; } }; @@ -62,7 +62,7 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) } // control waits for completion by RAII, but // it is done explicitly here for clarity - control.Wait(); + control.Complete(); }); } BENCHMARK(CCheckQueueSpeedPrevectorJob, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/cluster_linearize.cpp b/src/bench/cluster_linearize.cpp index de85741909..7d011975dd 100644 --- a/src/bench/cluster_linearize.cpp +++ b/src/bench/cluster_linearize.cpp @@ -4,7 +4,9 @@ #include <bench/bench.h> #include <cluster_linearize.h> +#include <test/util/cluster_linearize.h> #include <util/bitset.h> +#include <util/strencodings.h> #include <algorithm> #include <cassert> @@ -12,6 +14,7 @@ #include <vector> using namespace cluster_linearize; +using namespace util::hex_literals; namespace { @@ -25,7 +28,7 @@ DepGraph<SetType> MakeLinearGraph(ClusterIndex ntx) DepGraph<SetType> depgraph; for (ClusterIndex i = 0; i < ntx; ++i) { depgraph.AddTransaction({-int32_t(i), 1}); - if (i > 0) depgraph.AddDependency(i - 1, i); + if (i > 0) depgraph.AddDependencies(SetType::Singleton(i - 1), i); } return depgraph; } @@ -40,13 +43,13 @@ DepGraph<SetType> MakeWideGraph(ClusterIndex ntx) DepGraph<SetType> depgraph; for (ClusterIndex i = 0; i < ntx; ++i) { depgraph.AddTransaction({int32_t(i) + 1, 1}); - if (i > 0) depgraph.AddDependency(0, i); + if (i > 0) depgraph.AddDependencies(SetType::Singleton(0), i); } return depgraph; } -// Construct a difficult graph. These need at least sqrt(2^(n-1)) iterations in the best -// known algorithms (purely empirically determined). +// Construct a difficult graph. These need at least sqrt(2^(n-1)) iterations in the implemented +// algorithm (purely empirically determined). template<typename SetType> DepGraph<SetType> MakeHardGraph(ClusterIndex ntx) { @@ -67,19 +70,19 @@ DepGraph<SetType> MakeHardGraph(ClusterIndex ntx) depgraph.AddTransaction({1, 2}); } else if (i == 1) { depgraph.AddTransaction({14, 2}); - depgraph.AddDependency(0, 1); + depgraph.AddDependencies(SetType::Singleton(0), 1); } else if (i == 2) { depgraph.AddTransaction({6, 1}); - depgraph.AddDependency(2, 1); + depgraph.AddDependencies(SetType::Singleton(2), 1); } else if (i == 3) { depgraph.AddTransaction({5, 1}); - depgraph.AddDependency(2, 3); + depgraph.AddDependencies(SetType::Singleton(2), 3); } else if ((i & 1) == 0) { depgraph.AddTransaction({7, 1}); - depgraph.AddDependency(i - 1, i); + depgraph.AddDependencies(SetType::Singleton(i - 1), i); } else { depgraph.AddTransaction({5, 1}); - depgraph.AddDependency(i, 4); + depgraph.AddDependencies(SetType::Singleton(i), 4); } } else { // Even cluster size. @@ -95,33 +98,34 @@ DepGraph<SetType> MakeHardGraph(ClusterIndex ntx) depgraph.AddTransaction({1, 1}); } else if (i == 1) { depgraph.AddTransaction({3, 1}); - depgraph.AddDependency(0, 1); + depgraph.AddDependencies(SetType::Singleton(0), 1); } else if (i == 2) { depgraph.AddTransaction({1, 1}); - depgraph.AddDependency(0, 2); + depgraph.AddDependencies(SetType::Singleton(0), 2); } else if (i & 1) { depgraph.AddTransaction({4, 1}); - depgraph.AddDependency(i - 1, i); + depgraph.AddDependencies(SetType::Singleton(i - 1), i); } else { depgraph.AddTransaction({0, 1}); - depgraph.AddDependency(i, 3); + depgraph.AddDependencies(SetType::Singleton(i), 3); } } } return depgraph; } -/** Benchmark that does search-based candidate finding with 10000 iterations. +/** Benchmark that does search-based candidate finding with a specified number of iterations. * - * Its goal is measuring how much time every additional search iteration in linearization costs. + * Its goal is measuring how much time every additional search iteration in linearization costs, + * by running with a low and a high count, subtracting the results, and divided by the number + * iterations difference. */ template<typename SetType> -void BenchLinearizePerIterWorstCase(ClusterIndex ntx, benchmark::Bench& bench) +void BenchLinearizeWorstCase(ClusterIndex ntx, benchmark::Bench& bench, uint64_t iter_limit) { const auto depgraph = MakeHardGraph<SetType>(ntx); - const auto iter_limit = std::min<uint64_t>(10000, uint64_t{1} << (ntx / 2 - 1)); uint64_t rng_seed = 0; - bench.batch(iter_limit).unit("iters").run([&] { + bench.run([&] { SearchCandidateFinder finder(depgraph, rng_seed++); auto [candidate, iters_performed] = finder.FindCandidateSet(iter_limit, {}); assert(iters_performed == iter_limit); @@ -132,11 +136,12 @@ void BenchLinearizePerIterWorstCase(ClusterIndex ntx, benchmark::Bench& bench) * * Its goal is measuring how much time linearization may take without any search iterations. * - * If P is the resulting time of BenchLinearizePerIterWorstCase, and N is the resulting time of - * BenchLinearizeNoItersWorstCase*, then an invocation of Linearize with max_iterations=m should - * take no more than roughly N+m*P time. This may however be an overestimate, as the worst cases - * do not coincide (the ones that are worst for linearization without any search happen to be ones - * that do not need many search iterations). + * If P is the benchmarked per-iteration count (obtained by running BenchLinearizeWorstCase for a + * high and a low iteration count, subtracting them, and dividing by the difference in count), and + * N is the resulting time of BenchLinearizeNoItersWorstCase*, then an invocation of Linearize with + * max_iterations=m should take no more than roughly N+m*P time. This may however be an + * overestimate, as the worst cases do not coincide (the ones that are worst for linearization + * without any search happen to be ones that do not need many search iterations). * * This benchmark exercises a worst case for AncestorCandidateFinder, but for which improvement is * cheap. @@ -190,7 +195,7 @@ void BenchMergeLinearizationsWorstCase(ClusterIndex ntx, benchmark::Bench& bench DepGraph<SetType> depgraph; for (ClusterIndex i = 0; i < ntx; ++i) { depgraph.AddTransaction({i, 1}); - if (i) depgraph.AddDependency(0, i); + if (i) depgraph.AddDependencies(SetType::Singleton(0), i); } std::vector<ClusterIndex> lin1; std::vector<ClusterIndex> lin2; @@ -205,14 +210,57 @@ void BenchMergeLinearizationsWorstCase(ClusterIndex ntx, benchmark::Bench& bench }); } +template<size_t N> +void BenchLinearizeOptimally(benchmark::Bench& bench, const std::array<uint8_t, N>& serialized) +{ + // Determine how many transactions the serialized cluster has. + ClusterIndex num_tx{0}; + { + SpanReader reader{serialized}; + DepGraph<BitSet<128>> depgraph; + reader >> Using<DepGraphFormatter>(depgraph); + num_tx = depgraph.TxCount(); + assert(num_tx < 128); + } + + SpanReader reader{serialized}; + auto runner_fn = [&]<typename SetType>() noexcept { + DepGraph<SetType> depgraph; + reader >> Using<DepGraphFormatter>(depgraph); + uint64_t rng_seed = 0; + bench.run([&] { + auto res = Linearize(depgraph, /*max_iterations=*/10000000, rng_seed++); + assert(res.second); + }); + }; + + if (num_tx <= 32) { + runner_fn.template operator()<BitSet<32>>(); + } else if (num_tx <= 64) { + runner_fn.template operator()<BitSet<64>>(); + } else if (num_tx <= 96) { + runner_fn.template operator()<BitSet<96>>(); + } else if (num_tx <= 128) { + runner_fn.template operator()<BitSet<128>>(); + } else { + assert(false); + } +} + } // namespace -static void LinearizePerIter16TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase<BitSet<16>>(16, bench); } -static void LinearizePerIter32TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase<BitSet<32>>(32, bench); } -static void LinearizePerIter48TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase<BitSet<48>>(48, bench); } -static void LinearizePerIter64TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase<BitSet<64>>(64, bench); } -static void LinearizePerIter75TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase<BitSet<75>>(75, bench); } -static void LinearizePerIter99TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase<BitSet<99>>(99, bench); } +static void Linearize16TxWorstCase20Iters(benchmark::Bench& bench) { BenchLinearizeWorstCase<BitSet<16>>(16, bench, 20); } +static void Linearize16TxWorstCase120Iters(benchmark::Bench& bench) { BenchLinearizeWorstCase<BitSet<16>>(16, bench, 120); } +static void Linearize32TxWorstCase5000Iters(benchmark::Bench& bench) { BenchLinearizeWorstCase<BitSet<32>>(32, bench, 5000); } +static void Linearize32TxWorstCase15000Iters(benchmark::Bench& bench) { BenchLinearizeWorstCase<BitSet<32>>(32, bench, 15000); } +static void Linearize48TxWorstCase5000Iters(benchmark::Bench& bench) { BenchLinearizeWorstCase<BitSet<48>>(48, bench, 5000); } +static void Linearize48TxWorstCase15000Iters(benchmark::Bench& bench) { BenchLinearizeWorstCase<BitSet<48>>(48, bench, 15000); } +static void Linearize64TxWorstCase5000Iters(benchmark::Bench& bench) { BenchLinearizeWorstCase<BitSet<64>>(64, bench, 5000); } +static void Linearize64TxWorstCase15000Iters(benchmark::Bench& bench) { BenchLinearizeWorstCase<BitSet<64>>(64, bench, 15000); } +static void Linearize75TxWorstCase5000Iters(benchmark::Bench& bench) { BenchLinearizeWorstCase<BitSet<75>>(75, bench, 5000); } +static void Linearize75TxWorstCase15000Iters(benchmark::Bench& bench) { BenchLinearizeWorstCase<BitSet<75>>(75, bench, 15000); } +static void Linearize99TxWorstCase5000Iters(benchmark::Bench& bench) { BenchLinearizeWorstCase<BitSet<99>>(99, bench, 5000); } +static void Linearize99TxWorstCase15000Iters(benchmark::Bench& bench) { BenchLinearizeWorstCase<BitSet<99>>(99, bench, 15000); } static void LinearizeNoIters16TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc<BitSet<16>>(16, bench); } static void LinearizeNoIters32TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc<BitSet<32>>(32, bench); } @@ -242,12 +290,84 @@ static void MergeLinearizations64TxWorstCase(benchmark::Bench& bench) { BenchMer static void MergeLinearizations75TxWorstCase(benchmark::Bench& bench) { BenchMergeLinearizationsWorstCase<BitSet<75>>(75, bench); } static void MergeLinearizations99TxWorstCase(benchmark::Bench& bench) { BenchMergeLinearizationsWorstCase<BitSet<99>>(99, bench); } -BENCHMARK(LinearizePerIter16TxWorstCase, benchmark::PriorityLevel::HIGH); -BENCHMARK(LinearizePerIter32TxWorstCase, benchmark::PriorityLevel::HIGH); -BENCHMARK(LinearizePerIter48TxWorstCase, benchmark::PriorityLevel::HIGH); -BENCHMARK(LinearizePerIter64TxWorstCase, benchmark::PriorityLevel::HIGH); -BENCHMARK(LinearizePerIter75TxWorstCase, benchmark::PriorityLevel::HIGH); -BENCHMARK(LinearizePerIter99TxWorstCase, benchmark::PriorityLevel::HIGH); +// The following example clusters were constructed by replaying historical mempool activity, and +// selecting for ones that take many iterations (after the introduction of some but not all +// linearization algorithm optimizations). + +/* 2023-05-05T23:12:21Z 71, 521780, 543141,*/ +static constexpr auto BENCH_EXAMPLE_00 = "801081a5360092239efc6201810982ab58029b6b98c86803800eed7804800ecb7e058f2f878778068030d43407853e81902a08962a81d176098010b6620a8010b2280b8010da3a0c9f069da9580d800db11e0e9d719ad37a0f967897ed5210990e99fc0e11812c81982012804685823e0f0a893982b6040a10804682c146110a6e80db5c120a8010819806130a8079858f0c140a8054829a120c12803483a1760c116f81843c0d11718189000e11800d81ac2c0f11800d81e50e10117181c77c1111822e87f2601012815983d17211127180f2121212811584a21e1312800e80d1781412813c83e81815126f80ef5016126f80ff6c16126f80f66017126e80fd541812800d81942a1912800e80dd781a12800d81f96c1b12805282e7581b127180fd721c1271a918230b805fc11a220d8118a15a2d036f80e5002011817684d8241e346f80e1181c37805082fc04260024800d81f8621734803382b354270b12805182ca2e162f800e80d52e0d32803dc360201b850e818c400b318c49808a5a290210805181d65823142a800d81a34e0850800e81fb3c0851886994fc0a280b00082c805482d208032e28805e83ba380059801081cd4a0159811884f770002e0015e17280e49024300a0000000000000031803dcb48014200"_hex_u8; +/* 2023-12-06T09:30:01Z 81, 141675, 647053,*/ +static constexpr auto BENCH_EXAMPLE_01 = "b348f1fc4000f365818a9e2c01b44cf7ca0002b004f0b02003b33ef8ae3004b334f9e87005800d81c85e06b368fae26007b05ef2e14208be1a8093a50409b15cf5ee500a802c80a1420b802dea440c802ce50a0d802cdc320e802cd7220f802dd72210805380f74a118174f370126e96b32812127182c4701312817389d26414128035848c221512800e82bf3816126f81e4341712801082b228181280518af57418128040859a0019127182d0401a12803e858b641b127182c4421c126f82b3481d12811486b6301e12821d89e7281f126e8a8b421f127182d6642012806284c12021126e81d34822126e86a76222126e86d8102212805187b6542312800d82fc002412803d848e0e2512801082d27a26126e8589642612800e83a9602712800e83bd0028126e81ef1a29116e858d7228126f82db5e2912801083843c2a127181c93c2b126e85d0162b127181c5622c126e84f8262c12800f8392202d12800e82b66c2e126e81d0082f12803282d50430126e84f9003012805f84be6c3112846e88df0e2b12804080d44c340a8b31898808350a800ed760350b801083a1182b517182817e2a51800e82b6582951803583cb52420030806284cb6c204f7181d300204f82688ce0303e001d800e82bb200f488010808a182822a3289cd63041000a6fcd100a408a7caaa7024800002f803584e0741e27288f3386dd783b001000802683f27e004b8c44bcd0763f0000000000000000000100000e00"_hex_u8; +/* 2023-04-04T00:26:50Z 90, 99930, 529375,*/ +static constexpr auto BENCH_EXAMPLE_02 = "815b80b61e00800da63001cd378da70e028010991a03800e9d3e0480109708058010991a068010973a07da738fa72408de7491831009b35b88f0080a9d4485de180b71974e0c71974e0d80108e500eb27988a75a0f719632108061a56c11801087761280108a1413807893441480538c1415a606828806168010893e1780548c40188e4b80bb2c196eab3e1718805ed60e18188051c97a19188010cf781a1871b11e1b1871c5281c1880508080581d186e80b13c1e188035cf421f18805fe0482018804caa661f198035a9001f156e80cb701d1871a2281e1871ad281f18817380a16020186f98642118805ee04821198010b6702219800ea12623196eb67024198035808b0025196fa65c26198054ba1c2719807680bf7c28198053cd782919803d80b80429198051db5a2a198040d3742b19976584bb1c28196efc1c281971b21a29198052bc762a1971a2502b196eb73c2c19976381ab0c2a18806290543409862081c3423b00336fbc70224d80109e7c1c52805ebd5c1942800eb57016468034ba423405158118da28350416927480f4743000159f6a81c9462e00188051ec5e380e00800e9e420775800d9e26007c906c82f754251d0025870480f12c14280023800d9e26027e9e1385ed08102900001a804fac7a018001719856028001800da87e0180039b1a868b60064102246e9f42018005800da87e028005850d81d600026d862381a2200e0008230015831480a5480342000524803eeb32006e873582a4700a0100351300"_hex_u8; +/* 2023-05-08T15:51:59Z 87, 76869, 505222,*/ +static constexpr auto BENCH_EXAMPLE_03 = "c040b9e15a00b10eac842601805f85931802c104bae17403ae50aaa336049d76a9bf7005c55bbeab6606ae2aa9c72c07805e81992e08af7dab817a096e80a7e4520909803e92bd780a097185c76c0b096e98e7380b09850bb9953c0c09803389f6260d096f859d620e09803f88d3000f0971829c6e1009837690f6481109806285931811097181f56814076ea09b74120980408eb73213096f87853214096f86e2701509803f8c860016098a6fe6c3721709814f92a204180980628a8a441909803285df681a0980348498661b096e8290781c096e978e081c097187da1a1d097186c05c1e097185893c1f09805f8ad9002009800d84e74e21097183a67a22097182e23423097184b53a23096ea393062309840faddd46240980618eb732250980548bee6a2609807986883c2709718298402809815388b6582909805384ec742a097181b9142b096e97b5262b096e85e14e2c0980518abb5c2d09805489e75a2e09803187e3382f097180eb1c34046f87c34a2f098309a5c54430097186911831098054899c083209801083bc1033097081e02a3409805f848f0c35096e80d4343a057180c37040006f80a22438097180a0503f03816f8381444003803f80ef003f05800580a4283f066ef72845016efb91663e09923d808d8216470041803584837c46012f9247dc86684501268267a09610450222862184db68440712803585ea40440113835d97887805800b8723c7a40a4b00022f81529ae2143c0c1f80548b8f381b311980408e955c055e802589dc10037e801083b54602658010848130006700"_hex_u8; +/* 2023-05-01T19:32:10Z 35, 55747, 504128,*/ +static constexpr auto BENCH_EXAMPLE_04 = "801af95c00801af72801801af95c02873e85f2180202873e85f2180202873e85f21802028018fb2802068018fb2803068018fb2804068018fb2805068018fb2806068018fb2807068018fb2808068018fb2809068018fb280a068018fb280a058018fb280b058018fb280c058018fb280d058018fb280e058018fb280f058018fb2810058018fb2811058018fb2812058018fb2813058018fb2814058018fb2815058018fb2815048018fb2816048018fb2817048018fb2818048018fb2819048018fb281a048018fb281b04810d80d9481f00000100"_hex_u8; +/* 2023-02-27T17:06:38Z 60, 55680, 502749,*/ +static constexpr auto BENCH_EXAMPLE_05 = "b5108ab56600b26d89f85601b07383b01602b22683c96003b34a83d82e04b12f83b53a05b20e83c75a066e80840a06068040be0007066fb10608066fb2120906800eba320a06842b80b05a0a066eff420b067199300b068124c3140c0680618085180d066faa1c0e068010b4440f068051af541006800da1781106857881946812066eee1613068052b31014068324808d361506806180885c150671b03216066ef11017068052b63218066ef3521806803f80865419066e93441a068035a13e1b0680628085181c06806ec4481d068117e72c1e06719c721f068077c42420068159808d1821066eef0c21058010b90022056f9908230571993024058010b00a25058010b00a260580608087402705803fc10027068032b42828068051b6322906800db11e212a8324808d361933803ff400192f826381a7141a2f8032ac08152a800db54c044e8323808d3630010002018158d84000042d821cea12002807853580d462002d01891181d022002e00"_hex_u8; +/* 2023-04-20T22:25:49Z 99, 49100, 578622,*/ +static constexpr auto BENCH_EXAMPLE_06 = "bf3c87c14c008010955a01b21d85e07002800d946c036e8e3404b77f86c26605b33c85f55e06bd06879852078010970a08bd4b87cf00098123a7720ab2158687680b8054d4440b0a8062fa4c0c0a71ac400d0a80628081540e0a8010a2580f0a8054b676100a8032b85c110a6e9a40120a6e809012130a817f80c31e140a8175808674150a719d46160a8172d86415098033c1481609800da4181709800ada2e1809803dc85219098034b4041a096ef5501b098052d67c1c098051d3281d09800ebc4a1e098175808c641f098061c55020098078c85021096e8081141f0b6faf1e200b8061da68210b8062f000220b800ebc20230b8035d058240b8053de32250b8050b610250b6fad32260b803dc276270b803d80a610280b6ef812290b8052b6322a0b800eb57e2b0b8052bd062c0b719e522d0b71a3762e0b8010bb1e2f0b80109a78310a80109962320a8051a60c330a6f9f3e320b6e808b24330b719e40340b8117cc50350b803d80971a360b8051b930370b6f9e0a380b719b10390b8052a6003a0b6e808c76390a7195603a0a6f935c3b0a8054a31a3c0a803ce30c3b0b803fa3003c0b800dbe2a3d0b8f3480a84244058005851a44069d1bf824400b83098f284507719c723d4f6f9c1c3449719c722f4f6eb23c304f8061c5502e528061da682b4e8118bb724e022a8054b35028476e941c1d51815be02c4f01148557808e3a4f070e8104af464e001180329d364e010d805f9f6a421b9c3387aa744c0d4d71ac400b800881748098444710338173809b780b80008054d444292c12821dc040550403078b4682b4664517003f00"_hex_u8; +/* 2023-06-05T19:56:12Z 52, 44896, 540514,*/ +static constexpr auto BENCH_EXAMPLE_07 = "b317998a4000b40098d53e01b45b99814802b7289b940003b3699a9d1204b6619a807a05814682cb78050571d854060571d8540705800e808d7a0805803480c06a09056e8189280a056ffd060b05800d80ea7a0c05803c80b80c0c03803e80d86e0d036ed2280e03811581804a0f036fd34e1003805380eb6811036e81f60e12038010ec101204805f80e83a13048033809534140471e00a15048010f95816046e81fa301704805180a74c1705800d808f1018056fd55c1905800e8091481a056e80a76e1b05805f80e2741c0571809b021c05826382c8401d0571df201e05800e809d2c1f05850083e87c1f05811580af68200571f20a21056ff9042205803e80df1e23056e81956c24056e9f542604805180e83829000e800e8080621325803380b0402a020d6ef8100e2c8c4889a96a2c000f803580ce4c2c000b6e9f54062a803480c96406260500"_hex_u8; +/* 2023-12-05T23:48:44Z 69, 44283, 586734,*/ +static constexpr auto BENCH_EXAMPLE_08 = "83728ce80000b90befca1001806083b24002b40de6da3203b545e9c35c04b34beede3005b068e8883006d41c80b1e14c07b337e7841208b26beadb2e096e83892e090980518487380a096e82815c0a096e81ce3c0b097181db200c097181d4020d09810084ed600e096e96b0100f0971819a0210086e93da2e0f09803583ee5e1009803583c66c1109800d82bb6e1209800d81d56a1309803c82e622140971819f521509803d84a55c15057181d6161605806283ac5217056e949c5a18056e89e8641806815889e23419067181de321a066e8af2641a076e82a70a1b07803583f2081c076f81e76e1d076e81d33e1e07800d83b8761e086e82a5541f087181de302008805f84ad0021086e81c74022086e81bd3e23086e9288182408806184b3102409803283816025096e91ed662609830a88e70827096e81d14a27097181ce6028096e8cf03829097181883832016f81835c3103806181e0103203804180b8103204863584fe183304800de66434046e9e4c34056e81d6742f429213c0eb602e3d6483b06c283a6e81d73c263d6e82f9581831805485ab360e37805080c62609398b3189880838010603916db1f3583a03000110873199f8623c000000011100"_hex_u8; +/* 2023-04-14T19:36:52Z 77, 20418, 501117,*/ +static constexpr auto BENCH_EXAMPLE_09 = "bf2989d00400815bca5c01af1e86f97602800d9d6c03800d8a3404b47988866e05b36287f92e0680109f68078010991a08805ecf1208076e80933e09078062d01c0a078054b6760b078053b6760c076f9c1c0d078054b6760e0771af260f0771b17e10078032f57011078035d56812078054e1581307886b83dc301407817480d13013068005a6001406803d80821a15066ef3201606800ea2181706800da628180671ab1219068054db0c1a06719b001b06815b80a11c1c068050b9301d066fac2a1e068033ab481f06719b1020068035ab721e07803dc2761f0771ae3c20078040f60e210771ce282207800ea4322307882a81a66024078035ad4625076efe7e26078162808e1827078118bb7228076eac7428088010bf58290871a04c2a0871bc722b086fa8382c08803d80a0142d088035d6282e088051c30c2f086efc623008800d9f6231086f986432088117bb7237028010a63034068010c84e2740800ea64c2237832c80933e1f3b830880c454390208813c80955c3905068032c73611348010a03c093c837a808a101b278050ac34093a8051ac34291b8f3b8187401d28881a82cb3a3a0a37977b86d20843000028996686a7083f030f8078d3761b27106e995a08499070839b5a1131000b00"_hex_u8; +/* 2023-11-07T17:59:35Z 48, 4792, 498995,*/ +static constexpr auto BENCH_EXAMPLE_10 = "875f89aa1000b51ec09d7201c55cc7a72e02a11aa1fb3203b233a7f95204800ef56205b33ea9d13006803e80b26e07d90ec9dd4008b45eabbe6c09806080ca000a815984e8680a0a6f80925e0a0a803f80e1660c09937c94b7420d086e82f5640a086e80997e0b086f808d320c08800580a5640d086f8089100e08804080c9060f088115819a1c10086e82961a0f0a805f81bc0a100a6ff826110a6ef53e120a807584c60c110a6e818f32120a803c81c246130a805481d508140a8159838410150a7180a55c160a6f80821c170a6fe6101c066fe6101d06805080f854190a6e81b27c1a0a8155819c701e06805180ae0c21046e8b9a222501805180f53422001680f26880f8a62a220116803580da582007058153838e6e21000c800d80a712033a807681ae1c23000308834a82d36023020205815981e03a051a08001700"_hex_u8; +/* 2023-11-16T10:47:08Z 77, 473962, 486863,*/ +static constexpr auto BENCH_EXAMPLE_11 = "801980c06000801980c06001801980c06002801980c06003801980c06004801980c06005801980c06006801980c06007801980c06008801980c06009801980c0600a801980c0600b801980c0600c801980c0600d801980c0600e801980c0600f801980c060108019d12c11800f80b1601111800f80b1601111801080b1601111800f80b160100e800f80b160100f801980c060110f800f80b160140d801180b1601111801180b160100d801180b160120c801180b1600f10801180b1600f11801980c0601011800f80b160140e800f80b160110f801980c060170a801180b1601210801980c060140f800f80b1601311801980c0602005801180b1601f07800f80b1601b0c800fca7c1611812081f9601638812081f9601637812081fb001636801080b160142f801980c0600e2a801080b1600f2a801180b1600d25801980c0600e25800f80b1600d27801980c0600e27801980c0600d27801180b1600e26812080b1500c27812081f960201025812081f960200f27812081fc201d101c812081fc201d101d812081fc201d0f1f812081fc201d0f20812081f9601b1016800f80b1600a35800f80b1600a36800f80b1600e32801080b160122f812081f960280040812081fc20121d1b812081f960112713812081f960160d37812081fc20140d2b812081f960130d2d812081fc20130c2c812081fb001b0157812081fb001a0245812081fc20140030812081fc20092747812081fb000b152500"_hex_u8; +/* 2023-10-06T20:44:09Z 40, 341438, 341438,*/ +static constexpr auto BENCH_EXAMPLE_12 = "80318f4c0080318f4c0180318f4c0280318f4c0380318f4c0480318f4c0580318f4c0680318f4c078033a57807078033a57807078033a57807078033a57807078033a57807078033a57807078033a57807078033a578070780318f4c0e0180318f4c0d0380318f4c0c0580318f4c0b078033a57803128033a57803128033a57803128033a578031280318f4c0412810b9c28140300810c9c281303028033a57802188033a57802188033a5780218810c9c280b01108033a578001c810c9c2807050f8033a578001b810c98040700158033a578001c810c98040301158033a5780019806ca1240101118033a578001300"_hex_u8; +/* 2023-11-15T21:40:46Z 96, 23608, 138286,*/ +static constexpr auto BENCH_EXAMPLE_13 = "8060829f4000b157bab07a01b27cc2b16802b22fbce54603826480a95804803da81a05bc7bcac93806800de55207800daf0608805bc71809805bc7180a800d9d4a0b805bbc700c8152d7180d805bb9380e850a8886260f800d80d33410bf38d3d55011b41dc4eb6012bd70d2ce2e138d3596af7812137180cd501313805e81f7281413718092001513803d81f90016136e8b916c1713801081861a17106e80cd2a18106f80cc3c19106e80cf161911800d80fe781b107180d87c1c106e80fb081d10803e8286701d11800d81c4781f10804082a6002010801081912e21107180ff0021116e81da4a2310850b8b864023116e89db3224116e84ff7e2610897c95993427106f80bb1a240b803581c272250b8032828c10260b6e80d42a270b804082b35a280b800d80fe3e290b805cc0282312821d8697022b0b6e8add562c0b805281c8063007811883f1082313800d80fe3e24137180c9142513800d8380102613803382c00e2713805eb32228136e8494542913800e8186742913806082b74c2a1380528285782b13800d818f7a2c136e84a5562d1380508286702e136f80a46e3e04803f8191364102805481ad4c3d076e809a5a3e077180fe4032136e838b7233138c4790cf384106853584ab624206805b80932a4801806280966c48028168ef04400b7181bd524903806282db5c375b9316acbf703a599c68c5a454385c6e81d63e364a6f80ff64334e817485a6784f023171819536234e800d81826e1e498053829a12420018834c87cb14291d2e840e8bc94c1d2825800d81b7220368811783fe0e271f1f811783e758380f001ecd55809edf6e56000000003a815984ba76008010d54d80aebb4e2c22000000000000002c807682f150007a00"_hex_u8; +/* 2023-12-06T09:18:20Z 93, 68130, 122830,*/ +static constexpr auto BENCH_EXAMPLE_14 = "b26beadb2e00800d80ca0a01d41c80b1e14c02b068e8883003800d81af1604b34beede30056e80b14006b151f5d46c07b93e8085b02608b30cf98b1009b14ef6b3040ab176f6ab480bb7078082b8640c800d81c6460d802c80a8080e802c80a8080f802c80a14210802ce50a11802cd722127181ce6012126e81d14a13126e9b8b00141282428dd42c15128051828408150e6e81bd3e150f805f84ad00160f7181de30170f6e81c740180f800d83b876190f6e82a5541a0f6e81d33e1a106e82a70a1b106f81e76e1c10803583f2081d106e82d9401e106e96e4441f107181de321e12815889e2341f127182d60c20126e979d4e21126e8282262410800d82972c25106f838a5822126f82842a23127182d24a2412803e84bc2a2512800d83c81a26126e84f8142712805085a22c27126e889e6a2812801083aa50281280348598102912801082d5522a126e85865c2b127182c7602b1282468c82042c126e84972c2d12805485d93a2d12801083c7322e12815386e1582f126e84fb0c30126f82eb6c3011813a85b47a3111803f869f5c3211805181ed30370d6e84bf0a3411804180e1383809815883aa183a08815a8392203e05807681f140380c6e9e4c4005805485ab363255805183856030406e82f9582c45805185c1001b4f82418df1001a4e803283c50e430026800d83a6201a4b836886be3044010b8b318988084c0101803183a6120776800d828a1e087682338ae050301c33873199f8624d010032813986bc663c1034800d83a5220a6f800d82be52048000805183e364084907800d83cc4a018005815987b41e1832000017884b9dce72035035803284c11e00800885769d9538192f0000000002001000"_hex_u8; +/* 2023-12-14T02:02:29Z 55, 247754, 247754,*/ +static constexpr auto BENCH_EXAMPLE_15 = "801980c06000801980c06001801980c06002801980c06003801980c06004801980c06005801980c06006801980c06007801980c06008801980c06009801980c0600a801980c0600b801980c0600c801980c0600d801980c0600e801180b1600e0e801180b1600e0e801180b1600e0e801180b1600e0e801180b1600e0e801180b1600e0e801180b1600d07801180b1600f06801180b1600c0a801180b1600f08801180b1600c0c801180b1600c0d801180b1600c0e801180b160100b801180b1601309812081fc200e2a812081fc200e29812081fc200e28812081fc200e0e18812081fc200e0e17801980c060042e812081fc200e0d07812081fc200e0d08812081fc200e0c0a812081fc200e0d0a801980c060081e812081fc200f0c0c812081fc200f0c0d812081fc200f0c0e801180b160083a801180b1600426801980c0600b20801980c0600a22812081fc200f0b30801180b160022b801180b160022b812081fc20062422812081fc2006220b812081fc200c0a1e812081fc2012041a00"_hex_u8; +/* 2023-12-14T15:17:20Z 76, 102600, 103935,*/ +static constexpr auto BENCH_EXAMPLE_16 = "801980c06000801980c06001801980c06002801980c06003801980c06004801180b1600404801180b1600404801180b1600404801980c0600504801980c0600802801980c0600803801180b1600704801980c0600804801280b1600804812081fc200810812081fc20080f812081fc20080e801180b160080c800f80b160080d801980c060090d801180b160090e801980c0600a0e812181fc200a0c801180b1600a0d812181fd400a0c801980c0600a1c801980c0600916801180b1600719801180b160061b801980c0600d15801980c0600717812081fc200718801980c0600716801180b160072d801180b1600722801180b1600525801980c060091b801980c060071e801080b160071f801280b160061d812081fc20063a812181f960160815801280b1600525801980c0600625801180b1600626801980c0600726801980c0600536801180b160032b801980c060042b801280b160032d801980c060033e801180b160043e812181fc20100c27801080b160042f801980c0600342801180b1600442812081fc20150d25800f80b1600245812081fd40120619812081fc20040243812081fc20120c2c812081fd40120a1d812181fb00100623812081fc20030347812081fc20072126801980c0600236812081fc20040d2b812081fc20120328801980c0600237801180b1600337812081fc20052230801180b1600239812081fc2008242c812081fd4005112d812081fb00070b32812081f96011034700"_hex_u8; +/* 2023-12-15T07:12:29Z 98, 112693, 112730,*/ +static constexpr auto BENCH_EXAMPLE_17 = "801980c06000801980c06001801980c06002801980c06003801980c06004801980c06005801980c06006801180b1600606801180b1600606801180b1600606801180b1600606801280b1600606801180b1600606801180b1600606801980c0600d00801980c0600b03801980c0600b04801980c0600f01812081fc200a16812081fc200a15812081fc200a14812081fc200a13812081fd400a12812181fc200a11812181fc200a0f801180b1600a10801180b1600a10801980c0600a10801180b1600b10801180b1600b10801980c0600621801980c0600915801980c060041b801180b160051b801980c0600f12801980c0600f13801980c0600d15801980c0600c17801980c060072e800f80b160082e812181fc200d150e801980c0600922801180b1600923801980c0600823801180b1600623801180b1600a20801180b1600e1c801180b1600b20801180b1600b21801980c0600a3e800f80b1600b3e801980c0600931801180b1600a31812181fc20140325801180b1600a30801180b160054c801180b160043b801980c0600336812181fc200253812081f960090944812081fc2007003c801980c0600339801180b1600433801980c0600453801980c0600340801980c060033d801080b160043d812081f960070854801980c060045a801180b160055a801180b1600545801980c0600643801980c0600641801280b1600739801180b1600562812081fc20121f27812181fc20210137812181fc2016112f801980c0600259801980c0600156812181fc20053a31801180b160025c801180b1600257801980c0600357812081fc200d2d1e812181fc20102444812181fc20035a801180b160035b801980c0600751812181fc2007392a812181fc20025f801980c060045e801180b1600350812081fc20070f6f801180b1600263812181fc201b1322812181fc2011283b812081fc2002442100"_hex_u8; +/* 2023-12-16T02:25:33Z 99, 112399, 112399,*/ +static constexpr auto BENCH_EXAMPLE_18 = "801980c06000801980c06001801980c06002801980c06003801980c06004801980c06005801980c06006801980c06007801180b16008801180b16009801180b1600a801180b1600a0a801180b1600a0a801180b1600a0a801180b1600a0a801980c0600d06801180b1600b09801980c0601005801180b1600c0a801980c0600d0a801980c0601106801180b1600e0a801980c0601207801980c0601207801180b160100a812081e668100a812081e668100a812081e668100a801980c0601407801980c0601606812081fc201226812081fc201225812081fc201224812081fc201223801180b1600e21801980c0600b1e801180b1600c1e801180b1601316801980c060091b801980c0601312801980c0600a1c801180b160190e801180b1601315801180b1600e1b801180b1601713801180b1600f1c801980c0600d34801980c0600d30801980c060102e801980c060122d801980c0600b2a801980c0600b2a801980c0600b2b801180b1601122801180b1600e26801180b1601025801180b1600f26812081fc20280032812081fc20270034812081fc20250034801180b1600d4b801980c0600d457a809a000d46801980c0601044801980c0600e46801180b1600f43801180b160123f801180b160123e801180b1601130801180b1601131801180b1601131812081fc20230a36801980c0600a5a801180b1600a5b801980c0600a5b801180b1600b5b801980c0600b5a801180b1600f57801180b1600d3f801980c0600669801980c0600568801980c0600466801180b1600945801180b1600649801180b1600945812081fc2018234b812081fc20142534812081fc20142532812081fc20142530801180b160074d801180b1600a4b801180b1600a4a812081fc20221662812081fc200c0472812081fc20072e42812081fc20062c23812081fc20100572812081fc200f036c812081fc2001345100"_hex_u8; +/* 2023-03-31T19:24:02Z 78, 90393, 152832,*/ +static constexpr auto BENCH_EXAMPLE_19 = "800dd042008028b13c018028b13c028028b13c038029b13c048029b13c058029b13c0680299948078029b13c088029b13c09802899480a802899480b8028b13c0c80299e700d802899480e802999480f8029b13c10802999481180299948128028b13c138029b13c1480289e701580289948168028b13c1780289948188028994819802899481a802999481b802999481c802899481d802999481e8028b13c1f8029b13c20802999482180299948228028b13c2380298c242480289948258029b13c2680288c242780298c242880299e70298f5a80ea762a824780aa00292a82038090402429813fcf00152a8203809040142a813ff700112982038090402d002d813ff70028002c8203809040270024824780aa00270025820380904025002882038090401e022a82038090401d042782038090401c01298203809040190029813ff700170028813ff700140128807b9258120128841280f6402c01002e82038090402b00062b820380904027000031813ff70011192d82038090401d000129851981a9403a0000003b82038090400c182e813ff7000b0f2982038090401314141b807b925805192b84568190001121000334807bdd400149824780aa00001f2a813ff700003d0b8203809040050d1915807bdd4001498728828f400b010004050501000a050c851981a9400104050b061a0400"_hex_u8; + +static void LinearizeOptimallyExample00(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_00); } +static void LinearizeOptimallyExample01(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_01); } +static void LinearizeOptimallyExample02(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_02); } +static void LinearizeOptimallyExample03(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_03); } +static void LinearizeOptimallyExample04(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_04); } +static void LinearizeOptimallyExample05(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_05); } +static void LinearizeOptimallyExample06(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_06); } +static void LinearizeOptimallyExample07(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_07); } +static void LinearizeOptimallyExample08(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_08); } +static void LinearizeOptimallyExample09(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_09); } +static void LinearizeOptimallyExample10(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_10); } +static void LinearizeOptimallyExample11(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_11); } +static void LinearizeOptimallyExample12(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_12); } +static void LinearizeOptimallyExample13(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_13); } +static void LinearizeOptimallyExample14(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_14); } +static void LinearizeOptimallyExample15(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_15); } +static void LinearizeOptimallyExample16(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_16); } +static void LinearizeOptimallyExample17(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_17); } +static void LinearizeOptimallyExample18(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_18); } +static void LinearizeOptimallyExample19(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_19); } + +BENCHMARK(Linearize16TxWorstCase20Iters, benchmark::PriorityLevel::HIGH); +BENCHMARK(Linearize16TxWorstCase120Iters, benchmark::PriorityLevel::HIGH); +BENCHMARK(Linearize32TxWorstCase5000Iters, benchmark::PriorityLevel::HIGH); +BENCHMARK(Linearize32TxWorstCase15000Iters, benchmark::PriorityLevel::HIGH); +BENCHMARK(Linearize48TxWorstCase5000Iters, benchmark::PriorityLevel::HIGH); +BENCHMARK(Linearize48TxWorstCase15000Iters, benchmark::PriorityLevel::HIGH); +BENCHMARK(Linearize64TxWorstCase5000Iters, benchmark::PriorityLevel::HIGH); +BENCHMARK(Linearize64TxWorstCase15000Iters, benchmark::PriorityLevel::HIGH); +BENCHMARK(Linearize75TxWorstCase5000Iters, benchmark::PriorityLevel::HIGH); +BENCHMARK(Linearize75TxWorstCase15000Iters, benchmark::PriorityLevel::HIGH); +BENCHMARK(Linearize99TxWorstCase5000Iters, benchmark::PriorityLevel::HIGH); +BENCHMARK(Linearize99TxWorstCase15000Iters, benchmark::PriorityLevel::HIGH); BENCHMARK(LinearizeNoIters16TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); BENCHMARK(LinearizeNoIters32TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); @@ -276,3 +396,24 @@ BENCHMARK(MergeLinearizations48TxWorstCase, benchmark::PriorityLevel::HIGH); BENCHMARK(MergeLinearizations64TxWorstCase, benchmark::PriorityLevel::HIGH); BENCHMARK(MergeLinearizations75TxWorstCase, benchmark::PriorityLevel::HIGH); BENCHMARK(MergeLinearizations99TxWorstCase, benchmark::PriorityLevel::HIGH); + +BENCHMARK(LinearizeOptimallyExample00, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample01, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample02, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample03, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample04, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample05, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample06, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample07, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample08, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample09, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample10, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample11, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample12, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample13, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample14, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample15, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample16, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample17, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample18, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeOptimallyExample19, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 65da942ad7..a0799aea7a 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -192,10 +192,16 @@ static void SHA512(benchmark::Bench& bench) static void SipHash_32b(benchmark::Bench& bench) { - uint256 x; - uint64_t k1 = 0; + FastRandomContext rng{/*fDeterministic=*/true}; + auto k0{rng.rand64()}, k1{rng.rand64()}; + auto val{rng.rand256()}; + auto i{0U}; bench.run([&] { - *((uint64_t*)x.begin()) = SipHashUint256(0, ++k1, x); + ankerl::nanobench::doNotOptimizeAway(SipHashUint256(k0, k1, val)); + ++k0; + ++k1; + ++i; + val.data()[i % uint256::size()] ^= i & 0xFF; }); } diff --git a/src/bench/mempool_ephemeral_spends.cpp b/src/bench/mempool_ephemeral_spends.cpp new file mode 100644 index 0000000000..a8f2efecb9 --- /dev/null +++ b/src/bench/mempool_ephemeral_spends.cpp @@ -0,0 +1,87 @@ +// Copyright (c) 2011-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <consensus/amount.h> +#include <kernel/cs_main.h> +#include <policy/ephemeral_policy.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <sync.h> +#include <test/util/setup_common.h> +#include <test/util/txmempool.h> +#include <txmempool.h> +#include <util/check.h> + +#include <cstdint> +#include <memory> +#include <vector> + + +static void AddTx(const CTransactionRef& tx, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) +{ + int64_t nTime{0}; + unsigned int nHeight{1}; + uint64_t sequence{0}; + bool spendsCoinbase{false}; + unsigned int sigOpCost{4}; + uint64_t fee{0}; + LockPoints lp; + AddToMempool(pool, CTxMemPoolEntry( + tx, fee, nTime, nHeight, sequence, + spendsCoinbase, sigOpCost, lp)); +} + +static void MempoolCheckEphemeralSpends(benchmark::Bench& bench) +{ + const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + + int number_outputs{1000}; + if (bench.complexityN() > 1) { + number_outputs = static_cast<int>(bench.complexityN()); + } + + // Tx with many outputs + CMutableTransaction tx1; + tx1.vin.resize(1); + tx1.vout.resize(number_outputs); + for (size_t i = 0; i < tx1.vout.size(); i++) { + tx1.vout[i].scriptPubKey = CScript(); + // Each output progressively larger + tx1.vout[i].nValue = i * CENT; + } + + const auto& parent_txid = tx1.GetHash(); + + // Spends all outputs of tx1, other details don't matter + CMutableTransaction tx2; + tx2.vin.resize(tx1.vout.size()); + for (size_t i = 0; i < tx2.vin.size(); i++) { + tx2.vin[0].prevout.hash = parent_txid; + tx2.vin[0].prevout.n = i; + } + tx2.vout.resize(1); + + CTxMemPool& pool = *Assert(testing_setup->m_node.mempool); + LOCK2(cs_main, pool.cs); + // Create transaction references outside the "hot loop" + const CTransactionRef tx1_r{MakeTransactionRef(tx1)}; + const CTransactionRef tx2_r{MakeTransactionRef(tx2)}; + + AddTx(tx1_r, pool); + + uint32_t iteration{0}; + + TxValidationState dummy_state; + Txid dummy_txid; + + bench.run([&]() NO_THREAD_SAFETY_ANALYSIS { + + CheckEphemeralSpends({tx2_r}, /*dust_relay_rate=*/CFeeRate(iteration * COIN / 10), pool, dummy_state, dummy_txid); + iteration++; + }); +} + +BENCHMARK(MempoolCheckEphemeralSpends, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 84f873d183..aa2e8682e9 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -10,6 +10,7 @@ #include <script/script.h> #include <sync.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <util/check.h> @@ -26,7 +27,7 @@ static void AddTx(const CTransactionRef& tx, const CAmount& nFee, CTxMemPool& po bool spendsCoinbase = false; unsigned int sigOpCost = 4; LockPoints lp; - pool.addUnchecked(CTxMemPoolEntry( + AddToMempool(pool, CTxMemPoolEntry( tx, nFee, nTime, nHeight, sequence, spendsCoinbase, sigOpCost, lp)); } diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 67f689e4ea..fbac25db5f 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -10,6 +10,7 @@ #include <script/script.h> #include <sync.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <validation.h> @@ -28,7 +29,7 @@ static void AddTx(const CTransactionRef& tx, CTxMemPool& pool) EXCLUSIVE_LOCKS_R bool spendsCoinbase = false; unsigned int sigOpCost = 4; LockPoints lp; - pool.addUnchecked(CTxMemPoolEntry(tx, 1000, nTime, nHeight, sequence, spendsCoinbase, sigOpCost, lp)); + AddToMempool(pool, CTxMemPoolEntry(tx, 1000, nTime, nHeight, sequence, spendsCoinbase, sigOpCost, lp)); } struct Available { diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index 9b83c42693..adc3d18de0 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -87,6 +87,7 @@ static void PrevectorFillVectorDirect(benchmark::Bench& bench) { bench.run([&] { std::vector<prevector<28, T>> vec; + vec.reserve(260); for (size_t i = 0; i < 260; ++i) { vec.emplace_back(); } @@ -99,6 +100,7 @@ static void PrevectorFillVectorIndirect(benchmark::Bench& bench) { bench.run([&] { std::vector<prevector<28, T>> vec; + vec.reserve(260); for (size_t i = 0; i < 260; ++i) { // force allocation vec.emplace_back(29, T{}); diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index 6e8757bbd5..a61c6609bd 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -10,6 +10,7 @@ #include <script/script.h> #include <sync.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <univalue.h> #include <util/check.h> @@ -21,7 +22,7 @@ static void AddTx(const CTransactionRef& tx, const CAmount& fee, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) { LockPoints lp; - pool.addUnchecked(CTxMemPoolEntry(tx, fee, /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp)); + AddToMempool(pool, CTxMemPoolEntry(tx, fee, /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp)); } static void RpcMempool(benchmark::Bench& bench) diff --git a/src/bench/sign_transaction.cpp b/src/bench/sign_transaction.cpp index 3f0635711d..40219458f3 100644 --- a/src/bench/sign_transaction.cpp +++ b/src/bench/sign_transaction.cpp @@ -62,7 +62,7 @@ static void SignTransactionSingleInput(benchmark::Bench& bench, InputType input_ bench.minEpochIterations(100).run([&] { CMutableTransaction tx{unsigned_tx}; std::map<COutPoint, Coin> coins; - CScript prev_spk = prev_spks[(iter++) % prev_spks.size()]; + const CScript& prev_spk = prev_spks[(iter++) % prev_spks.size()]; coins[prevout] = Coin(CTxOut(10000, prev_spk), /*nHeightIn=*/100, /*fCoinBaseIn=*/false); std::map<int, bilingual_str> input_errors; bool complete = SignTransaction(tx, &keystore, coins, SIGHASH_ALL, input_errors); diff --git a/src/bench/streams_findbyte.cpp b/src/bench/streams_findbyte.cpp index 5098262e9a..004bf8ffc9 100644 --- a/src/bench/streams_findbyte.cpp +++ b/src/bench/streams_findbyte.cpp @@ -19,7 +19,7 @@ static void FindByte(benchmark::Bench& bench) uint8_t data[file_size] = {0}; data[file_size-1] = 1; file << data; - std::rewind(file.Get()); + file.seek(0, SEEK_SET); BufferedFile bf{file, /*nBufSize=*/file_size + 1, /*nRewindIn=*/file_size}; bench.run([&] { diff --git a/src/bench/wallet_create.cpp b/src/bench/wallet_create.cpp index 43b5b5c91e..3b916d7c39 100644 --- a/src/bench/wallet_create.cpp +++ b/src/bench/wallet_create.cpp @@ -3,7 +3,7 @@ // file COPYING or https://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <random.h> #include <support/allocators/secure.h> #include <test/util/setup_common.h> diff --git a/src/bench/wallet_ismine.cpp b/src/bench/wallet_ismine.cpp index 29e370ce29..5343814ab2 100644 --- a/src/bench/wallet_ismine.cpp +++ b/src/bench/wallet_ismine.cpp @@ -4,7 +4,7 @@ #include <addresstype.h> #include <bench/bench.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <key.h> #include <key_io.h> #include <script/descriptor.h> diff --git a/src/bench/wallet_loading.cpp b/src/bench/wallet_loading.cpp index 03459d37c1..5d92cfa0de 100644 --- a/src/bench/wallet_loading.cpp +++ b/src/bench/wallet_loading.cpp @@ -4,7 +4,7 @@ #include <addresstype.h> #include <bench/bench.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <consensus/amount.h> #include <outputtype.h> #include <primitives/transaction.h> diff --git a/src/bench/wallet_migration.cpp b/src/bench/wallet_migration.cpp new file mode 100644 index 0000000000..eff6c6b526 --- /dev/null +++ b/src/bench/wallet_migration.cpp @@ -0,0 +1,80 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include <bitcoin-build-config.h> // IWYU pragma: keep + +#include <bench/bench.h> +#include <interfaces/chain.h> +#include <node/context.h> +#include <test/util/mining.h> +#include <test/util/setup_common.h> +#include <wallet/test/util.h> +#include <wallet/context.h> +#include <wallet/receive.h> +#include <wallet/wallet.h> + +#include <optional> + +#if defined(USE_BDB) && defined(USE_SQLITE) // only enable benchmark when bdb and sqlite are enabled + +namespace wallet{ + +static void WalletMigration(benchmark::Bench& bench) +{ + const auto test_setup = MakeNoLogFileContext<TestingSetup>(); + + WalletContext context; + context.args = &test_setup->m_args; + context.chain = test_setup->m_node.chain.get(); + + // Number of imported watch only addresses + int NUM_WATCH_ONLY_ADDR = 20; + + // Setup legacy wallet + DatabaseOptions options; + options.use_unsafe_sync = true; + options.verify = false; + DatabaseStatus status; + bilingual_str error; + auto database = MakeWalletDatabase(fs::PathToString(test_setup->m_path_root / "legacy"), options, status, error); + uint64_t create_flags = 0; + auto wallet = TestLoadWallet(std::move(database), context, create_flags); + + // Add watch-only addresses + std::vector<CScript> scripts_watch_only; + for (int w = 0; w < NUM_WATCH_ONLY_ADDR; ++w) { + CKey key = GenerateRandomKey(); + LOCK(wallet->cs_wallet); + const CScript& script = scripts_watch_only.emplace_back(GetScriptForDestination(GetDestinationForKey(key.GetPubKey(), OutputType::LEGACY))); + bool res = wallet->ImportScriptPubKeys(strprintf("watch_%d", w), {script}, + /*have_solving_data=*/false, /*apply_label=*/true, /*timestamp=*/1); + assert(res); + } + + // Generate transactions and local addresses + for (int j = 0; j < 400; ++j) { + CMutableTransaction mtx; + mtx.vout.emplace_back(COIN, GetScriptForDestination(*Assert(wallet->GetNewDestination(OutputType::BECH32, strprintf("bench_%d", j))))); + mtx.vout.emplace_back(COIN, GetScriptForDestination(*Assert(wallet->GetNewDestination(OutputType::LEGACY, strprintf("legacy_%d", j))))); + mtx.vout.emplace_back(COIN, scripts_watch_only.at(j % NUM_WATCH_ONLY_ADDR)); + mtx.vin.resize(2); + wallet->AddToWallet(MakeTransactionRef(mtx), TxStateInactive{}, /*update_wtx=*/nullptr, /*fFlushOnClose=*/false, /*rescanning_old_block=*/true); + } + + // Unload so the migration process loads it + TestUnloadWallet(std::move(wallet)); + + bench.epochs(/*numEpochs=*/1).run([&] { + util::Result<MigrationResult> res = MigrateLegacyToDescriptor(fs::PathToString(test_setup->m_path_root / "legacy"), "", context); + assert(res); + assert(res->wallet); + assert(res->watchonly_wallet); + }); +} + +BENCHMARK(WalletMigration, benchmark::PriorityLevel::LOW); + +} // namespace wallet + +#endif // end USE_SQLITE && USE_BDB diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index ebe013b638..bab09f13d9 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -180,27 +180,6 @@ int main(int argc, char* argv[]) break; } - if (block.vtx.empty() || !block.vtx[0]->IsCoinBase()) { - std::cerr << "Block does not start with a coinbase" << std::endl; - break; - } - - uint256 hash = block.GetHash(); - { - LOCK(cs_main); - const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); - if (pindex) { - if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) { - std::cerr << "duplicate" << std::endl; - break; - } - if (pindex->nStatus & BLOCK_FAILED_MASK) { - std::cerr << "duplicate-invalid" << std::endl; - break; - } - } - } - { LOCK(cs_main); const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock); @@ -253,9 +232,6 @@ int main(int argc, char* argv[]) case BlockValidationResult::BLOCK_CONSENSUS: std::cerr << "invalid by consensus rules (excluding any below reasons)" << std::endl; break; - case BlockValidationResult::BLOCK_RECENT_CONSENSUS_CHANGE: - std::cerr << "Invalid by a change to consensus rules more recent than SegWit." << std::endl; - break; case BlockValidationResult::BLOCK_CACHED_INVALID: std::cerr << "this block was cached as being invalid and we didn't store the reason why" << std::endl; break; @@ -283,8 +259,6 @@ int main(int argc, char* argv[]) epilogue: // Without this precise shutdown sequence, there will be a lot of nullptr // dereferencing and UB. - if (chainman.m_thread_load.joinable()) chainman.m_thread_load.join(); - validation_signals.FlushBackgroundCallbacks(); { LOCK(cs_main); diff --git a/src/bitcoin-cli-res.rc b/src/bitcoin-cli-res.rc index d9e5dcf7fd..e0d4ee5f3c 100644 --- a/src/bitcoin-cli-res.rc +++ b/src/bitcoin-cli-res.rc @@ -15,14 +15,14 @@ BEGIN BLOCK "040904E4" // U.S. English - multilingual (hex) BEGIN VALUE "CompanyName", "Bitcoin" - VALUE "FileDescription", "bitcoin-cli (JSON-RPC client for " PACKAGE_NAME ")" - VALUE "FileVersion", PACKAGE_VERSION + VALUE "FileDescription", "bitcoin-cli (JSON-RPC client for " CLIENT_NAME ")" + VALUE "FileVersion", CLIENT_VERSION_STRING VALUE "InternalName", "bitcoin-cli" VALUE "LegalCopyright", COPYRIGHT_STR VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." VALUE "OriginalFilename", "bitcoin-cli.exe" VALUE "ProductName", "bitcoin-cli" - VALUE "ProductVersion", PACKAGE_VERSION + VALUE "ProductVersion", CLIENT_VERSION_STRING END END diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 7fcb440931..5c5965245b 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chainparamsbase.h> #include <clientversion.h> @@ -57,6 +57,7 @@ static const int DEFAULT_HTTP_CLIENT_TIMEOUT=900; static constexpr int DEFAULT_WAIT_CLIENT_TIMEOUT = 0; static const bool DEFAULT_NAMED=false; static const int CONTINUE_EXECUTION=-1; +static constexpr uint8_t NETINFO_MAX_LEVEL{4}; static constexpr int8_t UNKNOWN_NETWORK{-1}; // See GetNetworkName() in netbase.cpp static constexpr std::array NETWORKS{"not_publicly_routable", "ipv4", "ipv6", "onion", "i2p", "cjdns", "internal"}; @@ -81,7 +82,7 @@ static void SetupCliArgs(ArgsManager& argsman) argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); argsman.AddArg("-generate", strprintf("Generate blocks, equivalent to RPC getnewaddress followed by RPC generatetoaddress. Optional positional integer " "arguments are number of blocks to generate (default: %s) and maximum iterations to try (default: %s), equivalent to " @@ -90,10 +91,10 @@ static void SetupCliArgs(ArgsManager& argsman) ArgsManager::ALLOW_ANY, OptionsCategory::CLI_COMMANDS); argsman.AddArg("-addrinfo", "Get the number of addresses known to the node, per network and total, after filtering for quality and recency. The total number of addresses known to the node may be higher.", ArgsManager::ALLOW_ANY, OptionsCategory::CLI_COMMANDS); argsman.AddArg("-getinfo", "Get general information from the remote server. Note that unlike server-side RPC calls, the output of -getinfo is the result of multiple non-atomic requests. Some entries in the output may represent results from different states (e.g. wallet balance may be as of a different block from the chain state reported)", ArgsManager::ALLOW_ANY, OptionsCategory::CLI_COMMANDS); - argsman.AddArg("-netinfo", "Get network peer connection information from the remote server. An optional integer argument from 0 to 4 can be passed for different peers listings (default: 0). Pass \"help\" for detailed help documentation.", ArgsManager::ALLOW_ANY, OptionsCategory::CLI_COMMANDS); + argsman.AddArg("-netinfo", strprintf("Get network peer connection information from the remote server. An optional argument from 0 to %d can be passed for different peers listings (default: 0). If a non-zero value is passed, an additional \"outonly\" (or \"o\") argument can be passed to see outbound peers only. Pass \"help\" (or \"h\") for detailed help documentation.", NETINFO_MAX_LEVEL), ArgsManager::ALLOW_ANY, OptionsCategory::CLI_COMMANDS); SetupChainParamsBaseOptions(argsman); - argsman.AddArg("-color=<when>", strprintf("Color setting for CLI output (default: %s). Valid values: always, auto (add color codes when standard output is connected to a terminal and OS is not WIN32), never.", DEFAULT_COLOR_SETTING), ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); + argsman.AddArg("-color=<when>", strprintf("Color setting for CLI output (default: %s). Valid values: always, auto (add color codes when standard output is connected to a terminal and OS is not WIN32), never. Only applies to the output of -getinfo.", DEFAULT_COLOR_SETTING), ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); argsman.AddArg("-named", strprintf("Pass named instead of positional arguments (default: %s)", DEFAULT_NAMED), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-rpcclienttimeout=<n>", strprintf("Timeout in seconds during HTTP requests, or 0 for no timeout. (default: %d)", DEFAULT_HTTP_CLIENT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-rpcconnect=<ip>", strprintf("Send commands to node running on <ip> (default: %s)", DEFAULT_RPCCONNECT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -144,17 +145,23 @@ static int AppInitRPC(int argc, char* argv[]) tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); return EXIT_FAILURE; } - if (argc < 2 || HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { - std::string strUsage = PACKAGE_NAME " RPC client version " + FormatFullVersion() + "\n"; + if (argc < 2 || HelpRequested(gArgs) || gArgs.GetBoolArg("-version", false)) { + std::string strUsage = CLIENT_NAME " RPC client version " + FormatFullVersion() + "\n"; - if (gArgs.IsArgSet("-version")) { + if (gArgs.GetBoolArg("-version", false)) { strUsage += FormatParagraph(LicenseInfo()); } else { strUsage += "\n" - "Usage: bitcoin-cli [options] <command> [params] Send command to " PACKAGE_NAME "\n" - "or: bitcoin-cli [options] -named <command> [name=value]... Send command to " PACKAGE_NAME " (with named arguments)\n" - "or: bitcoin-cli [options] help List commands\n" - "or: bitcoin-cli [options] help <command> Get help for a command\n"; + "The bitcoin-cli utility provides a command line interface to interact with a " CLIENT_NAME " RPC server.\n" + "\nIt can be used to query network information, manage wallets, create or broadcast transactions, and control the " CLIENT_NAME " server.\n" + "\nUse the \"help\" command to list all commands. Use \"help <command>\" to show help for that command.\n" + "The -named option allows you to specify parameters using the key=value format, eliminating the need to pass unused positional parameters.\n" + "\n" + "Usage: bitcoin-cli [options] <command> [params]\n" + "or: bitcoin-cli [options] -named <command> [name=value]...\n" + "or: bitcoin-cli [options] help\n" + "or: bitcoin-cli [options] help <command>\n" + "\n"; strUsage += "\n" + gArgs.GetHelpMessage(); } @@ -379,7 +386,6 @@ public: class NetinfoRequestHandler : public BaseRequestHandler { private: - static constexpr uint8_t MAX_DETAIL_LEVEL{4}; std::array<std::array<uint16_t, NETWORKS.size() + 1>, 3> m_counts{{{}}}; //!< Peer counts by (in/out/total, networks/total) uint8_t m_block_relay_peers_count{0}; uint8_t m_manual_peers_count{0}; @@ -394,18 +400,21 @@ private: bool DetailsRequested() const { return m_details_level > 0 && m_details_level < 5; } bool IsAddressSelected() const { return m_details_level == 2 || m_details_level == 4; } bool IsVersionSelected() const { return m_details_level == 3 || m_details_level == 4; } + bool m_outbound_only_selected{false}; bool m_is_asmap_on{false}; size_t m_max_addr_length{0}; size_t m_max_addr_processed_length{5}; size_t m_max_addr_rate_limited_length{6}; size_t m_max_age_length{5}; size_t m_max_id_length{2}; + size_t m_max_services_length{6}; struct Peer { std::string addr; std::string sub_version; std::string conn_type; std::string network; std::string age; + std::string services; std::string transport_protocol_type; double min_ping; double ping; @@ -456,6 +465,15 @@ private: if (conn_type == "addr-fetch") return "addr"; return ""; } + std::string FormatServices(const UniValue& services) + { + std::string str; + for (size_t i = 0; i < services.size(); ++i) { + const std::string s{services[i].get_str()}; + str += s == "NETWORK_LIMITED" ? 'l' : s == "P2P_V2" ? '2' : ToLower(s[0]); + } + return str; + } public: static constexpr int ID_PEERINFO = 0; @@ -466,9 +484,18 @@ public: if (!args.empty()) { uint8_t n{0}; if (ParseUInt8(args.at(0), &n)) { - m_details_level = std::min(n, MAX_DETAIL_LEVEL); + m_details_level = std::min(n, NETINFO_MAX_LEVEL); } else { - throw std::runtime_error(strprintf("invalid -netinfo argument: %s\nFor more information, run: bitcoin-cli -netinfo help", args.at(0))); + throw std::runtime_error(strprintf("invalid -netinfo level argument: %s\nFor more information, run: bitcoin-cli -netinfo help", args.at(0))); + } + if (args.size() > 1) { + if (std::string_view s{args.at(1)}; n && (s == "o" || s == "outonly")) { + m_outbound_only_selected = true; + } else if (n) { + throw std::runtime_error(strprintf("invalid -netinfo outonly argument: %s\nFor more information, run: bitcoin-cli -netinfo help", s)); + } else { + throw std::runtime_error(strprintf("invalid -netinfo outonly argument: %s\nThe outonly argument is only valid for a level greater than 0 (the first argument). For more information, run: bitcoin-cli -netinfo help", s)); + } } } UniValue result(UniValue::VARR); @@ -503,6 +530,7 @@ public: ++m_counts.at(2).at(NETWORKS.size()); // total overall if (conn_type == "block-relay-only") ++m_block_relay_peers_count; if (conn_type == "manual") ++m_manual_peers_count; + if (m_outbound_only_selected && !is_outbound) continue; if (DetailsRequested()) { // Push data for this peer to the peers vector. const int peer_id{peer["id"].getInt<int>()}; @@ -519,28 +547,31 @@ public: const double ping{peer["pingtime"].isNull() ? -1 : peer["pingtime"].get_real()}; const std::string addr{peer["addr"].get_str()}; const std::string age{conn_time == 0 ? "" : ToString((time_now - conn_time) / 60)}; + const std::string services{FormatServices(peer["servicesnames"])}; const std::string sub_version{peer["subver"].get_str()}; const std::string transport{peer["transport_protocol_type"].isNull() ? "v1" : peer["transport_protocol_type"].get_str()}; const bool is_addr_relay_enabled{peer["addr_relay_enabled"].isNull() ? false : peer["addr_relay_enabled"].get_bool()}; const bool is_bip152_hb_from{peer["bip152_hb_from"].get_bool()}; const bool is_bip152_hb_to{peer["bip152_hb_to"].get_bool()}; - m_peers.push_back({addr, sub_version, conn_type, NETWORK_SHORT_NAMES[network_id], age, transport, min_ping, ping, addr_processed, addr_rate_limited, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_addr_relay_enabled, is_bip152_hb_from, is_bip152_hb_to, is_outbound, is_tx_relay}); + m_peers.push_back({addr, sub_version, conn_type, NETWORK_SHORT_NAMES[network_id], age, services, transport, min_ping, ping, addr_processed, addr_rate_limited, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_addr_relay_enabled, is_bip152_hb_from, is_bip152_hb_to, is_outbound, is_tx_relay}); m_max_addr_length = std::max(addr.length() + 1, m_max_addr_length); m_max_addr_processed_length = std::max(ToString(addr_processed).length(), m_max_addr_processed_length); m_max_addr_rate_limited_length = std::max(ToString(addr_rate_limited).length(), m_max_addr_rate_limited_length); m_max_age_length = std::max(age.length(), m_max_age_length); m_max_id_length = std::max(ToString(peer_id).length(), m_max_id_length); + m_max_services_length = std::max(services.length(), m_max_services_length); m_is_asmap_on |= (mapped_as != 0); } } // Generate report header. - std::string result{strprintf("%s client %s%s - server %i%s\n\n", PACKAGE_NAME, FormatFullVersion(), ChainToString(), networkinfo["protocolversion"].getInt<int>(), networkinfo["subversion"].get_str())}; + std::string result{strprintf("%s client %s%s - server %i%s\n\n", CLIENT_NAME, FormatFullVersion(), ChainToString(), networkinfo["protocolversion"].getInt<int>(), networkinfo["subversion"].get_str())}; // Report detailed peer connections list sorted by direction and minimum ping time. if (DetailsRequested() && !m_peers.empty()) { std::sort(m_peers.begin(), m_peers.end()); - result += strprintf("<-> type net v mping ping send recv txn blk hb %*s%*s%*s ", + result += strprintf("<-> type net %*s v mping ping send recv txn blk hb %*s%*s%*s ", + m_max_services_length, "serv", m_max_addr_processed_length, "addrp", m_max_addr_rate_limited_length, "addrl", m_max_age_length, "age"); @@ -549,10 +580,12 @@ public: for (const Peer& peer : m_peers) { std::string version{ToString(peer.version) + peer.sub_version}; result += strprintf( - "%3s %6s %5s %2s%7s%7s%5s%5s%5s%5s %2s %*s%*s%*s%*i %*s %-*s%s\n", + "%3s %6s %5s %*s %2s%7s%7s%5s%5s%5s%5s %2s %*s%*s%*s%*i %*s %-*s%s\n", peer.is_outbound ? "out" : "in", ConnectionTypeForNetinfo(peer.conn_type), peer.network, + m_max_services_length, // variable spacing + peer.services, (peer.transport_protocol_type.size() == 2 && peer.transport_protocol_type[0] == 'v') ? peer.transport_protocol_type[1] : ' ', PingTimeToString(peer.min_ping), PingTimeToString(peer.ping), @@ -575,7 +608,7 @@ public: IsAddressSelected() ? peer.addr : "", IsVersionSelected() && version != "0" ? version : ""); } - result += strprintf(" ms ms sec sec min min %*s\n\n", m_max_age_length, "min"); + result += strprintf(" %*s ms ms sec sec min min %*s\n\n", m_max_services_length, "", m_max_age_length, "min"); } // Report peer connection totals by type. @@ -632,25 +665,28 @@ public: } const std::string m_help_doc{ - "-netinfo level|\"help\" \n\n" + "-netinfo (level [outonly]) | help\n\n" "Returns a network peer connections dashboard with information from the remote server.\n" "This human-readable interface will change regularly and is not intended to be a stable API.\n" "Under the hood, -netinfo fetches the data by calling getpeerinfo and getnetworkinfo.\n" - + strprintf("An optional integer argument from 0 to %d can be passed for different peers listings; %d to 255 are parsed as %d.\n", MAX_DETAIL_LEVEL, MAX_DETAIL_LEVEL, MAX_DETAIL_LEVEL) + - "Pass \"help\" to see this detailed help documentation.\n" - "If more than one argument is passed, only the first one is read and parsed.\n" - "Suggestion: use with the Linux watch(1) command for a live dashboard; see example below.\n\n" + + strprintf("An optional argument from 0 to %d can be passed for different peers listings; values above %d up to 255 are parsed as %d.\n", NETINFO_MAX_LEVEL, NETINFO_MAX_LEVEL, NETINFO_MAX_LEVEL) + + "If that argument is passed, an optional additional \"outonly\" argument may be passed to obtain the listing with outbound peers only.\n" + "Pass \"help\" or \"h\" to see this detailed help documentation.\n" + "If more than two arguments are passed, only the first two are read and parsed.\n" + "Suggestion: use -netinfo with the Linux watch(1) command for a live dashboard; see example below.\n\n" "Arguments:\n" - + strprintf("1. level (integer 0-%d, optional) Specify the info level of the peers dashboard (default 0):\n", MAX_DETAIL_LEVEL) + + + strprintf("1. level (integer 0-%d, optional) Specify the info level of the peers dashboard (default 0):\n", NETINFO_MAX_LEVEL) + " 0 - Peer counts for each reachable network as well as for block relay peers\n" " and manual peers, and the list of local addresses and ports\n" " 1 - Like 0 but preceded by a peers listing (without address and version columns)\n" " 2 - Like 1 but with an address column\n" " 3 - Like 1 but with a version column\n" " 4 - Like 1 but with both address and version columns\n" - "2. help (string \"help\", optional) Print this help documentation instead of the dashboard.\n\n" + "2. outonly (\"outonly\" or \"o\", optional) Return the peers listing with outbound peers only, i.e. to save screen space\n" + " when a node has many inbound peers. Only valid if a level is passed.\n\n" + "help (\"help\" or \"h\", optional) Print this help documentation instead of the dashboard.\n\n" "Result:\n\n" - + strprintf("* The peers listing in levels 1-%d displays all of the peers sorted by direction and minimum ping time:\n\n", MAX_DETAIL_LEVEL) + + + strprintf("* The peers listing in levels 1-%d displays all of the peers sorted by direction and minimum ping time:\n\n", NETINFO_MAX_LEVEL) + " Column Description\n" " ------ -----------\n" " <-> Direction\n" @@ -663,22 +699,30 @@ public: " \"feeler\" - short-lived connection for testing addresses\n" " \"addr\" - address fetch; short-lived connection for requesting addresses\n" " net Network the peer connected through (\"ipv4\", \"ipv6\", \"onion\", \"i2p\", \"cjdns\", or \"npr\" (not publicly routable))\n" + " serv Services offered by the peer\n" + " \"n\" - NETWORK: peer can serve the full block chain\n" + " \"b\" - BLOOM: peer can handle bloom-filtered connections (see BIP 111)\n" + " \"w\" - WITNESS: peer can be asked for blocks and transactions with witness data (SegWit)\n" + " \"c\" - COMPACT_FILTERS: peer can handle basic block filter requests (see BIPs 157 and 158)\n" + " \"l\" - NETWORK_LIMITED: peer limited to serving only the last 288 blocks (~2 days)\n" + " \"2\" - P2P_V2: peer supports version 2 P2P transport protocol, as defined in BIP 324\n" + " \"u\" - UNKNOWN: unrecognized bit flag\n" " v Version of transport protocol used for the connection\n" " mping Minimum observed ping time, in milliseconds (ms)\n" " ping Last observed ping time, in milliseconds (ms)\n" " send Time since last message sent to the peer, in seconds\n" " recv Time since last message received from the peer, in seconds\n" " txn Time since last novel transaction received from the peer and accepted into our mempool, in minutes\n" - " \"*\" - we do not relay transactions to this peer (relaytxes is false)\n" + " \"*\" - we do not relay transactions to this peer (getpeerinfo \"relaytxes\" is false)\n" " blk Time since last novel block passing initial validity checks received from the peer, in minutes\n" " hb High-bandwidth BIP152 compact block relay\n" " \".\" (to) - we selected the peer as a high-bandwidth peer\n" " \"*\" (from) - the peer selected us as a high-bandwidth peer\n" " addrp Total number of addresses processed, excluding those dropped due to rate limiting\n" - " \".\" - we do not relay addresses to this peer (addr_relay_enabled is false)\n" + " \".\" - we do not relay addresses to this peer (getpeerinfo \"addr_relay_enabled\" is false)\n" " addrl Total number of addresses dropped due to rate limiting\n" " age Duration of connection to the peer, in minutes\n" - " asmap Mapped AS (Autonomous System) number in the BGP route to the peer, used for diversifying\n" + " asmap Mapped AS (Autonomous System) number at the end of the BGP route to the peer, used for diversifying\n" " peer selection (only displayed if the -asmap config option is set)\n" " id Peer index, in increasing order of peer connections since node startup\n" " address IP address and port of the peer\n" @@ -692,9 +736,11 @@ public: "The same, preceded by a peers listing without address and version columns\n" "> bitcoin-cli -netinfo 1\n\n" "Full dashboard\n" - + strprintf("> bitcoin-cli -netinfo %d\n\n", MAX_DETAIL_LEVEL) + + + strprintf("> bitcoin-cli -netinfo %d\n\n", NETINFO_MAX_LEVEL) + + "Full dashboard, but with outbound peers only\n" + + strprintf("> bitcoin-cli -netinfo %d outonly\n\n", NETINFO_MAX_LEVEL) + "Full live dashboard, adjust --interval or --no-title as needed (Linux)\n" - + strprintf("> watch --interval 1 --no-title bitcoin-cli -netinfo %d\n\n", MAX_DETAIL_LEVEL) + + + strprintf("> watch --interval 1 --no-title bitcoin-cli -netinfo %d\n\n", NETINFO_MAX_LEVEL) + "See this help\n" "> bitcoin-cli -netinfo help\n"}; }; @@ -950,7 +996,8 @@ static void ParseError(const UniValue& error, std::string& strPrint, int& nRet) strPrint += ("error message:\n" + err_msg.get_str()); } if (err_code.isNum() && err_code.getInt<int>() == RPC_WALLET_NOT_SPECIFIED) { - strPrint += "\nTry adding \"-rpcwallet=<filename>\" option to bitcoin-cli command line."; + strPrint += " Or for the CLI, specify the \"-rpcwallet=<walletname>\" option before the command"; + strPrint += " (run \"bitcoin-cli -h\" for help or \"bitcoin-cli listwallets\" to see which wallets are currently loaded)."; } } else { strPrint = "error: " + error.write(); @@ -1218,7 +1265,7 @@ static int CommandLineRPC(int argc, char *argv[]) if (gArgs.IsArgSet("-getinfo")) { rh.reset(new GetinfoRequestHandler()); } else if (gArgs.GetBoolArg("-netinfo", false)) { - if (!args.empty() && args.at(0) == "help") { + if (!args.empty() && (args.at(0) == "h" || args.at(0) == "help")) { tfm::format(std::cout, "%s\n", NetinfoRequestHandler().m_help_doc); return 0; } diff --git a/src/bitcoin-tx-res.rc b/src/bitcoin-tx-res.rc index 46e4fc9274..d5d7e3b7ca 100644 --- a/src/bitcoin-tx-res.rc +++ b/src/bitcoin-tx-res.rc @@ -16,13 +16,13 @@ BEGIN BEGIN VALUE "CompanyName", "Bitcoin" VALUE "FileDescription", "bitcoin-tx (CLI Bitcoin transaction editor utility)" - VALUE "FileVersion", PACKAGE_VERSION + VALUE "FileVersion", CLIENT_VERSION_STRING VALUE "InternalName", "bitcoin-tx" VALUE "LegalCopyright", COPYRIGHT_STR VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." VALUE "OriginalFilename", "bitcoin-tx.exe" VALUE "ProductName", "bitcoin-tx" - VALUE "ProductVersion", PACKAGE_VERSION + VALUE "ProductVersion", CLIENT_VERSION_STRING END END diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 89c03c1647..a627c24e86 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chainparamsbase.h> #include <clientversion.h> @@ -105,16 +105,19 @@ static int AppInitRawTx(int argc, char* argv[]) fCreateBlank = gArgs.GetBoolArg("-create", false); - if (argc < 2 || HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { + if (argc < 2 || HelpRequested(gArgs) || gArgs.GetBoolArg("-version", false)) { // First part of help message is specific to this utility - std::string strUsage = PACKAGE_NAME " bitcoin-tx utility version " + FormatFullVersion() + "\n"; + std::string strUsage = CLIENT_NAME " bitcoin-tx utility version " + FormatFullVersion() + "\n"; - if (gArgs.IsArgSet("-version")) { + if (gArgs.GetBoolArg("-version", false)) { strUsage += FormatParagraph(LicenseInfo()); } else { strUsage += "\n" - "Usage: bitcoin-tx [options] <hex-tx> [commands] Update hex-encoded bitcoin transaction\n" - "or: bitcoin-tx [options] -create [commands] Create hex-encoded bitcoin transaction\n" + "The bitcoin-tx tool is used for creating and modifying bitcoin transactions.\n\n" + "bitcoin-tx can be used with \"<hex-tx> [commands]\" to update a hex-encoded bitcoin transaction, or with \"-create [commands]\" to create a hex-encoded bitcoin transaction.\n" + "\n" + "Usage: bitcoin-tx [options] <hex-tx> [commands]\n" + "or: bitcoin-tx [options] -create [commands]\n" "\n"; strUsage += gArgs.GetHelpMessage(); } @@ -301,7 +304,7 @@ static void MutateTxAddOutAddr(CMutableTransaction& tx, const std::string& strIn CAmount value = ExtractAndValidateValue(vStrInputParts[0]); // extract and validate ADDRESS - std::string strAddr = vStrInputParts[1]; + const std::string& strAddr = vStrInputParts[1]; CTxDestination destination = DecodeDestination(strAddr); if (!IsValidDestination(destination)) { throw std::runtime_error("invalid TX output address"); @@ -334,7 +337,7 @@ static void MutateTxAddOutPubKey(CMutableTransaction& tx, const std::string& str bool bSegWit = false; bool bScriptHash = false; if (vStrInputParts.size() == 3) { - std::string flags = vStrInputParts[2]; + const std::string& flags = vStrInputParts[2]; bSegWit = (flags.find('W') != std::string::npos); bScriptHash = (flags.find('S') != std::string::npos); } @@ -395,7 +398,7 @@ static void MutateTxAddOutMultiSig(CMutableTransaction& tx, const std::string& s bool bSegWit = false; bool bScriptHash = false; if (vStrInputParts.size() == numkeys + 4) { - std::string flags = vStrInputParts.back(); + const std::string& flags = vStrInputParts.back(); bSegWit = (flags.find('W') != std::string::npos); bScriptHash = (flags.find('S') != std::string::npos); } @@ -470,14 +473,14 @@ static void MutateTxAddOutScript(CMutableTransaction& tx, const std::string& str CAmount value = ExtractAndValidateValue(vStrInputParts[0]); // extract and validate script - std::string strScript = vStrInputParts[1]; + const std::string& strScript = vStrInputParts[1]; CScript scriptPubKey = ParseScript(strScript); // Extract FLAGS bool bSegWit = false; bool bScriptHash = false; if (vStrInputParts.size() == 3) { - std::string flags = vStrInputParts.back(); + const std::string& flags = vStrInputParts.back(); bSegWit = (flags.find('W') != std::string::npos); bScriptHash = (flags.find('S') != std::string::npos); } diff --git a/src/bitcoin-util-res.rc b/src/bitcoin-util-res.rc index 0de8c5befa..e121d17f23 100644 --- a/src/bitcoin-util-res.rc +++ b/src/bitcoin-util-res.rc @@ -16,13 +16,13 @@ BEGIN BEGIN VALUE "CompanyName", "Bitcoin" VALUE "FileDescription", "bitcoin-util (CLI Bitcoin utility)" - VALUE "FileVersion", PACKAGE_VERSION + VALUE "FileVersion", CLIENT_VERSION_STRING VALUE "InternalName", "bitcoin-util" VALUE "LegalCopyright", COPYRIGHT_STR VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." VALUE "OriginalFilename", "bitcoin-util.exe" VALUE "ProductName", "bitcoin-util" - VALUE "ProductVersion", PACKAGE_VERSION + VALUE "ProductVersion", CLIENT_VERSION_STRING END END diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp index c8f5bc5026..ff2e4fb800 100644 --- a/src/bitcoin-util.cpp +++ b/src/bitcoin-util.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <arith_uint256.h> #include <chain.h> @@ -50,15 +50,18 @@ static int AppInitUtil(ArgsManager& args, int argc, char* argv[]) return EXIT_FAILURE; } - if (HelpRequested(args) || args.IsArgSet("-version")) { + if (HelpRequested(args) || args.GetBoolArg("-version", false)) { // First part of help message is specific to this utility - std::string strUsage = PACKAGE_NAME " bitcoin-util utility version " + FormatFullVersion() + "\n"; + std::string strUsage = CLIENT_NAME " bitcoin-util utility version " + FormatFullVersion() + "\n"; - if (args.IsArgSet("-version")) { + if (args.GetBoolArg("-version", false)) { strUsage += FormatParagraph(LicenseInfo()); } else { strUsage += "\n" - "Usage: bitcoin-util [options] [commands] Do stuff\n"; + "The bitcoin-util tool provides bitcoin related functionality that does not rely on the ability to access a running node. Available [commands] are listed below.\n" + "\n" + "Usage: bitcoin-util [options] [command]\n" + "or: bitcoin-util [options] grind <hex-block-header>\n"; strUsage += "\n" + args.GetHelpMessage(); } diff --git a/src/bitcoin-wallet-res.rc b/src/bitcoin-wallet-res.rc index d86ffbd9f1..4c7895f48d 100644 --- a/src/bitcoin-wallet-res.rc +++ b/src/bitcoin-wallet-res.rc @@ -15,14 +15,14 @@ BEGIN BLOCK "040904E4" // U.S. English - multilingual (hex) BEGIN VALUE "CompanyName", "Bitcoin" - VALUE "FileDescription", "bitcoin-wallet (CLI tool for " PACKAGE_NAME " wallets)" - VALUE "FileVersion", PACKAGE_VERSION + VALUE "FileDescription", "bitcoin-wallet (CLI tool for " CLIENT_NAME " wallets)" + VALUE "FileVersion", CLIENT_VERSION_STRING VALUE "InternalName", "bitcoin-wallet" VALUE "LegalCopyright", COPYRIGHT_STR VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." VALUE "OriginalFilename", "bitcoin-wallet.exe" VALUE "ProductName", "bitcoin-wallet" - VALUE "ProductVersion", PACKAGE_VERSION + VALUE "ProductVersion", CLIENT_VERSION_STRING END END diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 7d030abe97..033cf7a0f3 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -1,8 +1,8 @@ -// Copyright (c) 2016-2022 The Bitcoin Core developers +// Copyright (c) 2016-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chainparams.h> #include <chainparamsbase.h> @@ -34,7 +34,7 @@ static void SetupWalletToolArgs(ArgsManager& argsman) SetupChainParamsBaseOptions(argsman); argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); argsman.AddArg("-wallet=<wallet-name>", "Specify wallet name", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::OPTIONS); argsman.AddArg("-dumpfile=<file name>", "When used with 'dump', writes out the records to this file. When used with 'createfromdump', loads the records into a new wallet.", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); argsman.AddArg("-debug=<category>", "Output debugging information (default: 0).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); @@ -60,18 +60,19 @@ static std::optional<int> WalletAppInit(ArgsManager& args, int argc, char* argv[ return EXIT_FAILURE; } const bool missing_args{argc < 2}; - if (missing_args || HelpRequested(args) || args.IsArgSet("-version")) { - std::string strUsage = strprintf("%s bitcoin-wallet version", PACKAGE_NAME) + " " + FormatFullVersion() + "\n"; + if (missing_args || HelpRequested(args) || args.GetBoolArg("-version", false)) { + std::string strUsage = strprintf("%s bitcoin-wallet utility version", CLIENT_NAME) + " " + FormatFullVersion() + "\n"; - if (args.IsArgSet("-version")) { + if (args.GetBoolArg("-version", false)) { strUsage += FormatParagraph(LicenseInfo()); } else { strUsage += "\n" - "bitcoin-wallet is an offline tool for creating and interacting with " PACKAGE_NAME " wallet files.\n" - "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" - "To change the target wallet, use the -datadir, -wallet and -regtest/-signet/-testnet/-testnet4 arguments.\n\n" - "Usage:\n" - " bitcoin-wallet [options] <command>\n"; + "bitcoin-wallet is an offline tool for creating and interacting with " CLIENT_NAME " wallet files.\n\n" + "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n\n" + "To change the target wallet, use the -datadir, -wallet and (test)chain selection arguments.\n" + "\n" + "Usage: bitcoin-wallet [options] <command>\n" + "\n"; strUsage += "\n" + args.GetHelpMessage(); } tfm::format(std::cout, "%s", strUsage); diff --git a/src/bitcoind-res.rc b/src/bitcoind-res.rc index 353761dfa7..4d689d7e16 100644 --- a/src/bitcoind-res.rc +++ b/src/bitcoind-res.rc @@ -16,13 +16,13 @@ BEGIN BEGIN VALUE "CompanyName", "Bitcoin" VALUE "FileDescription", "bitcoind (Bitcoin node with a JSON-RPC server)" - VALUE "FileVersion", PACKAGE_VERSION + VALUE "FileVersion", CLIENT_VERSION_STRING VALUE "InternalName", "bitcoind" VALUE "LegalCopyright", COPYRIGHT_STR VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." VALUE "OriginalFilename", "bitcoind.exe" VALUE "ProductName", "bitcoind" - VALUE "ProductVersion", PACKAGE_VERSION + VALUE "ProductVersion", CLIENT_VERSION_STRING END END diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 1e27924dfd..5d5f06127d 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chainparams.h> #include <clientversion.h> @@ -135,13 +135,19 @@ static bool ParseArgs(NodeContext& node, int argc, char* argv[]) static bool ProcessInitCommands(ArgsManager& args) { // Process help and version before taking care about datadir - if (HelpRequested(args) || args.IsArgSet("-version")) { - std::string strUsage = PACKAGE_NAME " version " + FormatFullVersion() + "\n"; + if (HelpRequested(args) || args.GetBoolArg("-version", false)) { + std::string strUsage = CLIENT_NAME " daemon version " + FormatFullVersion() + "\n"; - if (args.IsArgSet("-version")) { + if (args.GetBoolArg("-version", false)) { strUsage += FormatParagraph(LicenseInfo()); } else { - strUsage += "\nUsage: bitcoind [options] Start " PACKAGE_NAME "\n" + strUsage += "\n" + "The " CLIENT_NAME " daemon (bitcoind) is a headless program that connects to the Bitcoin network to validate and relay transactions and blocks, as well as relaying addresses.\n\n" + "It provides the backbone of the Bitcoin network and its RPC, REST and ZMQ services can provide various transaction, block and address-related services.\n\n" + "There is an optional wallet component which provides transaction services.\n\n" + "It can be used in a headless environment or as part of a server setup.\n" + "\n" + "Usage: bitcoind [options]\n" "\n"; strUsage += args.GetHelpMessage(); } @@ -195,7 +201,7 @@ static bool AppInit(NodeContext& node) if (args.GetBoolArg("-daemon", DEFAULT_DAEMON) || args.GetBoolArg("-daemonwait", DEFAULT_DAEMONWAIT)) { #if HAVE_DECL_FORK - tfm::format(std::cout, PACKAGE_NAME " starting\n"); + tfm::format(std::cout, CLIENT_NAME " starting\n"); // Daemonize switch (fork_daemon(1, 0, daemon_ep)) { // don't chdir (1), do close FDs (0) @@ -274,7 +280,7 @@ MAIN_FUNCTION if (ProcessInitCommands(args)) return EXIT_SUCCESS; // Start application - if (!AppInit(node) || !Assert(node.shutdown)->wait()) { + if (!AppInit(node) || !Assert(node.shutdown_signal)->wait()) { node.exit_status = EXIT_FAILURE; } Interrupt(node); diff --git a/src/chain.h b/src/chain.h index c46392c535..13f7582385 100644 --- a/src/chain.h +++ b/src/chain.h @@ -178,7 +178,7 @@ public: //! Verification status of this block. See enum BlockStatus //! //! Note: this value is modified to show BLOCK_OPT_WITNESS during UTXO snapshot - //! load to avoid the block index being spuriously rewound. + //! load to avoid a spurious startup failure requiring -reindex. //! @sa NeedsRedownload //! @sa ActivateSnapshot uint32_t nStatus GUARDED_BY(::cs_main){0}; diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 68319e8e8b..7966b80a27 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -44,6 +44,7 @@ void ReadSigNetArgs(const ArgsManager& args, CChainParams::SigNetOptions& option void ReadRegTestArgs(const ArgsManager& args, CChainParams::RegTestOptions& options) { if (auto value = args.GetBoolArg("-fastprune")) options.fastprune = *value; + if (HasTestOption(args, "bip94")) options.enforce_bip94 = true; for (const std::string& arg : args.GetArgs("-testactivationheight")) { const auto found{arg.find('@')}; diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index aadd04e509..060d519d92 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -41,15 +41,15 @@ std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const ChainType chain) { switch (chain) { case ChainType::MAIN: - return std::make_unique<CBaseChainParams>("", 8332, 8334); + return std::make_unique<CBaseChainParams>("", 8332); case ChainType::TESTNET: - return std::make_unique<CBaseChainParams>("testnet3", 18332, 18334); + return std::make_unique<CBaseChainParams>("testnet3", 18332); case ChainType::TESTNET4: - return std::make_unique<CBaseChainParams>("testnet4", 48332, 48334); + return std::make_unique<CBaseChainParams>("testnet4", 48332); case ChainType::SIGNET: - return std::make_unique<CBaseChainParams>("signet", 38332, 38334); + return std::make_unique<CBaseChainParams>("signet", 38332); case ChainType::REGTEST: - return std::make_unique<CBaseChainParams>("regtest", 18443, 18445); + return std::make_unique<CBaseChainParams>("regtest", 18443); } assert(false); } diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index c75a70cb96..adbd6a5174 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -22,15 +22,13 @@ class CBaseChainParams public: const std::string& DataDir() const { return strDataDir; } uint16_t RPCPort() const { return m_rpc_port; } - uint16_t OnionServiceTargetPort() const { return m_onion_service_target_port; } CBaseChainParams() = delete; - CBaseChainParams(const std::string& data_dir, uint16_t rpc_port, uint16_t onion_service_target_port) - : m_rpc_port(rpc_port), m_onion_service_target_port(onion_service_target_port), strDataDir(data_dir) {} + CBaseChainParams(const std::string& data_dir, uint16_t rpc_port) + : m_rpc_port(rpc_port), strDataDir(data_dir) {} private: const uint16_t m_rpc_port; - const uint16_t m_onion_service_target_port; std::string strDataDir; }; diff --git a/src/checkqueue.h b/src/checkqueue.h index a1de000714..934f672ae3 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -5,25 +5,31 @@ #ifndef BITCOIN_CHECKQUEUE_H #define BITCOIN_CHECKQUEUE_H +#include <logging.h> #include <sync.h> #include <tinyformat.h> #include <util/threadnames.h> #include <algorithm> #include <iterator> +#include <optional> #include <vector> /** * Queue for verifications that have to be performed. * The verifications are represented by a type T, which must provide an - * operator(), returning a bool. + * operator(), returning an std::optional<R>. + * + * The overall result of the computation is std::nullopt if all invocations + * return std::nullopt, or one of the other results otherwise. * * One thread (the master) is assumed to push batches of verifications * onto the queue, where they are processed by N-1 worker threads. When * the master is done adding work, it temporarily joins the worker pool * as an N'th worker, until all jobs are done. + * */ -template <typename T> +template <typename T, typename R = std::remove_cvref_t<decltype(std::declval<T>()().value())>> class CCheckQueue { private: @@ -47,7 +53,7 @@ private: int nTotal GUARDED_BY(m_mutex){0}; //! The temporary evaluation result. - bool fAllOk GUARDED_BY(m_mutex){true}; + std::optional<R> m_result GUARDED_BY(m_mutex); /** * Number of verifications that haven't completed yet. @@ -62,24 +68,28 @@ private: std::vector<std::thread> m_worker_threads; bool m_request_stop GUARDED_BY(m_mutex){false}; - /** Internal function that does bulk of the verification work. */ - bool Loop(bool fMaster) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + /** Internal function that does bulk of the verification work. If fMaster, return the final result. */ + std::optional<R> Loop(bool fMaster) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) { std::condition_variable& cond = fMaster ? m_master_cv : m_worker_cv; std::vector<T> vChecks; vChecks.reserve(nBatchSize); unsigned int nNow = 0; - bool fOk = true; + std::optional<R> local_result; + bool do_work; do { { WAIT_LOCK(m_mutex, lock); // first do the clean-up of the previous loop run (allowing us to do it in the same critsect) if (nNow) { - fAllOk &= fOk; + if (local_result.has_value() && !m_result.has_value()) { + std::swap(local_result, m_result); + } nTodo -= nNow; - if (nTodo == 0 && !fMaster) + if (nTodo == 0 && !fMaster) { // We processed the last element; inform the master it can exit and return the result m_master_cv.notify_one(); + } } else { // first iteration nTotal++; @@ -88,18 +98,19 @@ private: while (queue.empty() && !m_request_stop) { if (fMaster && nTodo == 0) { nTotal--; - bool fRet = fAllOk; + std::optional<R> to_return = std::move(m_result); // reset the status for new work later - fAllOk = true; + m_result = std::nullopt; // return the current status - return fRet; + return to_return; } nIdle++; cond.wait(lock); // wait nIdle--; } if (m_request_stop) { - return false; + // return value does not matter, because m_request_stop is only set in the destructor. + return std::nullopt; } // Decide how many work units to process now. @@ -112,12 +123,15 @@ private: vChecks.assign(std::make_move_iterator(start_it), std::make_move_iterator(queue.end())); queue.erase(start_it, queue.end()); // Check whether we need to do work at all - fOk = fAllOk; + do_work = !m_result.has_value(); } // execute work - for (T& check : vChecks) - if (fOk) - fOk = check(); + if (do_work) { + for (T& check : vChecks) { + local_result = check(); + if (local_result.has_value()) break; + } + } vChecks.clear(); } while (true); } @@ -130,6 +144,7 @@ public: explicit CCheckQueue(unsigned int batch_size, int worker_threads_num) : nBatchSize(batch_size) { + LogInfo("Script verification uses %d additional threads", worker_threads_num); m_worker_threads.reserve(worker_threads_num); for (int n = 0; n < worker_threads_num; ++n) { m_worker_threads.emplace_back([this, n]() { @@ -146,8 +161,9 @@ public: CCheckQueue(CCheckQueue&&) = delete; CCheckQueue& operator=(CCheckQueue&&) = delete; - //! Wait until execution finishes, and return whether all evaluations were successful. - bool Wait() EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + //! Join the execution until completion. If at least one evaluation wasn't successful, return + //! its error. + std::optional<R> Complete() EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) { return Loop(true /* master thread */); } @@ -188,11 +204,11 @@ public: * RAII-style controller object for a CCheckQueue that guarantees the passed * queue is finished before continuing. */ -template <typename T> +template <typename T, typename R = std::remove_cvref_t<decltype(std::declval<T>()().value())>> class CCheckQueueControl { private: - CCheckQueue<T> * const pqueue; + CCheckQueue<T, R> * const pqueue; bool fDone; public: @@ -207,13 +223,12 @@ public: } } - bool Wait() + std::optional<R> Complete() { - if (pqueue == nullptr) - return true; - bool fRet = pqueue->Wait(); + if (pqueue == nullptr) return std::nullopt; + auto ret = pqueue->Complete(); fDone = true; - return fRet; + return ret; } void Add(std::vector<T>&& vChecks) @@ -226,7 +241,7 @@ public: ~CCheckQueueControl() { if (!fDone) - Wait(); + Complete(); if (pqueue != nullptr) { LEAVE_CRITICAL_SECTION(pqueue->m_control_mutex); } diff --git a/src/clientversion.cpp b/src/clientversion.cpp index 017366543d..af58c3023f 100644 --- a/src/clientversion.cpp +++ b/src/clientversion.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <clientversion.h> #include <util/string.h> @@ -20,17 +20,15 @@ using util::Join; * for both bitcoind and bitcoin-qt, to make it harder for attackers to * target servers or GUI users specifically. */ -const std::string CLIENT_NAME("Satoshi"); +const std::string UA_NAME("Satoshi"); -#ifdef HAVE_BUILD_INFO -#include <obj/build.h> -// The <obj/build.h>, which is generated by the build environment (cmake/script/GenerateBuildInfo.cmake), +#include <bitcoin-build-info.h> +// The <bitcoin-build-info.h>, which is generated by the build environment (cmake/script/GenerateBuildInfo.cmake), // could contain only one line of the following: // - "#define BUILD_GIT_TAG ...", if the top commit is tagged // - "#define BUILD_GIT_COMMIT ...", if the top commit is not tagged // - "// No build information available", if proper git information is not available -#endif //! git will put "#define GIT_COMMIT_ID ..." on the next line inside archives. $Format:%n#define GIT_COMMIT_ID "%H"$ @@ -38,7 +36,7 @@ const std::string CLIENT_NAME("Satoshi"); #define BUILD_DESC BUILD_GIT_TAG #define BUILD_SUFFIX "" #else - #define BUILD_DESC "v" PACKAGE_VERSION + #define BUILD_DESC "v" CLIENT_VERSION_STRING #if CLIENT_VERSION_IS_RELEASE #define BUILD_SUFFIX "" #elif defined(BUILD_GIT_COMMIT) @@ -73,7 +71,7 @@ std::string FormatSubVersion(const std::string& name, int nClientVersion, const std::string CopyrightHolders(const std::string& strPrefix) { - const auto copyright_devs = strprintf(_(COPYRIGHT_HOLDERS).translated, COPYRIGHT_HOLDERS_SUBSTITUTION); + const auto copyright_devs = strprintf(_(COPYRIGHT_HOLDERS), COPYRIGHT_HOLDERS_SUBSTITUTION).translated; std::string strCopyrightHolders = strPrefix + copyright_devs; // Make sure Bitcoin Core copyright is not removed by accident @@ -87,15 +85,17 @@ std::string LicenseInfo() { const std::string URL_SOURCE_CODE = "<https://github.com/bitcoin/bitcoin>"; - return CopyrightHolders(strprintf(_("Copyright (C) %i-%i").translated, 2009, COPYRIGHT_YEAR) + " ") + "\n" + + return CopyrightHolders(strprintf(_("Copyright (C) %i-%i"), 2009, COPYRIGHT_YEAR).translated + " ") + "\n" + "\n" + strprintf(_("Please contribute if you find %s useful. " - "Visit %s for further information about the software.").translated, PACKAGE_NAME, "<" PACKAGE_URL ">") + + "Visit %s for further information about the software."), + CLIENT_NAME, "<" CLIENT_URL ">") + .translated + "\n" + - strprintf(_("The source code is available from %s.").translated, URL_SOURCE_CODE) + + strprintf(_("The source code is available from %s."), URL_SOURCE_CODE).translated + "\n" + "\n" + _("This is experimental software.").translated + "\n" + - strprintf(_("Distributed under the MIT software license, see the accompanying file %s or %s").translated, "COPYING", "<https://opensource.org/licenses/MIT>") + + strprintf(_("Distributed under the MIT software license, see the accompanying file %s or %s"), "COPYING", "<https://opensource.org/licenses/MIT>").translated + "\n"; } diff --git a/src/clientversion.h b/src/clientversion.h index 73aaf868e4..03b96d0124 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -7,11 +7,11 @@ #include <util/macros.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep // Check that required client information is defined #if !defined(CLIENT_VERSION_MAJOR) || !defined(CLIENT_VERSION_MINOR) || !defined(CLIENT_VERSION_BUILD) || !defined(CLIENT_VERSION_IS_RELEASE) || !defined(COPYRIGHT_YEAR) -#error Client version information missing: version is not defined by bitcoin-config.h or in any other way +#error Client version information missing: version is not defined by bitcoin-build-config.h or in any other way #endif //! Copyright string used in Windows .rc files @@ -33,7 +33,7 @@ static const int CLIENT_VERSION = + 100 * CLIENT_VERSION_MINOR + 1 * CLIENT_VERSION_BUILD; -extern const std::string CLIENT_NAME; +extern const std::string UA_NAME; std::string FormatFullVersion(); diff --git a/src/cluster_linearize.h b/src/cluster_linearize.h index 607ae681d2..50b121d9e4 100644 --- a/src/cluster_linearize.h +++ b/src/cluster_linearize.h @@ -19,14 +19,6 @@ namespace cluster_linearize { -/** Data type to represent cluster input. - * - * cluster[i].first is tx_i's fee and size. - * cluster[i].second[j] is true iff tx_i spends one or more of tx_j's outputs. - */ -template<typename SetType> -using Cluster = std::vector<std::pair<FeeFrac, SetType>>; - /** Data type to represent transaction indices in clusters. */ using ClusterIndex = uint32_t; @@ -54,12 +46,23 @@ class DepGraph Entry(const FeeFrac& f, const SetType& a, const SetType& d) noexcept : feerate(f), ancestors(a), descendants(d) {} }; - /** Data for each transaction, in the same order as the Cluster it was constructed from. */ + /** Data for each transaction. */ std::vector<Entry> entries; + /** Which positions are used. */ + SetType m_used; + public: /** Equality operator (primarily for testing purposes). */ - friend bool operator==(const DepGraph&, const DepGraph&) noexcept = default; + friend bool operator==(const DepGraph& a, const DepGraph& b) noexcept + { + if (a.m_used != b.m_used) return false; + // Only compare the used positions within the entries vector. + for (auto idx : a.m_used) { + if (a.entries[idx] != b.entries[idx]) return false; + } + return true; + } // Default constructors. DepGraph() noexcept = default; @@ -68,58 +71,51 @@ public: DepGraph& operator=(const DepGraph&) noexcept = default; DepGraph& operator=(DepGraph&&) noexcept = default; - /** Construct a DepGraph object for ntx transactions, with no dependencies. + /** Construct a DepGraph object given another DepGraph and a mapping from old to new. * - * Complexity: O(N) where N=ntx. - **/ - explicit DepGraph(ClusterIndex ntx) noexcept - { - Assume(ntx <= SetType::Size()); - entries.resize(ntx); - for (ClusterIndex i = 0; i < ntx; ++i) { - entries[i].ancestors = SetType::Singleton(i); - entries[i].descendants = SetType::Singleton(i); - } - } - - /** Construct a DepGraph object given a cluster. + * @param depgraph The original DepGraph that is being remapped. + * + * @param mapping A Span such that mapping[i] gives the position in the new DepGraph + * for position i in the old depgraph. Its size must be equal to + * depgraph.PositionRange(). The value of mapping[i] is ignored if + * position i is a hole in depgraph (i.e., if !depgraph.Positions()[i]). + * + * @param pos_range The PositionRange() for the new DepGraph. It must equal the largest + * value in mapping for any used position in depgraph plus 1, or 0 if + * depgraph.TxCount() == 0. * - * Complexity: O(N^2) where N=cluster.size(). + * Complexity: O(N^2) where N=depgraph.TxCount(). */ - explicit DepGraph(const Cluster<SetType>& cluster) noexcept : entries(cluster.size()) + DepGraph(const DepGraph<SetType>& depgraph, Span<const ClusterIndex> mapping, ClusterIndex pos_range) noexcept : entries(pos_range) { - for (ClusterIndex i = 0; i < cluster.size(); ++i) { + Assume(mapping.size() == depgraph.PositionRange()); + Assume((pos_range == 0) == (depgraph.TxCount() == 0)); + for (ClusterIndex i : depgraph.Positions()) { + auto new_idx = mapping[i]; + Assume(new_idx < pos_range); + // Add transaction. + entries[new_idx].ancestors = SetType::Singleton(new_idx); + entries[new_idx].descendants = SetType::Singleton(new_idx); + m_used.Set(new_idx); // Fill in fee and size. - entries[i].feerate = cluster[i].first; - // Fill in direct parents as ancestors. - entries[i].ancestors = cluster[i].second; - // Make sure transactions are ancestors of themselves. - entries[i].ancestors.Set(i); - } - - // Propagate ancestor information. - for (ClusterIndex i = 0; i < entries.size(); ++i) { - // At this point, entries[a].ancestors[b] is true iff b is an ancestor of a and there - // is a path from a to b through the subgraph consisting of {a, b} union - // {0, 1, ..., (i-1)}. - SetType to_merge = entries[i].ancestors; - for (ClusterIndex j = 0; j < entries.size(); ++j) { - if (entries[j].ancestors[i]) { - entries[j].ancestors |= to_merge; - } - } + entries[new_idx].feerate = depgraph.entries[i].feerate; } - - // Fill in descendant information by transposing the ancestor information. - for (ClusterIndex i = 0; i < entries.size(); ++i) { - for (auto j : entries[i].ancestors) { - entries[j].descendants.Set(i); - } + for (ClusterIndex i : depgraph.Positions()) { + // Fill in dependencies by mapping direct parents. + SetType parents; + for (auto j : depgraph.GetReducedParents(i)) parents.Set(mapping[j]); + AddDependencies(parents, mapping[i]); } + // Verify that the provided pos_range was correct (no unused positions at the end). + Assume(m_used.None() ? (pos_range == 0) : (pos_range == m_used.Last() + 1)); } + /** Get the set of transactions positions in use. Complexity: O(1). */ + const SetType& Positions() const noexcept { return m_used; } + /** Get the range of positions in this DepGraph. All entries in Positions() are in [0, PositionRange() - 1]. */ + ClusterIndex PositionRange() const noexcept { return entries.size(); } /** Get the number of transactions in the graph. Complexity: O(1). */ - auto TxCount() const noexcept { return entries.size(); } + auto TxCount() const noexcept { return m_used.Count(); } /** Get the feerate of a given transaction i. Complexity: O(1). */ const FeeFrac& FeeRate(ClusterIndex i) const noexcept { return entries[i].feerate; } /** Get the mutable feerate of a given transaction i. Complexity: O(1). */ @@ -129,39 +125,120 @@ public: /** Get the descendants of a given transaction i. Complexity: O(1). */ const SetType& Descendants(ClusterIndex i) const noexcept { return entries[i].descendants; } - /** Add a new unconnected transaction to this transaction graph (at the end), and return its - * ClusterIndex. + /** Add a new unconnected transaction to this transaction graph (in the first available + * position), and return its ClusterIndex. * * Complexity: O(1) (amortized, due to resizing of backing vector). */ ClusterIndex AddTransaction(const FeeFrac& feefrac) noexcept { - Assume(TxCount() < SetType::Size()); - ClusterIndex new_idx = TxCount(); - entries.emplace_back(feefrac, SetType::Singleton(new_idx), SetType::Singleton(new_idx)); + static constexpr auto ALL_POSITIONS = SetType::Fill(SetType::Size()); + auto available = ALL_POSITIONS - m_used; + Assume(available.Any()); + ClusterIndex new_idx = available.First(); + if (new_idx == entries.size()) { + entries.emplace_back(feefrac, SetType::Singleton(new_idx), SetType::Singleton(new_idx)); + } else { + entries[new_idx] = Entry(feefrac, SetType::Singleton(new_idx), SetType::Singleton(new_idx)); + } + m_used.Set(new_idx); return new_idx; } - /** Modify this transaction graph, adding a dependency between a specified parent and child. + /** Remove the specified positions from this DepGraph. + * + * The specified positions will no longer be part of Positions(), and dependencies with them are + * removed. Note that due to DepGraph only tracking ancestors/descendants (and not direct + * dependencies), if a parent is removed while a grandparent remains, the grandparent will + * remain an ancestor. * * Complexity: O(N) where N=TxCount(). - **/ - void AddDependency(ClusterIndex parent, ClusterIndex child) noexcept + */ + void RemoveTransactions(const SetType& del) noexcept + { + m_used -= del; + // Remove now-unused trailing entries. + while (!entries.empty() && !m_used[entries.size() - 1]) { + entries.pop_back(); + } + // Remove the deleted transactions from ancestors/descendants of other transactions. Note + // that the deleted positions will retain old feerate and dependency information. This does + // not matter as they will be overwritten by AddTransaction if they get used again. + for (auto& entry : entries) { + entry.ancestors &= m_used; + entry.descendants &= m_used; + } + } + + /** Modify this transaction graph, adding multiple parents to a specified child. + * + * Complexity: O(N) where N=TxCount(). + */ + void AddDependencies(const SetType& parents, ClusterIndex child) noexcept { - // Bail out if dependency is already implied. - if (entries[child].ancestors[parent]) return; - // To each ancestor of the parent, add as descendants the descendants of the child. + Assume(m_used[child]); + Assume(parents.IsSubsetOf(m_used)); + // Compute the ancestors of parents that are not already ancestors of child. + SetType par_anc; + for (auto par : parents - Ancestors(child)) { + par_anc |= Ancestors(par); + } + par_anc -= Ancestors(child); + // Bail out if there are no such ancestors. + if (par_anc.None()) return; + // To each such ancestor, add as descendants the descendants of the child. const auto& chl_des = entries[child].descendants; - for (auto anc_of_par : Ancestors(parent)) { + for (auto anc_of_par : par_anc) { entries[anc_of_par].descendants |= chl_des; } - // To each descendant of the child, add as ancestors the ancestors of the parent. - const auto& par_anc = entries[parent].ancestors; + // To each descendant of the child, add those ancestors. for (auto dec_of_chl : Descendants(child)) { entries[dec_of_chl].ancestors |= par_anc; } } + /** Compute the (reduced) set of parents of node i in this graph. + * + * This returns the minimal subset of the parents of i whose ancestors together equal all of + * i's ancestors (unless i is part of a cycle of dependencies). Note that DepGraph does not + * store the set of parents; this information is inferred from the ancestor sets. + * + * Complexity: O(N) where N=Ancestors(i).Count() (which is bounded by TxCount()). + */ + SetType GetReducedParents(ClusterIndex i) const noexcept + { + SetType parents = Ancestors(i); + parents.Reset(i); + for (auto parent : parents) { + if (parents[parent]) { + parents -= Ancestors(parent); + parents.Set(parent); + } + } + return parents; + } + + /** Compute the (reduced) set of children of node i in this graph. + * + * This returns the minimal subset of the children of i whose descendants together equal all of + * i's descendants (unless i is part of a cycle of dependencies). Note that DepGraph does not + * store the set of children; this information is inferred from the descendant sets. + * + * Complexity: O(N) where N=Descendants(i).Count() (which is bounded by TxCount()). + */ + SetType GetReducedChildren(ClusterIndex i) const noexcept + { + SetType children = Descendants(i); + children.Reset(i); + for (auto child : children) { + if (children[child]) { + children -= Descendants(child); + children.Set(child); + } + } + return children; + } + /** Compute the aggregate feerate of a set of nodes in this graph. * * Complexity: O(N) where N=elems.Count(). @@ -215,7 +292,7 @@ public: * * Complexity: O(TxCount()). */ - bool IsConnected() const noexcept { return IsConnected(SetType::Fill(TxCount())); } + bool IsConnected() const noexcept { return IsConnected(m_used); } /** Append the entries of select to list in a topologically valid order. * @@ -257,6 +334,14 @@ struct SetInfo explicit SetInfo(const DepGraph<SetType>& depgraph, const SetType& txn) noexcept : transactions(txn), feerate(depgraph.FeeRate(txn)) {} + /** Add a transaction to this SetInfo (which must not yet be in it). */ + void Set(const DepGraph<SetType>& depgraph, ClusterIndex pos) noexcept + { + Assume(!transactions[pos]); + transactions.Set(pos); + feerate += depgraph.FeeRate(pos); + } + /** Add the transactions of other to this SetInfo (no overlap allowed). */ SetInfo& operator|=(const SetInfo& other) noexcept { @@ -457,11 +542,11 @@ public: */ AncestorCandidateFinder(const DepGraph<SetType>& depgraph LIFETIMEBOUND) noexcept : m_depgraph(depgraph), - m_todo{SetType::Fill(depgraph.TxCount())}, - m_ancestor_set_feerates(depgraph.TxCount()) + m_todo{depgraph.Positions()}, + m_ancestor_set_feerates(depgraph.PositionRange()) { // Precompute ancestor-set feerates. - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + for (ClusterIndex i : m_depgraph.Positions()) { /** The remaining ancestors for transaction i. */ SetType anc_to_add = m_depgraph.Ancestors(i); FeeFrac anc_feerate; @@ -506,6 +591,12 @@ public: return m_todo.None(); } + /** Count the number of remaining unlinearized transactions. */ + ClusterIndex NumRemaining() const noexcept + { + return m_todo.Count(); + } + /** Find the best (highest-feerate, smallest among those in case of a tie) ancestor set * among the remaining transactions. Requires !AllDone(). * @@ -541,23 +632,64 @@ class SearchCandidateFinder { /** Internal RNG. */ InsecureRandomContext m_rng; - /** Internal dependency graph for the cluster. */ - const DepGraph<SetType>& m_depgraph; - /** Which transactions are left to do (sorted indices). */ + /** m_sorted_to_original[i] is the original position that sorted transaction position i had. */ + std::vector<ClusterIndex> m_sorted_to_original; + /** m_original_to_sorted[i] is the sorted position original transaction position i has. */ + std::vector<ClusterIndex> m_original_to_sorted; + /** Internal dependency graph for the cluster (with transactions in decreasing individual + * feerate order). */ + DepGraph<SetType> m_sorted_depgraph; + /** Which transactions are left to do (indices in m_sorted_depgraph's order). */ SetType m_todo; + /** Given a set of transactions with sorted indices, get their original indices. */ + SetType SortedToOriginal(const SetType& arg) const noexcept + { + SetType ret; + for (auto pos : arg) ret.Set(m_sorted_to_original[pos]); + return ret; + } + + /** Given a set of transactions with original indices, get their sorted indices. */ + SetType OriginalToSorted(const SetType& arg) const noexcept + { + SetType ret; + for (auto pos : arg) ret.Set(m_original_to_sorted[pos]); + return ret; + } + public: /** Construct a candidate finder for a graph. * * @param[in] depgraph Dependency graph for the to-be-linearized cluster. * @param[in] rng_seed A random seed to control the search order. * - * Complexity: O(1). + * Complexity: O(N^2) where N=depgraph.Count(). */ - SearchCandidateFinder(const DepGraph<SetType>& depgraph LIFETIMEBOUND, uint64_t rng_seed) noexcept : + SearchCandidateFinder(const DepGraph<SetType>& depgraph, uint64_t rng_seed) noexcept : m_rng(rng_seed), - m_depgraph(depgraph), - m_todo(SetType::Fill(depgraph.TxCount())) {} + m_sorted_to_original(depgraph.TxCount()), + m_original_to_sorted(depgraph.PositionRange()) + { + // Determine reordering mapping, by sorting by decreasing feerate. Unused positions are + // not included, as they will never be looked up anyway. + ClusterIndex sorted_pos{0}; + for (auto i : depgraph.Positions()) { + m_sorted_to_original[sorted_pos++] = i; + } + std::sort(m_sorted_to_original.begin(), m_sorted_to_original.end(), [&](auto a, auto b) { + auto feerate_cmp = depgraph.FeeRate(a) <=> depgraph.FeeRate(b); + if (feerate_cmp == 0) return a < b; + return feerate_cmp > 0; + }); + // Compute reverse mapping. + for (ClusterIndex i = 0; i < m_sorted_to_original.size(); ++i) { + m_original_to_sorted[m_sorted_to_original[i]] = i; + } + // Compute reordered dependency graph. + m_sorted_depgraph = DepGraph(depgraph, m_original_to_sorted, m_sorted_to_original.size()); + m_todo = m_sorted_depgraph.Positions(); + } /** Check whether any unlinearized transactions remain. */ bool AllDone() const noexcept @@ -580,12 +712,15 @@ public: * be <= max_iterations. If strictly < max_iterations, the * returned subset is optimal. * - * Complexity: O(N * min(max_iterations, 2^N)) where N=depgraph.TxCount(). + * Complexity: possibly O(N * min(max_iterations, sqrt(2^N))) where N=depgraph.TxCount(). */ std::pair<SetInfo<SetType>, uint64_t> FindCandidateSet(uint64_t max_iterations, SetInfo<SetType> best) noexcept { Assume(!AllDone()); + // Convert the provided best to internal sorted indices. + best.transactions = OriginalToSorted(best.transactions); + /** Type for work queue items. */ struct WorkItem { @@ -596,16 +731,27 @@ public: /** Set of undecided transactions. This must be a subset of m_todo, and have no overlap * with inc. The set (inc | und) must be topologically valid. */ SetType und; + /** (Only when inc is not empty) The best feerate of any superset of inc that is also a + * subset of (inc | und), without requiring it to be topologically valid. It forms a + * conservative upper bound on how good a set this work item can give rise to. + * Transactions whose feerate is below best's are ignored when determining this value, + * which means it may technically be an underestimate, but if so, this work item + * cannot result in something that beats best anyway. */ + FeeFrac pot_feerate; /** Construct a new work item. */ - WorkItem(SetInfo<SetType>&& i, SetType&& u) noexcept : - inc(std::move(i)), und(std::move(u)) {} + WorkItem(SetInfo<SetType>&& i, SetType&& u, FeeFrac&& p_f) noexcept : + inc(std::move(i)), und(std::move(u)), pot_feerate(std::move(p_f)) + { + Assume(pot_feerate.IsEmpty() == inc.feerate.IsEmpty()); + } /** Swap two WorkItems. */ void Swap(WorkItem& other) noexcept { swap(inc, other.inc); swap(und, other.und); + swap(pot_feerate, other.pot_feerate); } }; @@ -613,39 +759,111 @@ public: VecDeque<WorkItem> queue; queue.reserve(std::max<size_t>(256, 2 * m_todo.Count())); - // Create an initial entry with m_todo as undecided. Also use it as best if not provided, - // so that during the work processing loop below, and during the add_fn/split_fn calls, we - // do not need to deal with the best=empty case. - if (best.feerate.IsEmpty()) best = SetInfo(m_depgraph, m_todo); - queue.emplace_back(SetInfo<SetType>{}, SetType{m_todo}); + // Create initial entries per connected component of m_todo. While clusters themselves are + // generally connected, this is not necessarily true after some parts have already been + // removed from m_todo. Without this, effort can be wasted on searching "inc" sets that + // span multiple components. + auto to_cover = m_todo; + do { + auto component = m_sorted_depgraph.FindConnectedComponent(to_cover); + to_cover -= component; + // If best is not provided, set it to the first component, so that during the work + // processing loop below, and during the add_fn/split_fn calls, we do not need to deal + // with the best=empty case. + if (best.feerate.IsEmpty()) best = SetInfo(m_sorted_depgraph, component); + queue.emplace_back(/*inc=*/SetInfo<SetType>{}, + /*und=*/std::move(component), + /*pot_feerate=*/FeeFrac{}); + } while (to_cover.Any()); /** Local copy of the iteration limit. */ uint64_t iterations_left = max_iterations; + /** The set of transactions in m_todo which have feerate > best's. */ + SetType imp = m_todo; + while (imp.Any()) { + ClusterIndex check = imp.Last(); + if (m_sorted_depgraph.FeeRate(check) >> best.feerate) break; + imp.Reset(check); + } + /** Internal function to add an item to the queue of elements to explore if there are any - * transactions left to split on, and to update best. + * transactions left to split on, possibly improving it before doing so, and to update + * best/imp. * * - inc: the "inc" value for the new work item (must be topological). * - und: the "und" value for the new work item ((inc | und) must be topological). */ auto add_fn = [&](SetInfo<SetType> inc, SetType und) noexcept { + /** SetInfo object with the set whose feerate will become the new work item's + * pot_feerate. It starts off equal to inc. */ + auto pot = inc; if (!inc.feerate.IsEmpty()) { + // Add entries to pot. We iterate over all undecided transactions whose feerate is + // higher than best. While undecided transactions of lower feerate may improve pot, + // the resulting pot feerate cannot possibly exceed best's (and this item will be + // skipped in split_fn anyway). + for (auto pos : imp & und) { + // Determine if adding transaction pos to pot (ignoring topology) would improve + // it. If not, we're done updating pot. This relies on the fact that + // m_sorted_depgraph, and thus the transactions iterated over, are in decreasing + // individual feerate order. + if (!(m_sorted_depgraph.FeeRate(pos) >> pot.feerate)) break; + pot.Set(m_sorted_depgraph, pos); + } + + // The "jump ahead" optimization: whenever pot has a topologically-valid subset, + // that subset can be added to inc. Any subset of (pot - inc) has the property that + // its feerate exceeds that of any set compatible with this work item (superset of + // inc, subset of (inc | und)). Thus, if T is a topological subset of pot, and B is + // the best topologically-valid set compatible with this work item, and (T - B) is + // non-empty, then (T | B) is better than B and also topological. This is in + // contradiction with the assumption that B is best. Thus, (T - B) must be empty, + // or T must be a subset of B. + // + // See https://delvingbitcoin.org/t/how-to-linearize-your-cluster/303 section 2.4. + const auto init_inc = inc.transactions; + for (auto pos : pot.transactions - inc.transactions) { + // If the transaction's ancestors are a subset of pot, we can add it together + // with its ancestors to inc. Just update the transactions here; the feerate + // update happens below. + auto anc_todo = m_sorted_depgraph.Ancestors(pos) & m_todo; + if (anc_todo.IsSubsetOf(pot.transactions)) inc.transactions |= anc_todo; + } + // Finally update und and inc's feerate to account for the added transactions. + und -= inc.transactions; + inc.feerate += m_sorted_depgraph.FeeRate(inc.transactions - init_inc); + // If inc's feerate is better than best's, remember it as our new best. if (inc.feerate > best.feerate) { best = inc; + // See if we can remove any entries from imp now. + while (imp.Any()) { + ClusterIndex check = imp.Last(); + if (m_sorted_depgraph.FeeRate(check) >> best.feerate) break; + imp.Reset(check); + } } + + // If no potential transactions exist beyond the already included ones, no + // improvement is possible anymore. + if (pot.feerate.size == inc.feerate.size) return; + // At this point und must be non-empty. If it were empty then pot would equal inc. + Assume(und.Any()); } else { Assume(inc.transactions.None()); + // If inc is empty, we just make sure there are undecided transactions left to + // split on. + if (und.None()) return; } - // Make sure there are undecided transactions left to split on. - if (und.None()) return; - // Actually construct a new work item on the queue. Due to the switch to DFS when queue // space runs out (see below), we know that no reallocation of the queue should ever // occur. Assume(queue.size() < queue.capacity()); - queue.emplace_back(std::move(inc), std::move(und)); + queue.emplace_back(/*inc=*/std::move(inc), + /*und=*/std::move(und), + /*pot_feerate=*/std::move(pot.feerate)); }; /** Internal process function. It takes an existing work item, and splits it in two: one @@ -659,18 +877,66 @@ public: Assume(elem.inc.transactions.IsSubsetOf(m_todo) && elem.und.IsSubsetOf(m_todo)); // Included transactions cannot be undecided. Assume(!elem.inc.transactions.Overlaps(elem.und)); + // If pot is empty, then so is inc. + Assume(elem.inc.feerate.IsEmpty() == elem.pot_feerate.IsEmpty()); + + const ClusterIndex first = elem.und.First(); + if (!elem.inc.feerate.IsEmpty()) { + // If no undecided transactions remain with feerate higher than best, this entry + // cannot be improved beyond best. + if (!elem.und.Overlaps(imp)) return; + // We can ignore any queue item whose potential feerate isn't better than the best + // seen so far. + if (elem.pot_feerate <= best.feerate) return; + } else { + // In case inc is empty use a simpler alternative check. + if (m_sorted_depgraph.FeeRate(first) <= best.feerate) return; + } - // Pick the first undecided transaction as the one to split on. - const ClusterIndex split = elem.und.First(); + // Decide which transaction to split on. Splitting is how new work items are added, and + // how progress is made. One split transaction is chosen among the queue item's + // undecided ones, and: + // - A work item is (potentially) added with that transaction plus its remaining + // descendants excluded (removed from the und set). + // - A work item is (potentially) added with that transaction plus its remaining + // ancestors included (added to the inc set). + // + // To decide what to split on, consider the undecided ancestors of the highest + // individual feerate undecided transaction. Pick the one which reduces the search space + // most. Let I(t) be the size of the undecided set after including t, and E(t) the size + // of the undecided set after excluding t. Then choose the split transaction t such + // that 2^I(t) + 2^E(t) is minimal, tie-breaking by highest individual feerate for t. + ClusterIndex split = 0; + const auto select = elem.und & m_sorted_depgraph.Ancestors(first); + Assume(select.Any()); + std::optional<std::pair<ClusterIndex, ClusterIndex>> split_counts; + for (auto t : select) { + // Call max = max(I(t), E(t)) and min = min(I(t), E(t)). Let counts = {max,min}. + // Sorting by the tuple counts is equivalent to sorting by 2^I(t) + 2^E(t). This + // expression is equal to 2^max + 2^min = 2^max * (1 + 1/2^(max - min)). The second + // factor (1 + 1/2^(max - min)) there is in (1,2]. Thus increasing max will always + // increase it, even when min decreases. Because of this, we can first sort by max. + std::pair<ClusterIndex, ClusterIndex> counts{ + (elem.und - m_sorted_depgraph.Ancestors(t)).Count(), + (elem.und - m_sorted_depgraph.Descendants(t)).Count()}; + if (counts.first < counts.second) std::swap(counts.first, counts.second); + // Remember the t with the lowest counts. + if (!split_counts.has_value() || counts < *split_counts) { + split = t; + split_counts = counts; + } + } + // Since there was at least one transaction in select, we must always find one. + Assume(split_counts.has_value()); // Add a work item corresponding to exclusion of the split transaction. - const auto& desc = m_depgraph.Descendants(split); + const auto& desc = m_sorted_depgraph.Descendants(split); add_fn(/*inc=*/elem.inc, /*und=*/elem.und - desc); // Add a work item corresponding to inclusion of the split transaction. - const auto anc = m_depgraph.Ancestors(split) & m_todo; - add_fn(/*inc=*/elem.inc.Add(m_depgraph, anc), + const auto anc = m_sorted_depgraph.Ancestors(split) & m_todo; + add_fn(/*inc=*/elem.inc.Add(m_sorted_depgraph, anc), /*und=*/elem.und - anc); // Account for the performed split. @@ -713,7 +979,9 @@ public: split_fn(std::move(elem)); } - // Return the found best set and the number of iterations performed. + // Return the found best set (converted to the original transaction indices), and the + // number of iterations performed. + best.transactions = SortedToOriginal(best.transactions); return {std::move(best), max_iterations - iterations_left}; } @@ -723,9 +991,10 @@ public: */ void MarkDone(const SetType& done) noexcept { - Assume(done.Any()); - Assume(done.IsSubsetOf(m_todo)); - m_todo -= done; + const auto done_sorted = OriginalToSorted(done); + Assume(done_sorted.Any()); + Assume(done_sorted.IsSubsetOf(m_todo)); + m_todo -= done_sorted; } }; @@ -744,7 +1013,7 @@ public: * - A boolean indicating whether the result is guaranteed to be * optimal. * - * Complexity: O(N * min(max_iterations + N, 2^N)) where N=depgraph.TxCount(). + * Complexity: possibly O(N * min(max_iterations + N, sqrt(2^N))) where N=depgraph.TxCount(). */ template<typename SetType> std::pair<std::vector<ClusterIndex>, bool> Linearize(const DepGraph<SetType>& depgraph, uint64_t max_iterations, uint64_t rng_seed, Span<const ClusterIndex> old_linearization = {}) noexcept @@ -756,10 +1025,20 @@ std::pair<std::vector<ClusterIndex>, bool> Linearize(const DepGraph<SetType>& de std::vector<ClusterIndex> linearization; AncestorCandidateFinder anc_finder(depgraph); - SearchCandidateFinder src_finder(depgraph, rng_seed); + std::optional<SearchCandidateFinder<SetType>> src_finder; linearization.reserve(depgraph.TxCount()); bool optimal = true; + // Treat the initialization of SearchCandidateFinder as taking N^2/64 (rounded up) iterations + // (largely due to the cost of constructing the internal sorted-by-feerate DepGraph inside + // SearchCandidateFinder), a rough approximation based on benchmark. If we don't have that + // many, don't start it. + uint64_t start_iterations = (uint64_t{depgraph.TxCount()} * depgraph.TxCount() + 63) / 64; + if (iterations_left > start_iterations) { + iterations_left -= start_iterations; + src_finder.emplace(depgraph, rng_seed); + } + /** Chunking of what remains of the old linearization. */ LinearizationChunking old_chunking(depgraph, old_linearization); @@ -772,12 +1051,22 @@ std::pair<std::vector<ClusterIndex>, bool> Linearize(const DepGraph<SetType>& de auto best = anc_finder.FindCandidateSet(); if (!best_prefix.feerate.IsEmpty() && best_prefix.feerate >= best.feerate) best = best_prefix; - // Invoke bounded search to update best, with up to half of our remaining iterations as - // limit. - uint64_t max_iterations_now = (iterations_left + 1) / 2; uint64_t iterations_done_now = 0; - std::tie(best, iterations_done_now) = src_finder.FindCandidateSet(max_iterations_now, best); - iterations_left -= iterations_done_now; + uint64_t max_iterations_now = 0; + if (src_finder) { + // Treat the invocation of SearchCandidateFinder::FindCandidateSet() as costing N/4 + // up-front (rounded up) iterations (largely due to the cost of connected-component + // splitting), a rough approximation based on benchmarks. + uint64_t base_iterations = (anc_finder.NumRemaining() + 3) / 4; + if (iterations_left > base_iterations) { + // Invoke bounded search to update best, with up to half of our remaining + // iterations as limit. + iterations_left -= base_iterations; + max_iterations_now = (iterations_left + 1) / 2; + std::tie(best, iterations_done_now) = src_finder->FindCandidateSet(max_iterations_now, best); + iterations_left -= iterations_done_now; + } + } if (iterations_done_now == max_iterations_now) { optimal = false; @@ -795,7 +1084,7 @@ std::pair<std::vector<ClusterIndex>, bool> Linearize(const DepGraph<SetType>& de // Update state to reflect best is no longer to be linearized. anc_finder.MarkDone(best.transactions); if (anc_finder.AllDone()) break; - src_finder.MarkDone(best.transactions); + if (src_finder) src_finder->MarkDone(best.transactions); if (old_chunking.NumChunksLeft() > 0) { old_chunking.MarkDone(best.transactions); } @@ -911,7 +1200,7 @@ void PostLinearize(const DepGraph<SetType>& depgraph, Span<ClusterIndex> lineari // During an even pass, the diagram above would correspond to linearization [2,3,0,1], with // groups [2] and [3,0,1]. - std::vector<TxEntry> entries(linearization.size() + 1); + std::vector<TxEntry> entries(depgraph.PositionRange() + 1); // Perform two passes over the linearization. for (int pass = 0; pass < 2; ++pass) { diff --git a/src/coins.cpp b/src/coins.cpp index a47ab8063e..24a102b0bc 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -9,7 +9,11 @@ #include <random.h> #include <util/trace.h> -bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } +TRACEPOINT_SEMAPHORE(utxocache, add); +TRACEPOINT_SEMAPHORE(utxocache, spent); +TRACEPOINT_SEMAPHORE(utxocache, uncache); + +std::optional<Coin> CCoinsView::GetCoin(const COutPoint& outpoint) const { return std::nullopt; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } std::vector<uint256> CCoinsView::GetHeadBlocks() const { return std::vector<uint256>(); } bool CCoinsView::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &hashBlock) { return false; } @@ -17,12 +21,11 @@ std::unique_ptr<CCoinsViewCursor> CCoinsView::Cursor() const { return nullptr; } bool CCoinsView::HaveCoin(const COutPoint &outpoint) const { - Coin coin; - return GetCoin(outpoint, coin); + return GetCoin(outpoint).has_value(); } CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) { } -bool CCoinsViewBacked::GetCoin(const COutPoint &outpoint, Coin &coin) const { return base->GetCoin(outpoint, coin); } +std::optional<Coin> CCoinsViewBacked::GetCoin(const COutPoint& outpoint) const { return base->GetCoin(outpoint); } bool CCoinsViewBacked::HaveCoin(const COutPoint &outpoint) const { return base->HaveCoin(outpoint); } uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } std::vector<uint256> CCoinsViewBacked::GetHeadBlocks() const { return base->GetHeadBlocks(); } @@ -45,26 +48,25 @@ size_t CCoinsViewCache::DynamicMemoryUsage() const { CCoinsMap::iterator CCoinsViewCache::FetchCoin(const COutPoint &outpoint) const { const auto [ret, inserted] = cacheCoins.try_emplace(outpoint); if (inserted) { - if (!base->GetCoin(outpoint, ret->second.coin)) { + if (auto coin{base->GetCoin(outpoint)}) { + ret->second.coin = std::move(*coin); + cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage(); + if (ret->second.coin.IsSpent()) { // TODO GetCoin cannot return spent coins + // The parent only has an empty entry for this outpoint; we can consider our version as fresh. + CCoinsCacheEntry::SetFresh(*ret, m_sentinel); + } + } else { cacheCoins.erase(ret); return cacheCoins.end(); } - if (ret->second.coin.IsSpent()) { - // The parent only has an empty entry for this outpoint; we can consider our version as fresh. - ret->second.AddFlags(CCoinsCacheEntry::FRESH, *ret, m_sentinel); - } - cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage(); } return ret; } -bool CCoinsViewCache::GetCoin(const COutPoint &outpoint, Coin &coin) const { - CCoinsMap::const_iterator it = FetchCoin(outpoint); - if (it != cacheCoins.end()) { - coin = it->second.coin; - return !coin.IsSpent(); - } - return false; +std::optional<Coin> CCoinsViewCache::GetCoin(const COutPoint& outpoint) const +{ + if (auto it{FetchCoin(outpoint)}; it != cacheCoins.end() && !it->second.coin.IsSpent()) return it->second.coin; + return std::nullopt; } void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possible_overwrite) { @@ -97,9 +99,10 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi fresh = !it->second.IsDirty(); } it->second.coin = std::move(coin); - it->second.AddFlags(CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0), *it, m_sentinel); + CCoinsCacheEntry::SetDirty(*it, m_sentinel); + if (fresh) CCoinsCacheEntry::SetFresh(*it, m_sentinel); cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); - TRACE5(utxocache, add, + TRACEPOINT(utxocache, add, outpoint.hash.data(), (uint32_t)outpoint.n, (uint32_t)it->second.coin.nHeight, @@ -109,13 +112,8 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi void CCoinsViewCache::EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin) { cachedCoinsUsage += coin.DynamicMemoryUsage(); - auto [it, inserted] = cacheCoins.emplace( - std::piecewise_construct, - std::forward_as_tuple(std::move(outpoint)), - std::forward_as_tuple(std::move(coin))); - if (inserted) { - it->second.AddFlags(CCoinsCacheEntry::DIRTY, *it, m_sentinel); - } + auto [it, inserted] = cacheCoins.try_emplace(std::move(outpoint), std::move(coin)); + if (inserted) CCoinsCacheEntry::SetDirty(*it, m_sentinel); } void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight, bool check_for_overwrite) { @@ -133,7 +131,7 @@ bool CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) { CCoinsMap::iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) return false; cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); - TRACE5(utxocache, spent, + TRACEPOINT(utxocache, spent, outpoint.hash.data(), (uint32_t)outpoint.n, (uint32_t)it->second.coin.nHeight, @@ -145,7 +143,7 @@ bool CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) { if (it->second.IsFresh()) { cacheCoins.erase(it); } else { - it->second.AddFlags(CCoinsCacheEntry::DIRTY, *it, m_sentinel); + CCoinsCacheEntry::SetDirty(*it, m_sentinel); it->second.coin.Clear(); } return true; @@ -205,13 +203,11 @@ bool CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &ha entry.coin = it->second.coin; } cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); - entry.AddFlags(CCoinsCacheEntry::DIRTY, *itUs, m_sentinel); + CCoinsCacheEntry::SetDirty(*itUs, m_sentinel); // We can mark it FRESH in the parent if it was FRESH in the child // Otherwise it might have just been flushed from the parent's cache // and already exist in the grandparent - if (it->second.IsFresh()) { - entry.AddFlags(CCoinsCacheEntry::FRESH, *itUs, m_sentinel); - } + if (it->second.IsFresh()) CCoinsCacheEntry::SetFresh(*itUs, m_sentinel); } } else { // Found the entry in the parent cache @@ -239,7 +235,7 @@ bool CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &ha itUs->second.coin = it->second.coin; } cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); - itUs->second.AddFlags(CCoinsCacheEntry::DIRTY, *itUs, m_sentinel); + CCoinsCacheEntry::SetDirty(*itUs, m_sentinel); // NOTE: It isn't safe to mark the coin as FRESH in the parent // cache. If it already existed and was spent in the parent // cache then marking it FRESH would prevent that spentness @@ -280,7 +276,7 @@ void CCoinsViewCache::Uncache(const COutPoint& hash) CCoinsMap::iterator it = cacheCoins.find(hash); if (it != cacheCoins.end() && !it->second.IsDirty() && !it->second.IsFresh()) { cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); - TRACE5(utxocache, uncache, + TRACEPOINT(utxocache, uncache, hash.hash.data(), (uint32_t)hash.n, (uint32_t)it->second.coin.nHeight, @@ -363,8 +359,8 @@ const Coin& AccessByTxid(const CCoinsViewCache& view, const Txid& txid) return coinEmpty; } -template <typename Func> -static bool ExecuteBackedWrapper(Func func, const std::vector<std::function<void()>>& err_callbacks) +template <typename ReturnType, typename Func> +static ReturnType ExecuteBackedWrapper(Func func, const std::vector<std::function<void()>>& err_callbacks) { try { return func(); @@ -381,10 +377,12 @@ static bool ExecuteBackedWrapper(Func func, const std::vector<std::function<void } } -bool CCoinsViewErrorCatcher::GetCoin(const COutPoint &outpoint, Coin &coin) const { - return ExecuteBackedWrapper([&]() { return CCoinsViewBacked::GetCoin(outpoint, coin); }, m_err_callbacks); +std::optional<Coin> CCoinsViewErrorCatcher::GetCoin(const COutPoint& outpoint) const +{ + return ExecuteBackedWrapper<std::optional<Coin>>([&]() { return CCoinsViewBacked::GetCoin(outpoint); }, m_err_callbacks); } -bool CCoinsViewErrorCatcher::HaveCoin(const COutPoint &outpoint) const { - return ExecuteBackedWrapper([&]() { return CCoinsViewBacked::HaveCoin(outpoint); }, m_err_callbacks); +bool CCoinsViewErrorCatcher::HaveCoin(const COutPoint& outpoint) const +{ + return ExecuteBackedWrapper<bool>([&]() { return CCoinsViewBacked::HaveCoin(outpoint); }, m_err_callbacks); } diff --git a/src/coins.h b/src/coins.h index 78b8eddacd..61fb4af642 100644 --- a/src/coins.h +++ b/src/coins.h @@ -110,7 +110,7 @@ struct CCoinsCacheEntry private: /** * These are used to create a doubly linked list of flagged entries. - * They are set in AddFlags and unset in ClearFlags. + * They are set in SetDirty, SetFresh, and unset in SetClean. * A flagged entry is any entry that is either DIRTY, FRESH, or both. * * DIRTY entries are tracked so that only modified entries can be passed to @@ -128,6 +128,21 @@ private: CoinsCachePair* m_next{nullptr}; uint8_t m_flags{0}; + //! Adding a flag requires a reference to the sentinel of the flagged pair linked list. + static void AddFlags(uint8_t flags, CoinsCachePair& pair, CoinsCachePair& sentinel) noexcept + { + Assume(flags & (DIRTY | FRESH)); + if (!pair.second.m_flags) { + Assume(!pair.second.m_prev && !pair.second.m_next); + pair.second.m_prev = sentinel.second.m_prev; + pair.second.m_next = &sentinel; + sentinel.second.m_prev = &pair; + pair.second.m_prev->second.m_next = &pair; + } + Assume(pair.second.m_prev && pair.second.m_next); + pair.second.m_flags |= flags; + } + public: Coin coin; // The actual cached data. @@ -156,52 +171,43 @@ public: explicit CCoinsCacheEntry(Coin&& coin_) noexcept : coin(std::move(coin_)) {} ~CCoinsCacheEntry() { - ClearFlags(); + SetClean(); } - //! Adding a flag also requires a self reference to the pair that contains - //! this entry in the CCoinsCache map and a reference to the sentinel of the - //! flagged pair linked list. - inline void AddFlags(uint8_t flags, CoinsCachePair& self, CoinsCachePair& sentinel) noexcept - { - Assume(&self.second == this); - if (!m_flags && flags) { - m_prev = sentinel.second.m_prev; - m_next = &sentinel; - sentinel.second.m_prev = &self; - m_prev->second.m_next = &self; - } - m_flags |= flags; - } - inline void ClearFlags() noexcept + static void SetDirty(CoinsCachePair& pair, CoinsCachePair& sentinel) noexcept { AddFlags(DIRTY, pair, sentinel); } + static void SetFresh(CoinsCachePair& pair, CoinsCachePair& sentinel) noexcept { AddFlags(FRESH, pair, sentinel); } + + void SetClean() noexcept { if (!m_flags) return; m_next->second.m_prev = m_prev; m_prev->second.m_next = m_next; m_flags = 0; + m_prev = m_next = nullptr; } - inline uint8_t GetFlags() const noexcept { return m_flags; } - inline bool IsDirty() const noexcept { return m_flags & DIRTY; } - inline bool IsFresh() const noexcept { return m_flags & FRESH; } + bool IsDirty() const noexcept { return m_flags & DIRTY; } + bool IsFresh() const noexcept { return m_flags & FRESH; } //! Only call Next when this entry is DIRTY, FRESH, or both - inline CoinsCachePair* Next() const noexcept { + CoinsCachePair* Next() const noexcept + { Assume(m_flags); return m_next; } //! Only call Prev when this entry is DIRTY, FRESH, or both - inline CoinsCachePair* Prev() const noexcept { + CoinsCachePair* Prev() const noexcept + { Assume(m_flags); return m_prev; } //! Only use this for initializing the linked list sentinel - inline void SelfRef(CoinsCachePair& self) noexcept + void SelfRef(CoinsCachePair& pair) noexcept { - Assume(&self.second == this); - m_prev = &self; - m_next = &self; + Assume(&pair.second == this); + m_prev = &pair; + m_next = &pair; // Set sentinel to DIRTY so we can call Next on it m_flags = DIRTY; } @@ -279,13 +285,13 @@ struct CoinsViewCacheCursor { const auto next_entry{current.second.Next()}; // If we are not going to erase the cache, we must still erase spent entries. - // Otherwise clear the flags on the entry. + // Otherwise, clear the state of the entry. if (!m_will_erase) { if (current.second.coin.IsSpent()) { m_usage -= current.second.coin.DynamicMemoryUsage(); m_map.erase(current.first); } else { - current.second.ClearFlags(); + current.second.SetClean(); } } return next_entry; @@ -303,11 +309,8 @@ private: class CCoinsView { public: - /** Retrieve the Coin (unspent transaction output) for a given outpoint. - * Returns true only when an unspent coin was found, which is returned in coin. - * When false is returned, coin's value is unspecified. - */ - virtual bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + //! Retrieve the Coin (unspent transaction output) for a given outpoint. + virtual std::optional<Coin> GetCoin(const COutPoint& outpoint) const; //! Just check whether a given outpoint is unspent. virtual bool HaveCoin(const COutPoint &outpoint) const; @@ -344,7 +347,7 @@ protected: public: CCoinsViewBacked(CCoinsView *viewIn); - bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + std::optional<Coin> GetCoin(const COutPoint& outpoint) const override; bool HaveCoin(const COutPoint &outpoint) const override; uint256 GetBestBlock() const override; std::vector<uint256> GetHeadBlocks() const override; @@ -384,7 +387,7 @@ public: CCoinsViewCache(const CCoinsViewCache &) = delete; // Standard CCoinsView methods - bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + std::optional<Coin> GetCoin(const COutPoint& outpoint) const override; bool HaveCoin(const COutPoint &outpoint) const override; uint256 GetBestBlock() const override; void SetBestBlock(const uint256 &hashBlock); @@ -514,7 +517,7 @@ public: m_err_callbacks.emplace_back(std::move(f)); } - bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + std::optional<Coin> GetCoin(const COutPoint& outpoint) const override; bool HaveCoin(const COutPoint &outpoint) const override; private: diff --git a/src/common/args.cpp b/src/common/args.cpp index f59d2b8f0f..833a0b28bd 100644 --- a/src/common/args.cpp +++ b/src/common/args.cpp @@ -184,7 +184,7 @@ bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::strin for (int i = 1; i < argc; i++) { std::string key(argv[i]); -#ifdef MAC_OSX +#ifdef __APPLE__ // At the first time when a user gets the "App downloaded from the // internet" warning, and clicks the Open button, macOS passes // a unique process serial number (PSN) as -psn_... command-line @@ -688,8 +688,8 @@ bool HelpRequested(const ArgsManager& args) void SetupHelpOptions(ArgsManager& args) { - args.AddArg("-?", "Print this help message and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - args.AddHiddenArgs({"-h", "-help"}); + args.AddArg("-help", "Print this help message and exit (also -h or -?)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + args.AddHiddenArgs({"-h", "-?"}); } static const int screenWidth = 79; @@ -709,6 +709,7 @@ std::string HelpMessageOpt(const std::string &option, const std::string &message const std::vector<std::string> TEST_OPTIONS_DOC{ "addrman (use deterministic addrman)", + "bip94 (enforce BIP94 consensus rules)", }; bool HasTestOption(const ArgsManager& args, const std::string& test_option) @@ -741,7 +742,7 @@ fs::path GetDefaultDataDir() pathRet = fs::path("/"); else pathRet = fs::path(pszHome); -#ifdef MAC_OSX +#ifdef __APPLE__ // macOS return pathRet / "Library/Application Support/Bitcoin"; #else diff --git a/src/common/config.cpp b/src/common/config.cpp index 98223fc3e3..fac4aa314c 100644 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -16,6 +16,7 @@ #include <algorithm> #include <cassert> #include <cstdlib> +#include <filesystem> #include <fstream> #include <iostream> #include <list> @@ -128,12 +129,18 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) } const auto conf_path{GetConfigFilePath()}; - std::ifstream stream{conf_path}; - - // not ok to have a config file specified that cannot be opened - if (IsArgSet("-conf") && !stream.good()) { - error = strprintf("specified config file \"%s\" could not be opened.", fs::PathToString(conf_path)); - return false; + std::ifstream stream; + if (!conf_path.empty()) { // path is empty when -noconf is specified + if (fs::is_directory(conf_path)) { + error = strprintf("Config file \"%s\" is a directory.", fs::PathToString(conf_path)); + return false; + } + stream = std::ifstream{conf_path}; + // If the file is explicitly specified, it must be readable + if (IsArgSet("-conf") && !stream.good()) { + error = strprintf("specified config file \"%s\" could not be opened.", fs::PathToString(conf_path)); + return false; + } } // ok to not have a config file if (stream.good()) { @@ -175,7 +182,12 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) const size_t default_includes = add_includes({}); for (const std::string& conf_file_name : conf_file_names) { - std::ifstream conf_file_stream{AbsPathForConfigVal(*this, fs::PathFromString(conf_file_name), /*net_specific=*/false)}; + const auto include_conf_path{AbsPathForConfigVal(*this, fs::PathFromString(conf_file_name), /*net_specific=*/false)}; + if (fs::is_directory(include_conf_path)) { + error = strprintf("Included config file \"%s\" is a directory.", fs::PathToString(include_conf_path)); + return false; + } + std::ifstream conf_file_stream{include_conf_path}; if (conf_file_stream.good()) { if (!ReadConfigStream(conf_file_stream, conf_file_name, error, ignore_invalid_keys)) { return false; @@ -213,7 +225,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) fs::path AbsPathForConfigVal(const ArgsManager& args, const fs::path& path, bool net_specific) { - if (path.is_absolute()) { + if (path.is_absolute() || path.empty()) { return path; } return fsbridge::AbsPathJoin(net_specific ? args.GetDataDirNet() : args.GetDataDirBase(), path); diff --git a/src/common/init.cpp b/src/common/init.cpp index 412d73aec7..5a70440468 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -62,29 +62,36 @@ std::optional<ConfigError> InitConfig(ArgsManager& args, SettingsAbortFn setting fs::create_directories(net_path / "wallets"); } - // Show an error or warning if there is a bitcoin.conf file in the + // Show an error or warn/log if there is a bitcoin.conf file in the // datadir that is being ignored. const fs::path base_config_path = base_path / BITCOIN_CONF_FILENAME; - if (fs::exists(base_config_path) && !fs::equivalent(orig_config_path, base_config_path)) { - const std::string cli_config_path = args.GetArg("-conf", ""); - const std::string config_source = cli_config_path.empty() - ? strprintf("data directory %s", fs::quoted(fs::PathToString(orig_datadir_path))) - : strprintf("command line argument %s", fs::quoted("-conf=" + cli_config_path)); - const std::string error = strprintf( - "Data directory %1$s contains a %2$s file which is ignored, because a different configuration file " - "%3$s from %4$s is being used instead. Possible ways to address this would be to:\n" - "- Delete or rename the %2$s file in data directory %1$s.\n" - "- Change datadir= or conf= options to specify one configuration file, not two, and use " - "includeconf= to include any other configuration files.\n" - "- Set allowignoredconf=1 option to treat this condition as a warning, not an error.", - fs::quoted(fs::PathToString(base_path)), - fs::quoted(BITCOIN_CONF_FILENAME), - fs::quoted(fs::PathToString(orig_config_path)), - config_source); - if (args.GetBoolArg("-allowignoredconf", false)) { - LogPrintf("Warning: %s\n", error); - } else { - return ConfigError{ConfigStatus::FAILED, Untranslated(error)}; + if (fs::exists(base_config_path)) { + if (orig_config_path.empty()) { + LogInfo( + "Data directory %s contains a %s file which is explicitly ignored using -noconf.", + fs::quoted(fs::PathToString(base_path)), + fs::quoted(BITCOIN_CONF_FILENAME)); + } else if (!fs::equivalent(orig_config_path, base_config_path)) { + const std::string cli_config_path = args.GetArg("-conf", ""); + const std::string config_source = cli_config_path.empty() + ? strprintf("data directory %s", fs::quoted(fs::PathToString(orig_datadir_path))) + : strprintf("command line argument %s", fs::quoted("-conf=" + cli_config_path)); + std::string error = strprintf( + "Data directory %1$s contains a %2$s file which is ignored, because a different configuration file " + "%3$s from %4$s is being used instead. Possible ways to address this would be to:\n" + "- Delete or rename the %2$s file in data directory %1$s.\n" + "- Change datadir= or conf= options to specify one configuration file, not two, and use " + "includeconf= to include any other configuration files.", + fs::quoted(fs::PathToString(base_path)), + fs::quoted(BITCOIN_CONF_FILENAME), + fs::quoted(fs::PathToString(orig_config_path)), + config_source); + if (args.GetBoolArg("-allowignoredconf", false)) { + LogWarning("%s", error); + } else { + error += "\n- Set allowignoredconf=1 option to treat this condition as a warning, not an error."; + return ConfigError{ConfigStatus::FAILED, Untranslated(error)}; + } } } diff --git a/src/common/netif.cpp b/src/common/netif.cpp new file mode 100644 index 0000000000..08f034a412 --- /dev/null +++ b/src/common/netif.cpp @@ -0,0 +1,303 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include <bitcoin-build-config.h> // IWYU pragma: keep + +#include <common/netif.h> + +#include <logging.h> +#include <netbase.h> +#include <util/check.h> +#include <util/sock.h> +#include <util/syserror.h> + +#if defined(__linux__) +#include <linux/rtnetlink.h> +#elif defined(__FreeBSD__) +#include <osreldate.h> +#if __FreeBSD_version >= 1400000 +// Workaround https://github.com/freebsd/freebsd-src/pull/1070. +#define typeof __typeof +#include <netlink/netlink.h> +#include <netlink/netlink_route.h> +#endif +#elif defined(WIN32) +#include <iphlpapi.h> +#elif defined(__APPLE__) +#include <net/route.h> +#include <sys/sysctl.h> +#endif + +namespace { + +// Linux and FreeBSD 14.0+. For FreeBSD 13.2 the code can be compiled but +// running it requires loading a special kernel module, otherwise socket(AF_NETLINK,...) +// will fail, so we skip that. +#if defined(__linux__) || (defined(__FreeBSD__) && __FreeBSD_version >= 1400000) + +std::optional<CNetAddr> QueryDefaultGatewayImpl(sa_family_t family) +{ + // Create a netlink socket. + auto sock{CreateSock(AF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE)}; + if (!sock) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "socket(AF_NETLINK): %s\n", NetworkErrorString(errno)); + return std::nullopt; + } + + // Send request. + struct { + nlmsghdr hdr; ///< Request header. + rtmsg data; ///< Request data, a "route message". + nlattr dst_hdr; ///< One attribute, conveying the route destination address. + char dst_data[16]; ///< Route destination address. To query the default route we use 0.0.0.0/0 or [::]/0. For IPv4 the first 4 bytes are used. + } request{}; + + // Whether to use the first 4 or 16 bytes from request.dst_data. + const size_t dst_data_len = family == AF_INET ? 4 : 16; + + request.hdr.nlmsg_type = RTM_GETROUTE; + request.hdr.nlmsg_flags = NLM_F_REQUEST; +#ifdef __linux__ + // Linux IPv4 / IPv6 - this must be present, otherwise no gateway is found + // FreeBSD IPv4 - does not matter, the gateway is found with or without this + // FreeBSD IPv6 - this must be absent, otherwise no gateway is found + request.hdr.nlmsg_flags |= NLM_F_DUMP; +#endif + request.hdr.nlmsg_len = NLMSG_LENGTH(sizeof(rtmsg) + sizeof(nlattr) + dst_data_len); + request.hdr.nlmsg_seq = 0; // Sequence number, used to match which reply is to which request. Irrelevant for us because we send just one request. + request.data.rtm_family = family; + request.data.rtm_dst_len = 0; // Prefix length. +#ifdef __FreeBSD__ + // Linux IPv4 / IPv6 this must be absent, otherwise no gateway is found + // FreeBSD IPv4 - does not matter, the gateway is found with or without this + // FreeBSD IPv6 - this must be present, otherwise no gateway is found + request.data.rtm_flags = RTM_F_PREFIX; +#endif + request.dst_hdr.nla_type = RTA_DST; + request.dst_hdr.nla_len = sizeof(nlattr) + dst_data_len; + + if (sock->Send(&request, request.hdr.nlmsg_len, 0) != static_cast<ssize_t>(request.hdr.nlmsg_len)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "send() to netlink socket: %s\n", NetworkErrorString(errno)); + return std::nullopt; + } + + // Receive response. + char response[4096]; + int64_t recv_result; + do { + recv_result = sock->Recv(response, sizeof(response), 0); + } while (recv_result < 0 && (errno == EINTR || errno == EAGAIN)); + if (recv_result < 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "recv() from netlink socket: %s\n", NetworkErrorString(errno)); + return std::nullopt; + } + + for (nlmsghdr* hdr = (nlmsghdr*)response; NLMSG_OK(hdr, recv_result); hdr = NLMSG_NEXT(hdr, recv_result)) { + rtmsg* r = (rtmsg*)NLMSG_DATA(hdr); + int remaining_len = RTM_PAYLOAD(hdr); + + // Iterate over the attributes. + rtattr *rta_gateway = nullptr; + int scope_id = 0; + for (rtattr* attr = RTM_RTA(r); RTA_OK(attr, remaining_len); attr = RTA_NEXT(attr, remaining_len)) { + if (attr->rta_type == RTA_GATEWAY) { + rta_gateway = attr; + } else if (attr->rta_type == RTA_OIF && sizeof(int) == RTA_PAYLOAD(attr)) { + std::memcpy(&scope_id, RTA_DATA(attr), sizeof(scope_id)); + } + } + + // Found gateway? + if (rta_gateway != nullptr) { + if (family == AF_INET && sizeof(in_addr) == RTA_PAYLOAD(rta_gateway)) { + in_addr gw; + std::memcpy(&gw, RTA_DATA(rta_gateway), sizeof(gw)); + return CNetAddr(gw); + } else if (family == AF_INET6 && sizeof(in6_addr) == RTA_PAYLOAD(rta_gateway)) { + in6_addr gw; + std::memcpy(&gw, RTA_DATA(rta_gateway), sizeof(gw)); + return CNetAddr(gw, scope_id); + } + } + } + + return std::nullopt; +} + +#elif defined(WIN32) + +std::optional<CNetAddr> QueryDefaultGatewayImpl(sa_family_t family) +{ + NET_LUID interface_luid = {}; + SOCKADDR_INET destination_address = {}; + MIB_IPFORWARD_ROW2 best_route = {}; + SOCKADDR_INET best_source_address = {}; + DWORD best_if_idx = 0; + DWORD status = 0; + + // Pass empty destination address of the requested type (:: or 0.0.0.0) to get interface of default route. + destination_address.si_family = family; + status = GetBestInterfaceEx((sockaddr*)&destination_address, &best_if_idx); + if (status != NO_ERROR) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Could not get best interface for default route: %s\n", NetworkErrorString(status)); + return std::nullopt; + } + + // Get best route to default gateway. + // Leave interface_luid at all-zeros to use interface index instead. + status = GetBestRoute2(&interface_luid, best_if_idx, nullptr, &destination_address, 0, &best_route, &best_source_address); + if (status != NO_ERROR) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Could not get best route for default route for interface index %d: %s\n", + best_if_idx, NetworkErrorString(status)); + return std::nullopt; + } + + Assume(best_route.NextHop.si_family == family); + if (family == AF_INET) { + return CNetAddr(best_route.NextHop.Ipv4.sin_addr); + } else if(family == AF_INET6) { + return CNetAddr(best_route.NextHop.Ipv6.sin6_addr, best_route.InterfaceIndex); + } + return std::nullopt; +} + +#elif defined(__APPLE__) + +#define ROUNDUP32(a) \ + ((a) > 0 ? (1 + (((a) - 1) | (sizeof(uint32_t) - 1))) : sizeof(uint32_t)) + +std::optional<CNetAddr> FromSockAddr(const struct sockaddr* addr) +{ + // Check valid length. Note that sa_len is not part of POSIX, and exists on MacOS and some BSDs only, so we can't + // do this check in SetSockAddr. + if (!(addr->sa_family == AF_INET && addr->sa_len == sizeof(struct sockaddr_in)) && + !(addr->sa_family == AF_INET6 && addr->sa_len == sizeof(struct sockaddr_in6))) { + return std::nullopt; + } + + // Fill in a CService from the sockaddr, then drop the port part. + CService service; + if (service.SetSockAddr(addr)) { + return (CNetAddr)service; + } + return std::nullopt; +} + +//! MacOS: Get default gateway from route table. See route(4) for the format. +std::optional<CNetAddr> QueryDefaultGatewayImpl(sa_family_t family) +{ + // net.route.0.inet[6].flags.gateway + int mib[] = {CTL_NET, PF_ROUTE, 0, family, NET_RT_FLAGS, RTF_GATEWAY}; + // The size of the available data is determined by calling sysctl() with oldp=nullptr. See sysctl(3). + size_t l = 0; + if (sysctl(/*name=*/mib, /*namelen=*/sizeof(mib) / sizeof(int), /*oldp=*/nullptr, /*oldlenp=*/&l, /*newp=*/nullptr, /*newlen=*/0) < 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Could not get sysctl length of routing table: %s\n", SysErrorString(errno)); + return std::nullopt; + } + std::vector<std::byte> buf(l); + if (sysctl(/*name=*/mib, /*namelen=*/sizeof(mib) / sizeof(int), /*oldp=*/buf.data(), /*oldlenp=*/&l, /*newp=*/nullptr, /*newlen=*/0) < 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Could not get sysctl data of routing table: %s\n", SysErrorString(errno)); + return std::nullopt; + } + // Iterate over messages (each message is a routing table entry). + for (size_t msg_pos = 0; msg_pos < buf.size(); ) { + if ((msg_pos + sizeof(rt_msghdr)) > buf.size()) return std::nullopt; + const struct rt_msghdr* rt = (const struct rt_msghdr*)(buf.data() + msg_pos); + const size_t next_msg_pos = msg_pos + rt->rtm_msglen; + if (rt->rtm_msglen < sizeof(rt_msghdr) || next_msg_pos > buf.size()) return std::nullopt; + // Iterate over addresses within message, get destination and gateway (if present). + // Address data starts after header. + size_t sa_pos = msg_pos + sizeof(struct rt_msghdr); + std::optional<CNetAddr> dst, gateway; + for (int i = 0; i < RTAX_MAX; i++) { + if (rt->rtm_addrs & (1 << i)) { + // 2 is just sa_len + sa_family, the theoretical minimum size of a socket address. + if ((sa_pos + 2) > next_msg_pos) return std::nullopt; + const struct sockaddr* sa = (const struct sockaddr*)(buf.data() + sa_pos); + if ((sa_pos + sa->sa_len) > next_msg_pos) return std::nullopt; + if (i == RTAX_DST) { + dst = FromSockAddr(sa); + } else if (i == RTAX_GATEWAY) { + gateway = FromSockAddr(sa); + } + // Skip sockaddr entries for bit flags we're not interested in, + // move cursor. + sa_pos += ROUNDUP32(sa->sa_len); + } + } + // Found default gateway? + if (dst && gateway && dst->IsBindAny()) { // Route to 0.0.0.0 or :: ? + return *gateway; + } + // Skip to next message. + msg_pos = next_msg_pos; + } + return std::nullopt; +} + +#else + +// Dummy implementation. +std::optional<CNetAddr> QueryDefaultGatewayImpl(sa_family_t) +{ + return std::nullopt; +} + +#endif + +} + +std::optional<CNetAddr> QueryDefaultGateway(Network network) +{ + Assume(network == NET_IPV4 || network == NET_IPV6); + + sa_family_t family; + if (network == NET_IPV4) { + family = AF_INET; + } else if(network == NET_IPV6) { + family = AF_INET6; + } else { + return std::nullopt; + } + + std::optional<CNetAddr> ret = QueryDefaultGatewayImpl(family); + + // It's possible for the default gateway to be 0.0.0.0 or ::0 on at least Windows + // for some routing strategies. If so, return as if no default gateway was found. + if (ret && !ret->IsBindAny()) { + return ret; + } else { + return std::nullopt; + } +} + +std::vector<CNetAddr> GetLocalAddresses() +{ + std::vector<CNetAddr> addresses; +#ifdef WIN32 + char pszHostName[256] = ""; + if (gethostname(pszHostName, sizeof(pszHostName)) != SOCKET_ERROR) { + addresses = LookupHost(pszHostName, 0, true); + } +#elif (HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS) + struct ifaddrs* myaddrs; + if (getifaddrs(&myaddrs) == 0) { + for (struct ifaddrs* ifa = myaddrs; ifa != nullptr; ifa = ifa->ifa_next) + { + if (ifa->ifa_addr == nullptr) continue; + if ((ifa->ifa_flags & IFF_UP) == 0) continue; + if ((ifa->ifa_flags & IFF_LOOPBACK) != 0) continue; + if (ifa->ifa_addr->sa_family == AF_INET) { + struct sockaddr_in* s4 = (struct sockaddr_in*)(ifa->ifa_addr); + addresses.emplace_back(s4->sin_addr); + } else if (ifa->ifa_addr->sa_family == AF_INET6) { + struct sockaddr_in6* s6 = (struct sockaddr_in6*)(ifa->ifa_addr); + addresses.emplace_back(s6->sin6_addr); + } + } + freeifaddrs(myaddrs); + } +#endif + return addresses; +} diff --git a/src/common/netif.h b/src/common/netif.h new file mode 100644 index 0000000000..55bc023be6 --- /dev/null +++ b/src/common/netif.h @@ -0,0 +1,19 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMMON_NETIF_H +#define BITCOIN_COMMON_NETIF_H + +#include <netaddress.h> + +#include <optional> + +//! Query the OS for the default gateway for `network`. This only makes sense for NET_IPV4 and NET_IPV6. +//! Returns std::nullopt if it cannot be found, or there is no support for this OS. +std::optional<CNetAddr> QueryDefaultGateway(Network network); + +//! Return all local non-loopback IPv4 and IPv6 network addresses. +std::vector<CNetAddr> GetLocalAddresses(); + +#endif // BITCOIN_COMMON_NETIF_H diff --git a/src/common/pcp.cpp b/src/common/pcp.cpp new file mode 100644 index 0000000000..3cc1cba924 --- /dev/null +++ b/src/common/pcp.cpp @@ -0,0 +1,524 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include <common/pcp.h> + +#include <common/netif.h> +#include <crypto/common.h> +#include <logging.h> +#include <netaddress.h> +#include <netbase.h> +#include <random.h> +#include <span.h> +#include <util/check.h> +#include <util/readwritefile.h> +#include <util/sock.h> +#include <util/strencodings.h> + +namespace { + +// RFC6886 NAT-PMP and RFC6887 Port Control Protocol (PCP) implementation. +// NAT-PMP and PCP use network byte order (big-endian). + +// NAT-PMP (v0) protocol constants. +//! NAT-PMP uses a fixed server port number (RFC6887 section 1.1). +constexpr uint16_t NATPMP_SERVER_PORT = 5351; +//! Version byte for NATPMP (RFC6886 1.1) +constexpr uint8_t NATPMP_VERSION = 0; +//! Request opcode base (RFC6886 3). +constexpr uint8_t NATPMP_REQUEST = 0x00; +//! Response opcode base (RFC6886 3). +constexpr uint8_t NATPMP_RESPONSE = 0x80; +//! Get external address (RFC6886 3.2) +constexpr uint8_t NATPMP_OP_GETEXTERNAL = 0x00; +//! Map TCP port (RFC6886 3.3) +constexpr uint8_t NATPMP_OP_MAP_TCP = 0x02; +//! Shared request header size in bytes. +constexpr size_t NATPMP_REQUEST_HDR_SIZE = 2; +//! Shared response header (minimum) size in bytes. +constexpr size_t NATPMP_RESPONSE_HDR_SIZE = 8; +//! GETEXTERNAL request size in bytes, including header (RFC6886 3.2). +constexpr size_t NATPMP_GETEXTERNAL_REQUEST_SIZE = NATPMP_REQUEST_HDR_SIZE + 0; +//! GETEXTERNAL response size in bytes, including header (RFC6886 3.2). +constexpr size_t NATPMP_GETEXTERNAL_RESPONSE_SIZE = NATPMP_RESPONSE_HDR_SIZE + 4; +//! MAP request size in bytes, including header (RFC6886 3.3). +constexpr size_t NATPMP_MAP_REQUEST_SIZE = NATPMP_REQUEST_HDR_SIZE + 10; +//! MAP response size in bytes, including header (RFC6886 3.3). +constexpr size_t NATPMP_MAP_RESPONSE_SIZE = NATPMP_RESPONSE_HDR_SIZE + 8; + +// Shared header offsets (RFC6886 3.2, 3.3), relative to start of packet. +//! Offset of version field in packets. +constexpr size_t NATPMP_HDR_VERSION_OFS = 0; +//! Offset of opcode field in packets +constexpr size_t NATPMP_HDR_OP_OFS = 1; +//! Offset of result code in packets. Result codes are 16 bit in NAT-PMP instead of 8 bit in PCP. +constexpr size_t NATPMP_RESPONSE_HDR_RESULT_OFS = 2; + +// GETEXTERNAL response offsets (RFC6886 3.2), relative to start of packet. +//! Returned external address +constexpr size_t NATPMP_GETEXTERNAL_RESPONSE_IP_OFS = 8; + +// MAP request offsets (RFC6886 3.3), relative to start of packet. +//! Internal port to be mapped. +constexpr size_t NATPMP_MAP_REQUEST_INTERNAL_PORT_OFS = 4; +//! Suggested external port for mapping. +constexpr size_t NATPMP_MAP_REQUEST_EXTERNAL_PORT_OFS = 6; +//! Requested port mapping lifetime in seconds. +constexpr size_t NATPMP_MAP_REQUEST_LIFETIME_OFS = 8; + +// MAP response offsets (RFC6886 3.3), relative to start of packet. +//! Internal port for mapping (will match internal port of request). +constexpr size_t NATPMP_MAP_RESPONSE_INTERNAL_PORT_OFS = 8; +//! External port for mapping. +constexpr size_t NATPMP_MAP_RESPONSE_EXTERNAL_PORT_OFS = 10; +//! Created port mapping lifetime in seconds. +constexpr size_t NATPMP_MAP_RESPONSE_LIFETIME_OFS = 12; + +// Relevant NETPMP result codes (RFC6886 3.5). +//! Result code representing success status. +constexpr uint8_t NATPMP_RESULT_SUCCESS = 0; +//! Result code representing unsupported version. +constexpr uint8_t NATPMP_RESULT_UNSUPP_VERSION = 1; +//! Result code representing lack of resources. +constexpr uint8_t NATPMP_RESULT_NO_RESOURCES = 4; + +//! Mapping of NATPMP result code to string (RFC6886 3.5). Result codes <=2 match PCP. +const std::map<uint8_t, std::string> NATPMP_RESULT_STR{ + {0, "SUCCESS"}, + {1, "UNSUPP_VERSION"}, + {2, "NOT_AUTHORIZED"}, + {3, "NETWORK_FAILURE"}, + {4, "NO_RESOURCES"}, + {5, "UNSUPP_OPCODE"}, +}; + +// PCP (v2) protocol constants. +//! Maximum packet size in bytes (RFC6887 section 7). +constexpr size_t PCP_MAX_SIZE = 1100; +//! PCP uses a fixed server port number (RFC6887 section 19.1). Shared with NAT-PMP. +constexpr uint16_t PCP_SERVER_PORT = NATPMP_SERVER_PORT; +//! Version byte. 0 is NAT-PMP (RFC6886), 1 is forbidden, 2 for PCP (RFC6887). +constexpr uint8_t PCP_VERSION = 2; +//! PCP Request Header. See RFC6887 section 7.1. Shared with NAT-PMP. +constexpr uint8_t PCP_REQUEST = NATPMP_REQUEST; // R = 0 +//! PCP Response Header. See RFC6887 section 7.2. Shared with NAT-PMP. +constexpr uint8_t PCP_RESPONSE = NATPMP_RESPONSE; // R = 1 +//! Map opcode. See RFC6887 section 19.2 +constexpr uint8_t PCP_OP_MAP = 0x01; +//! TCP protocol number (IANA). +constexpr uint16_t PCP_PROTOCOL_TCP = 6; +//! Request and response header size in bytes (RFC6887 section 7.1). +constexpr size_t PCP_HDR_SIZE = 24; +//! Map request and response size in bytes (RFC6887 section 11.1). +constexpr size_t PCP_MAP_SIZE = 36; + +// Header offsets shared between request and responses (RFC6887 7.1, 7.2), relative to start of packet. +//! Version field (1 byte). +constexpr size_t PCP_HDR_VERSION_OFS = NATPMP_HDR_VERSION_OFS; +//! Opcode field (1 byte). +constexpr size_t PCP_HDR_OP_OFS = NATPMP_HDR_OP_OFS; +//! Requested lifetime (request), granted lifetime (response) (4 bytes). +constexpr size_t PCP_HDR_LIFETIME_OFS = 4; + +// Request header offsets (RFC6887 7.1), relative to start of packet. +//! PCP client's IP address (16 bytes). +constexpr size_t PCP_REQUEST_HDR_IP_OFS = 8; + +// Response header offsets (RFC6887 7.2), relative to start of packet. +//! Result code (1 byte). +constexpr size_t PCP_RESPONSE_HDR_RESULT_OFS = 3; + +// MAP request/response offsets (RFC6887 11.1), relative to start of opcode-specific data. +//! Mapping nonce (12 bytes). +constexpr size_t PCP_MAP_NONCE_OFS = 0; +//! Protocol (1 byte). +constexpr size_t PCP_MAP_PROTOCOL_OFS = 12; +//! Internal port for mapping (2 bytes). +constexpr size_t PCP_MAP_INTERNAL_PORT_OFS = 16; +//! Suggested external port (request), assigned external port (response) (2 bytes). +constexpr size_t PCP_MAP_EXTERNAL_PORT_OFS = 18; +//! Suggested external IP (request), assigned external IP (response) (16 bytes). +constexpr size_t PCP_MAP_EXTERNAL_IP_OFS = 20; + +//! Result code representing success (RFC6887 7.4), shared with NAT-PMP. +constexpr uint8_t PCP_RESULT_SUCCESS = NATPMP_RESULT_SUCCESS; +//! Result code representing lack of resources (RFC6887 7.4). +constexpr uint8_t PCP_RESULT_NO_RESOURCES = 8; + +//! Mapping of PCP result code to string (RFC6887 7.4). Result codes <=2 match NAT-PMP. +const std::map<uint8_t, std::string> PCP_RESULT_STR{ + {0, "SUCCESS"}, + {1, "UNSUPP_VERSION"}, + {2, "NOT_AUTHORIZED"}, + {3, "MALFORMED_REQUEST"}, + {4, "UNSUPP_OPCODE"}, + {5, "UNSUPP_OPTION"}, + {6, "MALFORMED_OPTION"}, + {7, "NETWORK_FAILURE"}, + {8, "NO_RESOURCES"}, + {9, "UNSUPP_PROTOCOL"}, + {10, "USER_EX_QUOTA"}, + {11, "CANNOT_PROVIDE_EXTERNAL"}, + {12, "ADDRESS_MISMATCH"}, + {13, "EXCESSIVE_REMOTE_PEER"}, +}; + +//! Return human-readable string from NATPMP result code. +std::string NATPMPResultString(uint8_t result_code) +{ + auto result_i = NATPMP_RESULT_STR.find(result_code); + return strprintf("%s (code %d)", result_i == NATPMP_RESULT_STR.end() ? "(unknown)" : result_i->second, result_code); +} + +//! Return human-readable string from PCP result code. +std::string PCPResultString(uint8_t result_code) +{ + auto result_i = PCP_RESULT_STR.find(result_code); + return strprintf("%s (code %d)", result_i == PCP_RESULT_STR.end() ? "(unknown)" : result_i->second, result_code); +} + +//! Wrap address in IPv6 according to RFC6887. wrapped_addr needs to be able to store 16 bytes. +[[nodiscard]] bool PCPWrapAddress(Span<uint8_t> wrapped_addr, const CNetAddr &addr) +{ + Assume(wrapped_addr.size() == ADDR_IPV6_SIZE); + if (addr.IsIPv4()) { + struct in_addr addr4; + if (!addr.GetInAddr(&addr4)) return false; + // Section 5: "When the address field holds an IPv4 address, an IPv4-mapped IPv6 address [RFC4291] is used (::ffff:0:0/96)." + std::memcpy(wrapped_addr.data(), IPV4_IN_IPV6_PREFIX.data(), IPV4_IN_IPV6_PREFIX.size()); + std::memcpy(wrapped_addr.data() + IPV4_IN_IPV6_PREFIX.size(), &addr4, ADDR_IPV4_SIZE); + return true; + } else if (addr.IsIPv6()) { + struct in6_addr addr6; + if (!addr.GetIn6Addr(&addr6)) return false; + std::memcpy(wrapped_addr.data(), &addr6, ADDR_IPV6_SIZE); + return true; + } else { + return false; + } +} + +//! Unwrap PCP-encoded address according to RFC6887. +CNetAddr PCPUnwrapAddress(Span<const uint8_t> wrapped_addr) +{ + Assume(wrapped_addr.size() == ADDR_IPV6_SIZE); + if (util::HasPrefix(wrapped_addr, IPV4_IN_IPV6_PREFIX)) { + struct in_addr addr4; + std::memcpy(&addr4, wrapped_addr.data() + IPV4_IN_IPV6_PREFIX.size(), ADDR_IPV4_SIZE); + return CNetAddr(addr4); + } else { + struct in6_addr addr6; + std::memcpy(&addr6, wrapped_addr.data(), ADDR_IPV6_SIZE); + return CNetAddr(addr6); + } +} + +//! PCP or NAT-PMP send-receive loop. +std::optional<std::vector<uint8_t>> PCPSendRecv(Sock &sock, const std::string &protocol, Span<const uint8_t> request, int num_tries, + std::chrono::milliseconds timeout_per_try, + std::function<bool(Span<const uint8_t>)> check_packet) +{ + using namespace std::chrono; + // UDP is a potentially lossy protocol, so we try to send again a few times. + uint8_t response[PCP_MAX_SIZE]; + bool got_response = false; + int recvsz = 0; + for (int ntry = 0; !got_response && ntry < num_tries; ++ntry) { + if (ntry > 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "%s: Retrying (%d)\n", protocol, ntry); + } + // Dispatch packet to gateway. + if (sock.Send(request.data(), request.size(), 0) != static_cast<ssize_t>(request.size())) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "%s: Could not send request: %s\n", protocol, NetworkErrorString(WSAGetLastError())); + return std::nullopt; // Network-level error, probably no use retrying. + } + + // Wait for response(s) until we get a valid response, a network error, or time out. + auto cur_time = time_point_cast<milliseconds>(steady_clock::now()); + auto deadline = cur_time + timeout_per_try; + while ((cur_time = time_point_cast<milliseconds>(steady_clock::now())) < deadline) { + Sock::Event occurred = 0; + if (!sock.Wait(deadline - cur_time, Sock::RECV, &occurred)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "%s: Could not wait on socket: %s\n", protocol, NetworkErrorString(WSAGetLastError())); + return std::nullopt; // Network-level error, probably no use retrying. + } + if (!occurred) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "%s: Timeout\n", protocol); + break; // Retry. + } + + // Receive response. + recvsz = sock.Recv(response, sizeof(response), MSG_DONTWAIT); + if (recvsz < 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "%s: Could not receive response: %s\n", protocol, NetworkErrorString(WSAGetLastError())); + return std::nullopt; // Network-level error, probably no use retrying. + } + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "%s: Received response of %d bytes: %s\n", protocol, recvsz, HexStr(Span(response, recvsz))); + + if (check_packet(Span<uint8_t>(response, recvsz))) { + got_response = true; // Got expected response, break from receive loop as well as from retry loop. + break; + } + } + } + if (!got_response) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "%s: Giving up after %d tries\n", protocol, num_tries); + return std::nullopt; + } + return std::vector<uint8_t>(response, response + recvsz); +} + +} + +std::variant<MappingResult, MappingError> NATPMPRequestPortMap(const CNetAddr &gateway, uint16_t port, uint32_t lifetime, int num_tries, std::chrono::milliseconds timeout_per_try) +{ + struct sockaddr_storage dest_addr; + socklen_t dest_addrlen = sizeof(struct sockaddr_storage); + + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "natpmp: Requesting port mapping port %d from gateway %s\n", port, gateway.ToStringAddr()); + + // Validate gateway, make sure it's IPv4. NAT-PMP does not support IPv6. + if (!CService(gateway, PCP_SERVER_PORT).GetSockAddr((struct sockaddr*)&dest_addr, &dest_addrlen)) return MappingError::NETWORK_ERROR; + if (dest_addr.ss_family != AF_INET) return MappingError::NETWORK_ERROR; + + // Create IPv4 UDP socket + auto sock{CreateSock(AF_INET, SOCK_DGRAM, IPPROTO_UDP)}; + if (!sock) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Could not create UDP socket: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + + // Associate UDP socket to gateway. + if (sock->Connect((struct sockaddr*)&dest_addr, dest_addrlen) != 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Could not connect to gateway: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + + // Use getsockname to get the address toward the default gateway (the internal address). + struct sockaddr_in internal; + socklen_t internal_addrlen = sizeof(struct sockaddr_in); + if (sock->GetSockName((struct sockaddr*)&internal, &internal_addrlen) != 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Could not get sock name: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + + // Request external IP address (RFC6886 section 3.2). + std::vector<uint8_t> request(NATPMP_GETEXTERNAL_REQUEST_SIZE); + request[NATPMP_HDR_VERSION_OFS] = NATPMP_VERSION; + request[NATPMP_HDR_OP_OFS] = NATPMP_REQUEST | NATPMP_OP_GETEXTERNAL; + + auto recv_res = PCPSendRecv(*sock, "natpmp", request, num_tries, timeout_per_try, + [&](const Span<const uint8_t> response) -> bool { + if (response.size() < NATPMP_GETEXTERNAL_RESPONSE_SIZE) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Response too small\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + if (response[NATPMP_HDR_VERSION_OFS] != NATPMP_VERSION || response[NATPMP_HDR_OP_OFS] != (NATPMP_RESPONSE | NATPMP_OP_GETEXTERNAL)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Response to wrong command\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + return true; + }); + + struct in_addr external_addr; + if (recv_res) { + const std::span<const uint8_t> response = *recv_res; + + Assume(response.size() >= NATPMP_GETEXTERNAL_RESPONSE_SIZE); + uint16_t result_code = ReadBE16(response.data() + NATPMP_RESPONSE_HDR_RESULT_OFS); + if (result_code != NATPMP_RESULT_SUCCESS) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Getting external address failed with result %s\n", NATPMPResultString(result_code)); + return MappingError::PROTOCOL_ERROR; + } + + std::memcpy(&external_addr, response.data() + NATPMP_GETEXTERNAL_RESPONSE_IP_OFS, ADDR_IPV4_SIZE); + } else { + return MappingError::NETWORK_ERROR; + } + + // Create TCP mapping request (RFC6886 section 3.3). + request = std::vector<uint8_t>(NATPMP_MAP_REQUEST_SIZE); + request[NATPMP_HDR_VERSION_OFS] = NATPMP_VERSION; + request[NATPMP_HDR_OP_OFS] = NATPMP_REQUEST | NATPMP_OP_MAP_TCP; + WriteBE16(request.data() + NATPMP_MAP_REQUEST_INTERNAL_PORT_OFS, port); + WriteBE16(request.data() + NATPMP_MAP_REQUEST_EXTERNAL_PORT_OFS, port); + WriteBE32(request.data() + NATPMP_MAP_REQUEST_LIFETIME_OFS, lifetime); + + recv_res = PCPSendRecv(*sock, "natpmp", request, num_tries, timeout_per_try, + [&](const Span<const uint8_t> response) -> bool { + if (response.size() < NATPMP_MAP_RESPONSE_SIZE) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Response too small\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + if (response[0] != NATPMP_VERSION || response[1] != (NATPMP_RESPONSE | NATPMP_OP_MAP_TCP)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Response to wrong command\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + uint16_t internal_port = ReadBE16(response.data() + NATPMP_MAP_RESPONSE_INTERNAL_PORT_OFS); + if (internal_port != port) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Response port doesn't match request\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + return true; + }); + + if (recv_res) { + const std::span<uint8_t> response = *recv_res; + + Assume(response.size() >= NATPMP_MAP_RESPONSE_SIZE); + uint16_t result_code = ReadBE16(response.data() + NATPMP_RESPONSE_HDR_RESULT_OFS); + if (result_code != NATPMP_RESULT_SUCCESS) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Port mapping failed with result %s\n", NATPMPResultString(result_code)); + if (result_code == NATPMP_RESULT_NO_RESOURCES) { + return MappingError::NO_RESOURCES; + } + return MappingError::PROTOCOL_ERROR; + } + + uint32_t lifetime_ret = ReadBE32(response.data() + NATPMP_MAP_RESPONSE_LIFETIME_OFS); + uint16_t external_port = ReadBE16(response.data() + NATPMP_MAP_RESPONSE_EXTERNAL_PORT_OFS); + return MappingResult(NATPMP_VERSION, CService(internal.sin_addr, port), CService(external_addr, external_port), lifetime_ret); + } else { + return MappingError::NETWORK_ERROR; + } +} + +std::variant<MappingResult, MappingError> PCPRequestPortMap(const PCPMappingNonce &nonce, const CNetAddr &gateway, const CNetAddr &bind, uint16_t port, uint32_t lifetime, int num_tries, std::chrono::milliseconds timeout_per_try) +{ + struct sockaddr_storage dest_addr, bind_addr; + socklen_t dest_addrlen = sizeof(struct sockaddr_storage), bind_addrlen = sizeof(struct sockaddr_storage); + + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "pcp: Requesting port mapping for addr %s port %d from gateway %s\n", bind.ToStringAddr(), port, gateway.ToStringAddr()); + + // Validate addresses, make sure they're the same network family. + if (!CService(gateway, PCP_SERVER_PORT).GetSockAddr((struct sockaddr*)&dest_addr, &dest_addrlen)) return MappingError::NETWORK_ERROR; + if (!CService(bind, 0).GetSockAddr((struct sockaddr*)&bind_addr, &bind_addrlen)) return MappingError::NETWORK_ERROR; + if (dest_addr.ss_family != bind_addr.ss_family) return MappingError::NETWORK_ERROR; + + // Create UDP socket (IPv4 or IPv6 based on provided gateway). + auto sock{CreateSock(dest_addr.ss_family, SOCK_DGRAM, IPPROTO_UDP)}; + if (!sock) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Could not create UDP socket: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + + // Make sure that we send from requested destination address, anything else will be + // rejected by a security-conscious router. + if (sock->Bind((struct sockaddr*)&bind_addr, bind_addrlen) != 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Could not bind to address: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + + // Associate UDP socket to gateway. + if (sock->Connect((struct sockaddr*)&dest_addr, dest_addrlen) != 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Could not connect to gateway: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + + // Use getsockname to get the address toward the default gateway (the internal address), + // in case we don't know what address to map + // (this is only needed if bind is INADDR_ANY, but it doesn't hurt as an extra check). + struct sockaddr_storage internal_addr; + socklen_t internal_addrlen = sizeof(struct sockaddr_storage); + if (sock->GetSockName((struct sockaddr*)&internal_addr, &internal_addrlen) != 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Could not get sock name: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + CService internal; + if (!internal.SetSockAddr((struct sockaddr*)&internal_addr)) return MappingError::NETWORK_ERROR; + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "pcp: Internal address after connect: %s\n", internal.ToStringAddr()); + + // Build request packet. Make sure the packet is zeroed so that reserved fields are zero + // as required by the spec (and not potentially leak data). + // Make sure there's space for the request header and MAP specific request data. + std::vector<uint8_t> request(PCP_HDR_SIZE + PCP_MAP_SIZE); + // Fill in request header, See RFC6887 Figure 2. + size_t ofs = 0; + request[ofs + PCP_HDR_VERSION_OFS] = PCP_VERSION; + request[ofs + PCP_HDR_OP_OFS] = PCP_REQUEST | PCP_OP_MAP; + WriteBE32(request.data() + ofs + PCP_HDR_LIFETIME_OFS, lifetime); + if (!PCPWrapAddress(Span(request).subspan(ofs + PCP_REQUEST_HDR_IP_OFS, ADDR_IPV6_SIZE), internal)) return MappingError::NETWORK_ERROR; + + ofs += PCP_HDR_SIZE; + + // Fill in MAP request packet, See RFC6887 Figure 9. + // Randomize mapping nonce (this is repeated in the response, to be able to + // correlate requests and responses, and used to authenticate changes to the mapping). + std::memcpy(request.data() + ofs + PCP_MAP_NONCE_OFS, nonce.data(), PCP_MAP_NONCE_SIZE); + request[ofs + PCP_MAP_PROTOCOL_OFS] = PCP_PROTOCOL_TCP; + WriteBE16(request.data() + ofs + PCP_MAP_INTERNAL_PORT_OFS, port); + WriteBE16(request.data() + ofs + PCP_MAP_EXTERNAL_PORT_OFS, port); + if (!PCPWrapAddress(Span(request).subspan(ofs + PCP_MAP_EXTERNAL_IP_OFS, ADDR_IPV6_SIZE), bind)) return MappingError::NETWORK_ERROR; + + ofs += PCP_MAP_SIZE; + Assume(ofs == request.size()); + + // Receive loop. + bool is_natpmp = false; + auto recv_res = PCPSendRecv(*sock, "pcp", request, num_tries, timeout_per_try, + [&](const Span<const uint8_t> response) -> bool { + // Unsupported version according to RFC6887 appendix A and RFC6886 section 3.5, can fall back to NAT-PMP. + if (response.size() == NATPMP_RESPONSE_HDR_SIZE && response[PCP_HDR_VERSION_OFS] == NATPMP_VERSION && response[PCP_RESPONSE_HDR_RESULT_OFS] == NATPMP_RESULT_UNSUPP_VERSION) { + is_natpmp = true; + return true; // Let it through to caller. + } + if (response.size() < (PCP_HDR_SIZE + PCP_MAP_SIZE)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Response too small\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + if (response[PCP_HDR_VERSION_OFS] != PCP_VERSION || response[PCP_HDR_OP_OFS] != (PCP_RESPONSE | PCP_OP_MAP)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Response to wrong command\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + // Handle MAP opcode response. See RFC6887 Figure 10. + // Check that returned mapping nonce matches our request. + if (!std::ranges::equal(response.subspan(PCP_HDR_SIZE + PCP_MAP_NONCE_OFS, PCP_MAP_NONCE_SIZE), nonce)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Mapping nonce mismatch\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + uint8_t protocol = response[PCP_HDR_SIZE + 12]; + uint16_t internal_port = ReadBE16(response.data() + PCP_HDR_SIZE + 16); + if (protocol != PCP_PROTOCOL_TCP || internal_port != port) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Response protocol or port doesn't match request\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + return true; + }); + + if (!recv_res) { + return MappingError::NETWORK_ERROR; + } + if (is_natpmp) { + return MappingError::UNSUPP_VERSION; + } + + const std::span<const uint8_t> response = *recv_res; + // If we get here, we got a valid MAP response to our request. + // Check to see if we got the result we expected. + Assume(response.size() >= (PCP_HDR_SIZE + PCP_MAP_SIZE)); + uint8_t result_code = response[PCP_RESPONSE_HDR_RESULT_OFS]; + uint32_t lifetime_ret = ReadBE32(response.data() + PCP_HDR_LIFETIME_OFS); + uint16_t external_port = ReadBE16(response.data() + PCP_HDR_SIZE + PCP_MAP_EXTERNAL_PORT_OFS); + CNetAddr external_addr{PCPUnwrapAddress(response.subspan(PCP_HDR_SIZE + PCP_MAP_EXTERNAL_IP_OFS, ADDR_IPV6_SIZE))}; + if (result_code != PCP_RESULT_SUCCESS) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Mapping failed with result %s\n", PCPResultString(result_code)); + if (result_code == PCP_RESULT_NO_RESOURCES) { + return MappingError::NO_RESOURCES; + } + return MappingError::PROTOCOL_ERROR; + } + + return MappingResult(PCP_VERSION, CService(internal, port), CService(external_addr, external_port), lifetime_ret); +} + +std::string MappingResult::ToString() +{ + Assume(version == NATPMP_VERSION || version == PCP_VERSION); + return strprintf("%s:%s -> %s (for %ds)", + version == NATPMP_VERSION ? "natpmp" : "pcp", + external.ToStringAddrPort(), + internal.ToStringAddrPort(), + lifetime + ); +} diff --git a/src/common/pcp.h b/src/common/pcp.h new file mode 100644 index 0000000000..ce2273e140 --- /dev/null +++ b/src/common/pcp.h @@ -0,0 +1,68 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMMON_PCP_H +#define BITCOIN_COMMON_PCP_H + +#include <netaddress.h> + +#include <variant> + +// RFC6886 NAT-PMP and RFC6887 Port Control Protocol (PCP) implementation. +// NAT-PMP and PCP use network byte order (big-endian). + +//! Mapping nonce size in bytes (see RFC6887 section 11.1). +constexpr size_t PCP_MAP_NONCE_SIZE = 12; + +//! PCP mapping nonce. Arbitrary data chosen by the client to identify a mapping. +typedef std::array<uint8_t, PCP_MAP_NONCE_SIZE> PCPMappingNonce; + +//! Unsuccessful response to a port mapping. +enum class MappingError { + NETWORK_ERROR, ///< Any kind of network-level error. + PROTOCOL_ERROR, ///< Any kind of protocol-level error, except unsupported version or no resources. + UNSUPP_VERSION, ///< Unsupported protocol version. + NO_RESOURCES, ///< No resources available (port probably already mapped). +}; + +//! Successful response to a port mapping. +struct MappingResult { + MappingResult(uint8_t version, const CService &internal_in, const CService &external_in, uint32_t lifetime_in): + version(version), internal(internal_in), external(external_in), lifetime(lifetime_in) {} + //! Protocol version, one of NATPMP_VERSION or PCP_VERSION. + uint8_t version; + //! Internal host:port. + CService internal; + //! External host:port. + CService external; + //! Granted lifetime of binding (seconds). + uint32_t lifetime; + + //! Format mapping as string for logging. + std::string ToString(); +}; + +//! Try to open a port using RFC 6886 NAT-PMP. IPv4 only. +//! +//! * gateway: Destination address for PCP requests (usually the default gateway). +//! * port: Internal port, and desired external port. +//! * lifetime: Requested lifetime in seconds for mapping. The server may assign as shorter or longer lifetime. A lifetime of 0 deletes the mapping. +//! * num_tries: Number of tries in case of no response. +//! +//! Returns the external_ip:external_port of the mapping if successful, otherwise a MappingError. +std::variant<MappingResult, MappingError> NATPMPRequestPortMap(const CNetAddr &gateway, uint16_t port, uint32_t lifetime, int num_tries = 3, std::chrono::milliseconds timeout_per_try = std::chrono::milliseconds(1000)); + +//! Try to open a port using RFC 6887 Port Control Protocol (PCP). Handles IPv4 and IPv6. +//! +//! * nonce: Mapping cookie. Keep this the same over renewals. +//! * gateway: Destination address for PCP requests (usually the default gateway). +//! * bind: Specific local bind address for IPv6 pinholing. Set this as INADDR_ANY for IPv4. +//! * port: Internal port, and desired external port. +//! * lifetime: Requested lifetime in seconds for mapping. The server may assign as shorter or longer lifetime. A lifetime of 0 deletes the mapping. +//! * num_tries: Number of tries in case of no response. +//! +//! Returns the external_ip:external_port of the mapping if successful, otherwise a MappingError. +std::variant<MappingResult, MappingError> PCPRequestPortMap(const PCPMappingNonce &nonce, const CNetAddr &gateway, const CNetAddr &bind, uint16_t port, uint32_t lifetime, int num_tries = 3, std::chrono::milliseconds timeout_per_try = std::chrono::milliseconds(1000)); + +#endif // BITCOIN_COMMON_PCP_H diff --git a/src/common/run_command.cpp b/src/common/run_command.cpp index 67608b985f..1f6d51b4f4 100644 --- a/src/common/run_command.cpp +++ b/src/common/run_command.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <common/run_command.h> diff --git a/src/common/settings.cpp b/src/common/settings.cpp index c1520dacd2..046afca15d 100644 --- a/src/common/settings.cpp +++ b/src/common/settings.cpp @@ -4,7 +4,7 @@ #include <common/settings.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <tinyformat.h> #include <univalue.h> @@ -127,7 +127,7 @@ bool WriteSettings(const fs::path& path, SettingsValue out(SettingsValue::VOBJ); // Add auto-generated warning comment out.pushKV(SETTINGS_WARN_MSG_KEY, strprintf("This file is automatically generated and updated by %s. Please do not edit this file while the node " - "is running, as any changes might be ignored or overwritten.", PACKAGE_NAME)); + "is running, as any changes might be ignored or overwritten.", CLIENT_NAME)); // Push settings values for (const auto& value : values) { out.pushKVEnd(value.first, value.second); diff --git a/src/common/system.cpp b/src/common/system.cpp index 6d04c8a7bc..7af792db44 100644 --- a/src/common/system.cpp +++ b/src/common/system.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <common/system.h> @@ -70,7 +70,7 @@ void SetupEnvironment() #endif // On most POSIX systems (e.g. Linux, but not BSD) the environment's locale // may be invalid, in which case the "C.UTF-8" locale is used as fallback. -#if !defined(WIN32) && !defined(MAC_OSX) && !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__) +#if !defined(WIN32) && !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__) try { std::locale(""); // Raises a runtime error if current locale is invalid } catch (const std::runtime_error&) { diff --git a/src/common/system.h b/src/common/system.h index d9115d3b33..a4b56be9ac 100644 --- a/src/common/system.h +++ b/src/common/system.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_COMMON_SYSTEM_H #define BITCOIN_COMMON_SYSTEM_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <cstdint> #include <string> diff --git a/src/config/.empty b/src/config/.empty deleted file mode 100644 index e69de29bb2..0000000000 --- a/src/config/.empty +++ /dev/null diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp index af01902c92..7dd24e1868 100644 --- a/src/consensus/merkle.cpp +++ b/src/consensus/merkle.cpp @@ -4,6 +4,7 @@ #include <consensus/merkle.h> #include <hash.h> +#include <util/check.h> /* WARNING! If you're reading this because you're learning about crypto and/or designing a new system that will use merkle trees, keep in mind @@ -83,3 +84,108 @@ uint256 BlockWitnessMerkleRoot(const CBlock& block, bool* mutated) return ComputeMerkleRoot(std::move(leaves), mutated); } +/* This implements a constant-space merkle root/path calculator, limited to 2^32 leaves. */ +static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot, bool* pmutated, uint32_t leaf_pos, std::vector<uint256>* path) +{ + if (path) path->clear(); + Assume(leaves.size() <= UINT32_MAX); + if (leaves.size() == 0) { + if (pmutated) *pmutated = false; + if (proot) *proot = uint256(); + return; + } + bool mutated = false; + // count is the number of leaves processed so far. + uint32_t count = 0; + // inner is an array of eagerly computed subtree hashes, indexed by tree + // level (0 being the leaves). + // For example, when count is 25 (11001 in binary), inner[4] is the hash of + // the first 16 leaves, inner[3] of the next 8 leaves, and inner[0] equal to + // the last leaf. The other inner entries are undefined. + uint256 inner[32]; + // Which position in inner is a hash that depends on the matching leaf. + int matchlevel = -1; + // First process all leaves into 'inner' values. + while (count < leaves.size()) { + uint256 h = leaves[count]; + bool matchh = count == leaf_pos; + count++; + int level; + // For each of the lower bits in count that are 0, do 1 step. Each + // corresponds to an inner value that existed before processing the + // current leaf, and each needs a hash to combine it. + for (level = 0; !(count & ((uint32_t{1}) << level)); level++) { + if (path) { + if (matchh) { + path->push_back(inner[level]); + } else if (matchlevel == level) { + path->push_back(h); + matchh = true; + } + } + mutated |= (inner[level] == h); + h = Hash(inner[level], h); + } + // Store the resulting hash at inner position level. + inner[level] = h; + if (matchh) { + matchlevel = level; + } + } + // Do a final 'sweep' over the rightmost branch of the tree to process + // odd levels, and reduce everything to a single top value. + // Level is the level (counted from the bottom) up to which we've sweeped. + int level = 0; + // As long as bit number level in count is zero, skip it. It means there + // is nothing left at this level. + while (!(count & ((uint32_t{1}) << level))) { + level++; + } + uint256 h = inner[level]; + bool matchh = matchlevel == level; + while (count != ((uint32_t{1}) << level)) { + // If we reach this point, h is an inner value that is not the top. + // We combine it with itself (Bitcoin's special rule for odd levels in + // the tree) to produce a higher level one. + if (path && matchh) { + path->push_back(h); + } + h = Hash(h, h); + // Increment count to the value it would have if two entries at this + // level had existed. + count += ((uint32_t{1}) << level); + level++; + // And propagate the result upwards accordingly. + while (!(count & ((uint32_t{1}) << level))) { + if (path) { + if (matchh) { + path->push_back(inner[level]); + } else if (matchlevel == level) { + path->push_back(h); + matchh = true; + } + } + h = Hash(inner[level], h); + level++; + } + } + // Return result. + if (pmutated) *pmutated = mutated; + if (proot) *proot = h; +} + +static std::vector<uint256> ComputeMerklePath(const std::vector<uint256>& leaves, uint32_t position) { + std::vector<uint256> ret; + MerkleComputation(leaves, nullptr, nullptr, position, &ret); + return ret; +} + +std::vector<uint256> TransactionMerklePath(const CBlock& block, uint32_t position) +{ + std::vector<uint256> leaves; + leaves.resize(block.vtx.size()); + for (size_t s = 0; s < block.vtx.size(); s++) { + leaves[s] = block.vtx[s]->GetHash(); + } + return ComputeMerklePath(leaves, position); +} diff --git a/src/consensus/merkle.h b/src/consensus/merkle.h index 4ae5a5b897..c722cbe446 100644 --- a/src/consensus/merkle.h +++ b/src/consensus/merkle.h @@ -24,4 +24,14 @@ uint256 BlockMerkleRoot(const CBlock& block, bool* mutated = nullptr); */ uint256 BlockWitnessMerkleRoot(const CBlock& block, bool* mutated = nullptr); +/** + * Compute merkle path to the specified transaction + * + * @param[in] block the block + * @param[in] position transaction for which to calculate the merkle path (0 is the coinbase) + * + * @return merkle path ordered from the deepest + */ +std::vector<uint256> TransactionMerklePath(const CBlock& block, uint32_t position); + #endif // BITCOIN_CONSENSUS_MERKLE_H diff --git a/src/consensus/validation.h b/src/consensus/validation.h index 1556c7888f..dffa9f2dd6 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -23,14 +23,6 @@ static constexpr size_t MINIMUM_WITNESS_COMMITMENT{38}; enum class TxValidationResult { TX_RESULT_UNSET = 0, //!< initial value. Tx has not yet been rejected TX_CONSENSUS, //!< invalid by consensus rules - /** - * Invalid by a change to consensus rules more recent than SegWit. - * Currently unused as there are no such consensus rule changes, and any download - * sources realistically need to support SegWit in order to provide useful data, - * so differentiating between always-invalid and invalid-by-pre-SegWit-soft-fork - * is uninteresting. - */ - TX_RECENT_CONSENSUS_CHANGE, TX_INPUTS_NOT_STANDARD, //!< inputs (covered by txid) failed policy rules TX_NOT_STANDARD, //!< otherwise didn't meet our local policy rules TX_MISSING_INPUTS, //!< transaction was missing some of its inputs @@ -65,14 +57,6 @@ enum class TxValidationResult { enum class BlockValidationResult { BLOCK_RESULT_UNSET = 0, //!< initial value. Block has not yet been rejected BLOCK_CONSENSUS, //!< invalid by consensus rules (excluding any below reasons) - /** - * Invalid by a change to consensus rules more recent than SegWit. - * Currently unused as there are no such consensus rule changes, and any download - * sources realistically need to support SegWit in order to provide useful data, - * so differentiating between always-invalid and invalid-by-pre-SegWit-soft-fork - * is uninteresting. - */ - BLOCK_RECENT_CONSENSUS_CHANGE, BLOCK_CACHED_INVALID, //!< this block was cached as being invalid and we didn't store the reason why BLOCK_INVALID_HEADER, //!< invalid proof of work or time too old BLOCK_MUTATED, //!< the block's data didn't match the data committed to by the PoW diff --git a/src/core_io.h b/src/core_io.h index 9305bb7239..ce2e8f6712 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -10,6 +10,7 @@ #include <string> #include <vector> +#include <optional> class CBlock; class CBlockHeader; diff --git a/src/core_read.cpp b/src/core_read.cpp index 23f341c230..273dfb17e8 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -39,7 +39,7 @@ public: } mapOpNames[strName] = static_cast<opcodetype>(op); // Convenience: OP_ADD and just ADD are both recognized: - if (strName.compare(0, 3, "OP_") == 0) { // strName starts with "OP_" + if (strName.starts_with("OP_")) { mapOpNames[strName.substr(3)] = static_cast<opcodetype>(op); } } diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp index 4feed862cb..e756e0431e 100644 --- a/src/crypto/chacha20.cpp +++ b/src/crypto/chacha20.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 The Bitcoin Core developers +// Copyright (c) 2017-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -25,14 +25,14 @@ void ChaCha20Aligned::SetKey(Span<const std::byte> key) noexcept { assert(key.size() == KEYLEN); - input[0] = ReadLE32(UCharCast(key.data() + 0)); - input[1] = ReadLE32(UCharCast(key.data() + 4)); - input[2] = ReadLE32(UCharCast(key.data() + 8)); - input[3] = ReadLE32(UCharCast(key.data() + 12)); - input[4] = ReadLE32(UCharCast(key.data() + 16)); - input[5] = ReadLE32(UCharCast(key.data() + 20)); - input[6] = ReadLE32(UCharCast(key.data() + 24)); - input[7] = ReadLE32(UCharCast(key.data() + 28)); + input[0] = ReadLE32(key.data() + 0); + input[1] = ReadLE32(key.data() + 4); + input[2] = ReadLE32(key.data() + 8); + input[3] = ReadLE32(key.data() + 12); + input[4] = ReadLE32(key.data() + 16); + input[5] = ReadLE32(key.data() + 20); + input[6] = ReadLE32(key.data() + 24); + input[7] = ReadLE32(key.data() + 28); input[8] = 0; input[9] = 0; input[10] = 0; diff --git a/src/crypto/common.h b/src/crypto/common.h index 1dc4f3f55c..f151cbb625 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2020 The Bitcoin Core developers +// Copyright (c) 2014-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,76 +7,99 @@ #include <compat/endian.h> +#include <concepts> +#include <cstddef> #include <cstdint> #include <cstring> -uint16_t static inline ReadLE16(const unsigned char* ptr) +template <typename B> +concept ByteType = std::same_as<B, unsigned char> || std::same_as<B, std::byte>; + +template <ByteType B> +inline uint16_t ReadLE16(const B* ptr) { uint16_t x; memcpy(&x, ptr, 2); return le16toh_internal(x); } -uint32_t static inline ReadLE32(const unsigned char* ptr) +template <ByteType B> +inline uint32_t ReadLE32(const B* ptr) { uint32_t x; memcpy(&x, ptr, 4); return le32toh_internal(x); } -uint64_t static inline ReadLE64(const unsigned char* ptr) +template <ByteType B> +inline uint64_t ReadLE64(const B* ptr) { uint64_t x; memcpy(&x, ptr, 8); return le64toh_internal(x); } -void static inline WriteLE16(unsigned char* ptr, uint16_t x) +template <ByteType B> +inline void WriteLE16(B* ptr, uint16_t x) { uint16_t v = htole16_internal(x); memcpy(ptr, &v, 2); } -void static inline WriteLE32(unsigned char* ptr, uint32_t x) +template <ByteType B> +inline void WriteLE32(B* ptr, uint32_t x) { uint32_t v = htole32_internal(x); memcpy(ptr, &v, 4); } -void static inline WriteLE64(unsigned char* ptr, uint64_t x) +template <ByteType B> +inline void WriteLE64(B* ptr, uint64_t x) { uint64_t v = htole64_internal(x); memcpy(ptr, &v, 8); } -uint16_t static inline ReadBE16(const unsigned char* ptr) +template <ByteType B> +inline uint16_t ReadBE16(const B* ptr) { uint16_t x; memcpy(&x, ptr, 2); return be16toh_internal(x); } -uint32_t static inline ReadBE32(const unsigned char* ptr) +template <ByteType B> +inline uint32_t ReadBE32(const B* ptr) { uint32_t x; memcpy(&x, ptr, 4); return be32toh_internal(x); } -uint64_t static inline ReadBE64(const unsigned char* ptr) +template <ByteType B> +inline uint64_t ReadBE64(const B* ptr) { uint64_t x; memcpy(&x, ptr, 8); return be64toh_internal(x); } -void static inline WriteBE32(unsigned char* ptr, uint32_t x) +template <ByteType B> +inline void WriteBE16(B* ptr, uint16_t x) +{ + uint16_t v = htobe16_internal(x); + memcpy(ptr, &v, 2); +} + +template <ByteType B> +inline void WriteBE32(B* ptr, uint32_t x) { uint32_t v = htobe32_internal(x); memcpy(ptr, &v, 4); } -void static inline WriteBE64(unsigned char* ptr, uint64_t x) +template <ByteType B> +inline void WriteBE64(B* ptr, uint64_t x) { uint64_t v = htobe64_internal(x); memcpy(ptr, &v, 8); diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index deedc0a6d1..09c5d3123e 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <crypto/sha256.h> #include <crypto/common.h> diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index e0f153fd61..8fb366515a 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -147,6 +147,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) // on corruption in later versions. options.paranoid_checks = true; } + options.max_file_size = std::max(options.max_file_size, DBWRAPPER_MAX_FILE_SIZE); SetMaxOpenFiles(&options); return options; } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 63c2f99d2a..dd5daa7a1f 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -22,6 +22,7 @@ static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024; +static const size_t DBWRAPPER_MAX_FILE_SIZE = 32 << 20; // 32 MiB //! User-controlled performance and debug options. struct DBOptions { diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 69dd821dc0..5d906ffa0c 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -134,8 +134,6 @@ static bool multiUserAuthorized(std::string strUserPass) static bool RPCAuthorized(const std::string& strAuth, std::string& strAuthUsernameOut) { - if (strRPCUserColonPass.empty()) // Belt-and-suspenders measure if InitRPCAuthentication was not called - return false; if (strAuth.substr(0, 6) != "Basic ") return false; std::string_view strUserPass64 = TrimStringView(std::string_view{strAuth}.substr(6)); @@ -147,8 +145,9 @@ static bool RPCAuthorized(const std::string& strAuth, std::string& strAuthUserna if (strUserPass.find(':') != std::string::npos) strAuthUsernameOut = strUserPass.substr(0, strUserPass.find(':')); - //Check if authorized under single-user field - if (TimingResistantEqual(strUserPass, strRPCUserColonPass)) { + // Check if authorized under single-user field. + // (strRPCUserColonPass is empty when -norpccookiefile is specified). + if (!strRPCUserColonPass.empty() && TimingResistantEqual(strUserPass, strRPCUserColonPass)) { return true; } return multiUserAuthorized(strUserPass); @@ -294,22 +293,26 @@ static bool InitRPCAuthentication() { if (gArgs.GetArg("-rpcpassword", "") == "") { - LogInfo("Using random cookie authentication.\n"); - std::optional<fs::perms> cookie_perms{std::nullopt}; auto cookie_perms_arg{gArgs.GetArg("-rpccookieperms")}; if (cookie_perms_arg) { auto perm_opt = InterpretPermString(*cookie_perms_arg); if (!perm_opt) { - LogInfo("Invalid -rpccookieperms=%s; must be one of 'owner', 'group', or 'all'.\n", *cookie_perms_arg); + LogError("Invalid -rpccookieperms=%s; must be one of 'owner', 'group', or 'all'.", *cookie_perms_arg); return false; } cookie_perms = *perm_opt; } + assert(strRPCUserColonPass.empty()); // Only support initializing once if (!GenerateAuthCookie(&strRPCUserColonPass, cookie_perms)) { return false; } + if (strRPCUserColonPass.empty()) { + LogInfo("RPC authentication cookie file generation is disabled."); + } else { + LogInfo("Using random cookie authentication."); + } } else { LogPrintf("Config options rpcuser and rpcpassword will soon be deprecated. Locally-run instances may remove rpcuser to use cookie-based auth, or may be replaced with rpcauth. Please see share/rpcauth for rpcauth auth generation.\n"); strRPCUserColonPass = gArgs.GetArg("-rpcuser", "") + ":" + gArgs.GetArg("-rpcpassword", ""); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 0e5503a17f..88e640c377 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -2,12 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep - #include <httpserver.h> #include <chainparamsbase.h> #include <common/args.h> +#include <common/messages.h> #include <compat/compat.h> #include <logging.h> #include <netbase.h> @@ -43,6 +42,8 @@ #include <support/events.h> +using common::InvalidPortErrMsg; + /** Maximum size of http request (request line + headers) */ static const size_t MAX_HEADERS_SIZE = 8192; @@ -226,7 +227,7 @@ static bool InitHTTPAllowList() const CSubNet subnet{LookupSubNet(strAllow)}; if (!subnet.IsValid()) { uiInterface.ThreadSafeMessageBox( - strprintf(Untranslated("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24)."), strAllow), + Untranslated(strprintf("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24).", strAllow)), "", CClientUIInterface::MSG_ERROR); return false; } @@ -315,7 +316,7 @@ static void http_request_cb(struct evhttp_request* req, void* arg) if (i->exactMatch) match = (strURI == i->prefix); else - match = (strURI.substr(0, i->prefix.size()) == i->prefix); + match = strURI.starts_with(i->prefix); if (match) { path = strURI.substr(i->prefix.size()); break; @@ -374,7 +375,10 @@ static bool HTTPBindAddresses(struct evhttp* http) for (const std::string& strRPCBind : gArgs.GetArgs("-rpcbind")) { uint16_t port{http_port}; std::string host; - SplitHostPort(strRPCBind, port, host); + if (!SplitHostPort(strRPCBind, port, host)) { + LogError("%s\n", InvalidPortErrMsg("-rpcbind", strRPCBind).original); + return false; + } endpoints.emplace_back(host, port); } } diff --git a/src/httpserver.h b/src/httpserver.h index 33216a0119..6535dc6086 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -14,8 +14,17 @@ namespace util { class SignalInterrupt; } // namespace util -static const int DEFAULT_HTTP_THREADS=4; -static const int DEFAULT_HTTP_WORKQUEUE=16; +/** + * The default value for `-rpcthreads`. This number of threads will be created at startup. + */ +static const int DEFAULT_HTTP_THREADS=16; + +/** + * The default value for `-rpcworkqueue`. This is the maximum depth of the work queue, + * we don't allocate this number of work queue items upfront. + */ +static const int DEFAULT_HTTP_WORKQUEUE=64; + static const int DEFAULT_HTTP_SERVER_TIMEOUT=30; struct evhttp_request; diff --git a/src/index/base.cpp b/src/index/base.cpp index 6f9860415f..a8f9073d9f 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -31,7 +31,7 @@ template <typename... Args> void BaseIndex::FatalErrorf(util::ConstevalFormatString<sizeof...(Args)> fmt, const Args&... args) { auto message = tfm::format(fmt, args...); - node::AbortNode(m_chain->context()->shutdown, m_chain->context()->exit_status, Untranslated(message), m_chain->context()->warnings.get()); + node::AbortNode(m_chain->context()->shutdown_request, m_chain->context()->exit_status, Untranslated(message), m_chain->context()->warnings.get()); } CBlockLocator GetLocator(interfaces::Chain& chain, const uint256& block_hash) @@ -106,14 +106,14 @@ bool BaseIndex::Init() // best chain, we will rewind to the fork point during index sync const CBlockIndex* locator_index{m_chainstate->m_blockman.LookupBlockIndex(locator.vHave.at(0))}; if (!locator_index) { - return InitError(strprintf(Untranslated("%s: best block of the index not found. Please rebuild the index."), GetName())); + return InitError(Untranslated(strprintf("%s: best block of the index not found. Please rebuild the index.", GetName()))); } SetBestBlockIndex(locator_index); } // Child init const CBlockIndex* start_block = m_best_block_index.load(); - if (!CustomInit(start_block ? std::make_optional(interfaces::BlockKey{start_block->GetBlockHash(), start_block->nHeight}) : std::nullopt)) { + if (!CustomInit(start_block ? std::make_optional(interfaces::BlockRef{start_block->GetBlockHash(), start_block->nHeight}) : std::nullopt)) { return false; } diff --git a/src/index/base.h b/src/index/base.h index beb1575ab2..fbd9069a51 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -7,6 +7,7 @@ #include <dbwrapper.h> #include <interfaces/chain.h> +#include <interfaces/types.h> #include <util/string.h> #include <util/threadinterrupt.h> #include <validationinterface.h> @@ -107,7 +108,7 @@ protected: void ChainStateFlushed(ChainstateRole role, const CBlockLocator& locator) override; /// Initialize internal state from the database and block index. - [[nodiscard]] virtual bool CustomInit(const std::optional<interfaces::BlockKey>& block) { return true; } + [[nodiscard]] virtual bool CustomInit(const std::optional<interfaces::BlockRef>& block) { return true; } /// Write update index entries for a newly connected block. [[nodiscard]] virtual bool CustomAppend(const interfaces::BlockInfo& block) { return true; } @@ -118,7 +119,7 @@ protected: /// Rewind index to an earlier chain tip during a chain reorg. The tip must /// be an ancestor of the current best block. - [[nodiscard]] virtual bool CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) { return true; } + [[nodiscard]] virtual bool CustomRewind(const interfaces::BlockRef& current_tip, const interfaces::BlockRef& new_tip) { return true; } virtual DB& GetDB() const = 0; diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index 41bdca9df5..a808cc9085 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -112,7 +112,7 @@ BlockFilterIndex::BlockFilterIndex(std::unique_ptr<interfaces::Chain> chain, Blo m_filter_fileseq = std::make_unique<FlatFileSeq>(std::move(path), "fltr", FLTR_FILE_CHUNK_SIZE); } -bool BlockFilterIndex::CustomInit(const std::optional<interfaces::BlockKey>& block) +bool BlockFilterIndex::CustomInit(const std::optional<interfaces::BlockRef>& block) { if (!m_db->Read(DB_FILTER_POS, m_next_filter_pos)) { // Check that the cause of the read failure is that the key does not exist. Any other errors @@ -151,7 +151,7 @@ bool BlockFilterIndex::CustomCommit(CDBBatch& batch) LogError("%s: Failed to open filter file %d\n", __func__, pos.nFile); return false; } - if (!FileCommit(file.Get())) { + if (!file.Commit()) { LogError("%s: Failed to commit filter file %d\n", __func__, pos.nFile); return false; } @@ -201,11 +201,11 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile); return 0; } - if (!TruncateFile(last_file.Get(), pos.nPos)) { + if (!last_file.Truncate(pos.nPos)) { LogPrintf("%s: Failed to truncate filter file %d\n", __func__, pos.nFile); return 0; } - if (!FileCommit(last_file.Get())) { + if (!last_file.Commit()) { LogPrintf("%s: Failed to commit filter file %d\n", __func__, pos.nFile); return 0; } @@ -316,7 +316,7 @@ bool BlockFilterIndex::Write(const BlockFilter& filter, uint32_t block_height, c return true; } -bool BlockFilterIndex::CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) +bool BlockFilterIndex::CustomRewind(const interfaces::BlockRef& current_tip, const interfaces::BlockRef& new_tip) { CDBBatch batch(*m_db); std::unique_ptr<CDBIterator> db_it(m_db->NewIterator()); diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index cdb9563fb8..ccb4845ef5 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -52,13 +52,13 @@ private: std::optional<uint256> ReadFilterHeader(int height, const uint256& expected_block_hash); protected: - bool CustomInit(const std::optional<interfaces::BlockKey>& block) override; + bool CustomInit(const std::optional<interfaces::BlockRef>& block) override; bool CustomCommit(CDBBatch& batch) override; bool CustomAppend(const interfaces::BlockInfo& block) override; - bool CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) override; + bool CustomRewind(const interfaces::BlockRef& current_tip, const interfaces::BlockRef& new_tip) override; BaseIndex::DB& GetDB() const LIFETIMEBOUND override { return *m_db; } diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index dff8e50a4e..c950a18f3f 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -265,7 +265,7 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) return true; } -bool CoinStatsIndex::CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) +bool CoinStatsIndex::CustomRewind(const interfaces::BlockRef& current_tip, const interfaces::BlockRef& new_tip) { CDBBatch batch(*m_db); std::unique_ptr<CDBIterator> db_it(m_db->NewIterator()); @@ -304,7 +304,7 @@ bool CoinStatsIndex::CustomRewind(const interfaces::BlockKey& current_tip, const return true; } -static bool LookUpOne(const CDBWrapper& db, const interfaces::BlockKey& block, DBVal& result) +static bool LookUpOne(const CDBWrapper& db, const interfaces::BlockRef& block, DBVal& result) { // First check if the result is stored under the height index and the value // there matches the block hash. This should be the case if the block is on @@ -350,7 +350,7 @@ std::optional<CCoinsStats> CoinStatsIndex::LookUpStats(const CBlockIndex& block_ return stats; } -bool CoinStatsIndex::CustomInit(const std::optional<interfaces::BlockKey>& block) +bool CoinStatsIndex::CustomInit(const std::optional<interfaces::BlockRef>& block) { if (!m_db->Read(DB_MUHASH, m_muhash)) { // Check that the cause of the read failure is that the key does not diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h index d6322bfa7c..885b9e0a86 100644 --- a/src/index/coinstatsindex.h +++ b/src/index/coinstatsindex.h @@ -43,13 +43,13 @@ private: bool AllowPrune() const override { return true; } protected: - bool CustomInit(const std::optional<interfaces::BlockKey>& block) override; + bool CustomInit(const std::optional<interfaces::BlockRef>& block) override; bool CustomCommit(CDBBatch& batch) override; bool CustomAppend(const interfaces::BlockInfo& block) override; - bool CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) override; + bool CustomRewind(const interfaces::BlockRef& current_tip, const interfaces::BlockRef& new_tip) override; BaseIndex::DB& GetDB() const override { return *m_db; } diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 80f615ed0e..425a7f00a0 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -87,10 +87,7 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe CBlockHeader header; try { file >> header; - if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) { - LogError("%s: fseek(...) failed\n", __func__); - return false; - } + file.seek(postx.nTxOffset, SEEK_CUR); file >> TX_WITH_WITNESS(tx); } catch (const std::exception& e) { LogError("%s: Deserialize or I/O error - %s\n", __func__, e.what()); diff --git a/src/init.cpp b/src/init.cpp index f67835d7da..b5adcf6476 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <init.h> @@ -123,17 +123,19 @@ using node::ApplyArgsManOptions; using node::BlockManager; using node::CacheSizes; using node::CalculateCacheSizes; +using node::ChainstateLoadResult; +using node::ChainstateLoadStatus; using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINT_MODIFIED_FEE; using node::DEFAULT_STOPATHEIGHT; using node::DumpMempool; -using node::LoadMempool; +using node::ImportBlocks; using node::KernelNotifications; using node::LoadChainstate; +using node::LoadMempool; using node::MempoolPath; using node::NodeContext; using node::ShouldPersistMempool; -using node::ImportBlocks; using node::VerifyLoadedChainstate; using util::Join; using util::ReplaceAll; @@ -173,6 +175,8 @@ static fs::path GetPidFile(const ArgsManager& args) [[nodiscard]] static bool CreatePidFile(const ArgsManager& args) { + if (args.IsArgNegated("-pid")) return true; + std::ofstream file{GetPidFile(args)}; if (file) { #ifdef WIN32 @@ -205,7 +209,14 @@ void InitContext(NodeContext& node) g_shutdown.emplace(); node.args = &gArgs; - node.shutdown = &*g_shutdown; + node.shutdown_signal = &*g_shutdown; + node.shutdown_request = [&node] { + assert(node.shutdown_signal); + if (!(*node.shutdown_signal)()) return false; + // Wake any threads that may be waiting for the tip to change. + if (node.notifications) WITH_LOCK(node.notifications->m_tip_block_mutex, node.notifications->m_tip_block_cv.notify_all()); + return true; + }; } ////////////////////////////////////////////////////////////////////////////// @@ -233,7 +244,7 @@ void InitContext(NodeContext& node) bool ShutdownRequested(node::NodeContext& node) { - return bool{*Assert(node.shutdown)}; + return bool{*Assert(node.shutdown_signal)}; } #if HAVE_SYSTEM @@ -298,7 +309,7 @@ void Shutdown(NodeContext& node) StopTorControl(); - if (node.chainman && node.chainman->m_thread_load.joinable()) node.chainman->m_thread_load.join(); + if (node.background_init_thread.joinable()) node.background_init_thread.join(); // After everything has been shut down, but before things get flushed, stop the // the scheduler. After this point, SyncWithValidationInterfaceQueue() should not be called anymore // as this would prevent the shutdown from completing. @@ -429,20 +440,6 @@ static void registerSignalHandler(int signal, void(*handler)(int)) } #endif -static boost::signals2::connection rpc_notify_block_change_connection; -static void OnRPCStarted() -{ - rpc_notify_block_change_connection = uiInterface.NotifyBlockTip_connect(std::bind(RPCNotifyBlockChange, std::placeholders::_2)); -} - -static void OnRPCStopped() -{ - rpc_notify_block_change_connection.disconnect(); - RPCNotifyBlockChange(nullptr); - g_best_block_cv.notify_all(); - LogDebug(BCLog::RPC, "RPC stopped.\n"); -} - void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) { SetupHelpOptions(argsman); @@ -488,9 +485,9 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) argsman.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Disables automatic broadcast and rebroadcast of transactions, unless the source peer has the 'forcerelay' permission. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-coinstatsindex", strprintf("Maintain coinstats index used by the gettxoutsetinfo RPC (default: %u)", DEFAULT_COINSTATSINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-conf=<file>", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location (only useable from command line, not configuration file) (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); - argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (minimum %d, default: %d). Make sure you have enough RAM. In addition, unused memory allocated to the mempool is shared with this cache (see -maxmempool).", nMinDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -526,7 +523,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) argsman.AddArg("-addnode=<ip>", strprintf("Add a node to connect to and attempt to keep the connection open (see the addnode RPC help for more info). This option can be specified multiple times to add multiple nodes; connections are limited to %u at a time and are counted separately from the -maxconnections limit.", MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-asmap=<file>", strprintf("Specify asn mapping used for bucketing of the peers (default: %s). Relative paths will be prefixed by the net-specific datadir location.", DEFAULT_ASMAP_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-bantime=<n>", strprintf("Default duration (in seconds) of manually configured bans (default: %u)", DEFAULT_MISBEHAVING_BANTIME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-bind=<addr>[:<port>][=onion]", strprintf("Bind to given address and always listen on it (default: 0.0.0.0). Use [host]:port notation for IPv6. Append =onion to tag any incoming connections to that address and port as incoming Tor connections (default: 127.0.0.1:%u=onion, testnet3: 127.0.0.1:%u=onion, testnet4: 127.0.0.1:%u=onion, signet: 127.0.0.1:%u=onion, regtest: 127.0.0.1:%u=onion)", defaultBaseParams->OnionServiceTargetPort(), testnetBaseParams->OnionServiceTargetPort(), testnet4BaseParams->OnionServiceTargetPort(), signetBaseParams->OnionServiceTargetPort(), regtestBaseParams->OnionServiceTargetPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); + argsman.AddArg("-bind=<addr>[:<port>][=onion]", strprintf("Bind to given address and always listen on it (default: 0.0.0.0). Use [host]:port notation for IPv6. Append =onion to tag any incoming connections to that address and port as incoming Tor connections (default: 127.0.0.1:%u=onion, testnet3: 127.0.0.1:%u=onion, testnet4: 127.0.0.1:%u=onion, signet: 127.0.0.1:%u=onion, regtest: 127.0.0.1:%u=onion)", defaultChainParams->GetDefaultPort() + 1, testnetChainParams->GetDefaultPort() + 1, testnet4ChainParams->GetDefaultPort() + 1, signetChainParams->GetDefaultPort() + 1, regtestChainParams->GetDefaultPort() + 1), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-cjdnsreachable", "If set, then this host is configured for CJDNS (connecting to fc00::/8 addresses would lead us to the CJDNS network, see doc/cjdns.md) (default: 0)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-connect=<ip>", "Connect only to the specified node; -noconnect disables automatic connections (the rules for this peer are the same as for -addnode). This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-discover", "Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -553,7 +550,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-txreconciliation", strprintf("Enable transaction reconciliations per BIP 330 (default: %d)", DEFAULT_TXRECONCILIATION_ENABLE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); - argsman.AddArg("-port=<port>", strprintf("Listen for connections on <port> (default: %u, testnet3: %u, testnet4: %u, signet: %u, regtest: %u). Not relevant for I2P (see doc/i2p.md).", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), testnet4ChainParams->GetDefaultPort(), signetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); + argsman.AddArg("-port=<port>", strprintf("Listen for connections on <port> (default: %u, testnet3: %u, testnet4: %u, signet: %u, regtest: %u). Not relevant for I2P (see doc/i2p.md). If set to a value x, the default onion listening port will be set to x+1.", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), testnet4ChainParams->GetDefaultPort(), signetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); #ifdef HAVE_SOCKADDR_UN argsman.AddArg("-proxy=<ip:port|path>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled). May be a local file path prefixed with 'unix:' if the proxy supports it.", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_ELISION, OptionsCategory::CONNECTION); #else @@ -566,16 +563,9 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) argsman.AddArg("-peertimeout=<n>", strprintf("Specify a p2p connection timeout delay in seconds. After connecting to a peer, wait this amount of time before considering disconnection based on inactivity (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control host and port to use if onion listening enabled (default: %s). If no port is specified, the default port of %i will be used.", DEFAULT_TOR_CONTROL, DEFAULT_TOR_CONTROL_PORT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::CONNECTION); -#ifdef USE_UPNP - argsman.AddArg("-upnp", strprintf("Use UPnP to map the listening port (default: %u)", DEFAULT_UPNP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); -#else - hidden_args.emplace_back("-upnp"); -#endif -#ifdef USE_NATPMP - argsman.AddArg("-natpmp", strprintf("Use NAT-PMP to map the listening port (default: %u)", DEFAULT_NATPMP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); -#else - hidden_args.emplace_back("-natpmp"); -#endif // USE_NATPMP + // UPnP support was dropped. We keep `-upnp` as a hidden arg to display a more user friendly error when set. TODO: remove (here and below) for 30.0. NOTE: removing this option may prevent the GUI from starting, see https://github.com/bitcoin-core/gui/issues/843. + argsman.AddArg("-upnp", "", ArgsManager::ALLOW_ANY, OptionsCategory::HIDDEN); + argsman.AddArg("-natpmp", strprintf("Use PCP or NAT-PMP to map the listening port (default: %u)", DEFAULT_NATPMP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-whitebind=<[permissions@]addr>", "Bind to the given address and add permission flags to the peers connecting to it. " "Use [host]:port notation for IPv6. Allowed permissions: " + Join(NET_PERMISSIONS_DOC, ", ") + ". " "Specify multiple permissions separated by commas (default: download,noban,mempool,relay). Can be specified multiple times.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -633,7 +623,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", Ticks<std::chrono::seconds>(DEFAULT_MAX_TIP_AGE)), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-printpriority", strprintf("Log transaction fee rate in " + CURRENCY_UNIT + "/kvB when mining blocks (default: %u)", DEFAULT_PRINT_MODIFIED_FEE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-printpriority", strprintf("Log transaction fee rate in %s/kvB when mining blocks (default: %u)", CURRENCY_UNIT, DEFAULT_PRINT_MODIFIED_FEE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-uacomment=<cmt>", "Append comment to the user agent string", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); SetupChainParamsBaseOptions(argsman); @@ -649,7 +639,6 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) "is of this size or less (default: %u)", MAX_OP_RETURN_RELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); - argsman.AddArg("-mempoolfullrbf", strprintf("(DEPRECATED) Accept transaction replace-by-fee without requiring replaceability signaling (default: %u)", DEFAULT_MEMPOOL_FULL_RBF), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); argsman.AddArg("-permitbaremultisig", strprintf("Relay transactions creating non-P2SH multisig outputs (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); argsman.AddArg("-minrelaytxfee=<amt>", strprintf("Fees (in %s/kvB) smaller than this are considered zero fee for relaying, mining and transaction creation (default: %s)", @@ -676,7 +665,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) argsman.AddArg("-rpcuser=<user>", "Username for JSON-RPC connections", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::RPC); argsman.AddArg("-rpcwhitelist=<whitelist>", "Set a whitelist to filter incoming RPC calls for a specific user. The field <whitelist> comes in the format: <USERNAME>:<rpc 1>,<rpc 2>,...,<rpc n>. If multiple whitelists are set for a given user, they are set-intersected. See -rpcwhitelistdefault documentation for information on default whitelist behavior.", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); argsman.AddArg("-rpcwhitelistdefault", "Sets default behavior for rpc whitelisting. Unless rpcwhitelistdefault is set to 0, if any -rpcwhitelist is set, the rpc server acts as if all rpc users are subject to empty-unless-otherwise-specified whitelists. If rpcwhitelistdefault is set to 1 and no -rpcwhitelist is set, rpc server acts as if all rpc users are subject to empty whitelists.", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); - argsman.AddArg("-rpcworkqueue=<n>", strprintf("Set the depth of the work queue to service RPC calls (default: %d)", DEFAULT_HTTP_WORKQUEUE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::RPC); + argsman.AddArg("-rpcworkqueue=<n>", strprintf("Set the maximum depth of the work queue to service RPC calls (default: %d)", DEFAULT_HTTP_WORKQUEUE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::RPC); argsman.AddArg("-server", "Accept command line and JSON-RPC commands", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); if (can_listen_ipc) { argsman.AddArg("-ipcbind=<address>", "Bind to Unix socket address and listen for incoming connections. Valid address values are \"unix\" to listen on the default path, <datadir>/node.sock, or \"unix:/custom/path\" to specify a custom path. Can be specified multiple times to listen on multiple paths. Default behavior is not to listen on any path. If relative paths are specified, they are interpreted relative to the network data directory. If paths include any parent directory components and the parent directories do not exist, they will be created.", ArgsManager::ALLOW_ANY, OptionsCategory::IPC); @@ -694,21 +683,6 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) argsman.AddHiddenArgs(hidden_args); } -static bool fHaveGenesis = false; -static GlobalMutex g_genesis_wait_mutex; -static std::condition_variable g_genesis_wait_cv; - -static void BlockNotifyGenesisWait(const CBlockIndex* pBlockIndex) -{ - if (pBlockIndex != nullptr) { - { - LOCK(g_genesis_wait_mutex); - fHaveGenesis = true; - } - g_genesis_wait_cv.notify_all(); - } -} - #if HAVE_SYSTEM static void StartupNotify(const ArgsManager& args) { @@ -723,9 +697,7 @@ static void StartupNotify(const ArgsManager& args) static bool AppInitServers(NodeContext& node) { const ArgsManager& args = *Assert(node.args); - RPCServer::OnStarted(&OnRPCStarted); - RPCServer::OnStopped(&OnRPCStopped); - if (!InitHTTPServer(*Assert(node.shutdown))) { + if (!InitHTTPServer(*Assert(node.shutdown_signal))) { return false; } StartRPC(); @@ -766,8 +738,6 @@ void InitParameterInteraction(ArgsManager& args) LogInfo("parameter interaction: -proxy set -> setting -listen=0\n"); // to protect privacy, do not map ports when a proxy is set. The user may still specify -listen=1 // to listen locally, so don't rely on this happening through -listen below. - if (args.SoftSetBoolArg("-upnp", false)) - LogInfo("parameter interaction: -proxy set -> setting -upnp=0\n"); if (args.SoftSetBoolArg("-natpmp", false)) { LogInfo("parameter interaction: -proxy set -> setting -natpmp=0\n"); } @@ -778,8 +748,6 @@ void InitParameterInteraction(ArgsManager& args) if (!args.GetBoolArg("-listen", DEFAULT_LISTEN)) { // do not map ports or try to retrieve public IP when not listening (pointless) - if (args.SoftSetBoolArg("-upnp", false)) - LogInfo("parameter interaction: -listen=0 -> setting -upnp=0\n"); if (args.SoftSetBoolArg("-natpmp", false)) { LogInfo("parameter interaction: -listen=0 -> setting -natpmp=0\n"); } @@ -840,7 +808,7 @@ namespace { // Variables internal to initialization process only int nMaxConnections; int available_fds; -ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK_LIMITED | NODE_WITNESS); +ServiceFlags g_local_services = ServiceFlags(NODE_NETWORK_LIMITED | NODE_WITNESS); int64_t peer_connect_timeout; std::set<BlockFilterType> g_enabled_filter_types; @@ -903,6 +871,12 @@ bool AppInitParameterInteraction(const ArgsManager& args) // also see: InitParameterInteraction() + // We drop UPnP support but kept the arg as hidden for now to display a friendlier error to user who have the + // option in their config. TODO: remove (here and above) for version 30.0. + if (args.IsArgSet("-upnp")) { + InitWarning(_("Option '-upnp' is set but UPnP support was dropped in version 29.0. Consider using '-natpmp' instead.")); + } + // Error if network-specific options (-addnode, -connect, etc) are // specified in default section of config file, but not overridden // on the command line or in this chain's section of the config file. @@ -912,7 +886,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) } bilingual_str errors; for (const auto& arg : args.GetUnsuitableSectionOnlyArgs()) { - errors += strprintf(_("Config setting for %s only applied on %s network when in [%s] section.") + Untranslated("\n"), arg, ChainTypeToString(chain), ChainTypeToString(chain)); + errors += strprintf(_("Config setting for %s only applied on %s network when in [%s] section."), arg, ChainTypeToString(chain), ChainTypeToString(chain)) + Untranslated("\n"); } if (!errors.empty()) { @@ -927,7 +901,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) // Warn if unrecognized section name are present in the config file. bilingual_str warnings; for (const auto& section : args.GetUnrecognizedSections()) { - warnings += strprintf(Untranslated("%s:%i ") + _("Section [%s] is not recognized.") + Untranslated("\n"), section.m_file, section.m_line, section.m_name); + warnings += Untranslated(strprintf("%s:%i ", section.m_file, section.m_line)) + strprintf(_("Section [%s] is not recognized."), section.m_name) + Untranslated("\n"); } if (!warnings.empty()) { @@ -955,7 +929,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) // Signal NODE_P2P_V2 if BIP324 v2 transport is enabled. if (args.GetBoolArg("-v2transport", DEFAULT_V2_TRANSPORT)) { - nLocalServices = ServiceFlags(nLocalServices | NODE_P2P_V2); + g_local_services = ServiceFlags(g_local_services | NODE_P2P_V2); } // Signal NODE_COMPACT_FILTERS if peerblockfilters and basic filters index are both enabled. @@ -964,7 +938,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) return InitError(_("Cannot set -peerblockfilters without -blockfilterindex.")); } - nLocalServices = ServiceFlags(nLocalServices | NODE_COMPACT_FILTERS); + g_local_services = ServiceFlags(g_local_services | NODE_COMPACT_FILTERS); } if (args.GetIntArg("-prune", 0)) { @@ -1049,7 +1023,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) SetMockTime(args.GetIntArg("-mocktime", 0)); // SetMockTime(0) is a no-op if (args.GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) - nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); + g_local_services = ServiceFlags(g_local_services | NODE_BLOOM); if (args.IsArgSet("-test")) { if (chainparams.GetChainType() != ChainType::REGTEST) { @@ -1088,6 +1062,11 @@ bool AppInitParameterInteraction(const ArgsManager& args) if (!blockman_result) { return InitError(util::ErrorString(blockman_result)); } + CTxMemPool::Options mempool_opts{}; + auto mempool_result{ApplyArgsManOptions(args, chainparams, mempool_opts)}; + if (!mempool_result) { + return InitError(util::ErrorString(mempool_result)); + } } return true; @@ -1101,7 +1080,7 @@ static bool LockDataDirectory(bool probeOnly) case util::LockResult::ErrorWrite: return InitError(strprintf(_("Cannot write to data directory '%s'; check permissions."), fs::PathToString(datadir))); case util::LockResult::ErrorLock: - return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running."), fs::PathToString(datadir), PACKAGE_NAME)); + return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running."), fs::PathToString(datadir), CLIENT_NAME)); case util::LockResult::Success: return true; } // no default case, so the compiler can warn about missing cases assert(false); @@ -1113,11 +1092,11 @@ bool AppInitSanityChecks(const kernel::Context& kernel) auto result{kernel::SanityChecks(kernel)}; if (!result) { InitError(util::ErrorString(result)); - return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down."), PACKAGE_NAME)); + return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down."), CLIENT_NAME)); } if (!ECC_InitSanityCheck()) { - return InitError(strprintf(_("Elliptic curve cryptography sanity check failure. %s is shutting down."), PACKAGE_NAME)); + return InitError(strprintf(_("Elliptic curve cryptography sanity check failure. %s is shutting down."), CLIENT_NAME)); } // Probe the data directory lock to give an early error message, if possible @@ -1145,6 +1124,151 @@ bool AppInitInterfaces(NodeContext& node) return true; } +bool CheckHostPortOptions(const ArgsManager& args) { + for (const std::string port_option : { + "-port", + "-rpcport", + }) { + if (args.IsArgSet(port_option)) { + const std::string port = args.GetArg(port_option, ""); + uint16_t n; + if (!ParseUInt16(port, &n) || n == 0) { + return InitError(InvalidPortErrMsg(port_option, port)); + } + } + } + + for ([[maybe_unused]] const auto& [arg, unix] : std::vector<std::pair<std::string, bool>>{ + // arg name UNIX socket support + {"-i2psam", false}, + {"-onion", true}, + {"-proxy", true}, + {"-rpcbind", false}, + {"-torcontrol", false}, + {"-whitebind", false}, + {"-zmqpubhashblock", true}, + {"-zmqpubhashtx", true}, + {"-zmqpubrawblock", true}, + {"-zmqpubrawtx", true}, + {"-zmqpubsequence", true}, + }) { + for (const std::string& socket_addr : args.GetArgs(arg)) { + std::string host_out; + uint16_t port_out{0}; + if (!SplitHostPort(socket_addr, port_out, host_out)) { +#ifdef HAVE_SOCKADDR_UN + // Allow unix domain sockets for some options e.g. unix:/some/file/path + if (!unix || !socket_addr.starts_with(ADDR_PREFIX_UNIX)) { + return InitError(InvalidPortErrMsg(arg, socket_addr)); + } +#else + return InitError(InvalidPortErrMsg(arg, socket_addr)); +#endif + } + } + } + + return true; +} + +// A GUI user may opt to retry once with do_reindex set if there is a failure during chainstate initialization. +// The function therefore has to support re-entry. +static ChainstateLoadResult InitAndLoadChainstate( + NodeContext& node, + bool do_reindex, + const bool do_reindex_chainstate, + CacheSizes& cache_sizes, + const ArgsManager& args) +{ + const CChainParams& chainparams = Params(); + CTxMemPool::Options mempool_opts{ + .check_ratio = chainparams.DefaultConsistencyChecks() ? 1 : 0, + .signals = node.validation_signals.get(), + }; + Assert(ApplyArgsManOptions(args, chainparams, mempool_opts)); // no error can happen, already checked in AppInitParameterInteraction + bilingual_str mempool_error; + node.mempool = std::make_unique<CTxMemPool>(mempool_opts, mempool_error); + if (!mempool_error.empty()) { + return {ChainstateLoadStatus::FAILURE_FATAL, mempool_error}; + } + LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024)); + ChainstateManager::Options chainman_opts{ + .chainparams = chainparams, + .datadir = args.GetDataDirNet(), + .notifications = *node.notifications, + .signals = node.validation_signals.get(), + }; + Assert(ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction + BlockManager::Options blockman_opts{ + .chainparams = chainman_opts.chainparams, + .blocks_dir = args.GetBlocksDirPath(), + .notifications = chainman_opts.notifications, + }; + Assert(ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction + try { + node.chainman = std::make_unique<ChainstateManager>(*Assert(node.shutdown_signal), chainman_opts, blockman_opts); + } catch (std::exception& e) { + return {ChainstateLoadStatus::FAILURE_FATAL, Untranslated(strprintf("Failed to initialize ChainstateManager: %s", e.what()))}; + } + ChainstateManager& chainman = *node.chainman; + // This is defined and set here instead of inline in validation.h to avoid a hard + // dependency between validation and index/base, since the latter is not in + // libbitcoinkernel. + chainman.snapshot_download_completed = [&node]() { + if (!node.chainman->m_blockman.IsPruneMode()) { + LogPrintf("[snapshot] re-enabling NODE_NETWORK services\n"); + node.connman->AddLocalServices(NODE_NETWORK); + } + LogPrintf("[snapshot] restarting indexes\n"); + // Drain the validation interface queue to ensure that the old indexes + // don't have any pending work. + Assert(node.validation_signals)->SyncWithValidationInterfaceQueue(); + for (auto* index : node.indexes) { + index->Interrupt(); + index->Stop(); + if (!(index->Init() && index->StartBackgroundSync())) { + LogPrintf("[snapshot] WARNING failed to restart index %s on snapshot chain\n", index->GetName()); + } + } + }; + node::ChainstateLoadOptions options; + options.mempool = Assert(node.mempool.get()); + options.wipe_block_tree_db = do_reindex; + options.wipe_chainstate_db = do_reindex || do_reindex_chainstate; + options.prune = chainman.m_blockman.IsPruneMode(); + options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); + options.check_level = args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); + options.require_full_verification = args.IsArgSet("-checkblocks") || args.IsArgSet("-checklevel"); + options.coins_error_cb = [] { + uiInterface.ThreadSafeMessageBox( + _("Error reading from database, shutting down."), + "", CClientUIInterface::MSG_ERROR); + }; + uiInterface.InitMessage(_("Loading block index…").translated); + const auto load_block_index_start_time{SteadyClock::now()}; + auto catch_exceptions = [](auto&& f) { + try { + return f(); + } catch (const std::exception& e) { + LogError("%s\n", e.what()); + return std::make_tuple(node::ChainstateLoadStatus::FAILURE, _("Error loading databases")); + } + }; + auto [status, error] = catch_exceptions([&] { return LoadChainstate(chainman, cache_sizes, options); }); + if (status == node::ChainstateLoadStatus::SUCCESS) { + uiInterface.InitMessage(_("Verifying blocks…").translated); + if (chainman.m_blockman.m_have_pruned && options.check_blocks > MIN_BLOCKS_TO_KEEP) { + LogWarning("pruned datadir may not have more than %d blocks; only checking available blocks\n", + MIN_BLOCKS_TO_KEEP); + } + std::tie(status, error) = catch_exceptions([&] { return VerifyLoadedChainstate(chainman, options); }); + if (status == node::ChainstateLoadStatus::SUCCESS) { + LogPrintf(" block index %15dms\n", Ticks<std::chrono::milliseconds>(SteadyClock::now() - load_block_index_start_time)); + } + } + return {status, error}; +}; + bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) { const ArgsManager& args = *Assert(node.args); @@ -1193,7 +1317,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) constexpr uint64_t min_disk_space = 50 << 20; // 50 MB if (!CheckDiskSpace(args.GetBlocksDirPath(), min_disk_space)) { LogError("Shutting down due to lack of disk space!\n"); - if (!(*Assert(node.shutdown))()) { + if (!(Assert(node.shutdown_request))()) { LogError("Failed to send shutdown signal after disk space check\n"); } } @@ -1215,7 +1339,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) try { ipc->listenAddress(address); } catch (const std::exception& e) { - return InitError(strprintf(Untranslated("Unable to bind to IPC address '%s'. %s"), address, e.what())); + return InitError(Untranslated(strprintf("Unable to bind to IPC address '%s'. %s", address, e.what()))); } LogPrintf("Listening for IPC requests on address %s\n", address); } @@ -1232,6 +1356,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) RegisterZMQRPCCommands(tableRPC); #endif + // Check port numbers + if (!CheckHostPortOptions(args)) return false; + /* Start the RPC server already. It will be started in "warmup" mode * and not really process calls already (but it will signify connections * that the server is there and will be ready later). Warmup mode will @@ -1322,50 +1449,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) validation_signals.RegisterValidationInterface(fee_estimator); } - // Check port numbers - for (const std::string port_option : { - "-port", - "-rpcport", - }) { - if (args.IsArgSet(port_option)) { - const std::string port = args.GetArg(port_option, ""); - uint16_t n; - if (!ParseUInt16(port, &n) || n == 0) { - return InitError(InvalidPortErrMsg(port_option, port)); - } - } - } - - for ([[maybe_unused]] const auto& [arg, unix] : std::vector<std::pair<std::string, bool>>{ - // arg name UNIX socket support - {"-i2psam", false}, - {"-onion", true}, - {"-proxy", true}, - {"-rpcbind", false}, - {"-torcontrol", false}, - {"-whitebind", false}, - {"-zmqpubhashblock", true}, - {"-zmqpubhashtx", true}, - {"-zmqpubrawblock", true}, - {"-zmqpubrawtx", true}, - {"-zmqpubsequence", true}, - }) { - for (const std::string& socket_addr : args.GetArgs(arg)) { - std::string host_out; - uint16_t port_out{0}; - if (!SplitHostPort(socket_addr, port_out, host_out)) { -#ifdef HAVE_SOCKADDR_UN - // Allow unix domain sockets for some options e.g. unix:/some/file/path - if (!unix || socket_addr.find(ADDR_PREFIX_UNIX) != 0) { - return InitError(InvalidPortErrMsg(arg, socket_addr)); - } -#else - return InitError(InvalidPortErrMsg(arg, socket_addr)); -#endif - } - } - } - for (const std::string& socket_addr : args.GetArgs("-bind")) { std::string host_out; uint16_t port_out{0}; @@ -1382,7 +1465,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return InitError(strprintf(_("User Agent comment (%s) contains unsafe characters."), cmt)); uacomments.push_back(cmt); } - strSubVersion = FormatSubVersion(CLIENT_NAME, CLIENT_VERSION, uacomments); + strSubVersion = FormatSubVersion(UA_NAME, CLIENT_VERSION, uacomments); if (strSubVersion.size() > MAX_SUBVERSION_LENGTH) { return InitError(strprintf(_("Total length of network version string (%i) exceeds maximum length (%i). Reduce the number or size of uacomments."), strSubVersion.size(), MAX_SUBVERSION_LENGTH)); @@ -1515,22 +1598,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 7: load block chain - node.notifications = std::make_unique<KernelNotifications>(*Assert(node.shutdown), node.exit_status, *Assert(node.warnings)); - ReadNotificationArgs(args, *node.notifications); - ChainstateManager::Options chainman_opts{ - .chainparams = chainparams, - .datadir = args.GetDataDirNet(), - .notifications = *node.notifications, - .signals = &validation_signals, - }; - Assert(ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction - - BlockManager::Options blockman_opts{ - .chainparams = chainman_opts.chainparams, - .blocks_dir = args.GetBlocksDirPath(), - .notifications = chainman_opts.notifications, - }; - Assert(ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction + node.notifications = std::make_unique<KernelNotifications>(Assert(node.shutdown_request), node.exit_status, *Assert(node.warnings)); + auto& kernel_notifications{*node.notifications}; + ReadNotificationArgs(args, kernel_notifications); // cache size calculations CacheSizes cache_sizes = CalculateCacheSizes(args, g_enabled_filter_types.size()); @@ -1549,124 +1619,42 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) assert(!node.mempool); assert(!node.chainman); - CTxMemPool::Options mempool_opts{ - .check_ratio = chainparams.DefaultConsistencyChecks() ? 1 : 0, - .signals = &validation_signals, - }; - auto result{ApplyArgsManOptions(args, chainparams, mempool_opts)}; - if (!result) { - return InitError(util::ErrorString(result)); - } - bool do_reindex{args.GetBoolArg("-reindex", false)}; const bool do_reindex_chainstate{args.GetBoolArg("-reindex-chainstate", false)}; - for (bool fLoaded = false; !fLoaded && !ShutdownRequested(node);) { - bilingual_str mempool_error; - node.mempool = std::make_unique<CTxMemPool>(mempool_opts, mempool_error); - if (!mempool_error.empty()) { - return InitError(mempool_error); - } - LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024)); - - try { - node.chainman = std::make_unique<ChainstateManager>(*Assert(node.shutdown), chainman_opts, blockman_opts); - } catch (std::exception& e) { - return InitError(strprintf(Untranslated("Failed to initialize ChainstateManager: %s"), e.what())); - } - ChainstateManager& chainman = *node.chainman; - - // This is defined and set here instead of inline in validation.h to avoid a hard - // dependency between validation and index/base, since the latter is not in - // libbitcoinkernel. - chainman.snapshot_download_completed = [&node]() { - if (!node.chainman->m_blockman.IsPruneMode()) { - LogPrintf("[snapshot] re-enabling NODE_NETWORK services\n"); - node.connman->AddLocalServices(NODE_NETWORK); - } - - LogPrintf("[snapshot] restarting indexes\n"); - - // Drain the validation interface queue to ensure that the old indexes - // don't have any pending work. - Assert(node.validation_signals)->SyncWithValidationInterfaceQueue(); - - for (auto* index : node.indexes) { - index->Interrupt(); - index->Stop(); - if (!(index->Init() && index->StartBackgroundSync())) { - LogPrintf("[snapshot] WARNING failed to restart index %s on snapshot chain\n", index->GetName()); - } - } - }; - - node::ChainstateLoadOptions options; - options.mempool = Assert(node.mempool.get()); - options.wipe_block_tree_db = do_reindex; - options.wipe_chainstate_db = do_reindex || do_reindex_chainstate; - options.prune = chainman.m_blockman.IsPruneMode(); - options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); - options.check_level = args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); - options.require_full_verification = args.IsArgSet("-checkblocks") || args.IsArgSet("-checklevel"); - options.coins_error_cb = [] { - uiInterface.ThreadSafeMessageBox( - _("Error reading from database, shutting down."), - "", CClientUIInterface::MSG_ERROR); - }; - - uiInterface.InitMessage(_("Loading block index…").translated); - const auto load_block_index_start_time{SteadyClock::now()}; - auto catch_exceptions = [](auto&& f) { - try { - return f(); - } catch (const std::exception& e) { - LogError("%s\n", e.what()); - return std::make_tuple(node::ChainstateLoadStatus::FAILURE, _("Error opening block database")); - } - }; - auto [status, error] = catch_exceptions([&]{ return LoadChainstate(chainman, cache_sizes, options); }); - if (status == node::ChainstateLoadStatus::SUCCESS) { - uiInterface.InitMessage(_("Verifying blocks…").translated); - if (chainman.m_blockman.m_have_pruned && options.check_blocks > MIN_BLOCKS_TO_KEEP) { - LogWarning("pruned datadir may not have more than %d blocks; only checking available blocks\n", - MIN_BLOCKS_TO_KEEP); - } - std::tie(status, error) = catch_exceptions([&]{ return VerifyLoadedChainstate(chainman, options);}); - if (status == node::ChainstateLoadStatus::SUCCESS) { - fLoaded = true; - LogPrintf(" block index %15dms\n", Ticks<std::chrono::milliseconds>(SteadyClock::now() - load_block_index_start_time)); - } + // Chainstate initialization and loading may be retried once with reindexing by GUI users + auto [status, error] = InitAndLoadChainstate( + node, + do_reindex, + do_reindex_chainstate, + cache_sizes, + args); + if (status == ChainstateLoadStatus::FAILURE && !do_reindex && !ShutdownRequested(node)) { + // suggest a reindex + bool do_retry = uiInterface.ThreadSafeQuestion( + error + Untranslated(".\n\n") + _("Do you want to rebuild the databases now?"), + error.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", + "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); + if (!do_retry) { + return false; } - - if (status == node::ChainstateLoadStatus::FAILURE_FATAL || status == node::ChainstateLoadStatus::FAILURE_INCOMPATIBLE_DB || status == node::ChainstateLoadStatus::FAILURE_INSUFFICIENT_DBCACHE) { - return InitError(error); - } - - if (!fLoaded && !ShutdownRequested(node)) { - // first suggest a reindex - if (!do_reindex) { - bool fRet = uiInterface.ThreadSafeQuestion( - error + Untranslated(".\n\n") + _("Do you want to rebuild the block database now?"), - error.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", - "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); - if (fRet) { - do_reindex = true; - if (!Assert(node.shutdown)->reset()) { - LogError("Internal error: failed to reset shutdown signal.\n"); - } - } else { - LogError("Aborted block database rebuild. Exiting.\n"); - return false; - } - } else { - return InitError(error); - } + do_reindex = true; + if (!Assert(node.shutdown_signal)->reset()) { + LogError("Internal error: failed to reset shutdown signal.\n"); } + std::tie(status, error) = InitAndLoadChainstate( + node, + do_reindex, + do_reindex_chainstate, + cache_sizes, + args); + } + if (status != ChainstateLoadStatus::SUCCESS && status != ChainstateLoadStatus::INTERRUPTED) { + return InitError(error); } // As LoadBlockIndex can take several minutes, it's possible the user // requested to kill the GUI during the last operation. If so, exit. - // As the program has not fully started yet, Shutdown() is possibly overkill. if (ShutdownRequested(node)) { LogPrintf("Shutdown requested. Exiting.\n"); return false; @@ -1724,7 +1712,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Prior to setting NODE_NETWORK, check if we can provide historical blocks. if (!WITH_LOCK(chainman.GetMutex(), return chainman.BackgroundSyncInProgress())) { LogPrintf("Setting NODE_NETWORK on non-prune mode\n"); - nLocalServices = ServiceFlags(nLocalServices | NODE_NETWORK); + g_local_services = ServiceFlags(g_local_services | NODE_NETWORK); } else { LogPrintf("Running node in NODE_NETWORK_LIMITED mode until snapshot background sync completes\n"); } @@ -1762,15 +1750,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } } - // Either install a handler to notify us when genesis activates, or set fHaveGenesis directly. - // No locking, as this happens before any background thread is started. - boost::signals2::connection block_notify_genesis_wait_connection; - if (WITH_LOCK(chainman.GetMutex(), return chainman.ActiveChain().Tip() == nullptr)) { - block_notify_genesis_wait_connection = uiInterface.NotifyBlockTip_connect(std::bind(BlockNotifyGenesisWait, std::placeholders::_2)); - } else { - fHaveGenesis = true; - } - #if HAVE_SYSTEM const std::string block_notify = args.GetArg("-blocknotify", ""); if (!block_notify.empty()) { @@ -1789,13 +1768,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) vImportFiles.push_back(fs::PathFromString(strFile)); } - chainman.m_thread_load = std::thread(&util::TraceThread, "initload", [=, &chainman, &args, &node] { + node.background_init_thread = std::thread(&util::TraceThread, "initload", [=, &chainman, &args, &node] { ScheduleBatchPriority(); - // Import blocks + // Import blocks and ActivateBestChain() ImportBlocks(chainman, vImportFiles); if (args.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { LogPrintf("Stopping after block import\n"); - if (!(*Assert(node.shutdown))()) { + if (!(Assert(node.shutdown_request))()) { LogError("Failed to send shutdown signal after finishing block import\n"); } return; @@ -1814,16 +1793,22 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } }); - // Wait for genesis block to be processed + /* + * Wait for genesis block to be processed. Typically kernel_notifications.m_tip_block + * has already been set by a call to LoadChainTip() in CompleteChainstateInitialization(). + * But this is skipped if the chainstate doesn't exist yet or is being wiped: + * + * 1. first startup with an empty datadir + * 2. reindex + * 3. reindex-chainstate + * + * In these case it's connected by a call to ActivateBestChain() in the initload thread. + */ { - WAIT_LOCK(g_genesis_wait_mutex, lock); - // We previously could hang here if shutdown was requested prior to - // ImportBlocks getting started, so instead we just wait on a timer to - // check ShutdownRequested() regularly. - while (!fHaveGenesis && !ShutdownRequested(node)) { - g_genesis_wait_cv.wait_for(lock, std::chrono::milliseconds(500)); - } - block_notify_genesis_wait_connection.disconnect(); + WAIT_LOCK(kernel_notifications.m_tip_block_mutex, lock); + kernel_notifications.m_tip_block_cv.wait(lock, [&]() EXCLUSIVE_LOCKS_REQUIRED(kernel_notifications.m_tip_block_mutex) { + return kernel_notifications.TipBlock() || ShutdownRequested(node); + }); } if (ShutdownRequested(node)) { @@ -1832,17 +1817,17 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 12: start node - //// debug print int64_t best_block_time{}; { - LOCK(cs_main); + LOCK(chainman.GetMutex()); + const auto& tip{*Assert(chainman.ActiveTip())}; LogPrintf("block tree size = %u\n", chainman.BlockIndex().size()); - chain_active_height = chainman.ActiveChain().Height(); - best_block_time = chainman.ActiveChain().Tip() ? chainman.ActiveChain().Tip()->GetBlockTime() : chainman.GetParams().GenesisBlock().GetBlockTime(); + chain_active_height = tip.nHeight; + best_block_time = tip.GetBlockTime(); if (tip_info) { tip_info->block_height = chain_active_height; tip_info->block_time = best_block_time; - tip_info->verification_progress = GuessVerificationProgress(chainman.GetParams().TxData(), chainman.ActiveChain().Tip()); + tip_info->verification_progress = chainman.GuessVerificationProgress(&tip); } if (tip_info && chainman.m_best_header) { tip_info->header_height = chainman.m_best_header->nHeight; @@ -1852,11 +1837,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) LogPrintf("nBestHeight = %d\n", chain_active_height); if (node.peerman) node.peerman->SetBestBlock(chain_active_height, std::chrono::seconds{best_block_time}); - // Map ports with UPnP or NAT-PMP. - StartMapPort(args.GetBoolArg("-upnp", DEFAULT_UPNP), args.GetBoolArg("-natpmp", DEFAULT_NATPMP)); + // Map ports with NAT-PMP + StartMapPort(args.GetBoolArg("-natpmp", DEFAULT_NATPMP)); CConnman::Options connOptions; - connOptions.nLocalServices = nLocalServices; + connOptions.m_local_services = g_local_services; connOptions.m_max_automatic_connections = nMaxConnections; connOptions.uiInterface = &uiInterface; connOptions.m_banman = node.banman.get(); @@ -1873,6 +1858,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const uint16_t default_bind_port = static_cast<uint16_t>(args.GetIntArg("-port", Params().GetDefaultPort())); + const uint16_t default_bind_port_onion = default_bind_port + 1; + const auto BadPortWarning = [](const char* prefix, uint16_t port) { return strprintf(_("%s request to listen on port %u. This port is considered \"bad\" and " "thus it is unlikely that any peer will connect to it. See " @@ -1897,7 +1884,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const std::string network_type = bind_arg.substr(index + 1); if (network_type == "onion") { const std::string truncated_bind_arg = bind_arg.substr(0, index); - bind_addr = Lookup(truncated_bind_arg, BaseParams().OnionServiceTargetPort(), false); + bind_addr = Lookup(truncated_bind_arg, default_bind_port_onion, false); if (bind_addr.has_value()) { connOptions.onion_binds.push_back(bind_addr.value()); continue; @@ -1933,7 +1920,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } else if (!connOptions.vBinds.empty()) { onion_service_target = connOptions.vBinds.front(); } else { - onion_service_target = DefaultOnionServiceTarget(); + onion_service_target = DefaultOnionServiceTarget(default_bind_port_onion); connOptions.onion_binds.push_back(onion_service_target); } @@ -2011,11 +1998,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // cannot yet be called. Before we make it callable, we need to make sure // that the RPC's view of the best block is valid and consistent with // ChainstateManager's active tip. - // - // If we do not do this, RPC's view of the best block will be height=0 and - // hash=0x0. This will lead to erroroneous responses for things like - // waitforblockheight. - RPCNotifyBlockChange(WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip())); SetRPCWarmupFinished(); uiInterface.InitMessage(_("Done loading").translated); @@ -2074,7 +2056,7 @@ bool StartIndexBackgroundSync(NodeContext& node) const CBlockIndex* start_block = *indexes_start_block; if (!start_block) start_block = chainman.ActiveChain().Genesis(); if (!chainman.m_blockman.CheckBlockDataAvailability(*index_chain.Tip(), *Assert(start_block))) { - return InitError(strprintf(Untranslated("%s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"), older_index_name)); + return InitError(Untranslated(strprintf("%s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)", older_index_name))); } } diff --git a/src/init/common.cpp b/src/init/common.cpp index 36142c2b9a..70c4230cfd 100644 --- a/src/init/common.cpp +++ b/src/init/common.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <clientversion.h> #include <common/args.h> @@ -17,6 +17,7 @@ #include <util/translation.h> #include <algorithm> +#include <filesystem> #include <string> #include <vector> @@ -111,8 +112,8 @@ bool StartLogging(const ArgsManager& args) } } if (!LogInstance().StartLogging()) { - return InitError(strprintf(Untranslated("Could not open debug log file %s"), - fs::PathToString(LogInstance().m_file_path))); + return InitError(Untranslated(strprintf("Could not open debug log file %s", + fs::PathToString(LogInstance().m_file_path)))); } if (!LogInstance().m_log_timestamps) @@ -122,10 +123,13 @@ bool StartLogging(const ArgsManager& args) // Only log conf file usage message if conf file actually exists. fs::path config_file_path = args.GetConfigFilePath(); - if (fs::exists(config_file_path)) { + if (args.IsArgNegated("-conf")) { + LogInfo("Config file: <disabled>"); + } else if (fs::is_directory(config_file_path)) { + LogWarning("Config file: %s (is directory, not file)", fs::PathToString(config_file_path)); + } else if (fs::exists(config_file_path)) { LogPrintf("Config file: %s\n", fs::PathToString(config_file_path)); } else if (args.IsArgSet("-conf")) { - // Warn if no conf file exists at path provided by user InitWarning(strprintf(_("The specified config file %s does not exist"), fs::PathToString(config_file_path))); } else { // Not categorizing as "Warning" because it's the default behavior @@ -146,6 +150,6 @@ void LogPackageVersion() #else version_string += " (release build)"; #endif - LogPrintf(PACKAGE_NAME " version %s\n", version_string); + LogPrintf(CLIENT_NAME " version %s\n", version_string); } } // namespace init diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index be596b1765..4e858d1f89 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -41,12 +41,6 @@ namespace interfaces { class Handler; class Wallet; -//! Hash/height pair to help track and identify blocks. -struct BlockKey { - uint256 hash; - int height = -1; -}; - //! Helper for findBlock to selectively return pieces of block data. If block is //! found, data will be returned by setting specified output variables. If block //! is not found, output variables will keep their previous values. @@ -356,15 +350,22 @@ public: virtual common::SettingsValue getRwSetting(const std::string& name) = 0; //! Updates a setting in <datadir>/settings.json. + //! Null can be passed to erase the setting. There is intentionally no + //! support for writing null values to settings.json. //! Depending on the action returned by the update function, this will either //! update the setting in memory or write the updated settings to disk. virtual bool updateRwSetting(const std::string& name, const SettingsUpdate& update_function) = 0; //! Replace a setting in <datadir>/settings.json with a new value. - virtual bool overwriteRwSetting(const std::string& name, common::SettingsValue& value, bool write = true) = 0; + //! Null can be passed to erase the setting. + //! This method provides a simpler alternative to updateRwSetting when + //! atomically reading and updating the setting is not required. + virtual bool overwriteRwSetting(const std::string& name, common::SettingsValue value, SettingsAction action = SettingsAction::WRITE) = 0; //! Delete a given setting in <datadir>/settings.json. - virtual bool deleteRwSettings(const std::string& name, bool write = true) = 0; + //! This method provides a simpler alternative to overwriteRwSetting when + //! erasing a setting, for ease of use and readability. + virtual bool deleteRwSettings(const std::string& name, SettingsAction action = SettingsAction::WRITE) = 0; //! Synchronously send transactionAddedToMempool notifications about all //! current mempool transactions to the specified handler and return after diff --git a/src/interfaces/mining.h b/src/interfaces/mining.h index cebe97edb7..bc5955ded6 100644 --- a/src/interfaces/mining.h +++ b/src/interfaces/mining.h @@ -5,26 +5,61 @@ #ifndef BITCOIN_INTERFACES_MINING_H #define BITCOIN_INTERFACES_MINING_H -#include <node/types.h> -#include <uint256.h> - -#include <memory> -#include <optional> +#include <consensus/amount.h> // for CAmount +#include <interfaces/types.h> // for BlockRef +#include <node/types.h> // for BlockCreateOptions +#include <primitives/block.h> // for CBlock, CBlockHeader +#include <primitives/transaction.h> // for CTransactionRef +#include <stdint.h> // for int64_t +#include <uint256.h> // for uint256 +#include <util/time.h> // for MillisecondsDouble + +#include <memory> // for unique_ptr, shared_ptr +#include <optional> // for optional +#include <vector> // for vector namespace node { -struct CBlockTemplate; struct NodeContext; } // namespace node class BlockValidationState; -class CBlock; class CScript; namespace interfaces { +//! Block template interface +class BlockTemplate +{ +public: + virtual ~BlockTemplate() = default; + + virtual CBlockHeader getBlockHeader() = 0; + virtual CBlock getBlock() = 0; + + virtual std::vector<CAmount> getTxFees() = 0; + virtual std::vector<int64_t> getTxSigops() = 0; + + virtual CTransactionRef getCoinbaseTx() = 0; + virtual std::vector<unsigned char> getCoinbaseCommitment() = 0; + virtual int getWitnessCommitmentIndex() = 0; + + /** + * Compute merkle path to the coinbase transaction + * + * @return merkle path ordered from the deepest + */ + virtual std::vector<uint256> getCoinbaseMerklePath() = 0; + + /** + * Construct and broadcast the block. + * + * @returns if the block was processed, independent of block validity + */ + virtual bool submitSolution(uint32_t version, uint32_t timestamp, uint32_t nonce, CTransactionRef coinbase) = 0; +}; + //! Interface giving clients (RPC, Stratum v2 Template Provider in the future) //! ability to create block templates. - class Mining { public: @@ -36,42 +71,27 @@ public: //! Returns whether IBD is still in progress. virtual bool isInitialBlockDownload() = 0; - //! Returns the hash for the tip of this chain - virtual std::optional<uint256> getTipHash() = 0; - - /** - * Construct a new block template - * - * @param[in] script_pub_key the coinbase output - * @param[in] options options for creating the block - * @returns a block template - */ - virtual std::unique_ptr<node::CBlockTemplate> createNewBlock(const CScript& script_pub_key, const node::BlockCreateOptions& options={}) = 0; + //! Returns the hash and height for the tip of this chain + virtual std::optional<BlockRef> getTip() = 0; /** - * Processes new block. A valid new block is automatically relayed to peers. + * Waits for the connected tip to change. During node initialization, this will + * wait until the tip is connected. * - * @param[in] block The block we want to process. - * @param[out] new_block A boolean which is set to indicate if the block was first received via this call - * @returns If the block was processed, independently of block validity + * @param[in] current_tip block hash of the current chain tip. Function waits + * for the chain tip to differ from this. + * @param[in] timeout how long to wait for a new tip + * @returns Hash and height of the current chain tip after this call. */ - virtual bool processNewBlock(const std::shared_ptr<const CBlock>& block, bool* new_block) = 0; - - //! Return the number of transaction updates in the mempool, - //! used to decide whether to make a new block template. - virtual unsigned int getTransactionsUpdated() = 0; + virtual BlockRef waitTipChanged(uint256 current_tip, MillisecondsDouble timeout = MillisecondsDouble::max()) = 0; - /** - * Check a block is completely valid from start to finish. - * Only works on top of our current best block. - * Does not check proof-of-work. + /** + * Construct a new block template * - * @param[in] block the block to validate - * @param[in] check_merkle_root call CheckMerkleRoot() - * @param[out] state details of why a block failed to validate - * @returns false if it does not build on the current tip, or any of the checks fail + * @param[in] options options for creating the block + * @returns a block template */ - virtual bool testBlockValidity(const CBlock& block, bool check_merkle_root, BlockValidationState& state) = 0; + virtual std::unique_ptr<BlockTemplate> createNewBlock(const node::BlockCreateOptions& options = {}) = 0; //! Get internal node context. Useful for RPC and testing, //! but not accessible across processes. diff --git a/src/interfaces/node.h b/src/interfaces/node.h index b87c78db52..aebb438651 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -121,7 +121,7 @@ public: virtual void resetSettings() = 0; //! Map port. - virtual void mapPort(bool use_upnp, bool use_natpmp) = 0; + virtual void mapPort(bool use_pcp) = 0; //! Get proxy. virtual bool getProxy(Network net, Proxy& proxy_info) = 0; diff --git a/src/interfaces/types.h b/src/interfaces/types.h new file mode 100644 index 0000000000..e5edd301a7 --- /dev/null +++ b/src/interfaces/types.h @@ -0,0 +1,20 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INTERFACES_TYPES_H +#define BITCOIN_INTERFACES_TYPES_H + +#include <uint256.h> + +namespace interfaces { + +//! Hash/height pair to help track and identify blocks. +struct BlockRef { + uint256 hash; + int height = -1; +}; + +} // namespace interfaces + +#endif // BITCOIN_INTERFACES_TYPES_H diff --git a/src/ipc/CMakeLists.txt b/src/ipc/CMakeLists.txt index 94b1ceb54e..904d72f56e 100644 --- a/src/ipc/CMakeLists.txt +++ b/src/ipc/CMakeLists.txt @@ -3,16 +3,21 @@ # file COPYING or https://opensource.org/license/mit/. add_library(bitcoin_ipc STATIC EXCLUDE_FROM_ALL + capnp/mining.cpp capnp/protocol.cpp interfaces.cpp process.cpp ) target_capnp_sources(bitcoin_ipc ${PROJECT_SOURCE_DIR} - capnp/echo.capnp capnp/init.capnp + capnp/common.capnp + capnp/echo.capnp + capnp/init.capnp + capnp/mining.capnp ) target_link_libraries(bitcoin_ipc PRIVATE core_interface + univalue ) diff --git a/src/ipc/capnp/common-types.h b/src/ipc/capnp/common-types.h index 39e368491b..51af6a5f0a 100644 --- a/src/ipc/capnp/common-types.h +++ b/src/ipc/capnp/common-types.h @@ -6,6 +6,9 @@ #define BITCOIN_IPC_CAPNP_COMMON_TYPES_H #include <clientversion.h> +#include <interfaces/types.h> +#include <primitives/transaction.h> +#include <serialize.h> #include <streams.h> #include <univalue.h> @@ -16,33 +19,24 @@ namespace ipc { namespace capnp { -//! Use SFINAE to define Serializeable<T> trait which is true if type T has a -//! Serialize(stream) method, false otherwise. -template <typename T> -struct Serializable { -private: - template <typename C> - static std::true_type test(decltype(std::declval<C>().Serialize(std::declval<std::nullptr_t&>()))*); - template <typename> - static std::false_type test(...); - -public: - static constexpr bool value = decltype(test<T>(nullptr))::value; -}; +//! Construct a ParamStream wrapping a data stream with serialization parameters +//! needed to pass transaction objects between bitcoin processes. +//! In the future, more params may be added here to serialize other objects that +//! require serialization parameters. Params should just be chosen to serialize +//! objects completely and ensure that serializing and deserializing objects +//! with the specified parameters produces equivalent objects. It's also +//! harmless to specify serialization parameters here that are not used. +template <typename S> +auto Wrap(S& s) +{ + return ParamsStream{s, TX_WITH_WITNESS}; +} -//! Use SFINAE to define Unserializeable<T> trait which is true if type T has -//! an Unserialize(stream) method, false otherwise. +//! Detect if type has a deserialize_type constructor, which is +//! used to deserialize types like CTransaction that can't be unserialized into +//! existing objects because they are immutable. template <typename T> -struct Unserializable { -private: - template <typename C> - static std::true_type test(decltype(std::declval<C>().Unserialize(std::declval<std::nullptr_t&>()))*); - template <typename> - static std::false_type test(...); - -public: - static constexpr bool value = decltype(test<T>(nullptr))::value; -}; +concept Deserializable = std::is_constructible_v<T, ::deserialize_type, ::DataStream&>; } // namespace capnp } // namespace ipc @@ -50,42 +44,78 @@ public: namespace mp { //! Overload multiprocess library's CustomBuildField hook to allow any //! serializable object to be stored in a capnproto Data field or passed to a -//! canproto interface. Use Priority<1> so this hook has medium priority, and +//! capnproto interface. Use Priority<1> so this hook has medium priority, and //! higher priority hooks could take precedence over this one. template <typename LocalType, typename Value, typename Output> -void CustomBuildField( - TypeList<LocalType>, Priority<1>, InvokeContext& invoke_context, Value&& value, Output&& output, - // Enable if serializeable and if LocalType is not cv or reference - // qualified. If LocalType is cv or reference qualified, it is important to - // fall back to lower-priority Priority<0> implementation of this function - // that strips cv references, to prevent this CustomBuildField overload from - // taking precedence over more narrow overloads for specific LocalTypes. - std::enable_if_t<ipc::capnp::Serializable<LocalType>::value && - std::is_same_v<LocalType, std::remove_cv_t<std::remove_reference_t<LocalType>>>>* enable = nullptr) +void CustomBuildField(TypeList<LocalType>, Priority<1>, InvokeContext& invoke_context, Value&& value, Output&& output) +// Enable if serializeable and if LocalType is not cv or reference qualified. If +// LocalType is cv or reference qualified, it is important to fall back to +// lower-priority Priority<0> implementation of this function that strips cv +// references, to prevent this CustomBuildField overload from taking precedence +// over more narrow overloads for specific LocalTypes. +requires Serializable<LocalType, DataStream> && std::is_same_v<LocalType, std::remove_cv_t<std::remove_reference_t<LocalType>>> { DataStream stream; - value.Serialize(stream); + auto wrapper{ipc::capnp::Wrap(stream)}; + value.Serialize(wrapper); auto result = output.init(stream.size()); memcpy(result.begin(), stream.data(), stream.size()); } //! Overload multiprocess library's CustomReadField hook to allow any object //! with an Unserialize method to be read from a capnproto Data field or -//! returned from canproto interface. Use Priority<1> so this hook has medium +//! returned from capnproto interface. Use Priority<1> so this hook has medium //! priority, and higher priority hooks could take precedence over this one. template <typename LocalType, typename Input, typename ReadDest> -decltype(auto) -CustomReadField(TypeList<LocalType>, Priority<1>, InvokeContext& invoke_context, Input&& input, ReadDest&& read_dest, - std::enable_if_t<ipc::capnp::Unserializable<LocalType>::value>* enable = nullptr) +decltype(auto) CustomReadField(TypeList<LocalType>, Priority<1>, InvokeContext& invoke_context, Input&& input, ReadDest&& read_dest) +requires Unserializable<LocalType, DataStream> && (!ipc::capnp::Deserializable<LocalType>) { return read_dest.update([&](auto& value) { if (!input.has()) return; auto data = input.get(); SpanReader stream({data.begin(), data.end()}); - value.Unserialize(stream); + auto wrapper{ipc::capnp::Wrap(stream)}; + value.Unserialize(wrapper); }); } +//! Overload multiprocess library's CustomReadField hook to allow any object +//! with a deserialize constructor to be read from a capnproto Data field or +//! returned from capnproto interface. Use Priority<1> so this hook has medium +//! priority, and higher priority hooks could take precedence over this one. +template <typename LocalType, typename Input, typename ReadDest> +decltype(auto) CustomReadField(TypeList<LocalType>, Priority<1>, InvokeContext& invoke_context, Input&& input, ReadDest&& read_dest) +requires ipc::capnp::Deserializable<LocalType> +{ + assert(input.has()); + auto data = input.get(); + SpanReader stream({data.begin(), data.end()}); + auto wrapper{ipc::capnp::Wrap(stream)}; + return read_dest.construct(::deserialize, wrapper); +} + +//! Overload CustomBuildField and CustomReadField to serialize std::chrono +//! parameters and return values as numbers. +template <class Rep, class Period, typename Value, typename Output> +void CustomBuildField(TypeList<std::chrono::duration<Rep, Period>>, Priority<1>, InvokeContext& invoke_context, Value&& value, + Output&& output) +{ + static_assert(std::numeric_limits<decltype(output.get())>::lowest() <= std::numeric_limits<Rep>::lowest(), + "capnp type does not have enough range to hold lowest std::chrono::duration value"); + static_assert(std::numeric_limits<decltype(output.get())>::max() >= std::numeric_limits<Rep>::max(), + "capnp type does not have enough range to hold highest std::chrono::duration value"); + output.set(value.count()); +} + +template <class Rep, class Period, typename Input, typename ReadDest> +decltype(auto) CustomReadField(TypeList<std::chrono::duration<Rep, Period>>, Priority<1>, InvokeContext& invoke_context, + Input&& input, ReadDest&& read_dest) +{ + return read_dest.construct(input.get()); +} + +//! Overload CustomBuildField and CustomReadField to serialize UniValue +//! parameters and return values as JSON strings. template <typename Value, typename Output> void CustomBuildField(TypeList<UniValue>, Priority<1>, InvokeContext& invoke_context, Value&& value, Output&& output) { @@ -103,6 +133,33 @@ decltype(auto) CustomReadField(TypeList<UniValue>, Priority<1>, InvokeContext& i value.read(std::string_view{data.begin(), data.size()}); }); } + +//! Generic ::capnp::Data field builder for any C++ type that can be converted +//! to a span of bytes, like std::vector<char> or std::array<uint8_t>, or custom +//! blob types like uint256 or PKHash with data() and size() methods pointing to +//! bytes. +//! +//! Note: it might make sense to move this function into libmultiprocess, since +//! it is fairly generic. However this would require decreasing its priority so +//! it can be overridden, which would require more changes inside +//! libmultiprocess to avoid conflicting with the Priority<1> CustomBuildField +//! function it already provides for std::vector. Also, it might make sense to +//! provide a CustomReadField counterpart to this function, which could be +//! called to read C++ types that can be constructed from spans of bytes from +//! ::capnp::Data fields. But so far there hasn't been a need for this. +template <typename LocalType, typename Value, typename Output> +void CustomBuildField(TypeList<LocalType>, Priority<2>, InvokeContext& invoke_context, Value&& value, Output&& output) +requires + (std::is_same_v<decltype(output.get()), ::capnp::Data::Builder>) && + (std::convertible_to<Value, std::span<const std::byte>> || + std::convertible_to<Value, std::span<const char>> || + std::convertible_to<Value, std::span<const unsigned char>> || + std::convertible_to<Value, std::span<const signed char>>) +{ + auto data = std::span{value}; + auto result = output.init(data.size()); + memcpy(result.begin(), data.data(), data.size()); +} } // namespace mp #endif // BITCOIN_IPC_CAPNP_COMMON_TYPES_H diff --git a/src/ipc/capnp/common.capnp b/src/ipc/capnp/common.capnp new file mode 100644 index 0000000000..b3359f3f07 --- /dev/null +++ b/src/ipc/capnp/common.capnp @@ -0,0 +1,16 @@ +# Copyright (c) 2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xcd2c6232cb484a28; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("ipc::capnp::messages"); + +using Proxy = import "/mp/proxy.capnp"; +$Proxy.includeTypes("ipc/capnp/common-types.h"); + +struct BlockRef $Proxy.wrap("interfaces::BlockRef") { + hash @0 :Data; + height @1 :Int32; +} diff --git a/src/ipc/capnp/init-types.h b/src/ipc/capnp/init-types.h index 42031441b5..c3ddca27c0 100644 --- a/src/ipc/capnp/init-types.h +++ b/src/ipc/capnp/init-types.h @@ -6,5 +6,6 @@ #define BITCOIN_IPC_CAPNP_INIT_TYPES_H #include <ipc/capnp/echo.capnp.proxy-types.h> +#include <ipc/capnp/mining.capnp.proxy-types.h> #endif // BITCOIN_IPC_CAPNP_INIT_TYPES_H diff --git a/src/ipc/capnp/init.capnp b/src/ipc/capnp/init.capnp index e6d358c665..1001ee5336 100644 --- a/src/ipc/capnp/init.capnp +++ b/src/ipc/capnp/init.capnp @@ -10,11 +10,14 @@ $Cxx.namespace("ipc::capnp::messages"); using Proxy = import "/mp/proxy.capnp"; $Proxy.include("interfaces/echo.h"); $Proxy.include("interfaces/init.h"); +$Proxy.include("interfaces/mining.h"); $Proxy.includeTypes("ipc/capnp/init-types.h"); using Echo = import "echo.capnp"; +using Mining = import "mining.capnp"; interface Init $Proxy.wrap("interfaces::Init") { construct @0 (threadMap: Proxy.ThreadMap) -> (threadMap :Proxy.ThreadMap); makeEcho @1 (context :Proxy.Context) -> (result :Echo.Echo); + makeMining @2 (context :Proxy.Context) -> (result :Mining.Mining); } diff --git a/src/ipc/capnp/mining-types.h b/src/ipc/capnp/mining-types.h new file mode 100644 index 0000000000..2e60b43fcf --- /dev/null +++ b/src/ipc/capnp/mining-types.h @@ -0,0 +1,26 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_CAPNP_MINING_TYPES_H +#define BITCOIN_IPC_CAPNP_MINING_TYPES_H + +#include <interfaces/mining.h> +#include <ipc/capnp/common.capnp.proxy-types.h> +#include <ipc/capnp/common-types.h> +#include <ipc/capnp/mining.capnp.proxy.h> +#include <node/miner.h> +#include <node/types.h> +#include <validation.h> + +namespace mp { +// Custom serialization for BlockValidationState. +void CustomBuildMessage(InvokeContext& invoke_context, + const BlockValidationState& src, + ipc::capnp::messages::BlockValidationState::Builder&& builder); +void CustomReadMessage(InvokeContext& invoke_context, + const ipc::capnp::messages::BlockValidationState::Reader& reader, + BlockValidationState& dest); +} // namespace mp + +#endif // BITCOIN_IPC_CAPNP_MINING_TYPES_H diff --git a/src/ipc/capnp/mining.capnp b/src/ipc/capnp/mining.capnp new file mode 100644 index 0000000000..50b07bda70 --- /dev/null +++ b/src/ipc/capnp/mining.capnp @@ -0,0 +1,50 @@ +# Copyright (c) 2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xc77d03df6a41b505; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("ipc::capnp::messages"); + +using Common = import "common.capnp"; +using Proxy = import "/mp/proxy.capnp"; +$Proxy.include("interfaces/mining.h"); +$Proxy.includeTypes("ipc/capnp/mining-types.h"); + +interface Mining $Proxy.wrap("interfaces::Mining") { + isTestChain @0 (context :Proxy.Context) -> (result: Bool); + isInitialBlockDownload @1 (context :Proxy.Context) -> (result: Bool); + getTip @2 (context :Proxy.Context) -> (result: Common.BlockRef, hasResult: Bool); + waitTipChanged @3 (context :Proxy.Context, currentTip: Data, timeout: Float64) -> (result: Common.BlockRef); + createNewBlock @4 (options: BlockCreateOptions) -> (result: BlockTemplate); +} + +interface BlockTemplate $Proxy.wrap("interfaces::BlockTemplate") { + destroy @0 (context :Proxy.Context) -> (); + getBlockHeader @1 (context: Proxy.Context) -> (result: Data); + getBlock @2 (context: Proxy.Context) -> (result: Data); + getTxFees @3 (context: Proxy.Context) -> (result: List(Int64)); + getTxSigops @4 (context: Proxy.Context) -> (result: List(Int64)); + getCoinbaseTx @5 (context: Proxy.Context) -> (result: Data); + getCoinbaseCommitment @6 (context: Proxy.Context) -> (result: Data); + getWitnessCommitmentIndex @7 (context: Proxy.Context) -> (result: Int32); + getCoinbaseMerklePath @8 (context: Proxy.Context) -> (result: List(Data)); + submitSolution @9 (context: Proxy.Context, version: UInt32, timestamp: UInt32, nonce: UInt32, coinbase :Data) -> (result: Bool); +} + +struct BlockCreateOptions $Proxy.wrap("node::BlockCreateOptions") { + useMempool @0 :Bool $Proxy.name("use_mempool"); + coinbaseMaxAdditionalWeight @1 :UInt64 $Proxy.name("coinbase_max_additional_weight"); + coinbaseOutputMaxAdditionalSigops @2 :UInt64 $Proxy.name("coinbase_output_max_additional_sigops"); +} + +# Note: serialization of the BlockValidationState C++ type is somewhat fragile +# and using the struct can be awkward. It would be good if testBlockValidity +# method were changed to return validity information in a simpler format. +struct BlockValidationState { + mode @0 :Int32; + result @1 :Int32; + rejectReason @2 :Text; + debugMessage @3 :Text; +} diff --git a/src/ipc/capnp/mining.cpp b/src/ipc/capnp/mining.cpp new file mode 100644 index 0000000000..0f9533c1c7 --- /dev/null +++ b/src/ipc/capnp/mining.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <ipc/capnp/mining-types.h> +#include <ipc/capnp/mining.capnp.proxy-types.h> + +#include <mp/proxy-types.h> + +namespace mp { +void CustomBuildMessage(InvokeContext& invoke_context, + const BlockValidationState& src, + ipc::capnp::messages::BlockValidationState::Builder&& builder) +{ + if (src.IsValid()) { + builder.setMode(0); + } else if (src.IsInvalid()) { + builder.setMode(1); + } else if (src.IsError()) { + builder.setMode(2); + } else { + assert(false); + } + builder.setResult(static_cast<int>(src.GetResult())); + builder.setRejectReason(src.GetRejectReason()); + builder.setDebugMessage(src.GetDebugMessage()); +} + +void CustomReadMessage(InvokeContext& invoke_context, + const ipc::capnp::messages::BlockValidationState::Reader& reader, + BlockValidationState& dest) +{ + if (reader.getMode() == 0) { + assert(reader.getResult() == 0); + assert(reader.getRejectReason().size() == 0); + assert(reader.getDebugMessage().size() == 0); + } else if (reader.getMode() == 1) { + dest.Invalid(static_cast<BlockValidationResult>(reader.getResult()), reader.getRejectReason(), reader.getDebugMessage()); + } else if (reader.getMode() == 2) { + assert(reader.getResult() == 0); + dest.Error(reader.getRejectReason()); + assert(reader.getDebugMessage().size() == 0); + } else { + assert(false); + } +} +} // namespace mp diff --git a/src/ipc/process.cpp b/src/ipc/process.cpp index 432c365d8f..bdc541b654 100644 --- a/src/ipc/process.cpp +++ b/src/ipc/process.cpp @@ -72,7 +72,7 @@ static bool ParseAddress(std::string& address, struct sockaddr_un& addr, std::string& error) { - if (address.compare(0, 4, "unix") == 0 && (address.size() == 4 || address[4] == ':')) { + if (address == "unix" || address.starts_with("unix:")) { fs::path path; if (address.size() <= 5) { path = data_dir / fs::PathFromString(strprintf("%s.sock", RemovePrefixView(dest_exe_name, "bitcoin-"))); diff --git a/src/ipc/protocol.h b/src/ipc/protocol.h index b2ebf99e8c..cb964d802f 100644 --- a/src/ipc/protocol.h +++ b/src/ipc/protocol.h @@ -54,7 +54,7 @@ public: //! The optional `ready_fn` callback will be called after the event loop is //! created but before it is started. This can be useful in tests to trigger //! client connections from another thread as soon as the event loop is - //! available, but should not be neccessary in normal code which starts + //! available, but should not be necessary in normal code which starts //! clients and servers independently. virtual void serve(int fd, const char* exe_name, interfaces::Init& init, const std::function<void()>& ready_fn = {}) = 0; diff --git a/src/kernel/CMakeLists.txt b/src/kernel/CMakeLists.txt index 65d753b3af..2e07ba042a 100644 --- a/src/kernel/CMakeLists.txt +++ b/src/kernel/CMakeLists.txt @@ -33,6 +33,7 @@ add_library(bitcoinkernel ../node/blockstorage.cpp ../node/chainstate.cpp ../node/utxo_snapshot.cpp + ../policy/ephemeral_policy.cpp ../policy/feerate.cpp ../policy/packages.cpp ../policy/policy.cpp @@ -84,6 +85,7 @@ target_link_libraries(bitcoinkernel bitcoin_crypto leveldb secp256k1 + $<TARGET_NAME_IF_EXISTS:USDT::headers> PUBLIC Boost::headers ) @@ -119,18 +121,14 @@ if(NOT BUILD_SHARED_LIBS) set(all_kernel_static_link_libs "") get_target_static_link_libs(bitcoinkernel all_kernel_static_link_libs) + install(TARGETS ${all_kernel_static_link_libs} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT Kernel) + list(TRANSFORM all_kernel_static_link_libs PREPEND "-l") # LIBS_PRIVATE is substituted in the pkg-config file. - set(LIBS_PRIVATE "") - foreach(lib ${all_kernel_static_link_libs}) - install(TARGETS ${lib} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) - string(APPEND LIBS_PRIVATE " -l${lib}") - endforeach() - - string(STRIP "${LIBS_PRIVATE}" LIBS_PRIVATE) + list(JOIN all_kernel_static_link_libs " " LIBS_PRIVATE) endif() configure_file(${PROJECT_SOURCE_DIR}/libbitcoinkernel.pc.in ${PROJECT_BINARY_DIR}/libbitcoinkernel.pc @ONLY) -install(FILES ${PROJECT_BINARY_DIR}/libbitcoinkernel.pc DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") +install(FILES ${PROJECT_BINARY_DIR}/libbitcoinkernel.pc DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig" COMPONENT Kernel) include(GNUInstallDirs) install(TARGETS bitcoinkernel diff --git a/src/kernel/chainparams.cpp b/src/kernel/chainparams.cpp index 18c2026a88..349e14a5d6 100644 --- a/src/kernel/chainparams.cpp +++ b/src/kernel/chainparams.cpp @@ -73,7 +73,7 @@ static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesi static CBlock CreateGenesisBlock(uint32_t nTime, uint32_t nNonce, uint32_t nBits, int32_t nVersion, const CAmount& genesisReward) { const char* pszTimestamp = "The Times 03/Jan/2009 Chancellor on brink of second bailout for banks"; - const CScript genesisOutputScript = CScript() << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f"_hex_v_u8 << OP_CHECKSIG; + const CScript genesisOutputScript = CScript() << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f"_hex << OP_CHECKSIG; return CreateGenesisBlock(pszTimestamp, genesisOutputScript, nTime, nNonce, nBits, nVersion, genesisReward); } @@ -353,7 +353,7 @@ public: m_assumed_chain_state_size = 0; const char* testnet4_genesis_msg = "03/May/2024 000000000000000000001ebd58c244970b3aa9d783bb001011fbe8ea8e98e00e"; - const CScript testnet4_genesis_script = CScript() << "000000000000000000000000000000000000000000000000000000000000000000"_hex_v_u8 << OP_CHECKSIG; + const CScript testnet4_genesis_script = CScript() << "000000000000000000000000000000000000000000000000000000000000000000"_hex << OP_CHECKSIG; genesis = CreateGenesisBlock(testnet4_genesis_msg, testnet4_genesis_script, 1714777860, @@ -542,7 +542,7 @@ public: consensus.nPowTargetTimespan = 24 * 60 * 60; // one day consensus.nPowTargetSpacing = 10 * 60; consensus.fPowAllowMinDifficultyBlocks = true; - consensus.enforce_BIP94 = true; + consensus.enforce_BIP94 = opts.enforce_bip94; consensus.fPowNoRetargeting = true; consensus.nRuleChangeActivationThreshold = 108; // 75% for testchains consensus.nMinerConfirmationWindow = 144; // Faster than normal for regtest (144 instead of 2016) diff --git a/src/kernel/chainparams.h b/src/kernel/chainparams.h index c4584600fd..f7ea141c37 100644 --- a/src/kernel/chainparams.h +++ b/src/kernel/chainparams.h @@ -155,6 +155,7 @@ public: std::unordered_map<Consensus::DeploymentPos, VersionBitsParameters> version_bits_parameters{}; std::unordered_map<Consensus::BuriedDeployment, int> activation_heights{}; bool fastprune{false}; + bool enforce_bip94{false}; }; static std::unique_ptr<const CChainParams> RegTest(const RegTestOptions& options); diff --git a/src/kernel/mempool_options.h b/src/kernel/mempool_options.h index 4e1e24a11d..d57dbb393f 100644 --- a/src/kernel/mempool_options.h +++ b/src/kernel/mempool_options.h @@ -21,8 +21,6 @@ static constexpr unsigned int DEFAULT_MAX_MEMPOOL_SIZE_MB{300}; static constexpr unsigned int DEFAULT_BLOCKSONLY_MAX_MEMPOOL_SIZE_MB{5}; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static constexpr unsigned int DEFAULT_MEMPOOL_EXPIRY_HOURS{336}; -/** Default for -mempoolfullrbf, if the transaction replaceability signaling is ignored */ -static constexpr bool DEFAULT_MEMPOOL_FULL_RBF{true}; /** Whether to fall back to legacy V1 serialization when writing mempool.dat */ static constexpr bool DEFAULT_PERSIST_V1_DAT{false}; /** Default for -acceptnonstdtxn */ @@ -55,7 +53,6 @@ struct MemPoolOptions { std::optional<unsigned> max_datacarrier_bytes{DEFAULT_ACCEPT_DATACARRIER ? std::optional{MAX_OP_RETURN_RELAY} : std::nullopt}; bool permit_bare_multisig{DEFAULT_PERMIT_BAREMULTISIG}; bool require_standard{true}; - bool full_rbf{DEFAULT_MEMPOOL_FULL_RBF}; bool persist_v1_dat{DEFAULT_PERSIST_V1_DAT}; MemPoolLimits limits{}; diff --git a/src/key_io.cpp b/src/key_io.cpp index 29002afc45..6cece47e41 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -274,6 +274,9 @@ CExtKey DecodeExtKey(const std::string& str) key.Decode(data.data() + prefix.size()); } } + if (!data.empty()) { + memory_cleanse(data.data(), data.size()); + } return key; } diff --git a/src/logging.cpp b/src/logging.cpp index 9a54a12b42..5f055566ef 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -1,10 +1,11 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <logging.h> #include <memusage.h> +#include <util/check.h> #include <util/fs.h> #include <util/string.h> #include <util/threadnames.h> @@ -103,7 +104,6 @@ void BCLog::Logger::DisconnectTestLogger() m_cur_buffer_memusage = 0; m_buffer_lines_discarded = 0; m_msgs_before_open.clear(); - } void BCLog::Logger::DisableLogging() @@ -369,6 +369,8 @@ static size_t MemUsage(const BCLog::Logger::BufferedLog& buflog) void BCLog::Logger::FormatLogStrInPlace(std::string& str, BCLog::LogFlags category, BCLog::Level level, std::string_view source_file, int source_line, std::string_view logging_function, std::string_view threadname, SystemClock::time_point now, std::chrono::seconds mocktime) const { + if (!str.ends_with('\n')) str.push_back('\n'); + str.insert(0, GetLogPrefix(category, level)); if (m_log_sourcelocations) { @@ -392,21 +394,7 @@ void BCLog::Logger::LogPrintStr_(std::string_view str, std::string_view logging_ { std::string str_prefixed = LogEscapeMessage(str); - const bool starts_new_line = m_started_new_line; - m_started_new_line = !str.empty() && str[str.size()-1] == '\n'; - if (m_buffering) { - if (!starts_new_line) { - if (!m_msgs_before_open.empty()) { - m_msgs_before_open.back().str += str_prefixed; - m_cur_buffer_memusage += str_prefixed.size(); - return; - } else { - // unlikely edge case; add a marker that something was trimmed - str_prefixed.insert(0, "[...] "); - } - } - { BufferedLog buf{ .now=SystemClock::now(), @@ -436,9 +424,7 @@ void BCLog::Logger::LogPrintStr_(std::string_view str, std::string_view logging_ return; } - if (starts_new_line) { - FormatLogStrInPlace(str_prefixed, category, level, source_file, source_line, logging_function, util::ThreadGetInternalName(), SystemClock::now(), GetMockTime()); - } + FormatLogStrInPlace(str_prefixed, category, level, source_file, source_line, logging_function, util::ThreadGetInternalName(), SystemClock::now(), GetMockTime()); if (m_print_to_console) { // print to console diff --git a/src/logging.h b/src/logging.h index d583419d3a..fdc12c79b3 100644 --- a/src/logging.h +++ b/src/logging.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -105,13 +105,6 @@ namespace BCLog { size_t m_cur_buffer_memusage GUARDED_BY(m_cs){0}; size_t m_buffer_lines_discarded GUARDED_BY(m_cs){0}; - /** - * m_started_new_line is a state variable that will suppress printing of - * the timestamp when multiple calls are made that don't end in a - * newline. - */ - std::atomic_bool m_started_new_line{true}; - //! Category-specific log level. Overrides `m_log_level`. std::unordered_map<LogFlags, Level> m_category_log_levels GUARDED_BY(m_cs); @@ -245,28 +238,26 @@ static inline bool LogAcceptCategory(BCLog::LogFlags category, BCLog::Level leve /** Return true if str parses as a log category and set the flag */ bool GetLogCategory(BCLog::LogFlags& flag, std::string_view str); -// Be conservative when using functions that -// unconditionally log to debug.log! It should not be the case that an inbound -// peer can fill up a user's disk with debug.log entries. - template <typename... Args> -static inline void LogPrintf_(std::string_view logging_function, std::string_view source_file, const int source_line, const BCLog::LogFlags flag, const BCLog::Level level, const char* fmt, const Args&... args) +inline void LogPrintFormatInternal(std::string_view logging_function, std::string_view source_file, const int source_line, const BCLog::LogFlags flag, const BCLog::Level level, util::ConstevalFormatString<sizeof...(Args)> fmt, const Args&... args) { if (LogInstance().Enabled()) { std::string log_msg; try { log_msg = tfm::format(fmt, args...); } catch (tinyformat::format_error& fmterr) { - /* Original format string will have newline so don't add one here */ - log_msg = "Error \"" + std::string(fmterr.what()) + "\" while formatting log message: " + fmt; + log_msg = "Error \"" + std::string{fmterr.what()} + "\" while formatting log message: " + fmt.fmt; } LogInstance().LogPrintStr(log_msg, logging_function, source_file, source_line, flag, level); } } -#define LogPrintLevel_(category, level, ...) LogPrintf_(__func__, __FILE__, __LINE__, category, level, __VA_ARGS__) +#define LogPrintLevel_(category, level, ...) LogPrintFormatInternal(__func__, __FILE__, __LINE__, category, level, __VA_ARGS__) // Log unconditionally. +// Be conservative when using functions that unconditionally log to debug.log! +// It should not be the case that an inbound peer can fill up a user's storage +// with debug.log entries. #define LogInfo(...) LogPrintLevel_(BCLog::LogFlags::ALL, BCLog::Level::Info, __VA_ARGS__) #define LogWarning(...) LogPrintLevel_(BCLog::LogFlags::ALL, BCLog::Level::Warning, __VA_ARGS__) #define LogError(...) LogPrintLevel_(BCLog::LogFlags::ALL, BCLog::Level::Error, __VA_ARGS__) diff --git a/src/mapport.cpp b/src/mapport.cpp index 1920297be6..b7aadad6b4 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -2,33 +2,20 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep - #include <mapport.h> #include <clientversion.h> +#include <common/netif.h> +#include <common/pcp.h> #include <common/system.h> #include <logging.h> #include <net.h> #include <netaddress.h> #include <netbase.h> +#include <random.h> #include <util/thread.h> #include <util/threadinterrupt.h> -#ifdef USE_NATPMP -#include <compat/compat.h> -#include <natpmp.h> -#endif // USE_NATPMP - -#ifdef USE_UPNP -#include <miniupnpc/miniupnpc.h> -#include <miniupnpc/upnpcommands.h> -#include <miniupnpc/upnperrors.h> -// The minimum supported miniUPnPc API version is set to 17. This excludes -// versions with known vulnerabilities. -static_assert(MINIUPNPC_API_VERSION >= 17, "miniUPnPc API version >= 17 assumed"); -#endif // USE_UPNP - #include <atomic> #include <cassert> #include <chrono> @@ -36,7 +23,6 @@ static_assert(MINIUPNPC_API_VERSION >= 17, "miniUPnPc API version >= 17 assumed" #include <string> #include <thread> -#if defined(USE_NATPMP) || defined(USE_UPNP) static CThreadInterrupt g_mapport_interrupt; static std::thread g_mapport_thread; static std::atomic_uint g_mapport_enabled_protos{MapPortProtoFlag::NONE}; @@ -46,176 +32,96 @@ using namespace std::chrono_literals; static constexpr auto PORT_MAPPING_REANNOUNCE_PERIOD{20min}; static constexpr auto PORT_MAPPING_RETRY_PERIOD{5min}; -#ifdef USE_NATPMP -static uint16_t g_mapport_external_port = 0; -static bool NatpmpInit(natpmp_t* natpmp) -{ - const int r_init = initnatpmp(natpmp, /* detect gateway automatically */ 0, /* forced gateway - NOT APPLIED*/ 0); - if (r_init == 0) return true; - LogPrintf("natpmp: initnatpmp() failed with %d error.\n", r_init); - return false; -} - -static bool NatpmpDiscover(natpmp_t* natpmp, struct in_addr& external_ipv4_addr) -{ - const int r_send = sendpublicaddressrequest(natpmp); - if (r_send == 2 /* OK */) { - int r_read; - natpmpresp_t response; - do { - r_read = readnatpmpresponseorretry(natpmp, &response); - } while (r_read == NATPMP_TRYAGAIN); - - if (r_read == 0) { - external_ipv4_addr = response.pnu.publicaddress.addr; - return true; - } else if (r_read == NATPMP_ERR_NOGATEWAYSUPPORT) { - LogPrintf("natpmp: The gateway does not support NAT-PMP.\n"); - } else { - LogPrintf("natpmp: readnatpmpresponseorretry() for public address failed with %d error.\n", r_read); - } - } else { - LogPrintf("natpmp: sendpublicaddressrequest() failed with %d error.\n", r_send); - } - - return false; -} - -static bool NatpmpMapping(natpmp_t* natpmp, const struct in_addr& external_ipv4_addr, uint16_t private_port, bool& external_ip_discovered) +static bool ProcessPCP() { - const uint16_t suggested_external_port = g_mapport_external_port ? g_mapport_external_port : private_port; - const int r_send = sendnewportmappingrequest(natpmp, NATPMP_PROTOCOL_TCP, private_port, suggested_external_port, 3600 /*seconds*/); - if (r_send == 12 /* OK */) { - int r_read; - natpmpresp_t response; - do { - r_read = readnatpmpresponseorretry(natpmp, &response); - } while (r_read == NATPMP_TRYAGAIN); + // The same nonce is used for all mappings, this is allowed by the spec, and simplifies keeping track of them. + PCPMappingNonce pcp_nonce; + GetRandBytes(pcp_nonce); - if (r_read == 0) { - auto pm = response.pnu.newportmapping; - if (private_port == pm.privateport && pm.lifetime > 0) { - g_mapport_external_port = pm.mappedpublicport; - const CService external{external_ipv4_addr, pm.mappedpublicport}; - if (!external_ip_discovered && fDiscover) { - AddLocal(external, LOCAL_MAPPED); - external_ip_discovered = true; - } - LogPrintf("natpmp: Port mapping successful. External address = %s\n", external.ToStringAddrPort()); - return true; - } else { - LogPrintf("natpmp: Port mapping failed.\n"); + bool ret = false; + bool no_resources = false; + const uint16_t private_port = GetListenPort(); + // Multiply the reannounce period by two, as we'll try to renew approximately halfway. + const uint32_t requested_lifetime = std::chrono::seconds(PORT_MAPPING_REANNOUNCE_PERIOD * 2).count(); + uint32_t actual_lifetime = 0; + std::chrono::milliseconds sleep_time; + + // Local functor to handle result from PCP/NATPMP mapping. + auto handle_mapping = [&](std::variant<MappingResult, MappingError> &res) -> void { + if (MappingResult* mapping = std::get_if<MappingResult>(&res)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Info, "portmap: Added mapping %s\n", mapping->ToString()); + AddLocal(mapping->external, LOCAL_MAPPED); + ret = true; + actual_lifetime = std::min(actual_lifetime, mapping->lifetime); + } else if (MappingError *err = std::get_if<MappingError>(&res)) { + // Detailed error will already have been logged internally in respective Portmap function. + if (*err == MappingError::NO_RESOURCES) { + no_resources = true; } - } else if (r_read == NATPMP_ERR_NOGATEWAYSUPPORT) { - LogPrintf("natpmp: The gateway does not support NAT-PMP.\n"); - } else { - LogPrintf("natpmp: readnatpmpresponseorretry() for port mapping failed with %d error.\n", r_read); } - } else { - LogPrintf("natpmp: sendnewportmappingrequest() failed with %d error.\n", r_send); - } - - return false; -} + }; -static bool ProcessNatpmp() -{ - bool ret = false; - natpmp_t natpmp; - struct in_addr external_ipv4_addr; - if (NatpmpInit(&natpmp) && NatpmpDiscover(&natpmp, external_ipv4_addr)) { - bool external_ip_discovered = false; - const uint16_t private_port = GetListenPort(); - do { - ret = NatpmpMapping(&natpmp, external_ipv4_addr, private_port, external_ip_discovered); - } while (ret && g_mapport_interrupt.sleep_for(PORT_MAPPING_REANNOUNCE_PERIOD)); - g_mapport_interrupt.reset(); - - const int r_send = sendnewportmappingrequest(&natpmp, NATPMP_PROTOCOL_TCP, private_port, g_mapport_external_port, /* remove a port mapping */ 0); - g_mapport_external_port = 0; - if (r_send == 12 /* OK */) { - LogPrintf("natpmp: Port mapping removed successfully.\n"); + do { + actual_lifetime = requested_lifetime; + no_resources = false; // Set to true if there was any "no resources" error. + ret = false; // Set to true if any mapping succeeds. + + // IPv4 + std::optional<CNetAddr> gateway4 = QueryDefaultGateway(NET_IPV4); + if (!gateway4) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "portmap: Could not determine IPv4 default gateway\n"); } else { - LogPrintf("natpmp: sendnewportmappingrequest(0) failed with %d error.\n", r_send); + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "portmap: gateway [IPv4]: %s\n", gateway4->ToStringAddr()); + + // Open a port mapping on whatever local address we have toward the gateway. + struct in_addr inaddr_any; + inaddr_any.s_addr = htonl(INADDR_ANY); + auto res = PCPRequestPortMap(pcp_nonce, *gateway4, CNetAddr(inaddr_any), private_port, requested_lifetime); + MappingError* pcp_err = std::get_if<MappingError>(&res); + if (pcp_err && *pcp_err == MappingError::UNSUPP_VERSION) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "portmap: Got unsupported PCP version response, falling back to NAT-PMP\n"); + res = NATPMPRequestPortMap(*gateway4, private_port, requested_lifetime); + } + handle_mapping(res); } - } - closenatpmp(&natpmp); - return ret; -} -#endif // USE_NATPMP - -#ifdef USE_UPNP -static bool ProcessUpnp() -{ - bool ret = false; - std::string port = strprintf("%u", GetListenPort()); - const char * multicastif = nullptr; - const char * minissdpdpath = nullptr; - struct UPNPDev * devlist = nullptr; - char lanaddr[64]; - - int error = 0; - devlist = upnpDiscover(2000, multicastif, minissdpdpath, 0, 0, 2, &error); + // IPv6 + std::optional<CNetAddr> gateway6 = QueryDefaultGateway(NET_IPV6); + if (!gateway6) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "portmap: Could not determine IPv6 default gateway\n"); + } else { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "portmap: gateway [IPv6]: %s\n", gateway6->ToStringAddr()); - struct UPNPUrls urls; - struct IGDdatas data; - int r; -#if MINIUPNPC_API_VERSION <= 17 - r = UPNP_GetValidIGD(devlist, &urls, &data, lanaddr, sizeof(lanaddr)); -#else - r = UPNP_GetValidIGD(devlist, &urls, &data, lanaddr, sizeof(lanaddr), nullptr, 0); -#endif - if (r == 1) - { - if (fDiscover) { - char externalIPAddress[40]; - r = UPNP_GetExternalIPAddress(urls.controlURL, data.first.servicetype, externalIPAddress); - if (r != UPNPCOMMAND_SUCCESS) { - LogPrintf("UPnP: GetExternalIPAddress() returned %d\n", r); - } else { - if (externalIPAddress[0]) { - std::optional<CNetAddr> resolved{LookupHost(externalIPAddress, false)}; - if (resolved.has_value()) { - LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved->ToStringAddr()); - AddLocal(resolved.value(), LOCAL_MAPPED); - } - } else { - LogPrintf("UPnP: GetExternalIPAddress failed.\n"); - } + // Try to open pinholes for all routable local IPv6 addresses. + for (const auto &addr: GetLocalAddresses()) { + if (!addr.IsRoutable() || !addr.IsIPv6()) continue; + auto res = PCPRequestPortMap(pcp_nonce, *gateway6, addr, private_port, requested_lifetime); + handle_mapping(res); } } - std::string strDesc = PACKAGE_NAME " " + FormatFullVersion(); - - do { - r = UPNP_AddPortMapping(urls.controlURL, data.first.servicetype, port.c_str(), port.c_str(), lanaddr, strDesc.c_str(), "TCP", nullptr, "0"); + // Log message if we got NO_RESOURCES. + if (no_resources) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "portmap: At least one mapping failed because of a NO_RESOURCES error. This usually indicates that the port is already used on the router. If this is the only instance of bitcoin running on the network, this will resolve itself automatically. Otherwise, you might want to choose a different P2P port to prevent this conflict.\n"); + } - if (r != UPNPCOMMAND_SUCCESS) { - ret = false; - LogPrintf("AddPortMapping(%s, %s, %s) failed with code %d (%s)\n", port, port, lanaddr, r, strupnperror(r)); - break; - } else { - ret = true; - LogPrintf("UPnP Port Mapping successful.\n"); - } - } while (g_mapport_interrupt.sleep_for(PORT_MAPPING_REANNOUNCE_PERIOD)); - g_mapport_interrupt.reset(); + // Sanity-check returned lifetime. + if (actual_lifetime < 30) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "portmap: Got impossibly short mapping lifetime of %d seconds\n", actual_lifetime); + return false; + } + // RFC6887 11.2.1 recommends that clients send their first renewal packet at a time chosen with uniform random + // distribution in the range 1/2 to 5/8 of expiration time. + std::chrono::seconds sleep_time_min(actual_lifetime / 2); + std::chrono::seconds sleep_time_max(actual_lifetime * 5 / 8); + sleep_time = sleep_time_min + FastRandomContext().randrange<std::chrono::milliseconds>(sleep_time_max - sleep_time_min); + } while (ret && g_mapport_interrupt.sleep_for(sleep_time)); - r = UPNP_DeletePortMapping(urls.controlURL, data.first.servicetype, port.c_str(), "TCP", nullptr); - LogPrintf("UPNP_DeletePortMapping() returned: %d\n", r); - freeUPNPDevlist(devlist); devlist = nullptr; - FreeUPNPUrls(&urls); - } else { - LogPrintf("No valid UPnP IGDs found\n"); - freeUPNPDevlist(devlist); devlist = nullptr; - if (r != 0) - FreeUPNPUrls(&urls); - } + // We don't delete the mappings when the thread is interrupted because this would add additional complexity, so + // we rather just choose a fairly short expiry time. return ret; } -#endif // USE_UPNP static void ThreadMapPort() { @@ -223,23 +129,11 @@ static void ThreadMapPort() do { ok = false; -#ifdef USE_UPNP - // High priority protocol. - if (g_mapport_enabled_protos & MapPortProtoFlag::UPNP) { - g_mapport_current_proto = MapPortProtoFlag::UPNP; - ok = ProcessUpnp(); + if (g_mapport_enabled_protos & MapPortProtoFlag::PCP) { + g_mapport_current_proto = MapPortProtoFlag::PCP; + ok = ProcessPCP(); if (ok) continue; } -#endif // USE_UPNP - -#ifdef USE_NATPMP - // Low priority protocol. - if (g_mapport_enabled_protos & MapPortProtoFlag::NAT_PMP) { - g_mapport_current_proto = MapPortProtoFlag::NAT_PMP; - ok = ProcessNatpmp(); - if (ok) continue; - } -#endif // USE_NATPMP g_mapport_current_proto = MapPortProtoFlag::NONE; if (g_mapport_enabled_protos == MapPortProtoFlag::NONE) { @@ -275,15 +169,8 @@ static void DispatchMapPort() } if (g_mapport_enabled_protos & g_mapport_current_proto) { - // Enabling another protocol does not cause switching from the currently used one. return; } - - assert(g_mapport_thread.joinable()); - assert(!g_mapport_interrupt); - // Interrupt a protocol-specific loop in the ThreadUpnp() or in the ThreadNatpmp() - // to force trying the next protocol in the ThreadMapPort() loop. - g_mapport_interrupt(); } static void MapPortProtoSetEnabled(MapPortProtoFlag proto, bool enabled) @@ -295,10 +182,9 @@ static void MapPortProtoSetEnabled(MapPortProtoFlag proto, bool enabled) } } -void StartMapPort(bool use_upnp, bool use_natpmp) +void StartMapPort(bool use_pcp) { - MapPortProtoSetEnabled(MapPortProtoFlag::UPNP, use_upnp); - MapPortProtoSetEnabled(MapPortProtoFlag::NAT_PMP, use_natpmp); + MapPortProtoSetEnabled(MapPortProtoFlag::PCP, use_pcp); DispatchMapPort(); } @@ -317,18 +203,3 @@ void StopMapPort() g_mapport_interrupt.reset(); } } - -#else // #if defined(USE_NATPMP) || defined(USE_UPNP) -void StartMapPort(bool use_upnp, bool use_natpmp) -{ - // Intentionally left blank. -} -void InterruptMapPort() -{ - // Intentionally left blank. -} -void StopMapPort() -{ - // Intentionally left blank. -} -#endif // #if defined(USE_NATPMP) || defined(USE_UPNP) diff --git a/src/mapport.h b/src/mapport.h index 6f55c46f6c..8e33ede32f 100644 --- a/src/mapport.h +++ b/src/mapport.h @@ -5,17 +5,15 @@ #ifndef BITCOIN_MAPPORT_H #define BITCOIN_MAPPORT_H -static constexpr bool DEFAULT_UPNP = false; - static constexpr bool DEFAULT_NATPMP = false; enum MapPortProtoFlag : unsigned int { NONE = 0x00, - UPNP = 0x01, - NAT_PMP = 0x02, + // 0x01 was for UPnP, for which we dropped support. + PCP = 0x02, // PCP with NAT-PMP fallback. }; -void StartMapPort(bool use_upnp, bool use_natpmp); +void StartMapPort(bool use_pcp); void InterruptMapPort(); void StopMapPort(); diff --git a/src/memusage.h b/src/memusage.h index 08be66172e..9d9e549ef2 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -15,6 +15,7 @@ #include <map> #include <memory> #include <set> +#include <string> #include <vector> #include <unordered_map> #include <unordered_set> @@ -84,10 +85,22 @@ struct stl_shared_counter size_t weak_count; }; -template<typename X> -static inline size_t DynamicUsage(const std::vector<X>& v) +template<typename T, typename Allocator> +static inline size_t DynamicUsage(const std::vector<T, Allocator>& v) +{ + return MallocUsage(v.capacity() * sizeof(T)); +} + +static inline size_t DynamicUsage(const std::string& s) { - return MallocUsage(v.capacity() * sizeof(X)); + const char* s_ptr = reinterpret_cast<const char*>(&s); + // Don't count the dynamic memory used for string, if it resides in the + // "small string" optimization area (which stores data inside the object itself, up to some + // size; 15 bytes in modern libstdc++). + if (!std::less{}(s.data(), s_ptr) && !std::greater{}(s.data() + s.size(), s_ptr + sizeof(s))) { + return 0; + } + return MallocUsage(s.capacity()); } template<unsigned int N, typename X, typename S, typename D> diff --git a/src/net.cpp b/src/net.cpp index 257f67ed7b..8ea7f6ce44 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <net.h> @@ -12,6 +12,7 @@ #include <banman.h> #include <clientversion.h> #include <common/args.h> +#include <common/netif.h> #include <compat/compat.h> #include <consensus/consensus.h> #include <crypto/sha256.h> @@ -52,6 +53,8 @@ #include <optional> #include <unordered_map> +TRACEPOINT_SEMAPHORE(net, outbound_message); + /** Maximum number of block-relay-only anchor connections */ static constexpr size_t MAX_BLOCK_RELAY_ONLY_ANCHORS = 2; static_assert (MAX_BLOCK_RELAY_ONLY_ANCHORS <= static_cast<size_t>(MAX_BLOCK_RELAY_ONLY_CONNECTIONS), "MAX_BLOCK_RELAY_ONLY_ANCHORS must not exceed MAX_BLOCK_RELAY_ONLY_CONNECTIONS."); @@ -120,10 +123,12 @@ std::string strSubVersion; size_t CSerializedNetMsg::GetMemoryUsage() const noexcept { - // Don't count the dynamic memory used for the m_type string, by assuming it fits in the - // "small string" optimization area (which stores data inside the object itself, up to some - // size; 15 bytes in modern libstdc++). - return sizeof(*this) + memusage::DynamicUsage(data); + return sizeof(*this) + memusage::DynamicUsage(m_type) + memusage::DynamicUsage(data); +} + +size_t CNetMessage::GetMemoryUsage() const noexcept +{ + return sizeof(*this) + memusage::DynamicUsage(m_type) + m_recv.GetMemoryUsage(); } void CConnman::AddAddrFetch(const std::string& strDest) @@ -553,7 +558,6 @@ void CNode::CloseSocketDisconnect() fDisconnect = true; LOCK(m_sock_mutex); if (m_sock) { - LogDebug(BCLog::NET, "disconnecting peer=%d\n", id); m_sock.reset(); } m_i2p_sam_session.reset(); @@ -691,6 +695,18 @@ bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete) return true; } +std::string CNode::LogIP(bool log_ip) const +{ + return log_ip ? strprintf(" peeraddr=%s", addr.ToStringAddrPort()) : ""; +} + +std::string CNode::DisconnectMsg(bool log_ip) const +{ + return strprintf("disconnecting peer=%d%s", + GetId(), + LogIP(log_ip)); +} + V1Transport::V1Transport(const NodeId node_id) noexcept : m_magic_bytes{Params().MessageStart()}, m_node_id{node_id} { @@ -734,7 +750,7 @@ int V1Transport::readHeader(Span<const uint8_t> msg_bytes) // reject messages larger than MAX_SIZE or MAX_PROTOCOL_MESSAGE_LENGTH if (hdr.nMessageSize > MAX_SIZE || hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_LENGTH) { - LogDebug(BCLog::NET, "Header error: Size too large (%s, %u bytes), peer=%d\n", SanitizeString(hdr.GetCommand()), hdr.nMessageSize, m_node_id); + LogDebug(BCLog::NET, "Header error: Size too large (%s, %u bytes), peer=%d\n", SanitizeString(hdr.GetMessageType()), hdr.nMessageSize, m_node_id); return -1; } @@ -781,7 +797,7 @@ CNetMessage V1Transport::GetReceivedMessage(const std::chrono::microseconds time CNetMessage msg(std::move(vRecv)); // store message type string, time, and sizes - msg.m_type = hdr.GetCommand(); + msg.m_type = hdr.GetMessageType(); msg.m_time = time; msg.m_message_size = hdr.nMessageSize; msg.m_raw_message_size = hdr.nMessageSize + CMessageHeader::HEADER_SIZE; @@ -799,9 +815,9 @@ CNetMessage V1Transport::GetReceivedMessage(const std::chrono::microseconds time HexStr(hdr.pchChecksum), m_node_id); reject_message = true; - } else if (!hdr.IsCommandValid()) { + } else if (!hdr.IsMessageTypeValid()) { LogDebug(BCLog::NET, "Header error: Invalid message type (%s, %u bytes), peer=%d\n", - SanitizeString(hdr.GetCommand()), msg.m_message_size, m_node_id); + SanitizeString(hdr.GetMessageType()), msg.m_message_size, m_node_id); reject_message = true; } @@ -1183,7 +1199,7 @@ bool V2Transport::ProcessReceivedPacketBytes() noexcept // - 12 bytes of message type // - payload static constexpr size_t MAX_CONTENTS_LEN = - 1 + CMessageHeader::COMMAND_SIZE + + 1 + CMessageHeader::MESSAGE_TYPE_SIZE + std::min<size_t>(MAX_SIZE, MAX_PROTOCOL_MESSAGE_LENGTH); if (m_recv_buffer.size() == BIP324Cipher::LENGTH_LEN) { @@ -1398,12 +1414,12 @@ std::optional<std::string> V2Transport::GetMessageType(Span<const uint8_t>& cont } } - if (contents.size() < CMessageHeader::COMMAND_SIZE) { + if (contents.size() < CMessageHeader::MESSAGE_TYPE_SIZE) { return std::nullopt; // Long encoding needs 12 message type bytes. } size_t msg_type_len{0}; - while (msg_type_len < CMessageHeader::COMMAND_SIZE && contents[msg_type_len] != 0) { + while (msg_type_len < CMessageHeader::MESSAGE_TYPE_SIZE && contents[msg_type_len] != 0) { // Verify that message type bytes before the first 0x00 are in range. if (contents[msg_type_len] < ' ' || contents[msg_type_len] > 0x7F) { return {}; @@ -1411,13 +1427,13 @@ std::optional<std::string> V2Transport::GetMessageType(Span<const uint8_t>& cont ++msg_type_len; } std::string ret{reinterpret_cast<const char*>(contents.data()), msg_type_len}; - while (msg_type_len < CMessageHeader::COMMAND_SIZE) { + while (msg_type_len < CMessageHeader::MESSAGE_TYPE_SIZE) { // Verify that message type bytes after the first 0x00 are also 0x00. if (contents[msg_type_len] != 0) return {}; ++msg_type_len; } // Strip message type bytes of contents. - contents = contents.subspan(CMessageHeader::COMMAND_SIZE); + contents = contents.subspan(CMessageHeader::MESSAGE_TYPE_SIZE); return ret; } @@ -1469,9 +1485,9 @@ bool V2Transport::SetMessageToSend(CSerializedNetMsg& msg) noexcept } else { // Initialize with zeroes, and then write the message type string starting at offset 1. // This means contents[0] and the unused positions in contents[1..13] remain 0x00. - contents.resize(1 + CMessageHeader::COMMAND_SIZE + msg.data.size(), 0); + contents.resize(1 + CMessageHeader::MESSAGE_TYPE_SIZE + msg.data.size(), 0); std::copy(msg.m_type.begin(), msg.m_type.end(), contents.data() + 1); - std::copy(msg.data.begin(), msg.data.end(), contents.begin() + 1 + CMessageHeader::COMMAND_SIZE); + std::copy(msg.data.begin(), msg.data.end(), contents.begin() + 1 + CMessageHeader::MESSAGE_TYPE_SIZE); } // Construct ciphertext in send buffer. m_send_buffer.resize(contents.size() + BIP324Cipher::EXPANSION); @@ -1630,7 +1646,7 @@ std::pair<size_t, bool> CConnman::SocketSendData(CNode& node) const // error int nErr = WSAGetLastError(); if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) { - LogDebug(BCLog::NET, "socket send error for peer=%d: %s\n", node.GetId(), NetworkErrorString(nErr)); + LogDebug(BCLog::NET, "socket send error, %s: %s\n", node.DisconnectMsg(fLogIPs), NetworkErrorString(nErr)); node.CloseSocketDisconnect(); } } @@ -1874,7 +1890,7 @@ void CConnman::DisconnectNodes() // Disconnect any connected nodes for (CNode* pnode : m_nodes) { if (!pnode->fDisconnect) { - LogDebug(BCLog::NET, "Network not active, dropping peer=%d\n", pnode->GetId()); + LogDebug(BCLog::NET, "Network not active, %s\n", pnode->DisconnectMsg(fLogIPs)); pnode->fDisconnect = true; } } @@ -1966,26 +1982,43 @@ bool CConnman::InactivityCheck(const CNode& node) const if (!ShouldRunInactivityChecks(node, now)) return false; - if (last_recv.count() == 0 || last_send.count() == 0) { - LogDebug(BCLog::NET, "socket no message in first %i seconds, %d %d peer=%d\n", count_seconds(m_peer_connect_timeout), last_recv.count() != 0, last_send.count() != 0, node.GetId()); + bool has_received{last_recv.count() != 0}; + bool has_sent{last_send.count() != 0}; + + if (!has_received || !has_sent) { + std::string has_never; + if (!has_received) has_never += ", never received from peer"; + if (!has_sent) has_never += ", never sent to peer"; + LogDebug(BCLog::NET, + "socket no message in first %i seconds%s, %s\n", + count_seconds(m_peer_connect_timeout), + has_never, + node.DisconnectMsg(fLogIPs) + ); return true; } if (now > last_send + TIMEOUT_INTERVAL) { - LogDebug(BCLog::NET, "socket sending timeout: %is peer=%d\n", count_seconds(now - last_send), node.GetId()); + LogDebug(BCLog::NET, + "socket sending timeout: %is, %s\n", count_seconds(now - last_send), + node.DisconnectMsg(fLogIPs) + ); return true; } if (now > last_recv + TIMEOUT_INTERVAL) { - LogDebug(BCLog::NET, "socket receive timeout: %is peer=%d\n", count_seconds(now - last_recv), node.GetId()); + LogDebug(BCLog::NET, + "socket receive timeout: %is, %s\n", count_seconds(now - last_recv), + node.DisconnectMsg(fLogIPs) + ); return true; } if (!node.fSuccessfullyConnected) { if (node.m_transport->GetInfo().transport_type == TransportProtocolType::DETECTING) { - LogDebug(BCLog::NET, "V2 handshake timeout peer=%d\n", node.GetId()); + LogDebug(BCLog::NET, "V2 handshake timeout, %s\n", node.DisconnectMsg(fLogIPs)); } else { - LogDebug(BCLog::NET, "version handshake timeout peer=%d\n", node.GetId()); + LogDebug(BCLog::NET, "version handshake timeout, %s\n", node.DisconnectMsg(fLogIPs)); } return true; } @@ -2113,6 +2146,10 @@ void CConnman::SocketHandlerConnected(const std::vector<CNode*>& nodes, { bool notify = false; if (!pnode->ReceiveMsgBytes({pchBuf, (size_t)nBytes}, notify)) { + LogDebug(BCLog::NET, + "receiving message bytes failed, %s\n", + pnode->DisconnectMsg(fLogIPs) + ); pnode->CloseSocketDisconnect(); } RecordBytesRecv(nBytes); @@ -2125,7 +2162,7 @@ void CConnman::SocketHandlerConnected(const std::vector<CNode*>& nodes, { // socket closed gracefully if (!pnode->fDisconnect) { - LogDebug(BCLog::NET, "socket closed for peer=%d\n", pnode->GetId()); + LogDebug(BCLog::NET, "socket closed, %s\n", pnode->DisconnectMsg(fLogIPs)); } pnode->CloseSocketDisconnect(); } @@ -2136,7 +2173,7 @@ void CConnman::SocketHandlerConnected(const std::vector<CNode*>& nodes, if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) { if (!pnode->fDisconnect) { - LogDebug(BCLog::NET, "socket recv error for peer=%d: %s\n", pnode->GetId(), NetworkErrorString(nErr)); + LogDebug(BCLog::NET, "socket recv error, %s: %s\n", pnode->DisconnectMsg(fLogIPs), NetworkErrorString(nErr)); } pnode->CloseSocketDisconnect(); } @@ -2693,6 +2730,8 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect, Spa const auto current_time{NodeClock::now()}; int nTries = 0; + const auto reachable_nets{g_reachable_nets.All()}; + while (!interruptNet) { if (anchor && !m_anchors.empty()) { @@ -2724,7 +2763,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect, Spa if (!addr.IsValid()) { // No tried table collisions. Select a new table address // for our feeler. - std::tie(addr, addr_last_try) = addrman.Select(true); + std::tie(addr, addr_last_try) = addrman.Select(true, reachable_nets); } else if (AlreadyConnectedToAddress(addr)) { // If test-before-evict logic would have us connect to a // peer that we're already connected to, just mark that @@ -2733,14 +2772,16 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect, Spa // a currently-connected peer. addrman.Good(addr); // Select a new table address for our feeler instead. - std::tie(addr, addr_last_try) = addrman.Select(true); + std::tie(addr, addr_last_try) = addrman.Select(true, reachable_nets); } } else { // Not a feeler // If preferred_net has a value set, pick an extra outbound // peer from that network. The eviction logic in net_processing // ensures that a peer from another network will be evicted. - std::tie(addr, addr_last_try) = addrman.Select(false, preferred_net); + std::tie(addr, addr_last_try) = preferred_net.has_value() + ? addrman.Select(false, {*preferred_net}) + : addrman.Select(false, reachable_nets); } // Require outbound IPv4/IPv6 connections, other than feelers, to be to distinct network groups @@ -2945,7 +2986,7 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai return; pnode->grantOutbound = std::move(grant_outbound); - m_msgproc->InitializeNode(*pnode, nLocalServices); + m_msgproc->InitializeNode(*pnode, m_local_services); { LOCK(m_nodes_mutex); m_nodes.push_back(pnode); @@ -3049,14 +3090,14 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, socklen_t len = sizeof(sockaddr); if (!addrBind.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { - strError = strprintf(Untranslated("Bind address family for %s not supported"), addrBind.ToStringAddrPort()); + strError = Untranslated(strprintf("Bind address family for %s not supported", addrBind.ToStringAddrPort())); LogPrintLevel(BCLog::NET, BCLog::Level::Error, "%s\n", strError.original); return false; } std::unique_ptr<Sock> sock = CreateSock(addrBind.GetSAFamily(), SOCK_STREAM, IPPROTO_TCP); if (!sock) { - strError = strprintf(Untranslated("Couldn't open socket for incoming connections (socket returned error %s)"), NetworkErrorString(WSAGetLastError())); + strError = Untranslated(strprintf("Couldn't open socket for incoming connections (socket returned error %s)", NetworkErrorString(WSAGetLastError()))); LogPrintLevel(BCLog::NET, BCLog::Level::Error, "%s\n", strError.original); return false; } @@ -3064,7 +3105,7 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, // Allow binding if the port is still in TIME_WAIT state after // the program was closed and restarted. if (sock->SetSockOpt(SOL_SOCKET, SO_REUSEADDR, (sockopt_arg_type)&nOne, sizeof(int)) == SOCKET_ERROR) { - strError = strprintf(Untranslated("Error setting SO_REUSEADDR on socket: %s, continuing anyway"), NetworkErrorString(WSAGetLastError())); + strError = Untranslated(strprintf("Error setting SO_REUSEADDR on socket: %s, continuing anyway", NetworkErrorString(WSAGetLastError()))); LogPrintf("%s\n", strError.original); } @@ -3073,14 +3114,14 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, if (addrBind.IsIPv6()) { #ifdef IPV6_V6ONLY if (sock->SetSockOpt(IPPROTO_IPV6, IPV6_V6ONLY, (sockopt_arg_type)&nOne, sizeof(int)) == SOCKET_ERROR) { - strError = strprintf(Untranslated("Error setting IPV6_V6ONLY on socket: %s, continuing anyway"), NetworkErrorString(WSAGetLastError())); + strError = Untranslated(strprintf("Error setting IPV6_V6ONLY on socket: %s, continuing anyway", NetworkErrorString(WSAGetLastError()))); LogPrintf("%s\n", strError.original); } #endif #ifdef WIN32 int nProtLevel = PROTECTION_LEVEL_UNRESTRICTED; if (sock->SetSockOpt(IPPROTO_IPV6, IPV6_PROTECTION_LEVEL, (const char*)&nProtLevel, sizeof(int)) == SOCKET_ERROR) { - strError = strprintf(Untranslated("Error setting IPV6_PROTECTION_LEVEL on socket: %s, continuing anyway"), NetworkErrorString(WSAGetLastError())); + strError = Untranslated(strprintf("Error setting IPV6_PROTECTION_LEVEL on socket: %s, continuing anyway", NetworkErrorString(WSAGetLastError()))); LogPrintf("%s\n", strError.original); } #endif @@ -3089,7 +3130,7 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, if (sock->Bind(reinterpret_cast<struct sockaddr*>(&sockaddr), len) == SOCKET_ERROR) { int nErr = WSAGetLastError(); if (nErr == WSAEADDRINUSE) - strError = strprintf(_("Unable to bind to %s on this computer. %s is probably already running."), addrBind.ToStringAddrPort(), PACKAGE_NAME); + strError = strprintf(_("Unable to bind to %s on this computer. %s is probably already running."), addrBind.ToStringAddrPort(), CLIENT_NAME); else strError = strprintf(_("Unable to bind to %s on this computer (bind returned error %s)"), addrBind.ToStringAddrPort(), NetworkErrorString(nErr)); LogPrintLevel(BCLog::NET, BCLog::Level::Error, "%s\n", strError.original); @@ -3114,46 +3155,10 @@ void Discover() if (!fDiscover) return; -#ifdef WIN32 - // Get local host IP - char pszHostName[256] = ""; - if (gethostname(pszHostName, sizeof(pszHostName)) != SOCKET_ERROR) - { - const std::vector<CNetAddr> addresses{LookupHost(pszHostName, 0, true)}; - for (const CNetAddr& addr : addresses) - { - if (AddLocal(addr, LOCAL_IF)) - LogPrintf("%s: %s - %s\n", __func__, pszHostName, addr.ToStringAddr()); - } - } -#elif (HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS) - // Get local host ip - struct ifaddrs* myaddrs; - if (getifaddrs(&myaddrs) == 0) - { - for (struct ifaddrs* ifa = myaddrs; ifa != nullptr; ifa = ifa->ifa_next) - { - if (ifa->ifa_addr == nullptr) continue; - if ((ifa->ifa_flags & IFF_UP) == 0) continue; - if ((ifa->ifa_flags & IFF_LOOPBACK) != 0) continue; - if (ifa->ifa_addr->sa_family == AF_INET) - { - struct sockaddr_in* s4 = (struct sockaddr_in*)(ifa->ifa_addr); - CNetAddr addr(s4->sin_addr); - if (AddLocal(addr, LOCAL_IF)) - LogPrintf("%s: IPv4 %s: %s\n", __func__, ifa->ifa_name, addr.ToStringAddr()); - } - else if (ifa->ifa_addr->sa_family == AF_INET6) - { - struct sockaddr_in6* s6 = (struct sockaddr_in6*)(ifa->ifa_addr); - CNetAddr addr(s6->sin6_addr); - if (AddLocal(addr, LOCAL_IF)) - LogPrintf("%s: IPv6 %s: %s\n", __func__, ifa->ifa_name, addr.ToStringAddr()); - } - } - freeifaddrs(myaddrs); + for (const CNetAddr &addr: GetLocalAddresses()) { + if (AddLocal(addr, LOCAL_IF)) + LogPrintf("%s: %s\n", __func__, addr.ToStringAddr()); } -#endif } void CConnman::SetNetworkActive(bool active) @@ -3438,6 +3443,7 @@ void CConnman::StopNodes() std::vector<CNode*> nodes; WITH_LOCK(m_nodes_mutex, nodes.swap(m_nodes)); for (CNode* pnode : nodes) { + LogDebug(BCLog::NET, "%s\n", pnode->DisconnectMsg(fLogIPs)); pnode->CloseSocketDisconnect(); DeleteNode(pnode); } @@ -3742,7 +3748,7 @@ uint64_t CConnman::GetTotalBytesSent() const ServiceFlags CConnman::GetLocalServices() const { - return nLocalServices; + return m_local_services; } static std::unique_ptr<Transport> MakeTransport(NodeId id, bool use_v2transport, bool inbound) noexcept @@ -3803,7 +3809,7 @@ void CNode::MarkReceivedMsgsForProcessing() for (const auto& msg : vRecvMsg) { // vRecvMsg contains only completed CNetMessage // the single possible partially deserialized message are held by TransportDeserializer - nSizeAdded += msg.m_raw_message_size; + nSizeAdded += msg.GetMemoryUsage(); } LOCK(m_msg_process_queue_mutex); @@ -3820,7 +3826,7 @@ std::optional<std::pair<CNetMessage, bool>> CNode::PollMessage() std::list<CNetMessage> msgs; // Just take one message msgs.splice(msgs.begin(), m_msg_process_queue, m_msg_process_queue.begin()); - m_msg_process_queue_size -= msgs.front().m_raw_message_size; + m_msg_process_queue_size -= msgs.front().GetMemoryUsage(); fPauseRecv = m_msg_process_queue_size > m_recv_flood_size; return std::make_pair(std::move(msgs.front()), !m_msg_process_queue.empty()); @@ -3840,7 +3846,7 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) CaptureMessage(pnode->addr, msg.m_type, msg.data, /*is_incoming=*/false); } - TRACE6(net, outbound_message, + TRACEPOINT(net, outbound_message, pnode->GetId(), pnode->m_addr_name.c_str(), pnode->ConnectionTypeAsString().c_str(), @@ -3967,7 +3973,7 @@ static void CaptureMessageToFile(const CAddress& addr, ser_writedata64(f, now.count()); f << Span{msg_type}; - for (auto i = msg_type.length(); i < CMessageHeader::COMMAND_SIZE; ++i) { + for (auto i = msg_type.length(); i < CMessageHeader::MESSAGE_TYPE_SIZE; ++i) { f << uint8_t{'\0'}; } uint32_t size = data.size(); @@ -148,7 +148,7 @@ enum LOCAL_NONE, // unknown LOCAL_IF, // address a local interface listens on LOCAL_BIND, // address explicit bound to - LOCAL_MAPPED, // address reported by UPnP or NAT-PMP + LOCAL_MAPPED, // address reported by PCP LOCAL_MANUAL, // address explicitly specified (-externalip=) LOCAL_MAX @@ -245,6 +245,9 @@ public: CNetMessage(const CNetMessage&) = delete; CNetMessage& operator=(CNetMessage&&) = default; CNetMessage& operator=(const CNetMessage&) = delete; + + /** Compute total memory usage of this object (own memory + any dynamic memory). */ + size_t GetMemoryUsage() const noexcept; }; /** The Transport converts one connection's sent messages to wire bytes, and received bytes back. */ @@ -944,6 +947,22 @@ public: std::string ConnectionTypeAsString() const { return ::ConnectionTypeAsString(m_conn_type); } + /** + * Helper function to optionally log the IP address. + * + * @param[in] log_ip whether to include the IP address + * @return " peeraddr=..." or "" + */ + std::string LogIP(bool log_ip) const; + + /** + * Helper function to log disconnects. + * + * @param[in] log_ip whether to include the IP address + * @return "disconnecting peer=..." and optionally "peeraddr=..." + */ + std::string DisconnectMsg(bool log_ip) const; + /** A ping-pong round trip has completed successfully. Update latest and minimum ping times. */ void PongReceived(std::chrono::microseconds ping_time) { m_last_ping_time = ping_time; @@ -1035,7 +1054,7 @@ public: struct Options { - ServiceFlags nLocalServices = NODE_NONE; + ServiceFlags m_local_services = NODE_NONE; int m_max_automatic_connections = 0; CClientUIInterface* uiInterface = nullptr; NetEventsInterface* m_msgproc = nullptr; @@ -1065,7 +1084,7 @@ public: { AssertLockNotHeld(m_total_bytes_sent_mutex); - nLocalServices = connOptions.nLocalServices; + m_local_services = connOptions.m_local_services; m_max_automatic_connections = connOptions.m_max_automatic_connections; m_max_outbound_full_relay = std::min(MAX_OUTBOUND_FULL_RELAY_CONNECTIONS, m_max_automatic_connections); m_max_outbound_block_relay = std::min(MAX_BLOCK_RELAY_ONLY_CONNECTIONS, m_max_automatic_connections - m_max_outbound_full_relay); @@ -1152,7 +1171,7 @@ public: * Return all or many randomly selected addresses, optionally by network. * * @param[in] max_addresses Maximum number of addresses to return (0 = all). - * @param[in] max_pct Maximum percentage of addresses to return (0 = all). + * @param[in] max_pct Maximum percentage of addresses to return (0 = all). Value must be from 0 to 100. * @param[in] network Select only addresses of this network (nullopt = all). * @param[in] filtered Select only addresses that are considered high quality (false = all). */ @@ -1223,8 +1242,8 @@ public: //! Updates the local services that this node advertises to other peers //! during connection handshake. - void AddLocalServices(ServiceFlags services) { nLocalServices = ServiceFlags(nLocalServices | services); }; - void RemoveLocalServices(ServiceFlags services) { nLocalServices = ServiceFlags(nLocalServices & ~services); } + void AddLocalServices(ServiceFlags services) { m_local_services = ServiceFlags(m_local_services | services); }; + void RemoveLocalServices(ServiceFlags services) { m_local_services = ServiceFlags(m_local_services & ~services); } uint64_t GetMaxOutboundTarget() const EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); std::chrono::seconds GetMaxOutboundTimeframe() const; @@ -1470,7 +1489,7 @@ private: * * \sa Peer::our_services */ - std::atomic<ServiceFlags> nLocalServices; + std::atomic<ServiceFlags> m_local_services; std::unique_ptr<CSemaphore> semOutbound; std::unique_ptr<CSemaphore> semAddnode; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 6553393160..a19443c0f5 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -24,6 +24,7 @@ #include <netmessagemaker.h> #include <node/blockstorage.h> #include <node/timeoffsets.h> +#include <node/txdownloadman.h> #include <node/txreconciliation.h> #include <node/warnings.h> #include <policy/fees.h> @@ -56,6 +57,8 @@ using namespace util::hex_literals; +TRACEPOINT_SEMAPHORE(net, inbound_message); + /** Headers download timeout. * Timeout = base + per_header * (expected number of headers) */ static constexpr auto HEADERS_DOWNLOAD_TIMEOUT_BASE = 15min; @@ -88,22 +91,6 @@ static constexpr auto PING_INTERVAL{2min}; static const unsigned int MAX_LOCATOR_SZ = 101; /** The maximum number of entries in an 'inv' protocol message */ static const unsigned int MAX_INV_SZ = 50000; -/** Maximum number of in-flight transaction requests from a peer. It is not a hard limit, but the threshold at which - * point the OVERLOADED_PEER_TX_DELAY kicks in. */ -static constexpr int32_t MAX_PEER_TX_REQUEST_IN_FLIGHT = 100; -/** Maximum number of transactions to consider for requesting, per peer. It provides a reasonable DoS limit to - * per-peer memory usage spent on announcements, while covering peers continuously sending INVs at the maximum - * rate (by our own policy, see INVENTORY_BROADCAST_PER_SECOND) for several minutes, while not receiving - * the actual transaction (from any peer) in response to requests for them. */ -static constexpr int32_t MAX_PEER_TX_ANNOUNCEMENTS = 5000; -/** How long to delay requesting transactions via txids, if we have wtxid-relaying peers */ -static constexpr auto TXID_RELAY_DELAY{2s}; -/** How long to delay requesting transactions from non-preferred peers */ -static constexpr auto NONPREF_PEER_TX_DELAY{2s}; -/** How long to delay requesting transactions from overloaded peers (see MAX_PEER_TX_REQUEST_IN_FLIGHT). */ -static constexpr auto OVERLOADED_PEER_TX_DELAY{2s}; -/** How long to wait before downloading a transaction from an additional peer */ -static constexpr auto GETDATA_TX_INTERVAL{60s}; /** Limit to avoid sending big packets. Not used in processing incoming GETDATA for compatibility */ static const unsigned int MAX_GETDATA_SZ = 1000; /** Number of blocks that can be requested at any given time from a single peer. */ @@ -113,9 +100,6 @@ static const int MAX_BLOCKS_IN_TRANSIT_PER_PEER = 16; static constexpr auto BLOCK_STALLING_TIMEOUT_DEFAULT{2s}; /** Maximum timeout for stalling block download. */ static constexpr auto BLOCK_STALLING_TIMEOUT_MAX{64s}; -/** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends - * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ -static const unsigned int MAX_HEADERS_RESULTS = 2000; /** Maximum depth of blocks we're willing to serve as compact blocks to peers * when requested. For older blocks, a regular BLOCK response will be sent. */ static const int MAX_CMPCTBLOCK_DEPTH = 5; @@ -158,7 +142,7 @@ static constexpr unsigned int INVENTORY_BROADCAST_TARGET = INVENTORY_BROADCAST_P /** Maximum number of inventory items to send per transmission. */ static constexpr unsigned int INVENTORY_BROADCAST_MAX = 1000; static_assert(INVENTORY_BROADCAST_MAX >= INVENTORY_BROADCAST_TARGET, "INVENTORY_BROADCAST_MAX too low"); -static_assert(INVENTORY_BROADCAST_MAX <= MAX_PEER_TX_ANNOUNCEMENTS, "INVENTORY_BROADCAST_MAX too high"); +static_assert(INVENTORY_BROADCAST_MAX <= node::MAX_PEER_TX_ANNOUNCEMENTS, "INVENTORY_BROADCAST_MAX too high"); /** Average delay between feefilter broadcasts in seconds. */ static constexpr auto AVG_FEEFILTER_BROADCAST_INTERVAL{10min}; /** Maximum feefilter broadcast delay after significant change. */ @@ -224,6 +208,9 @@ struct Peer { /** Services this peer offered to us. */ std::atomic<ServiceFlags> m_their_services{NODE_NONE}; + //! Whether this peer is an inbound connection + const bool m_is_inbound; + /** Protects misbehavior data members */ Mutex m_misbehavior_mutex; /** Whether this peer should be disconnected and marked as discouraged (unless it has NetPermissionFlags::NoBan permission). */ @@ -394,9 +381,10 @@ struct Peer { * timestamp the peer sent in the version message. */ std::atomic<std::chrono::seconds> m_time_offset{0s}; - explicit Peer(NodeId id, ServiceFlags our_services) + explicit Peer(NodeId id, ServiceFlags our_services, bool is_inbound) : m_id{id} , m_our_services{our_services} + , m_is_inbound{is_inbound} {} private: @@ -476,11 +464,6 @@ struct CNodeState { //! Time of last new block announcement int64_t m_last_block_announcement{0}; - - //! Whether this peer is an inbound connection - const bool m_is_inbound; - - CNodeState(bool is_inbound) : m_is_inbound(is_inbound) {} }; class PeerManagerImpl final : public PeerManager @@ -519,6 +502,7 @@ public: std::optional<std::string> FetchBlock(NodeId peer_id, const CBlockIndex& block_index) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); + std::vector<TxOrphanage::OrphanTxBase> GetOrphanTransactions() override EXCLUSIVE_LOCKS_REQUIRED(!m_tx_download_mutex); PeerManagerInfo GetInfo() const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void SendPings() override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void RelayTransaction(const uint256& txid, const uint256& wtxid) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); @@ -583,12 +567,18 @@ private: bool MaybeDiscourageAndDisconnect(CNode& pnode, Peer& peer); /** Handle a transaction whose result was not MempoolAcceptResult::ResultType::VALID. - * @param[in] maybe_add_extra_compact_tx Whether this tx should be added to vExtraTxnForCompact. + * @param[in] first_time_failure Whether we should consider inserting into vExtraTxnForCompact, adding + * a new orphan to resolve, or looking for a package to submit. + * Set to true for transactions just received over p2p. * Set to false if the tx has already been rejected before, - * e.g. is an orphan, to avoid adding duplicate entries. - * Updates m_txrequest, m_lazy_recent_rejects, m_lazy_recent_rejects_reconsiderable, m_orphanage, and vExtraTxnForCompact. */ - void ProcessInvalidTx(NodeId nodeid, const CTransactionRef& tx, const TxValidationState& result, - bool maybe_add_extra_compact_tx) + * e.g. is already in the orphanage, to avoid adding duplicate entries. + * Updates m_txrequest, m_lazy_recent_rejects, m_lazy_recent_rejects_reconsiderable, m_orphanage, and vExtraTxnForCompact. + * + * @returns a PackageToValidate if this transaction has a reconsiderable failure and an eligible package was found, + * or std::nullopt otherwise. + */ + std::optional<node::PackageToValidate> ProcessInvalidTx(NodeId nodeid, const CTransactionRef& tx, const TxValidationState& result, + bool first_time_failure) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, m_tx_download_mutex); /** Handle a transaction whose result was MempoolAcceptResult::ResultType::VALID. @@ -596,40 +586,10 @@ private: void ProcessValidTx(NodeId nodeid, const CTransactionRef& tx, const std::list<CTransactionRef>& replaced_transactions) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, m_tx_download_mutex); - struct PackageToValidate { - const Package m_txns; - const std::vector<NodeId> m_senders; - /** Construct a 1-parent-1-child package. */ - explicit PackageToValidate(const CTransactionRef& parent, - const CTransactionRef& child, - NodeId parent_sender, - NodeId child_sender) : - m_txns{parent, child}, - m_senders {parent_sender, child_sender} - {} - - std::string ToString() const { - Assume(m_txns.size() == 2); - return strprintf("parent %s (wtxid=%s, sender=%d) + child %s (wtxid=%s, sender=%d)", - m_txns.front()->GetHash().ToString(), - m_txns.front()->GetWitnessHash().ToString(), - m_senders.front(), - m_txns.back()->GetHash().ToString(), - m_txns.back()->GetWitnessHash().ToString(), - m_senders.back()); - } - }; - /** Handle the results of package validation: calls ProcessValidTx and ProcessInvalidTx for * individual transactions, and caches rejection for the package as a group. */ - void ProcessPackageResult(const PackageToValidate& package_to_validate, const PackageMempoolAcceptResult& package_result) - EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, m_tx_download_mutex); - - /** Look for a child of this transaction in the orphanage to form a 1-parent-1-child package, - * skipping any combinations that have already been tried. Return the resulting package along with - * the senders of its respective transactions, or std::nullopt if no package is found. */ - std::optional<PackageToValidate> Find1P1CPackage(const CTransactionRef& ptx, NodeId nodeid) + void ProcessPackageResult(const node::PackageToValidate& package_to_validate, const PackageMempoolAcceptResult& package_result) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, m_tx_download_mutex); /** @@ -722,12 +682,6 @@ private: void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); - /** Register with TxRequestTracker that an INV has been received from a - * peer. The announcement parameters are decided in PeerManager and then - * passed to TxRequestTracker. */ - void AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std::chrono::microseconds current_time) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_tx_download_mutex); - /** Send a message to a peer */ void PushMessage(CNode& node, CSerializedNetMsg&& msg) const { m_connman.PushMessage(&node, std::move(msg)); } template <typename... Args> @@ -784,7 +738,8 @@ private: * - Each data structure's limits hold (m_orphanage max size, m_txrequest per-peer limits, etc). */ Mutex m_tx_download_mutex ACQUIRED_BEFORE(m_mempool.cs); - TxRequestTracker m_txrequest GUARDED_BY(m_tx_download_mutex); + node::TxDownloadManager m_txdownloadman GUARDED_BY(m_tx_download_mutex); + std::unique_ptr<TxReconciliationTracker> m_txreconciliation; /** The height of the best chain */ @@ -855,124 +810,6 @@ private: /** Stalling timeout for blocks in IBD */ std::atomic<std::chrono::seconds> m_block_stalling_timeout{BLOCK_STALLING_TIMEOUT_DEFAULT}; - /** Check whether we already have this gtxid in: - * - mempool - * - orphanage - * - m_lazy_recent_rejects - * - m_lazy_recent_rejects_reconsiderable (if include_reconsiderable = true) - * - m_lazy_recent_confirmed_transactions - * */ - bool AlreadyHaveTx(const GenTxid& gtxid, bool include_reconsiderable) - EXCLUSIVE_LOCKS_REQUIRED(m_tx_download_mutex); - - /** - * Filter for transactions that were recently rejected by the mempool. - * These are not rerequested until the chain tip changes, at which point - * the entire filter is reset. - * - * Without this filter we'd be re-requesting txs from each of our peers, - * increasing bandwidth consumption considerably. For instance, with 100 - * peers, half of which relay a tx we don't accept, that might be a 50x - * bandwidth increase. A flooding attacker attempting to roll-over the - * filter using minimum-sized, 60byte, transactions might manage to send - * 1000/sec if we have fast peers, so we pick 120,000 to give our peers a - * two minute window to send invs to us. - * - * Decreasing the false positive rate is fairly cheap, so we pick one in a - * million to make it highly unlikely for users to have issues with this - * filter. - * - * We typically only add wtxids to this filter. For non-segwit - * transactions, the txid == wtxid, so this only prevents us from - * re-downloading non-segwit transactions when communicating with - * non-wtxidrelay peers -- which is important for avoiding malleation - * attacks that could otherwise interfere with transaction relay from - * non-wtxidrelay peers. For communicating with wtxidrelay peers, having - * the reject filter store wtxids is exactly what we want to avoid - * redownload of a rejected transaction. - * - * In cases where we can tell that a segwit transaction will fail - * validation no matter the witness, we may add the txid of such - * transaction to the filter as well. This can be helpful when - * communicating with txid-relay peers or if we were to otherwise fetch a - * transaction via txid (eg in our orphan handling). - * - * Memory used: 1.3 MB - */ - std::unique_ptr<CRollingBloomFilter> m_lazy_recent_rejects GUARDED_BY(m_tx_download_mutex){nullptr}; - - CRollingBloomFilter& RecentRejectsFilter() EXCLUSIVE_LOCKS_REQUIRED(m_tx_download_mutex) - { - AssertLockHeld(m_tx_download_mutex); - - if (!m_lazy_recent_rejects) { - m_lazy_recent_rejects = std::make_unique<CRollingBloomFilter>(120'000, 0.000'001); - } - - return *m_lazy_recent_rejects; - } - - /** - * Filter for: - * (1) wtxids of transactions that were recently rejected by the mempool but are - * eligible for reconsideration if submitted with other transactions. - * (2) packages (see GetPackageHash) we have already rejected before and should not retry. - * - * Similar to m_lazy_recent_rejects, this filter is used to save bandwidth when e.g. all of our peers - * have larger mempools and thus lower minimum feerates than us. - * - * When a transaction's error is TxValidationResult::TX_RECONSIDERABLE (in a package or by - * itself), add its wtxid to this filter. When a package fails for any reason, add the combined - * hash to this filter. - * - * Upon receiving an announcement for a transaction, if it exists in this filter, do not - * download the txdata. When considering packages, if it exists in this filter, drop it. - * - * Reset this filter when the chain tip changes. - * - * Parameters are picked to be the same as m_lazy_recent_rejects, with the same rationale. - */ - std::unique_ptr<CRollingBloomFilter> m_lazy_recent_rejects_reconsiderable GUARDED_BY(m_tx_download_mutex){nullptr}; - - CRollingBloomFilter& RecentRejectsReconsiderableFilter() EXCLUSIVE_LOCKS_REQUIRED(m_tx_download_mutex) - { - AssertLockHeld(m_tx_download_mutex); - - if (!m_lazy_recent_rejects_reconsiderable) { - m_lazy_recent_rejects_reconsiderable = std::make_unique<CRollingBloomFilter>(120'000, 0.000'001); - } - - return *m_lazy_recent_rejects_reconsiderable; - } - - /* - * Filter for transactions that have been recently confirmed. - * We use this to avoid requesting transactions that have already been - * confirnmed. - * - * Blocks don't typically have more than 4000 transactions, so this should - * be at least six blocks (~1 hr) worth of transactions that we can store, - * inserting both a txid and wtxid for every observed transaction. - * If the number of transactions appearing in a block goes up, or if we are - * seeing getdata requests more than an hour after initial announcement, we - * can increase this number. - * The false positive rate of 1/1M should come out to less than 1 - * transaction per day that would be inadvertently ignored (which is the - * same probability that we have in the reject filter). - */ - std::unique_ptr<CRollingBloomFilter> m_lazy_recent_confirmed_transactions GUARDED_BY(m_tx_download_mutex){nullptr}; - - CRollingBloomFilter& RecentConfirmedTransactionsFilter() EXCLUSIVE_LOCKS_REQUIRED(m_tx_download_mutex) - { - AssertLockHeld(m_tx_download_mutex); - - if (!m_lazy_recent_confirmed_transactions) { - m_lazy_recent_confirmed_transactions = std::make_unique<CRollingBloomFilter>(48'000, 0.000'001); - } - - return *m_lazy_recent_confirmed_transactions; - } - /** * For sending `inv`s to inbound peers, we use a single (exponentially * distributed) timer for all peers. If we used a separate timer for each @@ -1015,7 +852,7 @@ private: bool IsBlockRequested(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Have we requested this block from an outbound peer */ - bool IsBlockRequestedFromOutbound(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool IsBlockRequestedFromOutbound(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main, !m_peer_mutex); /** Remove this block from our tracked requested blocks. Called if: * - the block has been received from a peer @@ -1099,7 +936,7 @@ private: * lNodesAnnouncingHeaderAndIDs, and keeping that list under a certain size by * removing the first element if necessary. */ - void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) EXCLUSIVE_LOCKS_REQUIRED(cs_main, !m_peer_mutex); /** Stack of nodes which we have set to announce using compact blocks */ std::list<NodeId> lNodesAnnouncingHeaderAndIDs GUARDED_BY(cs_main); @@ -1107,9 +944,6 @@ private: /** Number of peers from which we're downloading blocks. */ int m_peers_downloading_from GUARDED_BY(cs_main) = 0; - /** Storage for orphan information */ - TxOrphanage m_orphanage GUARDED_BY(m_tx_download_mutex); - void AddToCompactExtraTransactions(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex); /** Orphan/conflicted/etc transactions that are kept for compact block reconstruction. @@ -1302,8 +1136,8 @@ bool PeerManagerImpl::IsBlockRequestedFromOutbound(const uint256& hash) { for (auto range = mapBlocksInFlight.equal_range(hash); range.first != range.second; range.first++) { auto [nodeid, block_it] = range.first->second; - CNodeState& nodestate = *Assert(State(nodeid)); - if (!nodestate.m_is_inbound) return true; + PeerRef peer{GetPeerRef(nodeid)}; + if (peer && !peer->m_is_inbound) return true; } return false; @@ -1392,6 +1226,7 @@ void PeerManagerImpl::MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) if (m_opts.ignore_incoming_txs) return; CNodeState* nodestate = State(nodeid); + PeerRef peer{GetPeerRef(nodeid)}; if (!nodestate || !nodestate->m_provides_cmpctblocks) { // Don't request compact blocks if the peer has not signalled support return; @@ -1404,15 +1239,15 @@ void PeerManagerImpl::MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) lNodesAnnouncingHeaderAndIDs.push_back(nodeid); return; } - CNodeState *state = State(*it); - if (state != nullptr && !state->m_is_inbound) ++num_outbound_hb_peers; + PeerRef peer_ref{GetPeerRef(*it)}; + if (peer_ref && !peer_ref->m_is_inbound) ++num_outbound_hb_peers; } - if (nodestate->m_is_inbound) { + if (peer && peer->m_is_inbound) { // If we're adding an inbound HB peer, make sure we're not removing // our last outbound HB peer in the process. if (lNodesAnnouncingHeaderAndIDs.size() >= 3 && num_outbound_hb_peers == 1) { - CNodeState *remove_node = State(lNodesAnnouncingHeaderAndIDs.front()); - if (remove_node != nullptr && !remove_node->m_is_inbound) { + PeerRef remove_peer{GetPeerRef(lNodesAnnouncingHeaderAndIDs.front())}; + if (remove_peer && !remove_peer->m_is_inbound) { // Put the HB outbound peer in the second slot, so that it // doesn't get removed. std::swap(lNodesAnnouncingHeaderAndIDs.front(), *std::next(lNodesAnnouncingHeaderAndIDs.begin())); @@ -1680,34 +1515,6 @@ void PeerManagerImpl::PushNodeVersion(CNode& pnode, const Peer& peer) } } -void PeerManagerImpl::AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std::chrono::microseconds current_time) -{ - AssertLockHeld(::cs_main); // for State - AssertLockHeld(m_tx_download_mutex); // For m_txrequest - NodeId nodeid = node.GetId(); - if (!node.HasPermission(NetPermissionFlags::Relay) && m_txrequest.Count(nodeid) >= MAX_PEER_TX_ANNOUNCEMENTS) { - // Too many queued announcements from this peer - return; - } - const CNodeState* state = State(nodeid); - - // Decide the TxRequestTracker parameters for this announcement: - // - "preferred": if fPreferredDownload is set (= outbound, or NetPermissionFlags::NoBan permission) - // - "reqtime": current time plus delays for: - // - NONPREF_PEER_TX_DELAY for announcements from non-preferred connections - // - TXID_RELAY_DELAY for txid announcements while wtxid peers are available - // - OVERLOADED_PEER_TX_DELAY for announcements from peers which have at least - // MAX_PEER_TX_REQUEST_IN_FLIGHT requests in flight (and don't have NetPermissionFlags::Relay). - auto delay{0us}; - const bool preferred = state->fPreferredDownload; - if (!preferred) delay += NONPREF_PEER_TX_DELAY; - if (!gtxid.IsWtxid() && m_wtxid_relay_peers > 0) delay += TXID_RELAY_DELAY; - const bool overloaded = !node.HasPermission(NetPermissionFlags::Relay) && - m_txrequest.CountInFlight(nodeid) >= MAX_PEER_TX_REQUEST_IN_FLIGHT; - if (overloaded) delay += OVERLOADED_PEER_TX_DELAY; - m_txrequest.ReceivedInv(nodeid, gtxid, preferred, current_time + delay); -} - void PeerManagerImpl::UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) { LOCK(cs_main); @@ -1720,18 +1527,15 @@ void PeerManagerImpl::InitializeNode(const CNode& node, ServiceFlags our_service NodeId nodeid = node.GetId(); { LOCK(cs_main); // For m_node_states - m_node_states.emplace_hint(m_node_states.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(node.IsInboundConn())); - } - { - LOCK(m_tx_download_mutex); - assert(m_txrequest.Count(nodeid) == 0); + m_node_states.try_emplace(m_node_states.end(), nodeid); } + WITH_LOCK(m_tx_download_mutex, m_txdownloadman.CheckIsEmpty(nodeid)); if (NetPermissions::HasFlag(node.m_permission_flags, NetPermissionFlags::BloomFilter)) { our_services = static_cast<ServiceFlags>(our_services | NODE_BLOOM); } - PeerRef peer = std::make_shared<Peer>(nodeid, our_services); + PeerRef peer = std::make_shared<Peer>(nodeid, our_services, node.IsInboundConn()); { LOCK(m_peer_mutex); m_peer_map.emplace_hint(m_peer_map.end(), nodeid, peer); @@ -1793,8 +1597,7 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) } { LOCK(m_tx_download_mutex); - m_orphanage.EraseForPeer(nodeid); - m_txrequest.DisconnectedPeer(nodeid); + m_txdownloadman.DisconnectedPeer(nodeid); } if (m_txreconciliation) m_txreconciliation->ForgetPeer(nodeid); m_num_preferred_download_peers -= state->fPreferredDownload; @@ -1812,9 +1615,7 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) assert(m_peers_downloading_from == 0); assert(m_outbound_peers_with_protect_from_disconnect == 0); assert(m_wtxid_relay_peers == 0); - LOCK(m_tx_download_mutex); - assert(m_txrequest.Size() == 0); - assert(m_orphanage.Size() == 0); + WITH_LOCK(m_tx_download_mutex, m_txdownloadman.CheckIsEmpty()); } } // cs_main if (node.fSuccessfullyConnected && @@ -1920,6 +1721,12 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c return true; } +std::vector<TxOrphanage::OrphanTxBase> PeerManagerImpl::GetOrphanTransactions() +{ + LOCK(m_tx_download_mutex); + return m_txdownloadman.GetOrphanTransactions(); +} + PeerManagerInfo PeerManagerImpl::GetInfo() const { return PeerManagerInfo{ @@ -1968,15 +1775,9 @@ void PeerManagerImpl::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidati break; case BlockValidationResult::BLOCK_CACHED_INVALID: { - LOCK(cs_main); - CNodeState *node_state = State(nodeid); - if (node_state == nullptr) { - break; - } - // Discourage outbound (but not inbound) peers if on an invalid chain. // Exempt HB compact block peers. Manual connections are always protected from discouragement. - if (!via_compact_block && !node_state->m_is_inbound) { + if (peer && !via_compact_block && !peer->m_is_inbound) { if (peer) Misbehaving(*peer, message); return; } @@ -1991,7 +1792,6 @@ void PeerManagerImpl::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidati case BlockValidationResult::BLOCK_MISSING_PREV: if (peer) Misbehaving(*peer, message); return; - case BlockValidationResult::BLOCK_RECENT_CONSENSUS_CHANGE: case BlockValidationResult::BLOCK_TIME_FUTURE: break; } @@ -2011,7 +1811,6 @@ void PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationStat if (peer) Misbehaving(*peer, ""); return; // Conflicting (but not necessarily invalid) data or different policy: - case TxValidationResult::TX_RECENT_CONSENSUS_CHANGE: case TxValidationResult::TX_INPUTS_NOT_STANDARD: case TxValidationResult::TX_NOT_STANDARD: case TxValidationResult::TX_MISSING_INPUTS: @@ -2090,6 +1889,7 @@ PeerManagerImpl::PeerManagerImpl(CConnman& connman, AddrMan& addrman, m_banman(banman), m_chainman(chainman), m_mempool(pool), + m_txdownloadman(node::TxDownloadOptions{pool, m_rng, opts.max_orphan_txs, opts.deterministic_rng}), m_warnings{warnings}, m_opts{opts} { @@ -2126,8 +1926,7 @@ void PeerManagerImpl::ActiveTipChange(const CBlockIndex& new_tip, bool is_ibd) // If the chain tip has changed, previously rejected transactions might now be valid, e.g. due // to a timelock. Reset the rejection filters to give those transactions another chance if we // see them again. - RecentRejectsFilter().reset(); - RecentRejectsReconsiderableFilter().reset(); + m_txdownloadman.ActiveTipChange(); } } @@ -2162,30 +1961,13 @@ void PeerManagerImpl::BlockConnected( return; } LOCK(m_tx_download_mutex); - m_orphanage.EraseForBlock(*pblock); - - for (const auto& ptx : pblock->vtx) { - RecentConfirmedTransactionsFilter().insert(ptx->GetHash().ToUint256()); - if (ptx->HasWitness()) { - RecentConfirmedTransactionsFilter().insert(ptx->GetWitnessHash().ToUint256()); - } - m_txrequest.ForgetTxHash(ptx->GetHash()); - m_txrequest.ForgetTxHash(ptx->GetWitnessHash()); - } + m_txdownloadman.BlockConnected(pblock); } void PeerManagerImpl::BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) { - // To avoid relay problems with transactions that were previously - // confirmed, clear our filter of recently confirmed transactions whenever - // there's a reorg. - // This means that in a 1-block reorg (where 1 block is disconnected and - // then another block reconnected), our filter will drop to having only one - // block's worth of transactions in it, but that should be fine, since - // presumably the most common case of relaying a confirmed transaction - // should be just after a new block containing it is found. LOCK(m_tx_download_mutex); - RecentConfirmedTransactionsFilter().reset(); + m_txdownloadman.BlockDisconnected(); } /** @@ -2321,38 +2103,6 @@ void PeerManagerImpl::BlockChecked(const CBlock& block, const BlockValidationSta // Messages // - -bool PeerManagerImpl::AlreadyHaveTx(const GenTxid& gtxid, bool include_reconsiderable) -{ - AssertLockHeld(m_tx_download_mutex); - - const uint256& hash = gtxid.GetHash(); - - if (gtxid.IsWtxid()) { - // Normal query by wtxid. - if (m_orphanage.HaveTx(Wtxid::FromUint256(hash))) return true; - } else { - // Never query by txid: it is possible that the transaction in the orphanage has the same - // txid but a different witness, which would give us a false positive result. If we decided - // not to request the transaction based on this result, an attacker could prevent us from - // downloading a transaction by intentionally creating a malleated version of it. While - // only one (or none!) of these transactions can ultimately be confirmed, we have no way of - // discerning which one that is, so the orphanage can store multiple transactions with the - // same txid. - // - // While we won't query by txid, we can try to "guess" what the wtxid is based on the txid. - // A non-segwit transaction's txid == wtxid. Query this txid "casted" to a wtxid. This will - // help us find non-segwit transactions, saving bandwidth, and should have no false positives. - if (m_orphanage.HaveTx(Wtxid::FromUint256(hash))) return true; - } - - if (include_reconsiderable && RecentRejectsReconsiderableFilter().contains(hash)) return true; - - if (RecentConfirmedTransactionsFilter().contains(hash)) return true; - - return RecentRejectsFilter().contains(hash) || m_mempool.exists(gtxid); -} - bool PeerManagerImpl::AlreadyHaveBlock(const uint256& block_hash) { return m_chainman.m_blockman.LookupBlockIndex(block_hash) != nullptr; @@ -2488,7 +2238,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& (((m_chainman.m_best_header != nullptr) && (m_chainman.m_best_header->GetBlockTime() - pindex->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.IsMsgFilteredBlk()) && !pfrom.HasPermission(NetPermissionFlags::Download) // nodes with the download permission may exceed target ) { - LogDebug(BCLog::NET, "historical block serving limit reached, disconnect peer=%d\n", pfrom.GetId()); + LogDebug(BCLog::NET, "historical block serving limit reached, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } @@ -2497,7 +2247,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& if (!pfrom.HasPermission(NetPermissionFlags::NoBan) && ( (((peer.m_our_services & NODE_NETWORK_LIMITED) == NODE_NETWORK_LIMITED) && ((peer.m_our_services & NODE_NETWORK) != NODE_NETWORK) && (tip->nHeight - pindex->nHeight > (int)NODE_NETWORK_LIMITED_MIN_BLOCKS + 2 /* add two blocks buffer extension for possible races */) ) )) { - LogDebug(BCLog::NET, "Ignore block request below NODE_NETWORK_LIMITED threshold, disconnect peer=%d\n", pfrom.GetId()); + LogDebug(BCLog::NET, "Ignore block request below NODE_NETWORK_LIMITED threshold, %s\n", pfrom.DisconnectMsg(fLogIPs)); //disconnect node and prevent it from stalling (would otherwise wait for the missing block) pfrom.fDisconnect = true; return; @@ -2520,9 +2270,9 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& std::vector<uint8_t> block_data; if (!m_chainman.m_blockman.ReadRawBlockFromDisk(block_data, block_pos)) { if (WITH_LOCK(m_chainman.GetMutex(), return m_chainman.m_blockman.IsBlockPruned(*pindex))) { - LogDebug(BCLog::NET, "Block was pruned before it could be read, disconnect peer=%s\n", pfrom.GetId()); + LogDebug(BCLog::NET, "Block was pruned before it could be read, %s\n", pfrom.DisconnectMsg(fLogIPs)); } else { - LogError("Cannot load block from disk, disconnect peer=%d\n", pfrom.GetId()); + LogError("Cannot load block from disk, %s\n", pfrom.DisconnectMsg(fLogIPs)); } pfrom.fDisconnect = true; return; @@ -2534,9 +2284,9 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& std::shared_ptr<CBlock> pblockRead = std::make_shared<CBlock>(); if (!m_chainman.m_blockman.ReadBlockFromDisk(*pblockRead, block_pos)) { if (WITH_LOCK(m_chainman.GetMutex(), return m_chainman.m_blockman.IsBlockPruned(*pindex))) { - LogDebug(BCLog::NET, "Block was pruned before it could be read, disconnect peer=%s\n", pfrom.GetId()); + LogDebug(BCLog::NET, "Block was pruned before it could be read, %s\n", pfrom.DisconnectMsg(fLogIPs)); } else { - LogError("Cannot load block from disk, disconnect peer=%d\n", pfrom.GetId()); + LogError("Cannot load block from disk, %s\n", pfrom.DisconnectMsg(fLogIPs)); } pfrom.fDisconnect = true; return; @@ -2786,7 +2536,7 @@ bool PeerManagerImpl::CheckHeadersAreContinuous(const std::vector<CBlockHeader>& bool PeerManagerImpl::IsContinuationOfLowWorkHeadersSync(Peer& peer, CNode& pfrom, std::vector<CBlockHeader>& headers) { if (peer.m_headers_sync) { - auto result = peer.m_headers_sync->ProcessNextHeaders(headers, headers.size() == MAX_HEADERS_RESULTS); + auto result = peer.m_headers_sync->ProcessNextHeaders(headers, headers.size() == m_opts.max_headers_result); // If it is a valid continuation, we should treat the existing getheaders request as responded to. if (result.success) peer.m_last_getheaders_timestamp = {}; if (result.request_more) { @@ -2880,7 +2630,7 @@ bool PeerManagerImpl::TryLowWorkHeadersSync(Peer& peer, CNode& pfrom, const CBlo // Only try to sync with this peer if their headers message was full; // otherwise they don't have more headers after this so no point in // trying to sync their too-little-work chain. - if (headers.size() == MAX_HEADERS_RESULTS) { + if (headers.size() == m_opts.max_headers_result) { // Note: we could advance to the last header in this set that is // known to us, rather than starting at the first header (which we // may already have); however this is unlikely to matter much since @@ -3038,7 +2788,7 @@ void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, Peer& peer // the minimum chain work, even if a peer has a chain past our tip, // as an anti-DoS measure. if (pfrom.IsOutboundOrBlockRelayConn()) { - LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom.GetId()); + LogInfo("outbound peer headers chain has insufficient work, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; } } @@ -3192,7 +2942,7 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, assert(pindexLast); // Consider fetching more headers if we are not using our headers-sync mechanism. - if (nCount == MAX_HEADERS_RESULTS && !have_headers_sync) { + if (nCount == m_opts.max_headers_result && !have_headers_sync) { // Headers message had its maximum size; the peer may have more headers. if (MaybeSendGetHeaders(pfrom, GetLocator(pindexLast), peer)) { LogDebug(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", @@ -3200,7 +2950,7 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, } } - UpdatePeerStateForReceivedHeaders(pfrom, peer, *pindexLast, received_new_header, nCount == MAX_HEADERS_RESULTS); + UpdatePeerStateForReceivedHeaders(pfrom, peer, *pindexLast, received_new_header, nCount == m_opts.max_headers_result); // Consider immediately downloading blocks. HeadersDirectFetchBlocks(pfrom, peer, *pindexLast); @@ -3208,70 +2958,33 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, return; } -void PeerManagerImpl::ProcessInvalidTx(NodeId nodeid, const CTransactionRef& ptx, const TxValidationState& state, - bool maybe_add_extra_compact_tx) +std::optional<node::PackageToValidate> PeerManagerImpl::ProcessInvalidTx(NodeId nodeid, const CTransactionRef& ptx, const TxValidationState& state, + bool first_time_failure) { AssertLockNotHeld(m_peer_mutex); AssertLockHeld(g_msgproc_mutex); AssertLockHeld(m_tx_download_mutex); + PeerRef peer{GetPeerRef(nodeid)}; + LogDebug(BCLog::MEMPOOLREJ, "%s (wtxid=%s) from peer=%d was not accepted: %s\n", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), nodeid, state.ToString()); - if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { - return; - } else if (state.GetResult() != TxValidationResult::TX_WITNESS_STRIPPED) { - // We can add the wtxid of this transaction to our reject filter. - // Do not add txids of witness transactions or witness-stripped - // transactions to the filter, as they can have been malleated; - // adding such txids to the reject filter would potentially - // interfere with relay of valid transactions from peers that - // do not support wtxid-based relay. See - // https://github.com/bitcoin/bitcoin/issues/8279 for details. - // We can remove this restriction (and always add wtxids to - // the filter even for witness stripped transactions) once - // wtxid-based relay is broadly deployed. - // See also comments in https://github.com/bitcoin/bitcoin/pull/18044#discussion_r443419034 - // for concerns around weakening security of unupgraded nodes - // if we start doing this too early. - if (state.GetResult() == TxValidationResult::TX_RECONSIDERABLE) { - // If the result is TX_RECONSIDERABLE, add it to m_lazy_recent_rejects_reconsiderable - // because we should not download or submit this transaction by itself again, but may - // submit it as part of a package later. - RecentRejectsReconsiderableFilter().insert(ptx->GetWitnessHash().ToUint256()); - } else { - RecentRejectsFilter().insert(ptx->GetWitnessHash().ToUint256()); - } - m_txrequest.ForgetTxHash(ptx->GetWitnessHash()); - // If the transaction failed for TX_INPUTS_NOT_STANDARD, - // then we know that the witness was irrelevant to the policy - // failure, since this check depends only on the txid - // (the scriptPubKey being spent is covered by the txid). - // Add the txid to the reject filter to prevent repeated - // processing of this transaction in the event that child - // transactions are later received (resulting in - // parent-fetching by txid via the orphan-handling logic). - // We only add the txid if it differs from the wtxid, to avoid wasting entries in the - // rolling bloom filter. - if (state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && ptx->HasWitness()) { - RecentRejectsFilter().insert(ptx->GetHash().ToUint256()); - m_txrequest.ForgetTxHash(ptx->GetHash()); - } - if (maybe_add_extra_compact_tx && RecursiveDynamicUsage(*ptx) < 100000) { - AddToCompactExtraTransactions(ptx); - } + const auto& [add_extra_compact_tx, unique_parents, package_to_validate] = m_txdownloadman.MempoolRejectedTx(ptx, state, nodeid, first_time_failure); + + if (add_extra_compact_tx && RecursiveDynamicUsage(*ptx) < 100000) { + AddToCompactExtraTransactions(ptx); + } + for (const uint256& parent_txid : unique_parents) { + if (peer) AddKnownTx(*peer, parent_txid); } MaybePunishNodeForTx(nodeid, state); - // If the tx failed in ProcessOrphanTx, it should be removed from the orphanage unless the - // tx was still missing inputs. If the tx was not in the orphanage, EraseTx does nothing and returns 0. - if (Assume(state.GetResult() != TxValidationResult::TX_MISSING_INPUTS) && m_orphanage.EraseTx(ptx->GetWitnessHash()) > 0) { - LogDebug(BCLog::TXPACKAGES, " removed orphan tx %s (wtxid=%s)\n", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString()); - } + return package_to_validate; } void PeerManagerImpl::ProcessValidTx(NodeId nodeid, const CTransactionRef& tx, const std::list<CTransactionRef>& replaced_transactions) @@ -3280,14 +2993,7 @@ void PeerManagerImpl::ProcessValidTx(NodeId nodeid, const CTransactionRef& tx, c AssertLockHeld(g_msgproc_mutex); AssertLockHeld(m_tx_download_mutex); - // As this version of the transaction was acceptable, we can forget about any requests for it. - // No-op if the tx is not in txrequest. - m_txrequest.ForgetTxHash(tx->GetHash()); - m_txrequest.ForgetTxHash(tx->GetWitnessHash()); - - m_orphanage.AddChildrenToWorkSet(*tx); - // If it came from the orphanage, remove it. No-op if the tx is not in txorphanage. - m_orphanage.EraseTx(tx->GetWitnessHash()); + m_txdownloadman.MempoolAcceptedTx(tx); LogDebug(BCLog::MEMPOOL, "AcceptToMemoryPool: peer=%d: accepted %s (wtxid=%s) (poolsz %u txn, %u kB)\n", nodeid, @@ -3302,7 +3008,7 @@ void PeerManagerImpl::ProcessValidTx(NodeId nodeid, const CTransactionRef& tx, c } } -void PeerManagerImpl::ProcessPackageResult(const PackageToValidate& package_to_validate, const PackageMempoolAcceptResult& package_result) +void PeerManagerImpl::ProcessPackageResult(const node::PackageToValidate& package_to_validate, const PackageMempoolAcceptResult& package_result) { AssertLockNotHeld(m_peer_mutex); AssertLockHeld(g_msgproc_mutex); @@ -3312,7 +3018,7 @@ void PeerManagerImpl::ProcessPackageResult(const PackageToValidate& package_to_v const auto& senders = package_to_validate.m_senders; if (package_result.m_state.IsInvalid()) { - RecentRejectsReconsiderableFilter().insert(GetPackageHash(package)); + m_txdownloadman.MempoolRejectedPackage(package); } // We currently only expect to process 1-parent-1-child packages. Remove if this changes. if (!Assume(package.size() == 2)) return; @@ -3342,7 +3048,7 @@ void PeerManagerImpl::ProcessPackageResult(const PackageToValidate& package_to_v // added there when added to the orphanage or rejected for TX_RECONSIDERABLE. // This should be updated if package submission is ever used for transactions // that haven't already been validated before. - ProcessInvalidTx(nodeid, tx, tx_result.m_state, /*maybe_add_extra_compact_tx=*/false); + ProcessInvalidTx(nodeid, tx, tx_result.m_state, /*first_time_failure=*/false); break; } case MempoolAcceptResult::ResultType::MEMPOOL_ENTRY: @@ -3358,60 +3064,6 @@ void PeerManagerImpl::ProcessPackageResult(const PackageToValidate& package_to_v } } -std::optional<PeerManagerImpl::PackageToValidate> PeerManagerImpl::Find1P1CPackage(const CTransactionRef& ptx, NodeId nodeid) -{ - AssertLockNotHeld(m_peer_mutex); - AssertLockHeld(g_msgproc_mutex); - AssertLockHeld(m_tx_download_mutex); - - const auto& parent_wtxid{ptx->GetWitnessHash()}; - - Assume(RecentRejectsReconsiderableFilter().contains(parent_wtxid.ToUint256())); - - // Prefer children from this peer. This helps prevent censorship attempts in which an attacker - // sends lots of fake children for the parent, and we (unluckily) keep selecting the fake - // children instead of the real one provided by the honest peer. - const auto cpfp_candidates_same_peer{m_orphanage.GetChildrenFromSamePeer(ptx, nodeid)}; - - // These children should be sorted from newest to oldest. In the (probably uncommon) case - // of children that replace each other, this helps us accept the highest feerate (probably the - // most recent) one efficiently. - for (const auto& child : cpfp_candidates_same_peer) { - Package maybe_cpfp_package{ptx, child}; - if (!RecentRejectsReconsiderableFilter().contains(GetPackageHash(maybe_cpfp_package))) { - return PeerManagerImpl::PackageToValidate{ptx, child, nodeid, nodeid}; - } - } - - // If no suitable candidate from the same peer is found, also try children that were provided by - // a different peer. This is useful because sometimes multiple peers announce both transactions - // to us, and we happen to download them from different peers (we wouldn't have known that these - // 2 transactions are related). We still want to find 1p1c packages then. - // - // If we start tracking all announcers of orphans, we can restrict this logic to parent + child - // pairs in which both were provided by the same peer, i.e. delete this step. - const auto cpfp_candidates_different_peer{m_orphanage.GetChildrenFromDifferentPeer(ptx, nodeid)}; - - // Find the first 1p1c that hasn't already been rejected. We randomize the order to not - // create a bias that attackers can use to delay package acceptance. - // - // Create a random permutation of the indices. - std::vector<size_t> tx_indices(cpfp_candidates_different_peer.size()); - std::iota(tx_indices.begin(), tx_indices.end(), 0); - std::shuffle(tx_indices.begin(), tx_indices.end(), m_rng); - - for (const auto index : tx_indices) { - // If we already tried a package and failed for any reason, the combined hash was - // cached in m_lazy_recent_rejects_reconsiderable. - const auto [child_tx, child_sender] = cpfp_candidates_different_peer.at(index); - Package maybe_cpfp_package{ptx, child_tx}; - if (!RecentRejectsReconsiderableFilter().contains(GetPackageHash(maybe_cpfp_package))) { - return PeerManagerImpl::PackageToValidate{ptx, child_tx, nodeid, child_sender}; - } - } - return std::nullopt; -} - bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) { AssertLockHeld(g_msgproc_mutex); @@ -3419,7 +3071,7 @@ bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) CTransactionRef porphanTx = nullptr; - while (CTransactionRef porphanTx = m_orphanage.GetTxToReconsider(peer.m_id)) { + while (CTransactionRef porphanTx = m_txdownloadman.GetTxToReconsider(peer.m_id)) { const MempoolAcceptResult result = m_chainman.ProcessTransaction(porphanTx); const TxValidationState& state = result.m_state; const Txid& orphanHash = porphanTx->GetHash(); @@ -3440,7 +3092,7 @@ bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) state.GetResult() != TxValidationResult::TX_UNKNOWN && state.GetResult() != TxValidationResult::TX_NO_MEMPOOL && state.GetResult() != TxValidationResult::TX_RESULT_UNSET)) { - ProcessInvalidTx(peer.m_id, porphanTx, state, /*maybe_add_extra_compact_tx=*/false); + ProcessInvalidTx(peer.m_id, porphanTx, state, /*first_time_failure=*/false); } return true; } @@ -3459,8 +3111,8 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& node, Peer& peer, (filter_type == BlockFilterType::BASIC && (peer.m_our_services & NODE_COMPACT_FILTERS)); if (!supported_filter_type) { - LogDebug(BCLog::NET, "peer %d requested unsupported block filter type: %d\n", - node.GetId(), static_cast<uint8_t>(filter_type)); + LogDebug(BCLog::NET, "peer requested unsupported block filter type: %d, %s\n", + static_cast<uint8_t>(filter_type), node.DisconnectMsg(fLogIPs)); node.fDisconnect = true; return false; } @@ -3471,8 +3123,8 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& node, Peer& peer, // Check that the stop block exists and the peer would be allowed to fetch it. if (!stop_index || !BlockRequestAllowed(stop_index)) { - LogDebug(BCLog::NET, "peer %d requested invalid block hash: %s\n", - node.GetId(), stop_hash.ToString()); + LogDebug(BCLog::NET, "peer requested invalid block hash: %s, %s\n", + stop_hash.ToString(), node.DisconnectMsg(fLogIPs)); node.fDisconnect = true; return false; } @@ -3480,15 +3132,15 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& node, Peer& peer, uint32_t stop_height = stop_index->nHeight; if (start_height > stop_height) { - LogDebug(BCLog::NET, "peer %d sent invalid getcfilters/getcfheaders with " - "start height %d and stop height %d\n", - node.GetId(), start_height, stop_height); + LogDebug(BCLog::NET, "peer sent invalid getcfilters/getcfheaders with " + "start height %d and stop height %d, %s\n", + start_height, stop_height, node.DisconnectMsg(fLogIPs)); node.fDisconnect = true; return false; } if (stop_height - start_height >= max_height_diff) { - LogDebug(BCLog::NET, "peer %d requested too many cfilters/cfheaders: %d / %d\n", - node.GetId(), stop_height - start_height + 1, max_height_diff); + LogDebug(BCLog::NET, "peer requested too many cfilters/cfheaders: %d / %d, %s\n", + stop_height - start_height + 1, max_height_diff, node.DisconnectMsg(fLogIPs)); node.fDisconnect = true; return false; } @@ -3755,14 +3407,17 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (pfrom.ExpectServicesFromConn() && !HasAllDesirableServiceFlags(nServices)) { - LogDebug(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom.GetId(), nServices, GetDesirableServiceFlags(nServices)); + LogDebug(BCLog::NET, "peer does not offer the expected services (%08x offered, %08x expected), %s\n", + nServices, + GetDesirableServiceFlags(nServices), + pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } if (nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version - LogDebug(BCLog::NET, "peer=%d using obsolete version %i; disconnecting\n", pfrom.GetId(), nVersion); + LogDebug(BCLog::NET, "peer using obsolete version %i, %s\n", nVersion, pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } @@ -3913,15 +3568,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_addrman.Good(pfrom.addr); } - std::string remoteAddr; - if (fLogIPs) - remoteAddr = ", peeraddr=" + pfrom.addr.ToStringAddrPort(); - const auto mapped_as{m_connman.GetMappedAS(pfrom.addr)}; LogDebug(BCLog::NET, "receive version message: %s: version %d, blocks=%d, us=%s, txrelay=%d, peer=%d%s%s\n", cleanSubVer, pfrom.nVersion, peer->m_starting_height, addrMe.ToStringAddrPort(), fRelay, pfrom.GetId(), - remoteAddr, (mapped_as ? strprintf(", mapped_as=%d", mapped_as) : "")); + pfrom.LogIP(fLogIPs), (mapped_as ? strprintf(", mapped_as=%d", mapped_as) : "")); peer->m_time_offset = NodeSeconds{std::chrono::seconds{nTime}} - Now<NodeSeconds>(); if (!pfrom.IsInboundConn()) { @@ -3939,7 +3590,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Feeler connections exist only to verify if address is online. if (pfrom.IsFeelerConn()) { - LogDebug(BCLog::NET, "feeler connection completed peer=%d; disconnecting\n", pfrom.GetId()); + LogDebug(BCLog::NET, "feeler connection completed, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; } return; @@ -3965,7 +3616,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, pfrom.ConnectionTypeAsString(), TransportTypeAsString(pfrom.m_transport->GetInfo().transport_type), pfrom.nVersion.load(), peer->m_starting_height, - pfrom.GetId(), (fLogIPs ? strprintf(", peeraddr=%s", pfrom.addr.ToStringAddrPort()) : ""), + pfrom.GetId(), pfrom.LogIP(fLogIPs), (mapped_as ? strprintf(", mapped_as=%d", mapped_as) : "")); } @@ -4001,6 +3652,16 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, tx_relay->m_next_inv_send_time == 0s)); } + { + LOCK2(::cs_main, m_tx_download_mutex); + const CNodeState* state = State(pfrom.GetId()); + m_txdownloadman.ConnectedPeer(pfrom.GetId(), node::TxDownloadConnectionInfo { + .m_preferred = state->fPreferredDownload, + .m_relay_permissions = pfrom.HasPermission(NetPermissionFlags::Relay), + .m_wtxid_relay = peer->m_wtxid_relay, + }); + } + pfrom.fSuccessfullyConnected = true; return; } @@ -4033,7 +3694,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::WTXIDRELAY) { if (pfrom.fSuccessfullyConnected) { // Disconnect peers that send a wtxidrelay message after VERACK. - LogDebug(BCLog::NET, "wtxidrelay received after verack from peer=%d; disconnecting\n", pfrom.GetId()); + LogDebug(BCLog::NET, "wtxidrelay received after verack, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } @@ -4055,7 +3716,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::SENDADDRV2) { if (pfrom.fSuccessfullyConnected) { // Disconnect peers that send a SENDADDRV2 message after VERACK. - LogDebug(BCLog::NET, "sendaddrv2 received after verack from peer=%d; disconnecting\n", pfrom.GetId()); + LogDebug(BCLog::NET, "sendaddrv2 received after verack, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } @@ -4068,19 +3729,19 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // from switching announcement protocols after the connection is up. if (msg_type == NetMsgType::SENDTXRCNCL) { if (!m_txreconciliation) { - LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl from peer=%d ignored, as our node does not have txreconciliation enabled\n", pfrom.GetId()); + LogDebug(BCLog::NET, "sendtxrcncl from peer=%d ignored, as our node does not have txreconciliation enabled\n", pfrom.GetId()); return; } if (pfrom.fSuccessfullyConnected) { - LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl received after verack from peer=%d; disconnecting\n", pfrom.GetId()); + LogDebug(BCLog::NET, "sendtxrcncl received after verack, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } // Peer must not offer us reconciliations if we specified no tx relay support in VERSION. if (RejectIncomingTxs(pfrom)) { - LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl received from peer=%d to which we indicated no tx relay; disconnecting\n", pfrom.GetId()); + LogDebug(BCLog::NET, "sendtxrcncl received to which we indicated no tx relay, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } @@ -4090,7 +3751,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // eliminates them, so that this flag fully represents what we are looking for. const auto* tx_relay = peer->GetTxRelay(); if (!tx_relay || !WITH_LOCK(tx_relay->m_bloom_filter_mutex, return tx_relay->m_relay_txs)) { - LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl received from peer=%d which indicated no tx relay to us; disconnecting\n", pfrom.GetId()); + LogDebug(BCLog::NET, "sendtxrcncl received which indicated no tx relay to us, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } @@ -4103,16 +3764,16 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, peer_txreconcl_version, remote_salt); switch (result) { case ReconciliationRegisterResult::NOT_FOUND: - LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "Ignore unexpected txreconciliation signal from peer=%d\n", pfrom.GetId()); + LogDebug(BCLog::NET, "Ignore unexpected txreconciliation signal from peer=%d\n", pfrom.GetId()); break; case ReconciliationRegisterResult::SUCCESS: break; case ReconciliationRegisterResult::ALREADY_REGISTERED: - LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "txreconciliation protocol violation from peer=%d (sendtxrcncl received from already registered peer); disconnecting\n", pfrom.GetId()); + LogDebug(BCLog::NET, "txreconciliation protocol violation (sendtxrcncl received from already registered peer), %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; case ReconciliationRegisterResult::PROTOCOL_VIOLATION: - LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "txreconciliation protocol violation from peer=%d; disconnecting\n", pfrom.GetId()); + LogDebug(BCLog::NET, "txreconciliation protocol violation, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } @@ -4215,7 +3876,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // AddrFetch: Require multiple addresses to avoid disconnecting on self-announcements if (pfrom.IsAddrFetchConn() && vAddr.size() > 1) { - LogDebug(BCLog::NET, "addrfetch connection completed peer=%d; disconnecting\n", pfrom.GetId()); + LogDebug(BCLog::NET, "addrfetch connection completed, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; } return; @@ -4265,17 +3926,16 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } } else if (inv.IsGenTxMsg()) { if (reject_tx_invs) { - LogDebug(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); + LogDebug(BCLog::NET, "transaction (%s) inv sent in violation of protocol, %s\n", inv.hash.ToString(), pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } const GenTxid gtxid = ToGenTxid(inv); - const bool fAlreadyHave = AlreadyHaveTx(gtxid, /*include_reconsiderable=*/true); - LogDebug(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); - AddKnownTx(*peer, inv.hash); - if (!fAlreadyHave && !m_chainman.IsInitialBlockDownload()) { - AddTxAnnouncement(pfrom, gtxid, current_time); + + if (!m_chainman.IsInitialBlockDownload()) { + const bool fAlreadyHave{m_txdownloadman.AddTxAnnouncement(pfrom.GetId(), gtxid, current_time, /*p2p_inv=*/true)}; + LogDebug(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); } } else { LogDebug(BCLog::NET, "Unknown inv type \"%s\" received from peer=%d\n", inv.ToString(), pfrom.GetId()); @@ -4343,7 +4003,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, vRecv >> locator >> hashStop; if (locator.vHave.size() > MAX_LOCATOR_SZ) { - LogDebug(BCLog::NET, "getblocks locator size %lld > %d, disconnect peer=%d\n", locator.vHave.size(), MAX_LOCATOR_SZ, pfrom.GetId()); + LogDebug(BCLog::NET, "getblocks locator size %lld > %d, %s\n", locator.vHave.size(), MAX_LOCATOR_SZ, pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } @@ -4465,7 +4125,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, vRecv >> locator >> hashStop; if (locator.vHave.size() > MAX_LOCATOR_SZ) { - LogDebug(BCLog::NET, "getheaders locator size %lld > %d, disconnect peer=%d\n", locator.vHave.size(), MAX_LOCATOR_SZ, pfrom.GetId()); + LogDebug(BCLog::NET, "getheaders locator size %lld > %d, %s\n", locator.vHave.size(), MAX_LOCATOR_SZ, pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } @@ -4518,7 +4178,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // we must use CBlocks, as CBlockHeaders won't include the 0x00 nTx count at the end std::vector<CBlock> vHeaders; - int nLimit = MAX_HEADERS_RESULTS; + int nLimit = m_opts.max_headers_result; LogDebug(BCLog::NET, "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom.GetId()); for (; pindex; pindex = m_chainman.ActiveChain().Next(pindex)) { @@ -4567,22 +4227,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LOCK2(cs_main, m_tx_download_mutex); - m_txrequest.ReceivedResponse(pfrom.GetId(), txid); - if (tx.HasWitness()) m_txrequest.ReceivedResponse(pfrom.GetId(), wtxid); - - // We do the AlreadyHaveTx() check using wtxid, rather than txid - in the - // absence of witness malleation, this is strictly better, because the - // recent rejects filter may contain the wtxid but rarely contains - // the txid of a segwit transaction that has been rejected. - // In the presence of witness malleation, it's possible that by only - // doing the check with wtxid, we could overlook a transaction which - // was confirmed with a different witness, or exists in our mempool - // with a different witness, but this has limited downside: - // mempool validation does its own lookup of whether we have the txid - // already; and an adversary can already relay us old transactions - // (older than our recency filter) if trying to DoS us, without any need - // for witness malleation. - if (AlreadyHaveTx(GenTxid::Wtxid(wtxid), /*include_reconsiderable=*/true)) { + const auto& [should_validate, package_to_validate] = m_txdownloadman.ReceivedTx(pfrom.GetId(), ptx); + if (!should_validate) { if (pfrom.HasPermission(NetPermissionFlags::ForceRelay)) { // Always relay transactions received from peers with forcerelay // permission, even if they were already in the mempool, allowing @@ -4597,37 +4243,18 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } } - if (RecentRejectsReconsiderableFilter().contains(wtxid)) { - // When a transaction is already in m_lazy_recent_rejects_reconsiderable, we shouldn't submit - // it by itself again. However, look for a matching child in the orphanage, as it is - // possible that they succeed as a package. - LogDebug(BCLog::TXPACKAGES, "found tx %s (wtxid=%s) in reconsiderable rejects, looking for child in orphanage\n", - txid.ToString(), wtxid.ToString()); - if (auto package_to_validate{Find1P1CPackage(ptx, pfrom.GetId())}) { - const auto package_result{ProcessNewPackage(m_chainman.ActiveChainstate(), m_mempool, package_to_validate->m_txns, /*test_accept=*/false, /*client_maxfeerate=*/std::nullopt)}; - LogDebug(BCLog::TXPACKAGES, "package evaluation for %s: %s\n", package_to_validate->ToString(), - package_result.m_state.IsValid() ? "package accepted" : "package rejected"); - ProcessPackageResult(package_to_validate.value(), package_result); - } + if (package_to_validate) { + const auto package_result{ProcessNewPackage(m_chainman.ActiveChainstate(), m_mempool, package_to_validate->m_txns, /*test_accept=*/false, /*client_maxfeerate=*/std::nullopt)}; + LogDebug(BCLog::TXPACKAGES, "package evaluation for %s: %s\n", package_to_validate->ToString(), + package_result.m_state.IsValid() ? "package accepted" : "package rejected"); + ProcessPackageResult(package_to_validate.value(), package_result); } - // If a tx is detected by m_lazy_recent_rejects it is ignored. Because we haven't - // submitted the tx to our mempool, we won't have computed a DoS - // score for it or determined exactly why we consider it invalid. - // - // This means we won't penalize any peer subsequently relaying a DoSy - // tx (even if we penalized the first peer who gave it to us) because - // we have to account for m_lazy_recent_rejects showing false positives. In - // other words, we shouldn't penalize a peer if we aren't *sure* they - // submitted a DoSy tx. - // - // Note that m_lazy_recent_rejects doesn't just record DoSy or invalid - // transactions, but any tx not accepted by the mempool, which may be - // due to node policy (vs. consensus). So we can't blanket penalize a - // peer simply for relaying a tx that our m_lazy_recent_rejects has caught, - // regardless of false positives. return; } + // ReceivedTx should not be telling us to validate the tx and a package. + Assume(!package_to_validate.has_value()); + const MempoolAcceptResult result = m_chainman.ProcessTransaction(ptx); const TxValidationState& state = result.m_state; @@ -4635,90 +4262,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, ProcessValidTx(pfrom.GetId(), ptx, result.m_replaced_transactions); pfrom.m_last_tx_time = GetTime<std::chrono::seconds>(); } - else if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) - { - bool fRejectedParents = false; // It may be the case that the orphans parents have all been rejected - - // Deduplicate parent txids, so that we don't have to loop over - // the same parent txid more than once down below. - std::vector<uint256> unique_parents; - unique_parents.reserve(tx.vin.size()); - for (const CTxIn& txin : tx.vin) { - // We start with all parents, and then remove duplicates below. - unique_parents.push_back(txin.prevout.hash); - } - std::sort(unique_parents.begin(), unique_parents.end()); - unique_parents.erase(std::unique(unique_parents.begin(), unique_parents.end()), unique_parents.end()); - - // Distinguish between parents in m_lazy_recent_rejects and m_lazy_recent_rejects_reconsiderable. - // We can tolerate having up to 1 parent in m_lazy_recent_rejects_reconsiderable since we - // submit 1p1c packages. However, fail immediately if any are in m_lazy_recent_rejects. - std::optional<uint256> rejected_parent_reconsiderable; - for (const uint256& parent_txid : unique_parents) { - if (RecentRejectsFilter().contains(parent_txid)) { - fRejectedParents = true; - break; - } else if (RecentRejectsReconsiderableFilter().contains(parent_txid) && !m_mempool.exists(GenTxid::Txid(parent_txid))) { - // More than 1 parent in m_lazy_recent_rejects_reconsiderable: 1p1c will not be - // sufficient to accept this package, so just give up here. - if (rejected_parent_reconsiderable.has_value()) { - fRejectedParents = true; - break; - } - rejected_parent_reconsiderable = parent_txid; - } - } - if (!fRejectedParents) { - const auto current_time{GetTime<std::chrono::microseconds>()}; - - for (const uint256& parent_txid : unique_parents) { - // Here, we only have the txid (and not wtxid) of the - // inputs, so we only request in txid mode, even for - // wtxidrelay peers. - // Eventually we should replace this with an improved - // protocol for getting all unconfirmed parents. - const auto gtxid{GenTxid::Txid(parent_txid)}; - AddKnownTx(*peer, parent_txid); - // Exclude m_lazy_recent_rejects_reconsiderable: the missing parent may have been - // previously rejected for being too low feerate. This orphan might CPFP it. - if (!AlreadyHaveTx(gtxid, /*include_reconsiderable=*/false)) AddTxAnnouncement(pfrom, gtxid, current_time); - } - - if (m_orphanage.AddTx(ptx, pfrom.GetId())) { - AddToCompactExtraTransactions(ptx); - } - - // Once added to the orphan pool, a tx is considered AlreadyHave, and we shouldn't request it anymore. - m_txrequest.ForgetTxHash(tx.GetHash()); - m_txrequest.ForgetTxHash(tx.GetWitnessHash()); - - // DoS prevention: do not allow m_orphanage to grow unbounded (see CVE-2012-3789) - m_orphanage.LimitOrphans(m_opts.max_orphan_txs, m_rng); - } else { - LogDebug(BCLog::MEMPOOL, "not keeping orphan with rejected parents %s (wtxid=%s)\n", - tx.GetHash().ToString(), - tx.GetWitnessHash().ToString()); - // We will continue to reject this tx since it has rejected - // parents so avoid re-requesting it from other peers. - // Here we add both the txid and the wtxid, as we know that - // regardless of what witness is provided, we will not accept - // this, so we don't need to allow for redownload of this txid - // from any of our non-wtxidrelay peers. - RecentRejectsFilter().insert(tx.GetHash().ToUint256()); - RecentRejectsFilter().insert(tx.GetWitnessHash().ToUint256()); - m_txrequest.ForgetTxHash(tx.GetHash()); - m_txrequest.ForgetTxHash(tx.GetWitnessHash()); - } - } if (state.IsInvalid()) { - ProcessInvalidTx(pfrom.GetId(), ptx, state, /*maybe_add_extra_compact_tx=*/true); - } - // When a transaction fails for TX_RECONSIDERABLE, look for a matching child in the - // orphanage, as it is possible that they succeed as a package. - if (state.GetResult() == TxValidationResult::TX_RECONSIDERABLE) { - LogDebug(BCLog::TXPACKAGES, "tx %s (wtxid=%s) failed but reconsiderable, looking for child in orphanage\n", - txid.ToString(), wtxid.ToString()); - if (auto package_to_validate{Find1P1CPackage(ptx, pfrom.GetId())}) { + if (auto package_to_validate{ProcessInvalidTx(pfrom.GetId(), ptx, state, /*first_time_failure=*/true)}) { const auto package_result{ProcessNewPackage(m_chainman.ActiveChainstate(), m_mempool, package_to_validate->m_txns, /*test_accept=*/false, /*client_maxfeerate=*/std::nullopt)}; LogDebug(BCLog::TXPACKAGES, "package evaluation for %s: %s\n", package_to_validate->ToString(), package_result.m_state.IsValid() ? "package accepted" : "package rejected"); @@ -5002,7 +4547,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. unsigned int nCount = ReadCompactSize(vRecv); - if (nCount > MAX_HEADERS_RESULTS) { + if (nCount > m_opts.max_headers_result) { Misbehaving(*peer, strprintf("headers message size = %u", nCount)); return; } @@ -5121,7 +4666,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, { if (!pfrom.HasPermission(NetPermissionFlags::NoBan)) { - LogDebug(BCLog::NET, "mempool request with bloom filters disabled, disconnect peer=%d\n", pfrom.GetId()); + LogDebug(BCLog::NET, "mempool request with bloom filters disabled, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; } return; @@ -5131,7 +4676,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, { if (!pfrom.HasPermission(NetPermissionFlags::NoBan)) { - LogDebug(BCLog::NET, "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom.GetId()); + LogDebug(BCLog::NET, "mempool request with bandwidth limit reached, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; } return; @@ -5221,7 +4766,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::FILTERLOAD) { if (!(peer->m_our_services & NODE_BLOOM)) { - LogDebug(BCLog::NET, "filterload received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); + LogDebug(BCLog::NET, "filterload received despite not offering bloom services, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } @@ -5246,7 +4791,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::FILTERADD) { if (!(peer->m_our_services & NODE_BLOOM)) { - LogDebug(BCLog::NET, "filteradd received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); + LogDebug(BCLog::NET, "filteradd received despite not offering bloom services, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } @@ -5274,7 +4819,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::FILTERCLEAR) { if (!(peer->m_our_services & NODE_BLOOM)) { - LogDebug(BCLog::NET, "filterclear received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); + LogDebug(BCLog::NET, "filterclear received despite not offering bloom services, %s\n", pfrom.DisconnectMsg(fLogIPs)); pfrom.fDisconnect = true; return; } @@ -5321,16 +4866,16 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::NOTFOUND) { std::vector<CInv> vInv; vRecv >> vInv; - if (vInv.size() <= MAX_PEER_TX_ANNOUNCEMENTS + MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - LOCK(m_tx_download_mutex); + std::vector<uint256> tx_invs; + if (vInv.size() <= node::MAX_PEER_TX_ANNOUNCEMENTS + MAX_BLOCKS_IN_TRANSIT_PER_PEER) { for (CInv &inv : vInv) { if (inv.IsGenTxMsg()) { - // If we receive a NOTFOUND message for a tx we requested, mark the announcement for it as - // completed in TxRequestTracker. - m_txrequest.ReceivedResponse(pfrom.GetId(), inv.hash); + tx_invs.emplace_back(inv.hash); } } } + LOCK(m_tx_download_mutex); + m_txdownloadman.ReceivedNotFound(pfrom.GetId(), tx_invs); return; } @@ -5423,7 +4968,7 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt CNetMessage& msg{poll_result->first}; bool fMoreWork = poll_result->second; - TRACE6(net, inbound_message, + TRACEPOINT(net, inbound_message, pfrom->GetId(), pfrom->m_addr_name.c_str(), pfrom->ConnectionTypeAsString().c_str(), @@ -5449,7 +4994,7 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt // the extra work may not be noticed, possibly resulting in an // unnecessary 100ms delay) LOCK(m_tx_download_mutex); - if (m_orphanage.HaveTxToReconsider(peer->m_id)) fMoreWork = true; + if (m_txdownloadman.HaveMoreWork(peer->m_id)) fMoreWork = true; } catch (const std::exception& e) { LogDebug(BCLog::NET, "%s(%s, %u bytes): Exception '%s' (%s) caught\n", __func__, SanitizeString(msg.m_type), msg.m_message_size, e.what(), typeid(e).name()); } catch (...) { @@ -5495,7 +5040,7 @@ void PeerManagerImpl::ConsiderEviction(CNode& pto, Peer& peer, std::chrono::seco // message to give the peer a chance to update us. if (state.m_chain_sync.m_sent_getheaders) { // They've run out of time to catch up! - LogPrintf("Disconnecting outbound peer %d for old chain, best known block = %s\n", pto.GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>"); + LogInfo("Outbound peer has old chain, best known block = %s, %s\n", state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>", pto.DisconnectMsg(fLogIPs)); pto.fDisconnect = true; } else { assert(state.m_chain_sync.m_work_header); @@ -5896,7 +5441,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) const auto current_time{GetTime<std::chrono::microseconds>()}; if (pto->IsAddrFetchConn() && current_time - pto->m_connected > 10 * AVG_ADDRESS_BROADCAST_INTERVAL) { - LogDebug(BCLog::NET, "addrfetch connection timeout; disconnecting peer=%d\n", pto->GetId()); + LogDebug(BCLog::NET, "addrfetch connection timeout, %s\n", pto->DisconnectMsg(fLogIPs)); pto->fDisconnect = true; return true; } @@ -6240,7 +5785,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection // should only happen during initial block download. - LogPrintf("Peer=%d%s is stalling block download, disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); + LogInfo("Peer is stalling block download, %s\n", pto->DisconnectMsg(fLogIPs)); pto->fDisconnect = true; // Increase timeout for the next peer so that we don't disconnect multiple peers if our own // bandwidth is insufficient. @@ -6259,7 +5804,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); int nOtherPeersWithValidatedDownloads = m_peers_downloading_from - 1; if (current_time > state.m_downloading_since + std::chrono::seconds{consensusParams.nPowTargetSpacing} * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { - LogPrintf("Timeout downloading block %s from peer=%d%s, disconnecting\n", queuedBlock.pindex->GetBlockHash().ToString(), pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); + LogInfo("Timeout downloading block %s, %s\n", queuedBlock.pindex->GetBlockHash().ToString(), pto->DisconnectMsg(fLogIPs)); pto->fDisconnect = true; return true; } @@ -6275,11 +5820,11 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // disconnect our sync peer for stalling; we have bigger // problems if we can't get any outbound peers. if (!pto->HasPermission(NetPermissionFlags::NoBan)) { - LogPrintf("Timeout downloading headers from peer=%d%s, disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); + LogInfo("Timeout downloading headers, %s\n", pto->DisconnectMsg(fLogIPs)); pto->fDisconnect = true; return true; } else { - LogPrintf("Timeout downloading headers from noban peer=%d%s, not disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); + LogInfo("Timeout downloading headers from noban peer, not %s\n", pto->DisconnectMsg(fLogIPs)); // Reset the headers sync state so that we have a // chance to try downloading from a different peer. // Note: this will also result in at least one more @@ -6345,31 +5890,14 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // { LOCK(m_tx_download_mutex); - std::vector<std::pair<NodeId, GenTxid>> expired; - auto requestable = m_txrequest.GetRequestable(pto->GetId(), current_time, &expired); - for (const auto& entry : expired) { - LogDebug(BCLog::NET, "timeout of inflight %s %s from peer=%d\n", entry.second.IsWtxid() ? "wtx" : "tx", - entry.second.GetHash().ToString(), entry.first); - } - for (const GenTxid& gtxid : requestable) { - // Exclude m_lazy_recent_rejects_reconsiderable: we may be requesting a missing parent - // that was previously rejected for being too low feerate. - if (!AlreadyHaveTx(gtxid, /*include_reconsiderable=*/false)) { - LogDebug(BCLog::NET, "Requesting %s %s peer=%d\n", gtxid.IsWtxid() ? "wtx" : "tx", - gtxid.GetHash().ToString(), pto->GetId()); - vGetData.emplace_back(gtxid.IsWtxid() ? MSG_WTX : (MSG_TX | GetFetchFlags(*peer)), gtxid.GetHash()); - if (vGetData.size() >= MAX_GETDATA_SZ) { - MakeAndPushMessage(*pto, NetMsgType::GETDATA, vGetData); - vGetData.clear(); - } - m_txrequest.RequestedTx(pto->GetId(), gtxid.GetHash(), current_time + GETDATA_TX_INTERVAL); - } else { - // We have already seen this transaction, no need to download. This is just a belt-and-suspenders, as - // this should already be called whenever a transaction becomes AlreadyHaveTx(). - m_txrequest.ForgetTxHash(gtxid.GetHash()); + for (const GenTxid& gtxid : m_txdownloadman.GetRequestsToSend(pto->GetId(), current_time)) { + vGetData.emplace_back(gtxid.IsWtxid() ? MSG_WTX : (MSG_TX | GetFetchFlags(*peer)), gtxid.GetHash()); + if (vGetData.size() >= MAX_GETDATA_SZ) { + MakeAndPushMessage(*pto, NetMsgType::GETDATA, vGetData); + vGetData.clear(); } } - } // release m_tx_download_mutex + } if (!vGetData.empty()) MakeAndPushMessage(*pto, NetMsgType::GETDATA, vGetData); diff --git a/src/net_processing.h b/src/net_processing.h index a413db98e8..0d2dc59c5a 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -7,6 +7,7 @@ #define BITCOIN_NET_PROCESSING_H #include <net.h> +#include <txorphanage.h> #include <validationinterface.h> #include <chrono> @@ -31,6 +32,9 @@ static const bool DEFAULT_PEERBLOOMFILTERS = false; static const bool DEFAULT_PEERBLOCKFILTERS = false; /** Maximum number of outstanding CMPCTBLOCK requests for the same block. */ static const unsigned int MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK = 3; +/** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends + * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ +static const unsigned int MAX_HEADERS_RESULTS = 2000; struct CNodeStateStats { int nSyncHeight = -1; @@ -71,6 +75,9 @@ public: //! Whether or not the internal RNG behaves deterministically (this is //! a test-only option). bool deterministic_rng{false}; + //! Number of headers sent in one getheaders message result (this is + //! a test-only option). + uint32_t max_headers_result{MAX_HEADERS_RESULTS}; }; static std::unique_ptr<PeerManager> make(CConnman& connman, AddrMan& addrman, @@ -93,6 +100,8 @@ public: /** Get statistics from node state */ virtual bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const = 0; + virtual std::vector<TxOrphanage::OrphanTxBase> GetOrphanTransactions() = 0; + /** Get peer manager info. */ virtual PeerManagerInfo GetInfo() const = 0; diff --git a/src/netbase.cpp b/src/netbase.cpp index a6de72090e..eaca5a16c1 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <netbase.h> @@ -230,7 +230,7 @@ CService LookupNumeric(const std::string& name, uint16_t portDefault, DNSLookupF bool IsUnixSocketPath(const std::string& name) { #ifdef HAVE_SOCKADDR_UN - if (name.find(ADDR_PREFIX_UNIX) != 0) return false; + if (!name.starts_with(ADDR_PREFIX_UNIX)) return false; // Split off "unix:" prefix std::string str{name.substr(ADDR_PREFIX_UNIX.length())}; diff --git a/src/netbase.h b/src/netbase.h index 8ef6c28996..bf4d7ececc 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -134,6 +134,13 @@ public: return Contains(addr.GetNetwork()); } + [[nodiscard]] std::unordered_set<Network> All() const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + AssertLockNotHeld(m_mutex); + LOCK(m_mutex); + return m_reachable; + } + private: mutable Mutex m_mutex; diff --git a/src/node/abort.cpp b/src/node/abort.cpp index 8a17c41fd2..c15bf047c8 100644 --- a/src/node/abort.cpp +++ b/src/node/abort.cpp @@ -15,12 +15,12 @@ namespace node { -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message, node::Warnings* warnings) +void AbortNode(const std::function<bool()>& shutdown_request, std::atomic<int>& exit_status, const bilingual_str& message, node::Warnings* warnings) { if (warnings) warnings->Set(Warning::FATAL_INTERNAL_ERROR, message); InitError(_("A fatal internal error occurred, see debug.log for details: ") + message); exit_status.store(EXIT_FAILURE); - if (shutdown && !(*shutdown)()) { + if (shutdown_request && !shutdown_request()) { LogError("Failed to send shutdown signal\n"); }; } diff --git a/src/node/abort.h b/src/node/abort.h index c881af4634..c8514628bc 100644 --- a/src/node/abort.h +++ b/src/node/abort.h @@ -6,16 +6,13 @@ #define BITCOIN_NODE_ABORT_H #include <atomic> +#include <functional> struct bilingual_str; -namespace util { -class SignalInterrupt; -} // namespace util - namespace node { class Warnings; -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message, node::Warnings* warnings); +void AbortNode(const std::function<bool()>& shutdown_request, std::atomic<int>& exit_status, const bilingual_str& message, node::Warnings* warnings); } // namespace node #endif // BITCOIN_NODE_ABORT_H diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index b40ca0260b..07878a5602 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -683,11 +683,7 @@ bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos fileout << GetParams().MessageStart() << nSize; // Write undo data - long fileOutPos = ftell(fileout.Get()); - if (fileOutPos < 0) { - LogError("%s: ftell failed\n", __func__); - return false; - } + long fileOutPos = fileout.tell(); pos.nPos = (unsigned int)fileOutPos; fileout << blockundo; @@ -981,11 +977,7 @@ bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const fileout << GetParams().MessageStart() << nSize; // Write block - long fileOutPos = ftell(fileout.Get()); - if (fileOutPos < 0) { - LogError("%s: ftell failed\n", __func__); - return false; - } + long fileOutPos = fileout.tell(); pos.nPos = (unsigned int)fileOutPos; fileout << TX_WITH_WITNESS(block); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 03bc5f4600..ac6db0558d 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -430,6 +430,7 @@ public: void CleanupBlockRevFiles() const; }; +// Calls ActivateBestChain() even if no blocks are imported. void ImportBlocks(ChainstateManager& chainman, std::span<const fs::path> import_paths); } // namespace node diff --git a/src/node/caches.cpp b/src/node/caches.cpp index 7403f7ddea..dc4d98f592 100644 --- a/src/node/caches.cpp +++ b/src/node/caches.cpp @@ -13,7 +13,6 @@ CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes) { int64_t nTotalCache = (args.GetIntArg("-dbcache", nDefaultDbCache) << 20); nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache - nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache CacheSizes sizes; sizes.block_tree_db = std::min(nTotalCache / 8, nMaxBlockDBCache << 20); nTotalCache -= sizes.block_tree_db; diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index d7e6176be1..e56896fb22 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -41,12 +41,17 @@ static ChainstateLoadResult CompleteChainstateInitialization( // new BlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: pblocktree.reset(); - pblocktree = std::make_unique<BlockTreeDB>(DBParams{ - .path = chainman.m_options.datadir / "blocks" / "index", - .cache_bytes = static_cast<size_t>(cache_sizes.block_tree_db), - .memory_only = options.block_tree_db_in_memory, - .wipe_data = options.wipe_block_tree_db, - .options = chainman.m_options.block_tree_db}); + try { + pblocktree = std::make_unique<BlockTreeDB>(DBParams{ + .path = chainman.m_options.datadir / "blocks" / "index", + .cache_bytes = static_cast<size_t>(cache_sizes.block_tree_db), + .memory_only = options.block_tree_db_in_memory, + .wipe_data = options.wipe_block_tree_db, + .options = chainman.m_options.block_tree_db}); + } catch (dbwrapper_error& err) { + LogError("%s\n", err.what()); + return {ChainstateLoadStatus::FAILURE, _("Error opening block database")}; + } if (options.wipe_block_tree_db) { pblocktree->WriteReindexing(true); @@ -95,11 +100,12 @@ static ChainstateLoadResult CompleteChainstateInitialization( assert(chainman.m_total_coinstip_cache > 0); assert(chainman.m_total_coinsdb_cache > 0); - // Conservative value which is arbitrarily chosen, as it will ultimately be changed - // by a call to `chainman.MaybeRebalanceCaches()`. We just need to make sure - // that the sum of the two caches (40%) does not exceed the allowable amount - // during this temporary initialization state. - double init_cache_fraction = 0.2; + // If running with multiple chainstates, limit the cache sizes with a + // discount factor. If discounted the actual cache size will be + // recalculated by `chainman.MaybeRebalanceCaches()`. The discount factor + // is conservatively chosen such that the sum of the caches does not exceed + // the allowable amount during this temporary initialization state. + double init_cache_fraction = chainman.GetAll().size() > 1 ? 0.2 : 1.0; // At this point we're either in reindex or we've loaded a useful // block tree into BlockIndex()! @@ -107,10 +113,15 @@ static ChainstateLoadResult CompleteChainstateInitialization( for (Chainstate* chainstate : chainman.GetAll()) { LogPrintf("Initializing chainstate %s\n", chainstate->ToString()); - chainstate->InitCoinsDB( - /*cache_size_bytes=*/chainman.m_total_coinsdb_cache * init_cache_fraction, - /*in_memory=*/options.coins_db_in_memory, - /*should_wipe=*/options.wipe_chainstate_db); + try { + chainstate->InitCoinsDB( + /*cache_size_bytes=*/chainman.m_total_coinsdb_cache * init_cache_fraction, + /*in_memory=*/options.coins_db_in_memory, + /*should_wipe=*/options.wipe_chainstate_db); + } catch (dbwrapper_error& err) { + LogError("%s\n", err.what()); + return {ChainstateLoadStatus::FAILURE, _("Error opening coins database")}; + } if (options.coins_error_cb) { chainstate->CoinsErrorCatcher().AddReadErrCallback(options.coins_error_cb); @@ -236,7 +247,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize return {init_status, init_error}; } } else { - return {ChainstateLoadStatus::FAILURE, _( + return {ChainstateLoadStatus::FAILURE_FATAL, _( "UTXO snapshot failed to validate. " "Restart to resume normal initial block download, or try loading a different snapshot.")}; } diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp index b86d0b2991..0ac96c5514 100644 --- a/src/node/chainstatemanager_args.cpp +++ b/src/node/chainstatemanager_args.cpp @@ -35,7 +35,7 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManage if (auto min_work{uint256::FromUserHex(*value)}) { opts.minimum_chain_work = UintToArith256(*min_work); } else { - return util::Error{strprintf(Untranslated("Invalid minimum work specified (%s), must be up to %d hex digits"), *value, uint256::size() * 2)}; + return util::Error{Untranslated(strprintf("Invalid minimum work specified (%s), must be up to %d hex digits", *value, uint256::size() * 2))}; } } @@ -43,7 +43,7 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManage if (auto block_hash{uint256::FromUserHex(*value)}) { opts.assumed_valid_block = *block_hash; } else { - return util::Error{strprintf(Untranslated("Invalid assumevalid block hash specified (%s), must be up to %d hex digits (or 0 to disable)"), *value, uint256::size() * 2)}; + return util::Error{Untranslated(strprintf("Invalid assumevalid block hash specified (%s), must be up to %d hex digits (or 0 to disable)", *value, uint256::size() * 2))}; } } @@ -60,8 +60,7 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManage script_threads += GetNumCores(); } // Subtract 1 because the main thread counts towards the par threads. - opts.worker_threads_num = std::clamp(script_threads - 1, 0, MAX_SCRIPTCHECK_THREADS); - LogPrintf("Script verification uses %d additional threads\n", opts.worker_threads_num); + opts.worker_threads_num = script_threads - 1; if (auto max_size = args.GetIntArg("-maxsigcachesize")) { // 1. When supplied with a max_size of 0, both the signature cache and diff --git a/src/node/chainstatemanager_args.h b/src/node/chainstatemanager_args.h index b2cdba68b8..af13aa8d3c 100644 --- a/src/node/chainstatemanager_args.h +++ b/src/node/chainstatemanager_args.h @@ -10,8 +10,6 @@ class ArgsManager; -/** Maximum number of dedicated script-checking threads allowed */ -static constexpr int MAX_SCRIPTCHECK_THREADS{15}; /** -par default (number of script-checking threads, 0 = auto) */ static constexpr int DEFAULT_SCRIPTCHECK_THREADS{0}; diff --git a/src/node/coin.cpp b/src/node/coin.cpp index 221854c5f6..eba67f4cb5 100644 --- a/src/node/coin.cpp +++ b/src/node/coin.cpp @@ -16,10 +16,11 @@ void FindCoins(const NodeContext& node, std::map<COutPoint, Coin>& coins) LOCK2(cs_main, node.mempool->cs); CCoinsViewCache& chain_view = node.chainman->ActiveChainstate().CoinsTip(); CCoinsViewMemPool mempool_view(&chain_view, *node.mempool); - for (auto& coin : coins) { - if (!mempool_view.GetCoin(coin.first, coin.second)) { - // Either the coin is not in the CCoinsViewCache or is spent. Clear it. - coin.second.Clear(); + for (auto& [outpoint, coin] : coins) { + if (auto c{mempool_view.GetCoin(outpoint)}) { + coin = std::move(*c); + } else { + coin.Clear(); // Either the coin is not in the CCoinsViewCache or is spent } } } diff --git a/src/node/context.h b/src/node/context.h index a664fad80b..debc122120 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -9,6 +9,7 @@ #include <cstdlib> #include <functional> #include <memory> +#include <thread> #include <vector> class ArgsManager; @@ -58,8 +59,10 @@ struct NodeContext { std::unique_ptr<ECC_Context> ecc_context; //! Init interface for initializing current process and connecting to other processes. interfaces::Init* init{nullptr}; + //! Function to request a shutdown. + std::function<bool()> shutdown_request; //! Interrupt object used to track whether node shutdown was requested. - util::SignalInterrupt* shutdown{nullptr}; + util::SignalInterrupt* shutdown_signal{nullptr}; std::unique_ptr<AddrMan> addrman; std::unique_ptr<CConnman> connman; std::unique_ptr<CTxMemPool> mempool; @@ -86,6 +89,7 @@ struct NodeContext { std::atomic<int> exit_status{EXIT_SUCCESS}; //! Manages all the node warnings std::unique_ptr<node::Warnings> warnings; + std::thread background_init_thread; //! Declare default constructor and destructor that are not inline, so code //! instantiating the NodeContext struct doesn't need to #include class diff --git a/src/node/interface_ui.cpp b/src/node/interface_ui.cpp index 4f4d240d1b..c9dcd7a7fb 100644 --- a/src/node/interface_ui.cpp +++ b/src/node/interface_ui.cpp @@ -74,7 +74,7 @@ bool InitError(const bilingual_str& str, const std::vector<std::string>& details // functions which provide error details are ones that run during early init // before the GUI uiInterface is registered, so there's no point passing // main messages and details separately to uiInterface yet. - return InitError(details.empty() ? str : strprintf(Untranslated("%s:\n%s"), str, MakeUnorderedList(details))); + return InitError(details.empty() ? str : str + Untranslated(strprintf(":\n%s", MakeUnorderedList(details)))); } void InitWarning(const bilingual_str& str) diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 4a03183643..f3b8c6a072 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -8,6 +8,7 @@ #include <chain.h> #include <chainparams.h> #include <common/args.h> +#include <consensus/merkle.h> #include <consensus/validation.h> #include <deploymentstatus.h> #include <external_signer.h> @@ -17,6 +18,7 @@ #include <interfaces/handler.h> #include <interfaces/mining.h> #include <interfaces/node.h> +#include <interfaces/types.h> #include <interfaces/wallet.h> #include <kernel/chain.h> #include <kernel/context.h> @@ -33,6 +35,7 @@ #include <node/interface_ui.h> #include <node/mini_miner.h> #include <node/miner.h> +#include <node/kernel_notifications.h> #include <node/transaction.h> #include <node/types.h> #include <node/warnings.h> @@ -58,7 +61,7 @@ #include <validation.h> #include <validationinterface.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <any> #include <memory> @@ -67,6 +70,8 @@ #include <boost/signals2/signal.hpp> +using interfaces::BlockRef; +using interfaces::BlockTemplate; using interfaces::BlockTip; using interfaces::Chain; using interfaces::FoundBlock; @@ -130,9 +135,11 @@ public: } void startShutdown() override { - if (!(*Assert(Assert(m_context)->shutdown))()) { + NodeContext& ctx{*Assert(m_context)}; + if (!(Assert(ctx.shutdown_request))()) { LogError("Failed to send shutdown signal\n"); } + // Stop RPC for clean shutdown if any of waitfor* commands is executed. if (args().GetBoolArg("-server", false)) { InterruptRPC(); @@ -180,7 +187,7 @@ public: }); args().WriteSettingsFile(); } - void mapPort(bool use_upnp, bool use_natpmp) override { StartMapPort(use_upnp, use_natpmp); } + void mapPort(bool use_pcp) override { StartMapPort(use_pcp); } bool getProxy(Network net, Proxy& proxy_info) override { return GetProxy(net, proxy_info); } size_t getNodeCount(ConnectionDirection flags) override { @@ -317,7 +324,7 @@ public: } double getVerificationProgress() override { - return GuessVerificationProgress(chainman().GetParams().TxData(), WITH_LOCK(::cs_main, return chainman().ActiveChain().Tip())); + return chainman().GuessVerificationProgress(WITH_LOCK(chainman().GetMutex(), return chainman().ActiveChain().Tip())); } bool isInitialBlockDownload() override { @@ -351,9 +358,7 @@ public: std::optional<Coin> getUnspentOutput(const COutPoint& output) override { LOCK(::cs_main); - Coin coin; - if (chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin)) return coin; - return {}; + return chainman().ActiveChainstate().CoinsTip().GetCoin(output); } TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) override { @@ -401,9 +406,9 @@ public: } std::unique_ptr<Handler> handleNotifyBlockTip(NotifyBlockTipFn fn) override { - return MakeSignalHandler(::uiInterface.NotifyBlockTip_connect([fn](SynchronizationState sync_state, const CBlockIndex* block) { + return MakeSignalHandler(::uiInterface.NotifyBlockTip_connect([fn, this](SynchronizationState sync_state, const CBlockIndex* block) { fn(sync_state, BlockTip{block->nHeight, block->GetBlockTime(), block->GetBlockHash()}, - GuessVerificationProgress(Params().TxData(), block)); + chainman().GuessVerificationProgress(block)); })); } std::unique_ptr<Handler> handleNotifyHeaderTip(NotifyHeaderTipFn fn) override @@ -634,8 +639,8 @@ public: void findCoins(std::map<COutPoint, Coin>& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override { - LOCK(::cs_main); - return GuessVerificationProgress(chainman().GetParams().TxData(), chainman().m_blockman.LookupBlockIndex(block_hash)); + LOCK(chainman().GetMutex()); + return chainman().GuessVerificationProgress(chainman().m_blockman.LookupBlockIndex(block_hash)); } bool hasBlocks(const uint256& block_hash, int min_height, std::optional<int> max_height) override { @@ -819,29 +824,29 @@ public: { std::optional<interfaces::SettingsAction> action; args().LockSettings([&](common::Settings& settings) { - auto* ptr_value = common::FindKey(settings.rw_settings, name); - // Create value if it doesn't exist - auto& value = ptr_value ? *ptr_value : settings.rw_settings[name]; - action = update_settings_func(value); + if (auto* value = common::FindKey(settings.rw_settings, name)) { + action = update_settings_func(*value); + if (value->isNull()) settings.rw_settings.erase(name); + } else { + UniValue new_value; + action = update_settings_func(new_value); + if (!new_value.isNull()) settings.rw_settings[name] = std::move(new_value); + } }); if (!action) return false; // Now dump value to disk if requested - return *action == interfaces::SettingsAction::SKIP_WRITE || args().WriteSettingsFile(); + return *action != interfaces::SettingsAction::WRITE || args().WriteSettingsFile(); } - bool overwriteRwSetting(const std::string& name, common::SettingsValue& value, bool write) override + bool overwriteRwSetting(const std::string& name, common::SettingsValue value, interfaces::SettingsAction action) override { - if (value.isNull()) return deleteRwSettings(name, write); return updateRwSetting(name, [&](common::SettingsValue& settings) { settings = std::move(value); - return write ? interfaces::SettingsAction::WRITE : interfaces::SettingsAction::SKIP_WRITE; + return action; }); } - bool deleteRwSettings(const std::string& name, bool write) override + bool deleteRwSettings(const std::string& name, interfaces::SettingsAction action) override { - args().LockSettings([&](common::Settings& settings) { - settings.rw_settings.erase(name); - }); - return !write || args().WriteSettingsFile(); + return overwriteRwSetting(name, {}, action); } void requestMempoolTransactions(Notifications& notifications) override { @@ -863,6 +868,80 @@ public: NodeContext& m_node; }; +class BlockTemplateImpl : public BlockTemplate +{ +public: + explicit BlockTemplateImpl(std::unique_ptr<CBlockTemplate> block_template, NodeContext& node) : m_block_template(std::move(block_template)), m_node(node) + { + assert(m_block_template); + } + + CBlockHeader getBlockHeader() override + { + return m_block_template->block; + } + + CBlock getBlock() override + { + return m_block_template->block; + } + + std::vector<CAmount> getTxFees() override + { + return m_block_template->vTxFees; + } + + std::vector<int64_t> getTxSigops() override + { + return m_block_template->vTxSigOpsCost; + } + + CTransactionRef getCoinbaseTx() override + { + return m_block_template->block.vtx[0]; + } + + std::vector<unsigned char> getCoinbaseCommitment() override + { + return m_block_template->vchCoinbaseCommitment; + } + + int getWitnessCommitmentIndex() override + { + return GetWitnessCommitmentIndex(m_block_template->block); + } + + std::vector<uint256> getCoinbaseMerklePath() override + { + return TransactionMerklePath(m_block_template->block, 0); + } + + bool submitSolution(uint32_t version, uint32_t timestamp, uint32_t nonce, CTransactionRef coinbase) override + { + CBlock block{m_block_template->block}; + + if (block.vtx.size() == 0) { + block.vtx.push_back(coinbase); + } else { + block.vtx[0] = coinbase; + } + + block.nVersion = version; + block.nTime = timestamp; + block.nNonce = nonce; + + block.hashMerkleRoot = BlockMerkleRoot(block); + + auto block_ptr = std::make_shared<const CBlock>(block); + return chainman().ProcessNewBlock(block_ptr, /*force_processing=*/true, /*min_pow_checked=*/true, /*new_block=*/nullptr); + } + + const std::unique_ptr<CBlockTemplate> m_block_template; + + ChainstateManager& chainman() { return *Assert(m_node.chainman); } + NodeContext& m_node; +}; + class MinerImpl : public Mining { public: @@ -878,46 +957,40 @@ public: return chainman().IsInitialBlockDownload(); } - std::optional<uint256> getTipHash() override + std::optional<BlockRef> getTip() override { LOCK(::cs_main); CBlockIndex* tip{chainman().ActiveChain().Tip()}; if (!tip) return {}; - return tip->GetBlockHash(); - } - - bool processNewBlock(const std::shared_ptr<const CBlock>& block, bool* new_block) override - { - return chainman().ProcessNewBlock(block, /*force_processing=*/true, /*min_pow_checked=*/true, /*new_block=*/new_block); - } - - unsigned int getTransactionsUpdated() override - { - return context()->mempool->GetTransactionsUpdated(); + return BlockRef{tip->GetBlockHash(), tip->nHeight}; } - bool testBlockValidity(const CBlock& block, bool check_merkle_root, BlockValidationState& state) override + BlockRef waitTipChanged(uint256 current_tip, MillisecondsDouble timeout) override { - LOCK(cs_main); - CBlockIndex* tip{chainman().ActiveChain().Tip()}; - // Fail if the tip updated before the lock was taken - if (block.hashPrevBlock != tip->GetBlockHash()) { - state.Error("Block does not connect to current chain tip."); - return false; + if (timeout > std::chrono::years{100}) timeout = std::chrono::years{100}; // Upper bound to avoid UB in std::chrono + { + WAIT_LOCK(notifications().m_tip_block_mutex, lock); + notifications().m_tip_block_cv.wait_for(lock, timeout, [&]() EXCLUSIVE_LOCKS_REQUIRED(notifications().m_tip_block_mutex) { + // We need to wait for m_tip_block to be set AND for the value + // to differ from the current_tip value. + return (notifications().TipBlock() && notifications().TipBlock() != current_tip) || chainman().m_interrupt; + }); } - - return TestBlockValidity(state, chainman().GetParams(), chainman().ActiveChainstate(), block, tip, /*fCheckPOW=*/false, check_merkle_root); + // Must release m_tip_block_mutex before locking cs_main, to avoid deadlocks. + LOCK(::cs_main); + return BlockRef{chainman().ActiveChain().Tip()->GetBlockHash(), chainman().ActiveChain().Tip()->nHeight}; } - std::unique_ptr<CBlockTemplate> createNewBlock(const CScript& script_pub_key, const BlockCreateOptions& options) override + std::unique_ptr<BlockTemplate> createNewBlock(const BlockCreateOptions& options) override { BlockAssembler::Options assemble_options{options}; ApplyArgsManOptions(*Assert(m_node.args), assemble_options); - return BlockAssembler{chainman().ActiveChainstate(), context()->mempool.get(), assemble_options}.CreateNewBlock(script_pub_key); + return std::make_unique<BlockTemplateImpl>(BlockAssembler{chainman().ActiveChainstate(), context()->mempool.get(), assemble_options}.CreateNewBlock(), m_node); } NodeContext* context() override { return &m_node; } ChainstateManager& chainman() { return *Assert(m_node.chainman); } + KernelNotifications& notifications() { return *Assert(m_node.notifications); } NodeContext& m_node; }; } // namespace diff --git a/src/node/kernel_notifications.cpp b/src/node/kernel_notifications.cpp index 9894052a3a..550ffe74c4 100644 --- a/src/node/kernel_notifications.cpp +++ b/src/node/kernel_notifications.cpp @@ -4,7 +4,7 @@ #include <node/kernel_notifications.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chain.h> #include <common/args.h> @@ -50,9 +50,16 @@ namespace node { kernel::InterruptResult KernelNotifications::blockTip(SynchronizationState state, CBlockIndex& index) { + { + LOCK(m_tip_block_mutex); + Assume(index.GetBlockHash() != uint256::ZERO); + m_tip_block = index.GetBlockHash(); + m_tip_block_cv.notify_all(); + } + uiInterface.NotifyBlockTip(state, &index); if (m_stop_at_height && index.nHeight >= m_stop_at_height) { - if (!m_shutdown()) { + if (!m_shutdown_request()) { LogError("Failed to send shutdown signal after reaching stop height\n"); } return kernel::Interrupted{}; @@ -84,15 +91,22 @@ void KernelNotifications::warningUnset(kernel::Warning id) void KernelNotifications::flushError(const bilingual_str& message) { - AbortNode(&m_shutdown, m_exit_status, message, &m_warnings); + AbortNode(m_shutdown_request, m_exit_status, message, &m_warnings); } void KernelNotifications::fatalError(const bilingual_str& message) { - node::AbortNode(m_shutdown_on_fatal_error ? &m_shutdown : nullptr, + node::AbortNode(m_shutdown_on_fatal_error ? m_shutdown_request : nullptr, m_exit_status, message, &m_warnings); } +std::optional<uint256> KernelNotifications::TipBlock() +{ + AssertLockHeld(m_tip_block_mutex); + return m_tip_block; +}; + + void ReadNotificationArgs(const ArgsManager& args, KernelNotifications& notifications) { if (auto value{args.GetIntArg("-stopatheight")}) notifications.m_stop_at_height = *value; diff --git a/src/node/kernel_notifications.h b/src/node/kernel_notifications.h index e37f4d4e1e..f4174381da 100644 --- a/src/node/kernel_notifications.h +++ b/src/node/kernel_notifications.h @@ -7,8 +7,13 @@ #include <kernel/notifications_interface.h> +#include <sync.h> +#include <threadsafety.h> +#include <uint256.h> + #include <atomic> #include <cstdint> +#include <functional> class ArgsManager; class CBlockIndex; @@ -19,10 +24,6 @@ namespace kernel { enum class Warning; } // namespace kernel -namespace util { -class SignalInterrupt; -} // namespace util - namespace node { class Warnings; @@ -31,10 +32,10 @@ static constexpr int DEFAULT_STOPATHEIGHT{0}; class KernelNotifications : public kernel::Notifications { public: - KernelNotifications(util::SignalInterrupt& shutdown, std::atomic<int>& exit_status, node::Warnings& warnings) - : m_shutdown(shutdown), m_exit_status{exit_status}, m_warnings{warnings} {} + KernelNotifications(const std::function<bool()>& shutdown_request, std::atomic<int>& exit_status, node::Warnings& warnings) + : m_shutdown_request(shutdown_request), m_exit_status{exit_status}, m_warnings{warnings} {} - [[nodiscard]] kernel::InterruptResult blockTip(SynchronizationState state, CBlockIndex& index) override; + [[nodiscard]] kernel::InterruptResult blockTip(SynchronizationState state, CBlockIndex& index) override EXCLUSIVE_LOCKS_REQUIRED(!m_tip_block_mutex); void headerTip(SynchronizationState state, int64_t height, int64_t timestamp, bool presync) override; @@ -52,10 +53,20 @@ public: int m_stop_at_height{DEFAULT_STOPATHEIGHT}; //! Useful for tests, can be set to false to avoid shutdown on fatal error. bool m_shutdown_on_fatal_error{true}; + + Mutex m_tip_block_mutex; + std::condition_variable m_tip_block_cv GUARDED_BY(m_tip_block_mutex); + //! The block for which the last blockTip notification was received. + //! It's first set when the tip is connected during node initialization. + //! Might be unset during an early shutdown. + std::optional<uint256> TipBlock() EXCLUSIVE_LOCKS_REQUIRED(m_tip_block_mutex); + private: - util::SignalInterrupt& m_shutdown; + const std::function<bool()>& m_shutdown_request; std::atomic<int>& m_exit_status; node::Warnings& m_warnings; + + std::optional<uint256> m_tip_block GUARDED_BY(m_tip_block_mutex); }; void ReadNotificationArgs(const ArgsManager& args, KernelNotifications& notifications); diff --git a/src/node/mempool_args.cpp b/src/node/mempool_args.cpp index a488c1b149..e3bf5fb57c 100644 --- a/src/node/mempool_args.cpp +++ b/src/node/mempool_args.cpp @@ -89,12 +89,7 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainP mempool_opts.require_standard = !argsman.GetBoolArg("-acceptnonstdtxn", DEFAULT_ACCEPT_NON_STD_TXN); if (!chainparams.IsTestChain() && !mempool_opts.require_standard) { - return util::Error{strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.GetChainTypeString())}; - } - - mempool_opts.full_rbf = argsman.GetBoolArg("-mempoolfullrbf", mempool_opts.full_rbf); - if (!mempool_opts.full_rbf) { - LogInfo("Warning: mempoolfullrbf=0 set but deprecated and will be removed in a future release\n"); + return util::Error{Untranslated(strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.GetChainTypeString()))}; } mempool_opts.persist_v1_dat = argsman.GetBoolArg("-persistmempoolv1", mempool_opts.persist_v1_dat); diff --git a/src/node/mempool_persist.cpp b/src/node/mempool_persist.cpp index a265c2e12d..ff7de8c64a 100644 --- a/src/node/mempool_persist.cpp +++ b/src/node/mempool_persist.cpp @@ -199,8 +199,8 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock LogInfo("Writing %d unbroadcast transactions to file.\n", unbroadcast_txids.size()); file << unbroadcast_txids; - if (!skip_file_commit && !FileCommit(file.Get())) - throw std::runtime_error("FileCommit failed"); + if (!skip_file_commit && !file.Commit()) + throw std::runtime_error("Commit failed"); file.fclose(); if (!RenameOver(dump_path + ".new", dump_path)) { throw std::runtime_error("Rename failed"); diff --git a/src/node/miner.cpp b/src/node/miner.cpp index 97f6ac346a..5d7304b597 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -106,17 +106,13 @@ void BlockAssembler::resetBlock() nFees = 0; } -std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& scriptPubKeyIn) +std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock() { const auto time_start{SteadyClock::now()}; resetBlock(); pblocktemplate.reset(new CBlockTemplate()); - - if (!pblocktemplate.get()) { - return nullptr; - } CBlock* const pblock = &pblocktemplate->block; // pointer for convenience // Add dummy coinbase tx as first transaction @@ -142,8 +138,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc int nPackagesSelected = 0; int nDescendantsUpdated = 0; if (m_mempool) { - LOCK(m_mempool->cs); - addPackageTxs(*m_mempool, nPackagesSelected, nDescendantsUpdated); + addPackageTxs(nPackagesSelected, nDescendantsUpdated); } const auto time_1{SteadyClock::now()}; @@ -156,7 +151,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc coinbaseTx.vin.resize(1); coinbaseTx.vin[0].prevout.SetNull(); coinbaseTx.vout.resize(1); - coinbaseTx.vout[0].scriptPubKey = scriptPubKeyIn; + coinbaseTx.vout[0].scriptPubKey = m_options.coinbase_output_script; coinbaseTx.vout[0].nValue = nFees + GetBlockSubsidy(nHeight, chainparams.GetConsensus()); coinbaseTx.vin[0].scriptSig = CScript() << nHeight << OP_0; pblock->vtx[0] = MakeTransactionRef(std::move(coinbaseTx)); @@ -292,9 +287,10 @@ void BlockAssembler::SortForBlock(const CTxMemPool::setEntries& package, std::ve // Each time through the loop, we compare the best transaction in // mapModifiedTxs with the next transaction in the mempool to decide what // transaction package to work on next. -void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSelected, int& nDescendantsUpdated) +void BlockAssembler::addPackageTxs(int& nPackagesSelected, int& nDescendantsUpdated) { - AssertLockHeld(mempool.cs); + const auto& mempool{*Assert(m_mempool)}; + LOCK(mempool.cs); // mapModifiedTx will store sorted packages after they are modified // because some of their txs are already in the block diff --git a/src/node/miner.h b/src/node/miner.h index 1b82943766..f6461a8d55 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -169,8 +169,8 @@ public: explicit BlockAssembler(Chainstate& chainstate, const CTxMemPool* mempool, const Options& options); - /** Construct a new block template with coinbase to scriptPubKeyIn */ - std::unique_ptr<CBlockTemplate> CreateNewBlock(const CScript& scriptPubKeyIn); + /** Construct a new block template */ + std::unique_ptr<CBlockTemplate> CreateNewBlock(); inline static std::optional<int64_t> m_last_block_num_txs{}; inline static std::optional<int64_t> m_last_block_weight{}; @@ -187,8 +187,11 @@ private: // Methods for how to add transactions to a block. /** Add transactions based on feerate including unconfirmed ancestors * Increments nPackagesSelected / nDescendantsUpdated with corresponding - * statistics from the package selection (for logging statistics). */ - void addPackageTxs(const CTxMemPool& mempool, int& nPackagesSelected, int& nDescendantsUpdated) EXCLUSIVE_LOCKS_REQUIRED(mempool.cs); + * statistics from the package selection (for logging statistics). + * + * @pre BlockAssembler::m_mempool must not be nullptr + */ + void addPackageTxs(int& nPackagesSelected, int& nDescendantsUpdated) EXCLUSIVE_LOCKS_REQUIRED(!m_mempool->cs); // helper functions for addPackageTxs() /** Remove confirmed (inBlock) entries from given set */ diff --git a/src/node/txdownloadman.h b/src/node/txdownloadman.h new file mode 100644 index 0000000000..81b0c76e0a --- /dev/null +++ b/src/node/txdownloadman.h @@ -0,0 +1,179 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_TXDOWNLOADMAN_H +#define BITCOIN_NODE_TXDOWNLOADMAN_H + +#include <net.h> +#include <policy/packages.h> +#include <txorphanage.h> + +#include <cstdint> +#include <memory> + +class CBlock; +class CRollingBloomFilter; +class CTxMemPool; +class GenTxid; +class TxRequestTracker; +namespace node { +class TxDownloadManagerImpl; + +/** Maximum number of in-flight transaction requests from a peer. It is not a hard limit, but the threshold at which + * point the OVERLOADED_PEER_TX_DELAY kicks in. */ +static constexpr int32_t MAX_PEER_TX_REQUEST_IN_FLIGHT = 100; +/** Maximum number of transactions to consider for requesting, per peer. It provides a reasonable DoS limit to + * per-peer memory usage spent on announcements, while covering peers continuously sending INVs at the maximum + * rate (by our own policy, see INVENTORY_BROADCAST_PER_SECOND) for several minutes, while not receiving + * the actual transaction (from any peer) in response to requests for them. */ +static constexpr int32_t MAX_PEER_TX_ANNOUNCEMENTS = 5000; +/** How long to delay requesting transactions via txids, if we have wtxid-relaying peers */ +static constexpr auto TXID_RELAY_DELAY{2s}; +/** How long to delay requesting transactions from non-preferred peers */ +static constexpr auto NONPREF_PEER_TX_DELAY{2s}; +/** How long to delay requesting transactions from overloaded peers (see MAX_PEER_TX_REQUEST_IN_FLIGHT). */ +static constexpr auto OVERLOADED_PEER_TX_DELAY{2s}; +/** How long to wait before downloading a transaction from an additional peer */ +static constexpr auto GETDATA_TX_INTERVAL{60s}; +struct TxDownloadOptions { + /** Read-only reference to mempool. */ + const CTxMemPool& m_mempool; + /** RNG provided by caller. */ + FastRandomContext& m_rng; + /** Maximum number of transactions allowed in orphanage. */ + const uint32_t m_max_orphan_txs; + /** Instantiate TxRequestTracker as deterministic (used for tests). */ + bool m_deterministic_txrequest{false}; +}; +struct TxDownloadConnectionInfo { + /** Whether this peer is preferred for transaction download. */ + const bool m_preferred; + /** Whether this peer has Relay permissions. */ + const bool m_relay_permissions; + /** Whether this peer supports wtxid relay. */ + const bool m_wtxid_relay; +}; +struct PackageToValidate { + Package m_txns; + std::vector<NodeId> m_senders; + /** Construct a 1-parent-1-child package. */ + explicit PackageToValidate(const CTransactionRef& parent, + const CTransactionRef& child, + NodeId parent_sender, + NodeId child_sender) : + m_txns{parent, child}, + m_senders{parent_sender, child_sender} + {} + + // Move ctor + PackageToValidate(PackageToValidate&& other) : m_txns{std::move(other.m_txns)}, m_senders{std::move(other.m_senders)} {} + // Copy ctor + PackageToValidate(const PackageToValidate& other) = default; + + // Move assignment + PackageToValidate& operator=(PackageToValidate&& other) { + this->m_txns = std::move(other.m_txns); + this->m_senders = std::move(other.m_senders); + return *this; + } + + std::string ToString() const { + Assume(m_txns.size() == 2); + return strprintf("parent %s (wtxid=%s, sender=%d) + child %s (wtxid=%s, sender=%d)", + m_txns.front()->GetHash().ToString(), + m_txns.front()->GetWitnessHash().ToString(), + m_senders.front(), + m_txns.back()->GetHash().ToString(), + m_txns.back()->GetWitnessHash().ToString(), + m_senders.back()); + } +}; +struct RejectedTxTodo +{ + bool m_should_add_extra_compact_tx; + std::vector<uint256> m_unique_parents; + std::optional<PackageToValidate> m_package_to_validate; +}; + + +/** + * Class responsible for deciding what transactions to request and, once + * downloaded, whether and how to validate them. It is also responsible for + * deciding what transaction packages to validate and how to resolve orphan + * transactions. Its data structures include TxRequestTracker for scheduling + * requests, rolling bloom filters for remembering transactions that have + * already been {accepted, rejected, confirmed}, an orphanage, and a registry of + * each peer's transaction relay-related information. + * + * Caller needs to interact with TxDownloadManager: + * - ValidationInterface callbacks. + * - When a potential transaction relay peer connects or disconnects. + * - When a transaction or package is accepted or rejected from mempool + * - When a inv, notfound, or tx message is received + * - To get instructions for which getdata messages to send + * + * This class is not thread-safe. Access must be synchronized using an + * external mutex. + */ +class TxDownloadManager { + const std::unique_ptr<TxDownloadManagerImpl> m_impl; + +public: + explicit TxDownloadManager(const TxDownloadOptions& options); + ~TxDownloadManager(); + + // Responses to chain events. TxDownloadManager is not an actual client of ValidationInterface, these are called through PeerManager. + void ActiveTipChange(); + void BlockConnected(const std::shared_ptr<const CBlock>& pblock); + void BlockDisconnected(); + + /** Creates a new PeerInfo. Saves the connection info to calculate tx announcement delays later. */ + void ConnectedPeer(NodeId nodeid, const TxDownloadConnectionInfo& info); + + /** Deletes all txrequest announcements and orphans for a given peer. */ + void DisconnectedPeer(NodeId nodeid); + + /** Consider adding this tx hash to txrequest. Should be called whenever a new inv has been received. + * Also called internally when a transaction is missing parents so that we can request them. + * @param[in] p2p_inv When true, only add this announcement if we don't already have the tx. + * Returns true if this was a dropped inv (p2p_inv=true and we already have the tx), false otherwise. */ + bool AddTxAnnouncement(NodeId peer, const GenTxid& gtxid, std::chrono::microseconds now, bool p2p_inv); + + /** Get getdata requests to send. */ + std::vector<GenTxid> GetRequestsToSend(NodeId nodeid, std::chrono::microseconds current_time); + + /** Should be called when a notfound for a tx has been received. */ + void ReceivedNotFound(NodeId nodeid, const std::vector<uint256>& txhashes); + + /** Respond to successful transaction submission to mempool */ + void MempoolAcceptedTx(const CTransactionRef& tx); + + /** Respond to transaction rejected from mempool */ + RejectedTxTodo MempoolRejectedTx(const CTransactionRef& ptx, const TxValidationState& state, NodeId nodeid, bool first_time_failure); + + /** Respond to package rejected from mempool */ + void MempoolRejectedPackage(const Package& package); + + /** Marks a tx as ReceivedResponse in txrequest and checks whether AlreadyHaveTx. + * Return a bool indicating whether this tx should be validated. If false, optionally, a + * PackageToValidate. */ + std::pair<bool, std::optional<PackageToValidate>> ReceivedTx(NodeId nodeid, const CTransactionRef& ptx); + + /** Whether there are any orphans to reconsider for this peer. */ + bool HaveMoreWork(NodeId nodeid) const; + + /** Returns next orphan tx to consider, or nullptr if none exist. */ + CTransactionRef GetTxToReconsider(NodeId nodeid); + + /** Check that all data structures are empty. */ + void CheckIsEmpty() const; + + /** Check that all data structures that track per-peer information have nothing for this peer. */ + void CheckIsEmpty(NodeId nodeid) const; + + /** Wrapper for TxOrphanage::GetOrphanTransactions */ + std::vector<TxOrphanage::OrphanTxBase> GetOrphanTransactions() const; +}; +} // namespace node +#endif // BITCOIN_NODE_TXDOWNLOADMAN_H diff --git a/src/node/txdownloadman_impl.cpp b/src/node/txdownloadman_impl.cpp new file mode 100644 index 0000000000..f9635d049a --- /dev/null +++ b/src/node/txdownloadman_impl.cpp @@ -0,0 +1,536 @@ +// Copyright (c) 2024 +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/txdownloadman_impl.h> +#include <node/txdownloadman.h> + +#include <chain.h> +#include <consensus/validation.h> +#include <logging.h> +#include <txmempool.h> +#include <validation.h> +#include <validationinterface.h> + +namespace node { +// TxDownloadManager wrappers +TxDownloadManager::TxDownloadManager(const TxDownloadOptions& options) : + m_impl{std::make_unique<TxDownloadManagerImpl>(options)} +{} +TxDownloadManager::~TxDownloadManager() = default; + +void TxDownloadManager::ActiveTipChange() +{ + m_impl->ActiveTipChange(); +} +void TxDownloadManager::BlockConnected(const std::shared_ptr<const CBlock>& pblock) +{ + m_impl->BlockConnected(pblock); +} +void TxDownloadManager::BlockDisconnected() +{ + m_impl->BlockDisconnected(); +} +void TxDownloadManager::ConnectedPeer(NodeId nodeid, const TxDownloadConnectionInfo& info) +{ + m_impl->ConnectedPeer(nodeid, info); +} +void TxDownloadManager::DisconnectedPeer(NodeId nodeid) +{ + m_impl->DisconnectedPeer(nodeid); +} +bool TxDownloadManager::AddTxAnnouncement(NodeId peer, const GenTxid& gtxid, std::chrono::microseconds now, bool p2p_inv) +{ + return m_impl->AddTxAnnouncement(peer, gtxid, now, p2p_inv); +} +std::vector<GenTxid> TxDownloadManager::GetRequestsToSend(NodeId nodeid, std::chrono::microseconds current_time) +{ + return m_impl->GetRequestsToSend(nodeid, current_time); +} +void TxDownloadManager::ReceivedNotFound(NodeId nodeid, const std::vector<uint256>& txhashes) +{ + m_impl->ReceivedNotFound(nodeid, txhashes); +} +void TxDownloadManager::MempoolAcceptedTx(const CTransactionRef& tx) +{ + m_impl->MempoolAcceptedTx(tx); +} +RejectedTxTodo TxDownloadManager::MempoolRejectedTx(const CTransactionRef& ptx, const TxValidationState& state, NodeId nodeid, bool first_time_failure) +{ + return m_impl->MempoolRejectedTx(ptx, state, nodeid, first_time_failure); +} +void TxDownloadManager::MempoolRejectedPackage(const Package& package) +{ + m_impl->MempoolRejectedPackage(package); +} +std::pair<bool, std::optional<PackageToValidate>> TxDownloadManager::ReceivedTx(NodeId nodeid, const CTransactionRef& ptx) +{ + return m_impl->ReceivedTx(nodeid, ptx); +} +bool TxDownloadManager::HaveMoreWork(NodeId nodeid) const +{ + return m_impl->HaveMoreWork(nodeid); +} +CTransactionRef TxDownloadManager::GetTxToReconsider(NodeId nodeid) +{ + return m_impl->GetTxToReconsider(nodeid); +} +void TxDownloadManager::CheckIsEmpty() const +{ + m_impl->CheckIsEmpty(); +} +void TxDownloadManager::CheckIsEmpty(NodeId nodeid) const +{ + m_impl->CheckIsEmpty(nodeid); +} +std::vector<TxOrphanage::OrphanTxBase> TxDownloadManager::GetOrphanTransactions() const +{ + return m_impl->GetOrphanTransactions(); +} + +// TxDownloadManagerImpl +void TxDownloadManagerImpl::ActiveTipChange() +{ + RecentRejectsFilter().reset(); + RecentRejectsReconsiderableFilter().reset(); +} + +void TxDownloadManagerImpl::BlockConnected(const std::shared_ptr<const CBlock>& pblock) +{ + m_orphanage.EraseForBlock(*pblock); + + for (const auto& ptx : pblock->vtx) { + RecentConfirmedTransactionsFilter().insert(ptx->GetHash().ToUint256()); + if (ptx->HasWitness()) { + RecentConfirmedTransactionsFilter().insert(ptx->GetWitnessHash().ToUint256()); + } + m_txrequest.ForgetTxHash(ptx->GetHash()); + m_txrequest.ForgetTxHash(ptx->GetWitnessHash()); + } +} + +void TxDownloadManagerImpl::BlockDisconnected() +{ + // To avoid relay problems with transactions that were previously + // confirmed, clear our filter of recently confirmed transactions whenever + // there's a reorg. + // This means that in a 1-block reorg (where 1 block is disconnected and + // then another block reconnected), our filter will drop to having only one + // block's worth of transactions in it, but that should be fine, since + // presumably the most common case of relaying a confirmed transaction + // should be just after a new block containing it is found. + RecentConfirmedTransactionsFilter().reset(); +} + +bool TxDownloadManagerImpl::AlreadyHaveTx(const GenTxid& gtxid, bool include_reconsiderable) +{ + const uint256& hash = gtxid.GetHash(); + + if (gtxid.IsWtxid()) { + // Normal query by wtxid. + if (m_orphanage.HaveTx(Wtxid::FromUint256(hash))) return true; + } else { + // Never query by txid: it is possible that the transaction in the orphanage has the same + // txid but a different witness, which would give us a false positive result. If we decided + // not to request the transaction based on this result, an attacker could prevent us from + // downloading a transaction by intentionally creating a malleated version of it. While + // only one (or none!) of these transactions can ultimately be confirmed, we have no way of + // discerning which one that is, so the orphanage can store multiple transactions with the + // same txid. + // + // While we won't query by txid, we can try to "guess" what the wtxid is based on the txid. + // A non-segwit transaction's txid == wtxid. Query this txid "casted" to a wtxid. This will + // help us find non-segwit transactions, saving bandwidth, and should have no false positives. + if (m_orphanage.HaveTx(Wtxid::FromUint256(hash))) return true; + } + + if (include_reconsiderable && RecentRejectsReconsiderableFilter().contains(hash)) return true; + + if (RecentConfirmedTransactionsFilter().contains(hash)) return true; + + return RecentRejectsFilter().contains(hash) || m_opts.m_mempool.exists(gtxid); +} + +void TxDownloadManagerImpl::ConnectedPeer(NodeId nodeid, const TxDownloadConnectionInfo& info) +{ + // If already connected (shouldn't happen in practice), exit early. + if (m_peer_info.contains(nodeid)) return; + + m_peer_info.try_emplace(nodeid, info); + if (info.m_wtxid_relay) m_num_wtxid_peers += 1; +} + +void TxDownloadManagerImpl::DisconnectedPeer(NodeId nodeid) +{ + m_orphanage.EraseForPeer(nodeid); + m_txrequest.DisconnectedPeer(nodeid); + + if (auto it = m_peer_info.find(nodeid); it != m_peer_info.end()) { + if (it->second.m_connection_info.m_wtxid_relay) m_num_wtxid_peers -= 1; + m_peer_info.erase(it); + } + +} + +bool TxDownloadManagerImpl::AddTxAnnouncement(NodeId peer, const GenTxid& gtxid, std::chrono::microseconds now, bool p2p_inv) +{ + // If this is an inv received from a peer and we already have it, we can drop it. + // If this is a request for the parent of an orphan, we don't drop transactions that we already have. In particular, + // we *do* want to request parents that are in m_lazy_recent_rejects_reconsiderable, since they can be CPFP'd. + if (p2p_inv && AlreadyHaveTx(gtxid, /*include_reconsiderable=*/true)) return true; + + auto it = m_peer_info.find(peer); + if (it == m_peer_info.end()) return false; + const auto& info = it->second.m_connection_info; + if (!info.m_relay_permissions && m_txrequest.Count(peer) >= MAX_PEER_TX_ANNOUNCEMENTS) { + // Too many queued announcements for this peer + return false; + } + // Decide the TxRequestTracker parameters for this announcement: + // - "preferred": if fPreferredDownload is set (= outbound, or NetPermissionFlags::NoBan permission) + // - "reqtime": current time plus delays for: + // - NONPREF_PEER_TX_DELAY for announcements from non-preferred connections + // - TXID_RELAY_DELAY for txid announcements while wtxid peers are available + // - OVERLOADED_PEER_TX_DELAY for announcements from peers which have at least + // MAX_PEER_TX_REQUEST_IN_FLIGHT requests in flight (and don't have NetPermissionFlags::Relay). + auto delay{0us}; + if (!info.m_preferred) delay += NONPREF_PEER_TX_DELAY; + if (!gtxid.IsWtxid() && m_num_wtxid_peers > 0) delay += TXID_RELAY_DELAY; + const bool overloaded = !info.m_relay_permissions && m_txrequest.CountInFlight(peer) >= MAX_PEER_TX_REQUEST_IN_FLIGHT; + if (overloaded) delay += OVERLOADED_PEER_TX_DELAY; + + m_txrequest.ReceivedInv(peer, gtxid, info.m_preferred, now + delay); + + return false; +} + +std::vector<GenTxid> TxDownloadManagerImpl::GetRequestsToSend(NodeId nodeid, std::chrono::microseconds current_time) +{ + std::vector<GenTxid> requests; + std::vector<std::pair<NodeId, GenTxid>> expired; + auto requestable = m_txrequest.GetRequestable(nodeid, current_time, &expired); + for (const auto& entry : expired) { + LogDebug(BCLog::NET, "timeout of inflight %s %s from peer=%d\n", entry.second.IsWtxid() ? "wtx" : "tx", + entry.second.GetHash().ToString(), entry.first); + } + for (const GenTxid& gtxid : requestable) { + if (!AlreadyHaveTx(gtxid, /*include_reconsiderable=*/false)) { + LogDebug(BCLog::NET, "Requesting %s %s peer=%d\n", gtxid.IsWtxid() ? "wtx" : "tx", + gtxid.GetHash().ToString(), nodeid); + requests.emplace_back(gtxid); + m_txrequest.RequestedTx(nodeid, gtxid.GetHash(), current_time + GETDATA_TX_INTERVAL); + } else { + // We have already seen this transaction, no need to download. This is just a belt-and-suspenders, as + // this should already be called whenever a transaction becomes AlreadyHaveTx(). + m_txrequest.ForgetTxHash(gtxid.GetHash()); + } + } + return requests; +} + +void TxDownloadManagerImpl::ReceivedNotFound(NodeId nodeid, const std::vector<uint256>& txhashes) +{ + for (const auto& txhash : txhashes) { + // If we receive a NOTFOUND message for a tx we requested, mark the announcement for it as + // completed in TxRequestTracker. + m_txrequest.ReceivedResponse(nodeid, txhash); + } +} + +std::optional<PackageToValidate> TxDownloadManagerImpl::Find1P1CPackage(const CTransactionRef& ptx, NodeId nodeid) +{ + const auto& parent_wtxid{ptx->GetWitnessHash()}; + + Assume(RecentRejectsReconsiderableFilter().contains(parent_wtxid.ToUint256())); + + // Prefer children from this peer. This helps prevent censorship attempts in which an attacker + // sends lots of fake children for the parent, and we (unluckily) keep selecting the fake + // children instead of the real one provided by the honest peer. + const auto cpfp_candidates_same_peer{m_orphanage.GetChildrenFromSamePeer(ptx, nodeid)}; + + // These children should be sorted from newest to oldest. In the (probably uncommon) case + // of children that replace each other, this helps us accept the highest feerate (probably the + // most recent) one efficiently. + for (const auto& child : cpfp_candidates_same_peer) { + Package maybe_cpfp_package{ptx, child}; + if (!RecentRejectsReconsiderableFilter().contains(GetPackageHash(maybe_cpfp_package)) && + !RecentRejectsFilter().contains(child->GetHash().ToUint256())) { + return PackageToValidate{ptx, child, nodeid, nodeid}; + } + } + + // If no suitable candidate from the same peer is found, also try children that were provided by + // a different peer. This is useful because sometimes multiple peers announce both transactions + // to us, and we happen to download them from different peers (we wouldn't have known that these + // 2 transactions are related). We still want to find 1p1c packages then. + // + // If we start tracking all announcers of orphans, we can restrict this logic to parent + child + // pairs in which both were provided by the same peer, i.e. delete this step. + const auto cpfp_candidates_different_peer{m_orphanage.GetChildrenFromDifferentPeer(ptx, nodeid)}; + + // Find the first 1p1c that hasn't already been rejected. We randomize the order to not + // create a bias that attackers can use to delay package acceptance. + // + // Create a random permutation of the indices. + std::vector<size_t> tx_indices(cpfp_candidates_different_peer.size()); + std::iota(tx_indices.begin(), tx_indices.end(), 0); + std::shuffle(tx_indices.begin(), tx_indices.end(), m_opts.m_rng); + + for (const auto index : tx_indices) { + // If we already tried a package and failed for any reason, the combined hash was + // cached in m_lazy_recent_rejects_reconsiderable. + const auto [child_tx, child_sender] = cpfp_candidates_different_peer.at(index); + Package maybe_cpfp_package{ptx, child_tx}; + if (!RecentRejectsReconsiderableFilter().contains(GetPackageHash(maybe_cpfp_package)) && + !RecentRejectsFilter().contains(child_tx->GetHash().ToUint256())) { + return PackageToValidate{ptx, child_tx, nodeid, child_sender}; + } + } + return std::nullopt; +} + +void TxDownloadManagerImpl::MempoolAcceptedTx(const CTransactionRef& tx) +{ + // As this version of the transaction was acceptable, we can forget about any requests for it. + // No-op if the tx is not in txrequest. + m_txrequest.ForgetTxHash(tx->GetHash()); + m_txrequest.ForgetTxHash(tx->GetWitnessHash()); + + m_orphanage.AddChildrenToWorkSet(*tx); + // If it came from the orphanage, remove it. No-op if the tx is not in txorphanage. + m_orphanage.EraseTx(tx->GetWitnessHash()); +} + +node::RejectedTxTodo TxDownloadManagerImpl::MempoolRejectedTx(const CTransactionRef& ptx, const TxValidationState& state, NodeId nodeid, bool first_time_failure) +{ + const CTransaction& tx{*ptx}; + // Results returned to caller + // Whether we should call AddToCompactExtraTransactions at the end + bool add_extra_compact_tx{first_time_failure}; + // Hashes to pass to AddKnownTx later + std::vector<uint256> unique_parents; + // Populated if failure is reconsiderable and eligible package is found. + std::optional<node::PackageToValidate> package_to_validate; + + if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { + // Only process a new orphan if this is a first time failure, as otherwise it must be either + // already in orphanage or from 1p1c processing. + if (first_time_failure && !RecentRejectsFilter().contains(ptx->GetWitnessHash().ToUint256())) { + bool fRejectedParents = false; // It may be the case that the orphans parents have all been rejected + + // Deduplicate parent txids, so that we don't have to loop over + // the same parent txid more than once down below. + unique_parents.reserve(tx.vin.size()); + for (const CTxIn& txin : tx.vin) { + // We start with all parents, and then remove duplicates below. + unique_parents.push_back(txin.prevout.hash); + } + std::sort(unique_parents.begin(), unique_parents.end()); + unique_parents.erase(std::unique(unique_parents.begin(), unique_parents.end()), unique_parents.end()); + + // Distinguish between parents in m_lazy_recent_rejects and m_lazy_recent_rejects_reconsiderable. + // We can tolerate having up to 1 parent in m_lazy_recent_rejects_reconsiderable since we + // submit 1p1c packages. However, fail immediately if any are in m_lazy_recent_rejects. + std::optional<uint256> rejected_parent_reconsiderable; + for (const uint256& parent_txid : unique_parents) { + if (RecentRejectsFilter().contains(parent_txid)) { + fRejectedParents = true; + break; + } else if (RecentRejectsReconsiderableFilter().contains(parent_txid) && + !m_opts.m_mempool.exists(GenTxid::Txid(parent_txid))) { + // More than 1 parent in m_lazy_recent_rejects_reconsiderable: 1p1c will not be + // sufficient to accept this package, so just give up here. + if (rejected_parent_reconsiderable.has_value()) { + fRejectedParents = true; + break; + } + rejected_parent_reconsiderable = parent_txid; + } + } + if (!fRejectedParents) { + const auto current_time{GetTime<std::chrono::microseconds>()}; + + for (const uint256& parent_txid : unique_parents) { + // Here, we only have the txid (and not wtxid) of the + // inputs, so we only request in txid mode, even for + // wtxidrelay peers. + // Eventually we should replace this with an improved + // protocol for getting all unconfirmed parents. + const auto gtxid{GenTxid::Txid(parent_txid)}; + // Exclude m_lazy_recent_rejects_reconsiderable: the missing parent may have been + // previously rejected for being too low feerate. This orphan might CPFP it. + if (!AlreadyHaveTx(gtxid, /*include_reconsiderable=*/false)) { + AddTxAnnouncement(nodeid, gtxid, current_time, /*p2p_inv=*/false); + } + } + + // Potentially flip add_extra_compact_tx to false if AddTx returns false because the tx was already there + add_extra_compact_tx &= m_orphanage.AddTx(ptx, nodeid); + + // Once added to the orphan pool, a tx is considered AlreadyHave, and we shouldn't request it anymore. + m_txrequest.ForgetTxHash(tx.GetHash()); + m_txrequest.ForgetTxHash(tx.GetWitnessHash()); + + // DoS prevention: do not allow m_orphanage to grow unbounded (see CVE-2012-3789) + m_orphanage.LimitOrphans(m_opts.m_max_orphan_txs, m_opts.m_rng); + } else { + unique_parents.clear(); + LogDebug(BCLog::MEMPOOL, "not keeping orphan with rejected parents %s (wtxid=%s)\n", + tx.GetHash().ToString(), + tx.GetWitnessHash().ToString()); + // We will continue to reject this tx since it has rejected + // parents so avoid re-requesting it from other peers. + // Here we add both the txid and the wtxid, as we know that + // regardless of what witness is provided, we will not accept + // this, so we don't need to allow for redownload of this txid + // from any of our non-wtxidrelay peers. + RecentRejectsFilter().insert(tx.GetHash().ToUint256()); + RecentRejectsFilter().insert(tx.GetWitnessHash().ToUint256()); + m_txrequest.ForgetTxHash(tx.GetHash()); + m_txrequest.ForgetTxHash(tx.GetWitnessHash()); + } + } + } else if (state.GetResult() == TxValidationResult::TX_WITNESS_STRIPPED) { + add_extra_compact_tx = false; + } else { + // We can add the wtxid of this transaction to our reject filter. + // Do not add txids of witness transactions or witness-stripped + // transactions to the filter, as they can have been malleated; + // adding such txids to the reject filter would potentially + // interfere with relay of valid transactions from peers that + // do not support wtxid-based relay. See + // https://github.com/bitcoin/bitcoin/issues/8279 for details. + // We can remove this restriction (and always add wtxids to + // the filter even for witness stripped transactions) once + // wtxid-based relay is broadly deployed. + // See also comments in https://github.com/bitcoin/bitcoin/pull/18044#discussion_r443419034 + // for concerns around weakening security of unupgraded nodes + // if we start doing this too early. + if (state.GetResult() == TxValidationResult::TX_RECONSIDERABLE) { + // If the result is TX_RECONSIDERABLE, add it to m_lazy_recent_rejects_reconsiderable + // because we should not download or submit this transaction by itself again, but may + // submit it as part of a package later. + RecentRejectsReconsiderableFilter().insert(ptx->GetWitnessHash().ToUint256()); + + if (first_time_failure) { + // When a transaction fails for TX_RECONSIDERABLE, look for a matching child in the + // orphanage, as it is possible that they succeed as a package. + LogDebug(BCLog::TXPACKAGES, "tx %s (wtxid=%s) failed but reconsiderable, looking for child in orphanage\n", + ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString()); + package_to_validate = Find1P1CPackage(ptx, nodeid); + } + } else { + RecentRejectsFilter().insert(ptx->GetWitnessHash().ToUint256()); + } + m_txrequest.ForgetTxHash(ptx->GetWitnessHash()); + // If the transaction failed for TX_INPUTS_NOT_STANDARD, + // then we know that the witness was irrelevant to the policy + // failure, since this check depends only on the txid + // (the scriptPubKey being spent is covered by the txid). + // Add the txid to the reject filter to prevent repeated + // processing of this transaction in the event that child + // transactions are later received (resulting in + // parent-fetching by txid via the orphan-handling logic). + // We only add the txid if it differs from the wtxid, to avoid wasting entries in the + // rolling bloom filter. + if (state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && ptx->HasWitness()) { + RecentRejectsFilter().insert(ptx->GetHash().ToUint256()); + m_txrequest.ForgetTxHash(ptx->GetHash()); + } + } + + // If the tx failed in ProcessOrphanTx, it should be removed from the orphanage unless the + // tx was still missing inputs. If the tx was not in the orphanage, EraseTx does nothing and returns 0. + if (state.GetResult() != TxValidationResult::TX_MISSING_INPUTS && m_orphanage.EraseTx(ptx->GetWitnessHash()) > 0) { + LogDebug(BCLog::TXPACKAGES, " removed orphan tx %s (wtxid=%s)\n", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString()); + } + + return RejectedTxTodo{ + .m_should_add_extra_compact_tx = add_extra_compact_tx, + .m_unique_parents = std::move(unique_parents), + .m_package_to_validate = std::move(package_to_validate) + }; +} + +void TxDownloadManagerImpl::MempoolRejectedPackage(const Package& package) +{ + RecentRejectsReconsiderableFilter().insert(GetPackageHash(package)); +} + +std::pair<bool, std::optional<PackageToValidate>> TxDownloadManagerImpl::ReceivedTx(NodeId nodeid, const CTransactionRef& ptx) +{ + const uint256& txid = ptx->GetHash(); + const uint256& wtxid = ptx->GetWitnessHash(); + + // Mark that we have received a response + m_txrequest.ReceivedResponse(nodeid, txid); + if (ptx->HasWitness()) m_txrequest.ReceivedResponse(nodeid, wtxid); + + // First check if we should drop this tx. + // We do the AlreadyHaveTx() check using wtxid, rather than txid - in the + // absence of witness malleation, this is strictly better, because the + // recent rejects filter may contain the wtxid but rarely contains + // the txid of a segwit transaction that has been rejected. + // In the presence of witness malleation, it's possible that by only + // doing the check with wtxid, we could overlook a transaction which + // was confirmed with a different witness, or exists in our mempool + // with a different witness, but this has limited downside: + // mempool validation does its own lookup of whether we have the txid + // already; and an adversary can already relay us old transactions + // (older than our recency filter) if trying to DoS us, without any need + // for witness malleation. + if (AlreadyHaveTx(GenTxid::Wtxid(wtxid), /*include_reconsiderable=*/false)) { + // If a tx is detected by m_lazy_recent_rejects it is ignored. Because we haven't + // submitted the tx to our mempool, we won't have computed a DoS + // score for it or determined exactly why we consider it invalid. + // + // This means we won't penalize any peer subsequently relaying a DoSy + // tx (even if we penalized the first peer who gave it to us) because + // we have to account for m_lazy_recent_rejects showing false positives. In + // other words, we shouldn't penalize a peer if we aren't *sure* they + // submitted a DoSy tx. + // + // Note that m_lazy_recent_rejects doesn't just record DoSy or invalid + // transactions, but any tx not accepted by the mempool, which may be + // due to node policy (vs. consensus). So we can't blanket penalize a + // peer simply for relaying a tx that our m_lazy_recent_rejects has caught, + // regardless of false positives. + return {false, std::nullopt}; + } else if (RecentRejectsReconsiderableFilter().contains(wtxid)) { + // When a transaction is already in m_lazy_recent_rejects_reconsiderable, we shouldn't submit + // it by itself again. However, look for a matching child in the orphanage, as it is + // possible that they succeed as a package. + LogDebug(BCLog::TXPACKAGES, "found tx %s (wtxid=%s) in reconsiderable rejects, looking for child in orphanage\n", + txid.ToString(), wtxid.ToString()); + return {false, Find1P1CPackage(ptx, nodeid)}; + } + + + return {true, std::nullopt}; +} + +bool TxDownloadManagerImpl::HaveMoreWork(NodeId nodeid) +{ + return m_orphanage.HaveTxToReconsider(nodeid); +} + +CTransactionRef TxDownloadManagerImpl::GetTxToReconsider(NodeId nodeid) +{ + return m_orphanage.GetTxToReconsider(nodeid); +} + +void TxDownloadManagerImpl::CheckIsEmpty(NodeId nodeid) +{ + assert(m_txrequest.Count(nodeid) == 0); +} +void TxDownloadManagerImpl::CheckIsEmpty() +{ + assert(m_orphanage.Size() == 0); + assert(m_txrequest.Size() == 0); + assert(m_num_wtxid_peers == 0); +} +std::vector<TxOrphanage::OrphanTxBase> TxDownloadManagerImpl::GetOrphanTransactions() const +{ + return m_orphanage.GetOrphanTransactions(); +} +} // namespace node diff --git a/src/node/txdownloadman_impl.h b/src/node/txdownloadman_impl.h new file mode 100644 index 0000000000..8039ddb3cb --- /dev/null +++ b/src/node/txdownloadman_impl.h @@ -0,0 +1,194 @@ +// Copyright (c) 2024 +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#ifndef BITCOIN_NODE_TXDOWNLOADMAN_IMPL_H +#define BITCOIN_NODE_TXDOWNLOADMAN_IMPL_H + +#include <node/txdownloadman.h> + +#include <common/bloom.h> +#include <consensus/validation.h> +#include <kernel/chain.h> +#include <net.h> +#include <primitives/transaction.h> +#include <policy/packages.h> +#include <txorphanage.h> +#include <txrequest.h> + +class CTxMemPool; +namespace node { +class TxDownloadManagerImpl { +public: + TxDownloadOptions m_opts; + + /** Manages unvalidated tx data (orphan transactions for which we are downloading ancestors). */ + TxOrphanage m_orphanage; + /** Tracks candidates for requesting and downloading transaction data. */ + TxRequestTracker m_txrequest; + + /** + * Filter for transactions that were recently rejected by the mempool. + * These are not rerequested until the chain tip changes, at which point + * the entire filter is reset. + * + * Without this filter we'd be re-requesting txs from each of our peers, + * increasing bandwidth consumption considerably. For instance, with 100 + * peers, half of which relay a tx we don't accept, that might be a 50x + * bandwidth increase. A flooding attacker attempting to roll-over the + * filter using minimum-sized, 60byte, transactions might manage to send + * 1000/sec if we have fast peers, so we pick 120,000 to give our peers a + * two minute window to send invs to us. + * + * Decreasing the false positive rate is fairly cheap, so we pick one in a + * million to make it highly unlikely for users to have issues with this + * filter. + * + * We typically only add wtxids to this filter. For non-segwit + * transactions, the txid == wtxid, so this only prevents us from + * re-downloading non-segwit transactions when communicating with + * non-wtxidrelay peers -- which is important for avoiding malleation + * attacks that could otherwise interfere with transaction relay from + * non-wtxidrelay peers. For communicating with wtxidrelay peers, having + * the reject filter store wtxids is exactly what we want to avoid + * redownload of a rejected transaction. + * + * In cases where we can tell that a segwit transaction will fail + * validation no matter the witness, we may add the txid of such + * transaction to the filter as well. This can be helpful when + * communicating with txid-relay peers or if we were to otherwise fetch a + * transaction via txid (eg in our orphan handling). + * + * Memory used: 1.3 MB + */ + std::unique_ptr<CRollingBloomFilter> m_lazy_recent_rejects{nullptr}; + + CRollingBloomFilter& RecentRejectsFilter() + { + if (!m_lazy_recent_rejects) { + m_lazy_recent_rejects = std::make_unique<CRollingBloomFilter>(120'000, 0.000'001); + } + + return *m_lazy_recent_rejects; + } + + /** + * Filter for: + * (1) wtxids of transactions that were recently rejected by the mempool but are + * eligible for reconsideration if submitted with other transactions. + * (2) packages (see GetPackageHash) we have already rejected before and should not retry. + * + * Similar to m_lazy_recent_rejects, this filter is used to save bandwidth when e.g. all of our peers + * have larger mempools and thus lower minimum feerates than us. + * + * When a transaction's error is TxValidationResult::TX_RECONSIDERABLE (in a package or by + * itself), add its wtxid to this filter. When a package fails for any reason, add the combined + * hash to this filter. + * + * Upon receiving an announcement for a transaction, if it exists in this filter, do not + * download the txdata. When considering packages, if it exists in this filter, drop it. + * + * Reset this filter when the chain tip changes. + * + * Parameters are picked to be the same as m_lazy_recent_rejects, with the same rationale. + */ + std::unique_ptr<CRollingBloomFilter> m_lazy_recent_rejects_reconsiderable{nullptr}; + + CRollingBloomFilter& RecentRejectsReconsiderableFilter() + { + if (!m_lazy_recent_rejects_reconsiderable) { + m_lazy_recent_rejects_reconsiderable = std::make_unique<CRollingBloomFilter>(120'000, 0.000'001); + } + + return *m_lazy_recent_rejects_reconsiderable; + } + + /* + * Filter for transactions that have been recently confirmed. + * We use this to avoid requesting transactions that have already been + * confirmed. + * + * Blocks don't typically have more than 4000 transactions, so this should + * be at least six blocks (~1 hr) worth of transactions that we can store, + * inserting both a txid and wtxid for every observed transaction. + * If the number of transactions appearing in a block goes up, or if we are + * seeing getdata requests more than an hour after initial announcement, we + * can increase this number. + * The false positive rate of 1/1M should come out to less than 1 + * transaction per day that would be inadvertently ignored (which is the + * same probability that we have in the reject filter). + */ + std::unique_ptr<CRollingBloomFilter> m_lazy_recent_confirmed_transactions{nullptr}; + + CRollingBloomFilter& RecentConfirmedTransactionsFilter() + { + if (!m_lazy_recent_confirmed_transactions) { + m_lazy_recent_confirmed_transactions = std::make_unique<CRollingBloomFilter>(48'000, 0.000'001); + } + + return *m_lazy_recent_confirmed_transactions; + } + + TxDownloadManagerImpl(const TxDownloadOptions& options) : m_opts{options}, m_txrequest{options.m_deterministic_txrequest} {} + + struct PeerInfo { + /** Information relevant to scheduling tx requests. */ + const TxDownloadConnectionInfo m_connection_info; + + PeerInfo(const TxDownloadConnectionInfo& info) : m_connection_info{info} {} + }; + + /** Information for all of the peers we may download transactions from. This is not necessarily + * all peers we are connected to (no block-relay-only and temporary connections). */ + std::map<NodeId, PeerInfo> m_peer_info; + + /** Number of wtxid relay peers we have in m_peer_info. */ + uint32_t m_num_wtxid_peers{0}; + + void ActiveTipChange(); + void BlockConnected(const std::shared_ptr<const CBlock>& pblock); + void BlockDisconnected(); + + /** Check whether we already have this gtxid in: + * - mempool + * - orphanage + * - m_recent_rejects + * - m_recent_rejects_reconsiderable (if include_reconsiderable = true) + * - m_recent_confirmed_transactions + * */ + bool AlreadyHaveTx(const GenTxid& gtxid, bool include_reconsiderable); + + void ConnectedPeer(NodeId nodeid, const TxDownloadConnectionInfo& info); + void DisconnectedPeer(NodeId nodeid); + + /** Consider adding this tx hash to txrequest. Should be called whenever a new inv has been received. + * Also called internally when a transaction is missing parents so that we can request them. + */ + bool AddTxAnnouncement(NodeId peer, const GenTxid& gtxid, std::chrono::microseconds now, bool p2p_inv); + + /** Get getdata requests to send. */ + std::vector<GenTxid> GetRequestsToSend(NodeId nodeid, std::chrono::microseconds current_time); + + /** Marks a tx as ReceivedResponse in txrequest. */ + void ReceivedNotFound(NodeId nodeid, const std::vector<uint256>& txhashes); + + /** Look for a child of this transaction in the orphanage to form a 1-parent-1-child package, + * skipping any combinations that have already been tried. Return the resulting package along with + * the senders of its respective transactions, or std::nullopt if no package is found. */ + std::optional<PackageToValidate> Find1P1CPackage(const CTransactionRef& ptx, NodeId nodeid); + + void MempoolAcceptedTx(const CTransactionRef& tx); + RejectedTxTodo MempoolRejectedTx(const CTransactionRef& ptx, const TxValidationState& state, NodeId nodeid, bool first_time_failure); + void MempoolRejectedPackage(const Package& package); + + std::pair<bool, std::optional<PackageToValidate>> ReceivedTx(NodeId nodeid, const CTransactionRef& ptx); + + bool HaveMoreWork(NodeId nodeid); + CTransactionRef GetTxToReconsider(NodeId nodeid); + + void CheckIsEmpty(); + void CheckIsEmpty(NodeId nodeid); + + std::vector<TxOrphanage::OrphanTxBase> GetOrphanTransactions() const; +}; +} // namespace node +#endif // BITCOIN_NODE_TXDOWNLOADMAN_IMPL_H diff --git a/src/node/types.h b/src/node/types.h index 1302f1b127..4b0de084ab 100644 --- a/src/node/types.h +++ b/src/node/types.h @@ -3,8 +3,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. //! @file node/types.h is a home for public enum and struct type definitions -//! that are used by internally by node code, but also used externally by wallet -//! or GUI code. +//! that are used internally by node code, but also used externally by wallet, +//! mining or GUI code. //! //! This file is intended to define only simple types that do not have external //! dependencies. More complicated types should be defined in dedicated header @@ -14,6 +14,7 @@ #define BITCOIN_NODE_TYPES_H #include <cstddef> +#include <script/script.h> namespace node { enum class TransactionError { @@ -43,6 +44,22 @@ struct BlockCreateOptions { * transaction outputs. */ size_t coinbase_output_max_additional_sigops{400}; + /** + * Script to put in the coinbase transaction. The default is an + * anyone-can-spend dummy. + * + * Should only be used for tests, when the default doesn't suffice. + * + * Note that higher level code like the getblocktemplate RPC may omit the + * coinbase transaction entirely. It's instead constructed by pool software + * using fields like coinbasevalue, coinbaseaux and default_witness_commitment. + * This software typically also controls the payout outputs, even for solo + * mining. + * + * The size and sigops are not checked against + * coinbase_max_additional_weight and coinbase_output_max_additional_sigops. + */ + CScript coinbase_output_script{CScript() << OP_TRUE}; }; } // namespace node diff --git a/src/node/utxo_snapshot.cpp b/src/node/utxo_snapshot.cpp index 976421e455..ca5491bdc2 100644 --- a/src/node/utxo_snapshot.cpp +++ b/src/node/utxo_snapshot.cpp @@ -73,10 +73,10 @@ std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir) } afile >> base_blockhash; - if (std::fgetc(afile.Get()) != EOF) { + int64_t position = afile.tell(); + afile.seek(0, SEEK_END); + if (position != afile.tell()) { LogPrintf("[snapshot] warning: unexpected trailing data in %s\n", read_from_str); - } else if (std::ferror(afile.Get())) { - LogPrintf("[snapshot] warning: i/o error reading %s\n", read_from_str); } return base_blockhash; } diff --git a/src/node/warnings.cpp b/src/node/warnings.cpp index 87389e472b..255d8dba6e 100644 --- a/src/node/warnings.cpp +++ b/src/node/warnings.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <node/warnings.h> diff --git a/src/policy/ephemeral_policy.cpp b/src/policy/ephemeral_policy.cpp new file mode 100644 index 0000000000..b7d254dd63 --- /dev/null +++ b/src/policy/ephemeral_policy.cpp @@ -0,0 +1,94 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/validation.h> +#include <policy/ephemeral_policy.h> +#include <policy/feerate.h> +#include <policy/packages.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <txmempool.h> +#include <util/check.h> +#include <util/hasher.h> + +#include <algorithm> +#include <cstdint> +#include <map> +#include <memory> +#include <unordered_set> +#include <utility> +#include <vector> + +bool PreCheckEphemeralTx(const CTransaction& tx, CFeeRate dust_relay_rate, CAmount base_fee, CAmount mod_fee, TxValidationState& state) +{ + // We never want to give incentives to mine this transaction alone + if ((base_fee != 0 || mod_fee != 0) && !GetDust(tx, dust_relay_rate).empty()) { + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "dust", "tx with dust output must be 0-fee"); + } + + return true; +} + +bool CheckEphemeralSpends(const Package& package, CFeeRate dust_relay_rate, const CTxMemPool& tx_pool, TxValidationState& out_child_state, Txid& out_child_txid) +{ + if (!Assume(std::ranges::all_of(package, [](const auto& tx){return tx != nullptr;}))) { + // Bail out of spend checks if caller gave us an invalid package + return true; + } + + std::map<Txid, CTransactionRef> map_txid_ref; + for (const auto& tx : package) { + map_txid_ref[tx->GetHash()] = tx; + } + + for (const auto& tx : package) { + std::unordered_set<Txid, SaltedTxidHasher> processed_parent_set; + std::unordered_set<COutPoint, SaltedOutpointHasher> unspent_parent_dust; + + for (const auto& tx_input : tx->vin) { + const Txid& parent_txid{tx_input.prevout.hash}; + // Skip parents we've already checked dust for + if (processed_parent_set.contains(parent_txid)) continue; + + // We look for an in-package or in-mempool dependency + CTransactionRef parent_ref = nullptr; + if (auto it = map_txid_ref.find(parent_txid); it != map_txid_ref.end()) { + parent_ref = it->second; + } else { + parent_ref = tx_pool.get(parent_txid); + } + + // Check for dust on parents + if (parent_ref) { + for (uint32_t out_index = 0; out_index < parent_ref->vout.size(); out_index++) { + const auto& tx_output = parent_ref->vout[out_index]; + if (IsDust(tx_output, dust_relay_rate)) { + unspent_parent_dust.insert(COutPoint(parent_txid, out_index)); + } + } + } + + processed_parent_set.insert(parent_txid); + } + + if (unspent_parent_dust.empty()) { + continue; + } + + // Now that we have gathered parents' dust, make sure it's spent + // by the child + for (const auto& tx_input : tx->vin) { + unspent_parent_dust.erase(tx_input.prevout); + } + + if (!unspent_parent_dust.empty()) { + out_child_txid = tx->GetHash(); + out_child_state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "missing-ephemeral-spends", + strprintf("tx %s did not spend parent's ephemeral dust", out_child_txid.ToString())); + return false; + } + } + + return true; +} diff --git a/src/policy/ephemeral_policy.h b/src/policy/ephemeral_policy.h new file mode 100644 index 0000000000..4667f25b5a --- /dev/null +++ b/src/policy/ephemeral_policy.h @@ -0,0 +1,58 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_POLICY_EPHEMERAL_POLICY_H +#define BITCOIN_POLICY_EPHEMERAL_POLICY_H + +#include <consensus/amount.h> +#include <policy/packages.h> +#include <primitives/transaction.h> + +#include <optional> + +class CFeeRate; +class CTxMemPool; +class TxValidationState; + +/** These utility functions ensure that ephemeral dust is safely + * created and spent without unduly risking them entering the utxo + * set. + + * This is ensured by requiring: + * - PreCheckEphemeralTx checks are respected + * - The parent has no child (and 0-fee as implied above to disincentivize mining) + * - OR the parent transaction has exactly one child, and the dust is spent by that child + * + * Imagine three transactions: + * TxA, 0-fee with two outputs, one non-dust, one dust + * TxB, spends TxA's non-dust + * TxC, spends TxA's dust + * + * All the dust is spent if TxA+TxB+TxC is accepted, but the mining template may just pick + * up TxA+TxB rather than the three "legal configurations": + * 1) None + * 2) TxA+TxB+TxC + * 3) TxA+TxC + * By requiring the child transaction to sweep any dust from the parent txn, we ensure that + * there is a single child only, and this child, or the child's descendants, + * are the only way to bring fees. + */ + +/* All the following checks are only called if standardness rules are being applied. */ + +/** Must be called for each transaction once transaction fees are known. + * Does context-less checks about a single transaction. + * @returns false if the fee is non-zero and dust exists, populating state. True otherwise. + */ +bool PreCheckEphemeralTx(const CTransaction& tx, CFeeRate dust_relay_rate, CAmount base_fee, CAmount mod_fee, TxValidationState& state); + +/** Must be called for each transaction(package) if any dust is in the package. + * Checks that each transaction's parents have their dust spent by the child, + * where parents are either in the mempool or in the package itself. + * Sets out_child_state and out_child_txid on failure. + * @returns true if all dust is properly spent. + */ +bool CheckEphemeralSpends(const Package& package, CFeeRate dust_relay_rate, const CTxMemPool& tx_pool, TxValidationState& out_child_state, Txid& out_child_txid); + +#endif // BITCOIN_POLICY_EPHEMERAL_POLICY_H diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index a17faa3b99..e85b2f2caa 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -5,7 +5,6 @@ #include <policy/fees.h> -#include <clientversion.h> #include <common/system.h> #include <consensus/amount.h> #include <kernel/mempool_entry.h> @@ -32,6 +31,10 @@ #include <stdexcept> #include <utility> +// The current format written, and the version required to read. Must be +// increased to at least 289900+1 on the next breaking change. +constexpr int CURRENT_FEES_FILE_VERSION{149900}; + static constexpr double INF_FEERATE = 1e99; std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon) @@ -168,7 +171,7 @@ public: * Read saved state of estimation data from a file and replace all internal data structures and * variables with this state. */ - void Read(AutoFile& filein, int nFileVersion, size_t numBuckets); + void Read(AutoFile& filein, size_t numBuckets); }; @@ -414,7 +417,7 @@ void TxConfirmStats::Write(AutoFile& fileout) const fileout << Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(failAvg); } -void TxConfirmStats::Read(AutoFile& filein, int nFileVersion, size_t numBuckets) +void TxConfirmStats::Read(AutoFile& filein, size_t numBuckets) { // Read data file and do some very basic sanity checking // buckets and bucketMap are not updated yet, so don't access them @@ -961,8 +964,8 @@ bool CBlockPolicyEstimator::Write(AutoFile& fileout) const { try { LOCK(m_cs_fee_estimator); - fileout << 149900; // version required to read: 0.14.99 or later - fileout << CLIENT_VERSION; // version that wrote the file + fileout << CURRENT_FEES_FILE_VERSION; + fileout << int{0}; // Unused dummy field. Written files may contain any value in [0, 289900] fileout << nBestSeenHeight; if (BlockSpan() > HistoricalBlockSpan()/2) { fileout << firstRecordedHeight << nBestSeenHeight; @@ -976,7 +979,7 @@ bool CBlockPolicyEstimator::Write(AutoFile& fileout) const longStats->Write(fileout); } catch (const std::exception&) { - LogPrintf("CBlockPolicyEstimator::Write(): unable to write policy estimator data (non-fatal)\n"); + LogWarning("Unable to write policy estimator data (non-fatal)"); return false; } return true; @@ -986,10 +989,10 @@ bool CBlockPolicyEstimator::Read(AutoFile& filein) { try { LOCK(m_cs_fee_estimator); - int nVersionRequired, nVersionThatWrote; - filein >> nVersionRequired >> nVersionThatWrote; - if (nVersionRequired > CLIENT_VERSION) { - throw std::runtime_error(strprintf("up-version (%d) fee estimate file", nVersionRequired)); + int nVersionRequired, dummy; + filein >> nVersionRequired >> dummy; + if (nVersionRequired > CURRENT_FEES_FILE_VERSION) { + throw std::runtime_error{strprintf("File version (%d) too high to be read.", nVersionRequired)}; } // Read fee estimates file into temporary variables so existing data @@ -997,9 +1000,9 @@ bool CBlockPolicyEstimator::Read(AutoFile& filein) unsigned int nFileBestSeenHeight; filein >> nFileBestSeenHeight; - if (nVersionRequired < 149900) { - LogPrintf("%s: incompatible old fee estimation data (non-fatal). Version: %d\n", __func__, nVersionRequired); - } else { // New format introduced in 149900 + if (nVersionRequired < CURRENT_FEES_FILE_VERSION) { + LogWarning("Incompatible old fee estimation data (non-fatal). Version: %d", nVersionRequired); + } else { // nVersionRequired == CURRENT_FEES_FILE_VERSION unsigned int nFileHistoricalFirst, nFileHistoricalBest; filein >> nFileHistoricalFirst >> nFileHistoricalBest; if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) { @@ -1015,9 +1018,9 @@ bool CBlockPolicyEstimator::Read(AutoFile& filein) std::unique_ptr<TxConfirmStats> fileFeeStats(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE)); std::unique_ptr<TxConfirmStats> fileShortStats(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE)); std::unique_ptr<TxConfirmStats> fileLongStats(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); - fileFeeStats->Read(filein, nVersionThatWrote, numBuckets); - fileShortStats->Read(filein, nVersionThatWrote, numBuckets); - fileLongStats->Read(filein, nVersionThatWrote, numBuckets); + fileFeeStats->Read(filein, numBuckets); + fileShortStats->Read(filein, numBuckets); + fileLongStats->Read(filein, numBuckets); // Fee estimates file parsed correctly // Copy buckets from file and refresh our bucketmap @@ -1038,7 +1041,7 @@ bool CBlockPolicyEstimator::Read(AutoFile& filein) } } catch (const std::exception& e) { - LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal): %s\n",e.what()); + LogWarning("Unable to read policy estimator data (non-fatal): %s", e.what()); return false; } return true; diff --git a/src/policy/packages.h b/src/policy/packages.h index 3050320122..4b2350edac 100644 --- a/src/policy/packages.h +++ b/src/policy/packages.h @@ -89,8 +89,9 @@ bool IsChildWithParents(const Package& package); */ bool IsChildWithParentsTree(const Package& package); -/** Get the hash of these transactions' wtxids, concatenated in lexicographical order (treating the - * wtxids as little endian encoded uint256, smallest to largest). */ +/** Get the hash of the concatenated wtxids of transactions, with wtxids + * treated as a little-endian numbers and sorted in ascending numeric order. + */ uint256 GetPackageHash(const std::vector<CTransactionRef>& transactions); #endif // BITCOIN_POLICY_PACKAGES_H diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 68d879b5b8..ed33692823 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -67,6 +67,15 @@ bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) return (txout.nValue < GetDustThreshold(txout, dustRelayFeeIn)); } +std::vector<uint32_t> GetDust(const CTransaction& tx, CFeeRate dust_relay_rate) +{ + std::vector<uint32_t> dust_outputs; + for (uint32_t i{0}; i < tx.vout.size(); ++i) { + if (IsDust(tx.vout[i], dust_relay_rate)) dust_outputs.push_back(i); + } + return dust_outputs; +} + bool IsStandard(const CScript& scriptPubKey, const std::optional<unsigned>& max_datacarrier_bytes, TxoutType& whichType) { std::vector<std::vector<unsigned char> > vSolutions; @@ -141,12 +150,15 @@ bool IsStandardTx(const CTransaction& tx, const std::optional<unsigned>& max_dat else if ((whichType == TxoutType::MULTISIG) && (!permit_bare_multisig)) { reason = "bare-multisig"; return false; - } else if (IsDust(txout, dust_relay_fee)) { - reason = "dust"; - return false; } } + // Only MAX_DUST_OUTPUTS_PER_TX dust is permitted(on otherwise valid ephemeral dust) + if (GetDust(tx, dust_relay_fee).size() > MAX_DUST_OUTPUTS_PER_TX) { + reason = "dust"; + return false; + } + // only one OP_RETURN txout is permitted if (nDataOut > 1) { reason = "multi-op-return"; diff --git a/src/policy/policy.h b/src/policy/policy.h index a82488a28c..4412f2db87 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -77,6 +77,10 @@ static const unsigned int MAX_OP_RETURN_RELAY = 83; */ static constexpr unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT{10000}; +/** + * Maximum number of ephemeral dust outputs allowed. + */ +static constexpr unsigned int MAX_DUST_OUTPUTS_PER_TX{1}; /** * Mandatory script verification flags that all new transactions must comply with for @@ -127,6 +131,8 @@ bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFee); bool IsStandard(const CScript& scriptPubKey, const std::optional<unsigned>& max_datacarrier_bytes, TxoutType& whichType); +/** Get the vout index numbers of all dust outputs */ +std::vector<uint32_t> GetDust(const CTransaction& tx, CFeeRate dust_relay_rate); // Changing the default transaction version requires a two step process: first // adapting relay policy by bumping TX_MAX_STANDARD_VERSION, and then later diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 2ad79b6f99..bc76361fba 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -71,7 +71,7 @@ std::optional<std::string> GetEntriesForConflicts(const CTransaction& tx, // descendants (i.e. if multiple conflicts share a descendant, it will be counted multiple // times), but we just want to be conservative to avoid doing too much work. if (nConflictingCount > MAX_REPLACEMENT_CANDIDATES) { - return strprintf("rejecting replacement %s; too many potential replacements (%d > %d)\n", + return strprintf("rejecting replacement %s; too many potential replacements (%d > %d)", txid.ToString(), nConflictingCount, MAX_REPLACEMENT_CANDIDATES); @@ -184,14 +184,10 @@ std::optional<std::string> PaysForRBF(CAmount original_fees, return std::nullopt; } -std::optional<std::pair<DiagramCheckError, std::string>> ImprovesFeerateDiagram(CTxMemPool& pool, - const CTxMemPool::setEntries& direct_conflicts, - const CTxMemPool::setEntries& all_conflicts, - CAmount replacement_fees, - int64_t replacement_vsize) +std::optional<std::pair<DiagramCheckError, std::string>> ImprovesFeerateDiagram(CTxMemPool::ChangeSet& changeset) { // Require that the replacement strictly improves the mempool's feerate diagram. - const auto chunk_results{pool.CalculateChunksForRBF(replacement_fees, replacement_vsize, direct_conflicts, all_conflicts)}; + const auto chunk_results{changeset.CalculateChunksForRBF()}; if (!chunk_results.has_value()) { return std::make_pair(DiagramCheckError::UNCALCULABLE, util::ErrorString(chunk_results).original); diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 252fbec8e3..3cc0fc3149 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -117,19 +117,9 @@ std::optional<std::string> PaysForRBF(CAmount original_fees, /** * The replacement transaction must improve the feerate diagram of the mempool. - * @param[in] pool The mempool. - * @param[in] direct_conflicts Set of in-mempool txids corresponding to the direct conflicts i.e. - * input double-spends with the proposed transaction - * @param[in] all_conflicts Set of mempool entries corresponding to all transactions to be evicted - * @param[in] replacement_fees Fees of proposed replacement package - * @param[in] replacement_vsize Size of proposed replacement package + * @param[in] changeset The changeset containing proposed additions/removals * @returns error type and string if mempool diagram doesn't improve, otherwise std::nullopt. */ -std::optional<std::pair<DiagramCheckError, std::string>> ImprovesFeerateDiagram(CTxMemPool& pool, - const CTxMemPool::setEntries& direct_conflicts, - const CTxMemPool::setEntries& all_conflicts, - CAmount replacement_fees, - int64_t replacement_vsize) - EXCLUSIVE_LOCKS_REQUIRED(pool.cs); +std::optional<std::pair<DiagramCheckError, std::string>> ImprovesFeerateDiagram(CTxMemPool::ChangeSet& changeset); #endif // BITCOIN_POLICY_RBF_H diff --git a/src/pow.cpp b/src/pow.cpp index 50de8946be..bbcf39b593 100644 --- a/src/pow.cpp +++ b/src/pow.cpp @@ -9,6 +9,7 @@ #include <chain.h> #include <primitives/block.h> #include <uint256.h> +#include <util/check.h> unsigned int GetNextWorkRequired(const CBlockIndex* pindexLast, const CBlockHeader *pblock, const Consensus::Params& params) { @@ -134,8 +135,16 @@ bool PermittedDifficultyTransition(const Consensus::Params& params, int64_t heig return true; } +// Bypasses the actual proof of work check during fuzz testing with a simplified validation checking whether +// the most significant bit of the last byte of the hash is set. bool CheckProofOfWork(uint256 hash, unsigned int nBits, const Consensus::Params& params) { + if constexpr (G_FUZZING) return (hash.data()[31] & 0x80) == 0; + return CheckProofOfWorkImpl(hash, nBits, params); +} + +bool CheckProofOfWorkImpl(uint256 hash, unsigned int nBits, const Consensus::Params& params) +{ bool fNegative; bool fOverflow; arith_uint256 bnTarget; @@ -19,6 +19,7 @@ unsigned int CalculateNextWorkRequired(const CBlockIndex* pindexLast, int64_t nF /** Check whether a block hash satisfies the proof-of-work requirement specified by nBits */ bool CheckProofOfWork(uint256 hash, unsigned int nBits, const Consensus::Params&); +bool CheckProofOfWorkImpl(uint256 hash, unsigned int nBits, const Consensus::Params&); /** * Return false if the proof-of-work requirement specified by new_nbits at a diff --git a/src/prevector.h b/src/prevector.h index 0c47137910..d14e5f64e9 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -363,7 +363,8 @@ public: change_capacity(new_size + (new_size >> 1)); } T* ptr = item_ptr(p); - memmove(ptr + 1, ptr, (size() - p) * sizeof(T)); + T* dst = ptr + 1; + memmove(dst, ptr, (size() - p) * sizeof(T)); _size++; new(static_cast<void*>(ptr)) T(value); return iterator(ptr); @@ -376,7 +377,8 @@ public: change_capacity(new_size + (new_size >> 1)); } T* ptr = item_ptr(p); - memmove(ptr + count, ptr, (size() - p) * sizeof(T)); + T* dst = ptr + count; + memmove(dst, ptr, (size() - p) * sizeof(T)); _size += count; fill(item_ptr(p), count, value); } @@ -390,7 +392,8 @@ public: change_capacity(new_size + (new_size >> 1)); } T* ptr = item_ptr(p); - memmove(ptr + count, ptr, (size() - p) * sizeof(T)); + T* dst = ptr + count; + memmove(dst, ptr, (size() - p) * sizeof(T)); _size += count; fill(ptr, first, last); } diff --git a/src/protocol.cpp b/src/protocol.cpp index 0439d398c7..bdf0a66986 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -7,29 +7,29 @@ #include <common/system.h> -CMessageHeader::CMessageHeader(const MessageStartChars& pchMessageStartIn, const char* pszCommand, unsigned int nMessageSizeIn) +CMessageHeader::CMessageHeader(const MessageStartChars& pchMessageStartIn, const char* msg_type, unsigned int nMessageSizeIn) : pchMessageStart{pchMessageStartIn} { - // Copy the command name + // Copy the message type name size_t i = 0; - for (; i < COMMAND_SIZE && pszCommand[i] != 0; ++i) pchCommand[i] = pszCommand[i]; - assert(pszCommand[i] == 0); // Assert that the command name passed in is not longer than COMMAND_SIZE + for (; i < MESSAGE_TYPE_SIZE && msg_type[i] != 0; ++i) m_msg_type[i] = msg_type[i]; + assert(msg_type[i] == 0); // Assert that the message type name passed in is not longer than MESSAGE_TYPE_SIZE nMessageSize = nMessageSizeIn; } -std::string CMessageHeader::GetCommand() const +std::string CMessageHeader::GetMessageType() const { - return std::string(pchCommand, pchCommand + strnlen(pchCommand, COMMAND_SIZE)); + return std::string(m_msg_type, m_msg_type + strnlen(m_msg_type, MESSAGE_TYPE_SIZE)); } -bool CMessageHeader::IsCommandValid() const +bool CMessageHeader::IsMessageTypeValid() const { - // Check the command string for errors - for (const char* p1 = pchCommand; p1 < pchCommand + COMMAND_SIZE; ++p1) { + // Check the message type string for errors + for (const char* p1 = m_msg_type; p1 < m_msg_type + MESSAGE_TYPE_SIZE; ++p1) { if (*p1 == 0) { // Must be all zeros after the first zero - for (; p1 < pchCommand + COMMAND_SIZE; ++p1) { + for (; p1 < m_msg_type + MESSAGE_TYPE_SIZE; ++p1) { if (*p1 != 0) { return false; } @@ -55,7 +55,7 @@ bool operator<(const CInv& a, const CInv& b) return (a.type < b.type || (a.type == b.type && a.hash < b.hash)); } -std::string CInv::GetCommand() const +std::string CInv::GetMessageType() const { std::string cmd; if (type & MSG_WITNESS_FLAG) @@ -70,14 +70,14 @@ std::string CInv::GetCommand() const case MSG_FILTERED_BLOCK: return cmd.append(NetMsgType::MERKLEBLOCK); case MSG_CMPCT_BLOCK: return cmd.append(NetMsgType::CMPCTBLOCK); default: - throw std::out_of_range(strprintf("CInv::GetCommand(): type=%d unknown type", type)); + throw std::out_of_range(strprintf("CInv::GetMessageType(): type=%d unknown type", type)); } } std::string CInv::ToString() const { try { - return strprintf("%s %s", GetCommand(), hash.ToString()); + return strprintf("%s %s", GetMessageType(), hash.ToString()); } catch(const std::out_of_range &) { return strprintf("0x%08x %s", type, hash.ToString()); } diff --git a/src/protocol.h b/src/protocol.h index fd7cfddf3b..804597b86d 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -21,34 +21,34 @@ /** Message header. * (4) message start. - * (12) command. + * (12) message type. * (4) size. * (4) checksum. */ class CMessageHeader { public: - static constexpr size_t COMMAND_SIZE = 12; + static constexpr size_t MESSAGE_TYPE_SIZE = 12; static constexpr size_t MESSAGE_SIZE_SIZE = 4; static constexpr size_t CHECKSUM_SIZE = 4; - static constexpr size_t MESSAGE_SIZE_OFFSET = std::tuple_size_v<MessageStartChars> + COMMAND_SIZE; + static constexpr size_t MESSAGE_SIZE_OFFSET = std::tuple_size_v<MessageStartChars> + MESSAGE_TYPE_SIZE; static constexpr size_t CHECKSUM_OFFSET = MESSAGE_SIZE_OFFSET + MESSAGE_SIZE_SIZE; - static constexpr size_t HEADER_SIZE = std::tuple_size_v<MessageStartChars> + COMMAND_SIZE + MESSAGE_SIZE_SIZE + CHECKSUM_SIZE; + static constexpr size_t HEADER_SIZE = std::tuple_size_v<MessageStartChars> + MESSAGE_TYPE_SIZE + MESSAGE_SIZE_SIZE + CHECKSUM_SIZE; explicit CMessageHeader() = default; - /** Construct a P2P message header from message-start characters, a command and the size of the message. - * @note Passing in a `pszCommand` longer than COMMAND_SIZE will result in a run-time assertion error. + /** Construct a P2P message header from message-start characters, a message type and the size of the message. + * @note Passing in a `msg_type` longer than MESSAGE_TYPE_SIZE will result in a run-time assertion error. */ - CMessageHeader(const MessageStartChars& pchMessageStartIn, const char* pszCommand, unsigned int nMessageSizeIn); + CMessageHeader(const MessageStartChars& pchMessageStartIn, const char* msg_type, unsigned int nMessageSizeIn); - std::string GetCommand() const; - bool IsCommandValid() const; + std::string GetMessageType() const; + bool IsMessageTypeValid() const; - SERIALIZE_METHODS(CMessageHeader, obj) { READWRITE(obj.pchMessageStart, obj.pchCommand, obj.nMessageSize, obj.pchChecksum); } + SERIALIZE_METHODS(CMessageHeader, obj) { READWRITE(obj.pchMessageStart, obj.m_msg_type, obj.nMessageSize, obj.pchChecksum); } MessageStartChars pchMessageStart{}; - char pchCommand[COMMAND_SIZE]{}; + char m_msg_type[MESSAGE_TYPE_SIZE]{}; uint32_t nMessageSize{std::numeric_limits<uint32_t>::max()}; uint8_t pchChecksum[CHECKSUM_SIZE]{}; }; @@ -500,7 +500,7 @@ public: friend bool operator<(const CInv& a, const CInv& b); - std::string GetCommand() const; + std::string GetMessageType() const; std::string ToString() const; // Single-message helper methods diff --git a/src/pubkey.cpp b/src/pubkey.cpp index 0849d2a266..fb25ebd4ca 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -192,7 +192,10 @@ int ecdsa_signature_parse_der_lax(secp256k1_ecdsa_signature* sig, const unsigned * For an example script for calculating H, refer to the unit tests in * ./test/functional/test_framework/crypto/secp256k1.py */ -constexpr XOnlyPubKey XOnlyPubKey::NUMS_H{"50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0"_hex_u8}; +constexpr XOnlyPubKey XOnlyPubKey::NUMS_H{ + // Use immediate lambda to work around GCC-14 bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=117966 + []() consteval { return XOnlyPubKey{"50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0"_hex_u8}; }(), +}; std::vector<CKeyID> XOnlyPubKey::GetKeyIDs() const { diff --git a/src/qt/CMakeLists.txt b/src/qt/CMakeLists.txt index 297408a926..9f660264f9 100644 --- a/src/qt/CMakeLists.txt +++ b/src/qt/CMakeLists.txt @@ -35,6 +35,7 @@ endfunction() # - https://doc.qt.io/qt-5/cmake-manual.html set(CMAKE_AUTOMOC ON) +set(CMAKE_AUTOMOC_MOC_OPTIONS "-p${CMAKE_CURRENT_SOURCE_DIR}") set(CMAKE_AUTORCC ON) set(CMAKE_AUTOUIC ON) set(CMAKE_AUTOUIC_SEARCH_PATHS forms) @@ -133,9 +134,7 @@ target_link_libraries(bitcoinqt bitcoin_cli leveldb Boost::headers - $<TARGET_NAME_IF_EXISTS:NATPMP::NATPMP> - $<TARGET_NAME_IF_EXISTS:MiniUPnPc::MiniUPnPc> - $<TARGET_NAME_IF_EXISTS:PkgConfig::libqrencode> + $<TARGET_NAME_IF_EXISTS:QRencode::QRencode> $<$<PLATFORM_ID:Darwin>:-framework\ AppKit> $<$<CXX_COMPILER_ID:MSVC>:shlwapi> ) diff --git a/src/qt/README.md b/src/qt/README.md index 1c6f963ccf..3ecdd0888e 100644 --- a/src/qt/README.md +++ b/src/qt/README.md @@ -99,7 +99,7 @@ sudo apt-get install qtcreator #### Setup Qt Creator 1. Make sure you've installed all dependencies specified in your systems build instructions -2. Follow the compile instructions for your system, run `./configure` with the `--enable-debug` flag +2. Follow the compile instructions for your system, adding the `-DCMAKE_BUILD_TYPE=Debug` build flag 3. Start Qt Creator. At the start page, do: `New` -> `Import Project` -> `Import Existing Project` 4. Enter `bitcoin-qt` as the Project Name and enter the absolute path to `src/qt` as Location 5. Check over the file selection, you may need to select the `forms` directory (necessary if you intend to edit *.ui files) diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index ec415f0bac..eaaa9fc6fd 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/bitcoin.h> @@ -162,7 +162,7 @@ static void initTranslations(QTranslator &qtTranslatorBase, QTranslator &qtTrans static bool ErrorSettingsRead(const bilingual_str& error, const std::vector<std::string>& details) { - QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Reset | QMessageBox::Abort); + QMessageBox messagebox(QMessageBox::Critical, CLIENT_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Reset | QMessageBox::Abort); /*: Explanatory text shown on startup when the settings file cannot be read. Prompts user to make a choice between resetting or aborting. */ messagebox.setInformativeText(QObject::tr("Do you want to reset settings to default values, or to abort without making changes?")); @@ -181,7 +181,7 @@ static bool ErrorSettingsRead(const bilingual_str& error, const std::vector<std: static void ErrorSettingsWrite(const bilingual_str& error, const std::vector<std::string>& details) { - QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Ok); + QMessageBox messagebox(QMessageBox::Critical, CLIENT_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Ok); /*: Explanatory text shown on startup when the settings file could not be written. Prompts user to check that we have the ability to write to the file. Explains that the user has the option of running without a settings file.*/ @@ -260,7 +260,7 @@ bool BitcoinApplication::createOptionsModel(bool resetSettings) error.translated += tr("Settings file %1 might be corrupt or invalid.").arg(QString::fromStdString(quoted_path)).toStdString(); } InitError(error); - QMessageBox::critical(nullptr, PACKAGE_NAME, QString::fromStdString(error.translated)); + QMessageBox::critical(nullptr, CLIENT_NAME, QString::fromStdString(error.translated)); return false; } return true; @@ -442,8 +442,8 @@ void BitcoinApplication::handleRunawayException(const QString &message) { QMessageBox::critical( nullptr, tr("Runaway exception"), - tr("A fatal error occurred. %1 can no longer continue safely and will quit.").arg(PACKAGE_NAME) + - QLatin1String("<br><br>") + GUIUtil::MakeHtmlLink(message, PACKAGE_BUGREPORT)); + tr("A fatal error occurred. %1 can no longer continue safely and will quit.").arg(CLIENT_NAME) + + QLatin1String("<br><br>") + GUIUtil::MakeHtmlLink(message, CLIENT_BUGREPORT)); ::exit(EXIT_FAILURE); } @@ -453,8 +453,8 @@ void BitcoinApplication::handleNonFatalException(const QString& message) QMessageBox::warning( nullptr, tr("Internal error"), tr("An internal error occurred. %1 will attempt to continue safely. This is " - "an unexpected bug which can be reported as described below.").arg(PACKAGE_NAME) + - QLatin1String("<br><br>") + GUIUtil::MakeHtmlLink(message, PACKAGE_BUGREPORT)); + "an unexpected bug which can be reported as described below.").arg(CLIENT_NAME) + + QLatin1String("<br><br>") + GUIUtil::MakeHtmlLink(message, CLIENT_BUGREPORT)); } WId BitcoinApplication::getMainWinId() const @@ -529,9 +529,9 @@ int GuiMain(int argc, char* argv[]) SetupUIArgs(gArgs); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { - InitError(strprintf(Untranslated("Error parsing command line arguments: %s"), error)); + InitError(Untranslated(strprintf("Error parsing command line arguments: %s", error))); // Create a message box, because the gui has neither been created nor has subscribed to core signals - QMessageBox::critical(nullptr, PACKAGE_NAME, + QMessageBox::critical(nullptr, CLIENT_NAME, // message cannot be translated because translations have not been initialized QString::fromStdString("Error parsing command line arguments: %1.").arg(QString::fromStdString(error))); return EXIT_FAILURE; @@ -551,14 +551,14 @@ int GuiMain(int argc, char* argv[]) #endif if (payment_server_token_seen && arg.startsWith("-")) { InitError(Untranslated(strprintf("Options ('%s') cannot follow a BIP-21 payment URI", argv[i]))); - QMessageBox::critical(nullptr, PACKAGE_NAME, + QMessageBox::critical(nullptr, CLIENT_NAME, // message cannot be translated because translations have not been initialized QString::fromStdString("Options ('%1') cannot follow a BIP-21 payment URI").arg(QString::fromStdString(argv[i]))); return EXIT_FAILURE; } if (invalid_token) { InitError(Untranslated(strprintf("Command line contains unexpected token '%s', see bitcoin-qt -h for a list of options.", argv[i]))); - QMessageBox::critical(nullptr, PACKAGE_NAME, + QMessageBox::critical(nullptr, CLIENT_NAME, // message cannot be translated because translations have not been initialized QString::fromStdString("Command line contains unexpected token '%1', see bitcoin-qt -h for a list of options.").arg(QString::fromStdString(argv[i]))); return EXIT_FAILURE; @@ -582,8 +582,8 @@ int GuiMain(int argc, char* argv[]) // Show help message immediately after parsing command-line options (for "-lang") and setting locale, // but before showing splash screen. - if (HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { - HelpMessageDialog help(nullptr, gArgs.IsArgSet("-version")); + if (HelpRequested(gArgs) || gArgs.GetBoolArg("-version", false)) { + HelpMessageDialog help(nullptr, gArgs.GetBoolArg("-version", false)); help.showOrPrint(); return EXIT_SUCCESS; } @@ -613,7 +613,7 @@ int GuiMain(int argc, char* argv[]) } else if (error->status != common::ConfigStatus::ABORTED) { // Show a generic message in other cases, and no additional error // message in the case of a read error if the user decided to abort. - QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error: %1").arg(QString::fromStdString(error->message.translated))); + QMessageBox::critical(nullptr, CLIENT_NAME, QObject::tr("Error: %1").arg(QString::fromStdString(error->message.translated))); } return EXIT_FAILURE; } @@ -686,7 +686,7 @@ int GuiMain(int argc, char* argv[]) if (app.baseInitialize()) { app.requestInitialize(); #if defined(Q_OS_WIN) - WinShutdownMonitor::registerShutdownBlockReason(QObject::tr("%1 didn't yet exit safely…").arg(PACKAGE_NAME), (HWND)app.getMainWinId()); + WinShutdownMonitor::registerShutdownBlockReason(QObject::tr("%1 didn't yet exit safely…").arg(CLIENT_NAME), (HWND)app.getMainWinId()); #endif app.exec(); } else { diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 1423a8bbc6..52b117eed7 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_BITCOIN_H #define BITCOIN_QT_BITCOIN_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <interfaces/node.h> #include <qt/initexecutor.h> diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 6d66c7473b..f899a524f4 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/bitcoingui.h> @@ -294,15 +294,15 @@ void BitcoinGUI::createActions() quitAction->setStatusTip(tr("Quit application")); quitAction->setShortcut(QKeySequence(tr("Ctrl+Q"))); quitAction->setMenuRole(QAction::QuitRole); - aboutAction = new QAction(tr("&About %1").arg(PACKAGE_NAME), this); - aboutAction->setStatusTip(tr("Show information about %1").arg(PACKAGE_NAME)); + aboutAction = new QAction(tr("&About %1").arg(CLIENT_NAME), this); + aboutAction->setStatusTip(tr("Show information about %1").arg(CLIENT_NAME)); aboutAction->setMenuRole(QAction::AboutRole); aboutAction->setEnabled(false); aboutQtAction = new QAction(tr("About &Qt"), this); aboutQtAction->setStatusTip(tr("Show information about Qt")); aboutQtAction->setMenuRole(QAction::AboutQtRole); optionsAction = new QAction(tr("&Options…"), this); - optionsAction->setStatusTip(tr("Modify configuration options for %1").arg(PACKAGE_NAME)); + optionsAction->setStatusTip(tr("Modify configuration options for %1").arg(CLIENT_NAME)); optionsAction->setMenuRole(QAction::PreferencesRole); optionsAction->setEnabled(false); @@ -364,7 +364,7 @@ void BitcoinGUI::createActions() showHelpMessageAction = new QAction(tr("&Command-line options"), this); showHelpMessageAction->setMenuRole(QAction::NoRole); - showHelpMessageAction->setStatusTip(tr("Show the %1 help message to get a list with possible Bitcoin command-line options").arg(PACKAGE_NAME)); + showHelpMessageAction->setStatusTip(tr("Show the %1 help message to get a list with possible Bitcoin command-line options").arg(CLIENT_NAME)); m_mask_values_action = new QAction(tr("&Mask values"), this); m_mask_values_action->setShortcut(QKeySequence(Qt::CTRL | Qt::SHIFT | Qt::Key_M)); @@ -837,7 +837,7 @@ void BitcoinGUI::createTrayIcon() #ifndef Q_OS_MACOS if (QSystemTrayIcon::isSystemTrayAvailable()) { trayIcon = new QSystemTrayIcon(m_network_style->getTrayAndWindowIcon(), this); - QString toolTip = tr("%1 client").arg(PACKAGE_NAME) + " " + m_network_style->getTitleAddText(); + QString toolTip = tr("%1 client").arg(CLIENT_NAME) + " " + m_network_style->getTitleAddText(); trayIcon->setToolTip(toolTip); } #endif @@ -1230,7 +1230,7 @@ void BitcoinGUI::createWallet() void BitcoinGUI::message(const QString& title, QString message, unsigned int style, bool* ret, const QString& detailed_message) { // Default title. On macOS, the window title is ignored (as required by the macOS Guidelines). - QString strTitle{PACKAGE_NAME}; + QString strTitle{CLIENT_NAME}; // Default to information icon int nMBoxIcon = QMessageBox::Information; int nNotifyIcon = Notificator::Information; @@ -1489,7 +1489,7 @@ void BitcoinGUI::updateProxyIcon() void BitcoinGUI::updateWindowTitle() { - QString window_title = PACKAGE_NAME; + QString window_title = CLIENT_NAME; #ifdef ENABLE_WALLET if (walletFrame) { WalletModel* const wallet_model = walletFrame->currentWalletModel(); diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 73adbda5a5..32fb7488fb 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_BITCOINGUI_H #define BITCOIN_QT_BITCOINGUI_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/bitcoinunits.h> #include <qt/clientmodel.h> diff --git a/src/qt/bitcoinunits.cpp b/src/qt/bitcoinunits.cpp index fe3eb3240b..f73d83e4fc 100644 --- a/src/qt/bitcoinunits.cpp +++ b/src/qt/bitcoinunits.cpp @@ -166,7 +166,7 @@ bool BitcoinUnits::parse(Unit unit, const QString& value, CAmount* val_out) { return false; // More than one dot } - QString whole = parts[0]; + const QString& whole = parts[0]; QString decimals; if(parts.size() > 1) diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 5c70c2695c..fb81dee8da 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/clientmodel.h> diff --git a/src/qt/createwalletdialog.cpp b/src/qt/createwalletdialog.cpp index 3e8d1461e5..2908043d28 100644 --- a/src/qt/createwalletdialog.cpp +++ b/src/qt/createwalletdialog.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <interfaces/node.h> #include <qt/createwalletdialog.h> diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 99fb238772..9ab6b124e0 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -105,7 +105,7 @@ <item> <widget class="QLabel" name="databaseCacheLabel"> <property name="toolTip"> - <string extracomment="Tooltip text for Options window setting that sets the size of the database cache. Explains the corresponding effects of increasing/decreasing this value.">Maximum database cache size. A larger cache can contribute to faster sync, after which the benefit is less pronounced for most use cases. Lowering the cache size will reduce memory usage. Unused mempool memory is shared for this cache.</string> + <string extracomment="Tooltip text for Options window setting that sets the size of the database cache. Explains the corresponding effects of increasing/decreasing this value.">Maximum database cache size. Make sure you have enough RAM. A larger cache can contribute to faster sync, after which the benefit is less pronounced for most use cases. Lowering the cache size will reduce memory usage. Unused mempool memory is shared for this cache.</string> </property> <property name="text"> <string>Size of &database cache</string> @@ -316,22 +316,12 @@ </attribute> <layout class="QVBoxLayout" name="verticalLayout_Network"> <item> - <widget class="QCheckBox" name="mapPortUpnp"> - <property name="toolTip"> - <string>Automatically open the Bitcoin client port on the router. This only works when your router supports UPnP and it is enabled.</string> - </property> - <property name="text"> - <string>Map port using &UPnP</string> - </property> - </widget> - </item> - <item> <widget class="QCheckBox" name="mapPortNatpmp"> <property name="toolTip"> - <string>Automatically open the Bitcoin client port on the router. This only works when your router supports NAT-PMP and it is enabled. The external port could be random.</string> + <string>Automatically open the Bitcoin client port on the router. This only works when your router supports PCP or NAT-PMP and it is enabled. The external port could be random.</string> </property> <property name="text"> - <string>Map port using NA&T-PMP</string> + <string>Map port using PCP or NA&T-PMP</string> </property> </widget> </item> diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 26b42deb64..eeee13b224 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chainparams.h> #include <qt/intro.h> @@ -127,16 +127,16 @@ Intro::Intro(QWidget *parent, int64_t blockchain_size_gb, int64_t chain_state_si m_prune_target_gb{GetPruneTargetGB()} { ui->setupUi(this); - ui->welcomeLabel->setText(ui->welcomeLabel->text().arg(PACKAGE_NAME)); - ui->storageLabel->setText(ui->storageLabel->text().arg(PACKAGE_NAME)); + ui->welcomeLabel->setText(ui->welcomeLabel->text().arg(CLIENT_NAME)); + ui->storageLabel->setText(ui->storageLabel->text().arg(CLIENT_NAME)); ui->lblExplanation1->setText(ui->lblExplanation1->text() - .arg(PACKAGE_NAME) + .arg(CLIENT_NAME) .arg(m_blockchain_size_gb) .arg(2009) .arg(tr("Bitcoin")) ); - ui->lblExplanation2->setText(ui->lblExplanation2->text().arg(PACKAGE_NAME)); + ui->lblExplanation2->setText(ui->lblExplanation2->text().arg(CLIENT_NAME)); const int min_prune_target_GB = std::ceil(MIN_DISK_SPACE_FOR_BLOCK_FILES / 1e9); ui->pruneGB->setRange(min_prune_target_GB, std::numeric_limits<int>::max()); @@ -246,7 +246,7 @@ bool Intro::showIfNeeded(bool& did_show_intro, int64_t& prune_MiB) } break; } catch (const fs::filesystem_error&) { - QMessageBox::critical(nullptr, PACKAGE_NAME, + QMessageBox::critical(nullptr, CLIENT_NAME, tr("Error: Specified data directory \"%1\" cannot be created.").arg(dataDir)); /* fall through, back to choosing screen */ } @@ -389,7 +389,7 @@ void Intro::UpdatePruneLabels(bool prune_checked) //: Explanatory text on the capability of the current prune target. tr("(sufficient to restore backups %n day(s) old)", "", expected_backup_days)); ui->sizeWarningLabel->setText( - tr("%1 will download and store a copy of the Bitcoin block chain.").arg(PACKAGE_NAME) + " " + + tr("%1 will download and store a copy of the Bitcoin block chain.").arg(CLIENT_NAME) + " " + storageRequiresMsg.arg(m_required_space_gb) + " " + tr("The wallet will also be stored in this directory.") ); diff --git a/src/qt/locale/bitcoin_am.ts b/src/qt/locale/bitcoin_am.ts index 2fcf7a1e63..c02049b984 100644 --- a/src/qt/locale/bitcoin_am.ts +++ b/src/qt/locale/bitcoin_am.ts @@ -176,6 +176,10 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished">ቦርሳዎ ምስጢር ተደርጓል</translation> </message> <message> + <source>Back</source> + <translation type="unfinished">ተመለስ</translation> + </message> + <message> <source>Wallet to be encrypted</source> <translation type="unfinished">ለመመስጠር የተዘጋጀ ዋሌት</translation> </message> @@ -254,6 +258,18 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished">ስህተት፥ %1</translation> </message> <message> + <source>Embedded "%1"</source> + <translation type="unfinished">የተከተተ "%1"</translation> + </message> + <message> + <source>Default system font "%1"</source> + <translation type="unfinished">ነባሪ የስርዓት ቅርጸ-ቁምፊ "%1</translation> + </message> + <message> + <source>Custom…</source> + <translation type="unfinished">ብጁ…</translation> + </message> + <message> <source>Amount</source> <translation type="unfinished">መጠን</translation> </message> @@ -363,6 +379,14 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished">&ተቀበል</translation> </message> <message> + <source>&Change Passphrase…</source> + <translation type="unfinished">&የይለፍ ቃል ቀይር…</translation> + </message> + <message> + <source>Sign messages with your Bitcoin addresses to prove you own them</source> + <translation type="unfinished">በእርሶ የተያዙ መሆኑን ለማረጋገጥ በBitcoin አድራሻዎችዎ መልዕክቶችን ይፈርሙ</translation> + </message> + <message> <source>&File</source> <translation type="unfinished">&ፋይል</translation> </message> @@ -406,6 +430,14 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished">ዋሌት ዝጋ</translation> </message> <message> + <source>Migrate Wallet</source> + <translation type="unfinished">ዋሌትዎን ያዛውሩ</translation> + </message> + <message> + <source>Migrate a wallet</source> + <translation type="unfinished">ዋሌትዎን ያዛውሩ</translation> + </message> + <message> <source>Wallet Name</source> <extracomment>Label of the input field where the name of the wallet is entered.</extracomment> <translation type="unfinished">ዋሌት ስም</translation> @@ -423,6 +455,14 @@ Signing is only possible with addresses of the type 'legacy'.</source> </translation> </message> <message> + <source>Error creating wallet</source> + <translation type="unfinished">ዋሌትዎን ለፍጠር ተሳስተዋል </translation> + </message> + <message> + <source>Cannot create new wallet, the software was compiled without sqlite support (required for descriptor wallets)</source> + <translation type="unfinished">አዲስ ዋሌት መፍጠር አልተቻለም፣ ሶፍትዌሩ የተቀናበረው ያለ ስኩላይት ድጋፍ ነው (ለገላጭ ዋሌቶች ያስፈልጋል)</translation> + </message> + <message> <source>Error: %1</source> <translation type="unfinished">ስህተት፥ %1</translation> </message> @@ -485,6 +525,49 @@ Signing is only possible with addresses of the type 'legacy'.</source> </message> </context> <context> + <name>MigrateWalletActivity</name> + <message> + <source>Migrate wallet</source> + <translation type="unfinished">ዋሌት ያዛውሩ</translation> + </message> + <message> + <source>Migrating the wallet will convert this wallet to one or more descriptor wallets. A new wallet backup will need to be made. +If this wallet contains any watchonly scripts, a new wallet will be created which contains those watchonly scripts. +If this wallet contains any solvable but not watched scripts, a different and new wallet will be created which contains those scripts. + +The migration process will create a backup of the wallet before migrating. This backup file will be named <wallet name>-<timestamp>.legacy.bak and can be found in the directory for this wallet. In the event of an incorrect migration, the backup can be restored with the "Restore Wallet" functionality.</source> + <translation type="unfinished">ዋሌትን ማዛወር ይህንን ዋሌት አንድ ወይም ከዚያ በላይ ወደሆነ ገላጭ ዋሌቶች ይቀይረዋል።አዲስ ዋሌት ማዘጋጀት ያስፈልጋል ።ይህ ዋሌት ምንም ዓይነት የመመልከት ብቻ ስክሪፕቶችን የያዘ ከሆነ፣እነዚያን የመመልከት ብቻ ስክሪፕቶችን የያዘ አዲስ ዋሌት ይፈጠራል።ይህ ዋሌት ሊፈቱ የሚችሉ ነገር ግን የመመልከት ብቻ ያልሆኑ ስክሪፕቶችን የያዘ ከሆነ ፣ እነዚህን የያዘ አዲስ እና ልዩ የሆነ ዋሌት ይፈጠራል ።የማዛወር ሂደቱ ማዘዋወር ከመፈጸሙ በፊት የነዚህን ዋሌቶች መጠባበቂያ ቅጂ ይይዛል።ይህ መጠባበቂያ ቅጂ 1-2 legacy.bak ተብሎ ተሰይሞ በዋሌቱ ማውጫ ውስጥ ይገኛል።የተሳሳተ ዝውውር በሚከሰትበት ጊዜየመጠባበቂያ ቅጂው በ ዋሌት መመለሻ መተግበሪያ ውስጥ ይከማቻል።</translation> + </message> + <message> + <source>Migrate Wallet</source> + <translation type="unfinished">ዋሌትዎን ያዛውሩ</translation> + </message> + <message> + <source>Migrating Wallet <b>%1</b>…</source> + <translation type="unfinished">ዋሌት ማዘዋወር <b>%1</b>…</translation> + </message> + <message> + <source>The wallet '%1' was migrated successfully.</source> + <translation type="unfinished">ዋሌት '%1' በትክክል ተዛውሯል </translation> + </message> + <message> + <source>Watchonly scripts have been migrated to a new wallet named '%1'.</source> + <translation type="unfinished">የመመልከት ብቻ ስክሪፕቶች'%1'.ወደ ተሰኘው ዋሌት ተዛውረዋል </translation> + </message> + <message> + <source>Solvable but not watched scripts have been migrated to a new wallet named '%1'.</source> + <translation type="unfinished">ሊፈቱ የሚችሉ ነገር ግን የማይታዩ ስክሪፕቶች ወደ አዲስ ዋሌት ተዛውረዋል '%1'።</translation> + </message> + <message> + <source>Migration failed</source> + <translation type="unfinished">ዝውውሩ አልተሳካም </translation> + </message> + <message> + <source>Migration Successful</source> + <translation type="unfinished">ዝውውር ተሳክቷል </translation> + </message> +</context> +<context> <name>OpenWalletActivity</name> <message> <source>Open Wallet</source> @@ -502,6 +585,14 @@ Signing is only possible with addresses of the type 'legacy'.</source> <context> <name>CreateWalletDialog</name> <message> + <source>You are one step away from creating your new wallet!</source> + <translation type="unfinished">አዲሱን ዋሌትዎን ለመፍጠር አንድ እርምጃ ይቀርዎታል</translation> + </message> + <message> + <source>Please provide a name and, if desired, enable any advanced options</source> + <translation type="unfinished">እባክዎ ስም ያስገቡ እና ከተፈለገ ማንኛውንም የላቁ አማራጮችን ያንቁ</translation> + </message> + <message> <source>Wallet Name</source> <translation type="unfinished">ዋሌት ስም</translation> </message> @@ -590,6 +681,10 @@ Signing is only possible with addresses of the type 'legacy'.</source> <context> <name>OptionsDialog</name> <message> + <source>Font in the Overview tab: </source> + <translation type="unfinished">በአጠቃላይ እይታ ትር ውስጥ ያለ ፊደል</translation> + </message> + <message> <source>Error</source> <translation type="unfinished">ስህተት</translation> </message> @@ -602,6 +697,13 @@ Signing is only possible with addresses of the type 'legacy'.</source> </message> </context> <context> + <name>PSBTOperationsDialog</name> + <message> + <source>Sends %1 to %2</source> + <translation type="unfinished">%1 ወደ %2 ይልካል</translation> + </message> + </context> +<context> <name>PeerTableModel</name> <message> <source>Address</source> @@ -610,6 +712,56 @@ Signing is only possible with addresses of the type 'legacy'.</source> </message> </context> <context> + <name>RPCConsole</name> + <message> + <source>Local Addresses</source> + <translation type="unfinished">የአካባቢ አድራሻዎች</translation> + </message> + <message> + <source>Network addresses that your Bitcoin node is currently using to communicate with other nodes.</source> + <translation type="unfinished">የእርስዎ Bitcoin ኖድ ከሌሎች ኖዶች ጋር ለመገናኘት በአሁኑ ጊዜ እየተጠቀመበት ያለው የአውታረ መረብ አድራሻ።</translation> + </message> + <message> + <source>Hide Peers Detail</source> + <translation type="unfinished">የአቻዎችን ዝርዝር ደብቅ</translation> + </message> + <message> + <source>The transport layer version: %1</source> + <translation type="unfinished">የማጓጓዣ ንብርብር ስሪት፡ %1</translation> + </message> + <message> + <source>Transport</source> + <translation type="unfinished">መጓጓዣ</translation> + </message> + <message> + <source>Session ID</source> + <translation type="unfinished">የክፍለ ጊዜ መለያ</translation> + </message> + <message> + <source>The BIP324 session ID string in hex.</source> + <translation type="unfinished">የBIP324 ክፍለ ጊዜ መለያ ሕብረቁምፊ በሄክስ።</translation> + </message> + <message> + <source>detecting: peer could be v1 or v2</source> + <extracomment>Explanatory text for "detecting" transport type.</extracomment> + <translation type="unfinished">ማወቅ፡ እኩያ v1 ወይም v2 ሊሆን ይችላል።</translation> + </message> + <message> + <source>v1: unencrypted, plaintext transport protocol</source> + <extracomment>Explanatory text for v1 transport type.</extracomment> + <translation type="unfinished">v1፡ ያልተመሰጠረ፣ ግልጽ የጽሑፍ ትራንስፖርት ፕሮቶኮል</translation> + </message> + <message> + <source>v2: BIP324 encrypted transport protocol</source> + <extracomment>Explanatory text for v2 transport type.</extracomment> + <translation type="unfinished">v2፡ BIP324 የተመሰጠረ የትራንስፖርት ፕሮቶኮል</translation> + </message> + <message> + <source>Node window - [%1]</source> + <translation type="unfinished">የኖድ መስኮት - [%1]</translation> + </message> + </context> +<context> <name>ReceiveRequestDialog</name> <message> <source>Amount:</source> @@ -661,6 +813,10 @@ Signing is only possible with addresses of the type 'legacy'.</source> <source>Copy fee</source> <translation type="unfinished">ክፍያው ቅዳ</translation> </message> + <message> + <source>%1 from wallet '%2'</source> + <translation type="unfinished">%1 ከዋሌት %2'</translation> + </message> <message numerus="yes"> <source>Estimated to begin confirmation within %n block(s).</source> <translation type="unfinished"> @@ -674,6 +830,17 @@ Signing is only possible with addresses of the type 'legacy'.</source> </message> </context> <context> + <name>SignVerifyMessageDialog</name> + <message> + <source>You can sign messages/agreements with your legacy (P2PKH) addresses to prove you can receive bitcoins sent to them. Be careful not to sign anything vague or random, as phishing attacks may try to trick you into signing your identity over to them. Only sign fully-detailed statements you agree to.</source> + <translation type="unfinished">ወደ እነርሱ የተላኩ ቢትኮይን መቀበል እንደሚችሉ ለማረጋገጥ ከርስዎ (P2PKH) አድራሻዎች ጋር መልዕክቶችን/ስምምነቶችን መፈረም ይችላሉ። የማስገር ጥቃቶች እርስዎን ማንነትዎን በእነሱ ላይ እንዲፈርሙ ሊያታልሉዎት ስለሚችሉ ግልጽ ያልሆነ ወይም በዘፈቀደ ላለመፈረም ይጠንቀቁ። የተስማሙባቸውን ሙሉ ዝርዝር መግለጫዎች ብቻ ይፈርሙ።</translation> + </message> + <message> + <source>The entered address does not refer to a legacy (P2PKH) key. Message signing for SegWit and other non-P2PKH address types is not supported in this version of %1. Please check the address and try again.</source> + <translation type="unfinished">የገባው አድራሻ የቅርስ (P2PKH) ቁልፍን አያመለክትም። ለሴግዊት እና ሌሎች P2PKH ላልሆኑ የአድራሻ አይነቶች የመልዕክት መፈረም በዚህ የ%1 ስሪት ውስጥ አይደገፍም። እባክዎ አድራሻውን ያረጋግጡ እና እንደገና ይሞክሩ።</translation> + </message> + </context> +<context> <name>TransactionDesc</name> <message> <source>Date</source> @@ -687,6 +854,10 @@ Signing is only possible with addresses of the type 'legacy'.</source> </translation> </message> <message> + <source>%1 (Certificate was not verified)</source> + <translation type="unfinished">%1 (ማረጋገጫው አልተረጋገጠም)</translation> + </message> + <message> <source>Amount</source> <translation type="unfinished">መጠን</translation> </message> @@ -742,6 +913,17 @@ Signing is only possible with addresses of the type 'legacy'.</source> </message> </context> <context> + <name>WalletModel</name> + <message> + <source>Fee-bump PSBT copied to clipboard</source> + <translation type="unfinished">ከክፍያ-ነፃ PSBT ወደ ቅንጥብ ሰሌዳ ተቀድቷል።</translation> + </message> + <message> + <source>Signer error</source> + <translation type="unfinished">የፈራሚ ስህተት</translation> + </message> + </context> +<context> <name>WalletView</name> <message> <source>&Export</source> diff --git a/src/qt/locale/bitcoin_bn.ts b/src/qt/locale/bitcoin_bn.ts index d22624ad85..bfb104a7a3 100644 --- a/src/qt/locale/bitcoin_bn.ts +++ b/src/qt/locale/bitcoin_bn.ts @@ -276,6 +276,10 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished">আংশিক স্বাক্ষরিত বিটকয়েন লেনদেন লোড করুন</translation> </message> <message> + <source>Load PSBT from &clipboard…</source> + <translation type="unfinished">&ক্লিপবোর্ড থেকে আংশিক স্বাক্ষরিত বিটকয়েন লেনদেন আনুন</translation> + </message> + <message> <source>Load Partially Signed Bitcoin Transaction from clipboard</source> <translation type="unfinished">ক্লিপবোর্ড থেকে আংশিক স্বাক্ষরিত বিটকয়েন লেনদেন লোড করুন</translation> </message> diff --git a/src/qt/locale/bitcoin_de.ts b/src/qt/locale/bitcoin_de.ts index 474ce2a088..807a1c64ee 100644 --- a/src/qt/locale/bitcoin_de.ts +++ b/src/qt/locale/bitcoin_de.ts @@ -615,11 +615,15 @@ Das Signieren ist nur mit Adressen vom Typ 'Legacy' möglich.</translation> <source>Processing blocks on disk…</source> <translation type="unfinished">Verarbeite Blöcke auf Datenträger...</translation> </message> + <message> + <source>Connecting to peers…</source> + <translation type="unfinished">Verbindung zu Peers wird hergestellt…</translation> + </message> <message numerus="yes"> <source>Processed %n block(s) of transaction history.</source> <translation type="unfinished"> - <numerusform>Processed %n block(s) of transaction history.</numerusform> - <numerusform>Processed %n block(s) of transaction history.</numerusform> + <numerusform>%n Block der Transaktionshistorie prozessiert.</numerusform> + <numerusform>%n Block/Blöcke der Transaktionshistorie prozessiert.</numerusform> </translation> </message> <message> @@ -710,7 +714,7 @@ Das Signieren ist nur mit Adressen vom Typ 'Legacy' möglich.</translation> <message> <source>Show Peers tab</source> <extracomment>A context menu item. The "Peers tab" is an element of the "Node window".</extracomment> - <translation type="unfinished">Gegenstellen Reiter anzeigen</translation> + <translation type="unfinished">Reiter mit Peers anzeigen</translation> </message> <message> <source>Disable network activity</source> @@ -1420,7 +1424,7 @@ Während des Migrationsprozesses wird vor der Migration ein Backup der Wallet er </message> <message> <source>%1 is currently syncing. It will download headers and blocks from peers and validate them until reaching the tip of the block chain.</source> - <translation type="unfinished">%1 synchronisiert gerade. Es lädt Header und Blöcke von Gegenstellen und validiert sie bis zum Erreichen der Spitze der Blockchain.</translation> + <translation type="unfinished">%1 synchronisiert gerade. Es lädt Header und Blöcke von Peers und validiert sie bis zum Erreichen der Spitze der Blockchain.</translation> </message> <message> <source>Unknown. Syncing Headers (%1, %2%)…</source> @@ -1639,7 +1643,7 @@ Während des Migrationsprozesses wird vor der Migration ein Backup der Wallet er </message> <message> <source>Used for reaching peers via:</source> - <translation type="unfinished">Benutzt um Gegenstellen zu erreichen über:</translation> + <translation type="unfinished">Benutzt um Peers zu erreichen über:</translation> </message> <message> <source>&Window</source> @@ -1703,7 +1707,7 @@ Während des Migrationsprozesses wird vor der Migration ein Backup der Wallet er </message> <message> <source>Use separate SOCKS&5 proxy to reach peers via Tor onion services:</source> - <translation type="unfinished">Nutze separaten SOCKS&5-Proxy um Gegenstellen über Tor-Onion-Dienste zu erreichen:</translation> + <translation type="unfinished">Nutze separaten SOCKS&5-Proxy um Peers über Tor-Onion-Dienste zu erreichen:</translation> </message> <message> <source>&Cancel</source> @@ -2042,11 +2046,6 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI <translation type="unfinished">User-Agent</translation> </message> <message> - <source>Peer</source> - <extracomment>Title of Peers Table column which contains a unique number used to identify a connection.</extracomment> - <translation type="unfinished">Gegenstelle</translation> - </message> - <message> <source>Age</source> <extracomment>Title of Peers Table column which indicates the duration (length of time) since the peer connection started.</extracomment> <translation type="unfinished">Alter</translation> @@ -2171,6 +2170,14 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI <translation type="unfinished">Anzahl der Verbindungen</translation> </message> <message> + <source>Local Addresses</source> + <translation type="unfinished">Lokale Adressen</translation> + </message> + <message> + <source>Network addresses that your Bitcoin node is currently using to communicate with other nodes.</source> + <translation type="unfinished">Netzwerk-Adressen, die dein Bitcoin-Node aktuell verwendet, um mit anderen Nodes zu kommunizieren.</translation> + </message> + <message> <source>Block chain</source> <translation type="unfinished">Blockchain</translation> </message> @@ -2203,16 +2210,20 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI <translation type="unfinished">Gesendet</translation> </message> <message> - <source>&Peers</source> - <translation type="unfinished">&Gegenstellen</translation> - </message> - <message> <source>Banned peers</source> - <translation type="unfinished">Gesperrte Gegenstellen</translation> + <translation type="unfinished">Gesperrte Peers</translation> </message> <message> <source>Select a peer to view detailed information.</source> - <translation type="unfinished">Gegenstelle auswählen, um detaillierte Informationen zu erhalten.</translation> + <translation type="unfinished">Peers auswählen, um detaillierte Informationen zu erhalten.</translation> + </message> + <message> + <source>Hide Peers Detail</source> + <translation type="unfinished">Reiter mit Peers verstecken</translation> + </message> + <message> + <source>Ctrl+X</source> + <translation type="unfinished">Strg+X</translation> </message> <message> <source>The transport layer version: %1</source> @@ -2220,7 +2231,7 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI </message> <message> <source>Whether we relay transactions to this peer.</source> - <translation type="unfinished">Ob wir Adressen an diese Gegenstelle weiterleiten.</translation> + <translation type="unfinished">Ob wir Adressen an diesen Peer weiterleiten.</translation> </message> <message> <source>Transaction Relay</source> @@ -2244,7 +2255,7 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI </message> <message> <source>The mapped Autonomous System used for diversifying peer selection.</source> - <translation type="unfinished">Das zugeordnete autonome System zur Diversifizierung der Gegenstellen-Auswahl.</translation> + <translation type="unfinished">Das zugeordnete autonome System zur Diversifizierung der Peer-Auswahl.</translation> </message> <message> <source>Mapped AS</source> @@ -2253,7 +2264,7 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI <message> <source>Whether we relay addresses to this peer.</source> <extracomment>Tooltip text for the Address Relay field in the peer details area, which displays whether we relay addresses to this peer (Yes/No).</extracomment> - <translation type="unfinished">Ob wir Adressen an diese Gegenstelle weiterleiten.</translation> + <translation type="unfinished">Ob wir Adressen an diesen Peer weiterleiten.</translation> </message> <message> <source>Address Relay</source> @@ -2263,12 +2274,12 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI <message> <source>The total number of addresses received from this peer that were processed (excludes addresses that were dropped due to rate-limiting).</source> <extracomment>Tooltip text for the Addresses Processed field in the peer details area, which displays the total number of addresses received from this peer that were processed (excludes addresses that were dropped due to rate-limiting).</extracomment> - <translation type="unfinished">Die Gesamtzahl der von dieser Gegenstelle empfangenen Adressen, die aufgrund von Ratenbegrenzung verworfen (nicht verarbeitet) wurden.</translation> + <translation type="unfinished">Die Gesamtzahl der von diesem Peer empfangenen Adressen, die aufgrund von Ratenbegrenzung verworfen (nicht verarbeitet) wurden.</translation> </message> <message> <source>The total number of addresses received from this peer that were dropped (not processed) due to rate-limiting.</source> <extracomment>Tooltip text for the Addresses Rate-Limited field in the peer details area, which displays the total number of addresses received from this peer that were dropped (not processed) due to rate-limiting.</extracomment> - <translation type="unfinished">Die Gesamtzahl der von dieser Gegenstelle empfangenen Adressen, die aufgrund von Ratenbegrenzung verworfen (nicht verarbeitet) wurden.</translation> + <translation type="unfinished">Die Gesamtzahl der von diesem Peer empfangenen Adressen, die aufgrund von Ratenbegrenzung verworfen (nicht verarbeitet) wurden.</translation> </message> <message> <source>Addresses Processed</source> @@ -2306,7 +2317,7 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI </message> <message> <source>The direction and type of peer connection: %1</source> - <translation type="unfinished">Die Richtung und der Typ der Gegenstellen-Verbindung: %1</translation> + <translation type="unfinished">Die Richtung und der Typ der Peer-Verbindung: %1</translation> </message> <message> <source>Direction/Type</source> @@ -2318,7 +2329,7 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI </message> <message> <source>The network protocol this peer is connected through: IPv4, IPv6, Onion, I2P, or CJDNS.</source> - <translation type="unfinished">Das Netzwerkprotokoll, über das diese Gegenstelle verbunden ist, ist: IPv4, IPv6, Onion, I2P oder CJDNS.</translation> + <translation type="unfinished">Das Netzwerkprotokoll, über das dieser Peer verbunden ist, ist: IPv4, IPv6, Onion, I2P oder CJDNS.</translation> </message> <message> <source>Services</source> @@ -2338,7 +2349,7 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI </message> <message> <source>Elapsed time since a novel block passing initial validity checks was received from this peer.</source> - <translation type="unfinished">Abgelaufene Zeit seitdem ein neuer Block mit erfolgreichen initialen Gültigkeitsprüfungen von dieser Gegenstelle empfangen wurde.</translation> + <translation type="unfinished">Verstrichene Zeit, seit ein neuer Block, der initiale Validierungsprüfungen bestanden hat, von diesem Peer empfangen wurde.</translation> </message> <message> <source>Last Block</source> @@ -2347,7 +2358,7 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI <message> <source>Elapsed time since a novel transaction accepted into our mempool was received from this peer.</source> <extracomment>Tooltip text for the Last Transaction field in the peer details area.</extracomment> - <translation type="unfinished">Abgelaufene Zeit seit eine neue Transaktion, die in unseren Speicherpool hineingelassen wurde, von dieser Gegenstelle empfangen wurde.</translation> + <translation type="unfinished">Verstrichene Zeit, seit eine neue Transaktion, die in unseren Mempool aufgenommen wurde, von diesem Peer empfangen wurde.</translation> </message> <message> <source>Last Send</source> @@ -2416,7 +2427,7 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI <message> <source>Inbound: initiated by peer</source> <extracomment>Explanatory text for an inbound peer connection.</extracomment> - <translation type="unfinished">Eingehend: wurde von Gegenstelle initiiert</translation> + <translation type="unfinished">Eingehend: wurde vom Peer initiiert</translation> </message> <message> <source>Outbound Full Relay: default</source> @@ -2446,7 +2457,7 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI <message> <source>detecting: peer could be v1 or v2</source> <extracomment>Explanatory text for "detecting" transport type.</extracomment> - <translation type="unfinished">Erkennen: Peer könnte v1 oder v2 sein</translation> + <translation type="unfinished">Erkenne: Peer könnte v1 oder v2 sein</translation> </message> <message> <source>v1: unencrypted, plaintext transport protocol</source> @@ -2460,11 +2471,11 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI </message> <message> <source>we selected the peer for high bandwidth relay</source> - <translation type="unfinished">Wir haben die Gegenstelle zum Weiterleiten mit hoher Bandbreite ausgewählt</translation> + <translation type="unfinished">Wir haben den Peer zum Weiterleiten mit hoher Bandbreite ausgewählt</translation> </message> <message> <source>the peer selected us for high bandwidth relay</source> - <translation type="unfinished">Die Gegenstelle hat uns zum Weiterleiten mit hoher Bandbreite ausgewählt</translation> + <translation type="unfinished">Der Peer hat uns zum Weiterleiten mit hoher Bandbreite ausgewählt</translation> </message> <message> <source>no high bandwidth relay selected</source> @@ -2584,7 +2595,7 @@ Für weitere Informationen über diese Konsole, tippe %6. </message> <message> <source>(peer: %1)</source> - <translation type="unfinished">(Gegenstelle: %1)</translation> + <translation type="unfinished">(Peer: %1)</translation> </message> <message> <source>via %1</source> @@ -3996,7 +4007,7 @@ Gehen Sie zu Datei > Wallet Öffnen, um eine Wallet zu laden. </message> <message> <source>%s request to listen on port %u. This port is considered "bad" and thus it is unlikely that any peer will connect to it. See doc/p2p-bad-ports.md for details and a full list.</source> - <translation type="unfinished">%s Aufforderung, auf Port %u zu lauschen. Dieser Port wird als "schlecht" eingeschätzt und es ist daher unwahrscheinlich, dass sich Bitcoin Core Gegenstellen mit ihm verbinden. Siehe doc/p2p-bad-ports.md für Details und eine vollständige Liste.</translation> + <translation type="unfinished">%s Aufforderung, auf Port %u zu lauschen. Dieser Port wird als "schlecht" eingeschätzt und es ist daher unwahrscheinlich, dass sich Peers mit ihm verbinden. Siehe doc/p2p-bad-ports.md für Details und eine vollständige Liste.</translation> </message> <message> <source>Cannot downgrade wallet from version %i to version %i. Wallet version unchanged.</source> @@ -4161,7 +4172,7 @@ Bitte nutzen Sie entweder "bdb" oder "sqlite".</translation> </message> <message> <source>Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade.</source> - <translation type="unfinished">Warnung: Wir scheinen nicht vollständig mit unseren Gegenstellen übereinzustimmen! Sie oder die anderen Knoten müssen unter Umständen Ihre Client-Software aktualisieren.</translation> + <translation type="unfinished">Warnung: Wir scheinen nicht vollständig mit unseren Peers übereinzustimmen! Sie oder die anderen Knoten müssen unter Umständen Ihre Client-Software aktualisieren.</translation> </message> <message> <source>Witness data for blocks after height %d requires validation. Please restart with -reindex.</source> diff --git a/src/qt/locale/bitcoin_de_CH.ts b/src/qt/locale/bitcoin_de_CH.ts index a9a15f08ff..ffbc761ac1 100644 --- a/src/qt/locale/bitcoin_de_CH.ts +++ b/src/qt/locale/bitcoin_de_CH.ts @@ -2227,6 +2227,14 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI <translation type="unfinished">Anzahl der Verbindungen</translation> </message> <message> + <source>Local Addresses</source> + <translation type="unfinished">Lokale Adressen</translation> + </message> + <message> + <source>Network addresses that your Bitcoin node is currently using to communicate with other nodes.</source> + <translation type="unfinished">Netzwerk-Adressen, die dein Bitcoin-Node aktuell verwendet, um mit anderen Nodes zu kommunizieren.</translation> + </message> + <message> <source>Block chain</source> <translation type="unfinished">Blockchain</translation> </message> @@ -2271,6 +2279,14 @@ Wenn Sie diese Fehlermeldung erhalten, sollten Sie den Händler bitten, einen BI <translation type="unfinished">Gegenstelle auswählen, um detaillierte Informationen zu erhalten.</translation> </message> <message> + <source>Hide Peers Detail</source> + <translation type="unfinished">Gegenstellen Reiter verstecken</translation> + </message> + <message> + <source>Ctrl+X</source> + <translation type="unfinished">Strg+X</translation> + </message> + <message> <source>The transport layer version: %1</source> <translation type="unfinished">Die Transportschicht-Version: %1</translation> </message> diff --git a/src/qt/locale/bitcoin_gl_ES.ts b/src/qt/locale/bitcoin_gl_ES.ts index 77f9f3278c..47951fa6f2 100644 --- a/src/qt/locale/bitcoin_gl_ES.ts +++ b/src/qt/locale/bitcoin_gl_ES.ts @@ -2,10 +2,6 @@ <context> <name>AddressBookPage</name> <message> - <source>Right-click to edit address or label</source> - <translation type="unfinished">cd vcpkg/buildtrees/libvpx/srccd *./configuresed -i 's/CFLAGS+=-I/CFLAGS+=-fPIC -I/g' Makefilesed -i 's/CXXFLAGS+=-I/CXXFLAGS+=-fPIC -I/g' Makefilemakecp libvpx.a $HOME/vcpkg/installed/x64-linux/lib/cd</translation> - </message> - <message> <source>Create a new address</source> <translation type="unfinished">Crea un novo enderezo</translation> </message> @@ -94,6 +90,14 @@ Firmar é posible unicamente con enderezos de tipo 'legacy'.</translation> <translation type="unfinished">Houbo un erro tentando gardar a lista de enderezos en %1. Por favor proba de novo.</translation> </message> <message> + <source>Sending addresses - %1</source> + <translation type="unfinished">Enviando enderezos - %1</translation> + </message> + <message> + <source>Receiving addresses - %1</source> + <translation type="unfinished">Recibindo enderezos - %1</translation> + </message> + <message> <source>Exporting Failed</source> <translation type="unfinished">Exportación Fallida</translation> </message> diff --git a/src/qt/locale/bitcoin_ru.ts b/src/qt/locale/bitcoin_ru.ts index 204b66774e..7a54b45768 100644 --- a/src/qt/locale/bitcoin_ru.ts +++ b/src/qt/locale/bitcoin_ru.ts @@ -310,6 +310,10 @@ Signing is only possible with addresses of the type 'legacy'.</source> <source>unknown</source> <translation type="unfinished">неизвестно</translation> </message> + <message> + <source>Default system font "%1"</source> + <translation type="unfinished">Системный шрифт по умолчанию "%1"</translation> + </message> <message numerus="yes"> <source>%n second(s)</source> <translation type="unfinished"> @@ -1526,6 +1530,10 @@ The migration process will create a backup of the wallet before migrating. This <translation type="unfinished">Сворачивать вместо выхода из приложения при закрытии окна. Если данный параметр включён, приложение закроется только после нажатия "Выход" в меню.</translation> </message> <message> + <source>Font in the Overview tab: </source> + <translation type="unfinished">Шрифт на вкладке «Обзор»:</translation> + </message> + <message> <source>Options set in this dialog are overridden by the command line:</source> <translation type="unfinished">Параметры командной строки, которые переопределили параметры из этого окна:</translation> </message> @@ -1951,6 +1959,17 @@ The migration process will create a backup of the wallet before migrating. This </message> </context> <context> + <name>SignVerifyMessageDialog</name> + <message> + <source>You can sign messages/agreements with your legacy (P2PKH) addresses to prove you can receive bitcoins sent to them. Be careful not to sign anything vague or random, as phishing attacks may try to trick you into signing your identity over to them. Only sign fully-detailed statements you agree to.</source> + <translation type="unfinished">Вы можете подписывать сообщения/соглашения своими устаревшими (P2PKH) адресами, чтобы доказать, что вы можете получать биткоины на них. Будьте осторожны и не подписывайте непонятные или случайные сообщения, так как мошенники могут таким образом пытаться присвоить вашу личность. Подписывайте только такие сообщения, с которыми вы согласны вплоть до мелочей.</translation> + </message> + <message> + <source>The entered address does not refer to a legacy (P2PKH) key. Message signing for SegWit and other non-P2PKH address types is not supported in this version of %1. Please check the address and try again.</source> + <translation type="unfinished">Введенный адрес не относится к устаревшему (P2PKH) ключу. Подписывание сообщений для SegWit и других не--P2PKH типов адресов не поддерживается в этой версии %1. Пожалуйста, проверьте адрес и попробуйте ещё раз.</translation> + </message> + </context> +<context> <name>TransactionDesc</name> <message numerus="yes"> <source>matures in %n more block(s)</source> @@ -1961,6 +1980,10 @@ The migration process will create a backup of the wallet before migrating. This </translation> </message> <message> + <source>%1 (Certificate was not verified)</source> + <translation type="unfinished">%1 (Сертификат не был подтверждён)</translation> + </message> + <message> <source>Amount</source> <translation type="unfinished">Сумма</translation> </message> @@ -2634,5 +2657,21 @@ Unable to restore backup of wallet.</source> <source>Error: Unable to read all records in the database</source> <translation type="unfinished">Ошибка: не удалось прочитать все записи из базе данных</translation> </message> + <message> + <source>Failed to disconnect block.</source> + <translation type="unfinished">Не удалось отключить блок</translation> + </message> + <message> + <source>Failed to read block.</source> + <translation type="unfinished">Не удалось прочитать блок</translation> + </message> + <message> + <source>Failed to write block.</source> + <translation type="unfinished">Не удалось записать блок</translation> + </message> + <message> + <source>Wallet file creation failed: %s</source> + <translation type="unfinished">Не удалось создать кошелёк 1%s</translation> + </message> </context> </TS>
\ No newline at end of file diff --git a/src/qt/locale/bitcoin_sw.ts b/src/qt/locale/bitcoin_sw.ts index 199a0552b5..0497444d5e 100644 --- a/src/qt/locale/bitcoin_sw.ts +++ b/src/qt/locale/bitcoin_sw.ts @@ -325,7 +325,11 @@ Kutia sahihi kunawezekana tu kwa anwani za aina ya 'urithi'.</translation> <numerusform /> </translation> </message> - </context> + <message> + <source>default wallet</source> + <translation type="unfinished">mkoba chaguo-msingi</translation> + </message> +</context> <context> <name>BitcoinGUI</name> <message> @@ -414,10 +418,19 @@ Kutia sahihi kunawezekana tu kwa anwani za aina ya 'urithi'.</translation> <translation type="unfinished">&Chaguo...</translation> </message> <message> + <source>&Encrypt Wallet…</source> + <translation type="unfinished">&Simba Mkoba...</translation> + </message> + <message> <source>Encrypt the private keys that belong to your wallet</source> <translation type="unfinished">Funga funguo za siri zinazomiliki mkoba wako.</translation> </message> <message> + <source>&Backup Wallet…</source> + <translation type="unfinished">&Hifadhi Mkoba... +</translation> + </message> + <message> <source>&Change Passphrase…</source> <translation type="unfinished">&Badilisha Nenosiri...</translation> </message> @@ -430,10 +443,18 @@ Kutia sahihi kunawezekana tu kwa anwani za aina ya 'urithi'.</translation> <translation type="unfinished">Saini ujumbe na anwani zako za Bitcoin ili kuthibitisha umiliki wao.</translation> </message> <message> + <source>&Verify message…</source> + <translation type="unfinished">&Thibitisha ujumbe...</translation> + </message> + <message> <source>Verify messages to ensure they were signed with specified Bitcoin addresses</source> <translation type="unfinished">Hakikisha ujumbe umethibitishwa kuwa ulisainiwa na anwani za Bitcoin zilizotajwa</translation> </message> <message> + <source>&Load PSBT from file…</source> + <translation type="unfinished">&Pakia PSBT kutoka faili...</translation> + </message> + <message> <source>Open &URI…</source> <translation type="unfinished">Fungua &URI ...</translation> </message> @@ -509,6 +530,14 @@ Kutia sahihi kunawezekana tu kwa anwani za aina ya 'urithi'.</translation> </translation> </message> <message> + <source>%1 behind</source> + <translation type="unfinished">%1 nyuma</translation> + </message> + <message> + <source>Catching up…</source> + <translation type="unfinished">Inakamata...</translation> + </message> + <message> <source>Transactions after this will not yet be visible.</source> <translation type="unfinished">Shughuli baada ya hii bado hazitaonekana.</translation> </message> @@ -525,22 +554,135 @@ Kutia sahihi kunawezekana tu kwa anwani za aina ya 'urithi'.</translation> <translation type="unfinished">Habari</translation> </message> <message> + <source>Up to date</source> + <translation type="unfinished">Imesasishwa</translation> + </message> + <message> + <source>Load Partially Signed Bitcoin Transaction</source> + <translation type="unfinished">Pakia Muamala wa Bitcoin Uliosainiwa Kiasi</translation> + </message> + <message> + <source>Load PSBT from &clipboard…</source> + <translation type="unfinished">Pakia PSBT kutoka &clipboard...</translation> + </message> + <message> + <source>Load Partially Signed Bitcoin Transaction from clipboard</source> + <translation type="unfinished">Pakia Muamala wa Bitcoin Uliosainiwa Kiasi kutoka kwenye ubao wa kunakili</translation> + </message> + <message> + <source>Node window</source> + <translation type="unfinished">Dirisha la nodi</translation> + </message> + <message> + <source>Open node debugging and diagnostic console</source> + <translation type="unfinished">Fungua utatuzi wa nodi na koni ya uchunguzi</translation> + </message> + <message> + <source>&Sending addresses</source> + <translation type="unfinished">&Anwani za kutuma</translation> + </message> + <message> + <source>&Receiving addresses</source> + <translation type="unfinished">&Inapokea anwani</translation> + </message> + <message> + <source>Open a bitcoin: URI</source> + <translation type="unfinished">Fungua bitcoin: URI</translation> + </message> + <message> <source>Open Wallet</source> <translation type="unfinished">Fungua Pochi</translation> </message> <message> + <source>Open a wallet</source> + <translation type="unfinished">Fungua pochi</translation> + </message> + <message> <source>Close wallet</source> <translation type="unfinished">Funga pochi</translation> </message> <message> + <source>Restore Wallet…</source> + <extracomment>Name of the menu item that restores wallet from a backup file.</extracomment> + <translation type="unfinished">Rejesha Pochi...</translation> + </message> + <message> + <source>Restore a wallet from a backup file</source> + <extracomment>Status tip for Restore Wallet menu item</extracomment> + <translation type="unfinished">Rejesha mkoba kutoka kwa faili ya chelezo</translation> + </message> + <message> + <source>Close all wallets</source> + <translation type="unfinished">Funga pochi zote</translation> + </message> + <message> + <source>Migrate Wallet</source> + <translation type="unfinished">Hamisha Pochi</translation> + </message> + <message> + <source>Migrate a wallet</source> + <translation type="unfinished">Hamisha mkoba</translation> + </message> + <message> + <source>Show the %1 help message to get a list with possible Bitcoin command-line options</source> + <translation type="unfinished">Onyesha %1 ujumbe wa usaidizi ili kupata orodha na chaguo zinazowezekana za mstari wa amri za Bitcoin</translation> + </message> + <message> + <source>&Mask values</source> + <translation type="unfinished">&Funga maadili</translation> + </message> + <message> + <source>Mask the values in the Overview tab</source> + <translation type="unfinished">Ficha maadili kwenye kichupo cha Muhtasari</translation> + </message> + <message> + <source>No wallets available</source> + <translation type="unfinished">Hakuna pochi zinazopatikana</translation> + </message> + <message> + <source>Wallet Data</source> + <extracomment>Name of the wallet data file format.</extracomment> + <translation type="unfinished">Data ya Pochi</translation> + </message> + <message> + <source>Load Wallet Backup</source> + <extracomment>The title for Restore Wallet File Windows</extracomment> + <translation type="unfinished">Pakia Hifadhi Nakala ya Wallet</translation> + </message> + <message> + <source>Restore Wallet</source> + <extracomment>Title of pop-up window shown when the user is attempting to restore a wallet.</extracomment> + <translation type="unfinished">Rejesha Pochi</translation> + </message> + <message> <source>Wallet Name</source> <extracomment>Label of the input field where the name of the wallet is entered.</extracomment> <translation type="unfinished">Jina la Wallet</translation> </message> <message> + <source>&Window</source> + <translation type="unfinished">&Dirisha</translation> + </message> + <message> + <source>Zoom</source> + <translation type="unfinished">Kuza</translation> + </message> + <message> + <source>Main Window</source> + <translation type="unfinished">Dirisha Kuu</translation> + </message> + <message> + <source>%1 client</source> + <translation type="unfinished">%1 mteja</translation> + </message> + <message> <source>&Hide</source> <translation type="unfinished">&Ficha</translation> </message> + <message> + <source>S&how</source> + <translation type="unfinished">Jinsi & jinsi</translation> + </message> <message numerus="yes"> <source>%n active connection(s) to Bitcoin network.</source> <extracomment>A substring of the tooltip.</extracomment> @@ -550,15 +692,99 @@ Kutia sahihi kunawezekana tu kwa anwani za aina ya 'urithi'.</translation> </translation> </message> <message> + <source>Click for more actions.</source> + <extracomment>A substring of the tooltip. "More actions" are available via the context menu.</extracomment> + <translation type="unfinished">Bofya kwa vitendo zaidi.</translation> + </message> + <message> + <source>Show Peers tab</source> + <extracomment>A context menu item. The "Peers tab" is an element of the "Node window".</extracomment> + <translation type="unfinished">Onyesha kichupo cha Marika</translation> + </message> + <message> + <source>Disable network activity</source> + <extracomment>A context menu item.</extracomment> + <translation type="unfinished">Zima shughuli za mtandao</translation> + </message> + <message> + <source>Enable network activity</source> + <extracomment>A context menu item. The network activity was disabled previously.</extracomment> + <translation type="unfinished">Washa shughuli za mtandao</translation> + </message> + <message> + <source>Pre-syncing Headers (%1%)…</source> + <translation type="unfinished">Kusawazisha Vichwa vya awali (%1%)...</translation> + </message> + <message> + <source>Error creating wallet</source> + <translation type="unfinished">Hitilafu unapounda pochi</translation> + </message> + <message> + <source>Cannot create new wallet, the software was compiled without sqlite support (required for descriptor wallets)</source> + <translation type="unfinished">Haiwezi kuunda pochi mpya, programu iliundwa bila usaidizi wa sqlite (inahitajika kwa pochi za maelezo)</translation> + </message> + <message> <source>Error: %1</source> <translation type="unfinished">Kosa: %1</translation> </message> <message> + <source>Warning: %1</source> + <translation type="unfinished">Onyo: %1</translation> + </message> + <message> + <source>Date: %1 +</source> + <translation type="unfinished">Tarehe: %1</translation> + </message> + <message> + <source>Amount: %1 +</source> + <translation type="unfinished">Kiasi: %1 +</translation> + </message> + <message> + <source>Wallet: %1 +</source> + <translation type="unfinished">Pochi: %1 +</translation> + </message> + <message> + <source>Type: %1 +</source> + <translation type="unfinished">Aina: %1</translation> + </message> + <message> <source>Label: %1 </source> <translation type="unfinished">Chapa: %1 </translation> </message> + <message> + <source>Address: %1 +</source> + <translation type="unfinished">Anwani: %1 +</translation> + </message> + <message> + <source>Sent transaction</source> + <translation type="unfinished">Umetuma muamala</translation> + </message> + <message> + <source>Incoming transaction</source> + <translation type="unfinished">Muamala unaoingia</translation> + </message> + <message> + <source>HD key generation is <b>enabled</b></source> + <translation type="unfinished">Uzalishaji wa ufunguo wa HD ni <b>kuwezeshwa </b></translation> + </message> + <message> + <source>HD key generation is <b>disabled</b></source> + <translation type="unfinished">Uzalishaji wa ufunguo wa HD ni <b>kutowezeshwa</b></translation> + </message> + <message> + <source>Private key <b>disabled</b></source> + <translation type="unfinished">Ufunguo wa kibinafsi <b> umezimwa </b></translation> + </message> </context> <context> <name>CoinControlDialog</name> @@ -592,6 +818,13 @@ Kutia sahihi kunawezekana tu kwa anwani za aina ya 'urithi'.</translation> </message> </context> <context> + <name>MigrateWalletActivity</name> + <message> + <source>Migrate Wallet</source> + <translation type="unfinished">Hamisha Pochi</translation> + </message> + </context> +<context> <name>OpenWalletActivity</name> <message> <source>Open Wallet</source> @@ -602,6 +835,11 @@ Kutia sahihi kunawezekana tu kwa anwani za aina ya 'urithi'.</translation> <context> <name>RestoreWalletActivity</name> <message> + <source>Restore Wallet</source> + <extracomment>Title of progress window which is displayed when wallets are being restored.</extracomment> + <translation type="unfinished">Rejesha Pochi</translation> + </message> + <message> <source>Restore wallet warning</source> <extracomment>Title of message box which is displayed when the wallet is restored with some warning.</extracomment> <translation type="unfinished">Rejesha onyo la pochi</translation> @@ -617,6 +855,10 @@ Kutia sahihi kunawezekana tu kwa anwani za aina ya 'urithi'.</translation> <source>Closing the wallet for too long can result in having to resync the entire chain if pruning is enabled.</source> <translation type="unfinished">Kufunga pochi kwa muda mrefu sana kunaweza kusababisha kusawazisha tena mnyororo mzima ikiwa upogoaji umewezeshwa.</translation> </message> + <message> + <source>Close all wallets</source> + <translation type="unfinished">Funga pochi zote</translation> + </message> </context> <context> <name>CreateWalletDialog</name> @@ -730,6 +972,10 @@ Kutia sahihi kunawezekana tu kwa anwani za aina ya 'urithi'.</translation> <context> <name>OptionsDialog</name> <message> + <source>&Window</source> + <translation type="unfinished">&Dirisha</translation> + </message> + <message> <source>Error</source> <translation type="unfinished">Onyo</translation> </message> @@ -750,6 +996,13 @@ Kutia sahihi kunawezekana tu kwa anwani za aina ya 'urithi'.</translation> </message> </context> <context> + <name>RPCConsole</name> + <message> + <source>Node window</source> + <translation type="unfinished">Dirisha la nodi</translation> + </message> + </context> +<context> <name>ReceiveCoinsDialog</name> <message> <source>&Label:</source> @@ -925,6 +1178,11 @@ Kutia sahihi kunawezekana tu kwa anwani za aina ya 'urithi'.</translation> <source>Export the data in the current tab to a file</source> <translation type="unfinished">Toa data katika kichupo cha sasa hadi kwenye faili</translation> </message> + <message> + <source>Wallet Data</source> + <extracomment>Name of the wallet data file format.</extracomment> + <translation type="unfinished">Data ya Pochi</translation> + </message> </context> <context> <name>bitcoin-core</name> diff --git a/src/qt/locale/bitcoin_th.ts b/src/qt/locale/bitcoin_th.ts index 3706a7bd98..464e39cbac 100644 --- a/src/qt/locale/bitcoin_th.ts +++ b/src/qt/locale/bitcoin_th.ts @@ -1,5 +1,12 @@ <TS version="2.1" language="th"> <context> + <name>AskPassphraseDialog</name> + <message> + <source>Back</source> + <translation type="unfinished">ย้อนกลับ</translation> + </message> + </context> +<context> <name>QObject</name> <message> <source>%1 didn't yet exit safely…</source> diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index 7580f6b47a..e752628d98 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/modaloverlay.h> #include <qt/forms/ui_modaloverlay.h> @@ -31,7 +31,7 @@ ModalOverlay::ModalOverlay(bool enable_wallet, QWidget* parent) setVisible(false); if (!enable_wallet) { ui->infoText->setVisible(false); - ui->infoTextStrong->setText(tr("%1 is currently syncing. It will download headers and blocks from peers and validate them until reaching the tip of the block chain.").arg(PACKAGE_NAME)); + ui->infoTextStrong->setText(tr("%1 is currently syncing. It will download headers and blocks from peers and validate them until reaching the tip of the block chain.").arg(CLIENT_NAME)); } m_animation.setTargetObject(this); diff --git a/src/qt/notificator.cpp b/src/qt/notificator.cpp index 85bdeee49a..af97bb2143 100644 --- a/src/qt/notificator.cpp +++ b/src/qt/notificator.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/notificator.h> diff --git a/src/qt/notificator.h b/src/qt/notificator.h index 8808716aa4..932cfc4651 100644 --- a/src/qt/notificator.h +++ b/src/qt/notificator.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_NOTIFICATOR_H #define BITCOIN_QT_NOTIFICATOR_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <QIcon> #include <QObject> diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 4db2d6016c..f86b5229db 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/optionsdialog.h> #include <qt/forms/ui_optionsdialog.h> @@ -95,8 +95,7 @@ OptionsDialog::OptionsDialog(QWidget* parent, bool enableWallet) ui->verticalLayout->setStretchFactor(ui->tabWidget, 1); /* Main elements init */ - ui->databaseCache->setMinimum(nMinDbCache); - ui->databaseCache->setMaximum(nMaxDbCache); + ui->databaseCache->setRange(nMinDbCache, std::numeric_limits<int>::max()); ui->threadsScriptVerif->setMinimum(-GetNumCores()); ui->threadsScriptVerif->setMaximum(MAX_SCRIPTCHECK_THREADS); ui->pruneWarning->setVisible(false); @@ -106,13 +105,6 @@ OptionsDialog::OptionsDialog(QWidget* parent, bool enableWallet) connect(ui->prune, &QPushButton::toggled, ui->pruneSize, &QWidget::setEnabled); /* Network elements init */ -#ifndef USE_UPNP - ui->mapPortUpnp->setEnabled(false); -#endif -#ifndef USE_NATPMP - ui->mapPortNatpmp->setEnabled(false); -#endif - ui->proxyIp->setEnabled(false); ui->proxyPort->setEnabled(false); ui->proxyPort->setValidator(new QIntValidator(1, 65535, this)); @@ -147,7 +139,7 @@ OptionsDialog::OptionsDialog(QWidget* parent, bool enableWallet) } #ifdef ENABLE_EXTERNAL_SIGNER - ui->externalSignerPath->setToolTip(ui->externalSignerPath->toolTip().arg(PACKAGE_NAME)); + ui->externalSignerPath->setToolTip(ui->externalSignerPath->toolTip().arg(CLIENT_NAME)); #else //: "External signing" means using devices such as hardware wallets. ui->externalSignerPath->setToolTip(tr("Compiled without external signing support (required for external signing)")); @@ -156,12 +148,12 @@ OptionsDialog::OptionsDialog(QWidget* parent, bool enableWallet) /* Display elements init */ QDir translations(":translations"); - ui->bitcoinAtStartup->setToolTip(ui->bitcoinAtStartup->toolTip().arg(PACKAGE_NAME)); - ui->bitcoinAtStartup->setText(ui->bitcoinAtStartup->text().arg(PACKAGE_NAME)); + ui->bitcoinAtStartup->setToolTip(ui->bitcoinAtStartup->toolTip().arg(CLIENT_NAME)); + ui->bitcoinAtStartup->setText(ui->bitcoinAtStartup->text().arg(CLIENT_NAME)); - ui->openBitcoinConfButton->setToolTip(ui->openBitcoinConfButton->toolTip().arg(PACKAGE_NAME)); + ui->openBitcoinConfButton->setToolTip(ui->openBitcoinConfButton->toolTip().arg(CLIENT_NAME)); - ui->lang->setToolTip(ui->lang->toolTip().arg(PACKAGE_NAME)); + ui->lang->setToolTip(ui->lang->toolTip().arg(CLIENT_NAME)); ui->lang->addItem(QString("(") + tr("default") + QString(")"), QVariant("")); for (const QString &langStr : translations.entryList()) { @@ -170,8 +162,15 @@ OptionsDialog::OptionsDialog(QWidget* parent, bool enableWallet) /** check if the locale name consists of 2 parts (language_country) */ if(langStr.contains("_")) { - /** display language strings as "native language - native country (locale name)", e.g. "Deutsch - Deutschland (de)" */ - ui->lang->addItem(locale.nativeLanguageName() + QString(" - ") + locale.nativeCountryName() + QString(" (") + langStr + QString(")"), QVariant(langStr)); + /** display language strings as "native language - native country/territory (locale name)", e.g. "Deutsch - Deutschland (de)" */ + ui->lang->addItem(locale.nativeLanguageName() + QString(" - ") + +#if (QT_VERSION >= QT_VERSION_CHECK(6, 2, 0)) + locale.nativeTerritoryName() + +#else + locale.nativeCountryName() + +#endif + QString(" (") + langStr + QString(")"), QVariant(langStr)); + } else { @@ -297,7 +296,6 @@ void OptionsDialog::setMapper() mapper->addMapping(ui->m_enable_psbt_controls, OptionsModel::EnablePSBTControls); /* Network */ - mapper->addMapping(ui->mapPortUpnp, OptionsModel::MapPortUPnP); mapper->addMapping(ui->mapPortNatpmp, OptionsModel::MapPortNatpmp); mapper->addMapping(ui->allowIncoming, OptionsModel::Listen); mapper->addMapping(ui->enableServer, OptionsModel::Server); diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 0c21c6748d..a1dbc6248c 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/optionsmodel.h> @@ -41,7 +41,6 @@ static const char* SettingName(OptionsModel::OptionID option) case OptionsModel::ThreadsScriptVerif: return "par"; case OptionsModel::SpendZeroConfChange: return "spendzeroconfchange"; case OptionsModel::ExternalSignerPath: return "signer"; - case OptionsModel::MapPortUPnP: return "upnp"; case OptionsModel::MapPortNatpmp: return "natpmp"; case OptionsModel::Listen: return "listen"; case OptionsModel::Server: return "server"; @@ -215,7 +214,7 @@ bool OptionsModel::Init(bilingual_str& error) // These are shared with the core or have a command-line parameter // and we want command-line parameters to overwrite the GUI settings. - for (OptionID option : {DatabaseCache, ThreadsScriptVerif, SpendZeroConfChange, ExternalSignerPath, MapPortUPnP, + for (OptionID option : {DatabaseCache, ThreadsScriptVerif, SpendZeroConfChange, ExternalSignerPath, MapPortNatpmp, Listen, Server, Prune, ProxyUse, ProxyUseTor, Language}) { std::string setting = SettingName(option); if (node().isSettingIgnored(setting)) addOverriddenOption("-" + setting); @@ -320,10 +319,15 @@ static ProxySetting ParseProxyString(const QString& proxy) if (proxy.isEmpty()) { return default_val; } - // contains IP at index 0 and port at index 1 - QStringList ip_port = GUIUtil::SplitSkipEmptyParts(proxy, ":"); - if (ip_port.size() == 2) { - return {true, ip_port.at(0), ip_port.at(1)}; + uint16_t port{0}; + std::string hostname; + if (SplitHostPort(proxy.toStdString(), port, hostname) && port != 0) { + // Valid and port within the valid range + // Check if the hostname contains a colon, indicating an IPv6 address + if (hostname.find(':') != std::string::npos) { + hostname = "[" + hostname + "]"; // Wrap IPv6 address in brackets + } + return {true, QString::fromStdString(hostname), QString::number(port)}; } else { // Invalid: return default return default_val; } @@ -407,18 +411,8 @@ QVariant OptionsModel::getOption(OptionID option, const std::string& suffix) con return m_show_tray_icon; case MinimizeToTray: return fMinimizeToTray; - case MapPortUPnP: -#ifdef USE_UPNP - return SettingToBool(setting(), DEFAULT_UPNP); -#else - return false; -#endif // USE_UPNP case MapPortNatpmp: -#ifdef USE_NATPMP return SettingToBool(setting(), DEFAULT_NATPMP); -#else - return false; -#endif // USE_NATPMP case MinimizeOnClose: return fMinimizeOnClose; @@ -529,16 +523,10 @@ bool OptionsModel::setOption(OptionID option, const QVariant& value, const std:: fMinimizeToTray = value.toBool(); settings.setValue("fMinimizeToTray", fMinimizeToTray); break; - case MapPortUPnP: // core option - can be changed on-the-fly - if (changed()) { - update(value.toBool()); - node().mapPort(value.toBool(), getOption(MapPortNatpmp).toBool()); - } - break; case MapPortNatpmp: // core option - can be changed on-the-fly if (changed()) { update(value.toBool()); - node().mapPort(getOption(MapPortUPnP).toBool(), value.toBool()); + node().mapPort(value.toBool()); } break; case MinimizeOnClose: @@ -788,7 +776,6 @@ void OptionsModel::checkAndMigrate() migrate_setting(SpendZeroConfChange, "bSpendZeroConfChange"); migrate_setting(ExternalSignerPath, "external_signer_path"); #endif - migrate_setting(MapPortUPnP, "fUseUPnP"); migrate_setting(MapPortNatpmp, "fUseNatpmp"); migrate_setting(Listen, "fListen"); migrate_setting(Server, "server"); @@ -802,7 +789,7 @@ void OptionsModel::checkAndMigrate() // In case migrating QSettings caused any settings value to change, rerun // parameter interaction code to update other settings. This is particularly - // important for the -listen setting, which should cause -listenonion, -upnp, + // important for the -listen setting, which should cause -listenonion // and other settings to default to false if it was set to false. // (https://github.com/bitcoin-core/gui/issues/567). node().initParameterInteraction(); diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index b5ea6c783e..c24b35ed79 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -50,7 +50,6 @@ public: StartAtStartup, // bool ShowTrayIcon, // bool MinimizeToTray, // bool - MapPortUPnP, // bool MapPortNatpmp, // bool MinimizeOnClose, // bool ProxyUse, // bool diff --git a/src/qt/qrimagewidget.cpp b/src/qt/qrimagewidget.cpp index f6e712a047..e912dafa60 100644 --- a/src/qt/qrimagewidget.cpp +++ b/src/qt/qrimagewidget.cpp @@ -15,7 +15,7 @@ #include <QMouseEvent> #include <QPainter> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #ifdef USE_QRCODE #include <qrencode.h> diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index b4322ddc0f..a5ee6583e0 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -14,7 +14,7 @@ #include <QDialog> #include <QString> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep ReceiveRequestDialog::ReceiveRequestDialog(QWidget* parent) : QDialog(parent, GUIUtil::dialog_flags), diff --git a/src/qt/res/bitcoin-qt-res.rc b/src/qt/res/bitcoin-qt-res.rc index e590b407b0..9a379f06d4 100644 --- a/src/qt/res/bitcoin-qt-res.rc +++ b/src/qt/res/bitcoin-qt-res.rc @@ -1,5 +1,7 @@ IDI_ICON1 ICON DISCARDABLE "icons/bitcoin.ico" IDI_ICON2 ICON DISCARDABLE "icons/bitcoin_testnet.ico" +IDI_ICON3 ICON DISCARDABLE "icons/bitcoin_signet.ico" +IDI_ICON4 ICON DISCARDABLE "icons/bitcoin_testnet.ico" // testnet4 #include <windows.h> // needed for VERSIONINFO #include "../../clientversion.h" // holds the needed client version information @@ -20,13 +22,13 @@ BEGIN BLOCK "040904E4" // U.S. English - multilingual (hex) BEGIN VALUE "CompanyName", "Bitcoin" - VALUE "FileDescription", PACKAGE_NAME " (GUI node for Bitcoin)" + VALUE "FileDescription", CLIENT_NAME " (GUI node for Bitcoin)" VALUE "FileVersion", VER_FILEVERSION_STR VALUE "InternalName", "bitcoin-qt" VALUE "LegalCopyright", COPYRIGHT_STR VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." VALUE "OriginalFilename", "bitcoin-qt.exe" - VALUE "ProductName", PACKAGE_NAME + VALUE "ProductName", CLIENT_NAME VALUE "ProductVersion", VER_PRODUCTVERSION_STR END END diff --git a/src/qt/res/icons/bitcoin_signet.ico b/src/qt/res/icons/bitcoin_signet.ico Binary files differnew file mode 100644 index 0000000000..fb9be5153b --- /dev/null +++ b/src/qt/res/icons/bitcoin_signet.ico diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index ae3f9aa686..9d7c17ac91 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/rpcconsole.h> #include <qt/forms/ui_debugwindow.h> @@ -16,7 +16,9 @@ #include <qt/guiutil.h> #include <qt/peertablesortproxy.h> #include <qt/platformstyle.h> +#ifdef ENABLE_WALLET #include <qt/walletmodel.h> +#endif // ENABLE_WALLET #include <rpc/client.h> #include <rpc/server.h> #include <util/strencodings.h> @@ -534,7 +536,7 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty ui->peerHighBandwidthLabel->setToolTip(ui->peerHighBandwidthLabel->toolTip().arg(hb_list)); ui->dataDir->setToolTip(ui->dataDir->toolTip().arg(QString(nonbreaking_hyphen) + "datadir")); ui->blocksDir->setToolTip(ui->blocksDir->toolTip().arg(QString(nonbreaking_hyphen) + "blocksdir")); - ui->openDebugLogfileButton->setToolTip(ui->openDebugLogfileButton->toolTip().arg(PACKAGE_NAME)); + ui->openDebugLogfileButton->setToolTip(ui->openDebugLogfileButton->toolTip().arg(CLIENT_NAME)); if (platformStyle->getImagesOnButtons()) { ui->openDebugLogfileButton->setIcon(platformStyle->SingleColorIcon(":/icons/export")); @@ -912,7 +914,7 @@ void RPCConsole::clear(bool keep_prompt) "%7WARNING: Scammers have been active, telling users to type" " commands here, stealing their wallet contents. Do not use this console" " without fully understanding the ramifications of a command.%8") - .arg(PACKAGE_NAME, + .arg(CLIENT_NAME, "<b>" + ui->clearButton->shortcut().toString(QKeySequence::NativeText) + "</b>", "<b>" + ui->fontBiggerButton->shortcut().toString(QKeySequence::NativeText) + "</b>", "<b>" + ui->fontSmallerButton->shortcut().toString(QKeySequence::NativeText) + "</b>", diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index 4747e611d0..894ecb1fdf 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_RPCCONSOLE_H #define BITCOIN_QT_RPCCONSOLE_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/clientmodel.h> #include <qt/guiutil.h> diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 03173ec80e..0ee1b359fa 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/sendcoinsdialog.h> #include <qt/forms/ui_sendcoinsdialog.h> @@ -210,7 +210,7 @@ void SendCoinsDialog::setModel(WalletModel *_model) } } else if (model->wallet().privateKeysDisabled()) { ui->sendButton->setText(tr("Cr&eate Unsigned")); - ui->sendButton->setToolTip(tr("Creates a Partially Signed Bitcoin Transaction (PSBT) for use with e.g. an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(PACKAGE_NAME)); + ui->sendButton->setToolTip(tr("Creates a Partially Signed Bitcoin Transaction (PSBT) for use with e.g. an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(CLIENT_NAME)); } // set the smartfee-sliders default value (wallets default conf.target or last stored value) @@ -332,12 +332,12 @@ bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informa /*: Text to inform a user attempting to create a transaction of their current options. At this stage, a user can only create a PSBT. This string is displayed when private keys are disabled and an external signer is not available. */ - question_string.append(tr("Please, review your transaction proposal. This will produce a Partially Signed Bitcoin Transaction (PSBT) which you can save or copy and then sign with e.g. an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(PACKAGE_NAME)); + question_string.append(tr("Please, review your transaction proposal. This will produce a Partially Signed Bitcoin Transaction (PSBT) which you can save or copy and then sign with e.g. an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(CLIENT_NAME)); } else if (model->getOptionsModel()->getEnablePSBTControls()) { /*: Text to inform a user attempting to create a transaction of their current options. At this stage, a user can send their transaction or create a PSBT. This string is displayed when both private keys and PSBT controls are enabled. */ - question_string.append(tr("Please, review your transaction. You can create and send this transaction or create a Partially Signed Bitcoin Transaction (PSBT), which you can save or copy and then sign with, e.g., an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(PACKAGE_NAME)); + question_string.append(tr("Please, review your transaction. You can create and send this transaction or create a Partially Signed Bitcoin Transaction (PSBT), which you can save or copy and then sign with, e.g., an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(CLIENT_NAME)); } else { /*: Text to prompt a user to review the details of the transaction they are attempting to send. */ question_string.append(tr("Please, review your transaction.")); diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 012186ee4d..e46fe06162 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -11,7 +11,7 @@ #include <qt/walletmodel.h> #include <common/signmessage.h> // For MessageSign(), MessageVerify() -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <key_io.h> #include <wallet/wallet.h> @@ -124,7 +124,7 @@ void SignVerifyMessageDialog::on_signMessageButton_SM_clicked() if (!pkhash) { ui->addressIn_SM->setValid(false); ui->statusLabel_SM->setStyleSheet("QLabel { color: red; }"); - ui->statusLabel_SM->setText(tr("The entered address does not refer to a legacy (P2PKH) key. Message signing for SegWit and other non-P2PKH address types is not supported in this version of %1. Please check the address and try again.").arg(PACKAGE_NAME)); + ui->statusLabel_SM->setText(tr("The entered address does not refer to a legacy (P2PKH) key. Message signing for SegWit and other non-P2PKH address types is not supported in this version of %1. Please check the address and try again.").arg(CLIENT_NAME)); return; } @@ -222,7 +222,7 @@ void SignVerifyMessageDialog::on_verifyMessageButton_VM_clicked() return; case MessageVerificationResult::ERR_ADDRESS_NO_KEY: ui->addressIn_VM->setValid(false); - ui->statusLabel_VM->setText(tr("The entered address does not refer to a legacy (P2PKH) key. Message signing for SegWit and other non-P2PKH address types is not supported in this version of %1. Please check the address and try again.").arg(PACKAGE_NAME)); + ui->statusLabel_VM->setText(tr("The entered address does not refer to a legacy (P2PKH) key. Message signing for SegWit and other non-P2PKH address types is not supported in this version of %1. Please check the address and try again.").arg(CLIENT_NAME)); return; case MessageVerificationResult::ERR_MALFORMED_SIGNATURE: ui->signatureIn_VM->setValid(false); diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index ffd6689910..ccaa201811 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/splashscreen.h> @@ -39,7 +39,7 @@ SplashScreen::SplashScreen(const NetworkStyle* networkStyle) devicePixelRatio = static_cast<QGuiApplication*>(QCoreApplication::instance())->devicePixelRatio(); // define text to place - QString titleText = PACKAGE_NAME; + QString titleText = CLIENT_NAME; QString versionText = QString("Version %1").arg(QString::fromStdString(FormatFullVersion())); QString copyrightText = QString::fromUtf8(CopyrightHolders(strprintf("\xc2\xA9 %u-%u ", 2009, COPYRIGHT_YEAR)).c_str()); const QString& titleAddText = networkStyle->getTitleAddText(); diff --git a/src/qt/test/CMakeLists.txt b/src/qt/test/CMakeLists.txt index 582ed71466..e1e617661b 100644 --- a/src/qt/test/CMakeLists.txt +++ b/src/qt/test/CMakeLists.txt @@ -2,6 +2,8 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or https://opensource.org/license/mit/. +set(CMAKE_AUTOMOC_MOC_OPTIONS "-p${CMAKE_CURRENT_SOURCE_DIR}") + add_executable(test_bitcoin-qt apptests.cpp optiontests.cpp @@ -32,14 +34,6 @@ if(ENABLE_WALLET) ) endif() -if(NOT QT_IS_STATIC) - add_custom_command( - TARGET test_bitcoin-qt POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_PROPERTY:Qt5::QMinimalIntegrationPlugin,LOCATION_$<UPPER_CASE:$<CONFIG>>> $<TARGET_FILE_DIR:test_bitcoin-qt>/plugins/platforms - VERBATIM - ) -endif() - add_test(NAME test_bitcoin-qt COMMAND test_bitcoin-qt ) diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index f7d66f316e..3d5cb4a863 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -222,8 +222,8 @@ void AddressBookTests::addressBookTests() // framework when it tries to look up unimplemented cocoa functions, // and fails to handle returned nulls // (https://bugreports.qt.io/browse/QTBUG-49686). - QWARN("Skipping AddressBookTests on mac build with 'minimal' platform set due to Qt bugs. To run AppTests, invoke " - "with 'QT_QPA_PLATFORM=cocoa test_bitcoin-qt' on mac, or else use a linux or windows build."); + qWarning() << "Skipping AddressBookTests on mac build with 'minimal' platform set due to Qt bugs. To run AppTests, invoke " + "with 'QT_QPA_PLATFORM=cocoa test_bitcoin-qt' on mac, or else use a linux or windows build."; return; } #endif diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 10abcb00eb..73e04b55c8 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -60,8 +60,8 @@ void AppTests::appTests() // framework when it tries to look up unimplemented cocoa functions, // and fails to handle returned nulls // (https://bugreports.qt.io/browse/QTBUG-49686). - QWARN("Skipping AppTests on mac build with 'minimal' platform set due to Qt bugs. To run AppTests, invoke " - "with 'QT_QPA_PLATFORM=cocoa test_bitcoin-qt' on mac, or else use a linux or windows build."); + qWarning() << "Skipping AppTests on mac build with 'minimal' platform set due to Qt bugs. To run AppTests, invoke " + "with 'QT_QPA_PLATFORM=cocoa test_bitcoin-qt' on mac, or else use a linux or windows build."; return; } #endif diff --git a/src/qt/test/optiontests.cpp b/src/qt/test/optiontests.cpp index 0f82f65f3e..f8a1da8412 100644 --- a/src/qt/test/optiontests.cpp +++ b/src/qt/test/optiontests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <common/args.h> #include <init.h> @@ -37,7 +37,6 @@ void OptionTests::migrateSettings() QSettings settings; settings.setValue("nDatabaseCache", 600); settings.setValue("nThreadsScriptVerif", 12); - settings.setValue("fUseUPnP", false); settings.setValue("fListen", false); settings.setValue("bPrune", true); settings.setValue("nPruneSize", 3); @@ -50,7 +49,6 @@ void OptionTests::migrateSettings() QVERIFY(settings.contains("nDatabaseCache")); QVERIFY(settings.contains("nThreadsScriptVerif")); - QVERIFY(settings.contains("fUseUPnP")); QVERIFY(settings.contains("fListen")); QVERIFY(settings.contains("bPrune")); QVERIFY(settings.contains("nPruneSize")); @@ -64,7 +62,6 @@ void OptionTests::migrateSettings() QVERIFY(options.Init(error)); QVERIFY(!settings.contains("nDatabaseCache")); QVERIFY(!settings.contains("nThreadsScriptVerif")); - QVERIFY(!settings.contains("fUseUPnP")); QVERIFY(!settings.contains("fListen")); QVERIFY(!settings.contains("bPrune")); QVERIFY(!settings.contains("nPruneSize")); @@ -76,7 +73,7 @@ void OptionTests::migrateSettings() std::ifstream file(gArgs.GetDataDirNet() / "settings.json"); std::string default_warning = strprintf("This file is automatically generated and updated by %s. Please do not edit this file while the node " "is running, as any changes might be ignored or overwritten.", - PACKAGE_NAME); + CLIENT_NAME); QCOMPARE(std::string(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>()).c_str(), "{\n" " \"_warning_\": \""+ default_warning+"\",\n" " \"dbcache\": \"600\",\n" diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 72e8055425..0797d31a71 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -127,6 +127,11 @@ void RPCNestedTests::rpcNestedTests() RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest( abc , cba )"); QVERIFY(result == "[\"abc\",\"cba\"]"); +// Handle deprecated macro, can be removed once minimum Qt is at least 6.3.0. +#if (QT_VERSION >= QT_VERSION_CHECK(6, 3, 0)) +#undef QVERIFY_EXCEPTION_THROWN +#define QVERIFY_EXCEPTION_THROWN(expression, exceptiontype) QVERIFY_THROWS_EXCEPTION(exceptiontype, expression) +#endif QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo() .\n"), std::runtime_error); //invalid syntax QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo() getblockchaininfo()"), std::runtime_error); //invalid syntax RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo("); //tolerate non closing brackets if we have no arguments diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 958cc7ae88..f3430c7822 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <interfaces/init.h> #include <interfaces/node.h> @@ -54,11 +54,10 @@ int main(int argc, char* argv[]) gArgs.ForceSetArg("-discover", "0"); gArgs.ForceSetArg("-dnsseed", "0"); gArgs.ForceSetArg("-fixedseeds", "0"); - gArgs.ForceSetArg("-upnp", "0"); gArgs.ForceSetArg("-natpmp", "0"); std::string error; - if (!gArgs.ReadConfigFiles(error, true)) QWARN(error.c_str()); + if (!gArgs.ReadConfigFiles(error, true)) qWarning() << error.c_str(); // Prefer the "minimal" platform for the test instead of the normal default // platform ("xcb", "windows", or "cocoa") so tests can't unintentionally diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 6a573d284c..98dfe12f08 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -475,8 +475,8 @@ void WalletTests::walletTests() // framework when it tries to look up unimplemented cocoa functions, // and fails to handle returned nulls // (https://bugreports.qt.io/browse/QTBUG-49686). - QWARN("Skipping WalletTests on mac build with 'minimal' platform set due to Qt bugs. To run AppTests, invoke " - "with 'QT_QPA_PLATFORM=cocoa test_bitcoin-qt' on mac, or else use a linux or windows build."); + qWarning() << "Skipping WalletTests on mac build with 'minimal' platform set due to Qt bugs. To run AppTests, invoke " + "with 'QT_QPA_PLATFORM=cocoa test_bitcoin-qt' on mac, or else use a linux or windows build."; return; } #endif diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index f261c6409d..86c3770407 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/utilitydialog.h> @@ -33,11 +33,11 @@ HelpMessageDialog::HelpMessageDialog(QWidget *parent, bool about) : { ui->setupUi(this); - QString version = QString{PACKAGE_NAME} + " " + tr("version") + " " + QString::fromStdString(FormatFullVersion()); + QString version = QString{CLIENT_NAME} + " " + tr("version") + " " + QString::fromStdString(FormatFullVersion()); if (about) { - setWindowTitle(tr("About %1").arg(PACKAGE_NAME)); + setWindowTitle(tr("About %1").arg(CLIENT_NAME)); std::string licenseInfo = LicenseInfo(); /// HTML-format the license message from the core @@ -56,8 +56,11 @@ HelpMessageDialog::HelpMessageDialog(QWidget *parent, bool about) : ui->helpMessage->setVisible(false); } else { setWindowTitle(tr("Command-line options")); - QString header = "Usage: bitcoin-qt [command-line options] [URI]\n\n" - "Optional URI is a Bitcoin address in BIP21 URI format.\n"; + QString header = "The bitcoin-qt application provides a graphical interface for interacting with " CLIENT_NAME ".\n\n" + "It combines the core functionalities of bitcoind with a user-friendly interface for wallet management, transaction history, and network statistics.\n\n" + "It is suitable for users who prefer a graphical over a command-line interface.\n\n" + "You can optionally specify a payment [URI], in e.g. the BIP21 URI format.\n\n" + "Usage: bitcoin-qt [options] [URI]\n\n"; QTextCursor cursor(ui->helpMessage->document()); cursor.insertText(version); cursor.insertBlock(); @@ -141,7 +144,7 @@ ShutdownWindow::ShutdownWindow(QWidget *parent, Qt::WindowFlags f): { QVBoxLayout *layout = new QVBoxLayout(); layout->addWidget(new QLabel( - tr("%1 is shutting down…").arg(PACKAGE_NAME) + "<br /><br />" + + tr("%1 is shutting down…").arg(CLIENT_NAME) + "<br /><br />" + tr("Do not shut down the computer until this window disappears."))); setLayout(layout); diff --git a/src/qt/winshutdownmonitor.cpp b/src/qt/winshutdownmonitor.cpp index 0b5278c192..9ccd7028da 100644 --- a/src/qt/winshutdownmonitor.cpp +++ b/src/qt/winshutdownmonitor.cpp @@ -12,7 +12,11 @@ // If we don't want a message to be processed by Qt, return true and set result to // the value that the window procedure should return. Otherwise return false. +#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)) +bool WinShutdownMonitor::nativeEventFilter(const QByteArray &eventType, void *pMessage, qintptr *pnResult) +#else bool WinShutdownMonitor::nativeEventFilter(const QByteArray &eventType, void *pMessage, long *pnResult) +#endif { Q_UNUSED(eventType); diff --git a/src/qt/winshutdownmonitor.h b/src/qt/winshutdownmonitor.h index 060d8546e3..a8b626065d 100644 --- a/src/qt/winshutdownmonitor.h +++ b/src/qt/winshutdownmonitor.h @@ -20,7 +20,11 @@ public: WinShutdownMonitor(std::function<void()> shutdown_fn) : m_shutdown_fn{std::move(shutdown_fn)} {} /** Implements QAbstractNativeEventFilter interface for processing Windows messages */ +#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)) + bool nativeEventFilter(const QByteArray &eventType, void *pMessage, qintptr *pnResult) override; +#else bool nativeEventFilter(const QByteArray &eventType, void *pMessage, long *pnResult) override; +#endif /** Register the reason for blocking shutdown on Windows to allow clean client exit */ static void registerShutdownBlockReason(const QString& strReason, const HWND& mainWinId); diff --git a/src/random.cpp b/src/random.cpp index c89ee9f38b..4265c48022 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <random.h> @@ -34,7 +34,7 @@ #include <sys/time.h> #endif -#if defined(HAVE_GETRANDOM) || (defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX)) +#if defined(HAVE_GETRANDOM) || (defined(HAVE_GETENTROPY_RAND) && defined(__APPLE__)) #include <sys/random.h> #endif @@ -387,7 +387,7 @@ void GetOSRand(unsigned char *ent32) The function call is always successful. */ arc4random_buf(ent32, NUM_OS_RANDOM_BYTES); -#elif defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX) +#elif defined(HAVE_GETENTROPY_RAND) && defined(__APPLE__) if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) { RandFailure(); } @@ -599,7 +599,7 @@ void SeedPeriodic(CSHA512& hasher, RNGState& rng) noexcept // Add the events hasher into the mix rng.SeedEvents(hasher); - // Dynamic environment data (performance monitoring, ...) + // Dynamic environment data (clocks, resource usage, ...) auto old_size = hasher.Size(); RandAddDynamicEnv(hasher); LogDebug(BCLog::RAND, "Feeding %i bytes of dynamic environment data into RNG\n", hasher.Size() - old_size); @@ -616,7 +616,7 @@ void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept // Everything that the 'slow' seeder includes. SeedSlow(hasher, rng); - // Dynamic environment data (performance monitoring, ...) + // Dynamic environment data (clocks, resource usage, ...) auto old_size = hasher.Size(); RandAddDynamicEnv(hasher); @@ -671,9 +671,11 @@ void MakeRandDeterministicDANGEROUS(const uint256& seed) noexcept { GetRNGState().MakeDeterministic(seed); } +std::atomic<bool> g_used_g_prng{false}; // Only accessed from tests void GetRandBytes(Span<unsigned char> bytes) noexcept { + g_used_g_prng = true; ProcRand(bytes.data(), bytes.size(), RNGLevel::FAST, /*always_use_real_rng=*/false); } diff --git a/src/random.h b/src/random.h index 536e697cca..2a26ca986b 100644 --- a/src/random.h +++ b/src/random.h @@ -49,7 +49,7 @@ * * - RandAddPeriodic() seeds everything that fast seeding includes, but additionally: * - A high-precision timestamp - * - Dynamic environment data (performance monitoring, ...) + * - Dynamic environment data (clocks, resource usage, ...) * - Strengthen the entropy for 10 ms using repeated SHA512. * This is run once every minute. * diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 4754b597c5..7a46a5109b 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <randomenv.h> @@ -28,7 +28,6 @@ #ifdef WIN32 #include <windows.h> -#include <winreg.h> #else #include <fcntl.h> #include <netinet/in.h> @@ -64,45 +63,6 @@ extern char** environ; // NOLINT(readability-redundant-declaration): Necessary o namespace { -void RandAddSeedPerfmon(CSHA512& hasher) -{ -#ifdef WIN32 - // Seed with the entire set of perfmon data - - // This can take up to 2 seconds, so only do it every 10 minutes. - // Initialize last_perfmon to 0 seconds, we don't skip the first call. - static std::atomic<SteadyClock::time_point> last_perfmon{SteadyClock::time_point{0s}}; - auto last_time = last_perfmon.load(); - auto current_time = SteadyClock::now(); - if (current_time < last_time + 10min) return; - last_perfmon = current_time; - - std::vector<unsigned char> vData(250000, 0); - long ret = 0; - unsigned long nSize = 0; - const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data - while (true) { - nSize = vData.size(); - ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", nullptr, nullptr, vData.data(), &nSize); - if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) - break; - vData.resize(std::min((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially - } - RegCloseKey(HKEY_PERFORMANCE_DATA); - if (ret == ERROR_SUCCESS) { - hasher.Write(vData.data(), nSize); - memory_cleanse(vData.data(), nSize); - } else { - // Performance data is only a best-effort attempt at improving the - // situation when the OS randomness (and other sources) aren't - // adequate. As a result, failure to read it is isn't considered critical, - // so we don't call RandFailure(). - // TODO: Add logging when the logger is made functional before global - // constructors have been invoked. - } -#endif -} - /** Helper to easily feed data into a CSHA512. * * Note that this does not serialize the passed object (like stream.h's << operators do). @@ -227,8 +187,6 @@ void AddAllCPUID(CSHA512& hasher) void RandAddDynamicEnv(CSHA512& hasher) { - RandAddSeedPerfmon(hasher); - // Various clocks #ifdef WIN32 FILETIME ftime; diff --git a/src/rest.cpp b/src/rest.cpp index c42bc8e40c..3e4b8b37c6 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <rest.h> @@ -90,7 +90,7 @@ static NodeContext* GetNodeContext(const std::any& context, HTTPRequest* req) strprintf("%s:%d (%s)\n" "Internal bug detected: Node context not found!\n" "You may report this issue here: %s\n", - __FILE__, __LINE__, __func__, PACKAGE_BUGREPORT)); + __FILE__, __LINE__, __func__, CLIENT_BUGREPORT)); return nullptr; } return node_context; @@ -128,7 +128,7 @@ static ChainstateManager* GetChainman(const std::any& context, HTTPRequest* req) strprintf("%s:%d (%s)\n" "Internal bug detected: Chainman disabled or instance not found!\n" "You may report this issue here: %s\n", - __FILE__, __LINE__, __func__, PACKAGE_BUGREPORT)); + __FILE__, __LINE__, __func__, CLIENT_BUGREPORT)); return nullptr; } return node_context->chainman.get(); @@ -309,8 +309,11 @@ static bool rest_block(const std::any& context, if (!pblockindex) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } - if (chainman.m_blockman.IsBlockPruned(*pblockindex)) { - return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); + if (!(pblockindex->nStatus & BLOCK_HAVE_DATA)) { + if (chainman.m_blockman.IsBlockPruned(*pblockindex)) { + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); + } + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (not fully downloaded)"); } pos = pblockindex->GetBlockPos(); } @@ -867,10 +870,9 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: { auto process_utxos = [&vOutPoints, &outs, &hits, &active_height, &active_hash, &chainman](const CCoinsView& view, const CTxMemPool* mempool) EXCLUSIVE_LOCKS_REQUIRED(chainman.GetMutex()) { for (const COutPoint& vOutPoint : vOutPoints) { - Coin coin; - bool hit = (!mempool || !mempool->isSpent(vOutPoint)) && view.GetCoin(vOutPoint, coin); - hits.push_back(hit); - if (hit) outs.emplace_back(std::move(coin)); + auto coin = !mempool || !mempool->isSpent(vOutPoint) ? view.GetCoin(vOutPoint) : std::nullopt; + hits.push_back(coin.has_value()); + if (coin) outs.emplace_back(std::move(*coin)); } active_height = chainman.ActiveHeight(); active_hash = chainman.ActiveTip()->GetBlockHash(); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 7ffd03e71a..823d2303c8 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -22,6 +22,7 @@ #include <hash.h> #include <index/blockfilterindex.h> #include <index/coinstatsindex.h> +#include <interfaces/mining.h> #include <kernel/coinstats.h> #include <logging/timer.h> #include <net.h> @@ -54,28 +55,21 @@ #include <stdint.h> #include <condition_variable> +#include <iterator> #include <memory> #include <mutex> #include <optional> +#include <vector> using kernel::CCoinsStats; using kernel::CoinStatsHashType; +using interfaces::Mining; using node::BlockManager; using node::NodeContext; using node::SnapshotMetadata; using util::MakeUnorderedList; -struct CUpdatedBlock -{ - uint256 hash; - int height; -}; - -static GlobalMutex cs_blockchange; -static std::condition_variable cond_blockchange; -static CUpdatedBlock latestblock GUARDED_BY(cs_blockchange); - std::tuple<std::unique_ptr<CCoinsViewCursor>, CCoinsStats, const CBlockIndex*> PrepareUTXOSnapshot( Chainstate& chainstate, @@ -201,8 +195,10 @@ UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIn case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: CBlockUndo blockUndo; const bool is_not_pruned{WITH_LOCK(::cs_main, return !blockman.IsBlockPruned(blockindex))}; - const bool have_undo{is_not_pruned && blockman.UndoReadFromDisk(blockUndo, blockindex)}; - + bool have_undo{is_not_pruned && WITH_LOCK(::cs_main, return blockindex.nStatus & BLOCK_HAVE_UNDO)}; + if (have_undo && !blockman.UndoReadFromDisk(blockUndo, blockindex)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Undo data expected but can't be read. This could be due to disk corruption or a conflict with a pruning event."); + } for (size_t i = 0; i < block.vtx.size(); ++i) { const CTransactionRef& tx = block.vtx.at(i); // coinbase transaction (i.e. i == 0) doesn't have undo data @@ -260,21 +256,12 @@ static RPCHelpMan getbestblockhash() }; } -void RPCNotifyBlockChange(const CBlockIndex* pindex) -{ - if(pindex) { - LOCK(cs_blockchange); - latestblock.hash = pindex->GetBlockHash(); - latestblock.height = pindex->nHeight; - } - cond_blockchange.notify_all(); -} - static RPCHelpMan waitfornewblock() { return RPCHelpMan{"waitfornewblock", - "\nWaits for a specific new block and returns useful info about it.\n" - "\nReturns the current block on timeout or exit.\n", + "\nWaits for any new block and returns useful info about it.\n" + "\nReturns the current block on timeout or exit.\n" + "\nMake sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)", { {"timeout", RPCArg::Type::NUM, RPCArg::Default{0}, "Time in milliseconds to wait for a response. 0 indicates no timeout."}, }, @@ -293,17 +280,16 @@ static RPCHelpMan waitfornewblock() int timeout = 0; if (!request.params[0].isNull()) timeout = request.params[0].getInt<int>(); + if (timeout < 0) throw JSONRPCError(RPC_MISC_ERROR, "Negative timeout"); - CUpdatedBlock block; - { - WAIT_LOCK(cs_blockchange, lock); - block = latestblock; - if(timeout) - cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&block]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height != block.height || latestblock.hash != block.hash || !IsRPCRunning(); }); - else - cond_blockchange.wait(lock, [&block]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height != block.height || latestblock.hash != block.hash || !IsRPCRunning(); }); - block = latestblock; + NodeContext& node = EnsureAnyNodeContext(request.context); + Mining& miner = EnsureMining(node); + + auto block{CHECK_NONFATAL(miner.getTip()).value()}; + if (IsRPCRunning()) { + block = timeout ? miner.waitTipChanged(block.hash, std::chrono::milliseconds(timeout)) : miner.waitTipChanged(block.hash); } + UniValue ret(UniValue::VOBJ); ret.pushKV("hash", block.hash.GetHex()); ret.pushKV("height", block.height); @@ -316,7 +302,8 @@ static RPCHelpMan waitforblock() { return RPCHelpMan{"waitforblock", "\nWaits for a specific new block and returns useful info about it.\n" - "\nReturns the current block on timeout or exit.\n", + "\nReturns the current block on timeout or exit.\n" + "\nMake sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)", { {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "Block hash to wait for."}, {"timeout", RPCArg::Type::NUM, RPCArg::Default{0}, "Time in milliseconds to wait for a response. 0 indicates no timeout."}, @@ -339,15 +326,22 @@ static RPCHelpMan waitforblock() if (!request.params[1].isNull()) timeout = request.params[1].getInt<int>(); + if (timeout < 0) throw JSONRPCError(RPC_MISC_ERROR, "Negative timeout"); - CUpdatedBlock block; - { - WAIT_LOCK(cs_blockchange, lock); - if(timeout) - cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&hash]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.hash == hash || !IsRPCRunning();}); - else - cond_blockchange.wait(lock, [&hash]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.hash == hash || !IsRPCRunning(); }); - block = latestblock; + NodeContext& node = EnsureAnyNodeContext(request.context); + Mining& miner = EnsureMining(node); + + auto block{CHECK_NONFATAL(miner.getTip()).value()}; + const auto deadline{std::chrono::steady_clock::now() + 1ms * timeout}; + while (IsRPCRunning() && block.hash != hash) { + if (timeout) { + auto now{std::chrono::steady_clock::now()}; + if (now >= deadline) break; + const MillisecondsDouble remaining{deadline - now}; + block = miner.waitTipChanged(block.hash, remaining); + } else { + block = miner.waitTipChanged(block.hash); + } } UniValue ret(UniValue::VOBJ); @@ -363,7 +357,8 @@ static RPCHelpMan waitforblockheight() return RPCHelpMan{"waitforblockheight", "\nWaits for (at least) block height and returns the height and hash\n" "of the current tip.\n" - "\nReturns the current block on timeout or exit.\n", + "\nReturns the current block on timeout or exit.\n" + "\nMake sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)", { {"height", RPCArg::Type::NUM, RPCArg::Optional::NO, "Block height to wait for."}, {"timeout", RPCArg::Type::NUM, RPCArg::Default{0}, "Time in milliseconds to wait for a response. 0 indicates no timeout."}, @@ -386,16 +381,25 @@ static RPCHelpMan waitforblockheight() if (!request.params[1].isNull()) timeout = request.params[1].getInt<int>(); + if (timeout < 0) throw JSONRPCError(RPC_MISC_ERROR, "Negative timeout"); - CUpdatedBlock block; - { - WAIT_LOCK(cs_blockchange, lock); - if(timeout) - cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&height]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height >= height || !IsRPCRunning();}); - else - cond_blockchange.wait(lock, [&height]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height >= height || !IsRPCRunning(); }); - block = latestblock; + NodeContext& node = EnsureAnyNodeContext(request.context); + Mining& miner = EnsureMining(node); + + auto block{CHECK_NONFATAL(miner.getTip()).value()}; + const auto deadline{std::chrono::steady_clock::now() + 1ms * timeout}; + + while (IsRPCRunning() && block.height < height) { + if (timeout) { + auto now{std::chrono::steady_clock::now()}; + if (now >= deadline) break; + const MillisecondsDouble remaining{deadline - now}; + block = miner.waitTipChanged(block.hash, remaining); + } else { + block = miner.waitTipChanged(block.hash); + } } + UniValue ret(UniValue::VOBJ); ret.pushKV("hash", block.hash.GetHex()); ret.pushKV("height", block.height); @@ -597,20 +601,32 @@ static RPCHelpMan getblockheader() }; } +void CheckBlockDataAvailability(BlockManager& blockman, const CBlockIndex& blockindex, bool check_for_undo) +{ + AssertLockHeld(cs_main); + uint32_t flag = check_for_undo ? BLOCK_HAVE_UNDO : BLOCK_HAVE_DATA; + if (!(blockindex.nStatus & flag)) { + if (blockman.IsBlockPruned(blockindex)) { + throw JSONRPCError(RPC_MISC_ERROR, strprintf("%s not available (pruned data)", check_for_undo ? "Undo data" : "Block")); + } + if (check_for_undo) { + throw JSONRPCError(RPC_MISC_ERROR, "Undo data not available"); + } + throw JSONRPCError(RPC_MISC_ERROR, "Block not available (not fully downloaded)"); + } +} + static CBlock GetBlockChecked(BlockManager& blockman, const CBlockIndex& blockindex) { CBlock block; { LOCK(cs_main); - if (blockman.IsBlockPruned(blockindex)) { - throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); - } + CheckBlockDataAvailability(blockman, blockindex, /*check_for_undo=*/false); } if (!blockman.ReadBlockFromDisk(block, blockindex)) { - // Block not found on disk. This could be because we have the block - // header in our index but not yet have the block or did not accept the - // block. Or if the block was pruned right after we released the lock above. + // Block not found on disk. This shouldn't normally happen unless the block was + // pruned right after we released the lock above. throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); } @@ -623,16 +639,13 @@ static std::vector<uint8_t> GetRawBlockChecked(BlockManager& blockman, const CBl FlatFilePos pos{}; { LOCK(cs_main); - if (blockman.IsBlockPruned(blockindex)) { - throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); - } + CheckBlockDataAvailability(blockman, blockindex, /*check_for_undo=*/false); pos = blockindex.GetBlockPos(); } if (!blockman.ReadRawBlockFromDisk(data, pos)) { - // Block not found on disk. This could be because we have the block - // header in our index but not yet have the block or did not accept the - // block. Or if the block was pruned right after we released the lock above. + // Block not found on disk. This shouldn't normally happen unless the block was + // pruned right after we released the lock above. throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); } @@ -648,9 +661,7 @@ static CBlockUndo GetUndoChecked(BlockManager& blockman, const CBlockIndex& bloc { LOCK(cs_main); - if (blockman.IsBlockPruned(blockindex)) { - throw JSONRPCError(RPC_MISC_ERROR, "Undo data not available (pruned data)"); - } + CheckBlockDataAvailability(blockman, blockindex, /*check_for_undo=*/true); } if (!blockman.UndoReadFromDisk(blockUndo, blockindex)) { @@ -757,14 +768,7 @@ static RPCHelpMan getblock() { uint256 hash(ParseHashV(request.params[0], "blockhash")); - int verbosity = 1; - if (!request.params[1].isNull()) { - if (request.params[1].isBool()) { - verbosity = request.params[1].get_bool() ? 1 : 0; - } else { - verbosity = request.params[1].getInt<int>(); - } - } + int verbosity{ParseVerbosity(request.params[1], /*default_verbosity=*/1, /*allow_bool=*/true)}; const CBlockIndex* pblockindex; const CBlockIndex* tip; @@ -1131,35 +1135,32 @@ static RPCHelpMan gettxout() if (!request.params[2].isNull()) fMempool = request.params[2].get_bool(); - Coin coin; Chainstate& active_chainstate = chainman.ActiveChainstate(); CCoinsViewCache* coins_view = &active_chainstate.CoinsTip(); + std::optional<Coin> coin; if (fMempool) { const CTxMemPool& mempool = EnsureMemPool(node); LOCK(mempool.cs); CCoinsViewMemPool view(coins_view, mempool); - if (!view.GetCoin(out, coin) || mempool.isSpent(out)) { - return UniValue::VNULL; - } + if (!mempool.isSpent(out)) coin = view.GetCoin(out); } else { - if (!coins_view->GetCoin(out, coin)) { - return UniValue::VNULL; - } + coin = coins_view->GetCoin(out); } + if (!coin) return UniValue::VNULL; const CBlockIndex* pindex = active_chainstate.m_blockman.LookupBlockIndex(coins_view->GetBestBlock()); ret.pushKV("bestblock", pindex->GetBlockHash().GetHex()); - if (coin.nHeight == MEMPOOL_HEIGHT) { + if (coin->nHeight == MEMPOOL_HEIGHT) { ret.pushKV("confirmations", 0); } else { - ret.pushKV("confirmations", (int64_t)(pindex->nHeight - coin.nHeight + 1)); + ret.pushKV("confirmations", (int64_t)(pindex->nHeight - coin->nHeight + 1)); } - ret.pushKV("value", ValueFromAmount(coin.out.nValue)); + ret.pushKV("value", ValueFromAmount(coin->out.nValue)); UniValue o(UniValue::VOBJ); - ScriptToUniv(coin.out.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); + ScriptToUniv(coin->out.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); ret.pushKV("scriptPubKey", std::move(o)); - ret.pushKV("coinbase", (bool)coin.fCoinBase); + ret.pushKV("coinbase", (bool)coin->fCoinBase); return ret; }, @@ -1306,6 +1307,7 @@ RPCHelpMan getblockchaininfo() {RPCResult::Type::NUM, "pruneheight", /*optional=*/true, "height of the last block pruned, plus one (only present if pruning is enabled)"}, {RPCResult::Type::BOOL, "automatic_pruning", /*optional=*/true, "whether automatic pruning is enabled (only present if pruning is enabled)"}, {RPCResult::Type::NUM, "prune_target_size", /*optional=*/true, "the target size used by pruning (only present if automatic pruning is enabled)"}, + {RPCResult::Type::STR_HEX, "signet_challenge", /*optional=*/true, "the block challenge (aka. block script), in hexadecimal (only present if the current network is a signet)"}, (IsDeprecatedRPCEnabled("warnings") ? RPCResult{RPCResult::Type::STR, "warnings", "any network and blockchain warnings (DEPRECATED)"} : RPCResult{RPCResult::Type::ARR, "warnings", "any network and blockchain warnings (run with `-deprecatedrpc=warnings` to return the latest warning as a single string)", @@ -1335,7 +1337,7 @@ RPCHelpMan getblockchaininfo() obj.pushKV("difficulty", GetDifficulty(tip)); obj.pushKV("time", tip.GetBlockTime()); obj.pushKV("mediantime", tip.GetMedianTimePast()); - obj.pushKV("verificationprogress", GuessVerificationProgress(chainman.GetParams().TxData(), &tip)); + obj.pushKV("verificationprogress", chainman.GuessVerificationProgress(&tip)); obj.pushKV("initialblockdownload", chainman.IsInitialBlockDownload()); obj.pushKV("chainwork", tip.nChainWork.GetHex()); obj.pushKV("size_on_disk", chainman.m_blockman.CalculateCurrentUsage()); @@ -1350,6 +1352,11 @@ RPCHelpMan getblockchaininfo() obj.pushKV("prune_target_size", chainman.m_blockman.GetPruneTarget()); } } + if (chainman.GetParams().GetChainType() == ChainType::SIGNET) { + const std::vector<uint8_t>& signet_challenge = + chainman.GetParams().GetConsensus().signet_challenge; + obj.pushKV("signet_challenge", HexStr(signet_challenge)); + } NodeContext& node = EnsureAnyNodeContext(request.context); obj.pushKV("warnings", node::GetWarningsForRpc(*CHECK_NONFATAL(node.warnings), IsDeprecatedRPCEnabled("warnings"))); @@ -1648,6 +1655,7 @@ void ReconsiderBlock(ChainstateManager& chainman, uint256 block_hash) { } chainman.ActiveChainstate().ResetBlockFailureFlags(pblockindex); + chainman.RecalculateBestHeader(); } BlockValidationState state; @@ -2586,6 +2594,235 @@ static RPCHelpMan scanblocks() }; } +static RPCHelpMan getdescriptoractivity() +{ + return RPCHelpMan{"getdescriptoractivity", + "\nGet spend and receive activity associated with a set of descriptors for a set of blocks. " + "This command pairs well with the `relevant_blocks` output of `scanblocks()`.\n" + "This call may take several minutes. If you encounter timeouts, try specifying no RPC timeout (bitcoin-cli -rpcclienttimeout=0)", + { + RPCArg{"blockhashes", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "The list of blockhashes to examine for activity. Order doesn't matter. Must be along main chain or an error is thrown.\n", { + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A valid blockhash"}, + }}, + scan_objects_arg_desc, + {"include_mempool", RPCArg::Type::BOOL, RPCArg::Default{true}, "Whether to include unconfirmed activity"}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::ARR, "activity", "events", { + {RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "type", "always 'spend'"}, + {RPCResult::Type::STR_AMOUNT, "amount", "The total amount in " + CURRENCY_UNIT + " of the spent output"}, + {RPCResult::Type::STR_HEX, "blockhash", /*optional=*/true, "The blockhash this spend appears in (omitted if unconfirmed)"}, + {RPCResult::Type::NUM, "height", /*optional=*/true, "Height of the spend (omitted if unconfirmed)"}, + {RPCResult::Type::STR_HEX, "spend_txid", "The txid of the spending transaction"}, + {RPCResult::Type::NUM, "spend_vout", "The vout of the spend"}, + {RPCResult::Type::STR_HEX, "prevout_txid", "The txid of the prevout"}, + {RPCResult::Type::NUM, "prevout_vout", "The vout of the prevout"}, + {RPCResult::Type::OBJ, "prevout_spk", "", ScriptPubKeyDoc()}, + }}, + {RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "type", "always 'receive'"}, + {RPCResult::Type::STR_AMOUNT, "amount", "The total amount in " + CURRENCY_UNIT + " of the new output"}, + {RPCResult::Type::STR_HEX, "blockhash", /*optional=*/true, "The block that this receive is in (omitted if unconfirmed)"}, + {RPCResult::Type::NUM, "height", /*optional=*/true, "The height of the receive (omitted if unconfirmed)"}, + {RPCResult::Type::STR_HEX, "txid", "The txid of the receiving transaction"}, + {RPCResult::Type::NUM, "vout", "The vout of the receiving output"}, + {RPCResult::Type::OBJ, "output_spk", "", ScriptPubKeyDoc()}, + }}, + // TODO is the skip_type_check avoidable with a heterogeneous ARR? + }, /*skip_type_check=*/true}, + }, + }, + RPCExamples{ + HelpExampleCli("getdescriptoractivity", "'[\"000000000000000000001347062c12fded7c528943c8ce133987e2e2f5a840ee\"]' '[\"addr(bc1qzl6nsgqzu89a66l50cvwapnkw5shh23zarqkw9)\"]'") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + UniValue ret(UniValue::VOBJ); + UniValue activity(UniValue::VARR); + NodeContext& node = EnsureAnyNodeContext(request.context); + ChainstateManager& chainman = EnsureChainman(node); + + struct CompareByHeightAscending { + bool operator()(const CBlockIndex* a, const CBlockIndex* b) const { + return a->nHeight < b->nHeight; + } + }; + + std::set<const CBlockIndex*, CompareByHeightAscending> blockindexes_sorted; + + { + // Validate all given blockhashes, and ensure blocks are along a single chain. + LOCK(::cs_main); + for (const UniValue& blockhash : request.params[0].get_array().getValues()) { + uint256 bhash = ParseHashV(blockhash, "blockhash"); + CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(bhash); + if (!pindex) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + if (!chainman.ActiveChain().Contains(pindex)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Block is not in main chain"); + } + blockindexes_sorted.insert(pindex); + } + } + + std::set<CScript> scripts_to_watch; + + // Determine scripts to watch. + for (const UniValue& scanobject : request.params[1].get_array().getValues()) { + FlatSigningProvider provider; + std::vector<CScript> scripts = EvalDescriptorStringOrObject(scanobject, provider); + + for (const CScript& script : scripts) { + scripts_to_watch.insert(script); + } + } + + const auto AddSpend = [&]( + const CScript& spk, + const CAmount val, + const CTransactionRef& tx, + int vin, + const CTxIn& txin, + const CBlockIndex* index + ) { + UniValue event(UniValue::VOBJ); + UniValue spkUv(UniValue::VOBJ); + ScriptToUniv(spk, /*out=*/spkUv, /*include_hex=*/true, /*include_address=*/true); + + event.pushKV("type", "spend"); + event.pushKV("amount", ValueFromAmount(val)); + if (index) { + event.pushKV("blockhash", index->GetBlockHash().ToString()); + event.pushKV("height", index->nHeight); + } + event.pushKV("spend_txid", tx->GetHash().ToString()); + event.pushKV("spend_vin", vin); + event.pushKV("prevout_txid", txin.prevout.hash.ToString()); + event.pushKV("prevout_vout", txin.prevout.n); + event.pushKV("prevout_spk", spkUv); + + return event; + }; + + const auto AddReceive = [&](const CTxOut& txout, const CBlockIndex* index, int vout, const CTransactionRef& tx) { + UniValue event(UniValue::VOBJ); + UniValue spkUv(UniValue::VOBJ); + ScriptToUniv(txout.scriptPubKey, /*out=*/spkUv, /*include_hex=*/true, /*include_address=*/true); + + event.pushKV("type", "receive"); + event.pushKV("amount", ValueFromAmount(txout.nValue)); + if (index) { + event.pushKV("blockhash", index->GetBlockHash().ToString()); + event.pushKV("height", index->nHeight); + } + event.pushKV("txid", tx->GetHash().ToString()); + event.pushKV("vout", vout); + event.pushKV("output_spk", spkUv); + + return event; + }; + + BlockManager* blockman; + Chainstate& active_chainstate = chainman.ActiveChainstate(); + { + LOCK(::cs_main); + blockman = CHECK_NONFATAL(&active_chainstate.m_blockman); + } + + for (const CBlockIndex* blockindex : blockindexes_sorted) { + const CBlock block{GetBlockChecked(chainman.m_blockman, *blockindex)}; + const CBlockUndo block_undo{GetUndoChecked(*blockman, *blockindex)}; + + for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx = block.vtx.at(i); + + if (!tx->IsCoinBase()) { + // skip coinbase; spends can't happen there. + const auto& txundo = block_undo.vtxundo.at(i - 1); + + for (size_t vin_idx = 0; vin_idx < tx->vin.size(); ++vin_idx) { + const auto& coin = txundo.vprevout.at(vin_idx); + const auto& txin = tx->vin.at(vin_idx); + if (scripts_to_watch.contains(coin.out.scriptPubKey)) { + activity.push_back(AddSpend( + coin.out.scriptPubKey, coin.out.nValue, tx, vin_idx, txin, blockindex)); + } + } + } + + for (size_t vout_idx = 0; vout_idx < tx->vout.size(); ++vout_idx) { + const auto& vout = tx->vout.at(vout_idx); + if (scripts_to_watch.contains(vout.scriptPubKey)) { + activity.push_back(AddReceive(vout, blockindex, vout_idx, tx)); + } + } + } + } + + bool search_mempool = true; + if (!request.params[2].isNull()) { + search_mempool = request.params[2].get_bool(); + } + + if (search_mempool) { + const CTxMemPool& mempool = EnsureMemPool(node); + LOCK(::cs_main); + LOCK(mempool.cs); + const CCoinsViewCache& coins_view = &active_chainstate.CoinsTip(); + + for (const CTxMemPoolEntry& e : mempool.entryAll()) { + const auto& tx = e.GetSharedTx(); + + for (size_t vin_idx = 0; vin_idx < tx->vin.size(); ++vin_idx) { + CScript scriptPubKey; + CAmount value; + const auto& txin = tx->vin.at(vin_idx); + std::optional<Coin> coin = coins_view.GetCoin(txin.prevout); + + // Check if the previous output is in the chain + if (!coin) { + // If not found in the chain, check the mempool. Likely, this is a + // child transaction of another transaction in the mempool. + CTransactionRef prev_tx = CHECK_NONFATAL(mempool.get(txin.prevout.hash)); + + if (txin.prevout.n >= prev_tx->vout.size()) { + throw std::runtime_error("Invalid output index"); + } + const CTxOut& out = prev_tx->vout[txin.prevout.n]; + scriptPubKey = out.scriptPubKey; + value = out.nValue; + } else { + // Coin found in the chain + const CTxOut& out = coin->out; + scriptPubKey = out.scriptPubKey; + value = out.nValue; + } + + if (scripts_to_watch.contains(scriptPubKey)) { + UniValue event(UniValue::VOBJ); + activity.push_back(AddSpend( + scriptPubKey, value, tx, vin_idx, txin, nullptr)); + } + } + + for (size_t vout_idx = 0; vout_idx < tx->vout.size(); ++vout_idx) { + const auto& vout = tx->vout.at(vout_idx); + if (scripts_to_watch.contains(vout.scriptPubKey)) { + activity.push_back(AddReceive(vout, nullptr, vout_idx, tx)); + } + } + } + } + + ret.pushKV("activity", activity); + return ret; +}, + }; +} + static RPCHelpMan getblockfilter() { return RPCHelpMan{"getblockfilter", @@ -2711,7 +2948,7 @@ static RPCHelpMan dumptxoutset() return RPCHelpMan{ "dumptxoutset", "Write the serialized UTXO set to a file. This can be used in loadtxoutset afterwards if this snapshot height is supported in the chainparams as well.\n\n" - "Unless the the \"latest\" type is requested, the node will roll back to the requested height and network activity will be suspended during this process. " + "Unless the \"latest\" type is requested, the node will roll back to the requested height and network activity will be suspended during this process. " "Because of this it is discouraged to interact with the node in any other way during the execution of this call to avoid inconsistent results and race conditions, particularly RPCs that interact with blockstorage.\n\n" "This call may take several minutes. Make sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)", { @@ -3014,7 +3251,7 @@ static RPCHelpMan loadtxoutset() } }, RPCExamples{ - HelpExampleCli("loadtxoutset -rpcclienttimeout=0", "utxo.dat") + HelpExampleCli("-rpcclienttimeout=0 loadtxoutset", "utxo.dat") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { @@ -3107,7 +3344,7 @@ return RPCHelpMan{ data.pushKV("blocks", (int)chain.Height()); data.pushKV("bestblockhash", tip->GetBlockHash().GetHex()); data.pushKV("difficulty", GetDifficulty(*tip)); - data.pushKV("verificationprogress", GuessVerificationProgress(Params().TxData(), tip)); + data.pushKV("verificationprogress", chainman.GuessVerificationProgress(tip)); data.pushKV("coins_db_cache_bytes", cs.m_coinsdb_cache_size_bytes); data.pushKV("coins_tip_cache_bytes", cs.m_coinstip_cache_size_bytes); if (cs.m_from_snapshot_blockhash) { @@ -3153,6 +3390,7 @@ void RegisterBlockchainRPCCommands(CRPCTable& t) {"blockchain", &preciousblock}, {"blockchain", &scantxoutset}, {"blockchain", &scanblocks}, + {"blockchain", &getdescriptoractivity}, {"blockchain", &getblockfilter}, {"blockchain", &dumptxoutset}, {"blockchain", &loadtxoutset}, diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index b5a0382da1..89b9921d55 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -35,9 +35,6 @@ static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; */ double GetDifficulty(const CBlockIndex& blockindex); -/** Callback for when block tip changed. */ -void RPCNotifyBlockChange(const CBlockIndex*); - /** Block description to JSON */ UniValue blockToJSON(node::BlockManager& blockman, const CBlock& block, const CBlockIndex& tip, const CBlockIndex& blockindex, TxVerbosity verbosity) LOCKS_EXCLUDED(cs_main); @@ -60,5 +57,6 @@ UniValue CreateUTXOSnapshot( //! Return height of highest block that has been pruned, or std::nullopt if no blocks have been pruned std::optional<int> GetPruneHeight(const node::BlockManager& blockman, const CChain& chain) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); +void CheckBlockDataAvailability(node::BlockManager& blockman, const CBlockIndex& blockindex, bool check_for_undo) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); #endif // BITCOIN_RPC_BLOCKCHAIN_H diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 0112a261ce..1b711e3c5b 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -92,6 +92,9 @@ static const CRPCConvertParam vRPCConvertParams[] = { "scanblocks", 3, "stop_height" }, { "scanblocks", 5, "options" }, { "scanblocks", 5, "filter_false_positives" }, + { "getdescriptoractivity", 0, "blockhashes" }, + { "getdescriptoractivity", 1, "scanobjects" }, + { "getdescriptoractivity", 2, "include_mempool" }, { "scantxoutset", 1, "scanobjects" }, { "addmultisigaddress", 0, "nrequired" }, { "addmultisigaddress", 1, "keys" }, @@ -254,6 +257,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "keypoolrefill", 0, "newsize" }, { "getrawmempool", 0, "verbose" }, { "getrawmempool", 1, "mempool_sequence" }, + { "getorphantxs", 0, "verbosity" }, { "estimatesmartfee", 0, "conf_target" }, { "estimaterawfee", 0, "conf_target" }, { "estimaterawfee", 1, "threshold" }, diff --git a/src/rpc/external_signer.cpp b/src/rpc/external_signer.cpp index 3ad7a940e0..44de5443fa 100644 --- a/src/rpc/external_signer.cpp +++ b/src/rpc/external_signer.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <common/args.h> #include <common/system.h> diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index d61898260b..52260d5794 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -8,8 +8,10 @@ #include <node/mempool_persist.h> #include <chainparams.h> +#include <consensus/validation.h> #include <core_io.h> #include <kernel/mempool_entry.h> +#include <net_processing.h> #include <node/mempool_persist_args.h> #include <node/types.h> #include <policy/rbf.h> @@ -24,6 +26,7 @@ #include <util/moneystr.h> #include <util/strencodings.h> #include <util/time.h> +#include <util/vector.h> #include <utility> @@ -143,7 +146,8 @@ static RPCHelpMan testmempoolaccept() {RPCResult{RPCResult::Type::STR_HEX, "", "transaction wtxid in hex"}, }}, }}, - {RPCResult::Type::STR, "reject-reason", /*optional=*/true, "Rejection string (only present when 'allowed' is false)"}, + {RPCResult::Type::STR, "reject-reason", /*optional=*/true, "Rejection reason (only present when 'allowed' is false)"}, + {RPCResult::Type::STR, "reject-details", /*optional=*/true, "Rejection details (only present when 'allowed' is false and rejection details exist)"}, }}, } }, @@ -242,6 +246,7 @@ static RPCHelpMan testmempoolaccept() result_inner.pushKV("reject-reason", "missing-inputs"); } else { result_inner.pushKV("reject-reason", state.GetRejectReason()); + result_inner.pushKV("reject-details", state.ToString()); } } rpc_result.push_back(std::move(result_inner)); @@ -274,7 +279,7 @@ static std::vector<RPCResult> MempoolEntryDescription() {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "parent transaction id"}}}, RPCResult{RPCResult::Type::ARR, "spentby", "unconfirmed transactions spending outputs from this transaction", {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "child transaction id"}}}, - RPCResult{RPCResult::Type::BOOL, "bip125-replaceable", "Whether this transaction signals BIP125 replaceability or has an unconfirmed ancestor signaling BIP125 replaceability.\n"}, + RPCResult{RPCResult::Type::BOOL, "bip125-replaceable", "Whether this transaction signals BIP125 replaceability or has an unconfirmed ancestor signaling BIP125 replaceability. (DEPRECATED)\n"}, RPCResult{RPCResult::Type::BOOL, "unbroadcast", "Whether this transaction is currently unbroadcast (initial broadcast not yet acknowledged by any peers)"}, }; } @@ -679,7 +684,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) ret.pushKV("minrelaytxfee", ValueFromAmount(pool.m_opts.min_relay_feerate.GetFeePerK())); ret.pushKV("incrementalrelayfee", ValueFromAmount(pool.m_opts.incremental_relay_feerate.GetFeePerK())); ret.pushKV("unbroadcastcount", uint64_t{pool.GetUnbroadcastTxs().size()}); - ret.pushKV("fullrbf", pool.m_opts.full_rbf); + ret.pushKV("fullrbf", true); return ret; } @@ -701,7 +706,7 @@ static RPCHelpMan getmempoolinfo() {RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"}, {RPCResult::Type::NUM, "incrementalrelayfee", "minimum fee rate increment for mempool limiting or replacement in " + CURRENCY_UNIT + "/kvB"}, {RPCResult::Type::NUM, "unbroadcastcount", "Current number of transactions that haven't passed initial broadcast yet"}, - {RPCResult::Type::BOOL, "fullrbf", "True if the mempool accepts RBF without replaceability signaling inspection"}, + {RPCResult::Type::BOOL, "fullrbf", "True if the mempool accepts RBF without replaceability signaling inspection (DEPRECATED)"}, }}, RPCExamples{ HelpExampleCli("getmempoolinfo", "") @@ -812,6 +817,107 @@ static RPCHelpMan savemempool() }; } +static std::vector<RPCResult> OrphanDescription() +{ + return { + RPCResult{RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, + RPCResult{RPCResult::Type::STR_HEX, "wtxid", "The transaction witness hash in hex"}, + RPCResult{RPCResult::Type::NUM, "bytes", "The serialized transaction size in bytes"}, + RPCResult{RPCResult::Type::NUM, "vsize", "The virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted."}, + RPCResult{RPCResult::Type::NUM, "weight", "The transaction weight as defined in BIP 141."}, + RPCResult{RPCResult::Type::NUM_TIME, "entry", "The entry time into the orphanage expressed in " + UNIX_EPOCH_TIME}, + RPCResult{RPCResult::Type::NUM_TIME, "expiration", "The orphan expiration time expressed in " + UNIX_EPOCH_TIME}, + RPCResult{RPCResult::Type::ARR, "from", "", + { + RPCResult{RPCResult::Type::NUM, "peer_id", "Peer ID"}, + }}, + }; +} + +static UniValue OrphanToJSON(const TxOrphanage::OrphanTxBase& orphan) +{ + UniValue o(UniValue::VOBJ); + o.pushKV("txid", orphan.tx->GetHash().ToString()); + o.pushKV("wtxid", orphan.tx->GetWitnessHash().ToString()); + o.pushKV("bytes", orphan.tx->GetTotalSize()); + o.pushKV("vsize", GetVirtualTransactionSize(*orphan.tx)); + o.pushKV("weight", GetTransactionWeight(*orphan.tx)); + o.pushKV("entry", int64_t{TicksSinceEpoch<std::chrono::seconds>(orphan.nTimeExpire - ORPHAN_TX_EXPIRE_TIME)}); + o.pushKV("expiration", int64_t{TicksSinceEpoch<std::chrono::seconds>(orphan.nTimeExpire)}); + UniValue from(UniValue::VARR); + from.push_back(orphan.fromPeer); // only one fromPeer for now + o.pushKV("from", from); + return o; +} + +static RPCHelpMan getorphantxs() +{ + return RPCHelpMan{"getorphantxs", + "\nShows transactions in the tx orphanage.\n" + "\nEXPERIMENTAL warning: this call may be changed in future releases.\n", + { + {"verbosity", RPCArg::Type::NUM, RPCArg::Default{0}, "0 for an array of txids (may contain duplicates), 1 for an array of objects with tx details, and 2 for details from (1) and tx hex", + RPCArgOptions{.skip_type_check = true}}, + }, + { + RPCResult{"for verbose = 0", + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, + }}, + RPCResult{"for verbose = 1", + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::OBJ, "", "", OrphanDescription()}, + }}, + RPCResult{"for verbose = 2", + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::OBJ, "", "", + Cat<std::vector<RPCResult>>( + OrphanDescription(), + {{RPCResult::Type::STR_HEX, "hex", "The serialized, hex-encoded transaction data"}} + ) + }, + }}, + }, + RPCExamples{ + HelpExampleCli("getorphantxs", "2") + + HelpExampleRpc("getorphantxs", "2") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + const NodeContext& node = EnsureAnyNodeContext(request.context); + PeerManager& peerman = EnsurePeerman(node); + std::vector<TxOrphanage::OrphanTxBase> orphanage = peerman.GetOrphanTransactions(); + + int verbosity{ParseVerbosity(request.params[0], /*default_verbosity=*/0, /*allow_bool*/false)}; + + UniValue ret(UniValue::VARR); + + if (verbosity == 0) { + for (auto const& orphan : orphanage) { + ret.push_back(orphan.tx->GetHash().ToString()); + } + } else if (verbosity == 1) { + for (auto const& orphan : orphanage) { + ret.push_back(OrphanToJSON(orphan)); + } + } else if (verbosity == 2) { + for (auto const& orphan : orphanage) { + UniValue o{OrphanToJSON(orphan)}; + o.pushKV("hex", EncodeHexTx(*orphan.tx)); + ret.push_back(o); + } + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid verbosity value " + ToString(verbosity)); + } + + return ret; + }, + }; +} + static RPCHelpMan submitpackage() { return RPCHelpMan{"submitpackage", @@ -822,7 +928,7 @@ static RPCHelpMan submitpackage() , { {"package", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of raw transactions.\n" - "The package must solely consist of a child and its parents. None of the parents may depend on each other.\n" + "The package must solely consist of a child transaction and all of its unconfirmed parents, if any. None of the parents may depend on each other.\n" "The package must be topologically sorted, with the child being the last element in the array.", { {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, @@ -864,15 +970,15 @@ static RPCHelpMan submitpackage() }, }, RPCExamples{ - HelpExampleRpc("submitpackage", R"(["rawtx1", "rawtx2"])") + - HelpExampleCli("submitpackage", R"('["rawtx1", "rawtx2"]')") + HelpExampleRpc("submitpackage", R"(["raw-parent-tx-1", "raw-parent-tx-2", "raw-child-tx"])") + + HelpExampleCli("submitpackage", R"('["raw-tx-without-unconfirmed-parents"]')") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const UniValue raw_transactions = request.params[0].get_array(); - if (raw_transactions.size() < 2 || raw_transactions.size() > MAX_PACKAGE_COUNT) { + if (raw_transactions.empty() || raw_transactions.size() > MAX_PACKAGE_COUNT) { throw JSONRPCError(RPC_INVALID_PARAMETER, - "Array must contain between 2 and " + ToString(MAX_PACKAGE_COUNT) + " transactions."); + "Array must contain between 1 and " + ToString(MAX_PACKAGE_COUNT) + " transactions."); } // Fee check needs to be run with chainstate and package context @@ -903,7 +1009,8 @@ static RPCHelpMan submitpackage() txns.emplace_back(MakeTransactionRef(std::move(mtx))); } - if (!IsChildWithParentsTree(txns)) { + CHECK_NONFATAL(!txns.empty()); + if (txns.size() > 1 && !IsChildWithParentsTree(txns)) { throw JSONRPCTransactionError(TransactionError::INVALID_PACKAGE, "package topology disallowed. not child-with-parents or parents depend on each other."); } @@ -1027,6 +1134,7 @@ void RegisterMempoolRPCCommands(CRPCTable& t) {"blockchain", &getrawmempool}, {"blockchain", &importmempool}, {"blockchain", &savemempool}, + {"hidden", &getorphantxs}, {"rawtransactions", &submitpackage}, }; for (const auto& c : commands) { diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 375352b18d..7e5936fddf 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -1,9 +1,9 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chain.h> #include <chainparams.h> @@ -23,6 +23,7 @@ #include <node/context.h> #include <node/miner.h> #include <node/warnings.h> +#include <policy/ephemeral_policy.h> #include <pow.h> #include <rpc/blockchain.h> #include <rpc/mining.h> @@ -45,9 +46,9 @@ #include <memory> #include <stdint.h> -using node::BlockAssembler; -using node::CBlockTemplate; +using interfaces::BlockTemplate; using interfaces::Mining; +using node::BlockAssembler; using node::NodeContext; using node::RegenerateCommitments; using node::UpdateTime; @@ -130,7 +131,7 @@ static RPCHelpMan getnetworkhashps() }; } -static bool GenerateBlock(ChainstateManager& chainman, Mining& miner, CBlock& block, uint64_t& max_tries, std::shared_ptr<const CBlock>& block_out, bool process_new_block) +static bool GenerateBlock(ChainstateManager& chainman, CBlock&& block, uint64_t& max_tries, std::shared_ptr<const CBlock>& block_out, bool process_new_block) { block_out.reset(); block.hashMerkleRoot = BlockMerkleRoot(block); @@ -146,27 +147,26 @@ static bool GenerateBlock(ChainstateManager& chainman, Mining& miner, CBlock& bl return true; } - block_out = std::make_shared<const CBlock>(block); + block_out = std::make_shared<const CBlock>(std::move(block)); if (!process_new_block) return true; - if (!miner.processNewBlock(block_out, nullptr)) { + if (!chainman.ProcessNewBlock(block_out, /*force_processing=*/true, /*min_pow_checked=*/true, nullptr)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); } return true; } -static UniValue generateBlocks(ChainstateManager& chainman, Mining& miner, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) +static UniValue generateBlocks(ChainstateManager& chainman, Mining& miner, const CScript& coinbase_output_script, int nGenerate, uint64_t nMaxTries) { UniValue blockHashes(UniValue::VARR); while (nGenerate > 0 && !chainman.m_interrupt) { - std::unique_ptr<CBlockTemplate> pblocktemplate(miner.createNewBlock(coinbase_script)); - if (!pblocktemplate.get()) - throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); + std::unique_ptr<BlockTemplate> block_template(miner.createNewBlock({ .coinbase_output_script = coinbase_output_script })); + CHECK_NONFATAL(block_template); std::shared_ptr<const CBlock> block_out; - if (!GenerateBlock(chainman, miner, pblocktemplate->block, nMaxTries, block_out, /*process_new_block=*/true)) { + if (!GenerateBlock(chainman, block_template->getBlock(), nMaxTries, block_out, /*process_new_block=*/true)) { break; } @@ -236,9 +236,9 @@ static RPCHelpMan generatetodescriptor() const auto num_blocks{self.Arg<int>("num_blocks")}; const auto max_tries{self.Arg<uint64_t>("maxtries")}; - CScript coinbase_script; + CScript coinbase_output_script; std::string error; - if (!getScriptFromDescriptor(self.Arg<std::string>("descriptor"), coinbase_script, error)) { + if (!getScriptFromDescriptor(self.Arg<std::string>("descriptor"), coinbase_output_script, error)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); } @@ -246,7 +246,7 @@ static RPCHelpMan generatetodescriptor() Mining& miner = EnsureMining(node); ChainstateManager& chainman = EnsureChainman(node); - return generateBlocks(chainman, miner, coinbase_script, num_blocks, max_tries); + return generateBlocks(chainman, miner, coinbase_output_script, num_blocks, max_tries); }, }; } @@ -275,7 +275,7 @@ static RPCHelpMan generatetoaddress() RPCExamples{ "\nGenerate 11 blocks to myaddress\n" + HelpExampleCli("generatetoaddress", "11 \"myaddress\"") - + "If you are using the " PACKAGE_NAME " wallet, you can get a new address to send the newly generated bitcoin to with:\n" + + "If you are using the " CLIENT_NAME " wallet, you can get a new address to send the newly generated bitcoin to with:\n" + HelpExampleCli("getnewaddress", "") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue @@ -292,9 +292,9 @@ static RPCHelpMan generatetoaddress() Mining& miner = EnsureMining(node); ChainstateManager& chainman = EnsureChainman(node); - CScript coinbase_script = GetScriptForDestination(destination); + CScript coinbase_output_script = GetScriptForDestination(destination); - return generateBlocks(chainman, miner, coinbase_script, num_blocks, max_tries); + return generateBlocks(chainman, miner, coinbase_output_script, num_blocks, max_tries); }, }; } @@ -328,16 +328,16 @@ static RPCHelpMan generateblock() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const auto address_or_descriptor = request.params[0].get_str(); - CScript coinbase_script; + CScript coinbase_output_script; std::string error; - if (!getScriptFromDescriptor(address_or_descriptor, coinbase_script, error)) { + if (!getScriptFromDescriptor(address_or_descriptor, coinbase_output_script, error)) { const auto destination = DecodeDestination(address_or_descriptor); if (!IsValidDestination(destination)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Error: Invalid address or descriptor"); } - coinbase_script = GetScriptForDestination(destination); + coinbase_output_script = GetScriptForDestination(destination); } NodeContext& node = EnsureAnyNodeContext(request.context); @@ -371,30 +371,30 @@ static RPCHelpMan generateblock() ChainstateManager& chainman = EnsureChainman(node); { - std::unique_ptr<CBlockTemplate> blocktemplate{miner.createNewBlock(coinbase_script, {.use_mempool = false})}; - if (!blocktemplate) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); + LOCK(chainman.GetMutex()); + { + std::unique_ptr<BlockTemplate> block_template{miner.createNewBlock({.use_mempool = false, .coinbase_output_script = coinbase_output_script})}; + CHECK_NONFATAL(block_template); + + block = block_template->getBlock(); } - block = blocktemplate->block; - } - CHECK_NONFATAL(block.vtx.size() == 1); + CHECK_NONFATAL(block.vtx.size() == 1); - // Add transactions - block.vtx.insert(block.vtx.end(), txs.begin(), txs.end()); - RegenerateCommitments(block, chainman); + // Add transactions + block.vtx.insert(block.vtx.end(), txs.begin(), txs.end()); + RegenerateCommitments(block, chainman); - { BlockValidationState state; - if (!miner.testBlockValidity(block, /*check_merkle_root=*/false, state)) { - throw JSONRPCError(RPC_VERIFY_ERROR, strprintf("testBlockValidity failed: %s", state.ToString())); + if (!TestBlockValidity(state, chainman.GetParams(), chainman.ActiveChainstate(), block, chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock), /*fCheckPOW=*/false, /*fCheckMerkleRoot=*/false)) { + throw JSONRPCError(RPC_VERIFY_ERROR, strprintf("TestBlockValidity failed: %s", state.ToString())); } } std::shared_ptr<const CBlock> block_out; uint64_t max_tries{DEFAULT_MAX_TRIES}; - if (!GenerateBlock(chainman, miner, block, max_tries, block_out, process_new_block) || !block_out) { + if (!GenerateBlock(chainman, std::move(block), max_tries, block_out, process_new_block) || !block_out) { throw JSONRPCError(RPC_MISC_ERROR, "Failed to make block."); } @@ -425,6 +425,7 @@ static RPCHelpMan getmininginfo() {RPCResult::Type::NUM, "networkhashps", "The network hashes per second"}, {RPCResult::Type::NUM, "pooledtx", "The size of the mempool"}, {RPCResult::Type::STR, "chain", "current network name (" LIST_CHAIN_NAMES ")"}, + {RPCResult::Type::STR_HEX, "signet_challenge", /*optional=*/true, "The block challenge (aka. block script), in hexadecimal (only present if the current network is a signet)"}, (IsDeprecatedRPCEnabled("warnings") ? RPCResult{RPCResult::Type::STR, "warnings", "any network and blockchain warnings (DEPRECATED)"} : RPCResult{RPCResult::Type::ARR, "warnings", "any network and blockchain warnings (run with `-deprecatedrpc=warnings` to return the latest warning as a single string)", @@ -454,6 +455,11 @@ static RPCHelpMan getmininginfo() obj.pushKV("networkhashps", getnetworkhashps().HandleRequest(request)); obj.pushKV("pooledtx", (uint64_t)mempool.size()); obj.pushKV("chain", chainman.GetParams().GetChainTypeString()); + if (chainman.GetParams().GetChainType() == ChainType::SIGNET) { + const std::vector<uint8_t>& signet_challenge = + chainman.GetParams().GetConsensus().signet_challenge; + obj.pushKV("signet_challenge", HexStr(signet_challenge)); + } obj.pushKV("warnings", node::GetWarningsForRpc(*CHECK_NONFATAL(node.warnings), IsDeprecatedRPCEnabled("warnings"))); return obj; }, @@ -493,7 +499,15 @@ static RPCHelpMan prioritisetransaction() throw JSONRPCError(RPC_INVALID_PARAMETER, "Priority is no longer supported, dummy argument to prioritisetransaction must be 0."); } - EnsureAnyMemPool(request.context).PrioritiseTransaction(hash, nAmount); + CTxMemPool& mempool = EnsureAnyMemPool(request.context); + + // Non-0 fee dust transactions are not allowed for entry, and modification not allowed afterwards + const auto& tx = mempool.get(hash); + if (mempool.m_opts.require_standard && tx && !GetDust(*tx, mempool.m_opts.dust_relay_feerate).empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Priority is not supported for transactions with dust outputs."); + } + + mempool.PrioritiseTransaction(hash, nAmount); return true; }, }; @@ -619,8 +633,8 @@ static RPCHelpMan getblocktemplate() {RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR_HEX, "data", "transaction data encoded in hexadecimal (byte-for-byte)"}, - {RPCResult::Type::STR_HEX, "txid", "transaction id encoded in little-endian hexadecimal"}, - {RPCResult::Type::STR_HEX, "hash", "hash encoded in little-endian hexadecimal (including witness data)"}, + {RPCResult::Type::STR_HEX, "txid", "transaction hash excluding witness data, shown in byte-reversed hex"}, + {RPCResult::Type::STR_HEX, "hash", "transaction hash including witness data, shown in byte-reversed hex"}, {RPCResult::Type::ARR, "depends", "array of numbers", { {RPCResult::Type::NUM, "", "transactions before this one (by 1-based index in 'transactions' list) that must be present in the final block if this one is"}, @@ -663,7 +677,7 @@ static RPCHelpMan getblocktemplate() ChainstateManager& chainman = EnsureChainman(node); Mining& miner = EnsureMining(node); LOCK(cs_main); - uint256 tip{CHECK_NONFATAL(miner.getTipHash()).value()}; + uint256 tip{CHECK_NONFATAL(miner.getTip()).value().hash}; std::string strMode = "template"; UniValue lpval = NullUniValue; @@ -702,12 +716,12 @@ static RPCHelpMan getblocktemplate() return "duplicate-inconclusive"; } - // testBlockValidity only supports blocks built on the current Tip + // TestBlockValidity only supports blocks built on the current Tip if (block.hashPrevBlock != tip) { return "inconclusive-not-best-prevblk"; } BlockValidationState state; - miner.testBlockValidity(block, /*check_merkle_root=*/true, state); + TestBlockValidity(state, chainman.GetParams(), chainman.ActiveChainstate(), block, chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock), /*fCheckPOW=*/false, /*fCheckMerkleRoot=*/true); return BIP22ValidationResult(state); } @@ -726,21 +740,21 @@ static RPCHelpMan getblocktemplate() if (!miner.isTestChain()) { const CConnman& connman = EnsureConnman(node); if (connman.GetNodeCount(ConnectionDirection::Both) == 0) { - throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, CLIENT_NAME " is not connected!"); } if (miner.isInitialBlockDownload()) { - throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); + throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, CLIENT_NAME " is in initial sync and waiting for blocks..."); } } static unsigned int nTransactionsUpdatedLast; + const CTxMemPool& mempool = EnsureMemPool(node); if (!lpval.isNull()) { // Wait to respond until either the best block changes, OR a minute has passed and there are more transactions uint256 hashWatchedChain; - std::chrono::steady_clock::time_point checktxtime; unsigned int nTransactionsUpdatedLastLP; if (lpval.isStr()) @@ -761,24 +775,19 @@ static RPCHelpMan getblocktemplate() // Release lock while waiting LEAVE_CRITICAL_SECTION(cs_main); { - checktxtime = std::chrono::steady_clock::now() + std::chrono::minutes(1); - - WAIT_LOCK(g_best_block_mutex, lock); - while (g_best_block == hashWatchedChain && IsRPCRunning()) - { - if (g_best_block_cv.wait_until(lock, checktxtime) == std::cv_status::timeout) - { - // Timeout: Check transactions for update - // without holding the mempool lock to avoid deadlocks - if (miner.getTransactionsUpdated() != nTransactionsUpdatedLastLP) - break; - checktxtime += std::chrono::seconds(10); - } + MillisecondsDouble checktxtime{std::chrono::minutes(1)}; + while (tip == hashWatchedChain && IsRPCRunning()) { + tip = miner.waitTipChanged(hashWatchedChain, checktxtime).hash; + // Timeout: Check transactions for update + // without holding the mempool lock to avoid deadlocks + if (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLastLP) + break; + checktxtime = std::chrono::seconds(10); } } ENTER_CRITICAL_SECTION(cs_main); - tip = CHECK_NONFATAL(miner.getTipHash()).value(); + tip = CHECK_NONFATAL(miner.getTip()).value().hash; if (!IsRPCRunning()) throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); @@ -800,34 +809,32 @@ static RPCHelpMan getblocktemplate() // Update block static CBlockIndex* pindexPrev; static int64_t time_start; - static std::unique_ptr<CBlockTemplate> pblocktemplate; + static std::unique_ptr<BlockTemplate> block_template; if (!pindexPrev || pindexPrev->GetBlockHash() != tip || - (miner.getTransactionsUpdated() != nTransactionsUpdatedLast && GetTime() - time_start > 5)) + (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLast && GetTime() - time_start > 5)) { // Clear pindexPrev so future calls make a new block, despite any failures from here on pindexPrev = nullptr; // Store the pindexBest used before createNewBlock, to avoid races - nTransactionsUpdatedLast = miner.getTransactionsUpdated(); + nTransactionsUpdatedLast = mempool.GetTransactionsUpdated(); CBlockIndex* pindexPrevNew = chainman.m_blockman.LookupBlockIndex(tip); time_start = GetTime(); // Create new block - CScript scriptDummy = CScript() << OP_TRUE; - pblocktemplate = miner.createNewBlock(scriptDummy); - if (!pblocktemplate) { - throw JSONRPCError(RPC_OUT_OF_MEMORY, "Out of memory"); - } + block_template = miner.createNewBlock(); + CHECK_NONFATAL(block_template); + // Need to update only after we know createNewBlock succeeded pindexPrev = pindexPrevNew; } CHECK_NONFATAL(pindexPrev); - CBlock* pblock = &pblocktemplate->block; // pointer for convenience + CBlock block{block_template->getBlock()}; // Update nTime - UpdateTime(pblock, consensusParams, pindexPrev); - pblock->nNonce = 0; + UpdateTime(&block, consensusParams, pindexPrev); + block.nNonce = 0; // NOTE: If at some point we support pre-segwit miners post-segwit-activation, this needs to take segwit support into consideration const bool fPreSegWit = !DeploymentActiveAfter(pindexPrev, chainman, Consensus::DEPLOYMENT_SEGWIT); @@ -836,8 +843,11 @@ static RPCHelpMan getblocktemplate() UniValue transactions(UniValue::VARR); std::map<uint256, int64_t> setTxIndex; + std::vector<CAmount> tx_fees{block_template->getTxFees()}; + std::vector<CAmount> tx_sigops{block_template->getTxSigops()}; + int i = 0; - for (const auto& it : pblock->vtx) { + for (const auto& it : block.vtx) { const CTransaction& tx = *it; uint256 txHash = tx.GetHash(); setTxIndex[txHash] = i++; @@ -860,8 +870,8 @@ static RPCHelpMan getblocktemplate() entry.pushKV("depends", std::move(deps)); int index_in_template = i - 1; - entry.pushKV("fee", pblocktemplate->vTxFees[index_in_template]); - int64_t nTxSigOps = pblocktemplate->vTxSigOpsCost[index_in_template]; + entry.pushKV("fee", tx_fees.at(index_in_template)); + int64_t nTxSigOps{tx_sigops.at(index_in_template)}; if (fPreSegWit) { CHECK_NONFATAL(nTxSigOps % WITNESS_SCALE_FACTOR == 0); nTxSigOps /= WITNESS_SCALE_FACTOR; @@ -874,7 +884,7 @@ static RPCHelpMan getblocktemplate() UniValue aux(UniValue::VOBJ); - arith_uint256 hashTarget = arith_uint256().SetCompact(pblock->nBits); + arith_uint256 hashTarget = arith_uint256().SetCompact(block.nBits); UniValue aMutable(UniValue::VARR); aMutable.push_back("time"); @@ -904,7 +914,7 @@ static RPCHelpMan getblocktemplate() break; case ThresholdState::LOCKED_IN: // Ensure bit is set in block version - pblock->nVersion |= chainman.m_versionbitscache.Mask(consensusParams, pos); + block.nVersion |= chainman.m_versionbitscache.Mask(consensusParams, pos); [[fallthrough]]; case ThresholdState::STARTED: { @@ -913,7 +923,7 @@ static RPCHelpMan getblocktemplate() if (setClientRules.find(vbinfo.name) == setClientRules.end()) { if (!vbinfo.gbt_force) { // If the client doesn't support this, don't indicate it in the [default] version - pblock->nVersion &= ~chainman.m_versionbitscache.Mask(consensusParams, pos); + block.nVersion &= ~chainman.m_versionbitscache.Mask(consensusParams, pos); } } break; @@ -933,15 +943,15 @@ static RPCHelpMan getblocktemplate() } } } - result.pushKV("version", pblock->nVersion); + result.pushKV("version", block.nVersion); result.pushKV("rules", std::move(aRules)); result.pushKV("vbavailable", std::move(vbavailable)); result.pushKV("vbrequired", int(0)); - result.pushKV("previousblockhash", pblock->hashPrevBlock.GetHex()); + result.pushKV("previousblockhash", block.hashPrevBlock.GetHex()); result.pushKV("transactions", std::move(transactions)); result.pushKV("coinbaseaux", std::move(aux)); - result.pushKV("coinbasevalue", (int64_t)pblock->vtx[0]->vout[0].nValue); + result.pushKV("coinbasevalue", (int64_t)block.vtx[0]->vout[0].nValue); result.pushKV("longpollid", tip.GetHex() + ToString(nTransactionsUpdatedLast)); result.pushKV("target", hashTarget.GetHex()); result.pushKV("mintime", (int64_t)pindexPrev->GetMedianTimePast()+1); @@ -960,16 +970,16 @@ static RPCHelpMan getblocktemplate() if (!fPreSegWit) { result.pushKV("weightlimit", (int64_t)MAX_BLOCK_WEIGHT); } - result.pushKV("curtime", pblock->GetBlockTime()); - result.pushKV("bits", strprintf("%08x", pblock->nBits)); + result.pushKV("curtime", block.GetBlockTime()); + result.pushKV("bits", strprintf("%08x", block.nBits)); result.pushKV("height", (int64_t)(pindexPrev->nHeight+1)); if (consensusParams.signet_blocks) { result.pushKV("signet_challenge", HexStr(consensusParams.signet_challenge)); } - if (!pblocktemplate->vchCoinbaseCommitment.empty()) { - result.pushKV("default_witness_commitment", HexStr(pblocktemplate->vchCoinbaseCommitment)); + if (!block_template->getCoinbaseCommitment().empty()) { + result.pushKV("default_witness_commitment", HexStr(block_template->getCoinbaseCommitment())); } return result; @@ -1021,25 +1031,7 @@ static RPCHelpMan submitblock() throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block decode failed"); } - if (block.vtx.empty() || !block.vtx[0]->IsCoinBase()) { - throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block does not start with a coinbase"); - } - ChainstateManager& chainman = EnsureAnyChainman(request.context); - uint256 hash = block.GetHash(); - { - LOCK(cs_main); - const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); - if (pindex) { - if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) { - return "duplicate"; - } - if (pindex->nStatus & BLOCK_FAILED_MASK) { - return "duplicate-invalid"; - } - } - } - { LOCK(cs_main); const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock); @@ -1048,13 +1040,10 @@ static RPCHelpMan submitblock() } } - NodeContext& node = EnsureAnyNodeContext(request.context); - Mining& miner = EnsureMining(node); - bool new_block; auto sc = std::make_shared<submitblock_StateCatcher>(block.GetHash()); CHECK_NONFATAL(chainman.m_options.signals)->RegisterSharedValidationInterface(sc); - bool accepted = miner.processNewBlock(blockptr, /*new_block=*/&new_block); + bool accepted = chainman.ProcessNewBlock(blockptr, /*force_processing=*/true, /*min_pow_checked=*/true, /*new_block=*/&new_block); CHECK_NONFATAL(chainman.m_options.signals)->UnregisterSharedValidationInterface(sc); if (!new_block && accepted) { return "duplicate"; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 1119a3e668..bda07365e0 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -129,8 +129,8 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::STR, "addrbind", /*optional=*/true, "(ip:port) Bind address of the connection to the peer"}, {RPCResult::Type::STR, "addrlocal", /*optional=*/true, "(ip:port) Local address as reported by the peer"}, {RPCResult::Type::STR, "network", "Network (" + Join(GetNetworkNames(/*append_unroutable=*/true), ", ") + ")"}, - {RPCResult::Type::NUM, "mapped_as", /*optional=*/true, "The AS in the BGP route to the peer used for diversifying\n" - "peer selection (only available if the asmap config flag is set)"}, + {RPCResult::Type::NUM, "mapped_as", /*optional=*/true, "Mapped AS (Autonomous System) number at the end of the BGP route to the peer, used for diversifying\n" + "peer selection (only displayed if the -asmap config option is set)"}, {RPCResult::Type::STR_HEX, "services", "The services offered"}, {RPCResult::Type::ARR, "servicesnames", "the services offered, in human-readable form", { @@ -1024,7 +1024,7 @@ static RPCHelpMan sendmsgtopeer() "This RPC is for testing only.", { {"peer_id", RPCArg::Type::NUM, RPCArg::Optional::NO, "The peer to send the message to."}, - {"msg_type", RPCArg::Type::STR, RPCArg::Optional::NO, strprintf("The message type (maximum length %i)", CMessageHeader::COMMAND_SIZE)}, + {"msg_type", RPCArg::Type::STR, RPCArg::Optional::NO, strprintf("The message type (maximum length %i)", CMessageHeader::MESSAGE_TYPE_SIZE)}, {"msg", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The serialized message body to send, in hex, without a message header"}, }, RPCResult{RPCResult::Type::OBJ, "", "", std::vector<RPCResult>{}}, @@ -1033,8 +1033,8 @@ static RPCHelpMan sendmsgtopeer() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const NodeId peer_id{request.params[0].getInt<int64_t>()}; const std::string& msg_type{request.params[1].get_str()}; - if (msg_type.size() > CMessageHeader::COMMAND_SIZE) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Error: msg_type too long, max length is %i", CMessageHeader::COMMAND_SIZE)); + if (msg_type.size() > CMessageHeader::MESSAGE_TYPE_SIZE) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Error: msg_type too long, max length is %i", CMessageHeader::MESSAGE_TYPE_SIZE)); } auto msg{TryParseHex<unsigned char>(request.params[2].get_str())}; if (!msg.has_value()) { @@ -1102,12 +1102,12 @@ static RPCHelpMan getaddrmaninfo() }; } -UniValue AddrmanEntryToJSON(const AddrInfo& info, CConnman& connman) +UniValue AddrmanEntryToJSON(const AddrInfo& info, const CConnman& connman) { UniValue ret(UniValue::VOBJ); ret.pushKV("address", info.ToStringAddr()); - const auto mapped_as{connman.GetMappedAS(info)}; - if (mapped_as != 0) { + const uint32_t mapped_as{connman.GetMappedAS(info)}; + if (mapped_as) { ret.pushKV("mapped_as", mapped_as); } ret.pushKV("port", info.GetPort()); @@ -1116,14 +1116,14 @@ UniValue AddrmanEntryToJSON(const AddrInfo& info, CConnman& connman) ret.pushKV("network", GetNetworkName(info.GetNetClass())); ret.pushKV("source", info.source.ToStringAddr()); ret.pushKV("source_network", GetNetworkName(info.source.GetNetClass())); - const auto source_mapped_as{connman.GetMappedAS(info.source)}; - if (source_mapped_as != 0) { + const uint32_t source_mapped_as{connman.GetMappedAS(info.source)}; + if (source_mapped_as) { ret.pushKV("source_mapped_as", source_mapped_as); } return ret; } -UniValue AddrmanTableToJSON(const std::vector<std::pair<AddrInfo, AddressPosition>>& tableInfos, CConnman& connman) +UniValue AddrmanTableToJSON(const std::vector<std::pair<AddrInfo, AddressPosition>>& tableInfos, const CConnman& connman) { UniValue table(UniValue::VOBJ); for (const auto& e : tableInfos) { @@ -1150,14 +1150,14 @@ static RPCHelpMan getrawaddrman() {RPCResult::Type::OBJ_DYN, "table", "buckets with addresses in the address manager table ( new, tried )", { {RPCResult::Type::OBJ, "bucket/position", "the location in the address manager table (<bucket>/<position>)", { {RPCResult::Type::STR, "address", "The address of the node"}, - {RPCResult::Type::NUM, "mapped_as", /*optional=*/true, "The ASN mapped to the IP of this peer per our current ASMap"}, + {RPCResult::Type::NUM, "mapped_as", /*optional=*/true, "Mapped AS (Autonomous System) number at the end of the BGP route to the peer, used for diversifying peer selection (only displayed if the -asmap config option is set)"}, {RPCResult::Type::NUM, "port", "The port number of the node"}, {RPCResult::Type::STR, "network", "The network (" + Join(GetNetworkNames(), ", ") + ") of the address"}, {RPCResult::Type::NUM, "services", "The services offered by the node"}, {RPCResult::Type::NUM_TIME, "time", "The " + UNIX_EPOCH_TIME + " when the node was last seen"}, {RPCResult::Type::STR, "source", "The address that relayed the address to us"}, {RPCResult::Type::STR, "source_network", "The network (" + Join(GetNetworkNames(), ", ") + ") of the source address"}, - {RPCResult::Type::NUM, "source_mapped_as", /*optional=*/true, "The ASN mapped to the IP of this peer's source per our current ASMap"} + {RPCResult::Type::NUM, "source_mapped_as", /*optional=*/true, "Mapped AS (Autonomous System) number at the end of the BGP route to the source, used for diversifying peer selection (only displayed if the -asmap config option is set)"} }} }} } diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index af2e1d0eaa..5e36273cf4 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chainparams.h> #include <httpserver.h> diff --git a/src/rpc/output_script.cpp b/src/rpc/output_script.cpp index 49f3a81243..5de2847be9 100644 --- a/src/rpc/output_script.cpp +++ b/src/rpc/output_script.cpp @@ -123,6 +123,7 @@ static RPCHelpMan createmultisig() // Get the public keys const UniValue& keys = request.params[1].get_array(); std::vector<CPubKey> pubkeys; + pubkeys.reserve(keys.size()); for (unsigned int i = 0; i < keys.size(); ++i) { pubkeys.push_back(HexToPubKey(keys[i].get_str())); } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 21bc0e52f1..184fff9386 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -82,17 +82,6 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& } } -static std::vector<RPCResult> ScriptPubKeyDoc() { - return - { - {RPCResult::Type::STR, "asm", "Disassembly of the output script"}, - {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", "The raw output script bytes, hex-encoded"}, - {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::STR, "type", "The type (one of: " + GetAllOutputTypes() + ")"}, - }; -} - static std::vector<RPCResult> DecodeTxDoc(const std::string& txid_field_doc) { return { @@ -338,15 +327,7 @@ static RPCHelpMan getrawtransaction() throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "The genesis block coinbase is not considered an ordinary transaction and cannot be retrieved"); } - // Accept either a bool (true) or a num (>=0) to indicate verbosity. - int verbosity{0}; - if (!request.params[1].isNull()) { - if (request.params[1].isBool()) { - verbosity = request.params[1].get_bool(); - } else { - verbosity = request.params[1].getInt<int>(); - } - } + int verbosity{ParseVerbosity(request.params[1], /*default_verbosity=*/0, /*allow_bool=*/true)}; if (!request.params[2].isNull()) { LOCK(cs_main); @@ -405,11 +386,16 @@ static RPCHelpMan getrawtransaction() CBlockUndo blockUndo; CBlock block; - if (tx->IsCoinBase() || !blockindex || WITH_LOCK(::cs_main, return chainman.m_blockman.IsBlockPruned(*blockindex)) || - !(chainman.m_blockman.UndoReadFromDisk(blockUndo, *blockindex) && chainman.m_blockman.ReadBlockFromDisk(block, *blockindex))) { + if (tx->IsCoinBase() || !blockindex || WITH_LOCK(::cs_main, return !(blockindex->nStatus & BLOCK_HAVE_MASK))) { TxToJSON(*tx, hash_block, result, chainman.ActiveChainstate()); return result; } + if (!chainman.m_blockman.UndoReadFromDisk(blockUndo, *blockindex)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Undo data expected but can't be read. This could be due to disk corruption or a conflict with a pruning event."); + } + if (!chainman.m_blockman.ReadBlockFromDisk(block, *blockindex)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Block data expected but can't be read. This could be due to disk corruption or a conflict with a pruning event."); + } CTxUndo* undoTX {nullptr}; auto it = std::find_if(block.vtx.begin(), block.vtx.end(), [tx](CTransactionRef t){ return *t == *tx; }); diff --git a/src/rpc/register.h b/src/rpc/register.h index 65fd29ff08..17ed6c142c 100644 --- a/src/rpc/register.h +++ b/src/rpc/register.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_RPC_REGISTER_H #define BITCOIN_RPC_REGISTER_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep /** These are in one header file to avoid creating tons of single-function * headers for everything under src/rpc/ */ diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index afd98f8875..c0ae94bc4f 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -86,6 +86,9 @@ static const char* const COOKIEAUTH_FILE = ".cookie"; static fs::path GetAuthCookieFile(bool temp=false) { fs::path arg = gArgs.GetPathArg("-rpccookiefile", COOKIEAUTH_FILE); + if (arg.empty()) { + return {}; // -norpccookiefile was specified + } if (temp) { arg += ".tmp"; } @@ -106,9 +109,12 @@ bool GenerateAuthCookie(std::string* cookie_out, std::optional<fs::perms> cookie */ std::ofstream file; fs::path filepath_tmp = GetAuthCookieFile(true); + if (filepath_tmp.empty()) { + return true; // -norpccookiefile + } file.open(filepath_tmp); if (!file.is_open()) { - LogInfo("Unable to open cookie authentication file %s for writing\n", fs::PathToString(filepath_tmp)); + LogWarning("Unable to open cookie authentication file %s for writing", fs::PathToString(filepath_tmp)); return false; } file << cookie; @@ -116,14 +122,14 @@ bool GenerateAuthCookie(std::string* cookie_out, std::optional<fs::perms> cookie fs::path filepath = GetAuthCookieFile(false); if (!RenameOver(filepath_tmp, filepath)) { - LogInfo("Unable to rename cookie authentication file %s to %s\n", fs::PathToString(filepath_tmp), fs::PathToString(filepath)); + LogWarning("Unable to rename cookie authentication file %s to %s", fs::PathToString(filepath_tmp), fs::PathToString(filepath)); return false; } if (cookie_perms) { std::error_code code; fs::permissions(filepath, cookie_perms.value(), fs::perm_options::replace, code); if (code) { - LogInfo("Unable to set permissions on cookie authentication file %s\n", fs::PathToString(filepath_tmp)); + LogWarning("Unable to set permissions on cookie authentication file %s", fs::PathToString(filepath)); return false; } } @@ -142,6 +148,9 @@ bool GetAuthCookie(std::string *cookie_out) std::ifstream file; std::string cookie; fs::path filepath = GetAuthCookieFile(); + if (filepath.empty()) { + return true; // -norpccookiefile + } file.open(filepath); if (!file.is_open()) return false; diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 2c07a2ff08..34f19df256 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <rpc/server.h> @@ -11,6 +11,7 @@ #include <common/system.h> #include <logging.h> #include <node/context.h> +#include <node/kernel_notifications.h> #include <rpc/server_util.h> #include <rpc/util.h> #include <sync.h> @@ -18,8 +19,7 @@ #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> - -#include <boost/signals2/signal.hpp> +#include <validation.h> #include <cassert> #include <chrono> @@ -69,22 +69,6 @@ struct RPCCommandExecution } }; -static struct CRPCSignals -{ - boost::signals2::signal<void ()> Started; - boost::signals2::signal<void ()> Stopped; -} g_rpcSignals; - -void RPCServer::OnStarted(std::function<void ()> slot) -{ - g_rpcSignals.Started.connect(slot); -} - -void RPCServer::OnStopped(std::function<void ()> slot) -{ - g_rpcSignals.Stopped.connect(slot); -} - std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest& helpreq) const { std::string strRet; @@ -170,12 +154,12 @@ static RPCHelpMan help() static RPCHelpMan stop() { - static const std::string RESULT{PACKAGE_NAME " stopping"}; + static const std::string RESULT{CLIENT_NAME " stopping"}; return RPCHelpMan{"stop", // Also accept the hidden 'wait' integer argument (milliseconds) // For instance, 'stop 1000' makes the call wait 1 second before returning // to the client (intended for testing) - "\nRequest a graceful shutdown of " PACKAGE_NAME ".", + "\nRequest a graceful shutdown of " CLIENT_NAME ".", { {"wait", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "how long to wait in ms", RPCArgOptions{.hidden=true}}, }, @@ -185,7 +169,7 @@ static RPCHelpMan stop() { // Event loop will exit after current HTTP requests have been handled, so // this reply will get back to the client. - CHECK_NONFATAL((*CHECK_NONFATAL(EnsureAnyNodeContext(jsonRequest.context).shutdown))()); + CHECK_NONFATAL((CHECK_NONFATAL(EnsureAnyNodeContext(jsonRequest.context).shutdown_request))()); if (jsonRequest.params[0].isNum()) { UninterruptibleSleep(std::chrono::milliseconds{jsonRequest.params[0].getInt<int>()}); } @@ -297,7 +281,6 @@ void StartRPC() { LogDebug(BCLog::RPC, "Starting RPC\n"); g_rpc_running = true; - g_rpcSignals.Started(); } void InterruptRPC() @@ -316,11 +299,11 @@ void StopRPC() static std::once_flag g_rpc_stop_flag; // This function could be called twice if the GUI has been started with -server=1. assert(!g_rpc_running); - std::call_once(g_rpc_stop_flag, []() { + std::call_once(g_rpc_stop_flag, [&]() { LogDebug(BCLog::RPC, "Stopping RPC\n"); WITH_LOCK(g_deadline_timers_mutex, deadlineTimers.clear()); DeleteAuthCookie(); - g_rpcSignals.Stopped(); + LogDebug(BCLog::RPC, "RPC stopped.\n"); }); } diff --git a/src/rpc/server.h b/src/rpc/server.h index 56e8a63088..5a22279a58 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -18,12 +18,6 @@ class CRPCCommand; -namespace RPCServer -{ - void OnStarted(std::function<void ()> slot); - void OnStopped(std::function<void ()> slot); -} - /** Query whether RPC is running */ bool IsRPCRunning(); diff --git a/src/rpc/txoutproof.cpp b/src/rpc/txoutproof.cpp index 7958deb677..40294fda06 100644 --- a/src/rpc/txoutproof.cpp +++ b/src/rpc/txoutproof.cpp @@ -10,6 +10,7 @@ #include <merkleblock.h> #include <node/blockstorage.h> #include <primitives/transaction.h> +#include <rpc/blockchain.h> #include <rpc/server.h> #include <rpc/server_util.h> #include <rpc/util.h> @@ -96,6 +97,10 @@ static RPCHelpMan gettxoutproof() } } + { + LOCK(cs_main); + CheckBlockDataAvailability(chainman.m_blockman, *pblockindex, /*check_for_undo=*/false); + } CBlock block; if (!chainman.m_blockman.ReadBlockFromDisk(block, *pblockindex)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 678bac7a18..b1fbc25641 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <clientversion.h> #include <common/args.h> @@ -81,6 +81,21 @@ void RPCTypeCheckObj(const UniValue& o, } } +int ParseVerbosity(const UniValue& arg, int default_verbosity, bool allow_bool) +{ + if (!arg.isNull()) { + if (arg.isBool()) { + if (!allow_bool) { + throw JSONRPCError(RPC_TYPE_ERROR, "Verbosity was boolean but only integer allowed"); + } + return arg.get_bool(); // true = 1 + } else { + return arg.getInt<int>(); + } + } + return default_verbosity; +} + CAmount AmountFromValue(const UniValue& value, int decimals) { if (!value.isNum() && !value.isStr()) @@ -684,8 +699,8 @@ UniValue RPCHelpMan::HandleRequest(const JSONRPCRequest& request) const throw std::runtime_error{ strprintf("Internal bug detected: RPC call \"%s\" returned incorrect type:\n%s\n%s %s\nPlease report this issue here: %s\n", m_name, explain, - PACKAGE_NAME, FormatFullVersion(), - PACKAGE_BUGREPORT)}; + CLIENT_NAME, FormatFullVersion(), + CLIENT_BUGREPORT)}; } } return ret; @@ -1392,3 +1407,14 @@ void PushWarnings(const std::vector<bilingual_str>& warnings, UniValue& obj) if (warnings.empty()) return; obj.pushKV("warnings", BilingualStringsToUniValue(warnings)); } + +std::vector<RPCResult> ScriptPubKeyDoc() { + return + { + {RPCResult::Type::STR, "asm", "Disassembly of the output script"}, + {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, + {RPCResult::Type::STR_HEX, "hex", "The raw output script bytes, hex-encoded"}, + {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, + {RPCResult::Type::STR, "type", "The type (one of: " + GetAllOutputTypes() + ")"}, + }; +} diff --git a/src/rpc/util.h b/src/rpc/util.h index 23024376e0..5bb73d32ac 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -101,6 +101,17 @@ std::vector<unsigned char> ParseHexV(const UniValue& v, std::string_view name); std::vector<unsigned char> ParseHexO(const UniValue& o, std::string_view strKey); /** + * Parses verbosity from provided UniValue. + * + * @param[in] arg The verbosity argument as an int (0, 1, 2,...) or bool if allow_bool is set to true + * @param[in] default_verbosity The value to return if verbosity argument is null + * @param[in] allow_bool If true, allows arg to be a bool and parses it + * @returns An integer describing the verbosity level (e.g. 0, 1, 2, etc.) + * @throws JSONRPCError if allow_bool is false but arg provided is boolean + */ +int ParseVerbosity(const UniValue& arg, int default_verbosity, bool allow_bool); + +/** * Validate and return a CAmount from a UniValue number or string. * * @param[in] value UniValue number or string to parse. @@ -503,4 +514,6 @@ private: void PushWarnings(const UniValue& warnings, UniValue& obj); void PushWarnings(const std::vector<bilingual_str>& warnings, UniValue& obj); +std::vector<RPCResult> ScriptPubKeyDoc(); + #endif // BITCOIN_RPC_UTIL_H diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 5026470edc..2e1a30744e 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -1761,7 +1761,7 @@ struct KeyParser { std::vector<std::unique_ptr<DescriptorImpl>> ParseScript(uint32_t& key_exp_index, Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, std::string& error) { using namespace script; - + Assume(ctx == ParseScriptContext::TOP || ctx == ParseScriptContext::P2SH || ctx == ParseScriptContext::P2WSH || ctx == ParseScriptContext::P2TR); std::vector<std::unique_ptr<DescriptorImpl>> ret; auto expr = Expr(sp); if (Func("pk", expr)) { @@ -1787,10 +1787,6 @@ std::vector<std::unique_ptr<DescriptorImpl>> ParseScript(uint32_t& key_exp_index ret.emplace_back(std::make_unique<PKHDescriptor>(std::move(pubkey))); } return ret; - } else if (ctx != ParseScriptContext::P2TR && Func("pkh", expr)) { - // Under Taproot, always the Miniscript parser deal with it. - error = "Can only have pkh at top level, in sh(), wsh(), or in tr()"; - return {}; } if (ctx == ParseScriptContext::TOP && Func("combo", expr)) { auto pubkeys = ParsePubkey(key_exp_index, expr, ctx, out, error); diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 9d0e9b5e3c..a35306b693 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -51,8 +51,8 @@ bool CastToBool(const valtype& vch) * Script is a stack machine (like Forth) that evaluates a predicate * returning a bool indicating valid or not. There are no loops. */ -#define stacktop(i) (stack.at(stack.size()+(i))) -#define altstacktop(i) (altstack.at(altstack.size()+(i))) +#define stacktop(i) (stack.at(size_t(int64_t(stack.size()) + int64_t{i}))) +#define altstacktop(i) (altstack.at(size_t(int64_t(altstack.size()) + int64_t{i}))) static inline void popstack(std::vector<valtype>& stack) { if (stack.empty()) @@ -1303,7 +1303,7 @@ public: // Serialize the nSequence if (nInput != nIn && (fHashSingle || fHashNone)) // let the others update at will - ::Serialize(s, int{0}); + ::Serialize(s, int32_t{0}); else ::Serialize(s, txTo.vin[nInput].nSequence); } @@ -1565,7 +1565,7 @@ bool SignatureHashSchnorr(uint256& hash_out, ScriptExecutionData& execdata, cons } template <class T> -uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn, int nHashType, const CAmount& amount, SigVersion sigversion, const PrecomputedTransactionData* cache) +uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn, int32_t nHashType, const CAmount& amount, SigVersion sigversion, const PrecomputedTransactionData* cache) { assert(nIn < txTo.vin.size()); diff --git a/src/script/interpreter.h b/src/script/interpreter.h index 8ba0018c23..e2fb1998f0 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -240,7 +240,7 @@ extern const HashWriter HASHER_TAPLEAF; //!< Hasher with tag "TapLeaf" pre-fe extern const HashWriter HASHER_TAPBRANCH; //!< Hasher with tag "TapBranch" pre-fed to it. template <class T> -uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn, int nHashType, const CAmount& amount, SigVersion sigversion, const PrecomputedTransactionData* cache = nullptr); +uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn, int32_t nHashType, const CAmount& amount, SigVersion sigversion, const PrecomputedTransactionData* cache = nullptr); class BaseSignatureChecker { diff --git a/src/script/miniscript.cpp b/src/script/miniscript.cpp index 455bd56283..4b8d3673f9 100644 --- a/src/script/miniscript.cpp +++ b/src/script/miniscript.cpp @@ -1,14 +1,17 @@ -// Copyright (c) 2019-2022 The Bitcoin Core developers +// Copyright (c) 2019-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <string> +#include <limits> #include <vector> -#include <script/script.h> -#include <script/miniscript.h> -#include <serialize.h> -#include <assert.h> +#include <primitives/transaction.h> +#include <script/miniscript.h> +#include <script/script.h> +#include <script/solver.h> +#include <span.h> +#include <util/check.h> +#include <util/vector.h> namespace miniscript { namespace internal { diff --git a/src/script/miniscript.h b/src/script/miniscript.h index 58f24434f0..75f978457c 100644 --- a/src/script/miniscript.h +++ b/src/script/miniscript.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022 The Bitcoin Core developers +// Copyright (c) 2019-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,20 +6,24 @@ #define BITCOIN_SCRIPT_MINISCRIPT_H #include <algorithm> -#include <functional> -#include <numeric> +#include <compare> +#include <cstdint> +#include <cstdlib> +#include <iterator> #include <memory> #include <optional> -#include <string> +#include <set> +#include <stdexcept> +#include <tuple> +#include <utility> #include <vector> -#include <assert.h> -#include <cstdlib> - +#include <consensus/consensus.h> #include <policy/policy.h> -#include <primitives/transaction.h> +#include <script/interpreter.h> #include <script/parsing.h> #include <script/script.h> +#include <serialize.h> #include <span.h> #include <util/check.h> #include <util/strencodings.h> @@ -61,7 +65,7 @@ namespace miniscript { * - Is always "OP_SWAP [B]" or "OP_TOALTSTACK [B] OP_FROMALTSTACK". * - For example sc:pk_k(key) = OP_SWAP <key> OP_CHECKSIG * - * There a type properties that help reasoning about correctness: + * There are type properties that help reasoning about correctness: * - "z" Zero-arg: * - Is known to always consume exactly 0 stack elements. * - For example after(n) = <n> OP_CHECKLOCKTIMEVERIFY @@ -84,7 +88,7 @@ namespace miniscript { * - "e" Expression: * - This implies property 'd', but the dissatisfaction is nonmalleable. * - This generally requires 'e' for all subexpressions which are invoked for that - * dissatifsaction, and property 'f' for the unexecuted subexpressions in that case. + * dissatisfaction, and property 'f' for the unexecuted subexpressions in that case. * - Conflicts with type 'V'. * - "f" Forced: * - Dissatisfactions (if any) for this expression always involve at least one signature. @@ -150,7 +154,8 @@ public: }; //! Literal operator to construct Type objects. -inline consteval Type operator"" _mst(const char* c, size_t l) { +inline consteval Type operator""_mst(const char* c, size_t l) +{ Type typ{Type::Make(0)}; for (const char *p = c; p < c + l; p++) { @@ -1793,7 +1798,7 @@ inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) // Get threshold int next_comma = FindNextChar(in, ','); if (next_comma < 1) return false; - const auto k_to_integral{ToIntegral<int64_t>(std::string_view(in.begin(), next_comma))}; + const auto k_to_integral{ToIntegral<int64_t>(std::string_view(in.data(), next_comma))}; if (!k_to_integral.has_value()) return false; const int64_t k{k_to_integral.value()}; in = in.subspan(next_comma + 1); @@ -1949,7 +1954,7 @@ inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) } else if (Const("after(", in)) { int arg_size = FindNextChar(in, ')'); if (arg_size < 1) return {}; - const auto num{ToIntegral<int64_t>(std::string_view(in.begin(), arg_size))}; + const auto num{ToIntegral<int64_t>(std::string_view(in.data(), arg_size))}; if (!num.has_value() || *num < 1 || *num >= 0x80000000L) return {}; constructed.push_back(MakeNodeRef<Key>(internal::NoDupCheck{}, ctx.MsContext(), Fragment::AFTER, *num)); in = in.subspan(arg_size + 1); @@ -1957,7 +1962,7 @@ inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) } else if (Const("older(", in)) { int arg_size = FindNextChar(in, ')'); if (arg_size < 1) return {}; - const auto num{ToIntegral<int64_t>(std::string_view(in.begin(), arg_size))}; + const auto num{ToIntegral<int64_t>(std::string_view(in.data(), arg_size))}; if (!num.has_value() || *num < 1 || *num >= 0x80000000L) return {}; constructed.push_back(MakeNodeRef<Key>(internal::NoDupCheck{}, ctx.MsContext(), Fragment::OLDER, *num)); in = in.subspan(arg_size + 1); @@ -1969,7 +1974,7 @@ inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) } else if (Const("thresh(", in)) { int next_comma = FindNextChar(in, ','); if (next_comma < 1) return {}; - const auto k{ToIntegral<int64_t>(std::string_view(in.begin(), next_comma))}; + const auto k{ToIntegral<int64_t>(std::string_view(in.data(), next_comma))}; if (!k.has_value() || *k < 1) return {}; in = in.subspan(next_comma + 1); // n = 1 here because we read the first WRAPPED_EXPR before reaching THRESH diff --git a/src/script/script.h b/src/script/script.h index e3119cbe05..f457984980 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -17,6 +17,7 @@ #include <cstdint> #include <cstring> #include <limits> +#include <span> #include <stdexcept> #include <string> #include <type_traits> @@ -412,6 +413,32 @@ bool GetScriptOp(CScriptBase::const_iterator& pc, CScriptBase::const_iterator en /** Serialized script, used inside transaction inputs and outputs */ class CScript : public CScriptBase { +private: + inline void AppendDataSize(const uint32_t size) + { + if (size < OP_PUSHDATA1) { + insert(end(), static_cast<value_type>(size)); + } else if (size <= 0xff) { + insert(end(), OP_PUSHDATA1); + insert(end(), static_cast<value_type>(size)); + } else if (size <= 0xffff) { + insert(end(), OP_PUSHDATA2); + value_type data[2]; + WriteLE16(data, size); + insert(end(), std::cbegin(data), std::cend(data)); + } else { + insert(end(), OP_PUSHDATA4); + value_type data[4]; + WriteLE32(data, size); + insert(end(), std::cbegin(data), std::cend(data)); + } + } + + void AppendData(std::span<const value_type> data) + { + insert(end(), data.begin(), data.end()); + } + protected: CScript& push_int64(int64_t n) { @@ -463,35 +490,19 @@ public: return *this; } - CScript& operator<<(const std::vector<unsigned char>& b) LIFETIMEBOUND + CScript& operator<<(std::span<const std::byte> b) LIFETIMEBOUND { - if (b.size() < OP_PUSHDATA1) - { - insert(end(), (unsigned char)b.size()); - } - else if (b.size() <= 0xff) - { - insert(end(), OP_PUSHDATA1); - insert(end(), (unsigned char)b.size()); - } - else if (b.size() <= 0xffff) - { - insert(end(), OP_PUSHDATA2); - uint8_t _data[2]; - WriteLE16(_data, b.size()); - insert(end(), _data, _data + sizeof(_data)); - } - else - { - insert(end(), OP_PUSHDATA4); - uint8_t _data[4]; - WriteLE32(_data, b.size()); - insert(end(), _data, _data + sizeof(_data)); - } - insert(end(), b.begin(), b.end()); + AppendDataSize(b.size()); + AppendData({reinterpret_cast<const value_type*>(b.data()), b.size()}); return *this; } + // For compatibility reasons. In new code, prefer using std::byte instead of uint8_t. + CScript& operator<<(std::span<const value_type> b) LIFETIMEBOUND + { + return *this << std::as_bytes(b); + } + bool GetOp(const_iterator& pc, opcodetype& opcodeRet, std::vector<unsigned char>& vchRet) const { return GetScriptOp(pc, end(), opcodeRet, &vchRet); diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 9568348bf6..42db251359 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -694,27 +694,6 @@ void SignatureData::MergeSignatureData(SignatureData sigdata) signatures.insert(std::make_move_iterator(sigdata.signatures.begin()), std::make_move_iterator(sigdata.signatures.end())); } -bool SignSignature(const SigningProvider &provider, const CScript& fromPubKey, CMutableTransaction& txTo, unsigned int nIn, const CAmount& amount, int nHashType, SignatureData& sig_data) -{ - assert(nIn < txTo.vin.size()); - - MutableTransactionSignatureCreator creator(txTo, nIn, amount, nHashType); - - bool ret = ProduceSignature(provider, creator, fromPubKey, sig_data); - UpdateInput(txTo.vin.at(nIn), sig_data); - return ret; -} - -bool SignSignature(const SigningProvider &provider, const CTransaction& txFrom, CMutableTransaction& txTo, unsigned int nIn, int nHashType, SignatureData& sig_data) -{ - assert(nIn < txTo.vin.size()); - const CTxIn& txin = txTo.vin[nIn]; - assert(txin.prevout.n < txFrom.vout.size()); - const CTxOut& txout = txFrom.vout[txin.prevout.n]; - - return SignSignature(provider, txout.scriptPubKey, txTo, nIn, txout.nValue, nHashType, sig_data); -} - namespace { /** Dummy signature checker which accepts all signatures. */ class DummySignatureChecker final : public BaseSignatureChecker diff --git a/src/script/sign.h b/src/script/sign.h index 4edd5bf326..fe2c470bc6 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -97,25 +97,6 @@ struct SignatureData { /** Produce a script signature using a generic signature creator. */ bool ProduceSignature(const SigningProvider& provider, const BaseSignatureCreator& creator, const CScript& scriptPubKey, SignatureData& sigdata); -/** - * Produce a satisfying script (scriptSig or witness). - * - * @param provider Utility containing the information necessary to solve a script. - * @param fromPubKey The script to produce a satisfaction for. - * @param txTo The spending transaction. - * @param nIn The index of the input in `txTo` referring the output being spent. - * @param amount The value of the output being spent. - * @param nHashType Signature hash type. - * @param sig_data Additional data provided to solve a script. Filled with the resulting satisfying - * script and whether the satisfaction is complete. - * - * @return True if the produced script is entirely satisfying `fromPubKey`. - **/ -bool SignSignature(const SigningProvider &provider, const CScript& fromPubKey, CMutableTransaction& txTo, - unsigned int nIn, const CAmount& amount, int nHashType, SignatureData& sig_data); -bool SignSignature(const SigningProvider &provider, const CTransaction& txFrom, CMutableTransaction& txTo, - unsigned int nIn, int nHashType, SignatureData& sig_data); - /** Extract signature data from a transaction input, and insert it. */ SignatureData DataFromTransaction(const CMutableTransaction& tx, unsigned int nIn, const CTxOut& txout); void UpdateInput(CTxIn& input, const SignatureData& data); diff --git a/src/script/solver.h b/src/script/solver.h index 5a7b685a6f..5b945477c9 100644 --- a/src/script/solver.h +++ b/src/script/solver.h @@ -10,6 +10,7 @@ #include <attributes.h> #include <script/script.h> +#include <span.h> #include <string> #include <optional> @@ -17,7 +18,6 @@ #include <vector> class CPubKey; -template <typename C> class Span; enum class TxoutType { NONSTANDARD, diff --git a/src/secp256k1/.cirrus.yml b/src/secp256k1/.cirrus.yml index 0c1e01dc95..81a4f04328 100644 --- a/src/secp256k1/.cirrus.yml +++ b/src/secp256k1/.cirrus.yml @@ -22,9 +22,10 @@ env: RECOVERY: no EXTRAKEYS: no SCHNORRSIG: no + MUSIG: no ELLSWIFT: no ### test options - SECP256K1_TEST_ITERS: + SECP256K1_TEST_ITERS: 64 BENCH: yes SECP256K1_BENCH_ITERS: 2 CTIMETESTS: yes @@ -69,6 +70,7 @@ task: RECOVERY: yes EXTRAKEYS: yes SCHNORRSIG: yes + MUSIG: yes ELLSWIFT: yes matrix: # Currently only gcc-snapshot, the other compilers are tested on GHA with QEMU @@ -86,6 +88,7 @@ task: RECOVERY: yes EXTRAKEYS: yes SCHNORRSIG: yes + MUSIG: yes ELLSWIFT: yes WRAPPER_CMD: 'valgrind --error-exitcode=42' SECP256K1_TEST_ITERS: 2 diff --git a/src/secp256k1/.github/workflows/ci.yml b/src/secp256k1/.github/workflows/ci.yml index 0fc104d29b..54b2fab1c4 100644 --- a/src/secp256k1/.github/workflows/ci.yml +++ b/src/secp256k1/.github/workflows/ci.yml @@ -33,9 +33,10 @@ env: RECOVERY: 'no' EXTRAKEYS: 'no' SCHNORRSIG: 'no' + MUSIG: 'no' ELLSWIFT: 'no' ### test options - SECP256K1_TEST_ITERS: + SECP256K1_TEST_ITERS: 64 BENCH: 'yes' SECP256K1_BENCH_ITERS: 2 CTIMETESTS: 'yes' @@ -72,18 +73,18 @@ jobs: matrix: configuration: - env_vars: { WIDEMUL: 'int64', RECOVERY: 'yes' } - - env_vars: { WIDEMUL: 'int64', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes' } + - env_vars: { WIDEMUL: 'int64', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', MUSIG: 'yes', ELLSWIFT: 'yes' } - env_vars: { WIDEMUL: 'int128' } - env_vars: { WIDEMUL: 'int128_struct', ELLSWIFT: 'yes' } - - env_vars: { WIDEMUL: 'int128', RECOVERY: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes' } - - env_vars: { WIDEMUL: 'int128', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes' } + - env_vars: { WIDEMUL: 'int128', RECOVERY: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', MUSIG: 'yes', ELLSWIFT: 'yes' } + - env_vars: { WIDEMUL: 'int128', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', MUSIG: 'yes' } - env_vars: { WIDEMUL: 'int128', ASM: 'x86_64', ELLSWIFT: 'yes' } - - env_vars: { RECOVERY: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes' } - - env_vars: { CTIMETESTS: 'no', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', CPPFLAGS: '-DVERIFY' } + - env_vars: { RECOVERY: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', MUSIG: 'yes' } + - env_vars: { CTIMETESTS: 'no', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', MUSIG: 'yes', CPPFLAGS: '-DVERIFY' } - env_vars: { BUILD: 'distcheck', WITH_VALGRIND: 'no', CTIMETESTS: 'no', BENCH: 'no' } - env_vars: { CPPFLAGS: '-DDETERMINISTIC' } - env_vars: { CFLAGS: '-O0', CTIMETESTS: 'no' } - - env_vars: { CFLAGS: '-O1', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes' } + - env_vars: { CFLAGS: '-O1', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', MUSIG: 'yes', ELLSWIFT: 'yes' } - env_vars: { ECMULTGENKB: 2, ECMULTWINDOW: 2 } - env_vars: { ECMULTGENKB: 86, ECMULTWINDOW: 4 } cc: @@ -142,6 +143,7 @@ jobs: RECOVERY: 'yes' EXTRAKEYS: 'yes' SCHNORRSIG: 'yes' + MUSIG: 'yes' ELLSWIFT: 'yes' CC: ${{ matrix.cc }} @@ -187,6 +189,7 @@ jobs: RECOVERY: 'yes' EXTRAKEYS: 'yes' SCHNORRSIG: 'yes' + MUSIG: 'yes' ELLSWIFT: 'yes' CTIMETESTS: 'no' @@ -239,6 +242,7 @@ jobs: RECOVERY: 'yes' EXTRAKEYS: 'yes' SCHNORRSIG: 'yes' + MUSIG: 'yes' ELLSWIFT: 'yes' CTIMETESTS: 'no' @@ -285,6 +289,7 @@ jobs: RECOVERY: 'yes' EXTRAKEYS: 'yes' SCHNORRSIG: 'yes' + MUSIG: 'yes' ELLSWIFT: 'yes' CTIMETESTS: 'no' @@ -341,6 +346,7 @@ jobs: RECOVERY: 'yes' EXTRAKEYS: 'yes' SCHNORRSIG: 'yes' + MUSIG: 'yes' ELLSWIFT: 'yes' CTIMETESTS: 'no' @@ -394,6 +400,7 @@ jobs: RECOVERY: 'yes' EXTRAKEYS: 'yes' SCHNORRSIG: 'yes' + MUSIG: 'yes' ELLSWIFT: 'yes' CTIMETESTS: 'no' SECP256K1_TEST_ITERS: 2 @@ -446,6 +453,7 @@ jobs: RECOVERY: 'yes' EXTRAKEYS: 'yes' SCHNORRSIG: 'yes' + MUSIG: 'yes' ELLSWIFT: 'yes' CTIMETESTS: 'no' CFLAGS: '-fsanitize=undefined,address -g' @@ -511,6 +519,7 @@ jobs: RECOVERY: 'yes' EXTRAKEYS: 'yes' SCHNORRSIG: 'yes' + MUSIG: 'yes' ELLSWIFT: 'yes' CC: 'clang' SECP256K1_TEST_ITERS: 32 @@ -558,6 +567,7 @@ jobs: RECOVERY: 'yes' EXTRAKEYS: 'yes' SCHNORRSIG: 'yes' + MUSIG: 'yes' ELLSWIFT: 'yes' CTIMETESTS: 'no' @@ -602,9 +612,9 @@ jobs: if: ${{ always() }} x86_64-macos-native: - name: "x86_64: macOS Monterey, Valgrind" + name: "x86_64: macOS Ventura, Valgrind" # See: https://github.com/actions/runner-images#available-images. - runs-on: macos-12 + runs-on: macos-13 env: CC: 'clang' @@ -615,15 +625,15 @@ jobs: fail-fast: false matrix: env_vars: - - { WIDEMUL: 'int64', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes' } + - { WIDEMUL: 'int64', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', MUSIG: 'yes', ELLSWIFT: 'yes' } - { WIDEMUL: 'int128_struct', ECMULTGENKB: 2, ECMULTWINDOW: 4 } - - { WIDEMUL: 'int128', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes' } + - { WIDEMUL: 'int128', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', MUSIG: 'yes', ELLSWIFT: 'yes' } - { WIDEMUL: 'int128', RECOVERY: 'yes' } - - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes' } - - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes', CC: 'gcc' } - - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes', WRAPPER_CMD: 'valgrind --error-exitcode=42', SECP256K1_TEST_ITERS: 2 } - - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes', CC: 'gcc', WRAPPER_CMD: 'valgrind --error-exitcode=42', SECP256K1_TEST_ITERS: 2 } - - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes', CPPFLAGS: '-DVERIFY', CTIMETESTS: 'no' } + - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', MUSIG: 'yes', ELLSWIFT: 'yes' } + - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', MUSIG: 'yes', ELLSWIFT: 'yes', CC: 'gcc' } + - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', MUSIG: 'yes', ELLSWIFT: 'yes', WRAPPER_CMD: 'valgrind --error-exitcode=42', SECP256K1_TEST_ITERS: 2 } + - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', MUSIG: 'yes', ELLSWIFT: 'yes', CC: 'gcc', WRAPPER_CMD: 'valgrind --error-exitcode=42', SECP256K1_TEST_ITERS: 2 } + - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', EXTRAKEYS: 'yes', SCHNORRSIG: 'yes', MUSIG: 'yes', ELLSWIFT: 'yes', CPPFLAGS: '-DVERIFY', CTIMETESTS: 'no' } - BUILD: 'distcheck' steps: @@ -751,14 +761,14 @@ jobs: # Use the bash shell included with Git for Windows. shell: bash run: | - cd build/src/RelWithDebInfo && file *tests.exe bench*.exe libsecp256k1-*.dll || true + cd build/bin/RelWithDebInfo && file *tests.exe bench*.exe libsecp256k1-*.dll || true - name: Check run: | ctest -C RelWithDebInfo --test-dir build -j ([int]$env:NUMBER_OF_PROCESSORS + 1) - build\src\RelWithDebInfo\bench_ecmult.exe - build\src\RelWithDebInfo\bench_internal.exe - build\src\RelWithDebInfo\bench.exe + build\bin\RelWithDebInfo\bench_ecmult.exe + build\bin\RelWithDebInfo\bench_internal.exe + build\bin\RelWithDebInfo\bench.exe win64-native-headers: name: "x64 (MSVC): C++ (public headers)" @@ -790,6 +800,7 @@ jobs: RECOVERY: 'yes' EXTRAKEYS: 'yes' SCHNORRSIG: 'yes' + MUSIG: 'yes' ELLSWIFT: 'yes' steps: diff --git a/src/secp256k1/.gitignore b/src/secp256k1/.gitignore index 18e3259f59..bffba8cb2c 100644 --- a/src/secp256k1/.gitignore +++ b/src/secp256k1/.gitignore @@ -11,6 +11,7 @@ ecdh_example ecdsa_example schnorr_example ellswift_example +musig_example *.exe *.so *.a diff --git a/src/secp256k1/CHANGELOG.md b/src/secp256k1/CHANGELOG.md index fb82940627..ee447c0c1c 100644 --- a/src/secp256k1/CHANGELOG.md +++ b/src/secp256k1/CHANGELOG.md @@ -5,7 +5,26 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.6.0] - 2024-11-04 + +#### Added + - New module `musig` implements the MuSig2 multisignature scheme according to the [BIP 327 specification](https://github.com/bitcoin/bips/blob/master/bip-0327.mediawiki). See: + - Header file `include/secp256k1_musig.h` which defines the new API. + - Document `doc/musig.md` for further notes on API usage. + - Usage example `examples/musig.c`. + - New CMake variable `SECP256K1_APPEND_LDFLAGS` for appending linker flags to the build command. + +#### Changed + - API functions now use a significantly more robust method to clear secrets from the stack before returning. However, secret clearing remains a best-effort security measure and cannot guarantee complete removal. + - Any type `secp256k1_foo` can now be forward-declared using `typedef struct secp256k1_foo secp256k1_foo;` (or also `struct secp256k1_foo;` in C++). + - Organized CMake build artifacts into dedicated directories (`bin/` for executables, `lib/` for libraries) to improve build output structure and Windows shared library compatibility. + +#### Removed + - Removed the `secp256k1_scratch_space` struct and its associated functions `secp256k1_scratch_space_create` and `secp256k1_scratch_space_destroy` because the scratch space was unused in the API. + +#### ABI Compatibility +The symbols `secp256k1_scratch_space_create` and `secp256k1_scratch_space_destroy` were removed. +Otherwise, the library maintains backward compatibility with versions 0.3.x through 0.5.x. ## [0.5.1] - 2024-08-01 @@ -143,7 +162,7 @@ This version was in fact never released. The number was given by the build system since the introduction of autotools in Jan 2014 (ea0fe5a5bf0c04f9cc955b2966b614f5f378c6f6). Therefore, this version number does not uniquely identify a set of source files. -[unreleased]: https://github.com/bitcoin-core/secp256k1/compare/v0.5.1...HEAD +[0.6.0]: https://github.com/bitcoin-core/secp256k1/compare/v0.5.1...v0.6.0 [0.5.1]: https://github.com/bitcoin-core/secp256k1/compare/v0.5.0...v0.5.1 [0.5.0]: https://github.com/bitcoin-core/secp256k1/compare/v0.4.1...v0.5.0 [0.4.1]: https://github.com/bitcoin-core/secp256k1/compare/v0.4.0...v0.4.1 diff --git a/src/secp256k1/CMakeLists.txt b/src/secp256k1/CMakeLists.txt index d5a7a2de39..041bfa3dca 100644 --- a/src/secp256k1/CMakeLists.txt +++ b/src/secp256k1/CMakeLists.txt @@ -1,14 +1,19 @@ cmake_minimum_required(VERSION 3.16) +#============================= +# Project / Package metadata +#============================= project(libsecp256k1 # The package (a.k.a. release) version is based on semantic versioning 2.0.0 of # the API. All changes in experimental modules are treated as # backwards-compatible and therefore at most increase the minor version. - VERSION 0.5.2 + VERSION 0.6.0 DESCRIPTION "Optimized C library for ECDSA signatures and secret/public key operations on curve secp256k1." HOMEPAGE_URL "https://github.com/bitcoin-core/secp256k1" LANGUAGES C ) +enable_testing() +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) if(CMAKE_VERSION VERSION_LESS 3.21) # Emulates CMake 3.21+ behavior. @@ -26,15 +31,19 @@ endif() # https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html # All changes in experimental modules are treated as if they don't affect the # interface and therefore only increase the revision. -set(${PROJECT_NAME}_LIB_VERSION_CURRENT 4) -set(${PROJECT_NAME}_LIB_VERSION_REVISION 2) -set(${PROJECT_NAME}_LIB_VERSION_AGE 2) +set(${PROJECT_NAME}_LIB_VERSION_CURRENT 5) +set(${PROJECT_NAME}_LIB_VERSION_REVISION 0) +set(${PROJECT_NAME}_LIB_VERSION_AGE 0) +#============================= +# Language setup +#============================= set(CMAKE_C_STANDARD 90) set(CMAKE_C_EXTENSIONS OFF) -list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) - +#============================= +# Configurable options +#============================= option(BUILD_SHARED_LIBS "Build shared libraries." ON) option(SECP256K1_DISABLE_SHARED "Disable shared library. Overrides BUILD_SHARED_LIBS." OFF) if(SECP256K1_DISABLE_SHARED) @@ -51,6 +60,7 @@ option(SECP256K1_ENABLE_MODULE_ECDH "Enable ECDH module." ON) option(SECP256K1_ENABLE_MODULE_RECOVERY "Enable ECDSA pubkey recovery module." OFF) option(SECP256K1_ENABLE_MODULE_EXTRAKEYS "Enable extrakeys module." ON) option(SECP256K1_ENABLE_MODULE_SCHNORRSIG "Enable schnorrsig module." ON) +option(SECP256K1_ENABLE_MODULE_MUSIG "Enable musig module." ON) option(SECP256K1_ENABLE_MODULE_ELLSWIFT "Enable ElligatorSwift module." ON) # Processing must be done in a topological sorting of the dependency graph @@ -59,6 +69,14 @@ if(SECP256K1_ENABLE_MODULE_ELLSWIFT) add_compile_definitions(ENABLE_MODULE_ELLSWIFT=1) endif() +if(SECP256K1_ENABLE_MODULE_MUSIG) + if(DEFINED SECP256K1_ENABLE_MODULE_SCHNORRSIG AND NOT SECP256K1_ENABLE_MODULE_SCHNORRSIG) + message(FATAL_ERROR "Module dependency error: You have disabled the schnorrsig module explicitly, but it is required by the musig module.") + endif() + set(SECP256K1_ENABLE_MODULE_SCHNORRSIG ON) + add_compile_definitions(ENABLE_MODULE_MUSIG=1) +endif() + if(SECP256K1_ENABLE_MODULE_SCHNORRSIG) if(DEFINED SECP256K1_ENABLE_MODULE_EXTRAKEYS AND NOT SECP256K1_ENABLE_MODULE_EXTRAKEYS) message(FATAL_ERROR "Module dependency error: You have disabled the extrakeys module explicitly, but it is required by the schnorrsig module.") @@ -262,13 +280,6 @@ if(SECP256K1_BUILD_CTIME_TESTS) unset(msan_enabled) endif() -include(CTest) -# We do not use CTest's BUILD_TESTING because a single toggle for all tests is too coarse for our needs. -mark_as_advanced(BUILD_TESTING) -if(SECP256K1_BUILD_BENCHMARK OR SECP256K1_BUILD_TESTS OR SECP256K1_BUILD_EXHAUSTIVE_TESTS OR SECP256K1_BUILD_CTIME_TESTS OR SECP256K1_BUILD_EXAMPLES) - enable_testing() -endif() - set(SECP256K1_APPEND_CFLAGS "" CACHE STRING "Compiler flags that are appended to the command line after all other flags added by the build system. This variable is intended for debugging and special builds.") if(SECP256K1_APPEND_CFLAGS) # Appending to this low-level rule variable is the only way to @@ -284,6 +295,15 @@ if(SECP256K1_APPEND_LDFLAGS) string(APPEND CMAKE_C_LINK_EXECUTABLE " ${SECP256K1_APPEND_LDFLAGS}") endif() +if(NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/bin) +endif() +if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) +endif() +if(NOT CMAKE_ARCHIVE_OUTPUT_DIRECTORY) + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) +endif() add_subdirectory(src) if(SECP256K1_BUILD_EXAMPLES) add_subdirectory(examples) @@ -305,6 +325,7 @@ message(" ECDH ................................ ${SECP256K1_ENABLE_MODULE_ECDH} message(" ECDSA pubkey recovery ............... ${SECP256K1_ENABLE_MODULE_RECOVERY}") message(" extrakeys ........................... ${SECP256K1_ENABLE_MODULE_EXTRAKEYS}") message(" schnorrsig .......................... ${SECP256K1_ENABLE_MODULE_SCHNORRSIG}") +message(" musig ............................... ${SECP256K1_ENABLE_MODULE_MUSIG}") message(" ElligatorSwift ...................... ${SECP256K1_ENABLE_MODULE_ELLSWIFT}") message("Parameters:") message(" ecmult window size .................. ${SECP256K1_ECMULT_WINDOW_SIZE}") diff --git a/src/secp256k1/Makefile.am b/src/secp256k1/Makefile.am index 8723b53b2c..a95b4809d4 100644 --- a/src/secp256k1/Makefile.am +++ b/src/secp256k1/Makefile.am @@ -195,6 +195,17 @@ ellswift_example_LDFLAGS += -lbcrypt endif TESTS += ellswift_example endif +if ENABLE_MODULE_MUSIG +noinst_PROGRAMS += musig_example +musig_example_SOURCES = examples/musig.c +musig_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC +musig_example_LDADD = libsecp256k1.la +musig_example_LDFLAGS = -static +if BUILD_WINDOWS +musig_example_LDFLAGS += -lbcrypt +endif +TESTS += musig_example +endif endif ### Precomputed tables @@ -254,6 +265,7 @@ maintainer-clean-local: clean-testvectors ### Additional files to distribute EXTRA_DIST = autogen.sh CHANGELOG.md SECURITY.md EXTRA_DIST += doc/release-process.md doc/safegcd_implementation.md +EXTRA_DIST += doc/ellswift.md doc/musig.md EXTRA_DIST += examples/EXAMPLES_COPYING EXTRA_DIST += sage/gen_exhaustive_groups.sage EXTRA_DIST += sage/gen_split_lambda_constants.sage @@ -281,6 +293,10 @@ if ENABLE_MODULE_SCHNORRSIG include src/modules/schnorrsig/Makefile.am.include endif +if ENABLE_MODULE_MUSIG +include src/modules/musig/Makefile.am.include +endif + if ENABLE_MODULE_ELLSWIFT include src/modules/ellswift/Makefile.am.include endif diff --git a/src/secp256k1/README.md b/src/secp256k1/README.md index ed93e0519e..222e5fb768 100644 --- a/src/secp256k1/README.md +++ b/src/secp256k1/README.md @@ -21,6 +21,7 @@ Features: * Optional module for ECDH key exchange. * Optional module for Schnorr signatures according to [BIP-340](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki). * Optional module for ElligatorSwift key exchange according to [BIP-324](https://github.com/bitcoin/bips/blob/master/bip-0324.mediawiki). +* Optional module for MuSig2 Schnorr multi-signatures according to [BIP-327](https://github.com/bitcoin/bips/blob/master/bip-0327.mediawiki). Implementation details ---------------------- diff --git a/src/secp256k1/ci/ci.sh b/src/secp256k1/ci/ci.sh index a6c608c29c..3636deafa1 100755 --- a/src/secp256k1/ci/ci.sh +++ b/src/secp256k1/ci/ci.sh @@ -13,7 +13,7 @@ print_environment() { # does not rely on bash. for var in WERROR_CFLAGS MAKEFLAGS BUILD \ ECMULTWINDOW ECMULTGENKB ASM WIDEMUL WITH_VALGRIND EXTRAFLAGS \ - EXPERIMENTAL ECDH RECOVERY EXTRAKEYS SCHNORRSIG ELLSWIFT \ + EXPERIMENTAL ECDH RECOVERY EXTRAKEYS MUSIG SCHNORRSIG ELLSWIFT \ SECP256K1_TEST_ITERS BENCH SECP256K1_BENCH_ITERS CTIMETESTS\ EXAMPLES \ HOST WRAPPER_CMD \ @@ -79,6 +79,7 @@ esac --enable-module-ellswift="$ELLSWIFT" \ --enable-module-extrakeys="$EXTRAKEYS" \ --enable-module-schnorrsig="$SCHNORRSIG" \ + --enable-module-musig="$MUSIG" \ --enable-examples="$EXAMPLES" \ --enable-ctime-tests="$CTIMETESTS" \ --with-valgrind="$WITH_VALGRIND" \ diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index 6841543f59..f880a3578d 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -4,18 +4,18 @@ AC_PREREQ([2.60]) # the API. All changes in experimental modules are treated as # backwards-compatible and therefore at most increase the minor version. define(_PKG_VERSION_MAJOR, 0) -define(_PKG_VERSION_MINOR, 5) -define(_PKG_VERSION_PATCH, 2) -define(_PKG_VERSION_IS_RELEASE, false) +define(_PKG_VERSION_MINOR, 6) +define(_PKG_VERSION_PATCH, 0) +define(_PKG_VERSION_IS_RELEASE, true) # The library version is based on libtool versioning of the ABI. The set of # rules for updating the version can be found here: # https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html # All changes in experimental modules are treated as if they don't affect the # interface and therefore only increase the revision. -define(_LIB_VERSION_CURRENT, 4) -define(_LIB_VERSION_REVISION, 2) -define(_LIB_VERSION_AGE, 2) +define(_LIB_VERSION_CURRENT, 5) +define(_LIB_VERSION_REVISION, 0) +define(_LIB_VERSION_AGE, 0) AC_INIT([libsecp256k1],m4_join([.], _PKG_VERSION_MAJOR, _PKG_VERSION_MINOR, _PKG_VERSION_PATCH)m4_if(_PKG_VERSION_IS_RELEASE, [true], [], [-dev]),[https://github.com/bitcoin-core/secp256k1/issues],[libsecp256k1],[https://github.com/bitcoin-core/secp256k1]) @@ -184,6 +184,10 @@ AC_ARG_ENABLE(module_schnorrsig, AS_HELP_STRING([--enable-module-schnorrsig],[enable schnorrsig module [default=yes]]), [], [SECP_SET_DEFAULT([enable_module_schnorrsig], [yes], [yes])]) +AC_ARG_ENABLE(module_musig, + AS_HELP_STRING([--enable-module-musig],[enable MuSig2 module [default=yes]]), [], + [SECP_SET_DEFAULT([enable_module_musig], [yes], [yes])]) + AC_ARG_ENABLE(module_ellswift, AS_HELP_STRING([--enable-module-ellswift],[enable ElligatorSwift module [default=yes]]), [], [SECP_SET_DEFAULT([enable_module_ellswift], [yes], [yes])]) @@ -398,6 +402,14 @@ if test x"$enable_module_ellswift" = x"yes"; then SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ELLSWIFT=1" fi +if test x"$enable_module_musig" = x"yes"; then + if test x"$enable_module_schnorrsig" = x"no"; then + AC_MSG_ERROR([Module dependency error: You have disabled the schnorrsig module explicitly, but it is required by the musig module.]) + fi + enable_module_schnorrsig=yes + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_MUSIG=1" +fi + if test x"$enable_module_schnorrsig" = x"yes"; then if test x"$enable_module_extrakeys" = x"no"; then AC_MSG_ERROR([Module dependency error: You have disabled the extrakeys module explicitly, but it is required by the schnorrsig module.]) @@ -449,6 +461,7 @@ AM_CONDITIONAL([ENABLE_MODULE_ECDH], [test x"$enable_module_ecdh" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_RECOVERY], [test x"$enable_module_recovery" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_EXTRAKEYS], [test x"$enable_module_extrakeys" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_SCHNORRSIG], [test x"$enable_module_schnorrsig" = x"yes"]) +AM_CONDITIONAL([ENABLE_MODULE_MUSIG], [test x"$enable_module_musig" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_ELLSWIFT], [test x"$enable_module_ellswift" = x"yes"]) AM_CONDITIONAL([USE_EXTERNAL_ASM], [test x"$enable_external_asm" = x"yes"]) AM_CONDITIONAL([USE_ASM_ARM], [test x"$set_asm" = x"arm32"]) @@ -471,6 +484,7 @@ echo " module ecdh = $enable_module_ecdh" echo " module recovery = $enable_module_recovery" echo " module extrakeys = $enable_module_extrakeys" echo " module schnorrsig = $enable_module_schnorrsig" +echo " module musig = $enable_module_musig" echo " module ellswift = $enable_module_ellswift" echo echo " asm = $set_asm" diff --git a/src/secp256k1/doc/musig.md b/src/secp256k1/doc/musig.md new file mode 100644 index 0000000000..ae21f9b131 --- /dev/null +++ b/src/secp256k1/doc/musig.md @@ -0,0 +1,54 @@ +Notes on the musig module API +=========================== + +The following sections contain additional notes on the API of the musig module (`include/secp256k1_musig.h`). +A usage example can be found in `examples/musig.c`. + +## API misuse + +The musig API is designed with a focus on misuse resistance. +However, due to the interactive nature of the MuSig protocol, there are additional failure modes that are not present in regular (single-party) Schnorr signature creation. +While the results can be catastrophic (e.g. leaking of the secret key), it is unfortunately not possible for the musig implementation to prevent all such failure modes. + +Therefore, users of the musig module must take great care to make sure of the following: + +1. A unique nonce per signing session is generated in `secp256k1_musig_nonce_gen`. + See the corresponding comment in `include/secp256k1_musig.h` for how to ensure that. +2. The `secp256k1_musig_secnonce` structure is never copied or serialized. + See also the comment on `secp256k1_musig_secnonce` in `include/secp256k1_musig.h`. +3. Opaque data structures are never written to or read from directly. + Instead, only the provided accessor functions are used. + +## Key Aggregation and (Taproot) Tweaking + +Given a set of public keys, the aggregate public key is computed with `secp256k1_musig_pubkey_agg`. +A plain tweak can be added to the resulting public key with `secp256k1_ec_pubkey_tweak_add` by setting the `tweak32` argument to the hash defined in BIP 32. Similarly, a Taproot tweak can be added with `secp256k1_xonly_pubkey_tweak_add` by setting the `tweak32` argument to the TapTweak hash defined in BIP 341. +Both types of tweaking can be combined and invoked multiple times if the specific application requires it. + +## Signing + +This is covered by `examples/musig.c`. +Essentially, the protocol proceeds in the following steps: + +1. Generate a keypair with `secp256k1_keypair_create` and obtain the public key with `secp256k1_keypair_pub`. +2. Call `secp256k1_musig_pubkey_agg` with the pubkeys of all participants. +3. Optionally add a (Taproot) tweak with `secp256k1_musig_pubkey_xonly_tweak_add` and a plain tweak with `secp256k1_musig_pubkey_ec_tweak_add`. +4. Generate a pair of secret and public nonce with `secp256k1_musig_nonce_gen` and send the public nonce to the other signers. +5. Someone (not necessarily the signer) aggregates the public nonces with `secp256k1_musig_nonce_agg` and sends it to the signers. +6. Process the aggregate nonce with `secp256k1_musig_nonce_process`. +7. Create a partial signature with `secp256k1_musig_partial_sign`. +8. Verify the partial signatures (optional in some scenarios) with `secp256k1_musig_partial_sig_verify`. +9. Someone (not necessarily the signer) obtains all partial signatures and aggregates them into the final Schnorr signature using `secp256k1_musig_partial_sig_agg`. + +The aggregate signature can be verified with `secp256k1_schnorrsig_verify`. + +Steps 1 through 5 above can occur before or after the signers are aware of the message to be signed. +Whenever possible, it is recommended to generate the nonces only after the message is known. +This provides enhanced defense-in-depth measures, protecting against potential API misuse in certain scenarios. +However, it does require two rounds of communication during the signing process. +The alternative, generating the nonces in a pre-processing step before the message is known, eliminates these additional protective measures but allows for non-interactive signing. +Similarly, the API supports an alternative protocol flow where generating the aggregate key (steps 1 to 3) is allowed to happen after exchanging nonces (steps 4 to 5). + +## Verification + +A participant who wants to verify the partial signatures, but does not sign itself may do so using the above instructions except that the verifier skips steps 1, 4 and 7. diff --git a/src/secp256k1/examples/CMakeLists.txt b/src/secp256k1/examples/CMakeLists.txt index fd1ebce395..c9da9de6be 100644 --- a/src/secp256k1/examples/CMakeLists.txt +++ b/src/secp256k1/examples/CMakeLists.txt @@ -9,14 +9,7 @@ function(add_example name) $<$<PLATFORM_ID:Windows>:bcrypt> ) set(test_name ${name}_example) - add_test(NAME ${test_name} COMMAND ${target_name}) - if(BUILD_SHARED_LIBS AND MSVC) - # The DLL must reside either in the same folder where the executable is - # or somewhere in PATH. Using the latter option. - set_tests_properties(${test_name} PROPERTIES - ENVIRONMENT "PATH=$<TARGET_FILE_DIR:secp256k1>;$ENV{PATH}" - ) - endif() + add_test(NAME secp256k1_${test_name} COMMAND ${target_name}) endfunction() add_example(ecdsa) @@ -32,3 +25,7 @@ endif() if(SECP256K1_ENABLE_MODULE_ELLSWIFT) add_example(ellswift) endif() + +if(SECP256K1_ENABLE_MODULE_MUSIG) + add_example(musig) +endif() diff --git a/src/secp256k1/examples/ecdh.c b/src/secp256k1/examples/ecdh.c index d71fd2f604..13aa760b2d 100644 --- a/src/secp256k1/examples/ecdh.c +++ b/src/secp256k1/examples/ecdh.c @@ -42,18 +42,16 @@ int main(void) { assert(return_val); /*** Key Generation ***/ - - /* If the secret key is zero or out of range (bigger than secp256k1's - * order), we try to sample a new key. Note that the probability of this - * happening is negligible. */ - while (1) { - if (!fill_random(seckey1, sizeof(seckey1)) || !fill_random(seckey2, sizeof(seckey2))) { - printf("Failed to generate randomness\n"); - return 1; - } - if (secp256k1_ec_seckey_verify(ctx, seckey1) && secp256k1_ec_seckey_verify(ctx, seckey2)) { - break; - } + if (!fill_random(seckey1, sizeof(seckey1)) || !fill_random(seckey2, sizeof(seckey2))) { + printf("Failed to generate randomness\n"); + return 1; + } + /* If the secret key is zero or out of range (greater than secp256k1's + * order), we fail. Note that the probability of this occurring is negligible + * with a properly functioning random number generator. */ + if (!secp256k1_ec_seckey_verify(ctx, seckey1) || !secp256k1_ec_seckey_verify(ctx, seckey2)) { + printf("Generated secret key is invalid. This indicates an issue with the random number generator.\n"); + return 1; } /* Public key creation using a valid context with a verified secret key should never fail */ diff --git a/src/secp256k1/examples/ecdsa.c b/src/secp256k1/examples/ecdsa.c index d5c4613d9c..80ae9d46c5 100644 --- a/src/secp256k1/examples/ecdsa.c +++ b/src/secp256k1/examples/ecdsa.c @@ -49,18 +49,16 @@ int main(void) { assert(return_val); /*** Key Generation ***/ - - /* If the secret key is zero or out of range (bigger than secp256k1's - * order), we try to sample a new key. Note that the probability of this - * happening is negligible. */ - while (1) { - if (!fill_random(seckey, sizeof(seckey))) { - printf("Failed to generate randomness\n"); - return 1; - } - if (secp256k1_ec_seckey_verify(ctx, seckey)) { - break; - } + if (!fill_random(seckey, sizeof(seckey))) { + printf("Failed to generate randomness\n"); + return 1; + } + /* If the secret key is zero or out of range (greater than secp256k1's + * order), we fail. Note that the probability of this occurring is negligible + * with a properly functioning random number generator. */ + if (!secp256k1_ec_seckey_verify(ctx, seckey)) { + printf("Generated secret key is invalid. This indicates an issue with the random number generator.\n"); + return 1; } /* Public key creation using a valid context with a verified secret key should never fail */ diff --git a/src/secp256k1/examples/ellswift.c b/src/secp256k1/examples/ellswift.c index 52be7eebfb..afb2fee40b 100644 --- a/src/secp256k1/examples/ellswift.c +++ b/src/secp256k1/examples/ellswift.c @@ -47,18 +47,16 @@ int main(void) { assert(return_val); /*** Generate secret keys ***/ - - /* If the secret key is zero or out of range (bigger than secp256k1's - * order), we try to sample a new key. Note that the probability of this - * happening is negligible. */ - while (1) { - if (!fill_random(seckey1, sizeof(seckey1)) || !fill_random(seckey2, sizeof(seckey2))) { - printf("Failed to generate randomness\n"); - return 1; - } - if (secp256k1_ec_seckey_verify(ctx, seckey1) && secp256k1_ec_seckey_verify(ctx, seckey2)) { - break; - } + if (!fill_random(seckey1, sizeof(seckey1)) || !fill_random(seckey2, sizeof(seckey2))) { + printf("Failed to generate randomness\n"); + return 1; + } + /* If the secret key is zero or out of range (greater than secp256k1's + * order), we fail. Note that the probability of this occurring is negligible + * with a properly functioning random number generator. */ + if (!secp256k1_ec_seckey_verify(ctx, seckey1) || !secp256k1_ec_seckey_verify(ctx, seckey2)) { + printf("Generated secret key is invalid. This indicates an issue with the random number generator.\n"); + return 1; } /* Generate ElligatorSwift public keys. This should never fail with valid context and diff --git a/src/secp256k1/examples/musig.c b/src/secp256k1/examples/musig.c new file mode 100644 index 0000000000..0352dc40f3 --- /dev/null +++ b/src/secp256k1/examples/musig.c @@ -0,0 +1,260 @@ +/************************************************************************* + * To the extent possible under law, the author(s) have dedicated all * + * copyright and related and neighboring rights to the software in this * + * file to the public domain worldwide. This software is distributed * + * without any warranty. For the CC0 Public Domain Dedication, see * + * EXAMPLES_COPYING or https://creativecommons.org/publicdomain/zero/1.0 * + *************************************************************************/ + +/** This file demonstrates how to use the MuSig module to create a + * 3-of-3 multisignature. Additionally, see the documentation in + * include/secp256k1_musig.h and doc/musig.md. + */ + +#include <stdio.h> +#include <assert.h> +#include <string.h> + +#include <secp256k1.h> +#include <secp256k1_extrakeys.h> +#include <secp256k1_musig.h> +#include <secp256k1_schnorrsig.h> + +#include "examples_util.h" + +struct signer_secrets { + secp256k1_keypair keypair; + secp256k1_musig_secnonce secnonce; +}; + +struct signer { + secp256k1_pubkey pubkey; + secp256k1_musig_pubnonce pubnonce; + secp256k1_musig_partial_sig partial_sig; +}; + + /* Number of public keys involved in creating the aggregate signature */ +#define N_SIGNERS 3 +/* Create a key pair, store it in signer_secrets->keypair and signer->pubkey */ +static int create_keypair(const secp256k1_context* ctx, struct signer_secrets *signer_secrets, struct signer *signer) { + unsigned char seckey[32]; + + if (!fill_random(seckey, sizeof(seckey))) { + printf("Failed to generate randomness\n"); + return 0; + } + /* Try to create a keypair with a valid context. This only fails if the + * secret key is zero or out of range (greater than secp256k1's order). Note + * that the probability of this occurring is negligible with a properly + * functioning random number generator. */ + if (!secp256k1_keypair_create(ctx, &signer_secrets->keypair, seckey)) { + return 0; + } + if (!secp256k1_keypair_pub(ctx, &signer->pubkey, &signer_secrets->keypair)) { + return 0; + } + + secure_erase(seckey, sizeof(seckey)); + return 1; +} + +/* Tweak the pubkey corresponding to the provided keyagg cache, update the cache + * and return the tweaked aggregate pk. */ +static int tweak(const secp256k1_context* ctx, secp256k1_xonly_pubkey *agg_pk, secp256k1_musig_keyagg_cache *cache) { + secp256k1_pubkey output_pk; + /* For BIP 32 tweaking the plain_tweak is set to a hash as defined in BIP + * 32. */ + unsigned char plain_tweak[32] = "this could be a BIP32 tweak...."; + /* For Taproot tweaking the xonly_tweak is set to the TapTweak hash as + * defined in BIP 341 */ + unsigned char xonly_tweak[32] = "this could be a Taproot tweak.."; + + + /* Plain tweaking which, for example, allows deriving multiple child + * public keys from a single aggregate key using BIP32 */ + if (!secp256k1_musig_pubkey_ec_tweak_add(ctx, NULL, cache, plain_tweak)) { + return 0; + } + /* Note that we did not provide an output_pk argument, because the + * resulting pk is also saved in the cache and so if one is just interested + * in signing, the output_pk argument is unnecessary. On the other hand, if + * one is not interested in signing, the same output_pk can be obtained by + * calling `secp256k1_musig_pubkey_get` right after key aggregation to get + * the full pubkey and then call `secp256k1_ec_pubkey_tweak_add`. */ + + /* Xonly tweaking which, for example, allows creating Taproot commitments */ + if (!secp256k1_musig_pubkey_xonly_tweak_add(ctx, &output_pk, cache, xonly_tweak)) { + return 0; + } + /* Note that if we wouldn't care about signing, we can arrive at the same + * output_pk by providing the untweaked public key to + * `secp256k1_xonly_pubkey_tweak_add` (after converting it to an xonly pubkey + * if necessary with `secp256k1_xonly_pubkey_from_pubkey`). */ + + /* Now we convert the output_pk to an xonly pubkey to allow to later verify + * the Schnorr signature against it. For this purpose we can ignore the + * `pk_parity` output argument; we would need it if we would have to open + * the Taproot commitment. */ + if (!secp256k1_xonly_pubkey_from_pubkey(ctx, agg_pk, NULL, &output_pk)) { + return 0; + } + return 1; +} + +/* Sign a message hash with the given key pairs and store the result in sig */ +static int sign(const secp256k1_context* ctx, struct signer_secrets *signer_secrets, struct signer *signer, const secp256k1_musig_keyagg_cache *cache, const unsigned char *msg32, unsigned char *sig64) { + int i; + const secp256k1_musig_pubnonce *pubnonces[N_SIGNERS]; + const secp256k1_musig_partial_sig *partial_sigs[N_SIGNERS]; + /* The same for all signers */ + secp256k1_musig_session session; + secp256k1_musig_aggnonce agg_pubnonce; + + for (i = 0; i < N_SIGNERS; i++) { + unsigned char seckey[32]; + unsigned char session_secrand[32]; + /* Create random session ID. It is absolutely necessary that the session ID + * is unique for every call of secp256k1_musig_nonce_gen. Otherwise + * it's trivial for an attacker to extract the secret key! */ + if (!fill_random(session_secrand, sizeof(session_secrand))) { + return 0; + } + if (!secp256k1_keypair_sec(ctx, seckey, &signer_secrets[i].keypair)) { + return 0; + } + /* Initialize session and create secret nonce for signing and public + * nonce to send to the other signers. */ + if (!secp256k1_musig_nonce_gen(ctx, &signer_secrets[i].secnonce, &signer[i].pubnonce, session_secrand, seckey, &signer[i].pubkey, msg32, NULL, NULL)) { + return 0; + } + pubnonces[i] = &signer[i].pubnonce; + + secure_erase(seckey, sizeof(seckey)); + } + + /* Communication round 1: Every signer sends their pubnonce to the + * coordinator. The coordinator runs secp256k1_musig_nonce_agg and sends + * agg_pubnonce to each signer */ + if (!secp256k1_musig_nonce_agg(ctx, &agg_pubnonce, pubnonces, N_SIGNERS)) { + return 0; + } + + /* Every signer creates a partial signature */ + for (i = 0; i < N_SIGNERS; i++) { + /* Initialize the signing session by processing the aggregate nonce */ + if (!secp256k1_musig_nonce_process(ctx, &session, &agg_pubnonce, msg32, cache)) { + return 0; + } + /* partial_sign will clear the secnonce by setting it to 0. That's because + * you must _never_ reuse the secnonce (or use the same session_secrand to + * create a secnonce). If you do, you effectively reuse the nonce and + * leak the secret key. */ + if (!secp256k1_musig_partial_sign(ctx, &signer[i].partial_sig, &signer_secrets[i].secnonce, &signer_secrets[i].keypair, cache, &session)) { + return 0; + } + partial_sigs[i] = &signer[i].partial_sig; + } + /* Communication round 2: Every signer sends their partial signature to the + * coordinator, who verifies the partial signatures and aggregates them. */ + for (i = 0; i < N_SIGNERS; i++) { + /* To check whether signing was successful, it suffices to either verify + * the aggregate signature with the aggregate public key using + * secp256k1_schnorrsig_verify, or verify all partial signatures of all + * signers individually. Verifying the aggregate signature is cheaper but + * verifying the individual partial signatures has the advantage that it + * can be used to determine which of the partial signatures are invalid + * (if any), i.e., which of the partial signatures cause the aggregate + * signature to be invalid and thus the protocol run to fail. It's also + * fine to first verify the aggregate sig, and only verify the individual + * sigs if it does not work. + */ + if (!secp256k1_musig_partial_sig_verify(ctx, &signer[i].partial_sig, &signer[i].pubnonce, &signer[i].pubkey, cache, &session)) { + return 0; + } + } + return secp256k1_musig_partial_sig_agg(ctx, sig64, &session, partial_sigs, N_SIGNERS); +} + +int main(void) { + secp256k1_context* ctx; + int i; + struct signer_secrets signer_secrets[N_SIGNERS]; + struct signer signers[N_SIGNERS]; + const secp256k1_pubkey *pubkeys_ptr[N_SIGNERS]; + secp256k1_xonly_pubkey agg_pk; + secp256k1_musig_keyagg_cache cache; + unsigned char msg[32] = "this_could_be_the_hash_of_a_msg"; + unsigned char sig[64]; + + /* Create a secp256k1 context */ + ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); + printf("Creating key pairs......"); + fflush(stdout); + for (i = 0; i < N_SIGNERS; i++) { + if (!create_keypair(ctx, &signer_secrets[i], &signers[i])) { + printf("FAILED\n"); + return 1; + } + pubkeys_ptr[i] = &signers[i].pubkey; + } + printf("ok\n"); + + /* The aggregate public key produced by secp256k1_musig_pubkey_agg depends + * on the order of the provided public keys. If there is no canonical order + * of the signers, the individual public keys can optionally be sorted with + * secp256k1_ec_pubkey_sort to ensure that the aggregate public key is + * independent of the order of signers. */ + printf("Sorting public keys....."); + fflush(stdout); + if (!secp256k1_ec_pubkey_sort(ctx, pubkeys_ptr, N_SIGNERS)) { + printf("FAILED\n"); + return 1; + } + printf("ok\n"); + + printf("Combining public keys..."); + fflush(stdout); + /* If you just want to aggregate and not sign, you can call + * secp256k1_musig_pubkey_agg with the keyagg_cache argument set to NULL + * while providing a non-NULL agg_pk argument. */ + if (!secp256k1_musig_pubkey_agg(ctx, NULL, &cache, pubkeys_ptr, N_SIGNERS)) { + printf("FAILED\n"); + return 1; + } + printf("ok\n"); + printf("Tweaking................"); + fflush(stdout); + /* Optionally tweak the aggregate key */ + if (!tweak(ctx, &agg_pk, &cache)) { + printf("FAILED\n"); + return 1; + } + printf("ok\n"); + printf("Signing message........."); + fflush(stdout); + if (!sign(ctx, signer_secrets, signers, &cache, msg, sig)) { + printf("FAILED\n"); + return 1; + } + printf("ok\n"); + printf("Verifying signature....."); + fflush(stdout); + if (!secp256k1_schnorrsig_verify(ctx, sig, msg, 32, &agg_pk)) { + printf("FAILED\n"); + return 1; + } + printf("ok\n"); + + /* It's best practice to try to clear secrets from memory after using them. + * This is done because some bugs can allow an attacker to leak memory, for + * example through "out of bounds" array access (see Heartbleed), or the OS + * swapping them to disk. Hence, we overwrite secret key material with zeros. + * + * Here we are preventing these writes from being optimized out, as any good compiler + * will remove any writes that aren't used. */ + for (i = 0; i < N_SIGNERS; i++) { + secure_erase(&signer_secrets[i], sizeof(signer_secrets[i])); + } + secp256k1_context_destroy(ctx); + return 0; +} diff --git a/src/secp256k1/examples/schnorr.c b/src/secp256k1/examples/schnorr.c index 8d5d14bdaf..909fcaa1f3 100644 --- a/src/secp256k1/examples/schnorr.c +++ b/src/secp256k1/examples/schnorr.c @@ -43,20 +43,17 @@ int main(void) { assert(return_val); /*** Key Generation ***/ - - /* If the secret key is zero or out of range (bigger than secp256k1's - * order), we try to sample a new key. Note that the probability of this - * happening is negligible. */ - while (1) { - if (!fill_random(seckey, sizeof(seckey))) { - printf("Failed to generate randomness\n"); - return 1; - } - /* Try to create a keypair with a valid context, it should only fail if - * the secret key is zero or out of range. */ - if (secp256k1_keypair_create(ctx, &keypair, seckey)) { - break; - } + if (!fill_random(seckey, sizeof(seckey))) { + printf("Failed to generate randomness\n"); + return 1; + } + /* Try to create a keypair with a valid context. This only fails if the + * secret key is zero or out of range (greater than secp256k1's order). Note + * that the probability of this occurring is negligible with a properly + * functioning random number generator. */ + if (!secp256k1_keypair_create(ctx, &keypair, seckey)) { + printf("Generated secret key is invalid. This indicates an issue with the random number generator.\n"); + return 1; } /* Extract the X-only public key from the keypair. We pass NULL for diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index cfbdd528c2..c6e9417f05 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -49,19 +49,6 @@ extern "C" { */ typedef struct secp256k1_context_struct secp256k1_context; -/** Opaque data structure that holds rewritable "scratch space" - * - * The purpose of this structure is to replace dynamic memory allocations, - * because we target architectures where this may not be available. It is - * essentially a resizable (within specified parameters) block of bytes, - * which is initially created either by memory allocation or TODO as a pointer - * into some fixed rewritable space. - * - * Unlike the context object, this cannot safely be shared between threads - * without additional synchronization logic. - */ -typedef struct secp256k1_scratch_space_struct secp256k1_scratch_space; - /** Opaque data structure that holds a parsed and valid public key. * * The exact representation of data inside is implementation defined and not @@ -71,11 +58,11 @@ typedef struct secp256k1_scratch_space_struct secp256k1_scratch_space; * use secp256k1_ec_pubkey_serialize and secp256k1_ec_pubkey_parse. To * compare keys, use secp256k1_ec_pubkey_cmp. */ -typedef struct { +typedef struct secp256k1_pubkey { unsigned char data[64]; } secp256k1_pubkey; -/** Opaque data structured that holds a parsed ECDSA signature. +/** Opaque data structure that holds a parsed ECDSA signature. * * The exact representation of data inside is implementation defined and not * guaranteed to be portable between different platforms or versions. It is @@ -84,7 +71,7 @@ typedef struct { * comparison, use the secp256k1_ecdsa_signature_serialize_* and * secp256k1_ecdsa_signature_parse_* functions. */ -typedef struct { +typedef struct secp256k1_ecdsa_signature { unsigned char data[64]; } secp256k1_ecdsa_signature; @@ -147,6 +134,15 @@ typedef int (*secp256k1_nonce_function)( * 1. If using Libtool, it defines DLL_EXPORT automatically. * 2. In other cases, SECP256K1_DLL_EXPORT must be defined. */ # define SECP256K1_API extern __declspec (dllexport) +# else + /* Building libsecp256k1 as a static library on Windows. + * No declspec is needed, and so we would want the non-Windows-specific + * logic below take care of this case. However, this may result in setting + * __attribute__ ((visibility("default"))), which is supposed to be a noop + * on Windows but may trigger warnings when compiling with -flto due to a + * bug in GCC, see + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=116478 . */ +# define SECP256K1_API extern # endif /* The user must define SECP256K1_STATIC when consuming libsecp256k1 as a static * library on Windows. */ @@ -156,11 +152,12 @@ typedef int (*secp256k1_nonce_function)( # endif #endif #ifndef SECP256K1_API +/* All cases not captured by the Windows-specific logic. */ # if defined(__GNUC__) && (__GNUC__ >= 4) && defined(SECP256K1_BUILD) - /* Building libsecp256k1 on non-Windows using GCC or compatible. */ + /* Building libsecp256k1 using GCC or compatible. */ # define SECP256K1_API extern __attribute__ ((visibility ("default"))) # else - /* All cases not captured above. */ + /* Fall back to standard C's extern. */ # define SECP256K1_API extern # endif #endif @@ -392,29 +389,6 @@ SECP256K1_API void secp256k1_context_set_error_callback( const void *data ) SECP256K1_ARG_NONNULL(1); -/** Create a secp256k1 scratch space object. - * - * Returns: a newly created scratch space. - * Args: ctx: pointer to a context object. - * In: size: amount of memory to be available as scratch space. Some extra - * (<100 bytes) will be allocated for extra accounting. - */ -SECP256K1_API SECP256K1_WARN_UNUSED_RESULT secp256k1_scratch_space *secp256k1_scratch_space_create( - const secp256k1_context *ctx, - size_t size -) SECP256K1_ARG_NONNULL(1); - -/** Destroy a secp256k1 scratch space. - * - * The pointer may not be used afterwards. - * Args: ctx: pointer to a context object. - * scratch: space to destroy - */ -SECP256K1_API void secp256k1_scratch_space_destroy( - const secp256k1_context *ctx, - secp256k1_scratch_space *scratch -) SECP256K1_ARG_NONNULL(1); - /** Parse a variable-length public key into the pubkey object. * * Returns: 1 if the public key was fully valid. @@ -679,12 +653,14 @@ SECP256K1_API int secp256k1_ecdsa_sign( const void *ndata ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); -/** Verify an ECDSA secret key. +/** Verify an elliptic curve secret key. * * A secret key is valid if it is not 0 and less than the secp256k1 curve order * when interpreted as an integer (most significant byte first). The * probability of choosing a 32-byte string uniformly at random which is an - * invalid secret key is negligible. + * invalid secret key is negligible. However, if it does happen it should + * be assumed that the randomness source is severely broken and there should + * be no retry. * * Returns: 1: secret key is valid * 0: secret key is invalid diff --git a/src/secp256k1/include/secp256k1_extrakeys.h b/src/secp256k1/include/secp256k1_extrakeys.h index ad70b92f95..48c98693cf 100644 --- a/src/secp256k1/include/secp256k1_extrakeys.h +++ b/src/secp256k1/include/secp256k1_extrakeys.h @@ -19,7 +19,7 @@ extern "C" { * use secp256k1_xonly_pubkey_serialize and secp256k1_xonly_pubkey_parse. To * compare keys, use secp256k1_xonly_pubkey_cmp. */ -typedef struct { +typedef struct secp256k1_xonly_pubkey { unsigned char data[64]; } secp256k1_xonly_pubkey; @@ -30,7 +30,7 @@ typedef struct { * guaranteed to be portable between different platforms or versions. It is * however guaranteed to be 96 bytes in size, and can be safely copied/moved. */ -typedef struct { +typedef struct secp256k1_keypair { unsigned char data[96]; } secp256k1_keypair; @@ -155,10 +155,13 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_tweak_add_ const unsigned char *tweak32 ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5); -/** Compute the keypair for a secret key. +/** Compute the keypair for a valid secret key. * - * Returns: 1: secret was valid, keypair is ready to use - * 0: secret was invalid, try again with a different secret + * See the documentation of `secp256k1_ec_seckey_verify` for more information + * about the validity of secret keys. + * + * Returns: 1: secret key is valid + * 0: secret key is invalid * Args: ctx: pointer to a context object (not secp256k1_context_static). * Out: keypair: pointer to the created keypair. * In: seckey: pointer to a 32-byte secret key. diff --git a/src/secp256k1/include/secp256k1_musig.h b/src/secp256k1/include/secp256k1_musig.h new file mode 100644 index 0000000000..11b8f08c88 --- /dev/null +++ b/src/secp256k1/include/secp256k1_musig.h @@ -0,0 +1,588 @@ +#ifndef SECP256K1_MUSIG_H +#define SECP256K1_MUSIG_H + +#include "secp256k1_extrakeys.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stddef.h> +#include <stdint.h> + +/** This module implements BIP 327 "MuSig2 for BIP340-compatible + * Multi-Signatures" + * (https://github.com/bitcoin/bips/blob/master/bip-0327.mediawiki) + * v1.0.0. You can find an example demonstrating the musig module in + * examples/musig.c. + * + * The module also supports BIP 341 ("Taproot") public key tweaking. + * + * It is recommended to read the documentation in this include file carefully. + * Further notes on API usage can be found in doc/musig.md + * + * Since the first version of MuSig is essentially replaced by MuSig2, we use + * MuSig, musig and MuSig2 synonymously unless noted otherwise. + */ + +/** Opaque data structures + * + * The exact representation of data inside the opaque data structures is + * implementation defined and not guaranteed to be portable between different + * platforms or versions. With the exception of `secp256k1_musig_secnonce`, the + * data structures can be safely copied/moved. If you need to convert to a + * format suitable for storage, transmission, or comparison, use the + * corresponding serialization and parsing functions. + */ + +/** Opaque data structure that caches information about public key aggregation. + * + * Guaranteed to be 197 bytes in size. No serialization and parsing functions + * (yet). + */ +typedef struct secp256k1_musig_keyagg_cache { + unsigned char data[197]; +} secp256k1_musig_keyagg_cache; + +/** Opaque data structure that holds a signer's _secret_ nonce. + * + * Guaranteed to be 132 bytes in size. + * + * WARNING: This structure MUST NOT be copied or read or written to directly. A + * signer who is online throughout the whole process and can keep this + * structure in memory can use the provided API functions for a safe standard + * workflow. + * + * Copying this data structure can result in nonce reuse which will leak the + * secret signing key. + */ +typedef struct secp256k1_musig_secnonce { + unsigned char data[132]; +} secp256k1_musig_secnonce; + +/** Opaque data structure that holds a signer's public nonce. + * + * Guaranteed to be 132 bytes in size. Serialized and parsed with + * `musig_pubnonce_serialize` and `musig_pubnonce_parse`. + */ +typedef struct secp256k1_musig_pubnonce { + unsigned char data[132]; +} secp256k1_musig_pubnonce; + +/** Opaque data structure that holds an aggregate public nonce. + * + * Guaranteed to be 132 bytes in size. Serialized and parsed with + * `musig_aggnonce_serialize` and `musig_aggnonce_parse`. + */ +typedef struct secp256k1_musig_aggnonce { + unsigned char data[132]; +} secp256k1_musig_aggnonce; + +/** Opaque data structure that holds a MuSig session. + * + * This structure is not required to be kept secret for the signing protocol to + * be secure. Guaranteed to be 133 bytes in size. No serialization and parsing + * functions (yet). + */ +typedef struct secp256k1_musig_session { + unsigned char data[133]; +} secp256k1_musig_session; + +/** Opaque data structure that holds a partial MuSig signature. + * + * Guaranteed to be 36 bytes in size. Serialized and parsed with + * `musig_partial_sig_serialize` and `musig_partial_sig_parse`. + */ +typedef struct secp256k1_musig_partial_sig { + unsigned char data[36]; +} secp256k1_musig_partial_sig; + +/** Parse a signer's public nonce. + * + * Returns: 1 when the nonce could be parsed, 0 otherwise. + * Args: ctx: pointer to a context object + * Out: nonce: pointer to a nonce object + * In: in66: pointer to the 66-byte nonce to be parsed + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubnonce_parse( + const secp256k1_context *ctx, + secp256k1_musig_pubnonce *nonce, + const unsigned char *in66 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize a signer's public nonce + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * Out: out66: pointer to a 66-byte array to store the serialized nonce + * In: nonce: pointer to the nonce + */ +SECP256K1_API int secp256k1_musig_pubnonce_serialize( + const secp256k1_context *ctx, + unsigned char *out66, + const secp256k1_musig_pubnonce *nonce +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Parse an aggregate public nonce. + * + * Returns: 1 when the nonce could be parsed, 0 otherwise. + * Args: ctx: pointer to a context object + * Out: nonce: pointer to a nonce object + * In: in66: pointer to the 66-byte nonce to be parsed + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_aggnonce_parse( + const secp256k1_context *ctx, + secp256k1_musig_aggnonce *nonce, + const unsigned char *in66 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize an aggregate public nonce + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * Out: out66: pointer to a 66-byte array to store the serialized nonce + * In: nonce: pointer to the nonce + */ +SECP256K1_API int secp256k1_musig_aggnonce_serialize( + const secp256k1_context *ctx, + unsigned char *out66, + const secp256k1_musig_aggnonce *nonce +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Parse a MuSig partial signature. + * + * Returns: 1 when the signature could be parsed, 0 otherwise. + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: in32: pointer to the 32-byte signature to be parsed + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_partial_sig_parse( + const secp256k1_context *ctx, + secp256k1_musig_partial_sig *sig, + const unsigned char *in32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize a MuSig partial signature + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * Out: out32: pointer to a 32-byte array to store the serialized signature + * In: sig: pointer to the signature + */ +SECP256K1_API int secp256k1_musig_partial_sig_serialize( + const secp256k1_context *ctx, + unsigned char *out32, + const secp256k1_musig_partial_sig *sig +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Computes an aggregate public key and uses it to initialize a keyagg_cache + * + * Different orders of `pubkeys` result in different `agg_pk`s. + * + * Before aggregating, the pubkeys can be sorted with `secp256k1_ec_pubkey_sort` + * which ensures the same `agg_pk` result for the same multiset of pubkeys. + * This is useful to do before `pubkey_agg`, such that the order of pubkeys + * does not affect the aggregate public key. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: agg_pk: the MuSig-aggregated x-only public key. If you do not need it, + * this arg can be NULL. + * keyagg_cache: if non-NULL, pointer to a musig_keyagg_cache struct that + * is required for signing (or observing the signing session + * and verifying partial signatures). + * In: pubkeys: input array of pointers to public keys to aggregate. The order + * is important; a different order will result in a different + * aggregate public key. + * n_pubkeys: length of pubkeys array. Must be greater than 0. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubkey_agg( + const secp256k1_context *ctx, + secp256k1_xonly_pubkey *agg_pk, + secp256k1_musig_keyagg_cache *keyagg_cache, + const secp256k1_pubkey * const *pubkeys, + size_t n_pubkeys +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(4); + +/** Obtain the aggregate public key from a keyagg_cache. + * + * This is only useful if you need the non-xonly public key, in particular for + * plain (non-xonly) tweaking or batch-verifying multiple key aggregations + * (not implemented). + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: agg_pk: the MuSig-aggregated public key. + * In: keyagg_cache: pointer to a `musig_keyagg_cache` struct initialized by + * `musig_pubkey_agg` + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubkey_get( + const secp256k1_context *ctx, + secp256k1_pubkey *agg_pk, + const secp256k1_musig_keyagg_cache *keyagg_cache +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Apply plain "EC" tweaking to a public key in a given keyagg_cache by adding + * the generator multiplied with `tweak32` to it. This is useful for deriving + * child keys from an aggregate public key via BIP 32 where `tweak32` is set to + * a hash as defined in BIP 32. + * + * Callers are responsible for deriving `tweak32` in a way that does not reduce + * the security of MuSig (for example, by following BIP 32). + * + * The tweaking method is the same as `secp256k1_ec_pubkey_tweak_add`. So after + * the following pseudocode buf and buf2 have identical contents (absent + * earlier failures). + * + * secp256k1_musig_pubkey_agg(..., keyagg_cache, pubkeys, ...) + * secp256k1_musig_pubkey_get(..., agg_pk, keyagg_cache) + * secp256k1_musig_pubkey_ec_tweak_add(..., output_pk, tweak32, keyagg_cache) + * secp256k1_ec_pubkey_serialize(..., buf, ..., output_pk, ...) + * secp256k1_ec_pubkey_tweak_add(..., agg_pk, tweak32) + * secp256k1_ec_pubkey_serialize(..., buf2, ..., agg_pk, ...) + * + * This function is required if you want to _sign_ for a tweaked aggregate key. + * If you are only computing a public key but not intending to create a + * signature for it, use `secp256k1_ec_pubkey_tweak_add` instead. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: output_pubkey: pointer to a public key to store the result. Will be set + * to an invalid value if this function returns 0. If you + * do not need it, this arg can be NULL. + * In/Out: keyagg_cache: pointer to a `musig_keyagg_cache` struct initialized by + * `musig_pubkey_agg` + * In: tweak32: pointer to a 32-byte tweak. The tweak is valid if it passes + * `secp256k1_ec_seckey_verify` and is not equal to the + * secret key corresponding to the public key represented + * by keyagg_cache or its negation. For uniformly random + * 32-byte arrays the chance of being invalid is + * negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubkey_ec_tweak_add( + const secp256k1_context *ctx, + secp256k1_pubkey *output_pubkey, + secp256k1_musig_keyagg_cache *keyagg_cache, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Apply x-only tweaking to a public key in a given keyagg_cache by adding the + * generator multiplied with `tweak32` to it. This is useful for creating + * Taproot outputs where `tweak32` is set to a TapTweak hash as defined in BIP + * 341. + * + * Callers are responsible for deriving `tweak32` in a way that does not reduce + * the security of MuSig (for example, by following Taproot BIP 341). + * + * The tweaking method is the same as `secp256k1_xonly_pubkey_tweak_add`. So in + * the following pseudocode xonly_pubkey_tweak_add_check (absent earlier + * failures) returns 1. + * + * secp256k1_musig_pubkey_agg(..., agg_pk, keyagg_cache, pubkeys, ...) + * secp256k1_musig_pubkey_xonly_tweak_add(..., output_pk, keyagg_cache, tweak32) + * secp256k1_xonly_pubkey_serialize(..., buf, output_pk) + * secp256k1_xonly_pubkey_tweak_add_check(..., buf, ..., agg_pk, tweak32) + * + * This function is required if you want to _sign_ for a tweaked aggregate key. + * If you are only computing a public key but not intending to create a + * signature for it, use `secp256k1_xonly_pubkey_tweak_add` instead. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: output_pubkey: pointer to a public key to store the result. Will be set + * to an invalid value if this function returns 0. If you + * do not need it, this arg can be NULL. + * In/Out: keyagg_cache: pointer to a `musig_keyagg_cache` struct initialized by + * `musig_pubkey_agg` + * In: tweak32: pointer to a 32-byte tweak. The tweak is valid if it passes + * `secp256k1_ec_seckey_verify` and is not equal to the + * secret key corresponding to the public key represented + * by keyagg_cache or its negation. For uniformly random + * 32-byte arrays the chance of being invalid is + * negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubkey_xonly_tweak_add( + const secp256k1_context *ctx, + secp256k1_pubkey *output_pubkey, + secp256k1_musig_keyagg_cache *keyagg_cache, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Starts a signing session by generating a nonce + * + * This function outputs a secret nonce that will be required for signing and a + * corresponding public nonce that is intended to be sent to other signers. + * + * MuSig differs from regular Schnorr signing in that implementers _must_ take + * special care to not reuse a nonce. This can be ensured by following these rules: + * + * 1. Each call to this function must have a UNIQUE session_secrand32 that must + * NOT BE REUSED in subsequent calls to this function and must be KEPT + * SECRET (even from other signers). + * 2. If you already know the seckey, message or aggregate public key + * cache, they can be optionally provided to derive the nonce and increase + * misuse-resistance. The extra_input32 argument can be used to provide + * additional data that does not repeat in normal scenarios, such as the + * current time. + * 3. Avoid copying (or serializing) the secnonce. This reduces the possibility + * that it is used more than once for signing. + * + * If you don't have access to good randomness for session_secrand32, but you + * have access to a non-repeating counter, then see + * secp256k1_musig_nonce_gen_counter. + * + * Remember that nonce reuse will leak the secret key! + * Note that using the same seckey for multiple MuSig sessions is fine. + * + * Returns: 0 if the arguments are invalid and 1 otherwise + * Args: ctx: pointer to a context object (not secp256k1_context_static) + * Out: secnonce: pointer to a structure to store the secret nonce + * pubnonce: pointer to a structure to store the public nonce + * In/Out: + * session_secrand32: a 32-byte session_secrand32 as explained above. Must be unique to this + * call to secp256k1_musig_nonce_gen and must be uniformly + * random. If the function call is successful, the + * session_secrand32 buffer is invalidated to prevent reuse. + * In: + * seckey: the 32-byte secret key that will later be used for signing, if + * already known (can be NULL) + * pubkey: public key of the signer creating the nonce. The secnonce + * output of this function cannot be used to sign for any + * other public key. While the public key should correspond + * to the provided seckey, a mismatch will not cause the + * function to return 0. + * msg32: the 32-byte message that will later be signed, if already known + * (can be NULL) + * keyagg_cache: pointer to the keyagg_cache that was used to create the aggregate + * (and potentially tweaked) public key if already known + * (can be NULL) + * extra_input32: an optional 32-byte array that is input to the nonce + * derivation function (can be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_nonce_gen( + const secp256k1_context *ctx, + secp256k1_musig_secnonce *secnonce, + secp256k1_musig_pubnonce *pubnonce, + unsigned char *session_secrand32, + const unsigned char *seckey, + const secp256k1_pubkey *pubkey, + const unsigned char *msg32, + const secp256k1_musig_keyagg_cache *keyagg_cache, + const unsigned char *extra_input32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(6); + + +/** Alternative way to generate a nonce and start a signing session + * + * This function outputs a secret nonce that will be required for signing and a + * corresponding public nonce that is intended to be sent to other signers. + * + * This function differs from `secp256k1_musig_nonce_gen` by accepting a + * non-repeating counter value instead of a secret random value. This requires + * that a secret key is provided to `secp256k1_musig_nonce_gen_counter` + * (through the keypair argument), as opposed to `secp256k1_musig_nonce_gen` + * where the seckey argument is optional. + * + * MuSig differs from regular Schnorr signing in that implementers _must_ take + * special care to not reuse a nonce. This can be ensured by following these rules: + * + * 1. The nonrepeating_cnt argument must be a counter value that never repeats, + * i.e., you must never call `secp256k1_musig_nonce_gen_counter` twice with + * the same keypair and nonrepeating_cnt value. For example, this implies + * that if the same keypair is used with `secp256k1_musig_nonce_gen_counter` + * on multiple devices, none of the devices should have the same counter + * value as any other device. + * 2. If the seckey, message or aggregate public key cache is already available + * at this stage, any of these can be optionally provided, in which case + * they will be used in the derivation of the nonce and increase + * misuse-resistance. The extra_input32 argument can be used to provide + * additional data that does not repeat in normal scenarios, such as the + * current time. + * 3. Avoid copying (or serializing) the secnonce. This reduces the possibility + * that it is used more than once for signing. + * + * Remember that nonce reuse will leak the secret key! + * Note that using the same keypair for multiple MuSig sessions is fine. + * + * Returns: 0 if the arguments are invalid and 1 otherwise + * Args: ctx: pointer to a context object (not secp256k1_context_static) + * Out: secnonce: pointer to a structure to store the secret nonce + * pubnonce: pointer to a structure to store the public nonce + * In: + * nonrepeating_cnt: the value of a counter as explained above. Must be + * unique to this call to secp256k1_musig_nonce_gen. + * keypair: keypair of the signer creating the nonce. The secnonce + * output of this function cannot be used to sign for any + * other keypair. + * msg32: the 32-byte message that will later be signed, if already known + * (can be NULL) + * keyagg_cache: pointer to the keyagg_cache that was used to create the aggregate + * (and potentially tweaked) public key if already known + * (can be NULL) + * extra_input32: an optional 32-byte array that is input to the nonce + * derivation function (can be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_nonce_gen_counter( + const secp256k1_context *ctx, + secp256k1_musig_secnonce *secnonce, + secp256k1_musig_pubnonce *pubnonce, + uint64_t nonrepeating_cnt, + const secp256k1_keypair *keypair, + const unsigned char *msg32, + const secp256k1_musig_keyagg_cache *keyagg_cache, + const unsigned char *extra_input32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(5); + +/** Aggregates the nonces of all signers into a single nonce + * + * This can be done by an untrusted party to reduce the communication + * between signers. Instead of everyone sending nonces to everyone else, there + * can be one party receiving all nonces, aggregating the nonces with this + * function and then sending only the aggregate nonce back to the signers. + * + * If the aggregator does not compute the aggregate nonce correctly, the final + * signature will be invalid. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: aggnonce: pointer to an aggregate public nonce object for + * musig_nonce_process + * In: pubnonces: array of pointers to public nonces sent by the + * signers + * n_pubnonces: number of elements in the pubnonces array. Must be + * greater than 0. + */ +SECP256K1_API int secp256k1_musig_nonce_agg( + const secp256k1_context *ctx, + secp256k1_musig_aggnonce *aggnonce, + const secp256k1_musig_pubnonce * const *pubnonces, + size_t n_pubnonces +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Takes the aggregate nonce and creates a session that is required for signing + * and verification of partial signatures. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: session: pointer to a struct to store the session + * In: aggnonce: pointer to an aggregate public nonce object that is the + * output of musig_nonce_agg + * msg32: the 32-byte message to sign + * keyagg_cache: pointer to the keyagg_cache that was used to create the + * aggregate (and potentially tweaked) pubkey + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_nonce_process( + const secp256k1_context *ctx, + secp256k1_musig_session *session, + const secp256k1_musig_aggnonce *aggnonce, + const unsigned char *msg32, + const secp256k1_musig_keyagg_cache *keyagg_cache +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5); + +/** Produces a partial signature + * + * This function overwrites the given secnonce with zeros and will abort if given a + * secnonce that is all zeros. This is a best effort attempt to protect against nonce + * reuse. However, this is of course easily defeated if the secnonce has been + * copied (or serialized). Remember that nonce reuse will leak the secret key! + * + * For signing to succeed, the secnonce provided to this function must have + * been generated for the provided keypair. This means that when signing for a + * keypair consisting of a seckey and pubkey, the secnonce must have been + * created by calling musig_nonce_gen with that pubkey. Otherwise, the + * illegal_callback is called. + * + * This function does not verify the output partial signature, deviating from + * the BIP 327 specification. It is recommended to verify the output partial + * signature with `secp256k1_musig_partial_sig_verify` to prevent random or + * adversarially provoked computation errors. + * + * Returns: 0 if the arguments are invalid or the provided secnonce has already + * been used for signing, 1 otherwise + * Args: ctx: pointer to a context object + * Out: partial_sig: pointer to struct to store the partial signature + * In/Out: secnonce: pointer to the secnonce struct created in + * musig_nonce_gen that has been never used in a + * partial_sign call before and has been created for the + * keypair + * In: keypair: pointer to keypair to sign the message with + * keyagg_cache: pointer to the keyagg_cache that was output when the + * aggregate public key for this session + * session: pointer to the session that was created with + * musig_nonce_process + */ +SECP256K1_API int secp256k1_musig_partial_sign( + const secp256k1_context *ctx, + secp256k1_musig_partial_sig *partial_sig, + secp256k1_musig_secnonce *secnonce, + const secp256k1_keypair *keypair, + const secp256k1_musig_keyagg_cache *keyagg_cache, + const secp256k1_musig_session *session +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5) SECP256K1_ARG_NONNULL(6); + +/** Verifies an individual signer's partial signature + * + * The signature is verified for a specific signing session. In order to avoid + * accidentally verifying a signature from a different or non-existing signing + * session, you must ensure the following: + * 1. The `keyagg_cache` argument is identical to the one used to create the + * `session` with `musig_nonce_process`. + * 2. The `pubkey` argument must be identical to the one sent by the signer + * before aggregating it with `musig_pubkey_agg` to create the + * `keyagg_cache`. + * 3. The `pubnonce` argument must be identical to the one sent by the signer + * before aggregating it with `musig_nonce_agg` and using the result to + * create the `session` with `musig_nonce_process`. + * + * It is not required to call this function in regular MuSig sessions, because + * if any partial signature does not verify, the final signature will not + * verify either, so the problem will be caught. However, this function + * provides the ability to identify which specific partial signature fails + * verification. + * + * Returns: 0 if the arguments are invalid or the partial signature does not + * verify, 1 otherwise + * Args ctx: pointer to a context object + * In: partial_sig: pointer to partial signature to verify, sent by + * the signer associated with `pubnonce` and `pubkey` + * pubnonce: public nonce of the signer in the signing session + * pubkey: public key of the signer in the signing session + * keyagg_cache: pointer to the keyagg_cache that was output when the + * aggregate public key for this signing session + * session: pointer to the session that was created with + * `musig_nonce_process` + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_partial_sig_verify( + const secp256k1_context *ctx, + const secp256k1_musig_partial_sig *partial_sig, + const secp256k1_musig_pubnonce *pubnonce, + const secp256k1_pubkey *pubkey, + const secp256k1_musig_keyagg_cache *keyagg_cache, + const secp256k1_musig_session *session +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5) SECP256K1_ARG_NONNULL(6); + +/** Aggregates partial signatures + * + * Returns: 0 if the arguments are invalid, 1 otherwise (which does NOT mean + * the resulting signature verifies). + * Args: ctx: pointer to a context object + * Out: sig64: complete (but possibly invalid) Schnorr signature + * In: session: pointer to the session that was created with + * musig_nonce_process + * partial_sigs: array of pointers to partial signatures to aggregate + * n_sigs: number of elements in the partial_sigs array. Must be + * greater than 0. + */ +SECP256K1_API int secp256k1_musig_partial_sig_agg( + const secp256k1_context *ctx, + unsigned char *sig64, + const secp256k1_musig_session *session, + const secp256k1_musig_partial_sig * const *partial_sigs, + size_t n_sigs +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/secp256k1/include/secp256k1_recovery.h b/src/secp256k1/include/secp256k1_recovery.h index 341b8bac63..93a2e4ccbd 100644 --- a/src/secp256k1/include/secp256k1_recovery.h +++ b/src/secp256k1/include/secp256k1_recovery.h @@ -7,7 +7,7 @@ extern "C" { #endif -/** Opaque data structured that holds a parsed ECDSA signature, +/** Opaque data structure that holds a parsed ECDSA signature, * supporting pubkey recovery. * * The exact representation of data inside is implementation defined and not @@ -21,7 +21,7 @@ extern "C" { * recoverability) will have identical representation, so they can be * memcmp'ed. */ -typedef struct { +typedef struct secp256k1_ecdsa_recoverable_signature { unsigned char data[65]; } secp256k1_ecdsa_recoverable_signature; diff --git a/src/secp256k1/include/secp256k1_schnorrsig.h b/src/secp256k1/include/secp256k1_schnorrsig.h index 23163de2fb..013d4ee73d 100644 --- a/src/secp256k1/include/secp256k1_schnorrsig.h +++ b/src/secp256k1/include/secp256k1_schnorrsig.h @@ -79,7 +79,7 @@ SECP256K1_API const secp256k1_nonce_function_hardened secp256k1_nonce_function_b * secp256k1_nonce_function_bip340 is used, then ndata must be a * pointer to 32-byte auxiliary randomness as per BIP-340. */ -typedef struct { +typedef struct secp256k1_schnorrsig_extraparams { unsigned char magic[4]; secp256k1_nonce_function_hardened noncefp; void *ndata; diff --git a/src/secp256k1/src/CMakeLists.txt b/src/secp256k1/src/CMakeLists.txt index 4cbaeb914d..f31b8c8f55 100644 --- a/src/secp256k1/src/CMakeLists.txt +++ b/src/secp256k1/src/CMakeLists.txt @@ -87,12 +87,12 @@ endif() if(SECP256K1_BUILD_TESTS) add_executable(noverify_tests tests.c) target_link_libraries(noverify_tests secp256k1_precomputed secp256k1_asm) - add_test(NAME noverify_tests COMMAND noverify_tests) + add_test(NAME secp256k1_noverify_tests COMMAND noverify_tests) if(NOT CMAKE_BUILD_TYPE STREQUAL "Coverage") add_executable(tests tests.c) target_compile_definitions(tests PRIVATE VERIFY) target_link_libraries(tests secp256k1_precomputed secp256k1_asm) - add_test(NAME tests COMMAND tests) + add_test(NAME secp256k1_tests COMMAND tests) endif() endif() @@ -101,7 +101,7 @@ if(SECP256K1_BUILD_EXHAUSTIVE_TESTS) add_executable(exhaustive_tests tests_exhaustive.c) target_link_libraries(exhaustive_tests secp256k1_asm) target_compile_definitions(exhaustive_tests PRIVATE $<$<NOT:$<CONFIG:Coverage>>:VERIFY>) - add_test(NAME exhaustive_tests COMMAND exhaustive_tests) + add_test(NAME secp256k1_exhaustive_tests COMMAND exhaustive_tests) endif() if(SECP256K1_BUILD_CTIME_TESTS) @@ -132,6 +132,9 @@ if(SECP256K1_INSTALL) if(SECP256K1_ENABLE_MODULE_SCHNORRSIG) list(APPEND ${PROJECT_NAME}_headers "${PROJECT_SOURCE_DIR}/include/secp256k1_schnorrsig.h") endif() + if(SECP256K1_ENABLE_MODULE_MUSIG) + list(APPEND ${PROJECT_NAME}_headers "${PROJECT_SOURCE_DIR}/include/secp256k1_musig.h") + endif() if(SECP256K1_ENABLE_MODULE_ELLSWIFT) list(APPEND ${PROJECT_NAME}_headers "${PROJECT_SOURCE_DIR}/include/secp256k1_ellswift.h") endif() diff --git a/src/secp256k1/src/bench_ecmult.c b/src/secp256k1/src/bench_ecmult.c index 7dc52ad87b..3974af75f4 100644 --- a/src/secp256k1/src/bench_ecmult.c +++ b/src/secp256k1/src/bench_ecmult.c @@ -71,7 +71,7 @@ static void bench_ecmult_teardown_helper(bench_data* data, size_t* seckey_offset secp256k1_scalar sum_scalars; secp256k1_gej_set_infinity(&sum_output); - secp256k1_scalar_clear(&sum_scalars); + secp256k1_scalar_set_int(&sum_scalars, 0); for (i = 0; i < iters; ++i) { secp256k1_gej_add_var(&sum_output, &sum_output, &data->output[i], NULL); if (scalar_gen_offset != NULL) { diff --git a/src/secp256k1/src/ctime_tests.c b/src/secp256k1/src/ctime_tests.c index a384e83152..bbde863d96 100644 --- a/src/secp256k1/src/ctime_tests.c +++ b/src/secp256k1/src/ctime_tests.c @@ -5,6 +5,7 @@ ***********************************************************************/ #include <stdio.h> +#include <string.h> #include "../include/secp256k1.h" #include "assumptions.h" @@ -30,6 +31,10 @@ #include "../include/secp256k1_schnorrsig.h" #endif +#ifdef ENABLE_MODULE_MUSIG +#include "../include/secp256k1_musig.h" +#endif + #ifdef ENABLE_MODULE_ELLSWIFT #include "../include/secp256k1_ellswift.h" #endif @@ -180,6 +185,58 @@ static void run_tests(secp256k1_context *ctx, unsigned char *key) { CHECK(ret == 1); #endif +#ifdef ENABLE_MODULE_MUSIG + { + secp256k1_pubkey pk; + const secp256k1_pubkey *pk_ptr[1]; + secp256k1_xonly_pubkey agg_pk; + unsigned char session_secrand[32]; + uint64_t nonrepeating_cnt = 0; + secp256k1_musig_secnonce secnonce; + secp256k1_musig_pubnonce pubnonce; + const secp256k1_musig_pubnonce *pubnonce_ptr[1]; + secp256k1_musig_aggnonce aggnonce; + secp256k1_musig_keyagg_cache cache; + secp256k1_musig_session session; + secp256k1_musig_partial_sig partial_sig; + unsigned char extra_input[32]; + + pk_ptr[0] = &pk; + pubnonce_ptr[0] = &pubnonce; + SECP256K1_CHECKMEM_DEFINE(key, 32); + memcpy(session_secrand, key, sizeof(session_secrand)); + session_secrand[0] = session_secrand[0] + 1; + memcpy(extra_input, key, sizeof(extra_input)); + extra_input[0] = extra_input[0] + 2; + + CHECK(secp256k1_keypair_create(ctx, &keypair, key)); + CHECK(secp256k1_keypair_pub(ctx, &pk, &keypair)); + CHECK(secp256k1_musig_pubkey_agg(ctx, &agg_pk, &cache, pk_ptr, 1)); + + SECP256K1_CHECKMEM_UNDEFINE(key, 32); + SECP256K1_CHECKMEM_UNDEFINE(session_secrand, sizeof(session_secrand)); + SECP256K1_CHECKMEM_UNDEFINE(extra_input, sizeof(extra_input)); + ret = secp256k1_musig_nonce_gen(ctx, &secnonce, &pubnonce, session_secrand, key, &pk, msg, &cache, extra_input); + SECP256K1_CHECKMEM_DEFINE(&ret, sizeof(ret)); + CHECK(ret == 1); + ret = secp256k1_musig_nonce_gen_counter(ctx, &secnonce, &pubnonce, nonrepeating_cnt, &keypair, msg, &cache, extra_input); + SECP256K1_CHECKMEM_DEFINE(&ret, sizeof(ret)); + CHECK(ret == 1); + + CHECK(secp256k1_musig_nonce_agg(ctx, &aggnonce, pubnonce_ptr, 1)); + /* Make sure that previous tests don't undefine msg. It's not used as a secret here. */ + SECP256K1_CHECKMEM_DEFINE(msg, sizeof(msg)); + CHECK(secp256k1_musig_nonce_process(ctx, &session, &aggnonce, msg, &cache) == 1); + + ret = secp256k1_keypair_create(ctx, &keypair, key); + SECP256K1_CHECKMEM_DEFINE(&ret, sizeof(ret)); + CHECK(ret == 1); + ret = secp256k1_musig_partial_sign(ctx, &partial_sig, &secnonce, &keypair, &cache, &session); + SECP256K1_CHECKMEM_DEFINE(&ret, sizeof(ret)); + CHECK(ret == 1); + } +#endif + #ifdef ENABLE_MODULE_ELLSWIFT SECP256K1_CHECKMEM_UNDEFINE(key, 32); ret = secp256k1_ellswift_create(ctx, ellswift, key, NULL); diff --git a/src/secp256k1/src/ecmult_gen_impl.h b/src/secp256k1/src/ecmult_gen_impl.h index 2fbf623ca3..070a121308 100644 --- a/src/secp256k1/src/ecmult_gen_impl.h +++ b/src/secp256k1/src/ecmult_gen_impl.h @@ -277,8 +277,8 @@ static void secp256k1_ecmult_gen(const secp256k1_ecmult_gen_context *ctx, secp25 /* Cleanup. */ secp256k1_fe_clear(&neg); secp256k1_ge_clear(&add); - memset(&adds, 0, sizeof(adds)); - memset(&recoded, 0, sizeof(recoded)); + secp256k1_memclear(&adds, sizeof(adds)); + secp256k1_memclear(&recoded, sizeof(recoded)); } /* Setup blinding values for secp256k1_ecmult_gen. */ @@ -310,7 +310,7 @@ static void secp256k1_ecmult_gen_blind(secp256k1_ecmult_gen_context *ctx, const VERIFY_CHECK(seed32 != NULL); memcpy(keydata + 32, seed32, 32); secp256k1_rfc6979_hmac_sha256_initialize(&rng, keydata, 64); - memset(keydata, 0, sizeof(keydata)); + secp256k1_memclear(keydata, sizeof(keydata)); /* Compute projective blinding factor (cannot be 0). */ secp256k1_rfc6979_hmac_sha256_generate(&rng, nonce32, 32); @@ -325,16 +325,17 @@ static void secp256k1_ecmult_gen_blind(secp256k1_ecmult_gen_context *ctx, const * which secp256k1_gej_add_ge cannot handle. */ secp256k1_scalar_cmov(&b, &secp256k1_scalar_one, secp256k1_scalar_is_zero(&b)); secp256k1_rfc6979_hmac_sha256_finalize(&rng); - memset(nonce32, 0, 32); secp256k1_ecmult_gen(ctx, &gb, &b); secp256k1_scalar_negate(&b, &b); secp256k1_scalar_add(&ctx->scalar_offset, &b, &diff); secp256k1_ge_set_gej(&ctx->ge_offset, &gb); /* Clean up. */ + secp256k1_memclear(nonce32, sizeof(nonce32)); secp256k1_scalar_clear(&b); secp256k1_gej_clear(&gb); secp256k1_fe_clear(&f); + secp256k1_rfc6979_hmac_sha256_clear(&rng); } #endif /* SECP256K1_ECMULT_GEN_IMPL_H */ diff --git a/src/secp256k1/src/ecmult_impl.h b/src/secp256k1/src/ecmult_impl.h index 07d8477eb0..0b53b3fcb9 100644 --- a/src/secp256k1/src/ecmult_impl.h +++ b/src/secp256k1/src/ecmult_impl.h @@ -171,7 +171,9 @@ static int secp256k1_ecmult_wnaf(int *wnaf, int len, const secp256k1_scalar *a, VERIFY_CHECK(a != NULL); VERIFY_CHECK(2 <= w && w <= 31); - memset(wnaf, 0, len * sizeof(wnaf[0])); + for (bit = 0; bit < len; bit++) { + wnaf[bit] = 0; + } s = *a; if (secp256k1_scalar_get_bits_limb32(&s, 255, 1)) { @@ -179,6 +181,7 @@ static int secp256k1_ecmult_wnaf(int *wnaf, int len, const secp256k1_scalar *a, sign = -1; } + bit = 0; while (bit < len) { int now; int word; @@ -660,7 +663,6 @@ static int secp256k1_ecmult_pippenger_batch(const secp256k1_callback* error_call struct secp256k1_pippenger_state *state_space; size_t idx = 0; size_t point_idx = 0; - int i, j; int bucket_window; secp256k1_gej_set_infinity(r); @@ -708,18 +710,6 @@ static int secp256k1_ecmult_pippenger_batch(const secp256k1_callback* error_call } secp256k1_ecmult_pippenger_wnaf(buckets, bucket_window, state_space, r, scalars, points, idx); - - /* Clear data */ - for(i = 0; (size_t)i < idx; i++) { - secp256k1_scalar_clear(&scalars[i]); - state_space->ps[i].skew_na = 0; - for(j = 0; j < WNAF_SIZE(bucket_window+1); j++) { - state_space->wnaf_na[i * WNAF_SIZE(bucket_window+1) + j] = 0; - } - } - for(i = 0; i < 1<<bucket_window; i++) { - secp256k1_gej_clear(&buckets[i]); - } secp256k1_scratch_apply_checkpoint(error_callback, scratch, scratch_checkpoint); return 1; } diff --git a/src/secp256k1/src/field.h b/src/secp256k1/src/field.h index 8c65a3aff6..1f6ba7460f 100644 --- a/src/secp256k1/src/field.h +++ b/src/secp256k1/src/field.h @@ -81,7 +81,6 @@ static const secp256k1_fe secp256k1_const_beta = SECP256K1_FE_CONST( # define secp256k1_fe_normalizes_to_zero secp256k1_fe_impl_normalizes_to_zero # define secp256k1_fe_normalizes_to_zero_var secp256k1_fe_impl_normalizes_to_zero_var # define secp256k1_fe_set_int secp256k1_fe_impl_set_int -# define secp256k1_fe_clear secp256k1_fe_impl_clear # define secp256k1_fe_is_zero secp256k1_fe_impl_is_zero # define secp256k1_fe_is_odd secp256k1_fe_impl_is_odd # define secp256k1_fe_cmp_var secp256k1_fe_impl_cmp_var @@ -144,11 +143,7 @@ static int secp256k1_fe_normalizes_to_zero_var(const secp256k1_fe *r); */ static void secp256k1_fe_set_int(secp256k1_fe *r, int a); -/** Set a field element to 0. - * - * On input, a does not need to be initialized. - * On output, a represents 0, is normalized and has magnitude 0. - */ +/** Clear a field element to prevent leaking sensitive information. */ static void secp256k1_fe_clear(secp256k1_fe *a); /** Determine whether a represents field element 0. diff --git a/src/secp256k1/src/field_10x26_impl.h b/src/secp256k1/src/field_10x26_impl.h index 666068c712..ea14c27318 100644 --- a/src/secp256k1/src/field_10x26_impl.h +++ b/src/secp256k1/src/field_10x26_impl.h @@ -270,13 +270,6 @@ SECP256K1_INLINE static int secp256k1_fe_impl_is_odd(const secp256k1_fe *a) { return a->n[0] & 1; } -SECP256K1_INLINE static void secp256k1_fe_impl_clear(secp256k1_fe *a) { - int i; - for (i=0; i<10; i++) { - a->n[i] = 0; - } -} - static int secp256k1_fe_impl_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { int i; for (i = 9; i >= 0; i--) { diff --git a/src/secp256k1/src/field_5x52_impl.h b/src/secp256k1/src/field_5x52_impl.h index 76031f755e..46dca6b981 100644 --- a/src/secp256k1/src/field_5x52_impl.h +++ b/src/secp256k1/src/field_5x52_impl.h @@ -212,13 +212,6 @@ SECP256K1_INLINE static int secp256k1_fe_impl_is_odd(const secp256k1_fe *a) { return a->n[0] & 1; } -SECP256K1_INLINE static void secp256k1_fe_impl_clear(secp256k1_fe *a) { - int i; - for (i=0; i<5; i++) { - a->n[i] = 0; - } -} - static int secp256k1_fe_impl_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { int i; for (i = 4; i >= 0; i--) { diff --git a/src/secp256k1/src/field_impl.h b/src/secp256k1/src/field_impl.h index 989e9cdb2f..896507a3a4 100644 --- a/src/secp256k1/src/field_impl.h +++ b/src/secp256k1/src/field_impl.h @@ -18,6 +18,10 @@ #error "Please select wide multiplication implementation" #endif +SECP256K1_INLINE static void secp256k1_fe_clear(secp256k1_fe *a) { + secp256k1_memclear(a, sizeof(secp256k1_fe)); +} + SECP256K1_INLINE static int secp256k1_fe_equal(const secp256k1_fe *a, const secp256k1_fe *b) { secp256k1_fe na; SECP256K1_FE_VERIFY(a); @@ -232,15 +236,6 @@ SECP256K1_INLINE static void secp256k1_fe_add_int(secp256k1_fe *r, int a) { SECP256K1_FE_VERIFY(r); } -static void secp256k1_fe_impl_clear(secp256k1_fe *a); -SECP256K1_INLINE static void secp256k1_fe_clear(secp256k1_fe *a) { - a->magnitude = 0; - a->normalized = 1; - secp256k1_fe_impl_clear(a); - - SECP256K1_FE_VERIFY(a); -} - static int secp256k1_fe_impl_is_zero(const secp256k1_fe *a); SECP256K1_INLINE static int secp256k1_fe_is_zero(const secp256k1_fe *a) { SECP256K1_FE_VERIFY(a); diff --git a/src/secp256k1/src/group.h b/src/secp256k1/src/group.h index d81deb4264..992ff5c98c 100644 --- a/src/secp256k1/src/group.h +++ b/src/secp256k1/src/group.h @@ -174,6 +174,22 @@ static void secp256k1_ge_storage_cmov(secp256k1_ge_storage *r, const secp256k1_g /** Rescale a jacobian point by b which must be non-zero. Constant-time. */ static void secp256k1_gej_rescale(secp256k1_gej *r, const secp256k1_fe *b); +/** Convert a group element that is not infinity to a 64-byte array. The output + * array is platform-dependent. */ +static void secp256k1_ge_to_bytes(unsigned char *buf, const secp256k1_ge *a); + +/** Convert a 64-byte array into group element. This function assumes that the + * provided buffer correctly encodes a group element. */ +static void secp256k1_ge_from_bytes(secp256k1_ge *r, const unsigned char *buf); + +/** Convert a group element (that is allowed to be infinity) to a 64-byte + * array. The output array is platform-dependent. */ +static void secp256k1_ge_to_bytes_ext(unsigned char *data, const secp256k1_ge *ge); + +/** Convert a 64-byte array into a group element. This function assumes that the + * provided buffer is the output of secp256k1_ge_to_bytes_ext. */ +static void secp256k1_ge_from_bytes_ext(secp256k1_ge *ge, const unsigned char *data); + /** Determine if a point (which is assumed to be on the curve) is in the correct (sub)group of the curve. * * In normal mode, the used group is secp256k1, which has cofactor=1 meaning that every point on the curve is in the diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index 537be32ff6..c668fb2401 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -7,6 +7,8 @@ #ifndef SECP256K1_GROUP_IMPL_H #define SECP256K1_GROUP_IMPL_H +#include <string.h> + #include "field.h" #include "group.h" #include "util.h" @@ -281,36 +283,27 @@ static void secp256k1_ge_table_set_globalz(size_t len, secp256k1_ge *a, const se static void secp256k1_gej_set_infinity(secp256k1_gej *r) { r->infinity = 1; - secp256k1_fe_clear(&r->x); - secp256k1_fe_clear(&r->y); - secp256k1_fe_clear(&r->z); + secp256k1_fe_set_int(&r->x, 0); + secp256k1_fe_set_int(&r->y, 0); + secp256k1_fe_set_int(&r->z, 0); SECP256K1_GEJ_VERIFY(r); } static void secp256k1_ge_set_infinity(secp256k1_ge *r) { r->infinity = 1; - secp256k1_fe_clear(&r->x); - secp256k1_fe_clear(&r->y); + secp256k1_fe_set_int(&r->x, 0); + secp256k1_fe_set_int(&r->y, 0); SECP256K1_GE_VERIFY(r); } static void secp256k1_gej_clear(secp256k1_gej *r) { - r->infinity = 0; - secp256k1_fe_clear(&r->x); - secp256k1_fe_clear(&r->y); - secp256k1_fe_clear(&r->z); - - SECP256K1_GEJ_VERIFY(r); + secp256k1_memclear(r, sizeof(secp256k1_gej)); } static void secp256k1_ge_clear(secp256k1_ge *r) { - r->infinity = 0; - secp256k1_fe_clear(&r->x); - secp256k1_fe_clear(&r->y); - - SECP256K1_GE_VERIFY(r); + secp256k1_memclear(r, sizeof(secp256k1_ge)); } static int secp256k1_ge_set_xo_var(secp256k1_ge *r, const secp256k1_fe *x, int odd) { @@ -941,4 +934,41 @@ static int secp256k1_ge_x_frac_on_curve_var(const secp256k1_fe *xn, const secp25 return secp256k1_fe_is_square_var(&r); } +static void secp256k1_ge_to_bytes(unsigned char *buf, const secp256k1_ge *a) { + secp256k1_ge_storage s; + + /* We require that the secp256k1_ge_storage type is exactly 64 bytes. + * This is formally not guaranteed by the C standard, but should hold on any + * sane compiler in the real world. */ + STATIC_ASSERT(sizeof(secp256k1_ge_storage) == 64); + VERIFY_CHECK(!secp256k1_ge_is_infinity(a)); + secp256k1_ge_to_storage(&s, a); + memcpy(buf, &s, 64); +} + +static void secp256k1_ge_from_bytes(secp256k1_ge *r, const unsigned char *buf) { + secp256k1_ge_storage s; + + STATIC_ASSERT(sizeof(secp256k1_ge_storage) == 64); + memcpy(&s, buf, 64); + secp256k1_ge_from_storage(r, &s); +} + +static void secp256k1_ge_to_bytes_ext(unsigned char *data, const secp256k1_ge *ge) { + if (secp256k1_ge_is_infinity(ge)) { + memset(data, 0, 64); + } else { + secp256k1_ge_to_bytes(data, ge); + } +} + +static void secp256k1_ge_from_bytes_ext(secp256k1_ge *ge, const unsigned char *data) { + static const unsigned char zeros[64] = { 0 }; + if (secp256k1_memcmp_var(data, zeros, sizeof(zeros)) == 0) { + secp256k1_ge_set_infinity(ge); + } else { + secp256k1_ge_from_bytes(ge, data); + } +} + #endif /* SECP256K1_GROUP_IMPL_H */ diff --git a/src/secp256k1/src/hash.h b/src/secp256k1/src/hash.h index 4e0384cfbf..6d903ca7e0 100644 --- a/src/secp256k1/src/hash.h +++ b/src/secp256k1/src/hash.h @@ -19,6 +19,7 @@ typedef struct { static void secp256k1_sha256_initialize(secp256k1_sha256 *hash); static void secp256k1_sha256_write(secp256k1_sha256 *hash, const unsigned char *data, size_t size); static void secp256k1_sha256_finalize(secp256k1_sha256 *hash, unsigned char *out32); +static void secp256k1_sha256_clear(secp256k1_sha256 *hash); typedef struct { secp256k1_sha256 inner, outer; @@ -27,6 +28,7 @@ typedef struct { static void secp256k1_hmac_sha256_initialize(secp256k1_hmac_sha256 *hash, const unsigned char *key, size_t size); static void secp256k1_hmac_sha256_write(secp256k1_hmac_sha256 *hash, const unsigned char *data, size_t size); static void secp256k1_hmac_sha256_finalize(secp256k1_hmac_sha256 *hash, unsigned char *out32); +static void secp256k1_hmac_sha256_clear(secp256k1_hmac_sha256 *hash); typedef struct { unsigned char v[32]; @@ -37,5 +39,6 @@ typedef struct { static void secp256k1_rfc6979_hmac_sha256_initialize(secp256k1_rfc6979_hmac_sha256 *rng, const unsigned char *key, size_t keylen); static void secp256k1_rfc6979_hmac_sha256_generate(secp256k1_rfc6979_hmac_sha256 *rng, unsigned char *out, size_t outlen); static void secp256k1_rfc6979_hmac_sha256_finalize(secp256k1_rfc6979_hmac_sha256 *rng); +static void secp256k1_rfc6979_hmac_sha256_clear(secp256k1_rfc6979_hmac_sha256 *rng); #endif /* SECP256K1_HASH_H */ diff --git a/src/secp256k1/src/hash_impl.h b/src/secp256k1/src/hash_impl.h index 89f75ace74..956e0ea48b 100644 --- a/src/secp256k1/src/hash_impl.h +++ b/src/secp256k1/src/hash_impl.h @@ -171,6 +171,10 @@ static void secp256k1_sha256_initialize_tagged(secp256k1_sha256 *hash, const uns secp256k1_sha256_write(hash, buf, 32); } +static void secp256k1_sha256_clear(secp256k1_sha256 *hash) { + secp256k1_memclear(hash, sizeof(*hash)); +} + static void secp256k1_hmac_sha256_initialize(secp256k1_hmac_sha256 *hash, const unsigned char *key, size_t keylen) { size_t n; unsigned char rkey[64]; @@ -196,7 +200,7 @@ static void secp256k1_hmac_sha256_initialize(secp256k1_hmac_sha256 *hash, const rkey[n] ^= 0x5c ^ 0x36; } secp256k1_sha256_write(&hash->inner, rkey, sizeof(rkey)); - memset(rkey, 0, sizeof(rkey)); + secp256k1_memclear(rkey, sizeof(rkey)); } static void secp256k1_hmac_sha256_write(secp256k1_hmac_sha256 *hash, const unsigned char *data, size_t size) { @@ -207,10 +211,13 @@ static void secp256k1_hmac_sha256_finalize(secp256k1_hmac_sha256 *hash, unsigned unsigned char temp[32]; secp256k1_sha256_finalize(&hash->inner, temp); secp256k1_sha256_write(&hash->outer, temp, 32); - memset(temp, 0, 32); + secp256k1_memclear(temp, sizeof(temp)); secp256k1_sha256_finalize(&hash->outer, out32); } +static void secp256k1_hmac_sha256_clear(secp256k1_hmac_sha256 *hash) { + secp256k1_memclear(hash, sizeof(*hash)); +} static void secp256k1_rfc6979_hmac_sha256_initialize(secp256k1_rfc6979_hmac_sha256 *rng, const unsigned char *key, size_t keylen) { secp256k1_hmac_sha256 hmac; @@ -274,9 +281,11 @@ static void secp256k1_rfc6979_hmac_sha256_generate(secp256k1_rfc6979_hmac_sha256 } static void secp256k1_rfc6979_hmac_sha256_finalize(secp256k1_rfc6979_hmac_sha256 *rng) { - memset(rng->k, 0, 32); - memset(rng->v, 0, 32); - rng->retry = 0; + (void) rng; +} + +static void secp256k1_rfc6979_hmac_sha256_clear(secp256k1_rfc6979_hmac_sha256 *rng) { + secp256k1_memclear(rng, sizeof(*rng)); } #undef Round diff --git a/src/secp256k1/src/modinv32_impl.h b/src/secp256k1/src/modinv32_impl.h index 75eb354ff0..981d2abc6d 100644 --- a/src/secp256k1/src/modinv32_impl.h +++ b/src/secp256k1/src/modinv32_impl.h @@ -565,13 +565,12 @@ static void secp256k1_modinv32(secp256k1_modinv32_signed30 *x, const secp256k1_m /* g == 0 */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, 9, &SECP256K1_SIGNED30_ONE, 0) == 0); - /* |f| == 1, or (x == 0 and d == 0 and |f|=modulus) */ + /* |f| == 1, or (x == 0 and d == 0 and f == modulus) */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, 9, &SECP256K1_SIGNED30_ONE, -1) == 0 || secp256k1_modinv32_mul_cmp_30(&f, 9, &SECP256K1_SIGNED30_ONE, 1) == 0 || (secp256k1_modinv32_mul_cmp_30(x, 9, &SECP256K1_SIGNED30_ONE, 0) == 0 && secp256k1_modinv32_mul_cmp_30(&d, 9, &SECP256K1_SIGNED30_ONE, 0) == 0 && - (secp256k1_modinv32_mul_cmp_30(&f, 9, &modinfo->modulus, 1) == 0 || - secp256k1_modinv32_mul_cmp_30(&f, 9, &modinfo->modulus, -1) == 0))); + secp256k1_modinv32_mul_cmp_30(&f, 9, &modinfo->modulus, 1) == 0)); /* Optionally negate d, normalize to [0,modulus), and return it. */ secp256k1_modinv32_normalize_30(&d, f.v[8], modinfo); @@ -643,13 +642,12 @@ static void secp256k1_modinv32_var(secp256k1_modinv32_signed30 *x, const secp256 /* g == 0 */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, len, &SECP256K1_SIGNED30_ONE, 0) == 0); - /* |f| == 1, or (x == 0 and d == 0 and |f|=modulus) */ + /* |f| == 1, or (x == 0 and d == 0 and f == modulus) */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, len, &SECP256K1_SIGNED30_ONE, -1) == 0 || secp256k1_modinv32_mul_cmp_30(&f, len, &SECP256K1_SIGNED30_ONE, 1) == 0 || (secp256k1_modinv32_mul_cmp_30(x, 9, &SECP256K1_SIGNED30_ONE, 0) == 0 && secp256k1_modinv32_mul_cmp_30(&d, 9, &SECP256K1_SIGNED30_ONE, 0) == 0 && - (secp256k1_modinv32_mul_cmp_30(&f, len, &modinfo->modulus, 1) == 0 || - secp256k1_modinv32_mul_cmp_30(&f, len, &modinfo->modulus, -1) == 0))); + secp256k1_modinv32_mul_cmp_30(&f, len, &modinfo->modulus, 1) == 0)); /* Optionally negate d, normalize to [0,modulus), and return it. */ secp256k1_modinv32_normalize_30(&d, f.v[len - 1], modinfo); diff --git a/src/secp256k1/src/modinv64_impl.h b/src/secp256k1/src/modinv64_impl.h index 0dc1e80696..548787bedf 100644 --- a/src/secp256k1/src/modinv64_impl.h +++ b/src/secp256k1/src/modinv64_impl.h @@ -621,13 +621,12 @@ static void secp256k1_modinv64(secp256k1_modinv64_signed62 *x, const secp256k1_m /* g == 0 */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, 5, &SECP256K1_SIGNED62_ONE, 0) == 0); - /* |f| == 1, or (x == 0 and d == 0 and |f|=modulus) */ + /* |f| == 1, or (x == 0 and d == 0 and f == modulus) */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, 5, &SECP256K1_SIGNED62_ONE, -1) == 0 || secp256k1_modinv64_mul_cmp_62(&f, 5, &SECP256K1_SIGNED62_ONE, 1) == 0 || (secp256k1_modinv64_mul_cmp_62(x, 5, &SECP256K1_SIGNED62_ONE, 0) == 0 && secp256k1_modinv64_mul_cmp_62(&d, 5, &SECP256K1_SIGNED62_ONE, 0) == 0 && - (secp256k1_modinv64_mul_cmp_62(&f, 5, &modinfo->modulus, 1) == 0 || - secp256k1_modinv64_mul_cmp_62(&f, 5, &modinfo->modulus, -1) == 0))); + secp256k1_modinv64_mul_cmp_62(&f, 5, &modinfo->modulus, 1) == 0)); /* Optionally negate d, normalize to [0,modulus), and return it. */ secp256k1_modinv64_normalize_62(&d, f.v[4], modinfo); @@ -698,13 +697,12 @@ static void secp256k1_modinv64_var(secp256k1_modinv64_signed62 *x, const secp256 /* g == 0 */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, len, &SECP256K1_SIGNED62_ONE, 0) == 0); - /* |f| == 1, or (x == 0 and d == 0 and |f|=modulus) */ + /* |f| == 1, or (x == 0 and d == 0 and f == modulus) */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, len, &SECP256K1_SIGNED62_ONE, -1) == 0 || secp256k1_modinv64_mul_cmp_62(&f, len, &SECP256K1_SIGNED62_ONE, 1) == 0 || (secp256k1_modinv64_mul_cmp_62(x, 5, &SECP256K1_SIGNED62_ONE, 0) == 0 && secp256k1_modinv64_mul_cmp_62(&d, 5, &SECP256K1_SIGNED62_ONE, 0) == 0 && - (secp256k1_modinv64_mul_cmp_62(&f, len, &modinfo->modulus, 1) == 0 || - secp256k1_modinv64_mul_cmp_62(&f, len, &modinfo->modulus, -1) == 0))); + secp256k1_modinv64_mul_cmp_62(&f, len, &modinfo->modulus, 1) == 0)); /* Optionally negate d, normalize to [0,modulus), and return it. */ secp256k1_modinv64_normalize_62(&d, f.v[len - 1], modinfo); diff --git a/src/secp256k1/src/modules/ecdh/main_impl.h b/src/secp256k1/src/modules/ecdh/main_impl.h index 82b082a9f0..842b5359e3 100644 --- a/src/secp256k1/src/modules/ecdh/main_impl.h +++ b/src/secp256k1/src/modules/ecdh/main_impl.h @@ -19,6 +19,7 @@ static int ecdh_hash_function_sha256(unsigned char *output, const unsigned char secp256k1_sha256_write(&sha, &version, 1); secp256k1_sha256_write(&sha, x32, 32); secp256k1_sha256_finalize(&sha, output); + secp256k1_sha256_clear(&sha); return 1; } @@ -61,9 +62,11 @@ int secp256k1_ecdh(const secp256k1_context* ctx, unsigned char *output, const se ret = hashfp(output, x, y, data); - memset(x, 0, 32); - memset(y, 0, 32); + secp256k1_memclear(x, sizeof(x)); + secp256k1_memclear(y, sizeof(y)); secp256k1_scalar_clear(&s); + secp256k1_ge_clear(&pt); + secp256k1_gej_clear(&res); return !!ret & !overflow; } diff --git a/src/secp256k1/src/modules/ellswift/main_impl.h b/src/secp256k1/src/modules/ellswift/main_impl.h index b54ec08a22..745a969139 100644 --- a/src/secp256k1/src/modules/ellswift/main_impl.h +++ b/src/secp256k1/src/modules/ellswift/main_impl.h @@ -510,6 +510,7 @@ static int ellswift_xdh_hash_function_prefix(unsigned char *output, const unsign secp256k1_sha256_write(&sha, ell_b64, 64); secp256k1_sha256_write(&sha, x32, 32); secp256k1_sha256_finalize(&sha, output); + secp256k1_sha256_clear(&sha); return 1; } @@ -539,6 +540,7 @@ static int ellswift_xdh_hash_function_bip324(unsigned char* output, const unsign secp256k1_sha256_write(&sha, ell_b64, 64); secp256k1_sha256_write(&sha, x32, 32); secp256k1_sha256_finalize(&sha, output); + secp256k1_sha256_clear(&sha); return 1; } @@ -580,7 +582,7 @@ int secp256k1_ellswift_xdh(const secp256k1_context *ctx, unsigned char *output, /* Invoke hasher */ ret = hashfp(output, sx, ell_a64, ell_b64, data); - memset(sx, 0, 32); + secp256k1_memclear(sx, sizeof(sx)); secp256k1_fe_clear(&px); secp256k1_scalar_clear(&s); diff --git a/src/secp256k1/src/modules/musig/Makefile.am.include b/src/secp256k1/src/modules/musig/Makefile.am.include new file mode 100644 index 0000000000..796443c93b --- /dev/null +++ b/src/secp256k1/src/modules/musig/Makefile.am.include @@ -0,0 +1,8 @@ +include_HEADERS += include/secp256k1_musig.h +noinst_HEADERS += src/modules/musig/main_impl.h +noinst_HEADERS += src/modules/musig/keyagg.h +noinst_HEADERS += src/modules/musig/keyagg_impl.h +noinst_HEADERS += src/modules/musig/session.h +noinst_HEADERS += src/modules/musig/session_impl.h +noinst_HEADERS += src/modules/musig/tests_impl.h +noinst_HEADERS += src/modules/musig/vectors.h diff --git a/src/secp256k1/src/modules/musig/keyagg.h b/src/secp256k1/src/modules/musig/keyagg.h new file mode 100644 index 0000000000..a0b37252f8 --- /dev/null +++ b/src/secp256k1/src/modules/musig/keyagg.h @@ -0,0 +1,32 @@ +/*********************************************************************** + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php.* + ***********************************************************************/ + +#ifndef SECP256K1_MODULE_MUSIG_KEYAGG_H +#define SECP256K1_MODULE_MUSIG_KEYAGG_H + +#include "../../../include/secp256k1.h" +#include "../../../include/secp256k1_musig.h" + +#include "../../group.h" +#include "../../scalar.h" + +typedef struct { + secp256k1_ge pk; + /* If there is no "second" public key, second_pk is set to the point at + * infinity */ + secp256k1_ge second_pk; + unsigned char pks_hash[32]; + /* tweak is identical to value tacc[v] in the specification. */ + secp256k1_scalar tweak; + /* parity_acc corresponds to (1 - gacc[v])/2 in the spec. So if gacc[v] is + * -1, parity_acc is 1. Otherwise, parity_acc is 0. */ + int parity_acc; +} secp256k1_keyagg_cache_internal; + +static int secp256k1_keyagg_cache_load(const secp256k1_context* ctx, secp256k1_keyagg_cache_internal *cache_i, const secp256k1_musig_keyagg_cache *cache); + +static void secp256k1_musig_keyaggcoef(secp256k1_scalar *r, const secp256k1_keyagg_cache_internal *cache_i, secp256k1_ge *pk); + +#endif diff --git a/src/secp256k1/src/modules/musig/keyagg_impl.h b/src/secp256k1/src/modules/musig/keyagg_impl.h new file mode 100644 index 0000000000..0db4fce859 --- /dev/null +++ b/src/secp256k1/src/modules/musig/keyagg_impl.h @@ -0,0 +1,291 @@ +/*********************************************************************** + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php.* + ***********************************************************************/ + +#ifndef SECP256K1_MODULE_MUSIG_KEYAGG_IMPL_H +#define SECP256K1_MODULE_MUSIG_KEYAGG_IMPL_H + +#include <string.h> + +#include "keyagg.h" +#include "../../eckey.h" +#include "../../ecmult.h" +#include "../../field.h" +#include "../../group.h" +#include "../../hash.h" +#include "../../util.h" + +static const unsigned char secp256k1_musig_keyagg_cache_magic[4] = { 0xf4, 0xad, 0xbb, 0xdf }; + +/* A keyagg cache consists of + * - 4 byte magic set during initialization to allow detecting an uninitialized + * object. + * - 64 byte aggregate (and potentially tweaked) public key + * - 64 byte "second" public key (set to the point at infinity if not present) + * - 32 byte hash of all public keys + * - 1 byte the parity of the internal key (if tweaked, otherwise 0) + * - 32 byte tweak + */ +/* Requires that cache_i->pk is not infinity. */ +static void secp256k1_keyagg_cache_save(secp256k1_musig_keyagg_cache *cache, const secp256k1_keyagg_cache_internal *cache_i) { + unsigned char *ptr = cache->data; + memcpy(ptr, secp256k1_musig_keyagg_cache_magic, 4); + ptr += 4; + secp256k1_ge_to_bytes(ptr, &cache_i->pk); + ptr += 64; + secp256k1_ge_to_bytes_ext(ptr, &cache_i->second_pk); + ptr += 64; + memcpy(ptr, cache_i->pks_hash, 32); + ptr += 32; + *ptr = cache_i->parity_acc; + ptr += 1; + secp256k1_scalar_get_b32(ptr, &cache_i->tweak); +} + +static int secp256k1_keyagg_cache_load(const secp256k1_context* ctx, secp256k1_keyagg_cache_internal *cache_i, const secp256k1_musig_keyagg_cache *cache) { + const unsigned char *ptr = cache->data; + ARG_CHECK(secp256k1_memcmp_var(ptr, secp256k1_musig_keyagg_cache_magic, 4) == 0); + ptr += 4; + secp256k1_ge_from_bytes(&cache_i->pk, ptr); + ptr += 64; + secp256k1_ge_from_bytes_ext(&cache_i->second_pk, ptr); + ptr += 64; + memcpy(cache_i->pks_hash, ptr, 32); + ptr += 32; + cache_i->parity_acc = *ptr & 1; + ptr += 1; + secp256k1_scalar_set_b32(&cache_i->tweak, ptr, NULL); + return 1; +} + +/* Initializes SHA256 with fixed midstate. This midstate was computed by applying + * SHA256 to SHA256("KeyAgg list")||SHA256("KeyAgg list"). */ +static void secp256k1_musig_keyagglist_sha256(secp256k1_sha256 *sha) { + secp256k1_sha256_initialize(sha); + + sha->s[0] = 0xb399d5e0ul; + sha->s[1] = 0xc8fff302ul; + sha->s[2] = 0x6badac71ul; + sha->s[3] = 0x07c5b7f1ul; + sha->s[4] = 0x9701e2eful; + sha->s[5] = 0x2a72ecf8ul; + sha->s[6] = 0x201a4c7bul; + sha->s[7] = 0xab148a38ul; + sha->bytes = 64; +} + +/* Computes pks_hash = tagged_hash(pk[0], ..., pk[np-1]) */ +static int secp256k1_musig_compute_pks_hash(const secp256k1_context *ctx, unsigned char *pks_hash, const secp256k1_pubkey * const* pks, size_t np) { + secp256k1_sha256 sha; + size_t i; + + secp256k1_musig_keyagglist_sha256(&sha); + for (i = 0; i < np; i++) { + unsigned char ser[33]; + size_t ser_len = sizeof(ser); + if (!secp256k1_ec_pubkey_serialize(ctx, ser, &ser_len, pks[i], SECP256K1_EC_COMPRESSED)) { + return 0; + } + VERIFY_CHECK(ser_len == sizeof(ser)); + secp256k1_sha256_write(&sha, ser, sizeof(ser)); + } + secp256k1_sha256_finalize(&sha, pks_hash); + return 1; +} + +/* Initializes SHA256 with fixed midstate. This midstate was computed by applying + * SHA256 to SHA256("KeyAgg coefficient")||SHA256("KeyAgg coefficient"). */ +static void secp256k1_musig_keyaggcoef_sha256(secp256k1_sha256 *sha) { + secp256k1_sha256_initialize(sha); + + sha->s[0] = 0x6ef02c5aul; + sha->s[1] = 0x06a480deul; + sha->s[2] = 0x1f298665ul; + sha->s[3] = 0x1d1134f2ul; + sha->s[4] = 0x56a0b063ul; + sha->s[5] = 0x52da4147ul; + sha->s[6] = 0xf280d9d4ul; + sha->s[7] = 0x4484be15ul; + sha->bytes = 64; +} + +/* Compute KeyAgg coefficient which is constant 1 for the second pubkey and + * otherwise tagged_hash(pks_hash, pk) where pks_hash is the hash of public keys. + * second_pk is the point at infinity in case there is no second_pk. Assumes + * that pk is not the point at infinity and that the Y-coordinates of pk and + * second_pk are normalized. */ +static void secp256k1_musig_keyaggcoef_internal(secp256k1_scalar *r, const unsigned char *pks_hash, secp256k1_ge *pk, const secp256k1_ge *second_pk) { + VERIFY_CHECK(!secp256k1_ge_is_infinity(pk)); + + if (!secp256k1_ge_is_infinity(second_pk) + && secp256k1_ge_eq_var(pk, second_pk)) { + secp256k1_scalar_set_int(r, 1); + } else { + secp256k1_sha256 sha; + unsigned char buf[33]; + size_t buflen = sizeof(buf); + int ret; + secp256k1_musig_keyaggcoef_sha256(&sha); + secp256k1_sha256_write(&sha, pks_hash, 32); + ret = secp256k1_eckey_pubkey_serialize(pk, buf, &buflen, 1); +#ifdef VERIFY + /* Serialization does not fail since the pk is not the point at infinity + * (according to this function's precondition). */ + VERIFY_CHECK(ret && buflen == sizeof(buf)); +#else + (void) ret; +#endif + secp256k1_sha256_write(&sha, buf, sizeof(buf)); + secp256k1_sha256_finalize(&sha, buf); + secp256k1_scalar_set_b32(r, buf, NULL); + } +} + +/* Assumes that pk is not the point at infinity and that the Y-coordinates of pk + * and cache_i->second_pk are normalized. */ +static void secp256k1_musig_keyaggcoef(secp256k1_scalar *r, const secp256k1_keyagg_cache_internal *cache_i, secp256k1_ge *pk) { + secp256k1_musig_keyaggcoef_internal(r, cache_i->pks_hash, pk, &cache_i->second_pk); +} + +typedef struct { + const secp256k1_context *ctx; + /* pks_hash is the hash of the public keys */ + unsigned char pks_hash[32]; + const secp256k1_pubkey * const* pks; + secp256k1_ge second_pk; +} secp256k1_musig_pubkey_agg_ecmult_data; + +/* Callback for batch EC multiplication to compute keyaggcoef_0*P0 + keyaggcoef_1*P1 + ... */ +static int secp256k1_musig_pubkey_agg_callback(secp256k1_scalar *sc, secp256k1_ge *pt, size_t idx, void *data) { + secp256k1_musig_pubkey_agg_ecmult_data *ctx = (secp256k1_musig_pubkey_agg_ecmult_data *) data; + int ret; + ret = secp256k1_pubkey_load(ctx->ctx, pt, ctx->pks[idx]); +#ifdef VERIFY + /* pubkey_load can't fail because the same pks have already been loaded in + * `musig_compute_pks_hash` (and we test this). */ + VERIFY_CHECK(ret); +#else + (void) ret; +#endif + secp256k1_musig_keyaggcoef_internal(sc, ctx->pks_hash, pt, &ctx->second_pk); + return 1; +} + +int secp256k1_musig_pubkey_agg(const secp256k1_context* ctx, secp256k1_xonly_pubkey *agg_pk, secp256k1_musig_keyagg_cache *keyagg_cache, const secp256k1_pubkey * const* pubkeys, size_t n_pubkeys) { + secp256k1_musig_pubkey_agg_ecmult_data ecmult_data; + secp256k1_gej pkj; + secp256k1_ge pkp; + size_t i; + + VERIFY_CHECK(ctx != NULL); + if (agg_pk != NULL) { + memset(agg_pk, 0, sizeof(*agg_pk)); + } + ARG_CHECK(pubkeys != NULL); + ARG_CHECK(n_pubkeys > 0); + + ecmult_data.ctx = ctx; + ecmult_data.pks = pubkeys; + + secp256k1_ge_set_infinity(&ecmult_data.second_pk); + for (i = 1; i < n_pubkeys; i++) { + if (secp256k1_memcmp_var(pubkeys[0], pubkeys[i], sizeof(*pubkeys[0])) != 0) { + secp256k1_ge pk; + if (!secp256k1_pubkey_load(ctx, &pk, pubkeys[i])) { + return 0; + } + ecmult_data.second_pk = pk; + break; + } + } + + if (!secp256k1_musig_compute_pks_hash(ctx, ecmult_data.pks_hash, pubkeys, n_pubkeys)) { + return 0; + } + /* TODO: actually use optimized ecmult_multi algorithms by providing a + * scratch space */ + if (!secp256k1_ecmult_multi_var(&ctx->error_callback, NULL, &pkj, NULL, secp256k1_musig_pubkey_agg_callback, (void *) &ecmult_data, n_pubkeys)) { + /* In order to reach this line with the current implementation of + * ecmult_multi_var one would need to provide a callback that can + * fail. */ + return 0; + } + secp256k1_ge_set_gej(&pkp, &pkj); + secp256k1_fe_normalize_var(&pkp.y); + /* The resulting public key is infinity with negligible probability */ + VERIFY_CHECK(!secp256k1_ge_is_infinity(&pkp)); + if (keyagg_cache != NULL) { + secp256k1_keyagg_cache_internal cache_i = { 0 }; + cache_i.pk = pkp; + cache_i.second_pk = ecmult_data.second_pk; + memcpy(cache_i.pks_hash, ecmult_data.pks_hash, sizeof(cache_i.pks_hash)); + secp256k1_keyagg_cache_save(keyagg_cache, &cache_i); + } + + if (agg_pk != NULL) { + secp256k1_extrakeys_ge_even_y(&pkp); + secp256k1_xonly_pubkey_save(agg_pk, &pkp); + } + return 1; +} + +int secp256k1_musig_pubkey_get(const secp256k1_context* ctx, secp256k1_pubkey *agg_pk, const secp256k1_musig_keyagg_cache *keyagg_cache) { + secp256k1_keyagg_cache_internal cache_i; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(agg_pk != NULL); + memset(agg_pk, 0, sizeof(*agg_pk)); + ARG_CHECK(keyagg_cache != NULL); + + if (!secp256k1_keyagg_cache_load(ctx, &cache_i, keyagg_cache)) { + return 0; + } + secp256k1_pubkey_save(agg_pk, &cache_i.pk); + return 1; +} + +static int secp256k1_musig_pubkey_tweak_add_internal(const secp256k1_context* ctx, secp256k1_pubkey *output_pubkey, secp256k1_musig_keyagg_cache *keyagg_cache, const unsigned char *tweak32, int xonly) { + secp256k1_keyagg_cache_internal cache_i; + int overflow = 0; + secp256k1_scalar tweak; + + VERIFY_CHECK(ctx != NULL); + if (output_pubkey != NULL) { + memset(output_pubkey, 0, sizeof(*output_pubkey)); + } + ARG_CHECK(keyagg_cache != NULL); + ARG_CHECK(tweak32 != NULL); + + if (!secp256k1_keyagg_cache_load(ctx, &cache_i, keyagg_cache)) { + return 0; + } + secp256k1_scalar_set_b32(&tweak, tweak32, &overflow); + if (overflow) { + return 0; + } + if (xonly && secp256k1_extrakeys_ge_even_y(&cache_i.pk)) { + cache_i.parity_acc ^= 1; + secp256k1_scalar_negate(&cache_i.tweak, &cache_i.tweak); + } + secp256k1_scalar_add(&cache_i.tweak, &cache_i.tweak, &tweak); + if (!secp256k1_eckey_pubkey_tweak_add(&cache_i.pk, &tweak)) { + return 0; + } + /* eckey_pubkey_tweak_add fails if cache_i.pk is infinity */ + VERIFY_CHECK(!secp256k1_ge_is_infinity(&cache_i.pk)); + secp256k1_keyagg_cache_save(keyagg_cache, &cache_i); + if (output_pubkey != NULL) { + secp256k1_pubkey_save(output_pubkey, &cache_i.pk); + } + return 1; +} + +int secp256k1_musig_pubkey_ec_tweak_add(const secp256k1_context* ctx, secp256k1_pubkey *output_pubkey, secp256k1_musig_keyagg_cache *keyagg_cache, const unsigned char *tweak32) { + return secp256k1_musig_pubkey_tweak_add_internal(ctx, output_pubkey, keyagg_cache, tweak32, 0); +} + +int secp256k1_musig_pubkey_xonly_tweak_add(const secp256k1_context* ctx, secp256k1_pubkey *output_pubkey, secp256k1_musig_keyagg_cache *keyagg_cache, const unsigned char *tweak32) { + return secp256k1_musig_pubkey_tweak_add_internal(ctx, output_pubkey, keyagg_cache, tweak32, 1); +} + +#endif diff --git a/src/secp256k1/src/modules/musig/main_impl.h b/src/secp256k1/src/modules/musig/main_impl.h new file mode 100644 index 0000000000..a1311e4191 --- /dev/null +++ b/src/secp256k1/src/modules/musig/main_impl.h @@ -0,0 +1,12 @@ +/********************************************************************** + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef SECP256K1_MODULE_MUSIG_MAIN_H +#define SECP256K1_MODULE_MUSIG_MAIN_H + +#include "keyagg_impl.h" +#include "session_impl.h" + +#endif diff --git a/src/secp256k1/src/modules/musig/session.h b/src/secp256k1/src/modules/musig/session.h new file mode 100644 index 0000000000..d6d76bc6c1 --- /dev/null +++ b/src/secp256k1/src/modules/musig/session.h @@ -0,0 +1,24 @@ +/*********************************************************************** + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php.* + ***********************************************************************/ + +#ifndef SECP256K1_MODULE_MUSIG_SESSION_H +#define SECP256K1_MODULE_MUSIG_SESSION_H + +#include "../../../include/secp256k1.h" +#include "../../../include/secp256k1_musig.h" + +#include "../../scalar.h" + +typedef struct { + int fin_nonce_parity; + unsigned char fin_nonce[32]; + secp256k1_scalar noncecoef; + secp256k1_scalar challenge; + secp256k1_scalar s_part; +} secp256k1_musig_session_internal; + +static int secp256k1_musig_session_load(const secp256k1_context* ctx, secp256k1_musig_session_internal *session_i, const secp256k1_musig_session *session); + +#endif diff --git a/src/secp256k1/src/modules/musig/session_impl.h b/src/secp256k1/src/modules/musig/session_impl.h new file mode 100644 index 0000000000..dde3808582 --- /dev/null +++ b/src/secp256k1/src/modules/musig/session_impl.h @@ -0,0 +1,816 @@ +/*********************************************************************** + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php.* + ***********************************************************************/ + +#ifndef SECP256K1_MODULE_MUSIG_SESSION_IMPL_H +#define SECP256K1_MODULE_MUSIG_SESSION_IMPL_H + +#include <string.h> + +#include "../../../include/secp256k1.h" +#include "../../../include/secp256k1_extrakeys.h" +#include "../../../include/secp256k1_musig.h" + +#include "keyagg.h" +#include "session.h" +#include "../../eckey.h" +#include "../../hash.h" +#include "../../scalar.h" +#include "../../util.h" + +/* Outputs 33 zero bytes if the given group element is the point at infinity and + * otherwise outputs the compressed serialization */ +static void secp256k1_musig_ge_serialize_ext(unsigned char *out33, secp256k1_ge* ge) { + if (secp256k1_ge_is_infinity(ge)) { + memset(out33, 0, 33); + } else { + int ret; + size_t size = 33; + ret = secp256k1_eckey_pubkey_serialize(ge, out33, &size, 1); +#ifdef VERIFY + /* Serialize must succeed because the point is not at infinity */ + VERIFY_CHECK(ret && size == 33); +#else + (void) ret; +#endif + } +} + +/* Outputs the point at infinity if the given byte array is all zero, otherwise + * attempts to parse compressed point serialization. */ +static int secp256k1_musig_ge_parse_ext(secp256k1_ge* ge, const unsigned char *in33) { + unsigned char zeros[33] = { 0 }; + + if (secp256k1_memcmp_var(in33, zeros, sizeof(zeros)) == 0) { + secp256k1_ge_set_infinity(ge); + return 1; + } + if (!secp256k1_eckey_pubkey_parse(ge, in33, 33)) { + return 0; + } + return secp256k1_ge_is_in_correct_subgroup(ge); +} + +static const unsigned char secp256k1_musig_secnonce_magic[4] = { 0x22, 0x0e, 0xdc, 0xf1 }; + +static void secp256k1_musig_secnonce_save(secp256k1_musig_secnonce *secnonce, const secp256k1_scalar *k, const secp256k1_ge *pk) { + memcpy(&secnonce->data[0], secp256k1_musig_secnonce_magic, 4); + secp256k1_scalar_get_b32(&secnonce->data[4], &k[0]); + secp256k1_scalar_get_b32(&secnonce->data[36], &k[1]); + secp256k1_ge_to_bytes(&secnonce->data[68], pk); +} + +static int secp256k1_musig_secnonce_load(const secp256k1_context* ctx, secp256k1_scalar *k, secp256k1_ge *pk, const secp256k1_musig_secnonce *secnonce) { + int is_zero; + ARG_CHECK(secp256k1_memcmp_var(&secnonce->data[0], secp256k1_musig_secnonce_magic, 4) == 0); + /* We make very sure that the nonce isn't invalidated by checking the values + * in addition to the magic. */ + is_zero = secp256k1_is_zero_array(&secnonce->data[4], 2 * 32); + secp256k1_declassify(ctx, &is_zero, sizeof(is_zero)); + ARG_CHECK(!is_zero); + + secp256k1_scalar_set_b32(&k[0], &secnonce->data[4], NULL); + secp256k1_scalar_set_b32(&k[1], &secnonce->data[36], NULL); + secp256k1_ge_from_bytes(pk, &secnonce->data[68]); + return 1; +} + +/* If flag is true, invalidate the secnonce; otherwise leave it. Constant-time. */ +static void secp256k1_musig_secnonce_invalidate(const secp256k1_context* ctx, secp256k1_musig_secnonce *secnonce, int flag) { + secp256k1_memczero(secnonce->data, sizeof(secnonce->data), flag); + /* The flag argument is usually classified. So, the line above makes the + * magic and public key classified. However, we need both to be + * declassified. Note that we don't declassify the entire object, because if + * flag is 0, then k[0] and k[1] have not been zeroed. */ + secp256k1_declassify(ctx, secnonce->data, sizeof(secp256k1_musig_secnonce_magic)); + secp256k1_declassify(ctx, &secnonce->data[68], 64); +} + +static const unsigned char secp256k1_musig_pubnonce_magic[4] = { 0xf5, 0x7a, 0x3d, 0xa0 }; + +/* Saves two group elements into a pubnonce. Requires that none of the provided + * group elements is infinity. */ +static void secp256k1_musig_pubnonce_save(secp256k1_musig_pubnonce* nonce, const secp256k1_ge* ges) { + int i; + memcpy(&nonce->data[0], secp256k1_musig_pubnonce_magic, 4); + for (i = 0; i < 2; i++) { + secp256k1_ge_to_bytes(nonce->data + 4+64*i, &ges[i]); + } +} + +/* Loads two group elements from a pubnonce. Returns 1 unless the nonce wasn't + * properly initialized */ +static int secp256k1_musig_pubnonce_load(const secp256k1_context* ctx, secp256k1_ge* ges, const secp256k1_musig_pubnonce* nonce) { + int i; + + ARG_CHECK(secp256k1_memcmp_var(&nonce->data[0], secp256k1_musig_pubnonce_magic, 4) == 0); + for (i = 0; i < 2; i++) { + secp256k1_ge_from_bytes(&ges[i], nonce->data + 4 + 64*i); + } + return 1; +} + +static const unsigned char secp256k1_musig_aggnonce_magic[4] = { 0xa8, 0xb7, 0xe4, 0x67 }; + +static void secp256k1_musig_aggnonce_save(secp256k1_musig_aggnonce* nonce, const secp256k1_ge* ges) { + int i; + memcpy(&nonce->data[0], secp256k1_musig_aggnonce_magic, 4); + for (i = 0; i < 2; i++) { + secp256k1_ge_to_bytes_ext(&nonce->data[4 + 64*i], &ges[i]); + } +} + +static int secp256k1_musig_aggnonce_load(const secp256k1_context* ctx, secp256k1_ge* ges, const secp256k1_musig_aggnonce* nonce) { + int i; + + ARG_CHECK(secp256k1_memcmp_var(&nonce->data[0], secp256k1_musig_aggnonce_magic, 4) == 0); + for (i = 0; i < 2; i++) { + secp256k1_ge_from_bytes_ext(&ges[i], &nonce->data[4 + 64*i]); + } + return 1; +} + +static const unsigned char secp256k1_musig_session_cache_magic[4] = { 0x9d, 0xed, 0xe9, 0x17 }; + +/* A session consists of + * - 4 byte session cache magic + * - 1 byte the parity of the final nonce + * - 32 byte serialized x-only final nonce + * - 32 byte nonce coefficient b + * - 32 byte signature challenge hash e + * - 32 byte scalar s that is added to the partial signatures of the signers + */ +static void secp256k1_musig_session_save(secp256k1_musig_session *session, const secp256k1_musig_session_internal *session_i) { + unsigned char *ptr = session->data; + + memcpy(ptr, secp256k1_musig_session_cache_magic, 4); + ptr += 4; + *ptr = session_i->fin_nonce_parity; + ptr += 1; + memcpy(ptr, session_i->fin_nonce, 32); + ptr += 32; + secp256k1_scalar_get_b32(ptr, &session_i->noncecoef); + ptr += 32; + secp256k1_scalar_get_b32(ptr, &session_i->challenge); + ptr += 32; + secp256k1_scalar_get_b32(ptr, &session_i->s_part); +} + +static int secp256k1_musig_session_load(const secp256k1_context* ctx, secp256k1_musig_session_internal *session_i, const secp256k1_musig_session *session) { + const unsigned char *ptr = session->data; + + ARG_CHECK(secp256k1_memcmp_var(ptr, secp256k1_musig_session_cache_magic, 4) == 0); + ptr += 4; + session_i->fin_nonce_parity = *ptr; + ptr += 1; + memcpy(session_i->fin_nonce, ptr, 32); + ptr += 32; + secp256k1_scalar_set_b32(&session_i->noncecoef, ptr, NULL); + ptr += 32; + secp256k1_scalar_set_b32(&session_i->challenge, ptr, NULL); + ptr += 32; + secp256k1_scalar_set_b32(&session_i->s_part, ptr, NULL); + return 1; +} + +static const unsigned char secp256k1_musig_partial_sig_magic[4] = { 0xeb, 0xfb, 0x1a, 0x32 }; + +static void secp256k1_musig_partial_sig_save(secp256k1_musig_partial_sig* sig, secp256k1_scalar *s) { + memcpy(&sig->data[0], secp256k1_musig_partial_sig_magic, 4); + secp256k1_scalar_get_b32(&sig->data[4], s); +} + +static int secp256k1_musig_partial_sig_load(const secp256k1_context* ctx, secp256k1_scalar *s, const secp256k1_musig_partial_sig* sig) { + int overflow; + + ARG_CHECK(secp256k1_memcmp_var(&sig->data[0], secp256k1_musig_partial_sig_magic, 4) == 0); + secp256k1_scalar_set_b32(s, &sig->data[4], &overflow); + /* Parsed signatures can not overflow */ + VERIFY_CHECK(!overflow); + return 1; +} + +int secp256k1_musig_pubnonce_parse(const secp256k1_context* ctx, secp256k1_musig_pubnonce* nonce, const unsigned char *in66) { + secp256k1_ge ges[2]; + int i; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(nonce != NULL); + ARG_CHECK(in66 != NULL); + + for (i = 0; i < 2; i++) { + if (!secp256k1_eckey_pubkey_parse(&ges[i], &in66[33*i], 33)) { + return 0; + } + if (!secp256k1_ge_is_in_correct_subgroup(&ges[i])) { + return 0; + } + } + secp256k1_musig_pubnonce_save(nonce, ges); + return 1; +} + +int secp256k1_musig_pubnonce_serialize(const secp256k1_context* ctx, unsigned char *out66, const secp256k1_musig_pubnonce* nonce) { + secp256k1_ge ges[2]; + int i; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(out66 != NULL); + memset(out66, 0, 66); + ARG_CHECK(nonce != NULL); + + if (!secp256k1_musig_pubnonce_load(ctx, ges, nonce)) { + return 0; + } + for (i = 0; i < 2; i++) { + int ret; + size_t size = 33; + ret = secp256k1_eckey_pubkey_serialize(&ges[i], &out66[33*i], &size, 1); +#ifdef VERIFY + /* serialize must succeed because the point was just loaded */ + VERIFY_CHECK(ret && size == 33); +#else + (void) ret; +#endif + } + return 1; +} + +int secp256k1_musig_aggnonce_parse(const secp256k1_context* ctx, secp256k1_musig_aggnonce* nonce, const unsigned char *in66) { + secp256k1_ge ges[2]; + int i; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(nonce != NULL); + ARG_CHECK(in66 != NULL); + + for (i = 0; i < 2; i++) { + if (!secp256k1_musig_ge_parse_ext(&ges[i], &in66[33*i])) { + return 0; + } + } + secp256k1_musig_aggnonce_save(nonce, ges); + return 1; +} + +int secp256k1_musig_aggnonce_serialize(const secp256k1_context* ctx, unsigned char *out66, const secp256k1_musig_aggnonce* nonce) { + secp256k1_ge ges[2]; + int i; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(out66 != NULL); + memset(out66, 0, 66); + ARG_CHECK(nonce != NULL); + + if (!secp256k1_musig_aggnonce_load(ctx, ges, nonce)) { + return 0; + } + for (i = 0; i < 2; i++) { + secp256k1_musig_ge_serialize_ext(&out66[33*i], &ges[i]); + } + return 1; +} + +int secp256k1_musig_partial_sig_parse(const secp256k1_context* ctx, secp256k1_musig_partial_sig* sig, const unsigned char *in32) { + secp256k1_scalar tmp; + int overflow; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(sig != NULL); + ARG_CHECK(in32 != NULL); + + /* Ensure that using the signature will fail if parsing fails (and the user + * doesn't check the return value). */ + memset(sig, 0, sizeof(*sig)); + + secp256k1_scalar_set_b32(&tmp, in32, &overflow); + if (overflow) { + return 0; + } + secp256k1_musig_partial_sig_save(sig, &tmp); + return 1; +} + +int secp256k1_musig_partial_sig_serialize(const secp256k1_context* ctx, unsigned char *out32, const secp256k1_musig_partial_sig* sig) { + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(out32 != NULL); + ARG_CHECK(sig != NULL); + ARG_CHECK(secp256k1_memcmp_var(&sig->data[0], secp256k1_musig_partial_sig_magic, 4) == 0); + + memcpy(out32, &sig->data[4], 32); + return 1; +} + +/* Write optional inputs into the hash */ +static void secp256k1_nonce_function_musig_helper(secp256k1_sha256 *sha, unsigned int prefix_size, const unsigned char *data, unsigned char len) { + unsigned char zero[7] = { 0 }; + /* The spec requires length prefixes to be between 1 and 8 bytes + * (inclusive) */ + VERIFY_CHECK(prefix_size >= 1 && prefix_size <= 8); + /* Since the length of all input data fits in a byte, we can always pad the + * length prefix with prefix_size - 1 zero bytes. */ + secp256k1_sha256_write(sha, zero, prefix_size - 1); + if (data != NULL) { + secp256k1_sha256_write(sha, &len, 1); + secp256k1_sha256_write(sha, data, len); + } else { + len = 0; + secp256k1_sha256_write(sha, &len, 1); + } +} + +/* Initializes SHA256 with fixed midstate. This midstate was computed by applying + * SHA256 to SHA256("MuSig/aux")||SHA256("MuSig/aux"). */ +static void secp256k1_nonce_function_musig_sha256_tagged_aux(secp256k1_sha256 *sha) { + secp256k1_sha256_initialize(sha); + sha->s[0] = 0xa19e884bul; + sha->s[1] = 0xf463fe7eul; + sha->s[2] = 0x2f18f9a2ul; + sha->s[3] = 0xbeb0f9fful; + sha->s[4] = 0x0f37e8b0ul; + sha->s[5] = 0x06ebd26ful; + sha->s[6] = 0xe3b243d2ul; + sha->s[7] = 0x522fb150ul; + sha->bytes = 64; +} + +/* Initializes SHA256 with fixed midstate. This midstate was computed by applying + * SHA256 to SHA256("MuSig/nonce")||SHA256("MuSig/nonce"). */ +static void secp256k1_nonce_function_musig_sha256_tagged(secp256k1_sha256 *sha) { + secp256k1_sha256_initialize(sha); + sha->s[0] = 0x07101b64ul; + sha->s[1] = 0x18003414ul; + sha->s[2] = 0x0391bc43ul; + sha->s[3] = 0x0e6258eeul; + sha->s[4] = 0x29d26b72ul; + sha->s[5] = 0x8343937eul; + sha->s[6] = 0xb7a0a4fbul; + sha->s[7] = 0xff568a30ul; + sha->bytes = 64; +} + +static void secp256k1_nonce_function_musig(secp256k1_scalar *k, const unsigned char *session_secrand, const unsigned char *msg32, const unsigned char *seckey32, const unsigned char *pk33, const unsigned char *agg_pk32, const unsigned char *extra_input32) { + secp256k1_sha256 sha; + unsigned char rand[32]; + unsigned char i; + unsigned char msg_present; + + if (seckey32 != NULL) { + secp256k1_nonce_function_musig_sha256_tagged_aux(&sha); + secp256k1_sha256_write(&sha, session_secrand, 32); + secp256k1_sha256_finalize(&sha, rand); + for (i = 0; i < 32; i++) { + rand[i] ^= seckey32[i]; + } + } else { + memcpy(rand, session_secrand, sizeof(rand)); + } + + secp256k1_nonce_function_musig_sha256_tagged(&sha); + secp256k1_sha256_write(&sha, rand, sizeof(rand)); + secp256k1_nonce_function_musig_helper(&sha, 1, pk33, 33); + secp256k1_nonce_function_musig_helper(&sha, 1, agg_pk32, 32); + msg_present = msg32 != NULL; + secp256k1_sha256_write(&sha, &msg_present, 1); + if (msg_present) { + secp256k1_nonce_function_musig_helper(&sha, 8, msg32, 32); + } + secp256k1_nonce_function_musig_helper(&sha, 4, extra_input32, 32); + + for (i = 0; i < 2; i++) { + unsigned char buf[32]; + secp256k1_sha256 sha_tmp = sha; + secp256k1_sha256_write(&sha_tmp, &i, 1); + secp256k1_sha256_finalize(&sha_tmp, buf); + secp256k1_scalar_set_b32(&k[i], buf, NULL); + + /* Attempt to erase secret data */ + secp256k1_memclear(buf, sizeof(buf)); + secp256k1_sha256_clear(&sha_tmp); + } + secp256k1_memclear(rand, sizeof(rand)); + secp256k1_sha256_clear(&sha); +} + +static int secp256k1_musig_nonce_gen_internal(const secp256k1_context* ctx, secp256k1_musig_secnonce *secnonce, secp256k1_musig_pubnonce *pubnonce, const unsigned char *input_nonce, const unsigned char *seckey, const secp256k1_pubkey *pubkey, const unsigned char *msg32, const secp256k1_musig_keyagg_cache *keyagg_cache, const unsigned char *extra_input32) { + secp256k1_scalar k[2]; + secp256k1_ge nonce_pts[2]; + int i; + unsigned char pk_ser[33]; + size_t pk_ser_len = sizeof(pk_ser); + unsigned char aggpk_ser[32]; + unsigned char *aggpk_ser_ptr = NULL; + secp256k1_ge pk; + int pk_serialize_success; + int ret = 1; + + ARG_CHECK(pubnonce != NULL); + memset(pubnonce, 0, sizeof(*pubnonce)); + ARG_CHECK(pubkey != NULL); + ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); + + /* Check that the seckey is valid to be able to sign for it later. */ + if (seckey != NULL) { + secp256k1_scalar sk; + ret &= secp256k1_scalar_set_b32_seckey(&sk, seckey); + secp256k1_scalar_clear(&sk); + } + + if (keyagg_cache != NULL) { + secp256k1_keyagg_cache_internal cache_i; + if (!secp256k1_keyagg_cache_load(ctx, &cache_i, keyagg_cache)) { + return 0; + } + /* The loaded point cache_i.pk can not be the point at infinity. */ + secp256k1_fe_get_b32(aggpk_ser, &cache_i.pk.x); + aggpk_ser_ptr = aggpk_ser; + } + if (!secp256k1_pubkey_load(ctx, &pk, pubkey)) { + return 0; + } + pk_serialize_success = secp256k1_eckey_pubkey_serialize(&pk, pk_ser, &pk_ser_len, 1); + +#ifdef VERIFY + /* A pubkey cannot be the point at infinity */ + VERIFY_CHECK(pk_serialize_success); + VERIFY_CHECK(pk_ser_len == sizeof(pk_ser)); +#else + (void) pk_serialize_success; +#endif + + secp256k1_nonce_function_musig(k, input_nonce, msg32, seckey, pk_ser, aggpk_ser_ptr, extra_input32); + VERIFY_CHECK(!secp256k1_scalar_is_zero(&k[0])); + VERIFY_CHECK(!secp256k1_scalar_is_zero(&k[1])); + secp256k1_musig_secnonce_save(secnonce, k, &pk); + secp256k1_musig_secnonce_invalidate(ctx, secnonce, !ret); + + for (i = 0; i < 2; i++) { + secp256k1_gej nonce_ptj; + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &nonce_ptj, &k[i]); + secp256k1_ge_set_gej(&nonce_pts[i], &nonce_ptj); + secp256k1_declassify(ctx, &nonce_pts[i], sizeof(nonce_pts[i])); + secp256k1_scalar_clear(&k[i]); + secp256k1_gej_clear(&nonce_ptj); + } + /* None of the nonce_pts will be infinity because k != 0 with overwhelming + * probability */ + secp256k1_musig_pubnonce_save(pubnonce, nonce_pts); + return ret; +} + +int secp256k1_musig_nonce_gen(const secp256k1_context* ctx, secp256k1_musig_secnonce *secnonce, secp256k1_musig_pubnonce *pubnonce, unsigned char *session_secrand32, const unsigned char *seckey, const secp256k1_pubkey *pubkey, const unsigned char *msg32, const secp256k1_musig_keyagg_cache *keyagg_cache, const unsigned char *extra_input32) { + int ret = 1; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(secnonce != NULL); + memset(secnonce, 0, sizeof(*secnonce)); + ARG_CHECK(session_secrand32 != NULL); + + /* Check in constant time that the session_secrand32 is not 0 as a + * defense-in-depth measure that may protect against a faulty RNG. */ + ret &= !secp256k1_is_zero_array(session_secrand32, 32); + + /* We can declassify because branching on ret is only relevant when this + * function called with an invalid session_secrand32 argument */ + secp256k1_declassify(ctx, &ret, sizeof(ret)); + if (ret == 0) { + secp256k1_musig_secnonce_invalidate(ctx, secnonce, 1); + return 0; + } + + ret &= secp256k1_musig_nonce_gen_internal(ctx, secnonce, pubnonce, session_secrand32, seckey, pubkey, msg32, keyagg_cache, extra_input32); + + /* Set the session_secrand32 buffer to zero to prevent the caller from using + * nonce_gen multiple times with the same buffer. */ + secp256k1_memczero(session_secrand32, 32, ret); + return ret; +} + +int secp256k1_musig_nonce_gen_counter(const secp256k1_context* ctx, secp256k1_musig_secnonce *secnonce, secp256k1_musig_pubnonce *pubnonce, uint64_t nonrepeating_cnt, const secp256k1_keypair *keypair, const unsigned char *msg32, const secp256k1_musig_keyagg_cache *keyagg_cache, const unsigned char *extra_input32) { + unsigned char buf[32] = { 0 }; + unsigned char seckey[32]; + secp256k1_pubkey pubkey; + int ret; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(secnonce != NULL); + memset(secnonce, 0, sizeof(*secnonce)); + ARG_CHECK(keypair != NULL); + + secp256k1_write_be64(buf, nonrepeating_cnt); + /* keypair_sec and keypair_pub do not fail if the arguments are not NULL */ + ret = secp256k1_keypair_sec(ctx, seckey, keypair); + VERIFY_CHECK(ret); + ret = secp256k1_keypair_pub(ctx, &pubkey, keypair); + VERIFY_CHECK(ret); +#ifndef VERIFY + (void) ret; +#endif + + if (!secp256k1_musig_nonce_gen_internal(ctx, secnonce, pubnonce, buf, seckey, &pubkey, msg32, keyagg_cache, extra_input32)) { + return 0; + } + secp256k1_memclear(seckey, sizeof(seckey)); + return 1; +} + +static int secp256k1_musig_sum_pubnonces(const secp256k1_context* ctx, secp256k1_gej *summed_pubnonces, const secp256k1_musig_pubnonce * const* pubnonces, size_t n_pubnonces) { + size_t i; + int j; + + secp256k1_gej_set_infinity(&summed_pubnonces[0]); + secp256k1_gej_set_infinity(&summed_pubnonces[1]); + + for (i = 0; i < n_pubnonces; i++) { + secp256k1_ge nonce_pts[2]; + if (!secp256k1_musig_pubnonce_load(ctx, nonce_pts, pubnonces[i])) { + return 0; + } + for (j = 0; j < 2; j++) { + secp256k1_gej_add_ge_var(&summed_pubnonces[j], &summed_pubnonces[j], &nonce_pts[j], NULL); + } + } + return 1; +} + +int secp256k1_musig_nonce_agg(const secp256k1_context* ctx, secp256k1_musig_aggnonce *aggnonce, const secp256k1_musig_pubnonce * const* pubnonces, size_t n_pubnonces) { + secp256k1_gej aggnonce_ptsj[2]; + secp256k1_ge aggnonce_pts[2]; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(aggnonce != NULL); + ARG_CHECK(pubnonces != NULL); + ARG_CHECK(n_pubnonces > 0); + + if (!secp256k1_musig_sum_pubnonces(ctx, aggnonce_ptsj, pubnonces, n_pubnonces)) { + return 0; + } + secp256k1_ge_set_all_gej_var(aggnonce_pts, aggnonce_ptsj, 2); + secp256k1_musig_aggnonce_save(aggnonce, aggnonce_pts); + return 1; +} + +/* Initializes SHA256 with fixed midstate. This midstate was computed by applying + * SHA256 to SHA256("MuSig/noncecoef")||SHA256("MuSig/noncecoef"). */ +static void secp256k1_musig_compute_noncehash_sha256_tagged(secp256k1_sha256 *sha) { + secp256k1_sha256_initialize(sha); + sha->s[0] = 0x2c7d5a45ul; + sha->s[1] = 0x06bf7e53ul; + sha->s[2] = 0x89be68a6ul; + sha->s[3] = 0x971254c0ul; + sha->s[4] = 0x60ac12d2ul; + sha->s[5] = 0x72846dcdul; + sha->s[6] = 0x6c81212ful; + sha->s[7] = 0xde7a2500ul; + sha->bytes = 64; +} + +/* tagged_hash(aggnonce[0], aggnonce[1], agg_pk, msg) */ +static void secp256k1_musig_compute_noncehash(unsigned char *noncehash, secp256k1_ge *aggnonce, const unsigned char *agg_pk32, const unsigned char *msg) { + unsigned char buf[33]; + secp256k1_sha256 sha; + int i; + + secp256k1_musig_compute_noncehash_sha256_tagged(&sha); + for (i = 0; i < 2; i++) { + secp256k1_musig_ge_serialize_ext(buf, &aggnonce[i]); + secp256k1_sha256_write(&sha, buf, sizeof(buf)); + } + secp256k1_sha256_write(&sha, agg_pk32, 32); + secp256k1_sha256_write(&sha, msg, 32); + secp256k1_sha256_finalize(&sha, noncehash); +} + +/* out_nonce = nonce_pts[0] + b*nonce_pts[1] */ +static void secp256k1_effective_nonce(secp256k1_gej *out_nonce, const secp256k1_ge *nonce_pts, const secp256k1_scalar *b) { + secp256k1_gej tmp; + + secp256k1_gej_set_ge(&tmp, &nonce_pts[1]); + secp256k1_ecmult(out_nonce, &tmp, b, NULL); + secp256k1_gej_add_ge_var(out_nonce, out_nonce, &nonce_pts[0], NULL); +} + +static void secp256k1_musig_nonce_process_internal(int *fin_nonce_parity, unsigned char *fin_nonce, secp256k1_scalar *b, secp256k1_ge *aggnonce_pts, const unsigned char *agg_pk32, const unsigned char *msg) { + unsigned char noncehash[32]; + secp256k1_ge fin_nonce_pt; + secp256k1_gej fin_nonce_ptj; + + secp256k1_musig_compute_noncehash(noncehash, aggnonce_pts, agg_pk32, msg); + secp256k1_scalar_set_b32(b, noncehash, NULL); + /* fin_nonce = aggnonce_pts[0] + b*aggnonce_pts[1] */ + secp256k1_effective_nonce(&fin_nonce_ptj, aggnonce_pts, b); + secp256k1_ge_set_gej(&fin_nonce_pt, &fin_nonce_ptj); + if (secp256k1_ge_is_infinity(&fin_nonce_pt)) { + fin_nonce_pt = secp256k1_ge_const_g; + } + /* fin_nonce_pt is not the point at infinity */ + secp256k1_fe_normalize_var(&fin_nonce_pt.x); + secp256k1_fe_get_b32(fin_nonce, &fin_nonce_pt.x); + secp256k1_fe_normalize_var(&fin_nonce_pt.y); + *fin_nonce_parity = secp256k1_fe_is_odd(&fin_nonce_pt.y); +} + +int secp256k1_musig_nonce_process(const secp256k1_context* ctx, secp256k1_musig_session *session, const secp256k1_musig_aggnonce *aggnonce, const unsigned char *msg32, const secp256k1_musig_keyagg_cache *keyagg_cache) { + secp256k1_keyagg_cache_internal cache_i; + secp256k1_ge aggnonce_pts[2]; + unsigned char fin_nonce[32]; + secp256k1_musig_session_internal session_i; + unsigned char agg_pk32[32]; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(session != NULL); + ARG_CHECK(aggnonce != NULL); + ARG_CHECK(msg32 != NULL); + ARG_CHECK(keyagg_cache != NULL); + + if (!secp256k1_keyagg_cache_load(ctx, &cache_i, keyagg_cache)) { + return 0; + } + secp256k1_fe_get_b32(agg_pk32, &cache_i.pk.x); + + if (!secp256k1_musig_aggnonce_load(ctx, aggnonce_pts, aggnonce)) { + return 0; + } + + secp256k1_musig_nonce_process_internal(&session_i.fin_nonce_parity, fin_nonce, &session_i.noncecoef, aggnonce_pts, agg_pk32, msg32); + secp256k1_schnorrsig_challenge(&session_i.challenge, fin_nonce, msg32, 32, agg_pk32); + + /* If there is a tweak then set `challenge` times `tweak` to the `s`-part.*/ + secp256k1_scalar_set_int(&session_i.s_part, 0); + if (!secp256k1_scalar_is_zero(&cache_i.tweak)) { + secp256k1_scalar e_tmp; + secp256k1_scalar_mul(&e_tmp, &session_i.challenge, &cache_i.tweak); + if (secp256k1_fe_is_odd(&cache_i.pk.y)) { + secp256k1_scalar_negate(&e_tmp, &e_tmp); + } + session_i.s_part = e_tmp; + } + memcpy(session_i.fin_nonce, fin_nonce, sizeof(session_i.fin_nonce)); + secp256k1_musig_session_save(session, &session_i); + return 1; +} + +static void secp256k1_musig_partial_sign_clear(secp256k1_scalar *sk, secp256k1_scalar *k) { + secp256k1_scalar_clear(sk); + secp256k1_scalar_clear(&k[0]); + secp256k1_scalar_clear(&k[1]); +} + +int secp256k1_musig_partial_sign(const secp256k1_context* ctx, secp256k1_musig_partial_sig *partial_sig, secp256k1_musig_secnonce *secnonce, const secp256k1_keypair *keypair, const secp256k1_musig_keyagg_cache *keyagg_cache, const secp256k1_musig_session *session) { + secp256k1_scalar sk; + secp256k1_ge pk, keypair_pk; + secp256k1_scalar k[2]; + secp256k1_scalar mu, s; + secp256k1_keyagg_cache_internal cache_i; + secp256k1_musig_session_internal session_i; + int ret; + + VERIFY_CHECK(ctx != NULL); + + ARG_CHECK(secnonce != NULL); + /* Fails if the magic doesn't match */ + ret = secp256k1_musig_secnonce_load(ctx, k, &pk, secnonce); + /* Set nonce to zero to avoid nonce reuse. This will cause subsequent calls + * of this function to fail */ + memset(secnonce, 0, sizeof(*secnonce)); + if (!ret) { + secp256k1_musig_partial_sign_clear(&sk, k); + return 0; + } + + ARG_CHECK(partial_sig != NULL); + ARG_CHECK(keypair != NULL); + ARG_CHECK(keyagg_cache != NULL); + ARG_CHECK(session != NULL); + + if (!secp256k1_keypair_load(ctx, &sk, &keypair_pk, keypair)) { + secp256k1_musig_partial_sign_clear(&sk, k); + return 0; + } + ARG_CHECK(secp256k1_fe_equal(&pk.x, &keypair_pk.x) + && secp256k1_fe_equal(&pk.y, &keypair_pk.y)); + if (!secp256k1_keyagg_cache_load(ctx, &cache_i, keyagg_cache)) { + secp256k1_musig_partial_sign_clear(&sk, k); + return 0; + } + + /* Negate sk if secp256k1_fe_is_odd(&cache_i.pk.y)) XOR cache_i.parity_acc. + * This corresponds to the line "Let d = g⋅gacc⋅d' mod n" in the + * specification. */ + if ((secp256k1_fe_is_odd(&cache_i.pk.y) + != cache_i.parity_acc)) { + secp256k1_scalar_negate(&sk, &sk); + } + + /* Multiply KeyAgg coefficient */ + secp256k1_musig_keyaggcoef(&mu, &cache_i, &pk); + secp256k1_scalar_mul(&sk, &sk, &mu); + + if (!secp256k1_musig_session_load(ctx, &session_i, session)) { + secp256k1_musig_partial_sign_clear(&sk, k); + return 0; + } + + if (session_i.fin_nonce_parity) { + secp256k1_scalar_negate(&k[0], &k[0]); + secp256k1_scalar_negate(&k[1], &k[1]); + } + + /* Sign */ + secp256k1_scalar_mul(&s, &session_i.challenge, &sk); + secp256k1_scalar_mul(&k[1], &session_i.noncecoef, &k[1]); + secp256k1_scalar_add(&k[0], &k[0], &k[1]); + secp256k1_scalar_add(&s, &s, &k[0]); + secp256k1_musig_partial_sig_save(partial_sig, &s); + secp256k1_musig_partial_sign_clear(&sk, k); + return 1; +} + +int secp256k1_musig_partial_sig_verify(const secp256k1_context* ctx, const secp256k1_musig_partial_sig *partial_sig, const secp256k1_musig_pubnonce *pubnonce, const secp256k1_pubkey *pubkey, const secp256k1_musig_keyagg_cache *keyagg_cache, const secp256k1_musig_session *session) { + secp256k1_keyagg_cache_internal cache_i; + secp256k1_musig_session_internal session_i; + secp256k1_scalar mu, e, s; + secp256k1_gej pkj; + secp256k1_ge nonce_pts[2]; + secp256k1_gej rj; + secp256k1_gej tmp; + secp256k1_ge pkp; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(partial_sig != NULL); + ARG_CHECK(pubnonce != NULL); + ARG_CHECK(pubkey != NULL); + ARG_CHECK(keyagg_cache != NULL); + ARG_CHECK(session != NULL); + + if (!secp256k1_musig_session_load(ctx, &session_i, session)) { + return 0; + } + + if (!secp256k1_musig_pubnonce_load(ctx, nonce_pts, pubnonce)) { + return 0; + } + /* Compute "effective" nonce rj = nonce_pts[0] + b*nonce_pts[1] */ + /* TODO: use multiexp to compute -s*G + e*mu*pubkey + nonce_pts[0] + b*nonce_pts[1] */ + secp256k1_effective_nonce(&rj, nonce_pts, &session_i.noncecoef); + + if (!secp256k1_pubkey_load(ctx, &pkp, pubkey)) { + return 0; + } + if (!secp256k1_keyagg_cache_load(ctx, &cache_i, keyagg_cache)) { + return 0; + } + /* Multiplying the challenge by the KeyAgg coefficient is equivalent + * to multiplying the signer's public key by the coefficient, except + * much easier to do. */ + secp256k1_musig_keyaggcoef(&mu, &cache_i, &pkp); + secp256k1_scalar_mul(&e, &session_i.challenge, &mu); + + /* Negate e if secp256k1_fe_is_odd(&cache_i.pk.y)) XOR cache_i.parity_acc. + * This corresponds to the line "Let g' = g⋅gacc mod n" and the multiplication "g'⋅e" + * in the specification. */ + if (secp256k1_fe_is_odd(&cache_i.pk.y) + != cache_i.parity_acc) { + secp256k1_scalar_negate(&e, &e); + } + + if (!secp256k1_musig_partial_sig_load(ctx, &s, partial_sig)) { + return 0; + } + /* Compute -s*G + e*pkj + rj (e already includes the keyagg coefficient mu) */ + secp256k1_scalar_negate(&s, &s); + secp256k1_gej_set_ge(&pkj, &pkp); + secp256k1_ecmult(&tmp, &pkj, &e, &s); + if (session_i.fin_nonce_parity) { + secp256k1_gej_neg(&rj, &rj); + } + secp256k1_gej_add_var(&tmp, &tmp, &rj, NULL); + + return secp256k1_gej_is_infinity(&tmp); +} + +int secp256k1_musig_partial_sig_agg(const secp256k1_context* ctx, unsigned char *sig64, const secp256k1_musig_session *session, const secp256k1_musig_partial_sig * const* partial_sigs, size_t n_sigs) { + size_t i; + secp256k1_musig_session_internal session_i; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(sig64 != NULL); + ARG_CHECK(session != NULL); + ARG_CHECK(partial_sigs != NULL); + ARG_CHECK(n_sigs > 0); + + if (!secp256k1_musig_session_load(ctx, &session_i, session)) { + return 0; + } + for (i = 0; i < n_sigs; i++) { + secp256k1_scalar term; + if (!secp256k1_musig_partial_sig_load(ctx, &term, partial_sigs[i])) { + return 0; + } + secp256k1_scalar_add(&session_i.s_part, &session_i.s_part, &term); + } + secp256k1_scalar_get_b32(&sig64[32], &session_i.s_part); + memcpy(&sig64[0], session_i.fin_nonce, 32); + return 1; +} + +#endif diff --git a/src/secp256k1/src/modules/musig/tests_impl.h b/src/secp256k1/src/modules/musig/tests_impl.h new file mode 100644 index 0000000000..ce6ae1784d --- /dev/null +++ b/src/secp256k1/src/modules/musig/tests_impl.h @@ -0,0 +1,1143 @@ +/*********************************************************************** + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php.* + ***********************************************************************/ + +#ifndef SECP256K1_MODULE_MUSIG_TESTS_IMPL_H +#define SECP256K1_MODULE_MUSIG_TESTS_IMPL_H + +#include <stdlib.h> +#include <string.h> + +#include "../../../include/secp256k1.h" +#include "../../../include/secp256k1_extrakeys.h" +#include "../../../include/secp256k1_musig.h" + +#include "session.h" +#include "keyagg.h" +#include "../../scalar.h" +#include "../../field.h" +#include "../../group.h" +#include "../../hash.h" +#include "../../util.h" + +#include "vectors.h" + +static int create_keypair_and_pk(secp256k1_keypair *keypair, secp256k1_pubkey *pk, const unsigned char *sk) { + int ret; + secp256k1_keypair keypair_tmp; + ret = secp256k1_keypair_create(CTX, &keypair_tmp, sk); + ret &= secp256k1_keypair_pub(CTX, pk, &keypair_tmp); + if (keypair != NULL) { + *keypair = keypair_tmp; + } + return ret; +} + +/* Just a simple (non-tweaked) 2-of-2 MuSig aggregate, sign, verify + * test. */ +static void musig_simple_test(void) { + unsigned char sk[2][32]; + secp256k1_keypair keypair[2]; + secp256k1_musig_pubnonce pubnonce[2]; + const secp256k1_musig_pubnonce *pubnonce_ptr[2]; + secp256k1_musig_aggnonce aggnonce; + unsigned char msg[32]; + secp256k1_xonly_pubkey agg_pk; + secp256k1_musig_keyagg_cache keyagg_cache; + unsigned char session_secrand[2][32]; + secp256k1_musig_secnonce secnonce[2]; + secp256k1_pubkey pk[2]; + const secp256k1_pubkey *pk_ptr[2]; + secp256k1_musig_partial_sig partial_sig[2]; + const secp256k1_musig_partial_sig *partial_sig_ptr[2]; + unsigned char final_sig[64]; + secp256k1_musig_session session; + int i; + + testrand256(msg); + for (i = 0; i < 2; i++) { + testrand256(sk[i]); + pk_ptr[i] = &pk[i]; + pubnonce_ptr[i] = &pubnonce[i]; + partial_sig_ptr[i] = &partial_sig[i]; + + CHECK(create_keypair_and_pk(&keypair[i], &pk[i], sk[i])); + if (i == 0) { + testrand256(session_secrand[i]); + CHECK(secp256k1_musig_nonce_gen(CTX, &secnonce[i], &pubnonce[i], session_secrand[i], sk[i], &pk[i], NULL, NULL, NULL) == 1); + } else { + uint64_t nonrepeating_cnt = 0; + CHECK(secp256k1_musig_nonce_gen_counter(CTX, &secnonce[i], &pubnonce[i], nonrepeating_cnt, &keypair[i], NULL, NULL, NULL) == 1); + } + } + + CHECK(secp256k1_musig_pubkey_agg(CTX, &agg_pk, &keyagg_cache, pk_ptr, 2) == 1); + CHECK(secp256k1_musig_nonce_agg(CTX, &aggnonce, pubnonce_ptr, 2) == 1); + CHECK(secp256k1_musig_nonce_process(CTX, &session, &aggnonce, msg, &keyagg_cache) == 1); + + for (i = 0; i < 2; i++) { + CHECK(secp256k1_musig_partial_sign(CTX, &partial_sig[i], &secnonce[i], &keypair[i], &keyagg_cache, &session) == 1); + CHECK(secp256k1_musig_partial_sig_verify(CTX, &partial_sig[i], &pubnonce[i], &pk[i], &keyagg_cache, &session) == 1); + } + + CHECK(secp256k1_musig_partial_sig_agg(CTX, final_sig, &session, partial_sig_ptr, 2) == 1); + CHECK(secp256k1_schnorrsig_verify(CTX, final_sig, msg, sizeof(msg), &agg_pk) == 1); +} + +/* Generate two pubnonces such that both group elements of their sum (calculated + * with secp256k1_musig_sum_pubnonces) are infinity. */ +static void pubnonce_summing_to_inf(secp256k1_musig_pubnonce *pubnonce) { + secp256k1_ge ge[2]; + int i; + secp256k1_gej summed_pubnonces[2]; + const secp256k1_musig_pubnonce *pubnonce_ptr[2]; + + testutil_random_ge_test(&ge[0]); + testutil_random_ge_test(&ge[1]); + + for (i = 0; i < 2; i++) { + secp256k1_musig_pubnonce_save(&pubnonce[i], ge); + pubnonce_ptr[i] = &pubnonce[i]; + secp256k1_ge_neg(&ge[0], &ge[0]); + secp256k1_ge_neg(&ge[1], &ge[1]); + } + + secp256k1_musig_sum_pubnonces(CTX, summed_pubnonces, pubnonce_ptr, 2); + CHECK(secp256k1_gej_is_infinity(&summed_pubnonces[0])); + CHECK(secp256k1_gej_is_infinity(&summed_pubnonces[1])); +} + +int memcmp_and_randomize(unsigned char *value, const unsigned char *expected, size_t len) { + int ret; + size_t i; + ret = secp256k1_memcmp_var(value, expected, len); + for (i = 0; i < len; i++) { + value[i] = testrand_bits(8); + } + return ret; +} + +static void musig_api_tests(void) { + secp256k1_musig_partial_sig partial_sig[2]; + const secp256k1_musig_partial_sig *partial_sig_ptr[2]; + secp256k1_musig_partial_sig invalid_partial_sig; + const secp256k1_musig_partial_sig *invalid_partial_sig_ptr[2]; + unsigned char pre_sig[64]; + unsigned char buf[32]; + unsigned char sk[2][32]; + secp256k1_keypair keypair[2]; + secp256k1_keypair invalid_keypair; + unsigned char max64[64]; + unsigned char zeros132[132] = { 0 }; + unsigned char session_secrand[2][32]; + unsigned char nonrepeating_cnt = 0; + secp256k1_musig_secnonce secnonce[2]; + secp256k1_musig_secnonce secnonce_tmp; + secp256k1_musig_secnonce invalid_secnonce; + secp256k1_musig_pubnonce pubnonce[2]; + const secp256k1_musig_pubnonce *pubnonce_ptr[2]; + unsigned char pubnonce_ser[66]; + secp256k1_musig_pubnonce inf_pubnonce[2]; + const secp256k1_musig_pubnonce *inf_pubnonce_ptr[2]; + secp256k1_musig_pubnonce invalid_pubnonce; + const secp256k1_musig_pubnonce *invalid_pubnonce_ptr[1]; + secp256k1_musig_aggnonce aggnonce; + unsigned char aggnonce_ser[66]; + unsigned char msg[32]; + secp256k1_xonly_pubkey agg_pk; + secp256k1_pubkey full_agg_pk; + secp256k1_musig_keyagg_cache keyagg_cache; + secp256k1_musig_keyagg_cache invalid_keyagg_cache; + secp256k1_musig_session session; + secp256k1_musig_session invalid_session; + secp256k1_pubkey pk[2]; + const secp256k1_pubkey *pk_ptr[2]; + secp256k1_pubkey invalid_pk; + const secp256k1_pubkey *invalid_pk_ptr2[2]; + const secp256k1_pubkey *invalid_pk_ptr3[3]; + unsigned char tweak[32]; + int i; + + /** setup **/ + memset(max64, 0xff, sizeof(max64)); + memset(&invalid_keypair, 0, sizeof(invalid_keypair)); + memset(&invalid_pk, 0, sizeof(invalid_pk)); + memset(&invalid_secnonce, 0, sizeof(invalid_secnonce)); + memset(&invalid_partial_sig, 0, sizeof(invalid_partial_sig)); + pubnonce_summing_to_inf(inf_pubnonce); + /* Simulate structs being uninitialized by setting it to 0s. We don't want + * to produce undefined behavior by actually providing uninitialized + * structs. */ + memset(&invalid_keyagg_cache, 0, sizeof(invalid_keyagg_cache)); + memset(&invalid_pk, 0, sizeof(invalid_pk)); + memset(&invalid_pubnonce, 0, sizeof(invalid_pubnonce)); + memset(&invalid_session, 0, sizeof(invalid_session)); + + testrand256(msg); + testrand256(tweak); + for (i = 0; i < 2; i++) { + pk_ptr[i] = &pk[i]; + invalid_pk_ptr2[i] = &invalid_pk; + invalid_pk_ptr3[i] = &pk[i]; + pubnonce_ptr[i] = &pubnonce[i]; + inf_pubnonce_ptr[i] = &inf_pubnonce[i]; + partial_sig_ptr[i] = &partial_sig[i]; + invalid_partial_sig_ptr[i] = &partial_sig[i]; + testrand256(session_secrand[i]); + testrand256(sk[i]); + CHECK(create_keypair_and_pk(&keypair[i], &pk[i], sk[i])); + } + invalid_pubnonce_ptr[0] = &invalid_pubnonce; + invalid_partial_sig_ptr[0] = &invalid_partial_sig; + /* invalid_pk_ptr3 has two valid, one invalid pk, which is important to test + * musig_pubkey_agg */ + invalid_pk_ptr3[2] = &invalid_pk; + + /** main test body **/ + + /** Key aggregation **/ + CHECK(secp256k1_musig_pubkey_agg(CTX, &agg_pk, &keyagg_cache, pk_ptr, 2) == 1); + CHECK(secp256k1_musig_pubkey_agg(CTX, NULL, &keyagg_cache, pk_ptr, 2) == 1); + CHECK(secp256k1_musig_pubkey_agg(CTX, &agg_pk, NULL, pk_ptr, 2) == 1); + CHECK_ILLEGAL(CTX, secp256k1_musig_pubkey_agg(CTX, &agg_pk, &keyagg_cache, NULL, 2)); + CHECK(memcmp_and_randomize(agg_pk.data, zeros132, sizeof(agg_pk.data)) == 0); + CHECK_ILLEGAL(CTX, secp256k1_musig_pubkey_agg(CTX, &agg_pk, &keyagg_cache, invalid_pk_ptr2, 2)); + CHECK(memcmp_and_randomize(agg_pk.data, zeros132, sizeof(agg_pk.data)) == 0); + CHECK_ILLEGAL(CTX, secp256k1_musig_pubkey_agg(CTX, &agg_pk, &keyagg_cache, invalid_pk_ptr3, 3)); + CHECK(memcmp_and_randomize(agg_pk.data, zeros132, sizeof(agg_pk.data)) == 0); + CHECK_ILLEGAL(CTX, secp256k1_musig_pubkey_agg(CTX, &agg_pk, &keyagg_cache, pk_ptr, 0)); + CHECK(memcmp_and_randomize(agg_pk.data, zeros132, sizeof(agg_pk.data)) == 0); + CHECK_ILLEGAL(CTX, secp256k1_musig_pubkey_agg(CTX, &agg_pk, &keyagg_cache, NULL, 0)); + CHECK(memcmp_and_randomize(agg_pk.data, zeros132, sizeof(agg_pk.data)) == 0); + + CHECK(secp256k1_musig_pubkey_agg(CTX, &agg_pk, &keyagg_cache, pk_ptr, 2) == 1); + + /* pubkey_get */ + CHECK(secp256k1_musig_pubkey_get(CTX, &full_agg_pk, &keyagg_cache) == 1); + CHECK_ILLEGAL(CTX, secp256k1_musig_pubkey_get(CTX, NULL, &keyagg_cache)); + CHECK_ILLEGAL(CTX, secp256k1_musig_pubkey_get(CTX, &full_agg_pk, NULL)); + CHECK(secp256k1_memcmp_var(&full_agg_pk, zeros132, sizeof(full_agg_pk)) == 0); + + /** Tweaking **/ + { + int (*tweak_func[2]) (const secp256k1_context* ctx, secp256k1_pubkey *output_pubkey, secp256k1_musig_keyagg_cache *keyagg_cache, const unsigned char *tweak32); + tweak_func[0] = secp256k1_musig_pubkey_ec_tweak_add; + tweak_func[1] = secp256k1_musig_pubkey_xonly_tweak_add; + for (i = 0; i < 2; i++) { + secp256k1_pubkey tmp_output_pk; + secp256k1_musig_keyagg_cache tmp_keyagg_cache = keyagg_cache; + CHECK((*tweak_func[i])(CTX, &tmp_output_pk, &tmp_keyagg_cache, tweak) == 1); + /* Reset keyagg_cache */ + tmp_keyagg_cache = keyagg_cache; + CHECK((*tweak_func[i])(CTX, NULL, &tmp_keyagg_cache, tweak) == 1); + tmp_keyagg_cache = keyagg_cache; + CHECK_ILLEGAL(CTX, (*tweak_func[i])(CTX, &tmp_output_pk, NULL, tweak)); + CHECK(memcmp_and_randomize(tmp_output_pk.data, zeros132, sizeof(tmp_output_pk.data)) == 0); + tmp_keyagg_cache = keyagg_cache; + CHECK_ILLEGAL(CTX, (*tweak_func[i])(CTX, &tmp_output_pk, &tmp_keyagg_cache, NULL)); + CHECK(memcmp_and_randomize(tmp_output_pk.data, zeros132, sizeof(tmp_output_pk.data)) == 0); + tmp_keyagg_cache = keyagg_cache; + CHECK((*tweak_func[i])(CTX, &tmp_output_pk, &tmp_keyagg_cache, max64) == 0); + CHECK(memcmp_and_randomize(tmp_output_pk.data, zeros132, sizeof(tmp_output_pk.data)) == 0); + tmp_keyagg_cache = keyagg_cache; + /* Uninitialized keyagg_cache */ + CHECK_ILLEGAL(CTX, (*tweak_func[i])(CTX, &tmp_output_pk, &invalid_keyagg_cache, tweak)); + CHECK(memcmp_and_randomize(tmp_output_pk.data, zeros132, sizeof(tmp_output_pk.data)) == 0); + } + } + + /** Session creation with nonce_gen **/ + CHECK(secp256k1_musig_nonce_gen(CTX, &secnonce[0], &pubnonce[0], session_secrand[0], sk[0], &pk[0], msg, &keyagg_cache, max64) == 1); + /* nonce_gen, if successful, sets session_secrand to the zero array, which + * makes subsequent nonce_gen calls with the same session_secrand fail. So + * check that session_secrand is indeed the zero array and fill it with + * random values again. */ + CHECK(memcmp_and_randomize(session_secrand[0], zeros132, sizeof(session_secrand[0])) == 0); + + CHECK_ILLEGAL(STATIC_CTX, secp256k1_musig_nonce_gen(STATIC_CTX, &secnonce[0], &pubnonce[0], session_secrand[0], sk[0], &pk[0], msg, &keyagg_cache, max64)); + CHECK(memcmp_and_randomize(secnonce[0].data, zeros132, sizeof(secnonce[0].data)) == 0); + + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_gen(CTX, NULL, &pubnonce[0], session_secrand[0], sk[0], &pk[0], msg, &keyagg_cache, max64)); + + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_gen(CTX, &secnonce[0], NULL, session_secrand[0], sk[0], &pk[0], msg, &keyagg_cache, max64)); + CHECK(memcmp_and_randomize(secnonce[0].data, zeros132, sizeof(secnonce[0].data)) == 0); + + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_gen(CTX, &secnonce[0], &pubnonce[0], NULL, sk[0], &pk[0], msg, &keyagg_cache, max64)); + CHECK(memcmp_and_randomize(secnonce[0].data, zeros132, sizeof(secnonce[0].data)) == 0); + + /* session_secrand = 0 is disallowed because it indicates a faulty RNG */ + memcpy(&session_secrand[0], zeros132, sizeof(session_secrand[0])); + CHECK(secp256k1_musig_nonce_gen(CTX, &secnonce[0], &pubnonce[0], zeros132, sk[0], &pk[0], msg, &keyagg_cache, max64) == 0); + CHECK(memcmp_and_randomize(session_secrand[0], zeros132, sizeof(session_secrand[0])) == 0); + CHECK(memcmp_and_randomize(secnonce[0].data, zeros132, sizeof(secnonce[0].data)) == 0); + + CHECK(secp256k1_musig_nonce_gen(CTX, &secnonce[0], &pubnonce[0], session_secrand[0], NULL, &pk[0], msg, &keyagg_cache, max64) == 1); + CHECK(memcmp_and_randomize(session_secrand[0], zeros132, sizeof(session_secrand[0])) == 0); + + /* invalid seckey */ + CHECK(secp256k1_musig_nonce_gen(CTX, &secnonce[0], &pubnonce[0], session_secrand[0], max64, &pk[0], msg, &keyagg_cache, max64) == 0); + CHECK(memcmp_and_randomize(secnonce[0].data, zeros132, sizeof(secnonce[0].data)) == 0); + + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_gen(CTX, &secnonce[0], &pubnonce[0], session_secrand[0], sk[0], NULL, msg, &keyagg_cache, max64)); + CHECK(memcmp_and_randomize(secnonce[0].data, zeros132, sizeof(secnonce[0].data)) == 0); + + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_gen(CTX, &secnonce[0], &pubnonce[0], session_secrand[0], sk[0], &invalid_pk, msg, &keyagg_cache, max64)); + CHECK(memcmp_and_randomize(secnonce[0].data, zeros132, sizeof(secnonce[0].data)) == 0); + + CHECK(secp256k1_musig_nonce_gen(CTX, &secnonce[0], &pubnonce[0], session_secrand[0], sk[0], &pk[0], NULL, &keyagg_cache, max64) == 1); + CHECK(memcmp_and_randomize(session_secrand[0], zeros132, sizeof(session_secrand[0])) == 0); + + CHECK(secp256k1_musig_nonce_gen(CTX, &secnonce[0], &pubnonce[0], session_secrand[0], sk[0], &pk[0], msg, NULL, max64) == 1); + CHECK(memcmp_and_randomize(session_secrand[0], zeros132, sizeof(session_secrand[0])) == 0); + + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_gen(CTX, &secnonce[0], &pubnonce[0], session_secrand[0], sk[0], &pk[0], msg, &invalid_keyagg_cache, max64)); + CHECK(memcmp_and_randomize(secnonce[0].data, zeros132, sizeof(secnonce[0].data)) == 0); + + CHECK(secp256k1_musig_nonce_gen(CTX, &secnonce[0], &pubnonce[0], session_secrand[0], sk[0], &pk[0], msg, &keyagg_cache, NULL) == 1); + CHECK(memcmp_and_randomize(session_secrand[0], zeros132, sizeof(session_secrand[0])) == 0); + + /* Every in-argument except session_secrand and pubkey can be NULL */ + CHECK(secp256k1_musig_nonce_gen(CTX, &secnonce[0], &pubnonce[0], session_secrand[0], NULL, &pk[0], NULL, NULL, NULL) == 1); + CHECK(secp256k1_musig_nonce_gen(CTX, &secnonce[1], &pubnonce[1], session_secrand[1], sk[1], &pk[1], NULL, NULL, NULL) == 1); + + /** Session creation with nonce_gen_counter **/ + CHECK(secp256k1_musig_nonce_gen_counter(CTX, &secnonce[0], &pubnonce[0], nonrepeating_cnt, &keypair[0], msg, &keyagg_cache, max64) == 1); + CHECK_ILLEGAL(STATIC_CTX, secp256k1_musig_nonce_gen_counter(STATIC_CTX, &secnonce[0], &pubnonce[0], nonrepeating_cnt, &keypair[0], msg, &keyagg_cache, max64)); + CHECK(memcmp_and_randomize(secnonce[0].data, zeros132, sizeof(secnonce[0].data)) == 0); + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_gen_counter(CTX, NULL, &pubnonce[0], nonrepeating_cnt, &keypair[0], msg, &keyagg_cache, max64)); + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_gen_counter(CTX, &secnonce[0], NULL, nonrepeating_cnt, &keypair[0], msg, &keyagg_cache, max64)); + CHECK(memcmp_and_randomize(secnonce[0].data, zeros132, sizeof(secnonce[0].data)) == 0); + /* using nonce_gen_counter requires keypair */ + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_gen_counter(CTX, &secnonce[0], &pubnonce[0], nonrepeating_cnt, NULL, msg, &keyagg_cache, max64)); + CHECK(memcmp_and_randomize(secnonce[0].data, zeros132, sizeof(secnonce[0].data)) == 0); + /* invalid keypair */ + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_gen_counter(CTX, &secnonce[0], &pubnonce[0], nonrepeating_cnt, &invalid_keypair, msg, &keyagg_cache, max64)); + CHECK(memcmp_and_randomize(secnonce[0].data, zeros132, sizeof(secnonce[0].data)) == 0); + CHECK(secp256k1_musig_nonce_gen_counter(CTX, &secnonce[0], &pubnonce[0], nonrepeating_cnt, &keypair[0], NULL, &keyagg_cache, max64) == 1); + CHECK(secp256k1_musig_nonce_gen_counter(CTX, &secnonce[0], &pubnonce[0], nonrepeating_cnt, &keypair[0], msg, NULL, max64) == 1); + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_gen_counter(CTX, &secnonce[0], &pubnonce[0], nonrepeating_cnt, &keypair[0], msg, &invalid_keyagg_cache, max64)); + CHECK(memcmp_and_randomize(secnonce[0].data, zeros132, sizeof(secnonce[0].data)) == 0); + CHECK(secp256k1_musig_nonce_gen_counter(CTX, &secnonce[0], &pubnonce[0], nonrepeating_cnt,&keypair[0], msg, &keyagg_cache, NULL) == 1); + + /* Every in-argument except nonrepeating_cnt and keypair can be NULL */ + CHECK(secp256k1_musig_nonce_gen_counter(CTX, &secnonce[0], &pubnonce[0], nonrepeating_cnt, &keypair[0], NULL, NULL, NULL) == 1); + CHECK(secp256k1_musig_nonce_gen_counter(CTX, &secnonce[1], &pubnonce[1], nonrepeating_cnt, &keypair[1], NULL, NULL, NULL) == 1); + + + /** Serialize and parse public nonces **/ + CHECK_ILLEGAL(CTX, secp256k1_musig_pubnonce_serialize(CTX, NULL, &pubnonce[0])); + CHECK_ILLEGAL(CTX, secp256k1_musig_pubnonce_serialize(CTX, pubnonce_ser, NULL)); + CHECK(memcmp_and_randomize(pubnonce_ser, zeros132, sizeof(pubnonce_ser)) == 0); + CHECK_ILLEGAL(CTX, secp256k1_musig_pubnonce_serialize(CTX, pubnonce_ser, &invalid_pubnonce)); + CHECK(memcmp_and_randomize(pubnonce_ser, zeros132, sizeof(pubnonce_ser)) == 0); + CHECK(secp256k1_musig_pubnonce_serialize(CTX, pubnonce_ser, &pubnonce[0]) == 1); + + CHECK(secp256k1_musig_pubnonce_parse(CTX, &pubnonce[0], pubnonce_ser) == 1); + CHECK_ILLEGAL(CTX, secp256k1_musig_pubnonce_parse(CTX, NULL, pubnonce_ser)); + CHECK_ILLEGAL(CTX, secp256k1_musig_pubnonce_parse(CTX, &pubnonce[0], NULL)); + CHECK(secp256k1_musig_pubnonce_parse(CTX, &pubnonce[0], zeros132) == 0); + CHECK(secp256k1_musig_pubnonce_parse(CTX, &pubnonce[0], pubnonce_ser) == 1); + + { + /* Check that serialize and parse results in the same value */ + secp256k1_musig_pubnonce tmp; + CHECK(secp256k1_musig_pubnonce_serialize(CTX, pubnonce_ser, &pubnonce[0]) == 1); + CHECK(secp256k1_musig_pubnonce_parse(CTX, &tmp, pubnonce_ser) == 1); + CHECK(secp256k1_memcmp_var(&tmp, &pubnonce[0], sizeof(tmp)) == 0); + } + + /** Receive nonces and aggregate **/ + CHECK(secp256k1_musig_nonce_agg(CTX, &aggnonce, pubnonce_ptr, 2) == 1); + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_agg(CTX, NULL, pubnonce_ptr, 2)); + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_agg(CTX, &aggnonce, NULL, 2)); + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_agg(CTX, &aggnonce, pubnonce_ptr, 0)); + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_agg(CTX, &aggnonce, invalid_pubnonce_ptr, 1)); + CHECK(secp256k1_musig_nonce_agg(CTX, &aggnonce, inf_pubnonce_ptr, 2) == 1); + { + /* Check that the aggnonce encodes two points at infinity */ + secp256k1_ge aggnonce_pt[2]; + secp256k1_musig_aggnonce_load(CTX, aggnonce_pt, &aggnonce); + for (i = 0; i < 2; i++) { + secp256k1_ge_is_infinity(&aggnonce_pt[i]); + } + } + CHECK(secp256k1_musig_nonce_agg(CTX, &aggnonce, pubnonce_ptr, 2) == 1); + + /** Serialize and parse aggregate nonces **/ + CHECK(secp256k1_musig_aggnonce_serialize(CTX, aggnonce_ser, &aggnonce) == 1); + CHECK_ILLEGAL(CTX, secp256k1_musig_aggnonce_serialize(CTX, NULL, &aggnonce)); + CHECK_ILLEGAL(CTX, secp256k1_musig_aggnonce_serialize(CTX, aggnonce_ser, NULL)); + CHECK(memcmp_and_randomize(aggnonce_ser, zeros132, sizeof(aggnonce_ser)) == 0); + CHECK_ILLEGAL(CTX, secp256k1_musig_aggnonce_serialize(CTX, aggnonce_ser, (secp256k1_musig_aggnonce*) &invalid_pubnonce)); + CHECK(memcmp_and_randomize(aggnonce_ser, zeros132, sizeof(aggnonce_ser)) == 0); + CHECK(secp256k1_musig_aggnonce_serialize(CTX, aggnonce_ser, &aggnonce) == 1); + + CHECK(secp256k1_musig_aggnonce_parse(CTX, &aggnonce, aggnonce_ser) == 1); + CHECK_ILLEGAL(CTX, secp256k1_musig_aggnonce_parse(CTX, NULL, aggnonce_ser)); + CHECK_ILLEGAL(CTX, secp256k1_musig_aggnonce_parse(CTX, &aggnonce, NULL)); + CHECK(secp256k1_musig_aggnonce_parse(CTX, &aggnonce, zeros132) == 1); + CHECK(secp256k1_musig_aggnonce_parse(CTX, &aggnonce, aggnonce_ser) == 1); + + { + /* Check that serialize and parse results in the same value */ + secp256k1_musig_aggnonce tmp; + CHECK(secp256k1_musig_aggnonce_serialize(CTX, aggnonce_ser, &aggnonce) == 1); + CHECK(secp256k1_musig_aggnonce_parse(CTX, &tmp, aggnonce_ser) == 1); + CHECK(secp256k1_memcmp_var(&tmp, &aggnonce, sizeof(tmp)) == 0); + } + + /** Process nonces **/ + CHECK(secp256k1_musig_nonce_process(CTX, &session, &aggnonce, msg, &keyagg_cache) == 1); + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_process(CTX, NULL, &aggnonce, msg, &keyagg_cache)); + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_process(CTX, &session, NULL, msg, &keyagg_cache)); + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_process(CTX, &session, (secp256k1_musig_aggnonce*) &invalid_pubnonce, msg, &keyagg_cache)); + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_process(CTX, &session, &aggnonce, NULL, &keyagg_cache)); + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_process(CTX, &session, &aggnonce, msg, NULL)); + CHECK_ILLEGAL(CTX, secp256k1_musig_nonce_process(CTX, &session, &aggnonce, msg, &invalid_keyagg_cache)); + + CHECK(secp256k1_musig_nonce_process(CTX, &session, &aggnonce, msg, &keyagg_cache) == 1); + + memcpy(&secnonce_tmp, &secnonce[0], sizeof(secnonce_tmp)); + CHECK(secp256k1_musig_partial_sign(CTX, &partial_sig[0], &secnonce_tmp, &keypair[0], &keyagg_cache, &session) == 1); + /* The secnonce is set to 0 and subsequent signing attempts fail */ + CHECK(secp256k1_memcmp_var(&secnonce_tmp, zeros132, sizeof(secnonce_tmp)) == 0); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sign(CTX, &partial_sig[0], &secnonce_tmp, &keypair[0], &keyagg_cache, &session)); + memcpy(&secnonce_tmp, &secnonce[0], sizeof(secnonce_tmp)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sign(CTX, NULL, &secnonce_tmp, &keypair[0], &keyagg_cache, &session)); + memcpy(&secnonce_tmp, &secnonce[0], sizeof(secnonce_tmp)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sign(CTX, &partial_sig[0], NULL, &keypair[0], &keyagg_cache, &session)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sign(CTX, &partial_sig[0], &invalid_secnonce, &keypair[0], &keyagg_cache, &session)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sign(CTX, &partial_sig[0], &secnonce_tmp, NULL, &keyagg_cache, &session)); + memcpy(&secnonce_tmp, &secnonce[0], sizeof(secnonce_tmp)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sign(CTX, &partial_sig[0], &secnonce_tmp, &invalid_keypair, &keyagg_cache, &session)); + memcpy(&secnonce_tmp, &secnonce[0], sizeof(secnonce_tmp)); + { + unsigned char sk_tmp[32]; + secp256k1_keypair keypair_tmp; + testrand256(sk_tmp); + CHECK(secp256k1_keypair_create(CTX, &keypair_tmp, sk_tmp)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sign(CTX, &partial_sig[0], &secnonce_tmp, &keypair_tmp, &keyagg_cache, &session)); + memcpy(&secnonce_tmp, &secnonce[0], sizeof(secnonce_tmp)); + } + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sign(CTX, &partial_sig[0], &secnonce_tmp, &keypair[0], NULL, &session)); + memcpy(&secnonce_tmp, &secnonce[0], sizeof(secnonce_tmp)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sign(CTX, &partial_sig[0], &secnonce_tmp, &keypair[0], &invalid_keyagg_cache, &session)); + memcpy(&secnonce_tmp, &secnonce[0], sizeof(secnonce_tmp)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sign(CTX, &partial_sig[0], &secnonce_tmp, &keypair[0], &keyagg_cache, NULL)); + memcpy(&secnonce_tmp, &secnonce[0], sizeof(secnonce_tmp)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sign(CTX, &partial_sig[0], &secnonce_tmp, &keypair[0], &keyagg_cache, &invalid_session)); + memcpy(&secnonce_tmp, &secnonce[0], sizeof(secnonce_tmp)); + + CHECK(secp256k1_musig_partial_sign(CTX, &partial_sig[0], &secnonce[0], &keypair[0], &keyagg_cache, &session) == 1); + CHECK(secp256k1_musig_partial_sign(CTX, &partial_sig[1], &secnonce[1], &keypair[1], &keyagg_cache, &session) == 1); + + CHECK(secp256k1_musig_partial_sig_serialize(CTX, buf, &partial_sig[0]) == 1); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_serialize(CTX, NULL, &partial_sig[0])); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_serialize(CTX, buf, NULL)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_serialize(CTX, buf, &invalid_partial_sig)); + CHECK(secp256k1_musig_partial_sig_parse(CTX, &partial_sig[0], buf) == 1); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_parse(CTX, NULL, buf)); + { + /* Check that parsing failure results in an invalid sig */ + secp256k1_musig_partial_sig tmp; + CHECK(secp256k1_musig_partial_sig_parse(CTX, &tmp, max64) == 0); + CHECK(secp256k1_memcmp_var(&tmp, zeros132, sizeof(partial_sig[0])) == 0); + } + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_parse(CTX, &partial_sig[0], NULL)); + + { + /* Check that serialize and parse results in the same value */ + secp256k1_musig_partial_sig tmp; + CHECK(secp256k1_musig_partial_sig_serialize(CTX, buf, &partial_sig[0]) == 1); + CHECK(secp256k1_musig_partial_sig_parse(CTX, &tmp, buf) == 1); + CHECK(secp256k1_memcmp_var(&tmp, &partial_sig[0], sizeof(tmp)) == 0); + } + + /** Partial signature verification */ + CHECK(secp256k1_musig_partial_sig_verify(CTX, &partial_sig[0], &pubnonce[0], &pk[0], &keyagg_cache, &session) == 1); + CHECK(secp256k1_musig_partial_sig_verify(CTX, &partial_sig[1], &pubnonce[0], &pk[0], &keyagg_cache, &session) == 0); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_verify(CTX, NULL, &pubnonce[0], &pk[0], &keyagg_cache, &session)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_verify(CTX, &invalid_partial_sig, &pubnonce[0], &pk[0], &keyagg_cache, &session)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_verify(CTX, &partial_sig[0], NULL, &pk[0], &keyagg_cache, &session)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_verify(CTX, &partial_sig[0], &invalid_pubnonce, &pk[0], &keyagg_cache, &session)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_verify(CTX, &partial_sig[0], &pubnonce[0], NULL, &keyagg_cache, &session)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_verify(CTX, &partial_sig[0], &pubnonce[0], &invalid_pk, &keyagg_cache, &session)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_verify(CTX, &partial_sig[0], &pubnonce[0], &pk[0], NULL, &session)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_verify(CTX, &partial_sig[0], &pubnonce[0], &pk[0], &invalid_keyagg_cache, &session)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_verify(CTX, &partial_sig[0], &pubnonce[0], &pk[0], &keyagg_cache, NULL)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_verify(CTX, &partial_sig[0], &pubnonce[0], &pk[0], &keyagg_cache, &invalid_session)); + + CHECK(secp256k1_musig_partial_sig_verify(CTX, &partial_sig[0], &pubnonce[0], &pk[0], &keyagg_cache, &session) == 1); + CHECK(secp256k1_musig_partial_sig_verify(CTX, &partial_sig[1], &pubnonce[1], &pk[1], &keyagg_cache, &session) == 1); + + /** Signature aggregation and verification */ + CHECK(secp256k1_musig_partial_sig_agg(CTX, pre_sig, &session, partial_sig_ptr, 2) == 1); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_agg(CTX, NULL, &session, partial_sig_ptr, 2)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_agg(CTX, pre_sig, NULL, partial_sig_ptr, 2)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_agg(CTX, pre_sig, &invalid_session, partial_sig_ptr, 2)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_agg(CTX, pre_sig, &session, NULL, 2)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_agg(CTX, pre_sig, &session, invalid_partial_sig_ptr, 2)); + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sig_agg(CTX, pre_sig, &session, partial_sig_ptr, 0)); + CHECK(secp256k1_musig_partial_sig_agg(CTX, pre_sig, &session, partial_sig_ptr, 1) == 1); + CHECK(secp256k1_musig_partial_sig_agg(CTX, pre_sig, &session, partial_sig_ptr, 2) == 1); +} + +static void musig_nonce_bitflip(unsigned char **args, size_t n_flip, size_t n_bytes) { + secp256k1_scalar k1[2], k2[2]; + + secp256k1_nonce_function_musig(k1, args[0], args[1], args[2], args[3], args[4], args[5]); + testrand_flip(args[n_flip], n_bytes); + secp256k1_nonce_function_musig(k2, args[0], args[1], args[2], args[3], args[4], args[5]); + CHECK(secp256k1_scalar_eq(&k1[0], &k2[0]) == 0); + CHECK(secp256k1_scalar_eq(&k1[1], &k2[1]) == 0); +} + +static void musig_nonce_test(void) { + unsigned char *args[6]; + unsigned char session_secrand[32]; + unsigned char sk[32]; + unsigned char pk[33]; + unsigned char msg[32]; + unsigned char agg_pk[32]; + unsigned char extra_input[32]; + int i, j; + secp256k1_scalar k[6][2]; + + testrand_bytes_test(session_secrand, sizeof(session_secrand)); + testrand_bytes_test(sk, sizeof(sk)); + testrand_bytes_test(pk, sizeof(pk)); + testrand_bytes_test(msg, sizeof(msg)); + testrand_bytes_test(agg_pk, sizeof(agg_pk)); + testrand_bytes_test(extra_input, sizeof(extra_input)); + + /* Check that a bitflip in an argument results in different nonces. */ + args[0] = session_secrand; + args[1] = msg; + args[2] = sk; + args[3] = pk; + args[4] = agg_pk; + args[5] = extra_input; + for (i = 0; i < COUNT; i++) { + musig_nonce_bitflip(args, 0, sizeof(session_secrand)); + musig_nonce_bitflip(args, 1, sizeof(msg)); + musig_nonce_bitflip(args, 2, sizeof(sk)); + musig_nonce_bitflip(args, 3, sizeof(pk)); + musig_nonce_bitflip(args, 4, sizeof(agg_pk)); + musig_nonce_bitflip(args, 5, sizeof(extra_input)); + } + /* Check that if any argument is NULL, a different nonce is produced than if + * any other argument is NULL. */ + memcpy(msg, session_secrand, sizeof(msg)); + memcpy(sk, session_secrand, sizeof(sk)); + memcpy(pk, session_secrand, sizeof(session_secrand)); + memcpy(agg_pk, session_secrand, sizeof(agg_pk)); + memcpy(extra_input, session_secrand, sizeof(extra_input)); + secp256k1_nonce_function_musig(k[0], args[0], args[1], args[2], args[3], args[4], args[5]); + secp256k1_nonce_function_musig(k[1], args[0], NULL, args[2], args[3], args[4], args[5]); + secp256k1_nonce_function_musig(k[2], args[0], args[1], NULL, args[3], args[4], args[5]); + secp256k1_nonce_function_musig(k[3], args[0], args[1], args[2], NULL, args[4], args[5]); + secp256k1_nonce_function_musig(k[4], args[0], args[1], args[2], args[3], NULL, args[5]); + secp256k1_nonce_function_musig(k[5], args[0], args[1], args[2], args[3], args[4], NULL); + for (i = 0; i < 6; i++) { + CHECK(!secp256k1_scalar_eq(&k[i][0], &k[i][1])); + for (j = i+1; j < 6; j++) { + CHECK(!secp256k1_scalar_eq(&k[i][0], &k[j][0])); + CHECK(!secp256k1_scalar_eq(&k[i][1], &k[j][1])); + } + } +} + +static void sha256_tag_test_internal(secp256k1_sha256 *sha_tagged, unsigned char *tag, size_t taglen) { + secp256k1_sha256 sha; + secp256k1_sha256_initialize_tagged(&sha, tag, taglen); + test_sha256_eq(&sha, sha_tagged); +} + +/* Checks that the initialized tagged hashes have the expected + * state. */ +static void sha256_tag_test(void) { + secp256k1_sha256 sha; + { + char tag[] = "KeyAgg list"; + secp256k1_musig_keyagglist_sha256(&sha); + sha256_tag_test_internal(&sha, (unsigned char*)tag, sizeof(tag) - 1); + } + { + char tag[] = "KeyAgg coefficient"; + secp256k1_musig_keyaggcoef_sha256(&sha); + sha256_tag_test_internal(&sha, (unsigned char*)tag, sizeof(tag) - 1); + } + { + unsigned char tag[] = "MuSig/aux"; + secp256k1_nonce_function_musig_sha256_tagged_aux(&sha); + sha256_tag_test_internal(&sha, (unsigned char*)tag, sizeof(tag) - 1); + } + { + unsigned char tag[] = "MuSig/nonce"; + secp256k1_nonce_function_musig_sha256_tagged(&sha); + sha256_tag_test_internal(&sha, (unsigned char*)tag, sizeof(tag) - 1); + } + { + unsigned char tag[] = "MuSig/noncecoef"; + secp256k1_musig_compute_noncehash_sha256_tagged(&sha); + sha256_tag_test_internal(&sha, (unsigned char*)tag, sizeof(tag) - 1); + } +} + +/* Attempts to create a signature for the aggregate public key using given secret + * keys and keyagg_cache. */ +static void musig_tweak_test_helper(const secp256k1_xonly_pubkey* agg_pk, const unsigned char *sk0, const unsigned char *sk1, secp256k1_musig_keyagg_cache *keyagg_cache) { + secp256k1_pubkey pk[2]; + unsigned char session_secrand[2][32]; + unsigned char msg[32]; + secp256k1_musig_secnonce secnonce[2]; + secp256k1_musig_pubnonce pubnonce[2]; + const secp256k1_musig_pubnonce *pubnonce_ptr[2]; + secp256k1_musig_aggnonce aggnonce; + secp256k1_keypair keypair[2]; + secp256k1_musig_session session; + secp256k1_musig_partial_sig partial_sig[2]; + const secp256k1_musig_partial_sig *partial_sig_ptr[2]; + unsigned char final_sig[64]; + int i; + + for (i = 0; i < 2; i++) { + pubnonce_ptr[i] = &pubnonce[i]; + partial_sig_ptr[i] = &partial_sig[i]; + + testrand256(session_secrand[i]); + } + CHECK(create_keypair_and_pk(&keypair[0], &pk[0], sk0) == 1); + CHECK(create_keypair_and_pk(&keypair[1], &pk[1], sk1) == 1); + testrand256(msg); + + CHECK(secp256k1_musig_nonce_gen(CTX, &secnonce[0], &pubnonce[0], session_secrand[0], sk0, &pk[0], NULL, NULL, NULL) == 1); + CHECK(secp256k1_musig_nonce_gen(CTX, &secnonce[1], &pubnonce[1], session_secrand[1], sk1, &pk[1], NULL, NULL, NULL) == 1); + + CHECK(secp256k1_musig_nonce_agg(CTX, &aggnonce, pubnonce_ptr, 2) == 1); + CHECK(secp256k1_musig_nonce_process(CTX, &session, &aggnonce, msg, keyagg_cache) == 1); + + CHECK(secp256k1_musig_partial_sign(CTX, &partial_sig[0], &secnonce[0], &keypair[0], keyagg_cache, &session) == 1); + CHECK(secp256k1_musig_partial_sign(CTX, &partial_sig[1], &secnonce[1], &keypair[1], keyagg_cache, &session) == 1); + + CHECK(secp256k1_musig_partial_sig_verify(CTX, &partial_sig[0], &pubnonce[0], &pk[0], keyagg_cache, &session) == 1); + CHECK(secp256k1_musig_partial_sig_verify(CTX, &partial_sig[1], &pubnonce[1], &pk[1], keyagg_cache, &session) == 1); + + CHECK(secp256k1_musig_partial_sig_agg(CTX, final_sig, &session, partial_sig_ptr, 2) == 1); + CHECK(secp256k1_schnorrsig_verify(CTX, final_sig, msg, sizeof(msg), agg_pk) == 1); +} + +/* Create aggregate public key P[0], tweak multiple times (using xonly and + * plain tweaking) and test signing. */ +static void musig_tweak_test(void) { + unsigned char sk[2][32]; + secp256k1_pubkey pk[2]; + const secp256k1_pubkey *pk_ptr[2]; + secp256k1_musig_keyagg_cache keyagg_cache; + enum { N_TWEAKS = 8 }; + secp256k1_pubkey P[N_TWEAKS + 1]; + secp256k1_xonly_pubkey P_xonly[N_TWEAKS + 1]; + int i; + + /* Key Setup */ + for (i = 0; i < 2; i++) { + pk_ptr[i] = &pk[i]; + testrand256(sk[i]); + CHECK(create_keypair_and_pk(NULL, &pk[i], sk[i]) == 1); + } + /* Compute P0 = keyagg(pk0, pk1) and test signing for it */ + CHECK(secp256k1_musig_pubkey_agg(CTX, &P_xonly[0], &keyagg_cache, pk_ptr, 2) == 1); + musig_tweak_test_helper(&P_xonly[0], sk[0], sk[1], &keyagg_cache); + CHECK(secp256k1_musig_pubkey_get(CTX, &P[0], &keyagg_cache)); + + /* Compute Pi = f(Pj) + tweaki*G where where j = i-1 and try signing for + * that key. If xonly is set to true, the function f normalizes the input + * point to have an even X-coordinate ("xonly-tweaking"). + * Otherwise, the function f is the identity function. */ + for (i = 1; i <= N_TWEAKS; i++) { + unsigned char tweak[32]; + int P_parity; + int xonly = testrand_bits(1); + + testrand256(tweak); + if (xonly) { + CHECK(secp256k1_musig_pubkey_xonly_tweak_add(CTX, &P[i], &keyagg_cache, tweak) == 1); + } else { + CHECK(secp256k1_musig_pubkey_ec_tweak_add(CTX, &P[i], &keyagg_cache, tweak) == 1); + } + CHECK(secp256k1_xonly_pubkey_from_pubkey(CTX, &P_xonly[i], &P_parity, &P[i])); + /* Check that musig_pubkey_tweak_add produces same result as + * xonly_pubkey_tweak_add or ec_pubkey_tweak_add. */ + if (xonly) { + unsigned char P_serialized[32]; + CHECK(secp256k1_xonly_pubkey_serialize(CTX, P_serialized, &P_xonly[i])); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(CTX, P_serialized, P_parity, &P_xonly[i-1], tweak) == 1); + } else { + secp256k1_pubkey tmp_key = P[i-1]; + CHECK(secp256k1_ec_pubkey_tweak_add(CTX, &tmp_key, tweak)); + CHECK(secp256k1_memcmp_var(&tmp_key, &P[i], sizeof(tmp_key)) == 0); + } + /* Test signing for P[i] */ + musig_tweak_test_helper(&P_xonly[i], sk[0], sk[1], &keyagg_cache); + } +} + +int musig_vectors_keyagg_and_tweak(enum MUSIG_ERROR *error, + secp256k1_musig_keyagg_cache *keyagg_cache, + unsigned char *agg_pk_ser, + const unsigned char pubkeys33[][33], + const unsigned char tweaks32[][32], + size_t key_indices_len, + const size_t *key_indices, + size_t tweak_indices_len, + const size_t *tweak_indices, + const int *is_xonly) { + secp256k1_pubkey pubkeys[MUSIG_VECTORS_MAX_PUBKEYS]; + const secp256k1_pubkey *pk_ptr[MUSIG_VECTORS_MAX_PUBKEYS]; + int i; + secp256k1_pubkey agg_pk; + secp256k1_xonly_pubkey agg_pk_xonly; + + for (i = 0; i < (int)key_indices_len; i++) { + if (!secp256k1_ec_pubkey_parse(CTX, &pubkeys[i], pubkeys33[key_indices[i]], 33)) { + *error = MUSIG_PUBKEY; + return 0; + } + pk_ptr[i] = &pubkeys[i]; + } + if (!secp256k1_musig_pubkey_agg(CTX, NULL, keyagg_cache, pk_ptr, key_indices_len)) { + *error = MUSIG_OTHER; + return 0; + } + + for (i = 0; i < (int)tweak_indices_len; i++) { + if (is_xonly[i]) { + if (!secp256k1_musig_pubkey_xonly_tweak_add(CTX, NULL, keyagg_cache, tweaks32[tweak_indices[i]])) { + *error = MUSIG_TWEAK; + return 0; + } + } else { + if (!secp256k1_musig_pubkey_ec_tweak_add(CTX, NULL, keyagg_cache, tweaks32[tweak_indices[i]])) { + *error = MUSIG_TWEAK; + return 0; + } + } + } + if (!secp256k1_musig_pubkey_get(CTX, &agg_pk, keyagg_cache)) { + *error = MUSIG_OTHER; + return 0; + } + + if (!secp256k1_xonly_pubkey_from_pubkey(CTX, &agg_pk_xonly, NULL, &agg_pk)) { + *error = MUSIG_OTHER; + return 0; + } + + if (agg_pk_ser != NULL) { + if (!secp256k1_xonly_pubkey_serialize(CTX, agg_pk_ser, &agg_pk_xonly)) { + *error = MUSIG_OTHER; + return 0; + } + } + + return 1; +} + +static void musig_test_vectors_keyagg(void) { + size_t i; + const struct musig_key_agg_vector *vector = &musig_key_agg_vector; + + for (i = 0; i < sizeof(vector->valid_case)/sizeof(vector->valid_case[0]); i++) { + const struct musig_key_agg_valid_test_case *c = &vector->valid_case[i]; + enum MUSIG_ERROR error; + secp256k1_musig_keyagg_cache keyagg_cache; + unsigned char agg_pk[32]; + + CHECK(musig_vectors_keyagg_and_tweak(&error, &keyagg_cache, agg_pk, vector->pubkeys, vector->tweaks, c->key_indices_len, c->key_indices, 0, NULL, NULL)); + CHECK(secp256k1_memcmp_var(agg_pk, c->expected, sizeof(agg_pk)) == 0); + } + + for (i = 0; i < sizeof(vector->error_case)/sizeof(vector->error_case[0]); i++) { + const struct musig_key_agg_error_test_case *c = &vector->error_case[i]; + enum MUSIG_ERROR error; + secp256k1_musig_keyagg_cache keyagg_cache; + + CHECK(!musig_vectors_keyagg_and_tweak(&error, &keyagg_cache, NULL, vector->pubkeys, vector->tweaks, c->key_indices_len, c->key_indices, c->tweak_indices_len, c->tweak_indices, c->is_xonly)); + CHECK(c->error == error); + } +} + +static void musig_test_vectors_noncegen(void) { + size_t i; + const struct musig_nonce_gen_vector *vector = &musig_nonce_gen_vector; + + for (i = 0; i < sizeof(vector->test_case)/sizeof(vector->test_case[0]); i++) { + const struct musig_nonce_gen_test_case *c = &vector->test_case[i]; + secp256k1_musig_keyagg_cache keyagg_cache; + secp256k1_musig_keyagg_cache *keyagg_cache_ptr = NULL; + unsigned char session_secrand32[32]; + secp256k1_musig_secnonce secnonce; + secp256k1_musig_pubnonce pubnonce; + const unsigned char *sk = NULL; + const unsigned char *msg = NULL; + const unsigned char *extra_in = NULL; + secp256k1_pubkey pk; + unsigned char pubnonce66[66]; + + memcpy(session_secrand32, c->rand_, 32); + if (c->has_sk) { + sk = c->sk; + } + if (c->has_aggpk) { + /* Create keyagg_cache from aggpk */ + secp256k1_keyagg_cache_internal cache_i; + secp256k1_xonly_pubkey aggpk; + memset(&cache_i, 0, sizeof(cache_i)); + CHECK(secp256k1_xonly_pubkey_parse(CTX, &aggpk, c->aggpk)); + CHECK(secp256k1_xonly_pubkey_load(CTX, &cache_i.pk, &aggpk)); + secp256k1_keyagg_cache_save(&keyagg_cache, &cache_i); + keyagg_cache_ptr = &keyagg_cache; + } + if (c->has_msg) { + msg = c->msg; + } + if (c->has_extra_in) { + extra_in = c->extra_in; + } + + CHECK(secp256k1_ec_pubkey_parse(CTX, &pk, c->pk, sizeof(c->pk))); + CHECK(secp256k1_musig_nonce_gen(CTX, &secnonce, &pubnonce, session_secrand32, sk, &pk, msg, keyagg_cache_ptr, extra_in) == 1); + CHECK(secp256k1_memcmp_var(&secnonce.data[4], c->expected_secnonce, 2*32) == 0); + /* The last element of the secnonce is the public key (uncompressed in + * secp256k1_musig_secnonce, compressed in the test vector secnonce). */ + CHECK(secp256k1_memcmp_var(&secnonce.data[4+2*32], &pk, sizeof(pk)) == 0); + CHECK(secp256k1_memcmp_var(&c->expected_secnonce[2*32], c->pk, sizeof(c->pk)) == 0); + + CHECK(secp256k1_musig_pubnonce_serialize(CTX, pubnonce66, &pubnonce) == 1); + CHECK(sizeof(c->expected_pubnonce) == sizeof(pubnonce66)); + CHECK(secp256k1_memcmp_var(pubnonce66, c->expected_pubnonce, sizeof(pubnonce66)) == 0); + } +} + + +static void musig_test_vectors_nonceagg(void) { + size_t i; + int j; + const struct musig_nonce_agg_vector *vector = &musig_nonce_agg_vector; + + for (i = 0; i < sizeof(vector->valid_case)/sizeof(vector->valid_case[0]); i++) { + const struct musig_nonce_agg_test_case *c = &vector->valid_case[i]; + secp256k1_musig_pubnonce pubnonce[2]; + const secp256k1_musig_pubnonce *pubnonce_ptr[2]; + secp256k1_musig_aggnonce aggnonce; + unsigned char aggnonce66[66]; + + for (j = 0; j < 2; j++) { + CHECK(secp256k1_musig_pubnonce_parse(CTX, &pubnonce[j], vector->pnonces[c->pnonce_indices[j]]) == 1); + pubnonce_ptr[j] = &pubnonce[j]; + } + CHECK(secp256k1_musig_nonce_agg(CTX, &aggnonce, pubnonce_ptr, 2)); + CHECK(secp256k1_musig_aggnonce_serialize(CTX, aggnonce66, &aggnonce)); + CHECK(secp256k1_memcmp_var(aggnonce66, c->expected, 33) == 0); + } + for (i = 0; i < sizeof(vector->error_case)/sizeof(vector->error_case[0]); i++) { + const struct musig_nonce_agg_test_case *c = &vector->error_case[i]; + secp256k1_musig_pubnonce pubnonce[2]; + for (j = 0; j < 2; j++) { + int expected = c->invalid_nonce_idx != j; + CHECK(expected == secp256k1_musig_pubnonce_parse(CTX, &pubnonce[j], vector->pnonces[c->pnonce_indices[j]])); + } + } +} + +static void musig_test_set_secnonce(secp256k1_musig_secnonce *secnonce, const unsigned char *secnonce64, const secp256k1_pubkey *pubkey) { + secp256k1_ge pk; + secp256k1_scalar k[2]; + + secp256k1_scalar_set_b32(&k[0], &secnonce64[0], NULL); + secp256k1_scalar_set_b32(&k[1], &secnonce64[32], NULL); + CHECK(secp256k1_pubkey_load(CTX, &pk, pubkey)); + secp256k1_musig_secnonce_save(secnonce, k, &pk); +} + +static void musig_test_vectors_signverify(void) { + size_t i; + const struct musig_sign_verify_vector *vector = &musig_sign_verify_vector; + + for (i = 0; i < sizeof(vector->valid_case)/sizeof(vector->valid_case[0]); i++) { + const struct musig_valid_case *c = &vector->valid_case[i]; + enum MUSIG_ERROR error; + secp256k1_musig_keyagg_cache keyagg_cache; + secp256k1_pubkey pubkey; + secp256k1_musig_pubnonce pubnonce; + secp256k1_musig_aggnonce aggnonce; + secp256k1_musig_session session; + secp256k1_musig_partial_sig partial_sig; + secp256k1_musig_secnonce secnonce; + secp256k1_keypair keypair; + unsigned char partial_sig32[32]; + + CHECK(secp256k1_keypair_create(CTX, &keypair, vector->sk)); + CHECK(musig_vectors_keyagg_and_tweak(&error, &keyagg_cache, NULL, vector->pubkeys, NULL, c->key_indices_len, c->key_indices, 0, NULL, NULL)); + + CHECK(secp256k1_musig_aggnonce_parse(CTX, &aggnonce, vector->aggnonces[c->aggnonce_index])); + CHECK(secp256k1_musig_nonce_process(CTX, &session, &aggnonce, vector->msgs[c->msg_index], &keyagg_cache)); + + CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, vector->pubkeys[0], sizeof(vector->pubkeys[0]))); + musig_test_set_secnonce(&secnonce, vector->secnonces[0], &pubkey); + CHECK(secp256k1_musig_partial_sign(CTX, &partial_sig, &secnonce, &keypair, &keyagg_cache, &session)); + CHECK(secp256k1_musig_partial_sig_serialize(CTX, partial_sig32, &partial_sig)); + CHECK(secp256k1_memcmp_var(partial_sig32, c->expected, sizeof(partial_sig32)) == 0); + + CHECK(secp256k1_musig_pubnonce_parse(CTX, &pubnonce, vector->pubnonces[0])); + CHECK(secp256k1_musig_partial_sig_verify(CTX, &partial_sig, &pubnonce, &pubkey, &keyagg_cache, &session)); + } + for (i = 0; i < sizeof(vector->sign_error_case)/sizeof(vector->sign_error_case[0]); i++) { + const struct musig_sign_error_case *c = &vector->sign_error_case[i]; + enum MUSIG_ERROR error; + secp256k1_musig_keyagg_cache keyagg_cache; + secp256k1_pubkey pubkey; + secp256k1_musig_aggnonce aggnonce; + secp256k1_musig_session session; + secp256k1_musig_partial_sig partial_sig; + secp256k1_musig_secnonce secnonce; + secp256k1_keypair keypair; + int expected; + + if (i == 0) { + /* Skip this vector since the implementation does not error out when + * the signing key does not belong to any pubkey. */ + continue; + } + expected = c->error != MUSIG_PUBKEY; + CHECK(expected == musig_vectors_keyagg_and_tweak(&error, &keyagg_cache, NULL, vector->pubkeys, NULL, c->key_indices_len, c->key_indices, 0, NULL, NULL)); + CHECK(expected || c->error == error); + if (!expected) { + continue; + } + + expected = c->error != MUSIG_AGGNONCE; + CHECK(expected == secp256k1_musig_aggnonce_parse(CTX, &aggnonce, vector->aggnonces[c->aggnonce_index])); + if (!expected) { + continue; + } + CHECK(secp256k1_musig_nonce_process(CTX, &session, &aggnonce, vector->msgs[c->msg_index], &keyagg_cache)); + + CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, vector->pubkeys[0], sizeof(vector->pubkeys[0]))); + musig_test_set_secnonce(&secnonce, vector->secnonces[c->secnonce_index], &pubkey); + expected = c->error != MUSIG_SECNONCE; + if (expected) { + CHECK(secp256k1_musig_partial_sign(CTX, &partial_sig, &secnonce, &keypair, &keyagg_cache, &session)); + } else { + CHECK_ILLEGAL(CTX, secp256k1_musig_partial_sign(CTX, &partial_sig, &secnonce, &keypair, &keyagg_cache, &session)); + } + } + for (i = 0; i < sizeof(vector->verify_fail_case)/sizeof(vector->verify_fail_case[0]); i++) { + const struct musig_verify_fail_error_case *c = &vector->verify_fail_case[i]; + enum MUSIG_ERROR error; + secp256k1_musig_keyagg_cache keyagg_cache; + secp256k1_musig_aggnonce aggnonce; + secp256k1_musig_session session; + secp256k1_musig_partial_sig partial_sig; + enum { NUM_PUBNONCES = 3 }; + secp256k1_musig_pubnonce pubnonce[NUM_PUBNONCES]; + const secp256k1_musig_pubnonce *pubnonce_ptr[NUM_PUBNONCES]; + secp256k1_pubkey pubkey; + int expected; + size_t j; + + CHECK(NUM_PUBNONCES <= c->nonce_indices_len); + for (j = 0; j < c->nonce_indices_len; j++) { + CHECK(secp256k1_musig_pubnonce_parse(CTX, &pubnonce[j], vector->pubnonces[c->nonce_indices[j]])); + pubnonce_ptr[j] = &pubnonce[j]; + } + + CHECK(musig_vectors_keyagg_and_tweak(&error, &keyagg_cache, NULL, vector->pubkeys, NULL, c->key_indices_len, c->key_indices, 0, NULL, NULL)); + CHECK(secp256k1_musig_nonce_agg(CTX, &aggnonce, pubnonce_ptr, c->nonce_indices_len) == 1); + CHECK(secp256k1_musig_nonce_process(CTX, &session, &aggnonce, vector->msgs[c->msg_index], &keyagg_cache)); + + CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, vector->pubkeys[c->signer_index], sizeof(vector->pubkeys[0]))); + + expected = c->error != MUSIG_SIG; + CHECK(expected == secp256k1_musig_partial_sig_parse(CTX, &partial_sig, c->sig)); + if (!expected) { + continue; + } + expected = c->error != MUSIG_SIG_VERIFY; + CHECK(expected == secp256k1_musig_partial_sig_verify(CTX, &partial_sig, pubnonce, &pubkey, &keyagg_cache, &session)); + } + for (i = 0; i < sizeof(vector->verify_error_case)/sizeof(vector->verify_error_case[0]); i++) { + const struct musig_verify_fail_error_case *c = &vector->verify_error_case[i]; + enum MUSIG_ERROR error; + secp256k1_musig_keyagg_cache keyagg_cache; + secp256k1_musig_pubnonce pubnonce; + int expected; + + expected = c->error != MUSIG_PUBKEY; + CHECK(expected == musig_vectors_keyagg_and_tweak(&error, &keyagg_cache, NULL, vector->pubkeys, NULL, c->key_indices_len, c->key_indices, 0, NULL, NULL)); + CHECK(expected || c->error == error); + if (!expected) { + continue; + } + expected = c->error != MUSIG_PUBNONCE; + CHECK(expected == secp256k1_musig_pubnonce_parse(CTX, &pubnonce, vector->pubnonces[c->nonce_indices[c->signer_index]])); + } +} + +static void musig_test_vectors_tweak(void) { + size_t i; + const struct musig_tweak_vector *vector = &musig_tweak_vector; + secp256k1_pubkey pubkey; + secp256k1_musig_aggnonce aggnonce; + secp256k1_musig_secnonce secnonce; + + CHECK(secp256k1_musig_aggnonce_parse(CTX, &aggnonce, vector->aggnonce)); + CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, vector->pubkeys[0], sizeof(vector->pubkeys[0]))); + + for (i = 0; i < sizeof(vector->valid_case)/sizeof(vector->valid_case[0]); i++) { + const struct musig_tweak_case *c = &vector->valid_case[i]; + enum MUSIG_ERROR error; + secp256k1_musig_keyagg_cache keyagg_cache; + secp256k1_musig_pubnonce pubnonce; + secp256k1_musig_session session; + secp256k1_musig_partial_sig partial_sig; + secp256k1_keypair keypair; + unsigned char partial_sig32[32]; + + musig_test_set_secnonce(&secnonce, vector->secnonce, &pubkey); + + CHECK(secp256k1_keypair_create(CTX, &keypair, vector->sk)); + CHECK(musig_vectors_keyagg_and_tweak(&error, &keyagg_cache, NULL, vector->pubkeys, vector->tweaks, c->key_indices_len, c->key_indices, c->tweak_indices_len, c->tweak_indices, c->is_xonly)); + + CHECK(secp256k1_musig_nonce_process(CTX, &session, &aggnonce, vector->msg, &keyagg_cache)); + + CHECK(secp256k1_musig_partial_sign(CTX, &partial_sig, &secnonce, &keypair, &keyagg_cache, &session)); + CHECK(secp256k1_musig_partial_sig_serialize(CTX, partial_sig32, &partial_sig)); + CHECK(secp256k1_memcmp_var(partial_sig32, c->expected, sizeof(partial_sig32)) == 0); + + CHECK(secp256k1_musig_pubnonce_parse(CTX, &pubnonce, vector->pubnonces[c->nonce_indices[c->signer_index]])); + CHECK(secp256k1_musig_partial_sig_verify(CTX, &partial_sig, &pubnonce, &pubkey, &keyagg_cache, &session)); + } + for (i = 0; i < sizeof(vector->error_case)/sizeof(vector->error_case[0]); i++) { + const struct musig_tweak_case *c = &vector->error_case[i]; + enum MUSIG_ERROR error; + secp256k1_musig_keyagg_cache keyagg_cache; + CHECK(!musig_vectors_keyagg_and_tweak(&error, &keyagg_cache, NULL, vector->pubkeys, vector->tweaks, c->key_indices_len, c->key_indices, c->tweak_indices_len, c->tweak_indices, c->is_xonly)); + CHECK(error == MUSIG_TWEAK); + } +} + +static void musig_test_vectors_sigagg(void) { + size_t i, j; + const struct musig_sig_agg_vector *vector = &musig_sig_agg_vector; + + for (i = 0; i < sizeof(vector->valid_case)/sizeof(vector->valid_case[0]); i++) { + const struct musig_sig_agg_case *c = &vector->valid_case[i]; + enum MUSIG_ERROR error; + unsigned char final_sig[64]; + secp256k1_musig_keyagg_cache keyagg_cache; + unsigned char agg_pk32[32]; + secp256k1_xonly_pubkey agg_pk; + secp256k1_musig_aggnonce aggnonce; + secp256k1_musig_session session; + secp256k1_musig_partial_sig partial_sig[(sizeof(vector->psigs)/sizeof(vector->psigs[0]))]; + const secp256k1_musig_partial_sig *partial_sig_ptr[(sizeof(vector->psigs)/sizeof(vector->psigs[0]))]; + + CHECK(musig_vectors_keyagg_and_tweak(&error, &keyagg_cache, agg_pk32, vector->pubkeys, vector->tweaks, c->key_indices_len, c->key_indices, c->tweak_indices_len, c->tweak_indices, c->is_xonly)); + CHECK(secp256k1_musig_aggnonce_parse(CTX, &aggnonce, c->aggnonce)); + CHECK(secp256k1_musig_nonce_process(CTX, &session, &aggnonce, vector->msg, &keyagg_cache)); + for (j = 0; j < c->psig_indices_len; j++) { + CHECK(secp256k1_musig_partial_sig_parse(CTX, &partial_sig[j], vector->psigs[c->psig_indices[j]])); + partial_sig_ptr[j] = &partial_sig[j]; + } + + CHECK(secp256k1_musig_partial_sig_agg(CTX, final_sig, &session, partial_sig_ptr, c->psig_indices_len) == 1); + CHECK(secp256k1_memcmp_var(final_sig, c->expected, sizeof(final_sig)) == 0); + + CHECK(secp256k1_xonly_pubkey_parse(CTX, &agg_pk, agg_pk32)); + CHECK(secp256k1_schnorrsig_verify(CTX, final_sig, vector->msg, sizeof(vector->msg), &agg_pk) == 1); + } + for (i = 0; i < sizeof(vector->error_case)/sizeof(vector->error_case[0]); i++) { + const struct musig_sig_agg_case *c = &vector->error_case[i]; + secp256k1_musig_partial_sig partial_sig[(sizeof(vector->psigs)/sizeof(vector->psigs[0]))]; + for (j = 0; j < c->psig_indices_len; j++) { + int expected = c->invalid_sig_idx != (int)j; + CHECK(expected == secp256k1_musig_partial_sig_parse(CTX, &partial_sig[j], vector->psigs[c->psig_indices[j]])); + } + } +} + +/* Since the BIP doesn't provide static test vectors for nonce_gen_counter, we + * define a static test here */ +static void musig_test_static_nonce_gen_counter(void) { + secp256k1_musig_secnonce secnonce; + secp256k1_musig_pubnonce pubnonce; + unsigned char pubnonce66[66]; + secp256k1_pubkey pk; + secp256k1_keypair keypair; + uint64_t nonrepeating_cnt = 0; + unsigned char sk[32] = { + 0xEE, 0xC1, 0xCB, 0x7D, 0x1B, 0x72, 0x54, 0xC5, + 0xCA, 0xB0, 0xD9, 0xC6, 0x1A, 0xB0, 0x2E, 0x64, + 0x3D, 0x46, 0x4A, 0x59, 0xFE, 0x6C, 0x96, 0xA7, + 0xEF, 0xE8, 0x71, 0xF0, 0x7C, 0x5A, 0xEF, 0x54, + }; + unsigned char expected_secnonce[64] = { + 0x84, 0x2F, 0x13, 0x80, 0xCD, 0x17, 0xA1, 0x98, + 0xFC, 0x3D, 0xAD, 0x3B, 0x7D, 0xA7, 0x49, 0x29, + 0x41, 0xF4, 0x69, 0x76, 0xF2, 0x70, 0x2F, 0xF7, + 0xC6, 0x6F, 0x24, 0xF4, 0x72, 0x03, 0x6A, 0xF1, + 0xDA, 0x3F, 0x95, 0x2D, 0xDE, 0x4A, 0x2D, 0xA6, + 0xB6, 0x32, 0x57, 0x07, 0xCE, 0x87, 0xA4, 0xE3, + 0x61, 0x6D, 0x06, 0xFC, 0x5F, 0x81, 0xA9, 0xC9, + 0x93, 0x86, 0xD2, 0x0A, 0x99, 0xCE, 0xCF, 0x99, + }; + unsigned char expected_pubnonce[66] = { + 0x03, 0xA5, 0xB9, 0xB6, 0x90, 0x79, 0x42, 0xEA, + 0xCD, 0xDA, 0x49, 0xA3, 0x66, 0x01, 0x6E, 0xC2, + 0xE6, 0x24, 0x04, 0xA1, 0xBF, 0x4A, 0xB6, 0xD4, + 0xDB, 0x82, 0x06, 0x7B, 0xC3, 0xAD, 0xF0, 0x86, + 0xD7, 0x03, 0x32, 0x05, 0xDB, 0x9E, 0xB3, 0x4D, + 0x5C, 0x7C, 0xE0, 0x28, 0x48, 0xCA, 0xC6, 0x8A, + 0x83, 0xED, 0x73, 0xE3, 0x88, 0x34, 0x77, 0xF5, + 0x63, 0xF2, 0x3C, 0xE9, 0xA1, 0x1A, 0x77, 0x21, + 0xEC, 0x64, + }; + + CHECK(secp256k1_keypair_create(CTX, &keypair, sk)); + CHECK(secp256k1_keypair_pub(CTX, &pk, &keypair)); + CHECK(secp256k1_musig_nonce_gen_counter(CTX, &secnonce, &pubnonce, nonrepeating_cnt, &keypair, NULL, NULL, NULL) == 1); + + CHECK(secp256k1_memcmp_var(&secnonce.data[4], expected_secnonce, 2*32) == 0); + CHECK(secp256k1_memcmp_var(&secnonce.data[4+2*32], &pk, sizeof(pk)) == 0); + + CHECK(secp256k1_musig_pubnonce_serialize(CTX, pubnonce66, &pubnonce) == 1); + CHECK(secp256k1_memcmp_var(pubnonce66, expected_pubnonce, sizeof(pubnonce66)) == 0); +} + +static void run_musig_tests(void) { + int i; + + for (i = 0; i < COUNT; i++) { + musig_simple_test(); + } + musig_api_tests(); + musig_nonce_test(); + for (i = 0; i < COUNT; i++) { + /* Run multiple times to ensure that pk and nonce have different y + * parities */ + musig_tweak_test(); + } + sha256_tag_test(); + musig_test_vectors_keyagg(); + musig_test_vectors_noncegen(); + musig_test_vectors_nonceagg(); + musig_test_vectors_signverify(); + musig_test_vectors_tweak(); + musig_test_vectors_sigagg(); + + musig_test_static_nonce_gen_counter(); +} + +#endif diff --git a/src/secp256k1/src/modules/musig/vectors.h b/src/secp256k1/src/modules/musig/vectors.h new file mode 100644 index 0000000000..8407c2a69a --- /dev/null +++ b/src/secp256k1/src/modules/musig/vectors.h @@ -0,0 +1,346 @@ +/** + * Automatically generated by ./tools/test_vectors_musig2_generate.py. + * + * The test vectors for the KeySort function are included in this file. They can + * be found in src/modules/extrakeys/tests_impl.h. */ + +enum MUSIG_ERROR { + MUSIG_PUBKEY, + MUSIG_TWEAK, + MUSIG_PUBNONCE, + MUSIG_AGGNONCE, + MUSIG_SECNONCE, + MUSIG_SIG, + MUSIG_SIG_VERIFY, + MUSIG_OTHER +}; + +struct musig_key_agg_valid_test_case { + size_t key_indices_len; + size_t key_indices[4]; + unsigned char expected[32]; +}; + +struct musig_key_agg_error_test_case { + size_t key_indices_len; + size_t key_indices[4]; + size_t tweak_indices_len; + size_t tweak_indices[1]; + int is_xonly[1]; + enum MUSIG_ERROR error; +}; + +struct musig_key_agg_vector { + unsigned char pubkeys[7][33]; + unsigned char tweaks[2][32]; + struct musig_key_agg_valid_test_case valid_case[4]; + struct musig_key_agg_error_test_case error_case[5]; +}; + +static const struct musig_key_agg_vector musig_key_agg_vector = { + { + { 0x02, 0xF9, 0x30, 0x8A, 0x01, 0x92, 0x58, 0xC3, 0x10, 0x49, 0x34, 0x4F, 0x85, 0xF8, 0x9D, 0x52, 0x29, 0xB5, 0x31, 0xC8, 0x45, 0x83, 0x6F, 0x99, 0xB0, 0x86, 0x01, 0xF1, 0x13, 0xBC, 0xE0, 0x36, 0xF9 }, + { 0x03, 0xDF, 0xF1, 0xD7, 0x7F, 0x2A, 0x67, 0x1C, 0x5F, 0x36, 0x18, 0x37, 0x26, 0xDB, 0x23, 0x41, 0xBE, 0x58, 0xFE, 0xAE, 0x1D, 0xA2, 0xDE, 0xCE, 0xD8, 0x43, 0x24, 0x0F, 0x7B, 0x50, 0x2B, 0xA6, 0x59 }, + { 0x02, 0x35, 0x90, 0xA9, 0x4E, 0x76, 0x8F, 0x8E, 0x18, 0x15, 0xC2, 0xF2, 0x4B, 0x4D, 0x80, 0xA8, 0xE3, 0x14, 0x93, 0x16, 0xC3, 0x51, 0x8C, 0xE7, 0xB7, 0xAD, 0x33, 0x83, 0x68, 0xD0, 0x38, 0xCA, 0x66 }, + { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05 }, + { 0x02, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFC, 0x30 }, + { 0x04, 0xF9, 0x30, 0x8A, 0x01, 0x92, 0x58, 0xC3, 0x10, 0x49, 0x34, 0x4F, 0x85, 0xF8, 0x9D, 0x52, 0x29, 0xB5, 0x31, 0xC8, 0x45, 0x83, 0x6F, 0x99, 0xB0, 0x86, 0x01, 0xF1, 0x13, 0xBC, 0xE0, 0x36, 0xF9 }, + { 0x03, 0x93, 0x5F, 0x97, 0x2D, 0xA0, 0x13, 0xF8, 0x0A, 0xE0, 0x11, 0x89, 0x0F, 0xA8, 0x9B, 0x67, 0xA2, 0x7B, 0x7B, 0xE6, 0xCC, 0xB2, 0x4D, 0x32, 0x74, 0xD1, 0x8B, 0x2D, 0x40, 0x67, 0xF2, 0x61, 0xA9 } + }, + { + { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xBA, 0xAE, 0xDC, 0xE6, 0xAF, 0x48, 0xA0, 0x3B, 0xBF, 0xD2, 0x5E, 0x8C, 0xD0, 0x36, 0x41, 0x41 }, + { 0x25, 0x2E, 0x4B, 0xD6, 0x74, 0x10, 0xA7, 0x6C, 0xDF, 0x93, 0x3D, 0x30, 0xEA, 0xA1, 0x60, 0x82, 0x14, 0x03, 0x7F, 0x1B, 0x10, 0x5A, 0x01, 0x3E, 0xCC, 0xD3, 0xC5, 0xC1, 0x84, 0xA6, 0x11, 0x0B } + }, + { + { 3, { 0, 1, 2 }, { 0x90, 0x53, 0x9E, 0xED, 0xE5, 0x65, 0xF5, 0xD0, 0x54, 0xF3, 0x2C, 0xC0, 0xC2, 0x20, 0x12, 0x68, 0x89, 0xED, 0x1E, 0x5D, 0x19, 0x3B, 0xAF, 0x15, 0xAE, 0xF3, 0x44, 0xFE, 0x59, 0xD4, 0x61, 0x0C }}, + { 3, { 2, 1, 0 }, { 0x62, 0x04, 0xDE, 0x8B, 0x08, 0x34, 0x26, 0xDC, 0x6E, 0xAF, 0x95, 0x02, 0xD2, 0x70, 0x24, 0xD5, 0x3F, 0xC8, 0x26, 0xBF, 0x7D, 0x20, 0x12, 0x14, 0x8A, 0x05, 0x75, 0x43, 0x5D, 0xF5, 0x4B, 0x2B }}, + { 3, { 0, 0, 0 }, { 0xB4, 0x36, 0xE3, 0xBA, 0xD6, 0x2B, 0x8C, 0xD4, 0x09, 0x96, 0x9A, 0x22, 0x47, 0x31, 0xC1, 0x93, 0xD0, 0x51, 0x16, 0x2D, 0x8C, 0x5A, 0xE8, 0xB1, 0x09, 0x30, 0x61, 0x27, 0xDA, 0x3A, 0xA9, 0x35 }}, + { 4, { 0, 0, 1, 1 }, { 0x69, 0xBC, 0x22, 0xBF, 0xA5, 0xD1, 0x06, 0x30, 0x6E, 0x48, 0xA2, 0x06, 0x79, 0xDE, 0x1D, 0x73, 0x89, 0x38, 0x61, 0x24, 0xD0, 0x75, 0x71, 0xD0, 0xD8, 0x72, 0x68, 0x60, 0x28, 0xC2, 0x6A, 0x3E }}, + }, + { + { 2, { 0, 3 }, 0, { 0 }, { 0 }, MUSIG_PUBKEY }, + { 2, { 0, 4 }, 0, { 0 }, { 0 }, MUSIG_PUBKEY }, + { 2, { 5, 0 }, 0, { 0 }, { 0 }, MUSIG_PUBKEY }, + { 2, { 0, 1 }, 1, { 0 }, { 1 }, MUSIG_TWEAK }, + { 1, { 6 }, 1, { 1 }, { 0 }, MUSIG_TWEAK }, + }, +}; + +struct musig_nonce_gen_test_case { + unsigned char rand_[32]; + int has_sk; + unsigned char sk[32]; + unsigned char pk[33]; + int has_aggpk; + unsigned char aggpk[32]; + int has_msg; + unsigned char msg[32]; + int has_extra_in; + unsigned char extra_in[32]; + unsigned char expected_secnonce[97]; + unsigned char expected_pubnonce[66]; +}; + +struct musig_nonce_gen_vector { + struct musig_nonce_gen_test_case test_case[2]; +}; + +static const struct musig_nonce_gen_vector musig_nonce_gen_vector = { + { + { { 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F }, 1 , { 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02 }, { 0x02, 0x4D, 0x4B, 0x6C, 0xD1, 0x36, 0x10, 0x32, 0xCA, 0x9B, 0xD2, 0xAE, 0xB9, 0xD9, 0x00, 0xAA, 0x4D, 0x45, 0xD9, 0xEA, 0xD8, 0x0A, 0xC9, 0x42, 0x33, 0x74, 0xC4, 0x51, 0xA7, 0x25, 0x4D, 0x07, 0x66 }, 1 , { 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07 }, 1 , { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, 1 , { 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08 }, { 0xB1, 0x14, 0xE5, 0x02, 0xBE, 0xAA, 0x4E, 0x30, 0x1D, 0xD0, 0x8A, 0x50, 0x26, 0x41, 0x72, 0xC8, 0x4E, 0x41, 0x65, 0x0E, 0x6C, 0xB7, 0x26, 0xB4, 0x10, 0xC0, 0x69, 0x4D, 0x59, 0xEF, 0xFB, 0x64, 0x95, 0xB5, 0xCA, 0xF2, 0x8D, 0x04, 0x5B, 0x97, 0x3D, 0x63, 0xE3, 0xC9, 0x9A, 0x44, 0xB8, 0x07, 0xBD, 0xE3, 0x75, 0xFD, 0x6C, 0xB3, 0x9E, 0x46, 0xDC, 0x4A, 0x51, 0x17, 0x08, 0xD0, 0xE9, 0xD2, 0x02, 0x4D, 0x4B, 0x6C, 0xD1, 0x36, 0x10, 0x32, 0xCA, 0x9B, 0xD2, 0xAE, 0xB9, 0xD9, 0x00, 0xAA, 0x4D, 0x45, 0xD9, 0xEA, 0xD8, 0x0A, 0xC9, 0x42, 0x33, 0x74, 0xC4, 0x51, 0xA7, 0x25, 0x4D, 0x07, 0x66 }, { 0x02, 0xF7, 0xBE, 0x70, 0x89, 0xE8, 0x37, 0x6E, 0xB3, 0x55, 0x27, 0x23, 0x68, 0x76, 0x6B, 0x17, 0xE8, 0x8E, 0x7D, 0xB7, 0x20, 0x47, 0xD0, 0x5E, 0x56, 0xAA, 0x88, 0x1E, 0xA5, 0x2B, 0x3B, 0x35, 0xDF, 0x02, 0xC2, 0x9C, 0x80, 0x46, 0xFD, 0xD0, 0xDE, 0xD4, 0xC7, 0xE5, 0x58, 0x69, 0x13, 0x72, 0x00, 0xFB, 0xDB, 0xFE, 0x2E, 0xB6, 0x54, 0x26, 0x7B, 0x6D, 0x70, 0x13, 0x60, 0x2C, 0xAE, 0xD3, 0x11, 0x5A } }, + { { 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F }, 0 , { 0 }, { 0x02, 0xF9, 0x30, 0x8A, 0x01, 0x92, 0x58, 0xC3, 0x10, 0x49, 0x34, 0x4F, 0x85, 0xF8, 0x9D, 0x52, 0x29, 0xB5, 0x31, 0xC8, 0x45, 0x83, 0x6F, 0x99, 0xB0, 0x86, 0x01, 0xF1, 0x13, 0xBC, 0xE0, 0x36, 0xF9 }, 0 , { 0 }, 0 , { 0 }, 0 , { 0 }, { 0x89, 0xBD, 0xD7, 0x87, 0xD0, 0x28, 0x4E, 0x5E, 0x4D, 0x5F, 0xC5, 0x72, 0xE4, 0x9E, 0x31, 0x6B, 0xAB, 0x7E, 0x21, 0xE3, 0xB1, 0x83, 0x0D, 0xE3, 0x7D, 0xFE, 0x80, 0x15, 0x6F, 0xA4, 0x1A, 0x6D, 0x0B, 0x17, 0xAE, 0x8D, 0x02, 0x4C, 0x53, 0x67, 0x96, 0x99, 0xA6, 0xFD, 0x79, 0x44, 0xD9, 0xC4, 0xA3, 0x66, 0xB5, 0x14, 0xBA, 0xF4, 0x30, 0x88, 0xE0, 0x70, 0x8B, 0x10, 0x23, 0xDD, 0x28, 0x97, 0x02, 0xF9, 0x30, 0x8A, 0x01, 0x92, 0x58, 0xC3, 0x10, 0x49, 0x34, 0x4F, 0x85, 0xF8, 0x9D, 0x52, 0x29, 0xB5, 0x31, 0xC8, 0x45, 0x83, 0x6F, 0x99, 0xB0, 0x86, 0x01, 0xF1, 0x13, 0xBC, 0xE0, 0x36, 0xF9 }, { 0x02, 0xC9, 0x6E, 0x7C, 0xB1, 0xE8, 0xAA, 0x5D, 0xAC, 0x64, 0xD8, 0x72, 0x94, 0x79, 0x14, 0x19, 0x8F, 0x60, 0x7D, 0x90, 0xEC, 0xDE, 0x52, 0x00, 0xDE, 0x52, 0x97, 0x8A, 0xD5, 0xDE, 0xD6, 0x3C, 0x00, 0x02, 0x99, 0xEC, 0x51, 0x17, 0xC2, 0xD2, 0x9E, 0xDE, 0xE8, 0xA2, 0x09, 0x25, 0x87, 0xC3, 0x90, 0x9B, 0xE6, 0x94, 0xD5, 0xCF, 0xF0, 0x66, 0x7D, 0x6C, 0x02, 0xEA, 0x40, 0x59, 0xF7, 0xCD, 0x97, 0x86 } }, + }, +}; + +struct musig_nonce_agg_test_case { + size_t pnonce_indices[2]; + /* if valid case */ + unsigned char expected[66]; + /* if error case */ + int invalid_nonce_idx; +}; + +struct musig_nonce_agg_vector { + unsigned char pnonces[7][66]; + struct musig_nonce_agg_test_case valid_case[2]; + struct musig_nonce_agg_test_case error_case[3]; +}; + +static const struct musig_nonce_agg_vector musig_nonce_agg_vector = { + { + { 0x02, 0x01, 0x51, 0xC8, 0x0F, 0x43, 0x56, 0x48, 0xDF, 0x67, 0xA2, 0x2B, 0x74, 0x9C, 0xD7, 0x98, 0xCE, 0x54, 0xE0, 0x32, 0x1D, 0x03, 0x4B, 0x92, 0xB7, 0x09, 0xB5, 0x67, 0xD6, 0x0A, 0x42, 0xE6, 0x66, 0x03, 0xBA, 0x47, 0xFB, 0xC1, 0x83, 0x44, 0x37, 0xB3, 0x21, 0x2E, 0x89, 0xA8, 0x4D, 0x84, 0x25, 0xE7, 0xBF, 0x12, 0xE0, 0x24, 0x5D, 0x98, 0x26, 0x22, 0x68, 0xEB, 0xDC, 0xB3, 0x85, 0xD5, 0x06, 0x41 }, + { 0x03, 0xFF, 0x40, 0x6F, 0xFD, 0x8A, 0xDB, 0x9C, 0xD2, 0x98, 0x77, 0xE4, 0x98, 0x50, 0x14, 0xF6, 0x6A, 0x59, 0xF6, 0xCD, 0x01, 0xC0, 0xE8, 0x8C, 0xAA, 0x8E, 0x5F, 0x31, 0x66, 0xB1, 0xF6, 0x76, 0xA6, 0x02, 0x48, 0xC2, 0x64, 0xCD, 0xD5, 0x7D, 0x3C, 0x24, 0xD7, 0x99, 0x90, 0xB0, 0xF8, 0x65, 0x67, 0x4E, 0xB6, 0x2A, 0x0F, 0x90, 0x18, 0x27, 0x7A, 0x95, 0x01, 0x1B, 0x41, 0xBF, 0xC1, 0x93, 0xB8, 0x33 }, + { 0x02, 0x01, 0x51, 0xC8, 0x0F, 0x43, 0x56, 0x48, 0xDF, 0x67, 0xA2, 0x2B, 0x74, 0x9C, 0xD7, 0x98, 0xCE, 0x54, 0xE0, 0x32, 0x1D, 0x03, 0x4B, 0x92, 0xB7, 0x09, 0xB5, 0x67, 0xD6, 0x0A, 0x42, 0xE6, 0x66, 0x02, 0x79, 0xBE, 0x66, 0x7E, 0xF9, 0xDC, 0xBB, 0xAC, 0x55, 0xA0, 0x62, 0x95, 0xCE, 0x87, 0x0B, 0x07, 0x02, 0x9B, 0xFC, 0xDB, 0x2D, 0xCE, 0x28, 0xD9, 0x59, 0xF2, 0x81, 0x5B, 0x16, 0xF8, 0x17, 0x98 }, + { 0x03, 0xFF, 0x40, 0x6F, 0xFD, 0x8A, 0xDB, 0x9C, 0xD2, 0x98, 0x77, 0xE4, 0x98, 0x50, 0x14, 0xF6, 0x6A, 0x59, 0xF6, 0xCD, 0x01, 0xC0, 0xE8, 0x8C, 0xAA, 0x8E, 0x5F, 0x31, 0x66, 0xB1, 0xF6, 0x76, 0xA6, 0x03, 0x79, 0xBE, 0x66, 0x7E, 0xF9, 0xDC, 0xBB, 0xAC, 0x55, 0xA0, 0x62, 0x95, 0xCE, 0x87, 0x0B, 0x07, 0x02, 0x9B, 0xFC, 0xDB, 0x2D, 0xCE, 0x28, 0xD9, 0x59, 0xF2, 0x81, 0x5B, 0x16, 0xF8, 0x17, 0x98 }, + { 0x04, 0xFF, 0x40, 0x6F, 0xFD, 0x8A, 0xDB, 0x9C, 0xD2, 0x98, 0x77, 0xE4, 0x98, 0x50, 0x14, 0xF6, 0x6A, 0x59, 0xF6, 0xCD, 0x01, 0xC0, 0xE8, 0x8C, 0xAA, 0x8E, 0x5F, 0x31, 0x66, 0xB1, 0xF6, 0x76, 0xA6, 0x02, 0x48, 0xC2, 0x64, 0xCD, 0xD5, 0x7D, 0x3C, 0x24, 0xD7, 0x99, 0x90, 0xB0, 0xF8, 0x65, 0x67, 0x4E, 0xB6, 0x2A, 0x0F, 0x90, 0x18, 0x27, 0x7A, 0x95, 0x01, 0x1B, 0x41, 0xBF, 0xC1, 0x93, 0xB8, 0x33 }, + { 0x03, 0xFF, 0x40, 0x6F, 0xFD, 0x8A, 0xDB, 0x9C, 0xD2, 0x98, 0x77, 0xE4, 0x98, 0x50, 0x14, 0xF6, 0x6A, 0x59, 0xF6, 0xCD, 0x01, 0xC0, 0xE8, 0x8C, 0xAA, 0x8E, 0x5F, 0x31, 0x66, 0xB1, 0xF6, 0x76, 0xA6, 0x02, 0x48, 0xC2, 0x64, 0xCD, 0xD5, 0x7D, 0x3C, 0x24, 0xD7, 0x99, 0x90, 0xB0, 0xF8, 0x65, 0x67, 0x4E, 0xB6, 0x2A, 0x0F, 0x90, 0x18, 0x27, 0x7A, 0x95, 0x01, 0x1B, 0x41, 0xBF, 0xC1, 0x93, 0xB8, 0x31 }, + { 0x03, 0xFF, 0x40, 0x6F, 0xFD, 0x8A, 0xDB, 0x9C, 0xD2, 0x98, 0x77, 0xE4, 0x98, 0x50, 0x14, 0xF6, 0x6A, 0x59, 0xF6, 0xCD, 0x01, 0xC0, 0xE8, 0x8C, 0xAA, 0x8E, 0x5F, 0x31, 0x66, 0xB1, 0xF6, 0x76, 0xA6, 0x02, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFC, 0x30 } + }, + { + { { 0, 1 }, { 0x03, 0x5F, 0xE1, 0x87, 0x3B, 0x4F, 0x29, 0x67, 0xF5, 0x2F, 0xEA, 0x4A, 0x06, 0xAD, 0x5A, 0x8E, 0xCC, 0xBE, 0x9D, 0x0F, 0xD7, 0x30, 0x68, 0x01, 0x2C, 0x89, 0x4E, 0x2E, 0x87, 0xCC, 0xB5, 0x80, 0x4B, 0x02, 0x47, 0x25, 0x37, 0x73, 0x45, 0xBD, 0xE0, 0xE9, 0xC3, 0x3A, 0xF3, 0xC4, 0x3C, 0x0A, 0x29, 0xA9, 0x24, 0x9F, 0x2F, 0x29, 0x56, 0xFA, 0x8C, 0xFE, 0xB5, 0x5C, 0x85, 0x73, 0xD0, 0x26, 0x2D, 0xC8 }, 0 }, + { { 2, 3 }, { 0x03, 0x5F, 0xE1, 0x87, 0x3B, 0x4F, 0x29, 0x67, 0xF5, 0x2F, 0xEA, 0x4A, 0x06, 0xAD, 0x5A, 0x8E, 0xCC, 0xBE, 0x9D, 0x0F, 0xD7, 0x30, 0x68, 0x01, 0x2C, 0x89, 0x4E, 0x2E, 0x87, 0xCC, 0xB5, 0x80, 0x4B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 0 }, + }, + { + { { 0, 4 }, { 0 }, 1 }, + { { 5, 1 }, { 0 }, 0 }, + { { 6, 1 }, { 0 }, 0 }, + }, +}; + +/* Omit pubnonces in the test vectors because our partial signature verification + * implementation is able to accept the aggnonce directly. */ +struct musig_valid_case { + size_t key_indices_len; + size_t key_indices[3]; + size_t aggnonce_index; + size_t msg_index; + size_t signer_index; + unsigned char expected[32]; +}; + +struct musig_sign_error_case { + size_t key_indices_len; + size_t key_indices[3]; + size_t aggnonce_index; + size_t msg_index; + size_t secnonce_index; + enum MUSIG_ERROR error; +}; + +struct musig_verify_fail_error_case { + unsigned char sig[32]; + size_t key_indices_len; + size_t key_indices[3]; + size_t nonce_indices_len; + size_t nonce_indices[3]; + size_t msg_index; + size_t signer_index; + enum MUSIG_ERROR error; +}; + +struct musig_sign_verify_vector { + unsigned char sk[32]; + unsigned char pubkeys[4][33]; + unsigned char secnonces[2][194]; + unsigned char pubnonces[5][194]; + unsigned char aggnonces[5][66]; + unsigned char msgs[1][32]; + struct musig_valid_case valid_case[4]; + struct musig_sign_error_case sign_error_case[6]; + struct musig_verify_fail_error_case verify_fail_case[3]; + struct musig_verify_fail_error_case verify_error_case[2]; +}; + +static const struct musig_sign_verify_vector musig_sign_verify_vector = { + { 0x7F, 0xB9, 0xE0, 0xE6, 0x87, 0xAD, 0xA1, 0xEE, 0xBF, 0x7E, 0xCF, 0xE2, 0xF2, 0x1E, 0x73, 0xEB, 0xDB, 0x51, 0xA7, 0xD4, 0x50, 0x94, 0x8D, 0xFE, 0x8D, 0x76, 0xD7, 0xF2, 0xD1, 0x00, 0x76, 0x71 }, + { + { 0x03, 0x93, 0x5F, 0x97, 0x2D, 0xA0, 0x13, 0xF8, 0x0A, 0xE0, 0x11, 0x89, 0x0F, 0xA8, 0x9B, 0x67, 0xA2, 0x7B, 0x7B, 0xE6, 0xCC, 0xB2, 0x4D, 0x32, 0x74, 0xD1, 0x8B, 0x2D, 0x40, 0x67, 0xF2, 0x61, 0xA9 }, + { 0x02, 0xF9, 0x30, 0x8A, 0x01, 0x92, 0x58, 0xC3, 0x10, 0x49, 0x34, 0x4F, 0x85, 0xF8, 0x9D, 0x52, 0x29, 0xB5, 0x31, 0xC8, 0x45, 0x83, 0x6F, 0x99, 0xB0, 0x86, 0x01, 0xF1, 0x13, 0xBC, 0xE0, 0x36, 0xF9 }, + { 0x02, 0xDF, 0xF1, 0xD7, 0x7F, 0x2A, 0x67, 0x1C, 0x5F, 0x36, 0x18, 0x37, 0x26, 0xDB, 0x23, 0x41, 0xBE, 0x58, 0xFE, 0xAE, 0x1D, 0xA2, 0xDE, 0xCE, 0xD8, 0x43, 0x24, 0x0F, 0x7B, 0x50, 0x2B, 0xA6, 0x61 }, + { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07 } + }, + { + { 0x50, 0x8B, 0x81, 0xA6, 0x11, 0xF1, 0x00, 0xA6, 0xB2, 0xB6, 0xB2, 0x96, 0x56, 0x59, 0x08, 0x98, 0xAF, 0x48, 0x8B, 0xCF, 0x2E, 0x1F, 0x55, 0xCF, 0x22, 0xE5, 0xCF, 0xB8, 0x44, 0x21, 0xFE, 0x61, 0xFA, 0x27, 0xFD, 0x49, 0xB1, 0xD5, 0x00, 0x85, 0xB4, 0x81, 0x28, 0x5E, 0x1C, 0xA2, 0x05, 0xD5, 0x5C, 0x82, 0xCC, 0x1B, 0x31, 0xFF, 0x5C, 0xD5, 0x4A, 0x48, 0x98, 0x29, 0x35, 0x59, 0x01, 0xF7, 0x03, 0x93, 0x5F, 0x97, 0x2D, 0xA0, 0x13, 0xF8, 0x0A, 0xE0, 0x11, 0x89, 0x0F, 0xA8, 0x9B, 0x67, 0xA2, 0x7B, 0x7B, 0xE6, 0xCC, 0xB2, 0x4D, 0x32, 0x74, 0xD1, 0x8B, 0x2D, 0x40, 0x67, 0xF2, 0x61, 0xA9 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x93, 0x5F, 0x97, 0x2D, 0xA0, 0x13, 0xF8, 0x0A, 0xE0, 0x11, 0x89, 0x0F, 0xA8, 0x9B, 0x67, 0xA2, 0x7B, 0x7B, 0xE6, 0xCC, 0xB2, 0x4D, 0x32, 0x74, 0xD1, 0x8B, 0x2D, 0x40, 0x67, 0xF2, 0x61, 0xA9 } + }, + { + { 0x03, 0x37, 0xC8, 0x78, 0x21, 0xAF, 0xD5, 0x0A, 0x86, 0x44, 0xD8, 0x20, 0xA8, 0xF3, 0xE0, 0x2E, 0x49, 0x9C, 0x93, 0x18, 0x65, 0xC2, 0x36, 0x0F, 0xB4, 0x3D, 0x0A, 0x0D, 0x20, 0xDA, 0xFE, 0x07, 0xEA, 0x02, 0x87, 0xBF, 0x89, 0x1D, 0x2A, 0x6D, 0xEA, 0xEB, 0xAD, 0xC9, 0x09, 0x35, 0x2A, 0xA9, 0x40, 0x5D, 0x14, 0x28, 0xC1, 0x5F, 0x4B, 0x75, 0xF0, 0x4D, 0xAE, 0x64, 0x2A, 0x95, 0xC2, 0x54, 0x84, 0x80 }, + { 0x02, 0x79, 0xBE, 0x66, 0x7E, 0xF9, 0xDC, 0xBB, 0xAC, 0x55, 0xA0, 0x62, 0x95, 0xCE, 0x87, 0x0B, 0x07, 0x02, 0x9B, 0xFC, 0xDB, 0x2D, 0xCE, 0x28, 0xD9, 0x59, 0xF2, 0x81, 0x5B, 0x16, 0xF8, 0x17, 0x98, 0x02, 0x79, 0xBE, 0x66, 0x7E, 0xF9, 0xDC, 0xBB, 0xAC, 0x55, 0xA0, 0x62, 0x95, 0xCE, 0x87, 0x0B, 0x07, 0x02, 0x9B, 0xFC, 0xDB, 0x2D, 0xCE, 0x28, 0xD9, 0x59, 0xF2, 0x81, 0x5B, 0x16, 0xF8, 0x17, 0x98 }, + { 0x03, 0x2D, 0xE2, 0x66, 0x26, 0x28, 0xC9, 0x0B, 0x03, 0xF5, 0xE7, 0x20, 0x28, 0x4E, 0xB5, 0x2F, 0xF7, 0xD7, 0x1F, 0x42, 0x84, 0xF6, 0x27, 0xB6, 0x8A, 0x85, 0x3D, 0x78, 0xC7, 0x8E, 0x1F, 0xFE, 0x93, 0x03, 0xE4, 0xC5, 0x52, 0x4E, 0x83, 0xFF, 0xE1, 0x49, 0x3B, 0x90, 0x77, 0xCF, 0x1C, 0xA6, 0xBE, 0xB2, 0x09, 0x0C, 0x93, 0xD9, 0x30, 0x32, 0x10, 0x71, 0xAD, 0x40, 0xB2, 0xF4, 0x4E, 0x59, 0x90, 0x46 }, + { 0x02, 0x37, 0xC8, 0x78, 0x21, 0xAF, 0xD5, 0x0A, 0x86, 0x44, 0xD8, 0x20, 0xA8, 0xF3, 0xE0, 0x2E, 0x49, 0x9C, 0x93, 0x18, 0x65, 0xC2, 0x36, 0x0F, 0xB4, 0x3D, 0x0A, 0x0D, 0x20, 0xDA, 0xFE, 0x07, 0xEA, 0x03, 0x87, 0xBF, 0x89, 0x1D, 0x2A, 0x6D, 0xEA, 0xEB, 0xAD, 0xC9, 0x09, 0x35, 0x2A, 0xA9, 0x40, 0x5D, 0x14, 0x28, 0xC1, 0x5F, 0x4B, 0x75, 0xF0, 0x4D, 0xAE, 0x64, 0x2A, 0x95, 0xC2, 0x54, 0x84, 0x80 }, + { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x02, 0x87, 0xBF, 0x89, 0x1D, 0x2A, 0x6D, 0xEA, 0xEB, 0xAD, 0xC9, 0x09, 0x35, 0x2A, 0xA9, 0x40, 0x5D, 0x14, 0x28, 0xC1, 0x5F, 0x4B, 0x75, 0xF0, 0x4D, 0xAE, 0x64, 0x2A, 0x95, 0xC2, 0x54, 0x84, 0x80 } + }, + { + { 0x02, 0x84, 0x65, 0xFC, 0xF0, 0xBB, 0xDB, 0xCF, 0x44, 0x3A, 0xAB, 0xCC, 0xE5, 0x33, 0xD4, 0x2B, 0x4B, 0x5A, 0x10, 0x96, 0x6A, 0xC0, 0x9A, 0x49, 0x65, 0x5E, 0x8C, 0x42, 0xDA, 0xAB, 0x8F, 0xCD, 0x61, 0x03, 0x74, 0x96, 0xA3, 0xCC, 0x86, 0x92, 0x6D, 0x45, 0x2C, 0xAF, 0xCF, 0xD5, 0x5D, 0x25, 0x97, 0x2C, 0xA1, 0x67, 0x5D, 0x54, 0x93, 0x10, 0xDE, 0x29, 0x6B, 0xFF, 0x42, 0xF7, 0x2E, 0xEE, 0xA8, 0xC9 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x04, 0x84, 0x65, 0xFC, 0xF0, 0xBB, 0xDB, 0xCF, 0x44, 0x3A, 0xAB, 0xCC, 0xE5, 0x33, 0xD4, 0x2B, 0x4B, 0x5A, 0x10, 0x96, 0x6A, 0xC0, 0x9A, 0x49, 0x65, 0x5E, 0x8C, 0x42, 0xDA, 0xAB, 0x8F, 0xCD, 0x61, 0x03, 0x74, 0x96, 0xA3, 0xCC, 0x86, 0x92, 0x6D, 0x45, 0x2C, 0xAF, 0xCF, 0xD5, 0x5D, 0x25, 0x97, 0x2C, 0xA1, 0x67, 0x5D, 0x54, 0x93, 0x10, 0xDE, 0x29, 0x6B, 0xFF, 0x42, 0xF7, 0x2E, 0xEE, 0xA8, 0xC9 }, + { 0x02, 0x84, 0x65, 0xFC, 0xF0, 0xBB, 0xDB, 0xCF, 0x44, 0x3A, 0xAB, 0xCC, 0xE5, 0x33, 0xD4, 0x2B, 0x4B, 0x5A, 0x10, 0x96, 0x6A, 0xC0, 0x9A, 0x49, 0x65, 0x5E, 0x8C, 0x42, 0xDA, 0xAB, 0x8F, 0xCD, 0x61, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09 }, + { 0x02, 0x84, 0x65, 0xFC, 0xF0, 0xBB, 0xDB, 0xCF, 0x44, 0x3A, 0xAB, 0xCC, 0xE5, 0x33, 0xD4, 0x2B, 0x4B, 0x5A, 0x10, 0x96, 0x6A, 0xC0, 0x9A, 0x49, 0x65, 0x5E, 0x8C, 0x42, 0xDA, 0xAB, 0x8F, 0xCD, 0x61, 0x02, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFC, 0x30 } + }, + { + { 0xF9, 0x54, 0x66, 0xD0, 0x86, 0x77, 0x0E, 0x68, 0x99, 0x64, 0x66, 0x42, 0x19, 0x26, 0x6F, 0xE5, 0xED, 0x21, 0x5C, 0x92, 0xAE, 0x20, 0xBA, 0xB5, 0xC9, 0xD7, 0x9A, 0xDD, 0xDD, 0xF3, 0xC0, 0xCF } + }, + { + { 3, { 0, 1, 2 }, 0, 0, 0, { 0x01, 0x2A, 0xBB, 0xCB, 0x52, 0xB3, 0x01, 0x6A, 0xC0, 0x3A, 0xD8, 0x23, 0x95, 0xA1, 0xA4, 0x15, 0xC4, 0x8B, 0x93, 0xDE, 0xF7, 0x87, 0x18, 0xE6, 0x2A, 0x7A, 0x90, 0x05, 0x2F, 0xE2, 0x24, 0xFB }}, + { 3, { 1, 0, 2 }, 0, 0, 1, { 0x9F, 0xF2, 0xF7, 0xAA, 0xA8, 0x56, 0x15, 0x0C, 0xC8, 0x81, 0x92, 0x54, 0x21, 0x8D, 0x3A, 0xDE, 0xEB, 0x05, 0x35, 0x26, 0x90, 0x51, 0x89, 0x77, 0x24, 0xF9, 0xDB, 0x37, 0x89, 0x51, 0x3A, 0x52 }}, + { 3, { 1, 2, 0 }, 0, 0, 2, { 0xFA, 0x23, 0xC3, 0x59, 0xF6, 0xFA, 0xC4, 0xE7, 0x79, 0x6B, 0xB9, 0x3B, 0xC9, 0xF0, 0x53, 0x2A, 0x95, 0x46, 0x8C, 0x53, 0x9B, 0xA2, 0x0F, 0xF8, 0x6D, 0x7C, 0x76, 0xED, 0x92, 0x22, 0x79, 0x00 }}, + { 2, { 0, 1 }, 1, 0, 0, { 0xAE, 0x38, 0x60, 0x64, 0xB2, 0x61, 0x05, 0x40, 0x47, 0x98, 0xF7, 0x5D, 0xE2, 0xEB, 0x9A, 0xF5, 0xED, 0xA5, 0x38, 0x7B, 0x06, 0x4B, 0x83, 0xD0, 0x49, 0xCB, 0x7C, 0x5E, 0x08, 0x87, 0x95, 0x31 }}, + }, + { + { 2, { 1, 2 }, 0, 0, 0, MUSIG_PUBKEY }, + { 3, { 1, 0, 3 }, 0, 0, 0, MUSIG_PUBKEY }, + { 3, { 1, 2, 0 }, 2, 0, 0, MUSIG_AGGNONCE }, + { 3, { 1, 2, 0 }, 3, 0, 0, MUSIG_AGGNONCE }, + { 3, { 1, 2, 0 }, 4, 0, 0, MUSIG_AGGNONCE }, + { 3, { 0, 1, 2 }, 0, 0, 1, MUSIG_SECNONCE }, + }, + { + { { 0xFE, 0xD5, 0x44, 0x34, 0xAD, 0x4C, 0xFE, 0x95, 0x3F, 0xC5, 0x27, 0xDC, 0x6A, 0x5E, 0x5B, 0xE8, 0xF6, 0x23, 0x49, 0x07, 0xB7, 0xC1, 0x87, 0x55, 0x95, 0x57, 0xCE, 0x87, 0xA0, 0x54, 0x1C, 0x46 }, 3, { 0, 1, 2 }, 3, { 0, 1, 2 }, 0, 0, MUSIG_SIG_VERIFY }, + { { 0x01, 0x2A, 0xBB, 0xCB, 0x52, 0xB3, 0x01, 0x6A, 0xC0, 0x3A, 0xD8, 0x23, 0x95, 0xA1, 0xA4, 0x15, 0xC4, 0x8B, 0x93, 0xDE, 0xF7, 0x87, 0x18, 0xE6, 0x2A, 0x7A, 0x90, 0x05, 0x2F, 0xE2, 0x24, 0xFB }, 3, { 0, 1, 2 }, 3, { 0, 1, 2 }, 0, 1, MUSIG_SIG_VERIFY }, + { { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xBA, 0xAE, 0xDC, 0xE6, 0xAF, 0x48, 0xA0, 0x3B, 0xBF, 0xD2, 0x5E, 0x8C, 0xD0, 0x36, 0x41, 0x41 }, 3, { 0, 1, 2 }, 3, { 0, 1, 2 }, 0, 0, MUSIG_SIG }, + }, + { + { { 0x01, 0x2A, 0xBB, 0xCB, 0x52, 0xB3, 0x01, 0x6A, 0xC0, 0x3A, 0xD8, 0x23, 0x95, 0xA1, 0xA4, 0x15, 0xC4, 0x8B, 0x93, 0xDE, 0xF7, 0x87, 0x18, 0xE6, 0x2A, 0x7A, 0x90, 0x05, 0x2F, 0xE2, 0x24, 0xFB }, 3, { 0, 1, 2 }, 3, { 4, 1, 2 }, 0, 0, MUSIG_PUBNONCE }, + { { 0x01, 0x2A, 0xBB, 0xCB, 0x52, 0xB3, 0x01, 0x6A, 0xC0, 0x3A, 0xD8, 0x23, 0x95, 0xA1, 0xA4, 0x15, 0xC4, 0x8B, 0x93, 0xDE, 0xF7, 0x87, 0x18, 0xE6, 0x2A, 0x7A, 0x90, 0x05, 0x2F, 0xE2, 0x24, 0xFB }, 3, { 3, 1, 2 }, 3, { 0, 1, 2 }, 0, 0, MUSIG_PUBKEY }, + }, +}; + +struct musig_tweak_case { + size_t key_indices_len; + size_t key_indices[3]; + size_t nonce_indices_len; + size_t nonce_indices[3]; + size_t tweak_indices_len; + size_t tweak_indices[4]; + int is_xonly[4]; + size_t signer_index; + unsigned char expected[32]; +}; + +struct musig_tweak_vector { + unsigned char sk[32]; + unsigned char secnonce[97]; + unsigned char aggnonce[66]; + unsigned char msg[32]; + unsigned char pubkeys[3][33]; + unsigned char pubnonces[3][194]; + unsigned char tweaks[5][32]; + struct musig_tweak_case valid_case[5]; + struct musig_tweak_case error_case[1]; +}; + +static const struct musig_tweak_vector musig_tweak_vector = { + { 0x7F, 0xB9, 0xE0, 0xE6, 0x87, 0xAD, 0xA1, 0xEE, 0xBF, 0x7E, 0xCF, 0xE2, 0xF2, 0x1E, 0x73, 0xEB, 0xDB, 0x51, 0xA7, 0xD4, 0x50, 0x94, 0x8D, 0xFE, 0x8D, 0x76, 0xD7, 0xF2, 0xD1, 0x00, 0x76, 0x71 }, + { 0x50, 0x8B, 0x81, 0xA6, 0x11, 0xF1, 0x00, 0xA6, 0xB2, 0xB6, 0xB2, 0x96, 0x56, 0x59, 0x08, 0x98, 0xAF, 0x48, 0x8B, 0xCF, 0x2E, 0x1F, 0x55, 0xCF, 0x22, 0xE5, 0xCF, 0xB8, 0x44, 0x21, 0xFE, 0x61, 0xFA, 0x27, 0xFD, 0x49, 0xB1, 0xD5, 0x00, 0x85, 0xB4, 0x81, 0x28, 0x5E, 0x1C, 0xA2, 0x05, 0xD5, 0x5C, 0x82, 0xCC, 0x1B, 0x31, 0xFF, 0x5C, 0xD5, 0x4A, 0x48, 0x98, 0x29, 0x35, 0x59, 0x01, 0xF7, 0x03, 0x93, 0x5F, 0x97, 0x2D, 0xA0, 0x13, 0xF8, 0x0A, 0xE0, 0x11, 0x89, 0x0F, 0xA8, 0x9B, 0x67, 0xA2, 0x7B, 0x7B, 0xE6, 0xCC, 0xB2, 0x4D, 0x32, 0x74, 0xD1, 0x8B, 0x2D, 0x40, 0x67, 0xF2, 0x61, 0xA9 }, + { 0x02, 0x84, 0x65, 0xFC, 0xF0, 0xBB, 0xDB, 0xCF, 0x44, 0x3A, 0xAB, 0xCC, 0xE5, 0x33, 0xD4, 0x2B, 0x4B, 0x5A, 0x10, 0x96, 0x6A, 0xC0, 0x9A, 0x49, 0x65, 0x5E, 0x8C, 0x42, 0xDA, 0xAB, 0x8F, 0xCD, 0x61, 0x03, 0x74, 0x96, 0xA3, 0xCC, 0x86, 0x92, 0x6D, 0x45, 0x2C, 0xAF, 0xCF, 0xD5, 0x5D, 0x25, 0x97, 0x2C, 0xA1, 0x67, 0x5D, 0x54, 0x93, 0x10, 0xDE, 0x29, 0x6B, 0xFF, 0x42, 0xF7, 0x2E, 0xEE, 0xA8, 0xC9 }, + { 0xF9, 0x54, 0x66, 0xD0, 0x86, 0x77, 0x0E, 0x68, 0x99, 0x64, 0x66, 0x42, 0x19, 0x26, 0x6F, 0xE5, 0xED, 0x21, 0x5C, 0x92, 0xAE, 0x20, 0xBA, 0xB5, 0xC9, 0xD7, 0x9A, 0xDD, 0xDD, 0xF3, 0xC0, 0xCF }, + { + { 0x03, 0x93, 0x5F, 0x97, 0x2D, 0xA0, 0x13, 0xF8, 0x0A, 0xE0, 0x11, 0x89, 0x0F, 0xA8, 0x9B, 0x67, 0xA2, 0x7B, 0x7B, 0xE6, 0xCC, 0xB2, 0x4D, 0x32, 0x74, 0xD1, 0x8B, 0x2D, 0x40, 0x67, 0xF2, 0x61, 0xA9 }, + { 0x02, 0xF9, 0x30, 0x8A, 0x01, 0x92, 0x58, 0xC3, 0x10, 0x49, 0x34, 0x4F, 0x85, 0xF8, 0x9D, 0x52, 0x29, 0xB5, 0x31, 0xC8, 0x45, 0x83, 0x6F, 0x99, 0xB0, 0x86, 0x01, 0xF1, 0x13, 0xBC, 0xE0, 0x36, 0xF9 }, + { 0x02, 0xDF, 0xF1, 0xD7, 0x7F, 0x2A, 0x67, 0x1C, 0x5F, 0x36, 0x18, 0x37, 0x26, 0xDB, 0x23, 0x41, 0xBE, 0x58, 0xFE, 0xAE, 0x1D, 0xA2, 0xDE, 0xCE, 0xD8, 0x43, 0x24, 0x0F, 0x7B, 0x50, 0x2B, 0xA6, 0x59 } + }, + { + { 0x03, 0x37, 0xC8, 0x78, 0x21, 0xAF, 0xD5, 0x0A, 0x86, 0x44, 0xD8, 0x20, 0xA8, 0xF3, 0xE0, 0x2E, 0x49, 0x9C, 0x93, 0x18, 0x65, 0xC2, 0x36, 0x0F, 0xB4, 0x3D, 0x0A, 0x0D, 0x20, 0xDA, 0xFE, 0x07, 0xEA, 0x02, 0x87, 0xBF, 0x89, 0x1D, 0x2A, 0x6D, 0xEA, 0xEB, 0xAD, 0xC9, 0x09, 0x35, 0x2A, 0xA9, 0x40, 0x5D, 0x14, 0x28, 0xC1, 0x5F, 0x4B, 0x75, 0xF0, 0x4D, 0xAE, 0x64, 0x2A, 0x95, 0xC2, 0x54, 0x84, 0x80 }, + { 0x02, 0x79, 0xBE, 0x66, 0x7E, 0xF9, 0xDC, 0xBB, 0xAC, 0x55, 0xA0, 0x62, 0x95, 0xCE, 0x87, 0x0B, 0x07, 0x02, 0x9B, 0xFC, 0xDB, 0x2D, 0xCE, 0x28, 0xD9, 0x59, 0xF2, 0x81, 0x5B, 0x16, 0xF8, 0x17, 0x98, 0x02, 0x79, 0xBE, 0x66, 0x7E, 0xF9, 0xDC, 0xBB, 0xAC, 0x55, 0xA0, 0x62, 0x95, 0xCE, 0x87, 0x0B, 0x07, 0x02, 0x9B, 0xFC, 0xDB, 0x2D, 0xCE, 0x28, 0xD9, 0x59, 0xF2, 0x81, 0x5B, 0x16, 0xF8, 0x17, 0x98 }, + { 0x03, 0x2D, 0xE2, 0x66, 0x26, 0x28, 0xC9, 0x0B, 0x03, 0xF5, 0xE7, 0x20, 0x28, 0x4E, 0xB5, 0x2F, 0xF7, 0xD7, 0x1F, 0x42, 0x84, 0xF6, 0x27, 0xB6, 0x8A, 0x85, 0x3D, 0x78, 0xC7, 0x8E, 0x1F, 0xFE, 0x93, 0x03, 0xE4, 0xC5, 0x52, 0x4E, 0x83, 0xFF, 0xE1, 0x49, 0x3B, 0x90, 0x77, 0xCF, 0x1C, 0xA6, 0xBE, 0xB2, 0x09, 0x0C, 0x93, 0xD9, 0x30, 0x32, 0x10, 0x71, 0xAD, 0x40, 0xB2, 0xF4, 0x4E, 0x59, 0x90, 0x46 } + }, + { + { 0xE8, 0xF7, 0x91, 0xFF, 0x92, 0x25, 0xA2, 0xAF, 0x01, 0x02, 0xAF, 0xFF, 0x4A, 0x9A, 0x72, 0x3D, 0x96, 0x12, 0xA6, 0x82, 0xA2, 0x5E, 0xBE, 0x79, 0x80, 0x2B, 0x26, 0x3C, 0xDF, 0xCD, 0x83, 0xBB }, + { 0xAE, 0x2E, 0xA7, 0x97, 0xCC, 0x0F, 0xE7, 0x2A, 0xC5, 0xB9, 0x7B, 0x97, 0xF3, 0xC6, 0x95, 0x7D, 0x7E, 0x41, 0x99, 0xA1, 0x67, 0xA5, 0x8E, 0xB0, 0x8B, 0xCA, 0xFF, 0xDA, 0x70, 0xAC, 0x04, 0x55 }, + { 0xF5, 0x2E, 0xCB, 0xC5, 0x65, 0xB3, 0xD8, 0xBE, 0xA2, 0xDF, 0xD5, 0xB7, 0x5A, 0x4F, 0x45, 0x7E, 0x54, 0x36, 0x98, 0x09, 0x32, 0x2E, 0x41, 0x20, 0x83, 0x16, 0x26, 0xF2, 0x90, 0xFA, 0x87, 0xE0 }, + { 0x19, 0x69, 0xAD, 0x73, 0xCC, 0x17, 0x7F, 0xA0, 0xB4, 0xFC, 0xED, 0x6D, 0xF1, 0xF7, 0xBF, 0x99, 0x07, 0xE6, 0x65, 0xFD, 0xE9, 0xBA, 0x19, 0x6A, 0x74, 0xFE, 0xD0, 0xA3, 0xCF, 0x5A, 0xEF, 0x9D }, + { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xBA, 0xAE, 0xDC, 0xE6, 0xAF, 0x48, 0xA0, 0x3B, 0xBF, 0xD2, 0x5E, 0x8C, 0xD0, 0x36, 0x41, 0x41 } + }, + { + { 3, { 1, 2, 0 }, 3, { 1, 2, 0 }, 1, { 0 }, { 1 }, 2, { 0xE2, 0x8A, 0x5C, 0x66, 0xE6, 0x1E, 0x17, 0x8C, 0x2B, 0xA1, 0x9D, 0xB7, 0x7B, 0x6C, 0xF9, 0xF7, 0xE2, 0xF0, 0xF5, 0x6C, 0x17, 0x91, 0x8C, 0xD1, 0x31, 0x35, 0xE6, 0x0C, 0xC8, 0x48, 0xFE, 0x91 }}, + { 3, { 1, 2, 0 }, 3, { 1, 2, 0 }, 1, { 0 }, { 0 }, 2, { 0x38, 0xB0, 0x76, 0x77, 0x98, 0x25, 0x2F, 0x21, 0xBF, 0x57, 0x02, 0xC4, 0x80, 0x28, 0xB0, 0x95, 0x42, 0x83, 0x20, 0xF7, 0x3A, 0x4B, 0x14, 0xDB, 0x1E, 0x25, 0xDE, 0x58, 0x54, 0x3D, 0x2D, 0x2D }}, + { 3, { 1, 2, 0 }, 3, { 1, 2, 0 }, 2, { 0, 1 }, { 0, 1 }, 2, { 0x40, 0x8A, 0x0A, 0x21, 0xC4, 0xA0, 0xF5, 0xDA, 0xCA, 0xF9, 0x64, 0x6A, 0xD6, 0xEB, 0x6F, 0xEC, 0xD7, 0xF7, 0xA1, 0x1F, 0x03, 0xED, 0x1F, 0x48, 0xDF, 0xFF, 0x21, 0x85, 0xBC, 0x2C, 0x24, 0x08 }}, + { 3, { 1, 2, 0 }, 3, { 1, 2, 0 }, 4, { 0, 1, 2, 3 }, { 0, 0, 1, 1 }, 2, { 0x45, 0xAB, 0xD2, 0x06, 0xE6, 0x1E, 0x3D, 0xF2, 0xEC, 0x9E, 0x26, 0x4A, 0x6F, 0xEC, 0x82, 0x92, 0x14, 0x1A, 0x63, 0x3C, 0x28, 0x58, 0x63, 0x88, 0x23, 0x55, 0x41, 0xF9, 0xAD, 0xE7, 0x54, 0x35 }}, + { 3, { 1, 2, 0 }, 3, { 1, 2, 0 }, 4, { 0, 1, 2, 3 }, { 1, 0, 1, 0 }, 2, { 0xB2, 0x55, 0xFD, 0xCA, 0xC2, 0x7B, 0x40, 0xC7, 0xCE, 0x78, 0x48, 0xE2, 0xD3, 0xB7, 0xBF, 0x5E, 0xA0, 0xED, 0x75, 0x6D, 0xA8, 0x15, 0x65, 0xAC, 0x80, 0x4C, 0xCC, 0xA3, 0xE1, 0xD5, 0xD2, 0x39 }}, + }, + { + { 3, { 1, 2, 0 }, 3, { 1, 2, 0 }, 1, { 4 }, { 0 }, 2, { 0 }}, + }, +}; + +/* Omit pubnonces in the test vectors because they're only needed for + * implementations that do not directly accept an aggnonce. */ +struct musig_sig_agg_case { + size_t key_indices_len; + size_t key_indices[2]; + size_t tweak_indices_len; + size_t tweak_indices[3]; + int is_xonly[3]; + unsigned char aggnonce[66]; + size_t psig_indices_len; + size_t psig_indices[2]; + /* if valid case */ + unsigned char expected[64]; + /* if error case */ + int invalid_sig_idx; +}; + +struct musig_sig_agg_vector { + unsigned char pubkeys[4][33]; + unsigned char tweaks[3][32]; + unsigned char psigs[9][32]; + unsigned char msg[32]; + struct musig_sig_agg_case valid_case[4]; + struct musig_sig_agg_case error_case[1]; +}; + +static const struct musig_sig_agg_vector musig_sig_agg_vector = { + { + { 0x03, 0x93, 0x5F, 0x97, 0x2D, 0xA0, 0x13, 0xF8, 0x0A, 0xE0, 0x11, 0x89, 0x0F, 0xA8, 0x9B, 0x67, 0xA2, 0x7B, 0x7B, 0xE6, 0xCC, 0xB2, 0x4D, 0x32, 0x74, 0xD1, 0x8B, 0x2D, 0x40, 0x67, 0xF2, 0x61, 0xA9 }, + { 0x02, 0xD2, 0xDC, 0x6F, 0x5D, 0xF7, 0xC5, 0x6A, 0xCF, 0x38, 0xC7, 0xFA, 0x0A, 0xE7, 0xA7, 0x59, 0xAE, 0x30, 0xE1, 0x9B, 0x37, 0x35, 0x9D, 0xFD, 0xE0, 0x15, 0x87, 0x23, 0x24, 0xC7, 0xEF, 0x6E, 0x05 }, + { 0x03, 0xC7, 0xFB, 0x10, 0x1D, 0x97, 0xFF, 0x93, 0x0A, 0xCD, 0x0C, 0x67, 0x60, 0x85, 0x2E, 0xF6, 0x4E, 0x69, 0x08, 0x3D, 0xE0, 0xB0, 0x6A, 0xC6, 0x33, 0x57, 0x24, 0x75, 0x4B, 0xB4, 0xB0, 0x52, 0x2C }, + { 0x02, 0x35, 0x24, 0x33, 0xB2, 0x1E, 0x7E, 0x05, 0xD3, 0xB4, 0x52, 0xB8, 0x1C, 0xAE, 0x56, 0x6E, 0x06, 0xD2, 0xE0, 0x03, 0xEC, 0xE1, 0x6D, 0x10, 0x74, 0xAA, 0xBA, 0x42, 0x89, 0xE0, 0xE3, 0xD5, 0x81 } + }, + { + { 0xB5, 0x11, 0xDA, 0x49, 0x21, 0x82, 0xA9, 0x1B, 0x0F, 0xFB, 0x9A, 0x98, 0x02, 0x0D, 0x55, 0xF2, 0x60, 0xAE, 0x86, 0xD7, 0xEC, 0xBD, 0x03, 0x99, 0xC7, 0x38, 0x3D, 0x59, 0xA5, 0xF2, 0xAF, 0x7C }, + { 0xA8, 0x15, 0xFE, 0x04, 0x9E, 0xE3, 0xC5, 0xAA, 0xB6, 0x63, 0x10, 0x47, 0x7F, 0xBC, 0x8B, 0xCC, 0xCA, 0xC2, 0xF3, 0x39, 0x5F, 0x59, 0xF9, 0x21, 0xC3, 0x64, 0xAC, 0xD7, 0x8A, 0x2F, 0x48, 0xDC }, + { 0x75, 0x44, 0x8A, 0x87, 0x27, 0x4B, 0x05, 0x64, 0x68, 0xB9, 0x77, 0xBE, 0x06, 0xEB, 0x1E, 0x9F, 0x65, 0x75, 0x77, 0xB7, 0x32, 0x0B, 0x0A, 0x33, 0x76, 0xEA, 0x51, 0xFD, 0x42, 0x0D, 0x18, 0xA8 } + }, + { + { 0xB1, 0x5D, 0x2C, 0xD3, 0xC3, 0xD2, 0x2B, 0x04, 0xDA, 0xE4, 0x38, 0xCE, 0x65, 0x3F, 0x6B, 0x4E, 0xCF, 0x04, 0x2F, 0x42, 0xCF, 0xDE, 0xD7, 0xC4, 0x1B, 0x64, 0xAA, 0xF9, 0xB4, 0xAF, 0x53, 0xFB }, + { 0x61, 0x93, 0xD6, 0xAC, 0x61, 0xB3, 0x54, 0xE9, 0x10, 0x5B, 0xBD, 0xC8, 0x93, 0x7A, 0x34, 0x54, 0xA6, 0xD7, 0x05, 0xB6, 0xD5, 0x73, 0x22, 0xA5, 0xA4, 0x72, 0xA0, 0x2C, 0xE9, 0x9F, 0xCB, 0x64 }, + { 0x9A, 0x87, 0xD3, 0xB7, 0x9E, 0xC6, 0x72, 0x28, 0xCB, 0x97, 0x87, 0x8B, 0x76, 0x04, 0x9B, 0x15, 0xDB, 0xD0, 0x5B, 0x81, 0x58, 0xD1, 0x7B, 0x5B, 0x91, 0x14, 0xD3, 0xC2, 0x26, 0x88, 0x75, 0x05 }, + { 0x66, 0xF8, 0x2E, 0xA9, 0x09, 0x23, 0x68, 0x9B, 0x85, 0x5D, 0x36, 0xC6, 0xB7, 0xE0, 0x32, 0xFB, 0x99, 0x70, 0x30, 0x14, 0x81, 0xB9, 0x9E, 0x01, 0xCD, 0xB4, 0xD6, 0xAC, 0x7C, 0x34, 0x7A, 0x15 }, + { 0x4F, 0x5A, 0xEE, 0x41, 0x51, 0x08, 0x48, 0xA6, 0x44, 0x7D, 0xCD, 0x1B, 0xBC, 0x78, 0x45, 0x7E, 0xF6, 0x90, 0x24, 0x94, 0x4C, 0x87, 0xF4, 0x02, 0x50, 0xD3, 0xEF, 0x2C, 0x25, 0xD3, 0x3E, 0xFE }, + { 0xDD, 0xEF, 0x42, 0x7B, 0xBB, 0x84, 0x7C, 0xC0, 0x27, 0xBE, 0xFF, 0x4E, 0xDB, 0x01, 0x03, 0x81, 0x48, 0x91, 0x78, 0x32, 0x25, 0x3E, 0xBC, 0x35, 0x5F, 0xC3, 0x3F, 0x4A, 0x8E, 0x2F, 0xCC, 0xE4 }, + { 0x97, 0xB8, 0x90, 0xA2, 0x6C, 0x98, 0x1D, 0xA8, 0x10, 0x2D, 0x3B, 0xC2, 0x94, 0x15, 0x9D, 0x17, 0x1D, 0x72, 0x81, 0x0F, 0xDF, 0x7C, 0x6A, 0x69, 0x1D, 0xEF, 0x02, 0xF0, 0xF7, 0xAF, 0x3F, 0xDC }, + { 0x53, 0xFA, 0x9E, 0x08, 0xBA, 0x52, 0x43, 0xCB, 0xCB, 0x0D, 0x79, 0x7C, 0x5E, 0xE8, 0x3B, 0xC6, 0x72, 0x8E, 0x53, 0x9E, 0xB7, 0x6C, 0x2D, 0x0B, 0xF0, 0xF9, 0x71, 0xEE, 0x4E, 0x90, 0x99, 0x71 }, + { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xBA, 0xAE, 0xDC, 0xE6, 0xAF, 0x48, 0xA0, 0x3B, 0xBF, 0xD2, 0x5E, 0x8C, 0xD0, 0x36, 0x41, 0x41 } + }, + { 0x59, 0x9C, 0x67, 0xEA, 0x41, 0x0D, 0x00, 0x5B, 0x9D, 0xA9, 0x08, 0x17, 0xCF, 0x03, 0xED, 0x3B, 0x1C, 0x86, 0x8E, 0x4D, 0xA4, 0xED, 0xF0, 0x0A, 0x58, 0x80, 0xB0, 0x08, 0x2C, 0x23, 0x78, 0x69 }, + { + { 2, { 0, 1 }, 0, { 0 }, { 0 }, { 0x03, 0x41, 0x43, 0x27, 0x22, 0xC5, 0xCD, 0x02, 0x68, 0xD8, 0x29, 0xC7, 0x02, 0xCF, 0x0D, 0x1C, 0xBC, 0xE5, 0x70, 0x33, 0xEE, 0xD2, 0x01, 0xFD, 0x33, 0x51, 0x91, 0x38, 0x52, 0x27, 0xC3, 0x21, 0x0C, 0x03, 0xD3, 0x77, 0xF2, 0xD2, 0x58, 0xB6, 0x4A, 0xAD, 0xC0, 0xE1, 0x6F, 0x26, 0x46, 0x23, 0x23, 0xD7, 0x01, 0xD2, 0x86, 0x04, 0x6A, 0x2E, 0xA9, 0x33, 0x65, 0x65, 0x6A, 0xFD, 0x98, 0x75, 0x98, 0x2B }, 2, { 0, 1 }, { 0x04, 0x1D, 0xA2, 0x22, 0x23, 0xCE, 0x65, 0xC9, 0x2C, 0x9A, 0x0D, 0x6C, 0x2C, 0xAC, 0x82, 0x8A, 0xAF, 0x1E, 0xEE, 0x56, 0x30, 0x4F, 0xEC, 0x37, 0x1D, 0xDF, 0x91, 0xEB, 0xB2, 0xB9, 0xEF, 0x09, 0x12, 0xF1, 0x03, 0x80, 0x25, 0x85, 0x7F, 0xED, 0xEB, 0x3F, 0xF6, 0x96, 0xF8, 0xB9, 0x9F, 0xA4, 0xBB, 0x2C, 0x58, 0x12, 0xF6, 0x09, 0x5A, 0x2E, 0x00, 0x04, 0xEC, 0x99, 0xCE, 0x18, 0xDE, 0x1E }, 0 }, + { 2, { 0, 2 }, 0, { 0 }, { 0 }, { 0x02, 0x24, 0xAF, 0xD3, 0x6C, 0x90, 0x20, 0x84, 0x05, 0x8B, 0x51, 0xB5, 0xD3, 0x66, 0x76, 0xBB, 0xA4, 0xDC, 0x97, 0xC7, 0x75, 0x87, 0x37, 0x68, 0xE5, 0x88, 0x22, 0xF8, 0x7F, 0xE4, 0x37, 0xD7, 0x92, 0x02, 0x8C, 0xB1, 0x59, 0x29, 0x09, 0x9E, 0xEE, 0x2F, 0x5D, 0xAE, 0x40, 0x4C, 0xD3, 0x93, 0x57, 0x59, 0x1B, 0xA3, 0x2E, 0x9A, 0xF4, 0xE1, 0x62, 0xB8, 0xD3, 0xE7, 0xCB, 0x5E, 0xFE, 0x31, 0xCB, 0x20 }, 2, { 2, 3 }, { 0x10, 0x69, 0xB6, 0x7E, 0xC3, 0xD2, 0xF3, 0xC7, 0xC0, 0x82, 0x91, 0xAC, 0xCB, 0x17, 0xA9, 0xC9, 0xB8, 0xF2, 0x81, 0x9A, 0x52, 0xEB, 0x5D, 0xF8, 0x72, 0x6E, 0x17, 0xE7, 0xD6, 0xB5, 0x2E, 0x9F, 0x01, 0x80, 0x02, 0x60, 0xA7, 0xE9, 0xDA, 0xC4, 0x50, 0xF4, 0xBE, 0x52, 0x2D, 0xE4, 0xCE, 0x12, 0xBA, 0x91, 0xAE, 0xAF, 0x2B, 0x42, 0x79, 0x21, 0x9E, 0xF7, 0x4B, 0xE1, 0xD2, 0x86, 0xAD, 0xD9 }, 0 }, + { 2, { 0, 2 }, 1, { 0 }, { 0 }, { 0x02, 0x08, 0xC5, 0xC4, 0x38, 0xC7, 0x10, 0xF4, 0xF9, 0x6A, 0x61, 0xE9, 0xFF, 0x3C, 0x37, 0x75, 0x88, 0x14, 0xB8, 0xC3, 0xAE, 0x12, 0xBF, 0xEA, 0x0E, 0xD2, 0xC8, 0x7F, 0xF6, 0x95, 0x4F, 0xF1, 0x86, 0x02, 0x0B, 0x18, 0x16, 0xEA, 0x10, 0x4B, 0x4F, 0xCA, 0x2D, 0x30, 0x4D, 0x73, 0x3E, 0x0E, 0x19, 0xCE, 0xAD, 0x51, 0x30, 0x3F, 0xF6, 0x42, 0x0B, 0xFD, 0x22, 0x23, 0x35, 0xCA, 0xA4, 0x02, 0x91, 0x6D }, 2, { 4, 5 }, { 0x5C, 0x55, 0x8E, 0x1D, 0xCA, 0xDE, 0x86, 0xDA, 0x0B, 0x2F, 0x02, 0x62, 0x6A, 0x51, 0x2E, 0x30, 0xA2, 0x2C, 0xF5, 0x25, 0x5C, 0xAE, 0xA7, 0xEE, 0x32, 0xC3, 0x8E, 0x9A, 0x71, 0xA0, 0xE9, 0x14, 0x8B, 0xA6, 0xC0, 0xE6, 0xEC, 0x76, 0x83, 0xB6, 0x42, 0x20, 0xF0, 0x29, 0x86, 0x96, 0xF1, 0xB8, 0x78, 0xCD, 0x47, 0xB1, 0x07, 0xB8, 0x1F, 0x71, 0x88, 0x81, 0x2D, 0x59, 0x39, 0x71, 0xE0, 0xCC }, 0 }, + { 2, { 0, 3 }, 3, { 0, 1, 2 }, { 1, 0, 1 }, { 0x02, 0xB5, 0xAD, 0x07, 0xAF, 0xCD, 0x99, 0xB6, 0xD9, 0x2C, 0xB4, 0x33, 0xFB, 0xD2, 0xA2, 0x8F, 0xDE, 0xB9, 0x8E, 0xAE, 0x2E, 0xB0, 0x9B, 0x60, 0x14, 0xEF, 0x0F, 0x81, 0x97, 0xCD, 0x58, 0x40, 0x33, 0x02, 0xE8, 0x61, 0x69, 0x10, 0xF9, 0x29, 0x3C, 0xF6, 0x92, 0xC4, 0x9F, 0x35, 0x1D, 0xB8, 0x6B, 0x25, 0xE3, 0x52, 0x90, 0x1F, 0x0E, 0x23, 0x7B, 0xAF, 0xDA, 0x11, 0xF1, 0xC1, 0xCE, 0xF2, 0x9F, 0xFD }, 2, { 6, 7 }, { 0x83, 0x9B, 0x08, 0x82, 0x0B, 0x68, 0x1D, 0xBA, 0x8D, 0xAF, 0x4C, 0xC7, 0xB1, 0x04, 0xE8, 0xF2, 0x63, 0x8F, 0x93, 0x88, 0xF8, 0xD7, 0xA5, 0x55, 0xDC, 0x17, 0xB6, 0xE6, 0x97, 0x1D, 0x74, 0x26, 0xCE, 0x07, 0xBF, 0x6A, 0xB0, 0x1F, 0x1D, 0xB5, 0x0E, 0x4E, 0x33, 0x71, 0x92, 0x95, 0xF4, 0x09, 0x45, 0x72, 0xB7, 0x98, 0x68, 0xE4, 0x40, 0xFB, 0x3D, 0xEF, 0xD3, 0xFA, 0xC1, 0xDB, 0x58, 0x9E }, 0 }, + }, + { + { 2, { 0, 3 }, 3, { 0, 1, 2 }, { 1, 0, 1 }, { 0x02, 0xB5, 0xAD, 0x07, 0xAF, 0xCD, 0x99, 0xB6, 0xD9, 0x2C, 0xB4, 0x33, 0xFB, 0xD2, 0xA2, 0x8F, 0xDE, 0xB9, 0x8E, 0xAE, 0x2E, 0xB0, 0x9B, 0x60, 0x14, 0xEF, 0x0F, 0x81, 0x97, 0xCD, 0x58, 0x40, 0x33, 0x02, 0xE8, 0x61, 0x69, 0x10, 0xF9, 0x29, 0x3C, 0xF6, 0x92, 0xC4, 0x9F, 0x35, 0x1D, 0xB8, 0x6B, 0x25, 0xE3, 0x52, 0x90, 0x1F, 0x0E, 0x23, 0x7B, 0xAF, 0xDA, 0x11, 0xF1, 0xC1, 0xCE, 0xF2, 0x9F, 0xFD }, 2, { 7, 8 }, { 0 }, 1 }, + }, +}; +enum { MUSIG_VECTORS_MAX_PUBKEYS = 7 }; diff --git a/src/secp256k1/src/modules/schnorrsig/main_impl.h b/src/secp256k1/src/modules/schnorrsig/main_impl.h index 57f7eadd3c..82bba2f597 100644 --- a/src/secp256k1/src/modules/schnorrsig/main_impl.h +++ b/src/secp256k1/src/modules/schnorrsig/main_impl.h @@ -93,6 +93,7 @@ static int nonce_function_bip340(unsigned char *nonce32, const unsigned char *ms secp256k1_sha256_write(&sha, xonly_pk32, 32); secp256k1_sha256_write(&sha, msg, msglen); secp256k1_sha256_finalize(&sha, nonce32); + secp256k1_sha256_clear(&sha); return 1; } @@ -187,7 +188,8 @@ static int secp256k1_schnorrsig_sign_internal(const secp256k1_context* ctx, unsi secp256k1_memczero(sig64, 64, !ret); secp256k1_scalar_clear(&k); secp256k1_scalar_clear(&sk); - memset(seckey, 0, sizeof(seckey)); + secp256k1_memclear(seckey, sizeof(seckey)); + secp256k1_gej_clear(&rj); return ret; } diff --git a/src/secp256k1/src/scalar_4x64_impl.h b/src/secp256k1/src/scalar_4x64_impl.h index 4aaec85b99..807b9b70ab 100644 --- a/src/secp256k1/src/scalar_4x64_impl.h +++ b/src/secp256k1/src/scalar_4x64_impl.h @@ -29,13 +29,6 @@ #define SECP256K1_N_H_2 ((uint64_t)0xFFFFFFFFFFFFFFFFULL) #define SECP256K1_N_H_3 ((uint64_t)0x7FFFFFFFFFFFFFFFULL) -SECP256K1_INLINE static void secp256k1_scalar_clear(secp256k1_scalar *r) { - r->d[0] = 0; - r->d[1] = 0; - r->d[2] = 0; - r->d[3] = 0; -} - SECP256K1_INLINE static void secp256k1_scalar_set_int(secp256k1_scalar *r, unsigned int v) { r->d[0] = v; r->d[1] = 0; diff --git a/src/secp256k1/src/scalar_8x32_impl.h b/src/secp256k1/src/scalar_8x32_impl.h index c7d87b17d8..2610496052 100644 --- a/src/secp256k1/src/scalar_8x32_impl.h +++ b/src/secp256k1/src/scalar_8x32_impl.h @@ -38,17 +38,6 @@ #define SECP256K1_N_H_6 ((uint32_t)0xFFFFFFFFUL) #define SECP256K1_N_H_7 ((uint32_t)0x7FFFFFFFUL) -SECP256K1_INLINE static void secp256k1_scalar_clear(secp256k1_scalar *r) { - r->d[0] = 0; - r->d[1] = 0; - r->d[2] = 0; - r->d[3] = 0; - r->d[4] = 0; - r->d[5] = 0; - r->d[6] = 0; - r->d[7] = 0; -} - SECP256K1_INLINE static void secp256k1_scalar_set_int(secp256k1_scalar *r, unsigned int v) { r->d[0] = v; r->d[1] = 0; diff --git a/src/secp256k1/src/scalar_impl.h b/src/secp256k1/src/scalar_impl.h index 972d8041b0..dbb5b0a01c 100644 --- a/src/secp256k1/src/scalar_impl.h +++ b/src/secp256k1/src/scalar_impl.h @@ -27,6 +27,10 @@ static const secp256k1_scalar secp256k1_scalar_one = SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 1); static const secp256k1_scalar secp256k1_scalar_zero = SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 0); +SECP256K1_INLINE static void secp256k1_scalar_clear(secp256k1_scalar *r) { + secp256k1_memclear(r, sizeof(secp256k1_scalar)); +} + static int secp256k1_scalar_set_b32_seckey(secp256k1_scalar *r, const unsigned char *bin) { int overflow; secp256k1_scalar_set_b32(r, bin, &overflow); diff --git a/src/secp256k1/src/scalar_low_impl.h b/src/secp256k1/src/scalar_low_impl.h index 45d2f3e460..84e1a380a3 100644 --- a/src/secp256k1/src/scalar_low_impl.h +++ b/src/secp256k1/src/scalar_low_impl.h @@ -19,8 +19,6 @@ SECP256K1_INLINE static int secp256k1_scalar_is_even(const secp256k1_scalar *a) return !(*a & 1); } -SECP256K1_INLINE static void secp256k1_scalar_clear(secp256k1_scalar *r) { *r = 0; } - SECP256K1_INLINE static void secp256k1_scalar_set_int(secp256k1_scalar *r, unsigned int v) { *r = v % EXHAUSTIVE_TEST_ORDER; diff --git a/src/secp256k1/src/scratch.h b/src/secp256k1/src/scratch.h index 9dcb7581f6..6164330b39 100644 --- a/src/secp256k1/src/scratch.h +++ b/src/secp256k1/src/scratch.h @@ -21,6 +21,8 @@ typedef struct secp256k1_scratch_space_struct { size_t max_size; } secp256k1_scratch; +typedef struct secp256k1_scratch_space_struct secp256k1_scratch_space; + static secp256k1_scratch* secp256k1_scratch_create(const secp256k1_callback* error_callback, size_t max_size); static void secp256k1_scratch_destroy(const secp256k1_callback* error_callback, secp256k1_scratch* scratch); diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index 72d725a74e..a248519dfd 100644 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -220,12 +220,12 @@ void secp256k1_context_set_error_callback(secp256k1_context* ctx, void (*fun)(co ctx->error_callback.data = data; } -secp256k1_scratch_space* secp256k1_scratch_space_create(const secp256k1_context* ctx, size_t max_size) { +static secp256k1_scratch_space* secp256k1_scratch_space_create(const secp256k1_context* ctx, size_t max_size) { VERIFY_CHECK(ctx != NULL); return secp256k1_scratch_create(&ctx->error_callback, max_size); } -void secp256k1_scratch_space_destroy(const secp256k1_context *ctx, secp256k1_scratch_space* scratch) { +static void secp256k1_scratch_space_destroy(const secp256k1_context *ctx, secp256k1_scratch_space* scratch) { VERIFY_CHECK(ctx != NULL); secp256k1_scratch_destroy(&ctx->error_callback, scratch); } @@ -238,25 +238,13 @@ static SECP256K1_INLINE void secp256k1_declassify(const secp256k1_context* ctx, } static int secp256k1_pubkey_load(const secp256k1_context* ctx, secp256k1_ge* ge, const secp256k1_pubkey* pubkey) { - secp256k1_ge_storage s; - - /* We require that the secp256k1_ge_storage type is exactly 64 bytes. - * This is formally not guaranteed by the C standard, but should hold on any - * sane compiler in the real world. */ - STATIC_ASSERT(sizeof(secp256k1_ge_storage) == 64); - memcpy(&s, &pubkey->data[0], 64); - secp256k1_ge_from_storage(ge, &s); + secp256k1_ge_from_bytes(ge, pubkey->data); ARG_CHECK(!secp256k1_fe_is_zero(&ge->x)); return 1; } static void secp256k1_pubkey_save(secp256k1_pubkey* pubkey, secp256k1_ge* ge) { - secp256k1_ge_storage s; - - STATIC_ASSERT(sizeof(secp256k1_ge_storage) == 64); - VERIFY_CHECK(!secp256k1_ge_is_infinity(ge)); - secp256k1_ge_to_storage(&s, ge); - memcpy(&pubkey->data[0], &s, 64); + secp256k1_ge_to_bytes(pubkey->data, ge); } int secp256k1_ec_pubkey_parse(const secp256k1_context* ctx, secp256k1_pubkey* pubkey, const unsigned char *input, size_t inputlen) { @@ -506,11 +494,13 @@ static int nonce_function_rfc6979(unsigned char *nonce32, const unsigned char *m buffer_append(keydata, &offset, algo16, 16); } secp256k1_rfc6979_hmac_sha256_initialize(&rng, keydata, offset); - memset(keydata, 0, sizeof(keydata)); for (i = 0; i <= counter; i++) { secp256k1_rfc6979_hmac_sha256_generate(&rng, nonce32, 32); } secp256k1_rfc6979_hmac_sha256_finalize(&rng); + + secp256k1_memclear(keydata, sizeof(keydata)); + secp256k1_rfc6979_hmac_sha256_clear(&rng); return 1; } @@ -560,7 +550,7 @@ static int secp256k1_ecdsa_sign_inner(const secp256k1_context* ctx, secp256k1_sc * seckey. As a result is_sec_valid is included in ret only after ret was * used as a branching variable. */ ret &= is_sec_valid; - memset(nonce32, 0, 32); + secp256k1_memclear(nonce32, sizeof(nonce32)); secp256k1_scalar_clear(&msg); secp256k1_scalar_clear(&non); secp256k1_scalar_clear(&sec); @@ -607,6 +597,7 @@ static int secp256k1_ec_pubkey_create_helper(const secp256k1_ecmult_gen_context secp256k1_ecmult_gen(ecmult_gen_ctx, &pj, seckey_scalar); secp256k1_ge_set_gej(p, &pj); + secp256k1_gej_clear(&pj); return ret; } @@ -811,6 +802,7 @@ int secp256k1_tagged_sha256(const secp256k1_context* ctx, unsigned char *hash32, secp256k1_sha256_initialize_tagged(&sha, tag, taglen); secp256k1_sha256_write(&sha, msg, msglen); secp256k1_sha256_finalize(&sha, hash32); + secp256k1_sha256_clear(&sha); return 1; } @@ -830,6 +822,10 @@ int secp256k1_tagged_sha256(const secp256k1_context* ctx, unsigned char *hash32, # include "modules/schnorrsig/main_impl.h" #endif +#ifdef ENABLE_MODULE_MUSIG +# include "modules/musig/main_impl.h" +#endif + #ifdef ENABLE_MODULE_ELLSWIFT # include "modules/ellswift/main_impl.h" #endif diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index 70c15f870b..78533b11c2 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -37,7 +37,7 @@ #define CONDITIONAL_TEST(cnt, nam) if (COUNT < (cnt)) { printf("Skipping %s (iteration count too low)\n", nam); } else -static int COUNT = 64; +static int COUNT = 16; static secp256k1_context *CTX = NULL; static secp256k1_context *STATIC_CTX = NULL; @@ -3671,8 +3671,7 @@ static void test_ge(void) { secp256k1_fe zfi2, zfi3; secp256k1_gej_set_infinity(&gej[0]); - secp256k1_ge_clear(&ge[0]); - secp256k1_ge_set_gej_var(&ge[0], &gej[0]); + secp256k1_ge_set_infinity(&ge[0]); for (i = 0; i < runs; i++) { int j, k; secp256k1_ge g; @@ -3982,6 +3981,34 @@ static void test_add_neg_y_diff_x(void) { CHECK(secp256k1_gej_eq_ge_var(&sumj, &res)); } +static void test_ge_bytes(void) { + int i; + + for (i = 0; i < COUNT + 1; i++) { + unsigned char buf[64]; + secp256k1_ge p, q; + + if (i == 0) { + secp256k1_ge_set_infinity(&p); + } else { + testutil_random_ge_test(&p); + } + + if (!secp256k1_ge_is_infinity(&p)) { + secp256k1_ge_to_bytes(buf, &p); + + secp256k1_ge_from_bytes(&q, buf); + CHECK(secp256k1_ge_eq_var(&p, &q)); + + secp256k1_ge_from_bytes_ext(&q, buf); + CHECK(secp256k1_ge_eq_var(&p, &q)); + } + secp256k1_ge_to_bytes_ext(buf, &p); + secp256k1_ge_from_bytes_ext(&q, buf); + CHECK(secp256k1_ge_eq_var(&p, &q)); + } +} + static void run_ge(void) { int i; for (i = 0; i < COUNT * 32; i++) { @@ -3989,6 +4016,7 @@ static void run_ge(void) { } test_add_neg_y_diff_x(); test_intialized_inf(); + test_ge_bytes(); } static void test_gej_cmov(const secp256k1_gej *a, const secp256k1_gej *b) { @@ -4768,12 +4796,12 @@ static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi testutil_random_ge_test(&pt[ncount]); } - secp256k1_scalar_clear(&sc[0]); + secp256k1_scalar_set_int(&sc[0], 0); CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, 20)); - secp256k1_scalar_clear(&sc[1]); - secp256k1_scalar_clear(&sc[2]); - secp256k1_scalar_clear(&sc[3]); - secp256k1_scalar_clear(&sc[4]); + secp256k1_scalar_set_int(&sc[1], 0); + secp256k1_scalar_set_int(&sc[2], 0); + secp256k1_scalar_set_int(&sc[3], 0); + secp256k1_scalar_set_int(&sc[4], 0); CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, 6)); CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, 5)); CHECK(secp256k1_gej_is_infinity(&r)); @@ -5515,7 +5543,7 @@ static void run_ecmult_constants(void) { test_ecmult_constants_sha(1607366309u, 2048, expected32_8bit8); } - CONDITIONAL_TEST(35, "test_ecmult_constants_2bit") { + CONDITIONAL_TEST(16, "test_ecmult_constants_2bit") { test_ecmult_constants_2bit(); } } @@ -7419,6 +7447,10 @@ static void run_ecdsa_wycheproof(void) { # include "modules/schnorrsig/tests_impl.h" #endif +#ifdef ENABLE_MODULE_MUSIG +# include "modules/musig/tests_impl.h" +#endif + #ifdef ENABLE_MODULE_ELLSWIFT # include "modules/ellswift/tests_impl.h" #endif @@ -7438,6 +7470,18 @@ static void run_secp256k1_memczero_test(void) { CHECK(secp256k1_memcmp_var(buf1, buf2, sizeof(buf1)) == 0); } + +static void run_secp256k1_is_zero_array_test(void) { + unsigned char buf1[3] = {0, 1}; + unsigned char buf2[3] = {1, 0}; + + CHECK(secp256k1_is_zero_array(buf1, 0) == 1); + CHECK(secp256k1_is_zero_array(buf1, 1) == 1); + CHECK(secp256k1_is_zero_array(buf1, 2) == 0); + CHECK(secp256k1_is_zero_array(buf2, 1) == 0); + CHECK(secp256k1_is_zero_array(buf2, 2) == 0); +} + static void run_secp256k1_byteorder_tests(void) { { const uint32_t x = 0xFF03AB45; @@ -7771,12 +7815,17 @@ int main(int argc, char **argv) { run_schnorrsig_tests(); #endif +#ifdef ENABLE_MODULE_MUSIG + run_musig_tests(); +#endif + #ifdef ENABLE_MODULE_ELLSWIFT run_ellswift_tests(); #endif /* util tests */ run_secp256k1_memczero_test(); + run_secp256k1_is_zero_array_test(); run_secp256k1_byteorder_tests(); run_cmov_tests(); diff --git a/src/secp256k1/src/testutil.h b/src/secp256k1/src/testutil.h index fc56854dd3..64b3bb41c0 100644 --- a/src/secp256k1/src/testutil.h +++ b/src/secp256k1/src/testutil.h @@ -34,7 +34,7 @@ static void testutil_random_fe_magnitude(secp256k1_fe *fe, int m) { if (n == 0) { return; } - secp256k1_fe_clear(&zero); + secp256k1_fe_set_int(&zero, 0); secp256k1_fe_negate(&zero, &zero, 0); secp256k1_fe_mul_int_unchecked(&zero, n - 1); secp256k1_fe_add(fe, &zero); diff --git a/src/secp256k1/src/util.h b/src/secp256k1/src/util.h index ca73752ccc..88a4149421 100644 --- a/src/secp256k1/src/util.h +++ b/src/secp256k1/src/util.h @@ -8,11 +8,17 @@ #define SECP256K1_UTIL_H #include "../include/secp256k1.h" +#include "checkmem.h" +#include <string.h> #include <stdlib.h> #include <stdint.h> #include <stdio.h> #include <limits.h> +#if defined(_MSC_VER) +/* For SecureZeroMemory */ +#include <Windows.h> +#endif #define STR_(x) #x #define STR(x) STR_(x) @@ -192,14 +198,6 @@ static SECP256K1_INLINE void *checked_malloc(const secp256k1_callback* cb, size_ # endif #endif -#if defined(_WIN32) -# define I64FORMAT "I64d" -# define I64uFORMAT "I64u" -#else -# define I64FORMAT "lld" -# define I64uFORMAT "llu" -#endif - #if defined(__GNUC__) # define SECP256K1_GNUC_EXT __extension__ #else @@ -221,6 +219,34 @@ static SECP256K1_INLINE void secp256k1_memczero(void *s, size_t len, int flag) { } } +/* Cleanses memory to prevent leaking sensitive info. Won't be optimized out. */ +static SECP256K1_INLINE void secp256k1_memclear(void *ptr, size_t len) { +#if defined(_MSC_VER) + /* SecureZeroMemory is guaranteed not to be optimized out by MSVC. */ + SecureZeroMemory(ptr, len); +#elif defined(__GNUC__) + /* We use a memory barrier that scares the compiler away from optimizing out the memset. + * + * Quoting Adam Langley <agl@google.com> in commit ad1907fe73334d6c696c8539646c21b11178f20f + * in BoringSSL (ISC License): + * As best as we can tell, this is sufficient to break any optimisations that + * might try to eliminate "superfluous" memsets. + * This method is used in memzero_explicit() the Linux kernel, too. Its advantage is that it + * is pretty efficient, because the compiler can still implement the memset() efficently, + * just not remove it entirely. See "Dead Store Elimination (Still) Considered Harmful" by + * Yang et al. (USENIX Security 2017) for more background. + */ + memset(ptr, 0, len); + __asm__ __volatile__("" : : "r"(ptr) : "memory"); +#else + void *(*volatile const volatile_memset)(void *, int, size_t) = memset; + volatile_memset(ptr, 0, len); +#endif +#ifdef VERIFY + SECP256K1_CHECKMEM_UNDEFINE(ptr, len); +#endif +} + /** Semantics like memcmp. Variable-time. * * We use this to avoid possible compiler bugs with memcmp, e.g. @@ -239,6 +265,22 @@ static SECP256K1_INLINE int secp256k1_memcmp_var(const void *s1, const void *s2, return 0; } +/* Return 1 if all elements of array s are 0 and otherwise return 0. + * Constant-time. */ +static SECP256K1_INLINE int secp256k1_is_zero_array(const unsigned char *s, size_t len) { + unsigned char acc = 0; + int ret; + size_t i; + + for (i = 0; i < len; i++) { + acc |= s[i]; + } + ret = (acc == 0); + /* acc may contain secret values. Try to explicitly clear it. */ + secp256k1_memclear(&acc, sizeof(acc)); + return ret; +} + /** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. Both *r and *a must be initialized and non-negative.*/ static SECP256K1_INLINE void secp256k1_int_cmov(int *r, const int *a, int flag) { unsigned int mask0, mask1, r_masked, a_masked; diff --git a/src/secp256k1/tools/check-abi.sh b/src/secp256k1/tools/check-abi.sh index 94a98831e1..601a64ba96 100755 --- a/src/secp256k1/tools/check-abi.sh +++ b/src/secp256k1/tools/check-abi.sh @@ -49,7 +49,14 @@ checkout_and_build() { -DSECP256K1_BUILD_CTIME_TESTS=OFF \ -DSECP256K1_BUILD_EXAMPLES=OFF cmake --build . -j "$(nproc)" - abi-dumper src/libsecp256k1.so -o ABI.dump -lver "$2" -public-headers ../include/ + # FIXME: Just set LIBPATH to lib/libsecp256k1.so once version 0.6.0 is + # released. + if [ -f "src/libsecp256k1.so" ]; then + LIBPATH="src/libsecp256k1.so" + else + LIBPATH="lib/libsecp256k1.so" + fi + abi-dumper $LIBPATH -o ABI.dump -lver "$2" -public-headers ../include/ cd "$_orig_dir" } diff --git a/src/secp256k1/tools/test_vectors_musig2_generate.py b/src/secp256k1/tools/test_vectors_musig2_generate.py new file mode 100755 index 0000000000..97424419f3 --- /dev/null +++ b/src/secp256k1/tools/test_vectors_musig2_generate.py @@ -0,0 +1,656 @@ +#!/usr/bin/env python3 + +import sys +import json +import textwrap + +max_pubkeys = 0 + +if len(sys.argv) < 2: + print( + "This script converts BIP MuSig2 test vectors in a given directory to a C file that can be used in the test framework." + ) + print("Usage: %s <dir>" % sys.argv[0]) + sys.exit(1) + + +def hexstr_to_intarray(str): + return ", ".join([f"0x{b:02X}" for b in bytes.fromhex(str)]) + + +def create_init(name): + return """ +static const struct musig_%s_vector musig_%s_vector = { +""" % ( + name, + name, + ) + + +def init_array(key): + return textwrap.indent("{ %s },\n" % hexstr_to_intarray(data[key]), 4 * " ") + + +def init_arrays(key): + s = textwrap.indent("{\n", 4 * " ") + s += textwrap.indent( + ",\n".join(["{ %s }" % hexstr_to_intarray(x) for x in data[key]]), 8 * " " + ) + s += textwrap.indent("\n},\n", 4 * " ") + return s + + +def init_indices(array): + return " %d, { %s }" % ( + len(array), + ", ".join(map(str, array) if len(array) > 0 else "0"), + ) + + +def init_is_xonly(case): + if len(case["tweak_indices"]) > 0: + return ", ".join(map(lambda x: "1" if x else "0", case["is_xonly"])) + return "0" + + +def init_optional_expected(case): + return hexstr_to_intarray(case["expected"]) if "expected" in case else 0 + + +def init_cases(cases, f): + s = textwrap.indent("{\n", 4 * " ") + for (i, case) in enumerate(cases): + s += textwrap.indent("%s\n" % f(case), 8 * " ") + s += textwrap.indent("},\n", 4 * " ") + return s + + +def finish_init(): + return "};\n" + + +s = ( + """/** + * Automatically generated by %s. + * + * The test vectors for the KeySort function are included in this file. They can + * be found in src/modules/extrakeys/tests_impl.h. */ +""" + % sys.argv[0] +) + + +s += """ +enum MUSIG_ERROR { + MUSIG_PUBKEY, + MUSIG_TWEAK, + MUSIG_PUBNONCE, + MUSIG_AGGNONCE, + MUSIG_SECNONCE, + MUSIG_SIG, + MUSIG_SIG_VERIFY, + MUSIG_OTHER +}; +""" + +# key agg vectors +with open(sys.argv[1] + "/key_agg_vectors.json", "r") as f: + data = json.load(f) + + max_key_indices = max( + len(test_case["key_indices"]) for test_case in data["valid_test_cases"] + ) + max_tweak_indices = max( + len(test_case["tweak_indices"]) for test_case in data["error_test_cases"] + ) + num_pubkeys = len(data["pubkeys"]) + max_pubkeys = max(num_pubkeys, max_pubkeys) + num_tweaks = len(data["tweaks"]) + num_valid_cases = len(data["valid_test_cases"]) + num_error_cases = len(data["error_test_cases"]) + + # Add structures for valid and error cases + s += ( + """ +struct musig_key_agg_valid_test_case { + size_t key_indices_len; + size_t key_indices[%d]; + unsigned char expected[32]; +}; +""" + % max_key_indices + ) + s += """ +struct musig_key_agg_error_test_case { + size_t key_indices_len; + size_t key_indices[%d]; + size_t tweak_indices_len; + size_t tweak_indices[%d]; + int is_xonly[%d]; + enum MUSIG_ERROR error; +}; +""" % ( + max_key_indices, + max_tweak_indices, + max_tweak_indices, + ) + + # Add structure for entire vector + s += """ +struct musig_key_agg_vector { + unsigned char pubkeys[%d][33]; + unsigned char tweaks[%d][32]; + struct musig_key_agg_valid_test_case valid_case[%d]; + struct musig_key_agg_error_test_case error_case[%d]; +}; +""" % ( + num_pubkeys, + num_tweaks, + num_valid_cases, + num_error_cases, + ) + + s += create_init("key_agg") + # Add pubkeys and tweaks to the vector + s += init_arrays("pubkeys") + s += init_arrays("tweaks") + + # Add valid cases to the vector + s += init_cases( + data["valid_test_cases"], + lambda case: "{ %s, { %s }}," + % (init_indices(case["key_indices"]), hexstr_to_intarray(case["expected"])), + ) + + def comment_to_error(case): + comment = case["comment"] + if "public key" in comment.lower(): + return "MUSIG_PUBKEY" + elif "tweak" in comment.lower(): + return "MUSIG_TWEAK" + else: + sys.exit("Unknown error") + + # Add error cases to the vector + s += init_cases( + data["error_test_cases"], + lambda case: "{ %s, %s, { %s }, %s }," + % ( + init_indices(case["key_indices"]), + init_indices(case["tweak_indices"]), + init_is_xonly(case), + comment_to_error(case), + ), + ) + + s += finish_init() + +# nonce gen vectors +with open(sys.argv[1] + "/nonce_gen_vectors.json", "r") as f: + data = json.load(f) + + # The MuSig2 implementation only allows messages of length 32 + data["test_cases"] = list( + filter(lambda c: c["msg"] is None or len(c["msg"]) == 64, data["test_cases"]) + ) + + num_tests = len(data["test_cases"]) + + s += """ +struct musig_nonce_gen_test_case { + unsigned char rand_[32]; + int has_sk; + unsigned char sk[32]; + unsigned char pk[33]; + int has_aggpk; + unsigned char aggpk[32]; + int has_msg; + unsigned char msg[32]; + int has_extra_in; + unsigned char extra_in[32]; + unsigned char expected_secnonce[97]; + unsigned char expected_pubnonce[66]; +}; +""" + + s += ( + """ +struct musig_nonce_gen_vector { + struct musig_nonce_gen_test_case test_case[%d]; +}; +""" + % num_tests + ) + + s += create_init("nonce_gen") + + def init_array_maybe(array): + return "%d , { %s }" % ( + 0 if array is None else 1, + hexstr_to_intarray(array) if array is not None else 0, + ) + + s += init_cases( + data["test_cases"], + lambda case: "{ { %s }, %s, { %s }, %s, %s, %s, { %s }, { %s } }," + % ( + hexstr_to_intarray(case["rand_"]), + init_array_maybe(case["sk"]), + hexstr_to_intarray(case["pk"]), + init_array_maybe(case["aggpk"]), + init_array_maybe(case["msg"]), + init_array_maybe(case["extra_in"]), + hexstr_to_intarray(case["expected_secnonce"]), + hexstr_to_intarray(case["expected_pubnonce"]), + ), + ) + + s += finish_init() + +# nonce agg vectors +with open(sys.argv[1] + "/nonce_agg_vectors.json", "r") as f: + data = json.load(f) + + num_pnonces = len(data["pnonces"]) + num_valid_cases = len(data["valid_test_cases"]) + num_error_cases = len(data["error_test_cases"]) + + pnonce_indices_len = 2 + for case in data["valid_test_cases"] + data["error_test_cases"]: + assert len(case["pnonce_indices"]) == pnonce_indices_len + + # Add structures for valid and error cases + s += """ +struct musig_nonce_agg_test_case { + size_t pnonce_indices[2]; + /* if valid case */ + unsigned char expected[66]; + /* if error case */ + int invalid_nonce_idx; +}; +""" + # Add structure for entire vector + s += """ +struct musig_nonce_agg_vector { + unsigned char pnonces[%d][66]; + struct musig_nonce_agg_test_case valid_case[%d]; + struct musig_nonce_agg_test_case error_case[%d]; +}; +""" % ( + num_pnonces, + num_valid_cases, + num_error_cases, + ) + + s += create_init("nonce_agg") + s += init_arrays("pnonces") + + for cases in (data["valid_test_cases"], data["error_test_cases"]): + s += init_cases( + cases, + lambda case: "{ { %s }, { %s }, %d }," + % ( + ", ".join(map(str, case["pnonce_indices"])), + init_optional_expected(case), + case["error"]["signer"] if "error" in case else 0, + ), + ) + s += finish_init() + +# sign/verify vectors +with open(sys.argv[1] + "/sign_verify_vectors.json", "r") as f: + data = json.load(f) + + # The MuSig2 implementation only allows messages of length 32 + assert list(filter(lambda x: len(x) == 64, data["msgs"]))[0] == data["msgs"][0] + data["msgs"] = [data["msgs"][0]] + + def filter_msg32(k): + return list(filter(lambda x: x["msg_index"] == 0, data[k])) + + data["valid_test_cases"] = filter_msg32("valid_test_cases") + data["sign_error_test_cases"] = filter_msg32("sign_error_test_cases") + data["verify_error_test_cases"] = filter_msg32("verify_error_test_cases") + data["verify_fail_test_cases"] = filter_msg32("verify_fail_test_cases") + + num_pubkeys = len(data["pubkeys"]) + max_pubkeys = max(num_pubkeys, max_pubkeys) + num_secnonces = len(data["secnonces"]) + num_pubnonces = len(data["pnonces"]) + num_aggnonces = len(data["aggnonces"]) + num_msgs = len(data["msgs"]) + num_valid_cases = len(data["valid_test_cases"]) + num_sign_error_cases = len(data["sign_error_test_cases"]) + num_verify_fail_cases = len(data["verify_fail_test_cases"]) + num_verify_error_cases = len(data["verify_error_test_cases"]) + + all_cases = ( + data["valid_test_cases"] + + data["sign_error_test_cases"] + + data["verify_error_test_cases"] + + data["verify_fail_test_cases"] + ) + max_key_indices = max(len(test_case["key_indices"]) for test_case in all_cases) + max_nonce_indices = max( + len(test_case["nonce_indices"]) if "nonce_indices" in test_case else 0 + for test_case in all_cases + ) + # Add structures for valid and error cases + s += ( + """ +/* Omit pubnonces in the test vectors because our partial signature verification + * implementation is able to accept the aggnonce directly. */ +struct musig_valid_case { + size_t key_indices_len; + size_t key_indices[%d]; + size_t aggnonce_index; + size_t msg_index; + size_t signer_index; + unsigned char expected[32]; +}; +""" + % max_key_indices + ) + + s += ( + """ +struct musig_sign_error_case { + size_t key_indices_len; + size_t key_indices[%d]; + size_t aggnonce_index; + size_t msg_index; + size_t secnonce_index; + enum MUSIG_ERROR error; +}; +""" + % max_key_indices + ) + + s += """ +struct musig_verify_fail_error_case { + unsigned char sig[32]; + size_t key_indices_len; + size_t key_indices[%d]; + size_t nonce_indices_len; + size_t nonce_indices[%d]; + size_t msg_index; + size_t signer_index; + enum MUSIG_ERROR error; +}; +""" % ( + max_key_indices, + max_nonce_indices, + ) + + # Add structure for entire vector + s += """ +struct musig_sign_verify_vector { + unsigned char sk[32]; + unsigned char pubkeys[%d][33]; + unsigned char secnonces[%d][194]; + unsigned char pubnonces[%d][194]; + unsigned char aggnonces[%d][66]; + unsigned char msgs[%d][32]; + struct musig_valid_case valid_case[%d]; + struct musig_sign_error_case sign_error_case[%d]; + struct musig_verify_fail_error_case verify_fail_case[%d]; + struct musig_verify_fail_error_case verify_error_case[%d]; +}; +""" % ( + num_pubkeys, + num_secnonces, + num_pubnonces, + num_aggnonces, + num_msgs, + num_valid_cases, + num_sign_error_cases, + num_verify_fail_cases, + num_verify_error_cases, + ) + + s += create_init("sign_verify") + s += init_array("sk") + s += init_arrays("pubkeys") + s += init_arrays("secnonces") + s += init_arrays("pnonces") + s += init_arrays("aggnonces") + s += init_arrays("msgs") + + s += init_cases( + data["valid_test_cases"], + lambda case: "{ %s, %d, %d, %d, { %s }}," + % ( + init_indices(case["key_indices"]), + case["aggnonce_index"], + case["msg_index"], + case["signer_index"], + init_optional_expected(case), + ), + ) + + def sign_error(case): + comment = case["comment"] + if "pubkey" in comment or "public key" in comment: + return "MUSIG_PUBKEY" + elif "Aggregate nonce" in comment: + return "MUSIG_AGGNONCE" + elif "Secnonce" in comment: + return "MUSIG_SECNONCE" + else: + sys.exit("Unknown sign error") + + s += init_cases( + data["sign_error_test_cases"], + lambda case: "{ %s, %d, %d, %d, %s }," + % ( + init_indices(case["key_indices"]), + case["aggnonce_index"], + case["msg_index"], + case["secnonce_index"], + sign_error(case), + ), + ) + + def verify_error(case): + comment = case["comment"] + if "exceeds" in comment: + return "MUSIG_SIG" + elif "Wrong signer" in comment or "Wrong signature" in comment: + return "MUSIG_SIG_VERIFY" + elif "pubnonce" in comment: + return "MUSIG_PUBNONCE" + elif "pubkey" in comment: + return "MUSIG_PUBKEY" + else: + sys.exit("Unknown verify error") + + for cases in ("verify_fail_test_cases", "verify_error_test_cases"): + s += init_cases( + data[cases], + lambda case: "{ { %s }, %s, %s, %d, %d, %s }," + % ( + hexstr_to_intarray(case["sig"]), + init_indices(case["key_indices"]), + init_indices(case["nonce_indices"]), + case["msg_index"], + case["signer_index"], + verify_error(case), + ), + ) + + s += finish_init() + +# tweak vectors +with open(sys.argv[1] + "/tweak_vectors.json", "r") as f: + data = json.load(f) + + num_pubkeys = len(data["pubkeys"]) + max_pubkeys = max(num_pubkeys, max_pubkeys) + num_pubnonces = len(data["pnonces"]) + num_tweaks = len(data["tweaks"]) + num_valid_cases = len(data["valid_test_cases"]) + num_error_cases = len(data["error_test_cases"]) + + all_cases = data["valid_test_cases"] + data["error_test_cases"] + max_key_indices = max(len(test_case["key_indices"]) for test_case in all_cases) + max_tweak_indices = max(len(test_case["tweak_indices"]) for test_case in all_cases) + max_nonce_indices = max(len(test_case["nonce_indices"]) for test_case in all_cases) + # Add structures for valid and error cases + s += """ +struct musig_tweak_case { + size_t key_indices_len; + size_t key_indices[%d]; + size_t nonce_indices_len; + size_t nonce_indices[%d]; + size_t tweak_indices_len; + size_t tweak_indices[%d]; + int is_xonly[%d]; + size_t signer_index; + unsigned char expected[32]; +}; +""" % ( + max_key_indices, + max_nonce_indices, + max_tweak_indices, + max_tweak_indices, + ) + + # Add structure for entire vector + s += """ +struct musig_tweak_vector { + unsigned char sk[32]; + unsigned char secnonce[97]; + unsigned char aggnonce[66]; + unsigned char msg[32]; + unsigned char pubkeys[%d][33]; + unsigned char pubnonces[%d][194]; + unsigned char tweaks[%d][32]; + struct musig_tweak_case valid_case[%d]; + struct musig_tweak_case error_case[%d]; +}; +""" % ( + num_pubkeys, + num_pubnonces, + num_tweaks, + num_valid_cases, + num_error_cases, + ) + s += create_init("tweak") + s += init_array("sk") + s += init_array("secnonce") + s += init_array("aggnonce") + s += init_array("msg") + s += init_arrays("pubkeys") + s += init_arrays("pnonces") + s += init_arrays("tweaks") + + s += init_cases( + data["valid_test_cases"], + lambda case: "{ %s, %s, %s, { %s }, %d, { %s }}," + % ( + init_indices(case["key_indices"]), + init_indices(case["nonce_indices"]), + init_indices(case["tweak_indices"]), + init_is_xonly(case), + case["signer_index"], + init_optional_expected(case), + ), + ) + + s += init_cases( + data["error_test_cases"], + lambda case: "{ %s, %s, %s, { %s }, %d, { %s }}," + % ( + init_indices(case["key_indices"]), + init_indices(case["nonce_indices"]), + init_indices(case["tweak_indices"]), + init_is_xonly(case), + case["signer_index"], + init_optional_expected(case), + ), + ) + + s += finish_init() + +# sigagg vectors +with open(sys.argv[1] + "/sig_agg_vectors.json", "r") as f: + data = json.load(f) + + num_pubkeys = len(data["pubkeys"]) + max_pubkeys = max(num_pubkeys, max_pubkeys) + num_tweaks = len(data["tweaks"]) + num_psigs = len(data["psigs"]) + num_valid_cases = len(data["valid_test_cases"]) + num_error_cases = len(data["error_test_cases"]) + + all_cases = data["valid_test_cases"] + data["error_test_cases"] + max_key_indices = max(len(test_case["key_indices"]) for test_case in all_cases) + max_tweak_indices = max(len(test_case["tweak_indices"]) for test_case in all_cases) + max_psig_indices = max(len(test_case["psig_indices"]) for test_case in all_cases) + + # Add structures for valid and error cases + s += """ +/* Omit pubnonces in the test vectors because they're only needed for + * implementations that do not directly accept an aggnonce. */ +struct musig_sig_agg_case { + size_t key_indices_len; + size_t key_indices[%d]; + size_t tweak_indices_len; + size_t tweak_indices[%d]; + int is_xonly[%d]; + unsigned char aggnonce[66]; + size_t psig_indices_len; + size_t psig_indices[%d]; + /* if valid case */ + unsigned char expected[64]; + /* if error case */ + int invalid_sig_idx; +}; +""" % ( + max_key_indices, + max_tweak_indices, + max_tweak_indices, + max_psig_indices, + ) + + # Add structure for entire vector + s += """ +struct musig_sig_agg_vector { + unsigned char pubkeys[%d][33]; + unsigned char tweaks[%d][32]; + unsigned char psigs[%d][32]; + unsigned char msg[32]; + struct musig_sig_agg_case valid_case[%d]; + struct musig_sig_agg_case error_case[%d]; +}; +""" % ( + num_pubkeys, + num_tweaks, + num_psigs, + num_valid_cases, + num_error_cases, + ) + + s += create_init("sig_agg") + s += init_arrays("pubkeys") + s += init_arrays("tweaks") + s += init_arrays("psigs") + s += init_array("msg") + + for cases in (data["valid_test_cases"], data["error_test_cases"]): + s += init_cases( + cases, + lambda case: "{ %s, %s, { %s }, { %s }, %s, { %s }, %d }," + % ( + init_indices(case["key_indices"]), + init_indices(case["tweak_indices"]), + init_is_xonly(case), + hexstr_to_intarray(case["aggnonce"]), + init_indices(case["psig_indices"]), + init_optional_expected(case), + case["error"]["signer"] if "error" in case else 0, + ), + ) + s += finish_init() +s += "enum { MUSIG_VECTORS_MAX_PUBKEYS = %d };" % max_pubkeys +print(s) diff --git a/src/serialize.h b/src/serialize.h index 2af998f3c5..6384c95dba 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -264,6 +264,7 @@ template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_wri template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); } template <typename Stream, BasicByte B, int N> void Serialize(Stream& s, const B (&a)[N]) { s.write(MakeByteSpan(a)); } template <typename Stream, BasicByte B, std::size_t N> void Serialize(Stream& s, const std::array<B, N>& a) { s.write(MakeByteSpan(a)); } +template <typename Stream, BasicByte B, std::size_t N> void Serialize(Stream& s, std::span<B, N> span) { s.write(std::as_bytes(span)); } template <typename Stream, BasicByte B> void Serialize(Stream& s, Span<B> span) { s.write(AsBytes(span)); } template <typename Stream, CharNotInt8 V> void Unserialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t @@ -278,6 +279,7 @@ template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); } template <typename Stream, BasicByte B, int N> void Unserialize(Stream& s, B (&a)[N]) { s.read(MakeWritableByteSpan(a)); } template <typename Stream, BasicByte B, std::size_t N> void Unserialize(Stream& s, std::array<B, N>& a) { s.read(MakeWritableByteSpan(a)); } +template <typename Stream, BasicByte B, std::size_t N> void Unserialize(Stream& s, std::span<B, N> span) { s.read(std::as_writable_bytes(span)); } template <typename Stream, BasicByte B> void Unserialize(Stream& s, Span<B> span) { s.read(AsWritableBytes(span)); } template <typename Stream> inline void Serialize(Stream& s, bool a) { uint8_t f = a; ser_writedata8(s, f); } diff --git a/src/span.h b/src/span.h index 3c5028f0b7..93718f6189 100644 --- a/src/span.h +++ b/src/span.h @@ -248,9 +248,8 @@ template <typename T> T& SpanPopBack(Span<T>& span) { size_t size = span.size(); - ASSERT_IF_DEBUG(size > 0); - T& back = span[size - 1]; - span = Span<T>(span.data(), size - 1); + T& back = span.back(); + span = span.first(size - 1); return back; } diff --git a/src/streams.cpp b/src/streams.cpp index cdd36a86fe..cd496ee2be 100644 --- a/src/streams.cpp +++ b/src/streams.cpp @@ -2,23 +2,32 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or https://opensource.org/license/mit/. +#include <memusage.h> #include <span.h> #include <streams.h> +#include <util/fs_helpers.h> #include <array> +AutoFile::AutoFile(std::FILE* file, std::vector<std::byte> data_xor) + : m_file{file}, m_xor{std::move(data_xor)} +{ + if (!IsNull()) { + auto pos{std::ftell(m_file)}; + if (pos >= 0) m_position = pos; + } +} + std::size_t AutoFile::detail_fread(Span<std::byte> dst) { if (!m_file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr"); - if (m_xor.empty()) { - return std::fread(dst.data(), 1, dst.size(), m_file); - } else { - const auto init_pos{std::ftell(m_file)}; - if (init_pos < 0) throw std::ios_base::failure("AutoFile::read: ftell failed"); - std::size_t ret{std::fread(dst.data(), 1, dst.size(), m_file)}; - util::Xor(dst.subspan(0, ret), m_xor, init_pos); - return ret; + size_t ret = std::fread(dst.data(), 1, dst.size(), m_file); + if (!m_xor.empty()) { + if (!m_position.has_value()) throw std::ios_base::failure("AutoFile::read: position unknown"); + util::Xor(dst.subspan(0, ret), m_xor, *m_position); } + if (m_position.has_value()) *m_position += ret; + return ret; } void AutoFile::seek(int64_t offset, int origin) @@ -29,18 +38,23 @@ void AutoFile::seek(int64_t offset, int origin) if (std::fseek(m_file, offset, origin) != 0) { throw std::ios_base::failure(feof() ? "AutoFile::seek: end of file" : "AutoFile::seek: fseek failed"); } + if (origin == SEEK_SET) { + m_position = offset; + } else if (origin == SEEK_CUR && m_position.has_value()) { + *m_position += offset; + } else { + int64_t r{std::ftell(m_file)}; + if (r < 0) { + throw std::ios_base::failure("AutoFile::seek: ftell failed"); + } + m_position = r; + } } int64_t AutoFile::tell() { - if (IsNull()) { - throw std::ios_base::failure("AutoFile::tell: file handle is nullptr"); - } - int64_t r{std::ftell(m_file)}; - if (r < 0) { - throw std::ios_base::failure("AutoFile::tell: ftell failed"); - } - return r; + if (!m_position.has_value()) throw std::ios_base::failure("AutoFile::tell: position unknown"); + return *m_position; } void AutoFile::read(Span<std::byte> dst) @@ -60,6 +74,7 @@ void AutoFile::ignore(size_t nSize) throw std::ios_base::failure(feof() ? "AutoFile::ignore: end of file" : "AutoFile::ignore: fread failed"); } nSize -= nNow; + if (m_position.has_value()) *m_position += nNow; } } @@ -70,19 +85,34 @@ void AutoFile::write(Span<const std::byte> src) if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) { throw std::ios_base::failure("AutoFile::write: write failed"); } + if (m_position.has_value()) *m_position += src.size(); } else { - auto current_pos{std::ftell(m_file)}; - if (current_pos < 0) throw std::ios_base::failure("AutoFile::write: ftell failed"); + if (!m_position.has_value()) throw std::ios_base::failure("AutoFile::write: position unknown"); std::array<std::byte, 4096> buf; while (src.size() > 0) { auto buf_now{Span{buf}.first(std::min<size_t>(src.size(), buf.size()))}; std::copy(src.begin(), src.begin() + buf_now.size(), buf_now.begin()); - util::Xor(buf_now, m_xor, current_pos); + util::Xor(buf_now, m_xor, *m_position); if (std::fwrite(buf_now.data(), 1, buf_now.size(), m_file) != buf_now.size()) { throw std::ios_base::failure{"XorFile::write: failed"}; } src = src.subspan(buf_now.size()); - current_pos += buf_now.size(); + *m_position += buf_now.size(); } } } + +bool AutoFile::Commit() +{ + return ::FileCommit(m_file); +} + +bool AutoFile::Truncate(unsigned size) +{ + return ::TruncateFile(m_file, size); +} + +size_t DataStream::GetMemoryUsage() const noexcept +{ + return sizeof(*this) + memusage::DynamicUsage(vch); +} diff --git a/src/streams.h b/src/streams.h index c2a9dea287..e5316da7e7 100644 --- a/src/streams.h +++ b/src/streams.h @@ -79,7 +79,7 @@ public: memcpy(vchData.data() + nPos, src.data(), nOverwrite); } if (nOverwrite < src.size()) { - vchData.insert(vchData.end(), UCharCast(src.data()) + nOverwrite, UCharCast(src.end())); + vchData.insert(vchData.end(), UCharCast(src.data()) + nOverwrite, UCharCast(src.data() + src.size())); } nPos += src.size(); } @@ -277,6 +277,9 @@ public: { util::Xor(MakeWritableByteSpan(*this), MakeByteSpan(key)); } + + /** Compute total memory usage of this object (own memory + any dynamic memory). */ + size_t GetMemoryUsage() const noexcept; }; template <typename IStream> @@ -390,9 +393,10 @@ class AutoFile protected: std::FILE* m_file; std::vector<std::byte> m_xor; + std::optional<int64_t> m_position; public: - explicit AutoFile(std::FILE* file, std::vector<std::byte> data_xor={}) : m_file{file}, m_xor{std::move(data_xor)} {} + explicit AutoFile(std::FILE* file, std::vector<std::byte> data_xor={}); ~AutoFile() { fclose(); } @@ -419,12 +423,6 @@ public: return ret; } - /** Get wrapped FILE* without transfer of ownership. - * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the - * AutoFile outlives use of the passed pointer. - */ - std::FILE* Get() const { return m_file; } - /** Return true if the wrapped FILE* is nullptr, false otherwise. */ bool IsNull() const { return m_file == nullptr; } @@ -435,9 +433,18 @@ public: /** Implementation detail, only used internally. */ std::size_t detail_fread(Span<std::byte> dst); + /** Wrapper around fseek(). Will throw if seeking is not possible. */ void seek(int64_t offset, int origin); + + /** Find position within the file. Will throw if unknown. */ int64_t tell(); + /** Wrapper around FileCommit(). */ + bool Commit(); + + /** Wrapper around TruncateFile(). */ + bool Truncate(unsigned size); + // // Stream subset // diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index 4395567722..e7ffc9e201 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -74,7 +74,7 @@ secure_unique_ptr<T> make_secure_unique(Args&&... as) // initialize in place, and return as secure_unique_ptr try { - return secure_unique_ptr<T>(new (p) T(std::forward(as)...)); + return secure_unique_ptr<T>(new (p) T(std::forward<Args>(as)...)); } catch (...) { secure_allocator<T>().deallocate(p, 1); throw; diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 2c9957117c..859b913206 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -18,7 +18,6 @@ generate_header_from_raw(data/asmap.raw test::data) # SOURCES property is processed to gather test suite macros. add_executable(test_bitcoin main.cpp - $<TARGET_OBJECTS:bitcoin_consensus> ${CMAKE_CURRENT_BINARY_DIR}/data/asmap.raw.h ${CMAKE_CURRENT_BINARY_DIR}/data/base58_encode_decode.json.h ${CMAKE_CURRENT_BINARY_DIR}/data/bip341_wallet_vectors.json.h @@ -125,6 +124,7 @@ add_executable(test_bitcoin torcontrol_tests.cpp transaction_tests.cpp translation_tests.cpp + txdownload_tests.cpp txindex_tests.cpp txpackage_tests.cpp txreconciliation_tests.cpp @@ -135,6 +135,7 @@ add_executable(test_bitcoin util_string_tests.cpp util_tests.cpp util_threadnames_tests.cpp + util_trace_tests.cpp validation_block_tests.cpp validation_chainstate_tests.cpp validation_chainstatemanager_tests.cpp @@ -149,10 +150,11 @@ target_link_libraries(test_bitcoin test_util bitcoin_cli bitcoin_node + bitcoin_consensus minisketch secp256k1 Boost::headers - $<TARGET_NAME_IF_EXISTS:libevent::libevent> + libevent::extra ) if(ENABLE_WALLET) @@ -160,14 +162,6 @@ if(ENABLE_WALLET) endif() if(WITH_MULTIPROCESS) - add_library(bitcoin_ipc_test STATIC EXCLUDE_FROM_ALL - ipc_test.cpp - ) - - target_capnp_sources(bitcoin_ipc_test ${PROJECT_SOURCE_DIR} - ipc_test.capnp - ) - target_link_libraries(bitcoin_ipc_test PRIVATE core_interface diff --git a/src/test/README.md b/src/test/README.md index f71e0771bf..7e0f245ee8 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -10,14 +10,19 @@ The build system is set up to compile an executable called `test_bitcoin` that runs all of the unit tests. The main source file for the test library is found in `util/setup_common.cpp`. +The examples in this document assume the build directory is named +`build`. You'll need to adapt them if you named it differently. + ### Compiling/running unit tests Unit tests will be automatically compiled if dependencies were met during the generation of the Bitcoin Core build system and tests weren't explicitly disabled. -Assuming the build directory is named `build`, the unit tests can be run -with `ctest --test-dir build`, which includes unit tests from subtrees. +The unit tests can be run with `ctest --test-dir build`, which includes unit +tests from subtrees. + +Run `test_bitcoin --list_content` for the full list of tests. To run the unit tests manually, launch `build/src/test/test_bitcoin`. To recompile after a test file was modified, run `cmake --build build` and then run the test again. If you @@ -35,34 +40,45 @@ the `src/qt/test/test_main.cpp` file. ### Running individual tests -`test_bitcoin` accepts the command line arguments from the boost framework. -For example, to run just the `getarg_tests` suite of tests: +The `test_bitcoin` runner accepts command line arguments from the Boost +framework. To see the list of arguments that may be passed, run: + +``` +test_bitcoin --help +``` + +For example, to run only the tests in the `getarg_tests` file, with full logging: ```bash build/src/test/test_bitcoin --log_level=all --run_test=getarg_tests ``` -`log_level` controls the verbosity of the test framework, which logs when a -test case is entered, for example. +or -`test_bitcoin` also accepts some of the command line arguments accepted by -`bitcoind`. Use `--` to separate these sets of arguments: +```bash +build/src/test/test_bitcoin -l all -t getarg_tests +``` + +or to run only the doubledash test in `getarg_tests` ```bash -build/src/test/test_bitcoin --log_level=all --run_test=getarg_tests -- -printtoconsole=1 +build/src/test/test_bitcoin --run_test=getarg_tests/doubledash ``` -The `-printtoconsole=1` after the two dashes sends debug logging, which -normally goes only to `debug.log` within the data directory, also to the -standard terminal output. +The `--log_level=` (or `-l`) argument controls the verbosity of the test output. -... or to run just the doubledash test: +The `test_bitcoin` runner also accepts some of the command line arguments accepted by +`bitcoind`. Use `--` to separate these sets of arguments: ```bash -build/src/test/test_bitcoin --run_test=getarg_tests/doubledash +build/src/test/test_bitcoin --log_level=all --run_test=getarg_tests -- -printtoconsole=1 ``` -`test_bitcoin` creates a temporary working (data) directory with a randomly +The `-printtoconsole=1` after the two dashes sends debug logging, which +normally goes only to `debug.log` within the data directory, to the +standard terminal output as well. + +Running `test_bitcoin` creates a temporary working (data) directory with a randomly generated pathname within `test_common bitcoin/`, which in turn is within the system's temporary directory (see [`temp_directory_path`](https://en.cppreference.com/w/cpp/filesystem/temp_directory_path)). @@ -97,8 +113,6 @@ If you run an entire test suite, such as `--run_test=getarg_tests`, or all the t (by not specifying `--run_test`), a separate directory will be created for each individual test. -Run `test_bitcoin --help` for the full list of tests. - ### Adding test cases To add a new unit test file to our test suite, you need diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index da6ff77924..ddb1d5b43f 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -196,21 +196,21 @@ BOOST_AUTO_TEST_CASE(addrman_select) BOOST_AUTO_TEST_CASE(addrman_select_by_network) { auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); - BOOST_CHECK(!addrman->Select(/*new_only=*/true, NET_IPV4).first.IsValid()); - BOOST_CHECK(!addrman->Select(/*new_only=*/false, NET_IPV4).first.IsValid()); + BOOST_CHECK(!addrman->Select(/*new_only=*/true, {NET_IPV4}).first.IsValid()); + BOOST_CHECK(!addrman->Select(/*new_only=*/false, {NET_IPV4}).first.IsValid()); // add ipv4 address to the new table CNetAddr source = ResolveIP("252.2.2.2"); CService addr1 = ResolveService("250.1.1.1", 8333); BOOST_CHECK(addrman->Add({CAddress(addr1, NODE_NONE)}, source)); - BOOST_CHECK(addrman->Select(/*new_only=*/true, NET_IPV4).first == addr1); - BOOST_CHECK(addrman->Select(/*new_only=*/false, NET_IPV4).first == addr1); - BOOST_CHECK(!addrman->Select(/*new_only=*/false, NET_IPV6).first.IsValid()); - BOOST_CHECK(!addrman->Select(/*new_only=*/false, NET_ONION).first.IsValid()); - BOOST_CHECK(!addrman->Select(/*new_only=*/false, NET_I2P).first.IsValid()); - BOOST_CHECK(!addrman->Select(/*new_only=*/false, NET_CJDNS).first.IsValid()); - BOOST_CHECK(!addrman->Select(/*new_only=*/true, NET_CJDNS).first.IsValid()); + BOOST_CHECK(addrman->Select(/*new_only=*/true, {NET_IPV4}).first == addr1); + BOOST_CHECK(addrman->Select(/*new_only=*/false, {NET_IPV4}).first == addr1); + BOOST_CHECK(!addrman->Select(/*new_only=*/false, {NET_IPV6}).first.IsValid()); + BOOST_CHECK(!addrman->Select(/*new_only=*/false, {NET_ONION}).first.IsValid()); + BOOST_CHECK(!addrman->Select(/*new_only=*/false, {NET_I2P}).first.IsValid()); + BOOST_CHECK(!addrman->Select(/*new_only=*/false, {NET_CJDNS}).first.IsValid()); + BOOST_CHECK(!addrman->Select(/*new_only=*/true, {NET_CJDNS}).first.IsValid()); BOOST_CHECK(addrman->Select(/*new_only=*/false).first == addr1); // add I2P address to the new table @@ -218,25 +218,29 @@ BOOST_AUTO_TEST_CASE(addrman_select_by_network) i2p_addr.SetSpecial("udhdrtrcetjm5sxzskjyr5ztpeszydbh4dpl3pl4utgqqw2v4jna.b32.i2p"); BOOST_CHECK(addrman->Add({i2p_addr}, source)); - BOOST_CHECK(addrman->Select(/*new_only=*/true, NET_I2P).first == i2p_addr); - BOOST_CHECK(addrman->Select(/*new_only=*/false, NET_I2P).first == i2p_addr); - BOOST_CHECK(addrman->Select(/*new_only=*/false, NET_IPV4).first == addr1); - BOOST_CHECK(!addrman->Select(/*new_only=*/false, NET_IPV6).first.IsValid()); - BOOST_CHECK(!addrman->Select(/*new_only=*/false, NET_ONION).first.IsValid()); - BOOST_CHECK(!addrman->Select(/*new_only=*/false, NET_CJDNS).first.IsValid()); + BOOST_CHECK(addrman->Select(/*new_only=*/true, {NET_I2P}).first == i2p_addr); + BOOST_CHECK(addrman->Select(/*new_only=*/false, {NET_I2P}).first == i2p_addr); + BOOST_CHECK(addrman->Select(/*new_only=*/false, {NET_IPV4}).first == addr1); + std::unordered_set<Network> nets_with_entries = {NET_IPV4, NET_I2P}; + BOOST_CHECK(addrman->Select(/*new_only=*/false, nets_with_entries).first.IsValid()); + BOOST_CHECK(!addrman->Select(/*new_only=*/false, {NET_IPV6}).first.IsValid()); + BOOST_CHECK(!addrman->Select(/*new_only=*/false, {NET_ONION}).first.IsValid()); + BOOST_CHECK(!addrman->Select(/*new_only=*/false, {NET_CJDNS}).first.IsValid()); + std::unordered_set<Network> nets_without_entries = {NET_IPV6, NET_ONION, NET_CJDNS}; + BOOST_CHECK(!addrman->Select(/*new_only=*/false, nets_without_entries).first.IsValid()); // bump I2P address to tried table BOOST_CHECK(addrman->Good(i2p_addr)); - BOOST_CHECK(!addrman->Select(/*new_only=*/true, NET_I2P).first.IsValid()); - BOOST_CHECK(addrman->Select(/*new_only=*/false, NET_I2P).first == i2p_addr); + BOOST_CHECK(!addrman->Select(/*new_only=*/true, {NET_I2P}).first.IsValid()); + BOOST_CHECK(addrman->Select(/*new_only=*/false, {NET_I2P}).first == i2p_addr); // add another I2P address to the new table CAddress i2p_addr2; i2p_addr2.SetSpecial("c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p"); BOOST_CHECK(addrman->Add({i2p_addr2}, source)); - BOOST_CHECK(addrman->Select(/*new_only=*/true, NET_I2P).first == i2p_addr2); + BOOST_CHECK(addrman->Select(/*new_only=*/true, {NET_I2P}).first == i2p_addr2); // ensure that both new and tried table are selected from bool new_selected{false}; @@ -244,7 +248,7 @@ BOOST_AUTO_TEST_CASE(addrman_select_by_network) int counter = 256; while (--counter > 0 && (!new_selected || !tried_selected)) { - const CAddress selected{addrman->Select(/*new_only=*/false, NET_I2P).first}; + const CAddress selected{addrman->Select(/*new_only=*/false, {NET_I2P}).first}; BOOST_REQUIRE(selected == i2p_addr || selected == i2p_addr2); if (selected == i2p_addr) { tried_selected = true; @@ -277,7 +281,7 @@ BOOST_AUTO_TEST_CASE(addrman_select_special) // since the only ipv4 address is on the new table, ensure that the new // table gets selected even if new_only is false. if the table was being // selected at random, this test will sporadically fail - BOOST_CHECK(addrman->Select(/*new_only=*/false, NET_IPV4).first == addr1); + BOOST_CHECK(addrman->Select(/*new_only=*/false, {NET_IPV4}).first == addr1); } BOOST_AUTO_TEST_CASE(addrman_new_collisions) @@ -444,10 +448,21 @@ BOOST_AUTO_TEST_CASE(getaddr_unfiltered) CNetAddr source = ResolveIP("250.1.2.1"); BOOST_CHECK(addrman->Add({addr1, addr2}, source)); - // Filtered GetAddr should only return addr1 + // Set time on this addr so isTerrible = false + CAddress addr3 = CAddress(ResolveService("250.251.2.3", 9998), NODE_NONE); + addr3.nTime = Now<NodeSeconds>(); + addrman->Good(addr3, /*time=*/Now<NodeSeconds>()); + BOOST_CHECK(addrman->Add({addr3}, source)); + // The time is set, but after ADDRMAN_RETRIES unsuccessful attempts not + // retried in the last minute, this addr should be isTerrible = true + for (size_t i = 0; i < 3; ++i) { + addrman->Attempt(addr3, /*fCountFailure=*/true, /*time=*/Now<NodeSeconds>() - 61s); + } + + // GetAddr filtered by quality (i.e. not IsTerrible) should only return addr1 BOOST_CHECK_EQUAL(addrman->GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt).size(), 1U); - // Unfiltered GetAddr should return addr1 and addr2 - BOOST_CHECK_EQUAL(addrman->GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt, /*filtered=*/false).size(), 2U); + // Unfiltered GetAddr should return all addrs + BOOST_CHECK_EQUAL(addrman->GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt, /*filtered=*/false).size(), 3U); } BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 3a33bdb7ec..ed95a8831e 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) CBlock block(BuildBlockTestCase(rand_ctx)); LOCK2(cs_main, pool.cs); - pool.addUnchecked(entry.FromTx(block.vtx[2])); + AddToMempool(pool, entry.FromTx(block.vtx[2])); BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); // Do a simple ShortTxIDs RT @@ -151,7 +151,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) CBlock block(BuildBlockTestCase(rand_ctx)); LOCK2(cs_main, pool.cs); - pool.addUnchecked(entry.FromTx(block.vtx[2])); + AddToMempool(pool, entry.FromTx(block.vtx[2])); BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; @@ -222,7 +222,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) CBlock block(BuildBlockTestCase(rand_ctx)); LOCK2(cs_main, pool.cs); - pool.addUnchecked(entry.FromTx(block.vtx[1])); + AddToMempool(pool, entry.FromTx(block.vtx[1])); BOOST_CHECK_EQUAL(pool.get(block.vtx[1]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; @@ -322,7 +322,7 @@ BOOST_AUTO_TEST_CASE(ReceiveWithExtraTransactions) { extra_txn.resize(10); LOCK2(cs_main, pool.cs); - pool.addUnchecked(entry.FromTx(block.vtx[2])); + AddToMempool(pool, entry.FromTx(block.vtx[2])); BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); // Ensure the non_block_tx is actually not in the block for (const auto &block_tx : block.vtx) { diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index f6024f1ef3..4b576bb084 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -67,8 +67,9 @@ CBlock BuildChainTestingSetup::CreateBlock(const CBlockIndex* prev, const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey) { - BlockAssembler::Options options; - std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler{m_node.chainman->ActiveChainstate(), m_node.mempool.get(), options}.CreateNewBlock(scriptPubKey); + BlockAssembler::Options options; + options.coinbase_output_script = scriptPubKey; + std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler{m_node.chainman->ActiveChainstate(), m_node.mempool.get(), options}.CreateNewBlock(); CBlock& block = pblocktemplate->block; block.hashPrevBlock = prev->GetBlockHash(); block.nTime = prev->nTime + 1; @@ -144,7 +145,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) BOOST_REQUIRE(filter_index.StartBackgroundSync()); // Allow filter index to catch up with the block index. - IndexWaitSynced(filter_index, *Assert(m_node.shutdown)); + IndexWaitSynced(filter_index, *Assert(m_node.shutdown_signal)); // Check that filter index has all blocks that were in the chain before it started. { diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index 121f00bd25..c2b95dd861 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -28,13 +28,13 @@ BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) { const auto params {CreateChainParams(ArgsManager{}, ChainType::MAIN)}; - KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)}; + KernelNotifications notifications{Assert(m_node.shutdown_request), m_node.exit_status, *Assert(m_node.warnings)}; const BlockManager::Options blockman_opts{ .chainparams = *params, .blocks_dir = m_args.GetBlocksDirPath(), .notifications = notifications, }; - BlockManager blockman{*Assert(m_node.shutdown), blockman_opts}; + BlockManager blockman{*Assert(m_node.shutdown_signal), blockman_opts}; // simulate adding a genesis block normally BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); // simulate what happens during reindex @@ -135,13 +135,13 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_block_data_availability, TestChain100Setup) BOOST_AUTO_TEST_CASE(blockmanager_flush_block_file) { - KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)}; + KernelNotifications notifications{Assert(m_node.shutdown_request), m_node.exit_status, *Assert(m_node.warnings)}; node::BlockManager::Options blockman_opts{ .chainparams = Params(), .blocks_dir = m_args.GetBlocksDirPath(), .notifications = notifications, }; - BlockManager blockman{*Assert(m_node.shutdown), blockman_opts}; + BlockManager blockman{*Assert(m_node.shutdown_signal), blockman_opts}; // Test blocks with no transactions, not even a coinbase CBlock block1; diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index a9a1e47070..2463ce6da5 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -42,28 +42,26 @@ static const unsigned int QUEUE_BATCH_SIZE = 128; static const int SCRIPT_CHECK_THREADS = 3; struct FakeCheck { - bool operator()() const + std::optional<int> operator()() const { - return true; + return std::nullopt; } }; struct FakeCheckCheckCompletion { static std::atomic<size_t> n_calls; - bool operator()() + std::optional<int> operator()() { n_calls.fetch_add(1, std::memory_order_relaxed); - return true; + return std::nullopt; } }; -struct FailingCheck { - bool fails; - FailingCheck(bool _fails) : fails(_fails){}; - bool operator()() const - { - return !fails; - } +struct FixedCheck +{ + std::optional<int> m_result; + FixedCheck(std::optional<int> result) : m_result(result){}; + std::optional<int> operator()() const { return m_result; } }; struct UniqueCheck { @@ -71,11 +69,11 @@ struct UniqueCheck { static std::unordered_multiset<size_t> results GUARDED_BY(m); size_t check_id; UniqueCheck(size_t check_id_in) : check_id(check_id_in){}; - bool operator()() + std::optional<int> operator()() { LOCK(m); results.insert(check_id); - return true; + return std::nullopt; } }; @@ -83,9 +81,9 @@ struct UniqueCheck { struct MemoryCheck { static std::atomic<size_t> fake_allocated_memory; bool b {false}; - bool operator()() const + std::optional<int> operator()() const { - return true; + return std::nullopt; } MemoryCheck(const MemoryCheck& x) { @@ -110,9 +108,9 @@ struct FrozenCleanupCheck { static std::condition_variable cv; static std::mutex m; bool should_freeze{true}; - bool operator()() const + std::optional<int> operator()() const { - return true; + return std::nullopt; } FrozenCleanupCheck() = default; ~FrozenCleanupCheck() @@ -149,7 +147,7 @@ std::atomic<size_t> MemoryCheck::fake_allocated_memory{0}; // Queue Typedefs typedef CCheckQueue<FakeCheckCheckCompletion> Correct_Queue; typedef CCheckQueue<FakeCheck> Standard_Queue; -typedef CCheckQueue<FailingCheck> Failing_Queue; +typedef CCheckQueue<FixedCheck> Fixed_Queue; typedef CCheckQueue<UniqueCheck> Unique_Queue; typedef CCheckQueue<MemoryCheck> Memory_Queue; typedef CCheckQueue<FrozenCleanupCheck> FrozenCleanup_Queue; @@ -174,7 +172,7 @@ void CheckQueueTest::Correct_Queue_range(std::vector<size_t> range) total -= vChecks.size(); control.Add(std::move(vChecks)); } - BOOST_REQUIRE(control.Wait()); + BOOST_REQUIRE(!control.Complete().has_value()); BOOST_REQUIRE_EQUAL(FakeCheckCheckCompletion::n_calls, i); } } @@ -217,27 +215,27 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Correct_Random) } -/** Test that failing checks are caught */ +/** Test that distinct failing checks are caught */ BOOST_AUTO_TEST_CASE(test_CheckQueue_Catches_Failure) { - auto fail_queue = std::make_unique<Failing_Queue>(QUEUE_BATCH_SIZE, SCRIPT_CHECK_THREADS); + auto fixed_queue = std::make_unique<Fixed_Queue>(QUEUE_BATCH_SIZE, SCRIPT_CHECK_THREADS); for (size_t i = 0; i < 1001; ++i) { - CCheckQueueControl<FailingCheck> control(fail_queue.get()); + CCheckQueueControl<FixedCheck> control(fixed_queue.get()); size_t remaining = i; while (remaining) { size_t r = m_rng.randrange(10); - std::vector<FailingCheck> vChecks; + std::vector<FixedCheck> vChecks; vChecks.reserve(r); for (size_t k = 0; k < r && remaining; k++, remaining--) - vChecks.emplace_back(remaining == 1); + vChecks.emplace_back(remaining == 1 ? std::make_optional<int>(17 * i) : std::nullopt); control.Add(std::move(vChecks)); } - bool success = control.Wait(); + auto result = control.Complete(); if (i > 0) { - BOOST_REQUIRE(!success); - } else if (i == 0) { - BOOST_REQUIRE(success); + BOOST_REQUIRE(result.has_value() && *result == static_cast<int>(17 * i)); + } else { + BOOST_REQUIRE(!result.has_value()); } } } @@ -245,17 +243,17 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Catches_Failure) // future blocks, ie, the bad state is cleared. BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) { - auto fail_queue = std::make_unique<Failing_Queue>(QUEUE_BATCH_SIZE, SCRIPT_CHECK_THREADS); + auto fail_queue = std::make_unique<Fixed_Queue>(QUEUE_BATCH_SIZE, SCRIPT_CHECK_THREADS); for (auto times = 0; times < 10; ++times) { for (const bool end_fails : {true, false}) { - CCheckQueueControl<FailingCheck> control(fail_queue.get()); + CCheckQueueControl<FixedCheck> control(fail_queue.get()); { - std::vector<FailingCheck> vChecks; - vChecks.resize(100, false); - vChecks[99] = end_fails; + std::vector<FixedCheck> vChecks; + vChecks.resize(100, FixedCheck(std::nullopt)); + vChecks[99] = FixedCheck(end_fails ? std::make_optional<int>(2) : std::nullopt); control.Add(std::move(vChecks)); } - bool r =control.Wait(); + bool r = !control.Complete().has_value(); BOOST_REQUIRE(r != end_fails); } } @@ -329,8 +327,8 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) CCheckQueueControl<FrozenCleanupCheck> control(queue.get()); std::vector<FrozenCleanupCheck> vChecks(1); control.Add(std::move(vChecks)); - bool waitResult = control.Wait(); // Hangs here - assert(waitResult); + auto result = control.Complete(); // Hangs here + assert(!result); }); { std::unique_lock<std::mutex> l(FrozenCleanupCheck::m); @@ -360,6 +358,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) auto queue = std::make_unique<Standard_Queue>(QUEUE_BATCH_SIZE, SCRIPT_CHECK_THREADS); { std::vector<std::thread> tg; + tg.reserve(3); std::atomic<int> nThreads {0}; std::atomic<int> fails {0}; for (size_t i = 0; i < 3; ++i) { diff --git a/src/test/cluster_linearize_tests.cpp b/src/test/cluster_linearize_tests.cpp index d15e783ea1..265ccdc805 100644 --- a/src/test/cluster_linearize_tests.cpp +++ b/src/test/cluster_linearize_tests.cpp @@ -18,13 +18,24 @@ using namespace cluster_linearize; namespace { +/** Special magic value that indicates to TestDepGraphSerialization that a cluster entry represents + * a hole. */ +constexpr std::pair<FeeFrac, TestBitSet> HOLE{FeeFrac{0, 0x3FFFFF}, {}}; + template<typename SetType> -void TestDepGraphSerialization(const Cluster<SetType>& cluster, const std::string& hexenc) +void TestDepGraphSerialization(const std::vector<std::pair<FeeFrac, SetType>>& cluster, const std::string& hexenc) { - DepGraph depgraph(cluster); - - // Run normal sanity and correspondence checks, which includes a round-trip test. - VerifyDepGraphFromCluster(cluster, depgraph); + // Construct DepGraph from cluster argument. + DepGraph<SetType> depgraph; + SetType holes; + for (ClusterIndex i = 0; i < cluster.size(); ++i) { + depgraph.AddTransaction(cluster[i].first); + if (cluster[i] == HOLE) holes.Set(i); + } + for (ClusterIndex i = 0; i < cluster.size(); ++i) { + depgraph.AddDependencies(cluster[i].second, i); + } + depgraph.RemoveTransactions(holes); // There may be multiple serializations of the same graph, but DepGraphFormatter's serializer // only produces one of those. Verify that hexenc matches that canonical serialization. @@ -133,6 +144,34 @@ BOOST_AUTO_TEST_CASE(depgraph_ser_tests) skip insertion C): D,A,B,E,C */ "00" /* end of graph */ ); + + // Transactions: A(1,2), B(3,1), C(2,1), D(1,3), E(1,1). Deps: C->A, D->A, D->B, E->D. + // In order: [_, D, _, _, A, _, B, _, _, _, E, _, _, C] (_ being holes). Internally serialized + // in order A,B,C,D,E. + TestDepGraphSerialization<TestBitSet>( + {HOLE, {{1, 3}, {4, 6}}, HOLE, HOLE, {{1, 2}, {}}, HOLE, {{3, 1}, {}}, HOLE, HOLE, HOLE, {{1, 1}, {1}}, HOLE, HOLE, {{2, 1}, {4}}}, + "02" /* A size */ + "02" /* A fee */ + "03" /* A insertion position (3 holes): _, _, _, A */ + "01" /* B size */ + "06" /* B fee */ + "06" /* B insertion position (skip B->A dependency, skip 4 inserts, add 1 hole): _, _, _, A, _, B */ + "01" /* C size */ + "04" /* C fee */ + "01" /* C->A dependency (skip C->B dependency) */ + "0b" /* C insertion position (skip 6 inserts, add 5 holes): _, _, _, A, _, B, _, _, _, _, _, C */ + "03" /* D size */ + "02" /* D fee */ + "01" /* D->B dependency (skip D->C dependency) */ + "00" /* D->A dependency (no skips) */ + "0b" /* D insertion position (skip 11 inserts): _, D, _, _, A, _, B, _, _, _, _, _, C */ + "01" /* E size */ + "02" /* E fee */ + "00" /* E->D dependency (no skips) */ + "04" /* E insertion position (skip E->C dependency, E->B and E->A are implied, skip 3 + inserts): _, D, _, _, A, _, B, _, _, _, E, _, _, C */ + "00" /* end of graph */ + ); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 0d18cd0c2b..c46144b34b 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -15,6 +15,8 @@ #include <util/strencodings.h> #include <map> +#include <string> +#include <variant> #include <vector> #include <boost/test/unit_test.hpp> @@ -44,18 +46,14 @@ class CCoinsViewTest : public CCoinsView public: CCoinsViewTest(FastRandomContext& rng) : m_rng{rng} {} - [[nodiscard]] bool GetCoin(const COutPoint& outpoint, Coin& coin) const override + std::optional<Coin> GetCoin(const COutPoint& outpoint) const override { - std::map<COutPoint, Coin>::const_iterator it = map_.find(outpoint); - if (it == map_.end()) { - return false; - } - coin = it->second; - if (coin.IsSpent() && m_rng.randbool() == 0) { - // Randomly return false in case of an empty entry. - return false; + if (auto it{map_.find(outpoint)}; it != map_.end()) { + if (!it->second.IsSpent() || m_rng.randbool()) { + return it->second; // TODO spent coins shouldn't be returned + } } - return true; + return std::nullopt; } uint256 GetBestBlock() const override { return hashBestBlock_; } @@ -563,21 +561,58 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) } const static COutPoint OUTPOINT; -const static CAmount SPENT = -1; -const static CAmount ABSENT = -2; -const static CAmount FAIL = -3; -const static CAmount VALUE1 = 100; -const static CAmount VALUE2 = 200; -const static CAmount VALUE3 = 300; -const static char DIRTY = CCoinsCacheEntry::DIRTY; -const static char FRESH = CCoinsCacheEntry::FRESH; -const static char NO_ENTRY = -1; - -const static auto FLAGS = {char(0), FRESH, DIRTY, char(DIRTY | FRESH)}; -const static auto CLEAN_FLAGS = {char(0), FRESH}; -const static auto ABSENT_FLAGS = {NO_ENTRY}; - -static void SetCoinsValue(CAmount value, Coin& coin) +constexpr CAmount SPENT {-1}; +constexpr CAmount ABSENT{-2}; +constexpr CAmount VALUE1{100}; +constexpr CAmount VALUE2{200}; +constexpr CAmount VALUE3{300}; + +struct CoinEntry { + enum class State { CLEAN, DIRTY, FRESH, DIRTY_FRESH }; + + const CAmount value; + const State state; + + constexpr CoinEntry(const CAmount v, const State s) : value{v}, state{s} {} + + bool operator==(const CoinEntry& o) const = default; + friend std::ostream& operator<<(std::ostream& os, const CoinEntry& e) { return os << e.value << ", " << e.state; } + + constexpr bool IsDirtyFresh() const { return state == State::DIRTY_FRESH; } + constexpr bool IsDirty() const { return state == State::DIRTY || IsDirtyFresh(); } + constexpr bool IsFresh() const { return state == State::FRESH || IsDirtyFresh(); } + + static constexpr State ToState(const bool is_dirty, const bool is_fresh) { + if (is_dirty && is_fresh) return State::DIRTY_FRESH; + if (is_dirty) return State::DIRTY; + if (is_fresh) return State::FRESH; + return State::CLEAN; + } +}; + +using MaybeCoin = std::optional<CoinEntry>; +using CoinOrError = std::variant<MaybeCoin, std::string>; + +constexpr MaybeCoin MISSING {std::nullopt}; +constexpr MaybeCoin SPENT_DIRTY {{SPENT, CoinEntry::State::DIRTY}}; +constexpr MaybeCoin SPENT_DIRTY_FRESH {{SPENT, CoinEntry::State::DIRTY_FRESH}}; +constexpr MaybeCoin SPENT_FRESH {{SPENT, CoinEntry::State::FRESH}}; +constexpr MaybeCoin SPENT_CLEAN {{SPENT, CoinEntry::State::CLEAN}}; +constexpr MaybeCoin VALUE1_DIRTY {{VALUE1, CoinEntry::State::DIRTY}}; +constexpr MaybeCoin VALUE1_DIRTY_FRESH{{VALUE1, CoinEntry::State::DIRTY_FRESH}}; +constexpr MaybeCoin VALUE1_FRESH {{VALUE1, CoinEntry::State::FRESH}}; +constexpr MaybeCoin VALUE1_CLEAN {{VALUE1, CoinEntry::State::CLEAN}}; +constexpr MaybeCoin VALUE2_DIRTY {{VALUE2, CoinEntry::State::DIRTY}}; +constexpr MaybeCoin VALUE2_DIRTY_FRESH{{VALUE2, CoinEntry::State::DIRTY_FRESH}}; +constexpr MaybeCoin VALUE2_FRESH {{VALUE2, CoinEntry::State::FRESH}}; +constexpr MaybeCoin VALUE2_CLEAN {{VALUE2, CoinEntry::State::CLEAN}}; +constexpr MaybeCoin VALUE3_DIRTY {{VALUE3, CoinEntry::State::DIRTY}}; +constexpr MaybeCoin VALUE3_DIRTY_FRESH{{VALUE3, CoinEntry::State::DIRTY_FRESH}}; + +constexpr auto EX_OVERWRITE_UNSPENT{"Attempted to overwrite an unspent coin (when possible_overwrite is false)"}; +constexpr auto EX_FRESH_MISAPPLIED {"FRESH flag misapplied to coin that exists in parent cache"}; + +static void SetCoinsValue(const CAmount value, Coin& coin) { assert(value != ABSENT); coin.Clear(); @@ -589,45 +624,34 @@ static void SetCoinsValue(CAmount value, Coin& coin) } } -static size_t InsertCoinsMapEntry(CCoinsMap& map, CoinsCachePair& sentinel, CAmount value, char flags) +static size_t InsertCoinsMapEntry(CCoinsMap& map, CoinsCachePair& sentinel, const CoinEntry& cache_coin) { - if (value == ABSENT) { - assert(flags == NO_ENTRY); - return 0; - } - assert(flags != NO_ENTRY); CCoinsCacheEntry entry; - SetCoinsValue(value, entry.coin); - auto inserted = map.emplace(OUTPOINT, std::move(entry)); - assert(inserted.second); - inserted.first->second.AddFlags(flags, *inserted.first, sentinel); - return inserted.first->second.coin.DynamicMemoryUsage(); + SetCoinsValue(cache_coin.value, entry.coin); + auto [iter, inserted] = map.emplace(OUTPOINT, std::move(entry)); + assert(inserted); + if (cache_coin.IsDirty()) CCoinsCacheEntry::SetDirty(*iter, sentinel); + if (cache_coin.IsFresh()) CCoinsCacheEntry::SetFresh(*iter, sentinel); + return iter->second.coin.DynamicMemoryUsage(); } -void GetCoinsMapEntry(const CCoinsMap& map, CAmount& value, char& flags, const COutPoint& outp = OUTPOINT) +static MaybeCoin GetCoinsMapEntry(const CCoinsMap& map, const COutPoint& outp = OUTPOINT) { - auto it = map.find(outp); - if (it == map.end()) { - value = ABSENT; - flags = NO_ENTRY; - } else { - if (it->second.coin.IsSpent()) { - value = SPENT; - } else { - value = it->second.coin.out.nValue; - } - flags = it->second.GetFlags(); - assert(flags != NO_ENTRY); + if (auto it{map.find(outp)}; it != map.end()) { + return CoinEntry{ + it->second.coin.IsSpent() ? SPENT : it->second.coin.out.nValue, + CoinEntry::ToState(it->second.IsDirty(), it->second.IsFresh())}; } + return MISSING; } -void WriteCoinsViewEntry(CCoinsView& view, CAmount value, char flags) +static void WriteCoinsViewEntry(CCoinsView& view, const MaybeCoin& cache_coin) { CoinsCachePair sentinel{}; sentinel.second.SelfRef(sentinel); CCoinsMapMemoryResource resource; CCoinsMap map{0, CCoinsMap::hasher{}, CCoinsMap::key_equal{}, &resource}; - auto usage{InsertCoinsMapEntry(map, sentinel, value, flags)}; + auto usage{cache_coin ? InsertCoinsMapEntry(map, sentinel, *cache_coin) : 0}; auto cursor{CoinsViewCacheCursor(usage, sentinel, map, /*will_erase=*/true)}; BOOST_CHECK(view.BatchWrite(cursor, {})); } @@ -635,10 +659,11 @@ void WriteCoinsViewEntry(CCoinsView& view, CAmount value, char flags) class SingleEntryCacheTest { public: - SingleEntryCacheTest(CAmount base_value, CAmount cache_value, char cache_flags) + SingleEntryCacheTest(const CAmount base_value, const MaybeCoin& cache_coin) { - WriteCoinsViewEntry(base, base_value, base_value == ABSENT ? NO_ENTRY : DIRTY); - cache.usage() += InsertCoinsMapEntry(cache.map(), cache.sentinel(), cache_value, cache_flags); + auto base_cache_coin{base_value == ABSENT ? MISSING : CoinEntry{base_value, CoinEntry::State::DIRTY}}; + WriteCoinsViewEntry(base, base_cache_coin); + if (cache_coin) cache.usage() += InsertCoinsMapEntry(cache.map(), cache.sentinel(), *cache_coin); } CCoinsView root; @@ -646,17 +671,13 @@ public: CCoinsViewCacheTest cache{&base}; }; -static void CheckAccessCoin(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) +static void CheckAccessCoin(const CAmount base_value, const MaybeCoin& cache_coin, const MaybeCoin& expected) { - SingleEntryCacheTest test(base_value, cache_value, cache_flags); - test.cache.AccessCoin(OUTPOINT); + SingleEntryCacheTest test{base_value, cache_coin}; + auto& coin = test.cache.AccessCoin(OUTPOINT); + BOOST_CHECK_EQUAL(coin.IsSpent(), !test.cache.GetCoin(OUTPOINT)); test.cache.SelfTest(/*sanity_check=*/false); - - CAmount result_value; - char result_flags; - GetCoinsMapEntry(test.cache.map(), result_value, result_flags); - BOOST_CHECK_EQUAL(result_value, expected_value); - BOOST_CHECK_EQUAL(result_flags, expected_flags); + BOOST_CHECK_EQUAL(GetCoinsMapEntry(test.cache.map()), expected); } BOOST_AUTO_TEST_CASE(ccoins_access) @@ -664,121 +685,65 @@ BOOST_AUTO_TEST_CASE(ccoins_access) /* Check AccessCoin behavior, requesting a coin from a cache view layered on * top of a base view, and checking the resulting entry in the cache after * the access. - * - * Base Cache Result Cache Result - * Value Value Value Flags Flags + * Base Cache Expected */ - CheckAccessCoin(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckAccessCoin(ABSENT, SPENT , SPENT , 0 , 0 ); - CheckAccessCoin(ABSENT, SPENT , SPENT , FRESH , FRESH ); - CheckAccessCoin(ABSENT, SPENT , SPENT , DIRTY , DIRTY ); - CheckAccessCoin(ABSENT, SPENT , SPENT , DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoin(ABSENT, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoin(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoin(SPENT , ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckAccessCoin(SPENT , SPENT , SPENT , 0 , 0 ); - CheckAccessCoin(SPENT , SPENT , SPENT , FRESH , FRESH ); - CheckAccessCoin(SPENT , SPENT , SPENT , DIRTY , DIRTY ); - CheckAccessCoin(SPENT , SPENT , SPENT , DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoin(SPENT , VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoin(SPENT , VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoin(SPENT , VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoin(SPENT , VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoin(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); - CheckAccessCoin(VALUE1, SPENT , SPENT , 0 , 0 ); - CheckAccessCoin(VALUE1, SPENT , SPENT , FRESH , FRESH ); - CheckAccessCoin(VALUE1, SPENT , SPENT , DIRTY , DIRTY ); - CheckAccessCoin(VALUE1, SPENT , SPENT , DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoin(VALUE1, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoin(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + for (auto base_value : {ABSENT, SPENT, VALUE1}) { + CheckAccessCoin(base_value, MISSING, base_value == VALUE1 ? VALUE1_CLEAN : MISSING); + + CheckAccessCoin(base_value, SPENT_CLEAN, SPENT_CLEAN ); + CheckAccessCoin(base_value, SPENT_FRESH, SPENT_FRESH ); + CheckAccessCoin(base_value, SPENT_DIRTY, SPENT_DIRTY ); + CheckAccessCoin(base_value, SPENT_DIRTY_FRESH, SPENT_DIRTY_FRESH ); + + CheckAccessCoin(base_value, VALUE2_CLEAN, VALUE2_CLEAN ); + CheckAccessCoin(base_value, VALUE2_FRESH, VALUE2_FRESH ); + CheckAccessCoin(base_value, VALUE2_DIRTY, VALUE2_DIRTY ); + CheckAccessCoin(base_value, VALUE2_DIRTY_FRESH, VALUE2_DIRTY_FRESH); + } } -static void CheckSpendCoins(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) +static void CheckSpendCoins(const CAmount base_value, const MaybeCoin& cache_coin, const MaybeCoin& expected) { - SingleEntryCacheTest test(base_value, cache_value, cache_flags); + SingleEntryCacheTest test{base_value, cache_coin}; test.cache.SpendCoin(OUTPOINT); test.cache.SelfTest(); - - CAmount result_value; - char result_flags; - GetCoinsMapEntry(test.cache.map(), result_value, result_flags); - BOOST_CHECK_EQUAL(result_value, expected_value); - BOOST_CHECK_EQUAL(result_flags, expected_flags); -}; + BOOST_CHECK_EQUAL(GetCoinsMapEntry(test.cache.map()), expected); +} BOOST_AUTO_TEST_CASE(ccoins_spend) { /* Check SpendCoin behavior, requesting a coin from a cache view layered on * top of a base view, spending, and then checking * the resulting entry in the cache after the modification. - * - * Base Cache Result Cache Result - * Value Value Value Flags Flags + * Base Cache Expected */ - CheckSpendCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckSpendCoins(ABSENT, SPENT , SPENT , 0 , DIRTY ); - CheckSpendCoins(ABSENT, SPENT , ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(ABSENT, SPENT , SPENT , DIRTY , DIRTY ); - CheckSpendCoins(ABSENT, SPENT , ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(ABSENT, VALUE2, SPENT , 0 , DIRTY ); - CheckSpendCoins(ABSENT, VALUE2, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(ABSENT, VALUE2, SPENT , DIRTY , DIRTY ); - CheckSpendCoins(ABSENT, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(SPENT , ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckSpendCoins(SPENT , SPENT , SPENT , 0 , DIRTY ); - CheckSpendCoins(SPENT , SPENT , ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(SPENT , SPENT , SPENT , DIRTY , DIRTY ); - CheckSpendCoins(SPENT , SPENT , ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(SPENT , VALUE2, SPENT , 0 , DIRTY ); - CheckSpendCoins(SPENT , VALUE2, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(SPENT , VALUE2, SPENT , DIRTY , DIRTY ); - CheckSpendCoins(SPENT , VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(VALUE1, ABSENT, SPENT , NO_ENTRY , DIRTY ); - CheckSpendCoins(VALUE1, SPENT , SPENT , 0 , DIRTY ); - CheckSpendCoins(VALUE1, SPENT , ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(VALUE1, SPENT , SPENT , DIRTY , DIRTY ); - CheckSpendCoins(VALUE1, SPENT , ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(VALUE1, VALUE2, SPENT , 0 , DIRTY ); - CheckSpendCoins(VALUE1, VALUE2, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(VALUE1, VALUE2, SPENT , DIRTY , DIRTY ); - CheckSpendCoins(VALUE1, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + for (auto base_value : {ABSENT, SPENT, VALUE1}) { + CheckSpendCoins(base_value, MISSING, base_value == VALUE1 ? SPENT_DIRTY : MISSING); + + CheckSpendCoins(base_value, SPENT_CLEAN, SPENT_DIRTY); + CheckSpendCoins(base_value, SPENT_FRESH, MISSING ); + CheckSpendCoins(base_value, SPENT_DIRTY, SPENT_DIRTY); + CheckSpendCoins(base_value, SPENT_DIRTY_FRESH, MISSING ); + + CheckSpendCoins(base_value, VALUE2_CLEAN, SPENT_DIRTY); + CheckSpendCoins(base_value, VALUE2_FRESH, MISSING ); + CheckSpendCoins(base_value, VALUE2_DIRTY, SPENT_DIRTY); + CheckSpendCoins(base_value, VALUE2_DIRTY_FRESH, MISSING ); + } } -static void CheckAddCoinBase(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags, bool coinbase) +static void CheckAddCoin(const CAmount base_value, const MaybeCoin& cache_coin, const CAmount modify_value, const CoinOrError& expected, const bool coinbase) { - SingleEntryCacheTest test(base_value, cache_value, cache_flags); - - CAmount result_value; - char result_flags; - try { - CTxOut output; - output.nValue = modify_value; - test.cache.AddCoin(OUTPOINT, Coin(std::move(output), 1, coinbase), coinbase); + SingleEntryCacheTest test{base_value, cache_coin}; + bool possible_overwrite{coinbase}; + auto add_coin{[&] { test.cache.AddCoin(OUTPOINT, Coin{CTxOut{modify_value, CScript{}}, 1, coinbase}, possible_overwrite); }}; + if (auto* expected_coin{std::get_if<MaybeCoin>(&expected)}) { + add_coin(); test.cache.SelfTest(); - GetCoinsMapEntry(test.cache.map(), result_value, result_flags); - } catch (std::logic_error&) { - result_value = FAIL; - result_flags = NO_ENTRY; + BOOST_CHECK_EQUAL(GetCoinsMapEntry(test.cache.map()), *expected_coin); + } else { + BOOST_CHECK_EXCEPTION(add_coin(), std::logic_error, HasReason(std::get<std::string>(expected))); } - - BOOST_CHECK_EQUAL(result_value, expected_value); - BOOST_CHECK_EQUAL(result_flags, expected_flags); -} - -// Simple wrapper for CheckAddCoinBase function above that loops through -// different possible base_values, making sure each one gives the same results. -// This wrapper lets the coins_add test below be shorter and less repetitive, -// while still verifying that the CoinsViewCache::AddCoin implementation -// ignores base values. -template <typename... Args> -static void CheckAddCoin(Args&&... args) -{ - for (const CAmount base_value : {ABSENT, SPENT, VALUE1}) - CheckAddCoinBase(base_value, std::forward<Args>(args)...); } BOOST_AUTO_TEST_CASE(ccoins_add) @@ -786,48 +751,44 @@ BOOST_AUTO_TEST_CASE(ccoins_add) /* Check AddCoin behavior, requesting a new coin from a cache view, * writing a modification to the coin, and then checking the resulting * entry in the cache after the modification. Verify behavior with the - * AddCoin possible_overwrite argument set to false, and to true. - * - * Cache Write Result Cache Result possible_overwrite - * Value Value Value Flags Flags + * AddCoin coinbase argument set to false, and to true. + * Base Cache Write Expected Coinbase */ - CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); - CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); - CheckAddCoin(SPENT , VALUE3, VALUE3, 0 , DIRTY|FRESH, false); - CheckAddCoin(SPENT , VALUE3, VALUE3, 0 , DIRTY , true ); - CheckAddCoin(SPENT , VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); - CheckAddCoin(SPENT , VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckAddCoin(SPENT , VALUE3, VALUE3, DIRTY , DIRTY , false); - CheckAddCoin(SPENT , VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckAddCoin(SPENT , VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); - CheckAddCoin(SPENT , VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); - CheckAddCoin(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); - CheckAddCoin(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckAddCoin(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); - CheckAddCoin(VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY , NO_ENTRY , false); - CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY|FRESH, NO_ENTRY , false); - CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + for (auto base_value : {ABSENT, SPENT, VALUE1}) { + CheckAddCoin(base_value, MISSING, VALUE3, VALUE3_DIRTY_FRESH, false); + CheckAddCoin(base_value, MISSING, VALUE3, VALUE3_DIRTY, true ); + + CheckAddCoin(base_value, SPENT_CLEAN, VALUE3, VALUE3_DIRTY_FRESH, false); + CheckAddCoin(base_value, SPENT_CLEAN, VALUE3, VALUE3_DIRTY, true ); + CheckAddCoin(base_value, SPENT_FRESH, VALUE3, VALUE3_DIRTY_FRESH, false); + CheckAddCoin(base_value, SPENT_FRESH, VALUE3, VALUE3_DIRTY_FRESH, true ); + CheckAddCoin(base_value, SPENT_DIRTY, VALUE3, VALUE3_DIRTY, false); + CheckAddCoin(base_value, SPENT_DIRTY, VALUE3, VALUE3_DIRTY, true ); + CheckAddCoin(base_value, SPENT_DIRTY_FRESH, VALUE3, VALUE3_DIRTY_FRESH, false); + CheckAddCoin(base_value, SPENT_DIRTY_FRESH, VALUE3, VALUE3_DIRTY_FRESH, true ); + + CheckAddCoin(base_value, VALUE2_CLEAN, VALUE3, EX_OVERWRITE_UNSPENT, false); + CheckAddCoin(base_value, VALUE2_CLEAN, VALUE3, VALUE3_DIRTY, true ); + CheckAddCoin(base_value, VALUE2_FRESH, VALUE3, EX_OVERWRITE_UNSPENT, false); + CheckAddCoin(base_value, VALUE2_FRESH, VALUE3, VALUE3_DIRTY_FRESH, true ); + CheckAddCoin(base_value, VALUE2_DIRTY, VALUE3, EX_OVERWRITE_UNSPENT, false); + CheckAddCoin(base_value, VALUE2_DIRTY, VALUE3, VALUE3_DIRTY, true ); + CheckAddCoin(base_value, VALUE2_DIRTY_FRESH, VALUE3, EX_OVERWRITE_UNSPENT, false); + CheckAddCoin(base_value, VALUE2_DIRTY_FRESH, VALUE3, VALUE3_DIRTY_FRESH, true ); + } } -void CheckWriteCoins(CAmount parent_value, CAmount child_value, CAmount expected_value, char parent_flags, char child_flags, char expected_flags) +static void CheckWriteCoins(const MaybeCoin& parent, const MaybeCoin& child, const CoinOrError& expected) { - SingleEntryCacheTest test(ABSENT, parent_value, parent_flags); - - CAmount result_value; - char result_flags; - try { - WriteCoinsViewEntry(test.cache, child_value, child_flags); + SingleEntryCacheTest test{ABSENT, parent}; + auto write_coins{[&] { WriteCoinsViewEntry(test.cache, child); }}; + if (auto* expected_coin{std::get_if<MaybeCoin>(&expected)}) { + write_coins(); test.cache.SelfTest(/*sanity_check=*/false); - GetCoinsMapEntry(test.cache.map(), result_value, result_flags); - } catch (std::logic_error&) { - result_value = FAIL; - result_flags = NO_ENTRY; + BOOST_CHECK_EQUAL(GetCoinsMapEntry(test.cache.map()), *expected_coin); + } else { + BOOST_CHECK_EXCEPTION(write_coins(), std::logic_error, HasReason(std::get<std::string>(expected))); } - - BOOST_CHECK_EQUAL(result_value, expected_value); - BOOST_CHECK_EQUAL(result_flags, expected_flags); } BOOST_AUTO_TEST_CASE(ccoins_write) @@ -835,68 +796,74 @@ BOOST_AUTO_TEST_CASE(ccoins_write) /* Check BatchWrite behavior, flushing one entry from a child cache to a * parent cache, and checking the resulting entry in the parent cache * after the write. - * - * Parent Child Result Parent Child Result - * Value Value Value Flags Flags Flags + * Parent Child Expected */ - CheckWriteCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY , NO_ENTRY ); - CheckWriteCoins(ABSENT, SPENT , SPENT , NO_ENTRY , DIRTY , DIRTY ); - CheckWriteCoins(ABSENT, SPENT , ABSENT, NO_ENTRY , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(ABSENT, VALUE2, VALUE2, NO_ENTRY , DIRTY , DIRTY ); - CheckWriteCoins(ABSENT, VALUE2, VALUE2, NO_ENTRY , DIRTY|FRESH, DIRTY|FRESH); - CheckWriteCoins(SPENT , ABSENT, SPENT , 0 , NO_ENTRY , 0 ); - CheckWriteCoins(SPENT , ABSENT, SPENT , FRESH , NO_ENTRY , FRESH ); - CheckWriteCoins(SPENT , ABSENT, SPENT , DIRTY , NO_ENTRY , DIRTY ); - CheckWriteCoins(SPENT , ABSENT, SPENT , DIRTY|FRESH, NO_ENTRY , DIRTY|FRESH); - CheckWriteCoins(SPENT , SPENT , SPENT , 0 , DIRTY , DIRTY ); - CheckWriteCoins(SPENT , SPENT , SPENT , 0 , DIRTY|FRESH, DIRTY ); - CheckWriteCoins(SPENT , SPENT , ABSENT, FRESH , DIRTY , NO_ENTRY ); - CheckWriteCoins(SPENT , SPENT , ABSENT, FRESH , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(SPENT , SPENT , SPENT , DIRTY , DIRTY , DIRTY ); - CheckWriteCoins(SPENT , SPENT , SPENT , DIRTY , DIRTY|FRESH, DIRTY ); - CheckWriteCoins(SPENT , SPENT , ABSENT, DIRTY|FRESH, DIRTY , NO_ENTRY ); - CheckWriteCoins(SPENT , SPENT , ABSENT, DIRTY|FRESH, DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(SPENT , VALUE2, VALUE2, 0 , DIRTY , DIRTY ); - CheckWriteCoins(SPENT , VALUE2, VALUE2, 0 , DIRTY|FRESH, DIRTY ); - CheckWriteCoins(SPENT , VALUE2, VALUE2, FRESH , DIRTY , DIRTY|FRESH); - CheckWriteCoins(SPENT , VALUE2, VALUE2, FRESH , DIRTY|FRESH, DIRTY|FRESH); - CheckWriteCoins(SPENT , VALUE2, VALUE2, DIRTY , DIRTY , DIRTY ); - CheckWriteCoins(SPENT , VALUE2, VALUE2, DIRTY , DIRTY|FRESH, DIRTY ); - CheckWriteCoins(SPENT , VALUE2, VALUE2, DIRTY|FRESH, DIRTY , DIRTY|FRESH); - CheckWriteCoins(SPENT , VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH, DIRTY|FRESH); - CheckWriteCoins(VALUE1, ABSENT, VALUE1, 0 , NO_ENTRY , 0 ); - CheckWriteCoins(VALUE1, ABSENT, VALUE1, FRESH , NO_ENTRY , FRESH ); - CheckWriteCoins(VALUE1, ABSENT, VALUE1, DIRTY , NO_ENTRY , DIRTY ); - CheckWriteCoins(VALUE1, ABSENT, VALUE1, DIRTY|FRESH, NO_ENTRY , DIRTY|FRESH); - CheckWriteCoins(VALUE1, SPENT , SPENT , 0 , DIRTY , DIRTY ); - CheckWriteCoins(VALUE1, SPENT , FAIL , 0 , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(VALUE1, SPENT , ABSENT, FRESH , DIRTY , NO_ENTRY ); - CheckWriteCoins(VALUE1, SPENT , FAIL , FRESH , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(VALUE1, SPENT , SPENT , DIRTY , DIRTY , DIRTY ); - CheckWriteCoins(VALUE1, SPENT , FAIL , DIRTY , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(VALUE1, SPENT , ABSENT, DIRTY|FRESH, DIRTY , NO_ENTRY ); - CheckWriteCoins(VALUE1, SPENT , FAIL , DIRTY|FRESH, DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(VALUE1, VALUE2, VALUE2, 0 , DIRTY , DIRTY ); - CheckWriteCoins(VALUE1, VALUE2, FAIL , 0 , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(VALUE1, VALUE2, VALUE2, FRESH , DIRTY , DIRTY|FRESH); - CheckWriteCoins(VALUE1, VALUE2, FAIL , FRESH , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY , DIRTY ); - CheckWriteCoins(VALUE1, VALUE2, FAIL , DIRTY , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY , DIRTY|FRESH); - CheckWriteCoins(VALUE1, VALUE2, FAIL , DIRTY|FRESH, DIRTY|FRESH, NO_ENTRY ); - - // The checks above omit cases where the child flags are not DIRTY, since + CheckWriteCoins(MISSING, MISSING, MISSING ); + CheckWriteCoins(MISSING, SPENT_DIRTY, SPENT_DIRTY ); + CheckWriteCoins(MISSING, SPENT_DIRTY_FRESH, MISSING ); + CheckWriteCoins(MISSING, VALUE2_DIRTY, VALUE2_DIRTY ); + CheckWriteCoins(MISSING, VALUE2_DIRTY_FRESH, VALUE2_DIRTY_FRESH ); + CheckWriteCoins(SPENT_CLEAN, MISSING, SPENT_CLEAN ); + CheckWriteCoins(SPENT_FRESH, MISSING, SPENT_FRESH ); + CheckWriteCoins(SPENT_DIRTY, MISSING, SPENT_DIRTY ); + CheckWriteCoins(SPENT_DIRTY_FRESH, MISSING, SPENT_DIRTY_FRESH ); + + CheckWriteCoins(SPENT_CLEAN, SPENT_DIRTY, SPENT_DIRTY ); + CheckWriteCoins(SPENT_CLEAN, SPENT_DIRTY_FRESH, SPENT_DIRTY ); + CheckWriteCoins(SPENT_FRESH, SPENT_DIRTY, MISSING ); + CheckWriteCoins(SPENT_FRESH, SPENT_DIRTY_FRESH, MISSING ); + CheckWriteCoins(SPENT_DIRTY, SPENT_DIRTY, SPENT_DIRTY ); + CheckWriteCoins(SPENT_DIRTY, SPENT_DIRTY_FRESH, SPENT_DIRTY ); + CheckWriteCoins(SPENT_DIRTY_FRESH, SPENT_DIRTY, MISSING ); + CheckWriteCoins(SPENT_DIRTY_FRESH, SPENT_DIRTY_FRESH, MISSING ); + + CheckWriteCoins(SPENT_CLEAN, VALUE2_DIRTY, VALUE2_DIRTY ); + CheckWriteCoins(SPENT_CLEAN, VALUE2_DIRTY_FRESH, VALUE2_DIRTY ); + CheckWriteCoins(SPENT_FRESH, VALUE2_DIRTY, VALUE2_DIRTY_FRESH ); + CheckWriteCoins(SPENT_FRESH, VALUE2_DIRTY_FRESH, VALUE2_DIRTY_FRESH ); + CheckWriteCoins(SPENT_DIRTY, VALUE2_DIRTY, VALUE2_DIRTY ); + CheckWriteCoins(SPENT_DIRTY, VALUE2_DIRTY_FRESH, VALUE2_DIRTY ); + CheckWriteCoins(SPENT_DIRTY_FRESH, VALUE2_DIRTY, VALUE2_DIRTY_FRESH ); + CheckWriteCoins(SPENT_DIRTY_FRESH, VALUE2_DIRTY_FRESH, VALUE2_DIRTY_FRESH ); + + CheckWriteCoins(VALUE1_CLEAN, MISSING, VALUE1_CLEAN ); + CheckWriteCoins(VALUE1_FRESH, MISSING, VALUE1_FRESH ); + CheckWriteCoins(VALUE1_DIRTY, MISSING, VALUE1_DIRTY ); + CheckWriteCoins(VALUE1_DIRTY_FRESH, MISSING, VALUE1_DIRTY_FRESH ); + CheckWriteCoins(VALUE1_CLEAN, SPENT_DIRTY, SPENT_DIRTY ); + CheckWriteCoins(VALUE1_CLEAN, SPENT_DIRTY_FRESH, EX_FRESH_MISAPPLIED); + CheckWriteCoins(VALUE1_FRESH, SPENT_DIRTY, MISSING ); + CheckWriteCoins(VALUE1_FRESH, SPENT_DIRTY_FRESH, EX_FRESH_MISAPPLIED); + CheckWriteCoins(VALUE1_DIRTY, SPENT_DIRTY, SPENT_DIRTY ); + CheckWriteCoins(VALUE1_DIRTY, SPENT_DIRTY_FRESH, EX_FRESH_MISAPPLIED); + CheckWriteCoins(VALUE1_DIRTY_FRESH, SPENT_DIRTY, MISSING ); + CheckWriteCoins(VALUE1_DIRTY_FRESH, SPENT_DIRTY_FRESH, EX_FRESH_MISAPPLIED); + + CheckWriteCoins(VALUE1_CLEAN, VALUE2_DIRTY, VALUE2_DIRTY ); + CheckWriteCoins(VALUE1_CLEAN, VALUE2_DIRTY_FRESH, EX_FRESH_MISAPPLIED); + CheckWriteCoins(VALUE1_FRESH, VALUE2_DIRTY, VALUE2_DIRTY_FRESH ); + CheckWriteCoins(VALUE1_FRESH, VALUE2_DIRTY_FRESH, EX_FRESH_MISAPPLIED); + CheckWriteCoins(VALUE1_DIRTY, VALUE2_DIRTY, VALUE2_DIRTY ); + CheckWriteCoins(VALUE1_DIRTY, VALUE2_DIRTY_FRESH, EX_FRESH_MISAPPLIED); + CheckWriteCoins(VALUE1_DIRTY_FRESH, VALUE2_DIRTY, VALUE2_DIRTY_FRESH ); + CheckWriteCoins(VALUE1_DIRTY_FRESH, VALUE2_DIRTY_FRESH, EX_FRESH_MISAPPLIED); + + // The checks above omit cases where the child state is not DIRTY, since // they would be too repetitive (the parent cache is never updated in these // cases). The loop below covers these cases and makes sure the parent cache // is always left unchanged. - for (const CAmount parent_value : {ABSENT, SPENT, VALUE1}) - for (const CAmount child_value : {ABSENT, SPENT, VALUE2}) - for (const char parent_flags : parent_value == ABSENT ? ABSENT_FLAGS : FLAGS) - for (const char child_flags : child_value == ABSENT ? ABSENT_FLAGS : CLEAN_FLAGS) - CheckWriteCoins(parent_value, child_value, parent_value, parent_flags, child_flags, parent_flags); + for (const MaybeCoin& parent : {MISSING, + SPENT_CLEAN, SPENT_DIRTY, SPENT_FRESH, SPENT_DIRTY_FRESH, + VALUE1_CLEAN, VALUE1_DIRTY, VALUE1_FRESH, VALUE1_DIRTY_FRESH}) { + for (const MaybeCoin& child : {MISSING, + SPENT_CLEAN, SPENT_FRESH, + VALUE2_CLEAN, VALUE2_FRESH}) { + auto expected{CoinOrError{parent}}; // TODO test failure cases as well + CheckWriteCoins(parent, child, expected); + } + } } - struct FlushTest : BasicTestingSetup { Coin MakeCoin() { @@ -924,8 +891,6 @@ void TestFlushBehavior( std::vector<std::unique_ptr<CCoinsViewCacheTest>>& all_caches, bool do_erasing_flush) { - CAmount value; - char flags; size_t cache_usage; size_t cache_size; @@ -959,9 +924,7 @@ void TestFlushBehavior( BOOST_CHECK(!base.HaveCoin(outp)); BOOST_CHECK(view->HaveCoin(outp)); - GetCoinsMapEntry(view->map(), value, flags, outp); - BOOST_CHECK_EQUAL(value, coin.out.nValue); - BOOST_CHECK_EQUAL(flags, DIRTY|FRESH); + BOOST_CHECK_EQUAL(GetCoinsMapEntry(view->map(), outp), CoinEntry(coin.out.nValue, CoinEntry::State::DIRTY_FRESH)); // --- 2. Flushing all caches (without erasing) // @@ -973,9 +936,7 @@ void TestFlushBehavior( // --- 3. Ensuring the entry still exists in the cache and has been written to parent // - GetCoinsMapEntry(view->map(), value, flags, outp); - BOOST_CHECK_EQUAL(value, coin.out.nValue); - BOOST_CHECK_EQUAL(flags, 0); // Flags should have been wiped. + BOOST_CHECK_EQUAL(GetCoinsMapEntry(view->map(), outp), CoinEntry(coin.out.nValue, CoinEntry::State::CLEAN)); // State should have been wiped. // Both views should now have the coin. BOOST_CHECK(base.HaveCoin(outp)); @@ -993,14 +954,9 @@ void TestFlushBehavior( // --- 5. Ensuring the entry is no longer in the cache // - GetCoinsMapEntry(view->map(), value, flags, outp); - BOOST_CHECK_EQUAL(value, ABSENT); - BOOST_CHECK_EQUAL(flags, NO_ENTRY); - + BOOST_CHECK(!GetCoinsMapEntry(view->map(), outp)); view->AccessCoin(outp); - GetCoinsMapEntry(view->map(), value, flags, outp); - BOOST_CHECK_EQUAL(value, coin.out.nValue); - BOOST_CHECK_EQUAL(flags, 0); + BOOST_CHECK_EQUAL(GetCoinsMapEntry(view->map(), outp), CoinEntry(coin.out.nValue, CoinEntry::State::CLEAN)); } // Can't overwrite an entry without specifying that an overwrite is @@ -1014,9 +970,7 @@ void TestFlushBehavior( BOOST_CHECK(view->SpendCoin(outp)); // The coin should be in the cache, but spent and marked dirty. - GetCoinsMapEntry(view->map(), value, flags, outp); - BOOST_CHECK_EQUAL(value, SPENT); - BOOST_CHECK_EQUAL(flags, DIRTY); + BOOST_CHECK_EQUAL(GetCoinsMapEntry(view->map(), outp), SPENT_DIRTY); BOOST_CHECK(!view->HaveCoin(outp)); // Coin should be considered spent in `view`. BOOST_CHECK(base.HaveCoin(outp)); // But coin should still be unspent in `base`. @@ -1067,10 +1021,7 @@ void TestFlushBehavior( all_caches[0]->AddCoin(outp, std::move(coin), false); // Coin should be FRESH in the cache. - GetCoinsMapEntry(all_caches[0]->map(), value, flags, outp); - BOOST_CHECK_EQUAL(value, coin_val); - BOOST_CHECK_EQUAL(flags, DIRTY|FRESH); - + BOOST_CHECK_EQUAL(GetCoinsMapEntry(all_caches[0]->map(), outp), CoinEntry(coin_val, CoinEntry::State::DIRTY_FRESH)); // Base shouldn't have seen coin. BOOST_CHECK(!base.HaveCoin(outp)); @@ -1078,9 +1029,7 @@ void TestFlushBehavior( all_caches[0]->Sync(); // Ensure there is no sign of the coin after spend/flush. - GetCoinsMapEntry(all_caches[0]->map(), value, flags, outp); - BOOST_CHECK_EQUAL(value, ABSENT); - BOOST_CHECK_EQUAL(flags, NO_ENTRY); + BOOST_CHECK(!GetCoinsMapEntry(all_caches[0]->map(), outp)); BOOST_CHECK(!all_caches[0]->HaveCoinInCache(outp)); BOOST_CHECK(!base.HaveCoin(outp)); } diff --git a/src/test/coinscachepair_tests.cpp b/src/test/coinscachepair_tests.cpp index 61840f1f09..0c208e93df 100644 --- a/src/test/coinscachepair_tests.cpp +++ b/src/test/coinscachepair_tests.cpp @@ -19,9 +19,9 @@ std::list<CoinsCachePair> CreatePairs(CoinsCachePair& sentinel) nodes.emplace_back(); auto node{std::prev(nodes.end())}; - node->second.AddFlags(CCoinsCacheEntry::DIRTY, *node, sentinel); + CCoinsCacheEntry::SetDirty(*node, sentinel); - BOOST_CHECK_EQUAL(node->second.GetFlags(), CCoinsCacheEntry::DIRTY); + BOOST_CHECK(node->second.IsDirty() && !node->second.IsFresh()); BOOST_CHECK_EQUAL(node->second.Next(), &sentinel); BOOST_CHECK_EQUAL(sentinel.second.Prev(), &(*node)); @@ -48,12 +48,12 @@ BOOST_AUTO_TEST_CASE(linked_list_iteration) BOOST_CHECK_EQUAL(node, &sentinel); // Check iterating through pairs is identical to iterating through a list - // Clear the flags during iteration + // Clear the state during iteration node = sentinel.second.Next(); for (const auto& expected : nodes) { BOOST_CHECK_EQUAL(&expected, node); auto next = node->second.Next(); - node->second.ClearFlags(); + node->second.SetClean(); node = next; } BOOST_CHECK_EQUAL(node, &sentinel); @@ -63,7 +63,7 @@ BOOST_AUTO_TEST_CASE(linked_list_iteration) // Delete the nodes from the list to make sure there are no dangling pointers for (auto it{nodes.begin()}; it != nodes.end(); it = nodes.erase(it)) { - BOOST_CHECK_EQUAL(it->second.GetFlags(), 0); + BOOST_CHECK(!it->second.IsDirty() && !it->second.IsFresh()); } } @@ -74,8 +74,8 @@ BOOST_AUTO_TEST_CASE(linked_list_iterate_erase) auto nodes{CreatePairs(sentinel)}; // Check iterating through pairs is identical to iterating through a list - // Erase the nodes as we iterate through, but don't clear flags - // The flags will be cleared by the CCoinsCacheEntry's destructor + // Erase the nodes as we iterate through, but don't clear state + // The state will be cleared by the CCoinsCacheEntry's destructor auto node{sentinel.second.Next()}; for (auto expected{nodes.begin()}; expected != nodes.end(); expected = nodes.erase(expected)) { BOOST_CHECK_EQUAL(&(*expected), node); @@ -104,10 +104,10 @@ BOOST_AUTO_TEST_CASE(linked_list_random_deletion) // sentinel->n1->n3->n4->sentinel nodes.erase(n2); // Check that n1 now points to n3, and n3 still points to n4 - // Also check that flags were not altered - BOOST_CHECK_EQUAL(n1->second.GetFlags(), CCoinsCacheEntry::DIRTY); + // Also check that state was not altered + BOOST_CHECK(n1->second.IsDirty() && !n1->second.IsFresh()); BOOST_CHECK_EQUAL(n1->second.Next(), &(*n3)); - BOOST_CHECK_EQUAL(n3->second.GetFlags(), CCoinsCacheEntry::DIRTY); + BOOST_CHECK(n3->second.IsDirty() && !n3->second.IsFresh()); BOOST_CHECK_EQUAL(n3->second.Next(), &(*n4)); BOOST_CHECK_EQUAL(n3->second.Prev(), &(*n1)); @@ -115,8 +115,8 @@ BOOST_AUTO_TEST_CASE(linked_list_random_deletion) // sentinel->n3->n4->sentinel nodes.erase(n1); // Check that sentinel now points to n3, and n3 still points to n4 - // Also check that flags were not altered - BOOST_CHECK_EQUAL(n3->second.GetFlags(), CCoinsCacheEntry::DIRTY); + // Also check that state was not altered + BOOST_CHECK(n3->second.IsDirty() && !n3->second.IsFresh()); BOOST_CHECK_EQUAL(sentinel.second.Next(), &(*n3)); BOOST_CHECK_EQUAL(n3->second.Next(), &(*n4)); BOOST_CHECK_EQUAL(n3->second.Prev(), &sentinel); @@ -125,8 +125,8 @@ BOOST_AUTO_TEST_CASE(linked_list_random_deletion) // sentinel->n3->sentinel nodes.erase(n4); // Check that sentinel still points to n3, and n3 points to sentinel - // Also check that flags were not altered - BOOST_CHECK_EQUAL(n3->second.GetFlags(), CCoinsCacheEntry::DIRTY); + // Also check that state was not altered + BOOST_CHECK(n3->second.IsDirty() && !n3->second.IsFresh()); BOOST_CHECK_EQUAL(sentinel.second.Next(), &(*n3)); BOOST_CHECK_EQUAL(n3->second.Next(), &sentinel); BOOST_CHECK_EQUAL(sentinel.second.Prev(), &(*n3)); @@ -139,77 +139,56 @@ BOOST_AUTO_TEST_CASE(linked_list_random_deletion) BOOST_CHECK_EQUAL(sentinel.second.Prev(), &sentinel); } -BOOST_AUTO_TEST_CASE(linked_list_add_flags) +BOOST_AUTO_TEST_CASE(linked_list_set_state) { CoinsCachePair sentinel; sentinel.second.SelfRef(sentinel); CoinsCachePair n1; CoinsCachePair n2; - // Check that adding 0 flag has no effect - n1.second.AddFlags(0, n1, sentinel); - BOOST_CHECK_EQUAL(n1.second.GetFlags(), 0); - BOOST_CHECK_EQUAL(sentinel.second.Next(), &sentinel); - BOOST_CHECK_EQUAL(sentinel.second.Prev(), &sentinel); - - // Check that adding DIRTY flag inserts it into linked list and sets flags - n1.second.AddFlags(CCoinsCacheEntry::DIRTY, n1, sentinel); - BOOST_CHECK_EQUAL(n1.second.GetFlags(), CCoinsCacheEntry::DIRTY); + // Check that setting DIRTY inserts it into linked list and sets state + CCoinsCacheEntry::SetDirty(n1, sentinel); + BOOST_CHECK(n1.second.IsDirty() && !n1.second.IsFresh()); BOOST_CHECK_EQUAL(n1.second.Next(), &sentinel); BOOST_CHECK_EQUAL(n1.second.Prev(), &sentinel); BOOST_CHECK_EQUAL(sentinel.second.Next(), &n1); BOOST_CHECK_EQUAL(sentinel.second.Prev(), &n1); - // Check that adding FRESH flag on new node inserts it after n1 - n2.second.AddFlags(CCoinsCacheEntry::FRESH, n2, sentinel); - BOOST_CHECK_EQUAL(n2.second.GetFlags(), CCoinsCacheEntry::FRESH); + // Check that setting FRESH on new node inserts it after n1 + CCoinsCacheEntry::SetFresh(n2, sentinel); + BOOST_CHECK(n2.second.IsFresh() && !n2.second.IsDirty()); BOOST_CHECK_EQUAL(n2.second.Next(), &sentinel); BOOST_CHECK_EQUAL(n2.second.Prev(), &n1); BOOST_CHECK_EQUAL(n1.second.Next(), &n2); BOOST_CHECK_EQUAL(sentinel.second.Prev(), &n2); - // Check that adding 0 flag has no effect, and doesn't change position - n1.second.AddFlags(0, n1, sentinel); - BOOST_CHECK_EQUAL(n1.second.GetFlags(), CCoinsCacheEntry::DIRTY); + // Check that we can set extra state, but they don't change our position + CCoinsCacheEntry::SetFresh(n1, sentinel); + BOOST_CHECK(n1.second.IsDirty() && n1.second.IsFresh()); BOOST_CHECK_EQUAL(n1.second.Next(), &n2); BOOST_CHECK_EQUAL(n1.second.Prev(), &sentinel); BOOST_CHECK_EQUAL(sentinel.second.Next(), &n1); BOOST_CHECK_EQUAL(n2.second.Prev(), &n1); - // Check that we can add extra flags, but they don't change our position - n1.second.AddFlags(CCoinsCacheEntry::FRESH, n1, sentinel); - BOOST_CHECK_EQUAL(n1.second.GetFlags(), CCoinsCacheEntry::DIRTY | CCoinsCacheEntry::FRESH); - BOOST_CHECK_EQUAL(n1.second.Next(), &n2); - BOOST_CHECK_EQUAL(n1.second.Prev(), &sentinel); - BOOST_CHECK_EQUAL(sentinel.second.Next(), &n1); - BOOST_CHECK_EQUAL(n2.second.Prev(), &n1); - - // Check that we can clear flags then re-add them - n1.second.ClearFlags(); - BOOST_CHECK_EQUAL(n1.second.GetFlags(), 0); - BOOST_CHECK_EQUAL(sentinel.second.Next(), &n2); - BOOST_CHECK_EQUAL(sentinel.second.Prev(), &n2); - BOOST_CHECK_EQUAL(n2.second.Next(), &sentinel); - BOOST_CHECK_EQUAL(n2.second.Prev(), &sentinel); - - // Check that calling `ClearFlags` with 0 flags has no effect - n1.second.ClearFlags(); - BOOST_CHECK_EQUAL(n1.second.GetFlags(), 0); + // Check that we can clear state then re-set it + n1.second.SetClean(); + BOOST_CHECK(!n1.second.IsDirty() && !n1.second.IsFresh()); BOOST_CHECK_EQUAL(sentinel.second.Next(), &n2); BOOST_CHECK_EQUAL(sentinel.second.Prev(), &n2); BOOST_CHECK_EQUAL(n2.second.Next(), &sentinel); BOOST_CHECK_EQUAL(n2.second.Prev(), &sentinel); - // Adding 0 still has no effect - n1.second.AddFlags(0, n1, sentinel); + // Calling `SetClean` a second time has no effect + n1.second.SetClean(); + BOOST_CHECK(!n1.second.IsDirty() && !n1.second.IsFresh()); BOOST_CHECK_EQUAL(sentinel.second.Next(), &n2); BOOST_CHECK_EQUAL(sentinel.second.Prev(), &n2); BOOST_CHECK_EQUAL(n2.second.Next(), &sentinel); BOOST_CHECK_EQUAL(n2.second.Prev(), &sentinel); - // But adding DIRTY re-inserts it after n2 - n1.second.AddFlags(CCoinsCacheEntry::DIRTY, n1, sentinel); - BOOST_CHECK_EQUAL(n1.second.GetFlags(), CCoinsCacheEntry::DIRTY); + // Adding DIRTY re-inserts it after n2 + CCoinsCacheEntry::SetDirty(n1, sentinel); + BOOST_CHECK(n1.second.IsDirty() && !n1.second.IsFresh()); BOOST_CHECK_EQUAL(n2.second.Next(), &n1); BOOST_CHECK_EQUAL(n1.second.Prev(), &n2); BOOST_CHECK_EQUAL(n1.second.Next(), &sentinel); diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 08814c1499..e09aad05e9 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -35,7 +35,7 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) BOOST_REQUIRE(coin_stats_index.StartBackgroundSync()); - IndexWaitSynced(coin_stats_index, *Assert(m_node.shutdown)); + IndexWaitSynced(coin_stats_index, *Assert(m_node.shutdown_signal)); // Check that CoinStatsIndex works for genesis block. const CBlockIndex* genesis_block_index; @@ -86,7 +86,7 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup) CoinStatsIndex index{interfaces::MakeChain(m_node), 1 << 20}; BOOST_REQUIRE(index.Init()); BOOST_REQUIRE(index.StartBackgroundSync()); - IndexWaitSynced(index, *Assert(m_node.shutdown)); + IndexWaitSynced(index, *Assert(m_node.shutdown_signal)); std::shared_ptr<const CBlock> new_block; CBlockIndex* new_block_index = nullptr; { diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index bb4c8c7093..f08c4d4428 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -224,6 +224,7 @@ void test_cache_erase_parallel(size_t megabytes) /** Spin up 3 threads to run contains with erase. */ std::vector<std::thread> threads; + threads.reserve(3); /** Erase the first quarter */ for (uint32_t x = 0; x < 3; ++x) /** Each thread is emplaced with x copy-by-value diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 24b8f2f793..288ffc66eb 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -161,7 +161,8 @@ void DoCheck(std::string prv, std::string pub, const std::string& norm_pub, int // We must be able to estimate the max satisfaction size for any solvable descriptor top descriptor (but combo). const bool is_nontop_or_nonsolvable{!parse_priv->IsSolvable() || !parse_priv->GetOutputType()}; const auto max_sat_maxsig{parse_priv->MaxSatisfactionWeight(true)}; - const auto max_sat_nonmaxsig{parse_priv->MaxSatisfactionWeight(true)}; + const auto max_sat_nonmaxsig{parse_priv->MaxSatisfactionWeight(false)}; + BOOST_CHECK(max_sat_nonmaxsig <= max_sat_maxsig); const auto max_elems{parse_priv->MaxSatisfactionElems()}; const bool is_input_size_info_set{max_sat_maxsig && max_sat_nonmaxsig && max_elems}; BOOST_CHECK_MESSAGE(is_input_size_info_set || is_nontop_or_nonsolvable, prv); diff --git a/src/test/feefrac_tests.cpp b/src/test/feefrac_tests.cpp index 5af3c3d7ed..41c9c0a633 100644 --- a/src/test/feefrac_tests.cpp +++ b/src/test/feefrac_tests.cpp @@ -15,7 +15,7 @@ BOOST_AUTO_TEST_CASE(feefrac_operators) FeeFrac sum{1500, 400}; FeeFrac diff{500, -200}; FeeFrac empty{0, 0}; - FeeFrac zero_fee{0, 1}; // zero-fee allowed + [[maybe_unused]] FeeFrac zero_fee{0, 1}; // zero-fee allowed BOOST_CHECK(empty == FeeFrac{}); // same as no-args diff --git a/src/test/fuzz/CMakeLists.txt b/src/test/fuzz/CMakeLists.txt index 165add2e5a..f65ed62b2d 100644 --- a/src/test/fuzz/CMakeLists.txt +++ b/src/test/fuzz/CMakeLists.txt @@ -72,9 +72,11 @@ add_executable(fuzz netbase_dns_lookup.cpp node_eviction.cpp p2p_handshake.cpp + p2p_headers_presync.cpp p2p_transport_serialization.cpp package_eval.cpp parse_hd_keypath.cpp + parse_iso8601.cpp parse_numbers.cpp parse_script.cpp parse_univalue.cpp @@ -116,12 +118,16 @@ add_executable(fuzz timeoffsets.cpp torcontrol.cpp transaction.cpp + txdownloadman.cpp tx_in.cpp tx_out.cpp tx_pool.cpp txorphan.cpp txrequest.cpp - utxo_snapshot.cpp + # Visual Studio 2022 version 17.12 introduced a bug + # that causes an internal compiler error. + # See: https://github.com/bitcoin/bitcoin/issues/31303 + $<$<VERSION_LESS:${MSVC_VERSION},1942>:utxo_snapshot.cpp> utxo_total_supply.cpp validation_load_mempool.cpp vecdeque.cpp @@ -138,7 +144,7 @@ target_link_libraries(fuzz univalue secp256k1 Boost::headers - $<TARGET_NAME_IF_EXISTS:libevent::libevent> + libevent::extra ) if(ENABLE_WALLET) diff --git a/src/test/fuzz/FuzzedDataProvider.h b/src/test/fuzz/FuzzedDataProvider.h index 5903ed8379..e57b95b630 100644 --- a/src/test/fuzz/FuzzedDataProvider.h +++ b/src/test/fuzz/FuzzedDataProvider.h @@ -18,6 +18,7 @@ #include <climits> #include <cstddef> #include <cstdint> +#include <cstdlib> #include <cstring> #include <initializer_list> #include <limits> diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index c324b4fdd9..dedea85080 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -39,15 +39,9 @@ void initialize_addrman() g_setup = testing_setup.get(); } -[[nodiscard]] inline NetGroupManager ConsumeNetGroupManager(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); - if (!SanityCheckASMap(asmap, 128)) asmap.clear(); - return NetGroupManager(asmap); -} - FUZZ_TARGET(data_stream_addr_man, .init = initialize_addrman) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; DataStream data_stream = ConsumeDataStream(fuzzed_data_provider); NetGroupManager netgroupman{ConsumeNetGroupManager(fuzzed_data_provider)}; @@ -118,121 +112,20 @@ void FillAddrman(AddrMan& addrman, FuzzedDataProvider& fuzzed_data_provider) } } -class AddrManDeterministic : public AddrMan -{ -public: - explicit AddrManDeterministic(const NetGroupManager& netgroupman, FuzzedDataProvider& fuzzed_data_provider) - : AddrMan(netgroupman, /*deterministic=*/true, GetCheckRatio()) - { - WITH_LOCK(m_impl->cs, m_impl->insecure_rand.Reseed(ConsumeUInt256(fuzzed_data_provider))); - } - - /** - * Compare with another AddrMan. - * This compares: - * - the values in `mapInfo` (the keys aka ids are ignored) - * - vvNew entries refer to the same addresses - * - vvTried entries refer to the same addresses - */ - bool operator==(const AddrManDeterministic& other) const - { - LOCK2(m_impl->cs, other.m_impl->cs); - - if (m_impl->mapInfo.size() != other.m_impl->mapInfo.size() || m_impl->nNew != other.m_impl->nNew || - m_impl->nTried != other.m_impl->nTried) { - return false; - } - - // Check that all values in `mapInfo` are equal to all values in `other.mapInfo`. - // Keys may be different. - - auto addrinfo_hasher = [](const AddrInfo& a) { - CSipHasher hasher(0, 0); - auto addr_key = a.GetKey(); - auto source_key = a.source.GetAddrBytes(); - hasher.Write(TicksSinceEpoch<std::chrono::seconds>(a.m_last_success)); - hasher.Write(a.nAttempts); - hasher.Write(a.nRefCount); - hasher.Write(a.fInTried); - hasher.Write(a.GetNetwork()); - hasher.Write(a.source.GetNetwork()); - hasher.Write(addr_key.size()); - hasher.Write(source_key.size()); - hasher.Write(addr_key); - hasher.Write(source_key); - return (size_t)hasher.Finalize(); - }; - - auto addrinfo_eq = [](const AddrInfo& lhs, const AddrInfo& rhs) { - return std::tie(static_cast<const CService&>(lhs), lhs.source, lhs.m_last_success, lhs.nAttempts, lhs.nRefCount, lhs.fInTried) == - std::tie(static_cast<const CService&>(rhs), rhs.source, rhs.m_last_success, rhs.nAttempts, rhs.nRefCount, rhs.fInTried); - }; - - using Addresses = std::unordered_set<AddrInfo, decltype(addrinfo_hasher), decltype(addrinfo_eq)>; - - const size_t num_addresses{m_impl->mapInfo.size()}; - - Addresses addresses{num_addresses, addrinfo_hasher, addrinfo_eq}; - for (const auto& [id, addr] : m_impl->mapInfo) { - addresses.insert(addr); - } - - Addresses other_addresses{num_addresses, addrinfo_hasher, addrinfo_eq}; - for (const auto& [id, addr] : other.m_impl->mapInfo) { - other_addresses.insert(addr); - } - - if (addresses != other_addresses) { - return false; - } - - auto IdsReferToSameAddress = [&](int id, int other_id) EXCLUSIVE_LOCKS_REQUIRED(m_impl->cs, other.m_impl->cs) { - if (id == -1 && other_id == -1) { - return true; - } - if ((id == -1 && other_id != -1) || (id != -1 && other_id == -1)) { - return false; - } - return m_impl->mapInfo.at(id) == other.m_impl->mapInfo.at(other_id); - }; - - // Check that `vvNew` contains the same addresses as `other.vvNew`. Notice - `vvNew[i][j]` - // contains just an id and the address is to be found in `mapInfo.at(id)`. The ids - // themselves may differ between `vvNew` and `other.vvNew`. - for (size_t i = 0; i < ADDRMAN_NEW_BUCKET_COUNT; ++i) { - for (size_t j = 0; j < ADDRMAN_BUCKET_SIZE; ++j) { - if (!IdsReferToSameAddress(m_impl->vvNew[i][j], other.m_impl->vvNew[i][j])) { - return false; - } - } - } - - // Same for `vvTried`. - for (size_t i = 0; i < ADDRMAN_TRIED_BUCKET_COUNT; ++i) { - for (size_t j = 0; j < ADDRMAN_BUCKET_SIZE; ++j) { - if (!IdsReferToSameAddress(m_impl->vvTried[i][j], other.m_impl->vvTried[i][j])) { - return false; - } - } - } - - return true; - } -}; - FUZZ_TARGET(addrman, .init = initialize_addrman) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); NetGroupManager netgroupman{ConsumeNetGroupManager(fuzzed_data_provider)}; - auto addr_man_ptr = std::make_unique<AddrManDeterministic>(netgroupman, fuzzed_data_provider); + auto addr_man_ptr = std::make_unique<AddrManDeterministic>(netgroupman, fuzzed_data_provider, GetCheckRatio()); if (fuzzed_data_provider.ConsumeBool()) { const std::vector<uint8_t> serialized_data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; DataStream ds{serialized_data}; try { ds >> *addr_man_ptr; } catch (const std::ios_base::failure&) { - addr_man_ptr = std::make_unique<AddrManDeterministic>(netgroupman, fuzzed_data_provider); + addr_man_ptr = std::make_unique<AddrManDeterministic>(netgroupman, fuzzed_data_provider, GetCheckRatio()); } } AddrManDeterministic& addr_man = *addr_man_ptr; @@ -282,10 +175,18 @@ FUZZ_TARGET(addrman, .init = initialize_addrman) network = fuzzed_data_provider.PickValueInArray(ALL_NETWORKS); } auto max_addresses = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096); - auto max_pct = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096); + auto max_pct = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 100); auto filtered = fuzzed_data_provider.ConsumeBool(); (void)const_addr_man.GetAddr(max_addresses, max_pct, network, filtered); - (void)const_addr_man.Select(fuzzed_data_provider.ConsumeBool(), network); + + std::unordered_set<Network> nets; + for (const auto& net : ALL_NETWORKS) { + if (fuzzed_data_provider.ConsumeBool()) { + nets.insert(net); + } + } + (void)const_addr_man.Select(fuzzed_data_provider.ConsumeBool(), nets); + std::optional<bool> in_new; if (fuzzed_data_provider.ConsumeBool()) { in_new = fuzzed_data_provider.ConsumeBool(); @@ -298,12 +199,13 @@ FUZZ_TARGET(addrman, .init = initialize_addrman) // Check that serialize followed by unserialize produces the same addrman. FUZZ_TARGET(addrman_serdeser, .init = initialize_addrman) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); NetGroupManager netgroupman{ConsumeNetGroupManager(fuzzed_data_provider)}; - AddrManDeterministic addr_man1{netgroupman, fuzzed_data_provider}; - AddrManDeterministic addr_man2{netgroupman, fuzzed_data_provider}; + AddrManDeterministic addr_man1{netgroupman, fuzzed_data_provider, GetCheckRatio()}; + AddrManDeterministic addr_man2{netgroupman, fuzzed_data_provider, GetCheckRatio()}; DataStream data_stream{}; diff --git a/src/test/fuzz/autofile.cpp b/src/test/fuzz/autofile.cpp index 45316b6b21..81761c7bf9 100644 --- a/src/test/fuzz/autofile.cpp +++ b/src/test/fuzz/autofile.cpp @@ -56,7 +56,6 @@ FUZZ_TARGET(autofile) WriteToStream(fuzzed_data_provider, auto_file); }); } - (void)auto_file.Get(); (void)auto_file.IsNull(); if (fuzzed_data_provider.ConsumeBool()) { FILE* f = auto_file.release(); diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp index 4165cc6b2c..bbe8e12b2f 100644 --- a/src/test/fuzz/banman.cpp +++ b/src/test/fuzz/banman.cpp @@ -42,6 +42,7 @@ static bool operator==(const CBanEntry& lhs, const CBanEntry& rhs) FUZZ_TARGET(banman, .init = initialize_banman) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); fs::path banlist_file = gArgs.GetDataDirNet() / "fuzzed_banlist"; diff --git a/src/test/fuzz/blockfilter.cpp b/src/test/fuzz/blockfilter.cpp index 3adc114515..17e99ac94d 100644 --- a/src/test/fuzz/blockfilter.cpp +++ b/src/test/fuzz/blockfilter.cpp @@ -6,6 +6,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/util/random.h> #include <cstdint> #include <optional> @@ -14,6 +15,7 @@ FUZZ_TARGET(blockfilter) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::optional<BlockFilter> block_filter = ConsumeDeserializable<BlockFilter>(fuzzed_data_provider); if (!block_filter) { diff --git a/src/test/fuzz/checkqueue.cpp b/src/test/fuzz/checkqueue.cpp index 6320b500b6..6b93886c71 100644 --- a/src/test/fuzz/checkqueue.cpp +++ b/src/test/fuzz/checkqueue.cpp @@ -19,9 +19,10 @@ struct DumbCheck { { } - bool operator()() const + std::optional<int> operator()() const { - return result; + if (result) return std::nullopt; + return 1; } }; } // namespace @@ -45,7 +46,7 @@ FUZZ_TARGET(checkqueue) check_queue_1.Add(std::move(checks_1)); } if (fuzzed_data_provider.ConsumeBool()) { - (void)check_queue_1.Wait(); + (void)check_queue_1.Complete(); } CCheckQueueControl<DumbCheck> check_queue_control{&check_queue_2}; @@ -53,6 +54,6 @@ FUZZ_TARGET(checkqueue) check_queue_control.Add(std::move(checks_2)); } if (fuzzed_data_provider.ConsumeBool()) { - (void)check_queue_control.Wait(); + (void)check_queue_control.Complete(); } } diff --git a/src/test/fuzz/cluster_linearize.cpp b/src/test/fuzz/cluster_linearize.cpp index 2dfdfbb41d..5b3770636a 100644 --- a/src/test/fuzz/cluster_linearize.cpp +++ b/src/test/fuzz/cluster_linearize.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <cluster_linearize.h> +#include <random.h> #include <serialize.h> #include <streams.h> #include <test/fuzz/fuzz.h> @@ -36,7 +37,7 @@ class SimpleCandidateFinder public: /** Construct an SimpleCandidateFinder for a given graph. */ SimpleCandidateFinder(const DepGraph<SetType>& depgraph LIFETIMEBOUND) noexcept : - m_depgraph(depgraph), m_todo{SetType::Fill(depgraph.TxCount())} {} + m_depgraph(depgraph), m_todo{depgraph.Positions()} {} /** Remove a set of transactions from the set of to-be-linearized ones. */ void MarkDone(SetType select) noexcept { m_todo -= select; } @@ -106,7 +107,7 @@ class ExhaustiveCandidateFinder public: /** Construct an ExhaustiveCandidateFinder for a given graph. */ ExhaustiveCandidateFinder(const DepGraph<SetType>& depgraph LIFETIMEBOUND) noexcept : - m_depgraph(depgraph), m_todo{SetType::Fill(depgraph.TxCount())} {} + m_depgraph(depgraph), m_todo{depgraph.Positions()} {} /** Remove a set of transactions from the set of to-be-linearized ones. */ void MarkDone(SetType select) noexcept { m_todo -= select; } @@ -152,7 +153,7 @@ std::pair<std::vector<ClusterIndex>, bool> SimpleLinearize(const DepGraph<SetTyp { std::vector<ClusterIndex> linearization; SimpleCandidateFinder finder(depgraph); - SetType todo = SetType::Fill(depgraph.TxCount()); + SetType todo = depgraph.Positions(); bool optimal = true; while (todo.Any()) { auto [candidate, iterations_done] = finder.FindCandidateSet(max_iterations); @@ -165,6 +166,23 @@ std::pair<std::vector<ClusterIndex>, bool> SimpleLinearize(const DepGraph<SetTyp return {std::move(linearization), optimal}; } +/** Stitch connected components together in a DepGraph, guaranteeing its corresponding cluster is connected. */ +template<typename BS> +void MakeConnected(DepGraph<BS>& depgraph) +{ + auto todo = depgraph.Positions(); + auto comp = depgraph.FindConnectedComponent(todo); + Assume(depgraph.IsConnected(comp)); + todo -= comp; + while (todo.Any()) { + auto nextcomp = depgraph.FindConnectedComponent(todo); + Assume(depgraph.IsConnected(nextcomp)); + depgraph.AddDependencies(BS::Singleton(comp.Last()), nextcomp.First()); + todo -= nextcomp; + comp = nextcomp; + } +} + /** Given a dependency graph, and a todo set, read a topological subset of todo from reader. */ template<typename SetType> SetType ReadTopologicalSet(const DepGraph<SetType>& depgraph, const SetType& todo, SpanReader& reader) @@ -188,7 +206,7 @@ template<typename BS> std::vector<ClusterIndex> ReadLinearization(const DepGraph<BS>& depgraph, SpanReader& reader) { std::vector<ClusterIndex> linearization; - TestBitSet todo = TestBitSet::Fill(depgraph.TxCount()); + TestBitSet todo = depgraph.Positions(); // In every iteration one topologically-valid transaction is appended to linearization. while (todo.Any()) { // Compute the set of transactions with no not-yet-included ancestors. @@ -223,59 +241,157 @@ std::vector<ClusterIndex> ReadLinearization(const DepGraph<BS>& depgraph, SpanRe } // namespace -FUZZ_TARGET(clusterlin_add_dependency) -{ - // Verify that computing a DepGraph from a cluster, or building it step by step using AddDependency - // have the same effect. - - // Construct a cluster of a certain length, with no dependencies. - FuzzedDataProvider provider(buffer.data(), buffer.size()); - auto num_tx = provider.ConsumeIntegralInRange<ClusterIndex>(2, 32); - Cluster<TestBitSet> cluster(num_tx, std::pair{FeeFrac{0, 1}, TestBitSet{}}); - // Construct the corresponding DepGraph object (also no dependencies). - DepGraph depgraph(cluster); - SanityCheck(depgraph); - // Read (parent, child) pairs, and add them to the cluster and depgraph. - LIMITED_WHILE(provider.remaining_bytes() > 0, TestBitSet::Size() * TestBitSet::Size()) { - auto parent = provider.ConsumeIntegralInRange<ClusterIndex>(0, num_tx - 1); - auto child = provider.ConsumeIntegralInRange<ClusterIndex>(0, num_tx - 2); - child += (child >= parent); - cluster[child].second.Set(parent); - depgraph.AddDependency(parent, child); - assert(depgraph.Ancestors(child)[parent]); - assert(depgraph.Descendants(parent)[child]); - } - // Sanity check the result. - SanityCheck(depgraph); - // Verify that the resulting DepGraph matches one recomputed from the cluster. - assert(DepGraph(cluster) == depgraph); -} - -FUZZ_TARGET(clusterlin_cluster_serialization) +FUZZ_TARGET(clusterlin_depgraph_sim) { - // Verify that any graph of transactions has its ancestry correctly computed by DepGraph, and - // if it is a DAG, that it can be serialized as a DepGraph in a way that roundtrips. This - // guarantees that any acyclic cluster has a corresponding DepGraph serialization. + // Simulation test to verify the full behavior of DepGraph. FuzzedDataProvider provider(buffer.data(), buffer.size()); - // Construct a cluster in a naive way (using a FuzzedDataProvider-based serialization). - Cluster<TestBitSet> cluster; - auto num_tx = provider.ConsumeIntegralInRange<ClusterIndex>(1, 32); - cluster.resize(num_tx); - for (ClusterIndex i = 0; i < num_tx; ++i) { - cluster[i].first.size = provider.ConsumeIntegralInRange<int32_t>(1, 0x3fffff); - cluster[i].first.fee = provider.ConsumeIntegralInRange<int64_t>(-0x8000000000000, 0x7ffffffffffff); - for (ClusterIndex j = 0; j < num_tx; ++j) { - if (i == j) continue; - if (provider.ConsumeBool()) cluster[i].second.Set(j); + /** Real DepGraph being tested. */ + DepGraph<TestBitSet> real; + /** Simulated DepGraph (sim[i] is std::nullopt if position i does not exist; otherwise, + * sim[i]->first is its individual feerate, and sim[i]->second is its set of ancestors. */ + std::array<std::optional<std::pair<FeeFrac, TestBitSet>>, TestBitSet::Size()> sim; + /** The number of non-nullopt position in sim. */ + ClusterIndex num_tx_sim{0}; + + /** Read a valid index of a transaction from the provider. */ + auto idx_fn = [&]() { + auto offset = provider.ConsumeIntegralInRange<ClusterIndex>(0, num_tx_sim - 1); + for (ClusterIndex i = 0; i < sim.size(); ++i) { + if (!sim[i].has_value()) continue; + if (offset == 0) return i; + --offset; + } + assert(false); + return ClusterIndex(-1); + }; + + /** Read a valid subset of the transactions from the provider. */ + auto subset_fn = [&]() { + auto range = (uint64_t{1} << num_tx_sim) - 1; + const auto mask = provider.ConsumeIntegralInRange<uint64_t>(0, range); + auto mask_shifted = mask; + TestBitSet subset; + for (ClusterIndex i = 0; i < sim.size(); ++i) { + if (!sim[i].has_value()) continue; + if (mask_shifted & 1) { + subset.Set(i); + } + mask_shifted >>= 1; + } + assert(mask_shifted == 0); + return subset; + }; + + /** Read any set of transactions from the provider (including unused positions). */ + auto set_fn = [&]() { + auto range = (uint64_t{1} << sim.size()) - 1; + const auto mask = provider.ConsumeIntegralInRange<uint64_t>(0, range); + TestBitSet set; + for (ClusterIndex i = 0; i < sim.size(); ++i) { + if ((mask >> i) & 1) { + set.Set(i); + } + } + return set; + }; + + /** Propagate ancestor information in sim. */ + auto anc_update_fn = [&]() { + while (true) { + bool updates{false}; + for (ClusterIndex chl = 0; chl < sim.size(); ++chl) { + if (!sim[chl].has_value()) continue; + for (auto par : sim[chl]->second) { + if (!sim[chl]->second.IsSupersetOf(sim[par]->second)) { + sim[chl]->second |= sim[par]->second; + updates = true; + } + } + } + if (!updates) break; + } + }; + + /** Compare the state of transaction i in the simulation with the real one. */ + auto check_fn = [&](ClusterIndex i) { + // Compare used positions. + assert(real.Positions()[i] == sim[i].has_value()); + if (sim[i].has_value()) { + // Compare feerate. + assert(real.FeeRate(i) == sim[i]->first); + // Compare ancestors (note that SanityCheck verifies correspondence between ancestors + // and descendants, so we can restrict ourselves to ancestors here). + assert(real.Ancestors(i) == sim[i]->second); } + }; + + LIMITED_WHILE(provider.remaining_bytes() > 0, 1000) { + uint8_t command = provider.ConsumeIntegral<uint8_t>(); + if (num_tx_sim == 0 || ((command % 3) <= 0 && num_tx_sim < TestBitSet::Size())) { + // AddTransaction. + auto fee = provider.ConsumeIntegralInRange<int64_t>(-0x8000000000000, 0x7ffffffffffff); + auto size = provider.ConsumeIntegralInRange<int32_t>(1, 0x3fffff); + FeeFrac feerate{fee, size}; + // Apply to DepGraph. + auto idx = real.AddTransaction(feerate); + // Verify that the returned index is correct. + assert(!sim[idx].has_value()); + for (ClusterIndex i = 0; i < TestBitSet::Size(); ++i) { + if (!sim[i].has_value()) { + assert(idx == i); + break; + } + } + // Update sim. + sim[idx] = {feerate, TestBitSet::Singleton(idx)}; + ++num_tx_sim; + continue; + } + if ((command % 3) <= 1 && num_tx_sim > 0) { + // AddDependencies. + ClusterIndex child = idx_fn(); + auto parents = subset_fn(); + // Apply to DepGraph. + real.AddDependencies(parents, child); + // Apply to sim. + sim[child]->second |= parents; + continue; + } + if (num_tx_sim > 0) { + // Remove transactions. + auto del = set_fn(); + // Propagate all ancestry information before deleting anything in the simulation (as + // intermediary transactions may be deleted which impact connectivity). + anc_update_fn(); + // Compare the state of the transactions being deleted. + for (auto i : del) check_fn(i); + // Apply to DepGraph. + real.RemoveTransactions(del); + // Apply to sim. + for (ClusterIndex i = 0; i < sim.size(); ++i) { + if (sim[i].has_value()) { + if (del[i]) { + --num_tx_sim; + sim[i] = std::nullopt; + } else { + sim[i]->second -= del; + } + } + } + continue; + } + // This should be unreachable (one of the 3 above actions should always be possible). + assert(false); } - // Construct dependency graph, and verify it matches the cluster (which includes a round-trip - // check for the serialization). - DepGraph depgraph(cluster); - VerifyDepGraphFromCluster(cluster, depgraph); + // Compare the real obtained depgraph against the simulation. + anc_update_fn(); + for (ClusterIndex i = 0; i < sim.size(); ++i) check_fn(i); + assert(real.TxCount() == num_tx_sim); + // Sanity check the result (which includes round-tripping serialization, if applicable). + SanityCheck(real); } FUZZ_TARGET(clusterlin_depgraph_serialization) @@ -305,7 +421,7 @@ FUZZ_TARGET(clusterlin_components) reader >> Using<DepGraphFormatter>(depgraph); } catch (const std::ios_base::failure&) {} - TestBitSet todo = TestBitSet::Fill(depgraph.TxCount()); + TestBitSet todo = depgraph.Positions(); while (todo.Any()) { // Find a connected component inside todo. auto component = depgraph.FindConnectedComponent(todo); @@ -316,7 +432,7 @@ FUZZ_TARGET(clusterlin_components) // If todo is the entire graph, and the entire graph is connected, then the component must // be the entire graph. - if (todo == TestBitSet::Fill(depgraph.TxCount())) { + if (todo == depgraph.Positions()) { assert((component == todo) == depgraph.IsConnected()); } @@ -353,7 +469,7 @@ FUZZ_TARGET(clusterlin_components) reader >> VARINT(subset_bits); } catch (const std::ios_base::failure&) {} TestBitSet subset; - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + for (ClusterIndex i : depgraph.Positions()) { if (todo[i]) { if (subset_bits & 1) subset.Set(i); subset_bits >>= 1; @@ -369,6 +485,20 @@ FUZZ_TARGET(clusterlin_components) assert(depgraph.FindConnectedComponent(todo).None()); } +FUZZ_TARGET(clusterlin_make_connected) +{ + // Verify that MakeConnected makes graphs connected. + + SpanReader reader(buffer); + DepGraph<TestBitSet> depgraph; + try { + reader >> Using<DepGraphFormatter>(depgraph); + } catch (const std::ios_base::failure&) {} + MakeConnected(depgraph); + SanityCheck(depgraph); + assert(depgraph.IsConnected()); +} + FUZZ_TARGET(clusterlin_chunking) { // Verify the correctness of the ChunkLinearization function. @@ -392,13 +522,13 @@ FUZZ_TARGET(clusterlin_chunking) } // Naively recompute the chunks (each is the highest-feerate prefix of what remains). - auto todo = TestBitSet::Fill(depgraph.TxCount()); + auto todo = depgraph.Positions(); for (const auto& chunk_feerate : chunking) { assert(todo.Any()); SetInfo<TestBitSet> accumulator, best; for (ClusterIndex idx : linearization) { if (todo[idx]) { - accumulator |= SetInfo(depgraph, idx); + accumulator.Set(depgraph, idx); if (best.feerate.IsEmpty() || accumulator.feerate >> best.feerate) { best = accumulator; } @@ -423,10 +553,11 @@ FUZZ_TARGET(clusterlin_ancestor_finder) } catch (const std::ios_base::failure&) {} AncestorCandidateFinder anc_finder(depgraph); - auto todo = TestBitSet::Fill(depgraph.TxCount()); + auto todo = depgraph.Positions(); while (todo.Any()) { // Call the ancestor finder's FindCandidateSet for what remains of the graph. assert(!anc_finder.AllDone()); + assert(todo.Count() == anc_finder.NumRemaining()); auto best_anc = anc_finder.FindCandidateSet(); // Sanity check the result. assert(best_anc.transactions.Any()); @@ -458,6 +589,7 @@ FUZZ_TARGET(clusterlin_ancestor_finder) anc_finder.MarkDone(del_set); } assert(anc_finder.AllDone()); + assert(anc_finder.NumRemaining() == 0); } static constexpr auto MAX_SIMPLE_ITERATIONS = 300000; @@ -468,13 +600,17 @@ FUZZ_TARGET(clusterlin_search_finder) // and comparing with the results from SimpleCandidateFinder, ExhaustiveCandidateFinder, and // AncestorCandidateFinder. - // Retrieve an RNG seed and a depgraph from the fuzz input. + // Retrieve an RNG seed, a depgraph, and whether to make it connected, from the fuzz input. SpanReader reader(buffer); DepGraph<TestBitSet> depgraph; uint64_t rng_seed{0}; + uint8_t make_connected{1}; try { - reader >> Using<DepGraphFormatter>(depgraph) >> rng_seed; + reader >> Using<DepGraphFormatter>(depgraph) >> rng_seed >> make_connected; } catch (const std::ios_base::failure&) {} + // The most complicated graphs are connected ones (other ones just split up). Optionally force + // the graph to be connected. + if (make_connected) MakeConnected(depgraph); // Instantiate ALL the candidate finders. SearchCandidateFinder src_finder(depgraph, rng_seed); @@ -482,12 +618,13 @@ FUZZ_TARGET(clusterlin_search_finder) ExhaustiveCandidateFinder exh_finder(depgraph); AncestorCandidateFinder anc_finder(depgraph); - auto todo = TestBitSet::Fill(depgraph.TxCount()); + auto todo = depgraph.Positions(); while (todo.Any()) { assert(!src_finder.AllDone()); assert(!smp_finder.AllDone()); assert(!exh_finder.AllDone()); assert(!anc_finder.AllDone()); + assert(anc_finder.NumRemaining() == todo.Count()); // For each iteration, read an iteration count limit from the fuzz input. uint64_t max_iterations = 1; @@ -513,9 +650,17 @@ FUZZ_TARGET(clusterlin_search_finder) assert(found.transactions.IsSupersetOf(depgraph.Ancestors(i) & todo)); } - // At most 2^N-1 iterations can be required: the number of non-empty subsets a graph with N - // transactions has. - assert(iterations_done <= ((uint64_t{1} << todo.Count()) - 1)); + // At most 2^(N-1) iterations can be required: the maximum number of non-empty topological + // subsets a (connected) cluster with N transactions can have. Even when the cluster is no + // longer connected after removing certain transactions, this holds, because the connected + // components are searched separately. + assert(iterations_done <= (uint64_t{1} << (todo.Count() - 1))); + // Additionally, test that no more than sqrt(2^N)+1 iterations are required. This is just + // an empirical bound that seems to hold, without proof. Still, add a test for it so we + // can learn about counterexamples if they exist. + if (iterations_done >= 1 && todo.Count() <= 63) { + Assume((iterations_done - 1) * (iterations_done - 1) <= uint64_t{1} << todo.Count()); + } // Perform quality checks only if SearchCandidateFinder claims an optimal result. if (iterations_done < max_iterations) { @@ -562,6 +707,7 @@ FUZZ_TARGET(clusterlin_search_finder) assert(smp_finder.AllDone()); assert(exh_finder.AllDone()); assert(anc_finder.AllDone()); + assert(anc_finder.NumRemaining() == 0); } FUZZ_TARGET(clusterlin_linearization_chunking) @@ -576,7 +722,7 @@ FUZZ_TARGET(clusterlin_linearization_chunking) } catch (const std::ios_base::failure&) {} // Retrieve a topologically-valid subset of depgraph. - auto todo = TestBitSet::Fill(depgraph.TxCount()); + auto todo = depgraph.Positions(); auto subset = SetInfo(depgraph, ReadTopologicalSet(depgraph, todo, reader)); // Retrieve a valid linearization for depgraph. @@ -621,7 +767,7 @@ FUZZ_TARGET(clusterlin_linearization_chunking) SetInfo<TestBitSet> accumulator, best; for (auto j : linearization) { if (todo[j] && !combined[j]) { - accumulator |= SetInfo(depgraph, j); + accumulator.Set(depgraph, j); if (best.feerate.IsEmpty() || accumulator.feerate > best.feerate) { best = accumulator; } @@ -685,14 +831,19 @@ FUZZ_TARGET(clusterlin_linearize) { // Verify the behavior of Linearize(). - // Retrieve an RNG seed, an iteration count, and a depgraph from the fuzz input. + // Retrieve an RNG seed, an iteration count, a depgraph, and whether to make it connected from + // the fuzz input. SpanReader reader(buffer); DepGraph<TestBitSet> depgraph; uint64_t rng_seed{0}; uint64_t iter_count{0}; + uint8_t make_connected{1}; try { - reader >> VARINT(iter_count) >> Using<DepGraphFormatter>(depgraph) >> rng_seed; + reader >> VARINT(iter_count) >> Using<DepGraphFormatter>(depgraph) >> rng_seed >> make_connected; } catch (const std::ios_base::failure&) {} + // The most complicated graphs are connected ones (other ones just split up). Optionally force + // the graph to be connected. + if (make_connected) MakeConnected(depgraph); // Optionally construct an old linearization for it. std::vector<ClusterIndex> old_linearization; @@ -721,12 +872,24 @@ FUZZ_TARGET(clusterlin_linearize) } // If the iteration count is sufficiently high, an optimal linearization must be found. - // Each linearization step can use up to 2^k iterations, with steps k=1..n. That sum is - // 2 * (2^n - 1) + // Each linearization step can use up to 2^(k-1) iterations, with steps k=1..n. That sum is + // 2^n - 1. const uint64_t n = depgraph.TxCount(); - if (n <= 18 && iter_count > 2U * ((uint64_t{1} << n) - 1U)) { + if (n <= 19 && iter_count > (uint64_t{1} << n)) { assert(optimal); } + // Additionally, if the assumption of sqrt(2^k)+1 iterations per step holds, plus ceil(k/4) + // start-up cost per step, plus ceil(n^2/64) start-up cost overall, we can compute the upper + // bound for a whole linearization (summing for k=1..n) using the Python expression + // [sum((k+3)//4 + int(math.sqrt(2**k)) + 1 for k in range(1, n + 1)) + (n**2 + 63) // 64 for n in range(0, 35)]: + static constexpr uint64_t MAX_OPTIMAL_ITERS[] = { + 0, 4, 8, 12, 18, 26, 37, 51, 70, 97, 133, 182, 251, 346, 480, 666, 927, 1296, 1815, 2545, + 3576, 5031, 7087, 9991, 14094, 19895, 28096, 39690, 56083, 79263, 112041, 158391, 223936, + 316629, 447712 + }; + if (n < std::size(MAX_OPTIMAL_ITERS) && iter_count >= MAX_OPTIMAL_ITERS[n]) { + Assume(optimal); + } // If Linearize claims optimal result, run quality tests. if (optimal) { @@ -742,8 +905,8 @@ FUZZ_TARGET(clusterlin_linearize) // Only for very small clusters, test every topologically-valid permutation. if (depgraph.TxCount() <= 7) { - std::vector<ClusterIndex> perm_linearization(depgraph.TxCount()); - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) perm_linearization[i] = i; + std::vector<ClusterIndex> perm_linearization; + for (ClusterIndex i : depgraph.Positions()) perm_linearization.push_back(i); // Iterate over all valid permutations. do { // Determine whether perm_linearization is topological. @@ -827,30 +990,30 @@ FUZZ_TARGET(clusterlin_postlinearize_tree) // Now construct a new graph, copying the nodes, but leaving only the first parent (even // direction) or the first child (odd direction). DepGraph<TestBitSet> depgraph_tree; - for (ClusterIndex i = 0; i < depgraph_gen.TxCount(); ++i) { - depgraph_tree.AddTransaction(depgraph_gen.FeeRate(i)); + for (ClusterIndex i = 0; i < depgraph_gen.PositionRange(); ++i) { + if (depgraph_gen.Positions()[i]) { + depgraph_tree.AddTransaction(depgraph_gen.FeeRate(i)); + } else { + // For holes, add a dummy transaction which is deleted below, so that non-hole + // transactions retain their position. + depgraph_tree.AddTransaction(FeeFrac{}); + } } + depgraph_tree.RemoveTransactions(TestBitSet::Fill(depgraph_gen.PositionRange()) - depgraph_gen.Positions()); + if (direction & 1) { for (ClusterIndex i = 0; i < depgraph_gen.TxCount(); ++i) { - auto children = depgraph_gen.Descendants(i) - TestBitSet::Singleton(i); - // Remove descendants that are children of other descendants. - for (auto j : children) { - if (!children[j]) continue; - children -= depgraph_gen.Descendants(j); - children.Set(j); + auto children = depgraph_gen.GetReducedChildren(i); + if (children.Any()) { + depgraph_tree.AddDependencies(TestBitSet::Singleton(i), children.First()); } - if (children.Any()) depgraph_tree.AddDependency(i, children.First()); } } else { for (ClusterIndex i = 0; i < depgraph_gen.TxCount(); ++i) { - auto parents = depgraph_gen.Ancestors(i) - TestBitSet::Singleton(i); - // Remove ancestors that are parents of other ancestors. - for (auto j : parents) { - if (!parents[j]) continue; - parents -= depgraph_gen.Ancestors(j); - parents.Set(j); + auto parents = depgraph_gen.GetReducedParents(i); + if (parents.Any()) { + depgraph_tree.AddDependencies(TestBitSet::Singleton(parents.First()), i); } - if (parents.Any()) depgraph_tree.AddDependency(parents.First(), i); } } diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 368c69819a..9c6aa6e7a1 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -128,7 +128,8 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) { CCoinsCacheEntry coins_cache_entry; - const auto flags{fuzzed_data_provider.ConsumeIntegral<uint8_t>()}; + const auto dirty{fuzzed_data_provider.ConsumeBool()}; + const auto fresh{fuzzed_data_provider.ConsumeBool()}; if (fuzzed_data_provider.ConsumeBool()) { coins_cache_entry.coin = random_coin; } else { @@ -140,7 +141,8 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) coins_cache_entry.coin = *opt_coin; } auto it{coins_map.emplace(random_out_point, std::move(coins_cache_entry)).first}; - it->second.AddFlags(flags, *it, sentinel); + if (dirty) CCoinsCacheEntry::SetDirty(*it, sentinel); + if (fresh) CCoinsCacheEntry::SetFresh(*it, sentinel); usage += it->second.coin.DynamicMemoryUsage(); } bool expected_code_path = false; @@ -162,22 +164,20 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) const bool exists_using_access_coin = !(coin_using_access_coin == EMPTY_COIN); const bool exists_using_have_coin = coins_view_cache.HaveCoin(random_out_point); const bool exists_using_have_coin_in_cache = coins_view_cache.HaveCoinInCache(random_out_point); - Coin coin_using_get_coin; - const bool exists_using_get_coin = coins_view_cache.GetCoin(random_out_point, coin_using_get_coin); - if (exists_using_get_coin) { - assert(coin_using_get_coin == coin_using_access_coin); + if (auto coin{coins_view_cache.GetCoin(random_out_point)}) { + assert(*coin == coin_using_access_coin); + assert(exists_using_access_coin && exists_using_have_coin_in_cache && exists_using_have_coin); + } else { + assert(!exists_using_access_coin && !exists_using_have_coin_in_cache && !exists_using_have_coin); } - assert((exists_using_access_coin && exists_using_have_coin_in_cache && exists_using_have_coin && exists_using_get_coin) || - (!exists_using_access_coin && !exists_using_have_coin_in_cache && !exists_using_have_coin && !exists_using_get_coin)); // If HaveCoin on the backend is true, it must also be on the cache if the coin wasn't spent. const bool exists_using_have_coin_in_backend = backend_coins_view.HaveCoin(random_out_point); if (!coin_using_access_coin.IsSpent() && exists_using_have_coin_in_backend) { assert(exists_using_have_coin); } - Coin coin_using_backend_get_coin; - if (backend_coins_view.GetCoin(random_out_point, coin_using_backend_get_coin)) { + if (auto coin{backend_coins_view.GetCoin(random_out_point)}) { assert(exists_using_have_coin_in_backend); - // Note we can't assert that `coin_using_get_coin == coin_using_backend_get_coin` because the coin in + // Note we can't assert that `coin_using_get_coin == *coin` because the coin in // the cache may have been modified but not yet flushed. } else { assert(!exists_using_have_coin_in_backend); diff --git a/src/test/fuzz/coinscache_sim.cpp b/src/test/fuzz/coinscache_sim.cpp index 8e717e96b4..6000d52fc9 100644 --- a/src/test/fuzz/coinscache_sim.cpp +++ b/src/test/fuzz/coinscache_sim.cpp @@ -137,9 +137,8 @@ struct CacheLevel /** Class for the base of the hierarchy (roughly simulating a memory-backed CCoinsViewDB). * - * The initial state consists of the empty UTXO set, though coins whose output index - * is 3 (mod 5) always have GetCoin() succeed (but returning an IsSpent() coin unless a UTXO - * exists). Coins whose output index is 4 (mod 5) have GetCoin() always succeed after being spent. + * The initial state consists of the empty UTXO set. + * Coins whose output index is 4 (mod 5) have GetCoin() always succeed after being spent. * This exercises code paths with spent, non-DIRTY cache entries. */ class CoinsViewBottom final : public CCoinsView @@ -147,19 +146,11 @@ class CoinsViewBottom final : public CCoinsView std::map<COutPoint, Coin> m_data; public: - bool GetCoin(const COutPoint& outpoint, Coin& coin) const final + std::optional<Coin> GetCoin(const COutPoint& outpoint) const final { - auto it = m_data.find(outpoint); - if (it == m_data.end()) { - if ((outpoint.n % 5) == 3) { - coin.Clear(); - return true; - } - return false; - } else { - coin = it->second; - return true; - } + // TODO GetCoin shouldn't return spent coins + if (auto it = m_data.find(outpoint); it != m_data.end()) return it->second; + return std::nullopt; } bool HaveCoin(const COutPoint& outpoint) const final @@ -270,17 +261,16 @@ FUZZ_TARGET(coinscache_sim) // Look up in simulation data. auto sim = lookup(outpointidx); // Look up in real caches. - Coin realcoin; - auto real = caches.back()->GetCoin(data.outpoints[outpointidx], realcoin); + auto realcoin = caches.back()->GetCoin(data.outpoints[outpointidx]); // Compare results. if (!sim.has_value()) { - assert(!real || realcoin.IsSpent()); + assert(!realcoin || realcoin->IsSpent()); } else { - assert(real && !realcoin.IsSpent()); + assert(realcoin && !realcoin->IsSpent()); const auto& simcoin = data.coins[sim->first]; - assert(realcoin.out == simcoin.out); - assert(realcoin.fCoinBase == simcoin.fCoinBase); - assert(realcoin.nHeight == sim->second); + assert(realcoin->out == simcoin.out); + assert(realcoin->fCoinBase == simcoin.fCoinBase); + assert(realcoin->nHeight == sim->second); } }, @@ -465,16 +455,15 @@ FUZZ_TARGET(coinscache_sim) // Compare the bottom coinsview (not a CCoinsViewCache) with sim_cache[0]. for (uint32_t outpointidx = 0; outpointidx < NUM_OUTPOINTS; ++outpointidx) { - Coin realcoin; - bool real = bottom.GetCoin(data.outpoints[outpointidx], realcoin); + auto realcoin = bottom.GetCoin(data.outpoints[outpointidx]); auto sim = lookup(outpointidx, 0); if (!sim.has_value()) { - assert(!real || realcoin.IsSpent()); + assert(!realcoin || realcoin->IsSpent()); } else { - assert(real && !realcoin.IsSpent()); - assert(realcoin.out == data.coins[sim->first].out); - assert(realcoin.fCoinBase == data.coins[sim->first].fCoinBase); - assert(realcoin.nHeight == sim->second); + assert(realcoin && !realcoin->IsSpent()); + assert(realcoin->out == data.coins[sim->first].out); + assert(realcoin->fCoinBase == data.coins[sim->first].fCoinBase); + assert(realcoin->nHeight == sim->second); } } } diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index beefc9d82e..a62d227da8 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -20,6 +20,12 @@ namespace { const TestingSetup* g_setup; + +int32_t GetCheckRatio() +{ + return std::clamp<int32_t>(g_setup->m_node.args->GetIntArg("-checkaddrman", 0), 0, 1000000); +} + } // namespace void initialize_connman() @@ -30,12 +36,25 @@ void initialize_connman() FUZZ_TARGET(connman, .init = initialize_connman) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); + auto netgroupman{ConsumeNetGroupManager(fuzzed_data_provider)}; + auto addr_man_ptr{std::make_unique<AddrManDeterministic>(netgroupman, fuzzed_data_provider, GetCheckRatio())}; + if (fuzzed_data_provider.ConsumeBool()) { + const std::vector<uint8_t> serialized_data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; + DataStream ds{serialized_data}; + try { + ds >> *addr_man_ptr; + } catch (const std::ios_base::failure&) { + addr_man_ptr = std::make_unique<AddrManDeterministic>(netgroupman, fuzzed_data_provider, GetCheckRatio()); + } + } + AddrManDeterministic& addr_man{*addr_man_ptr}; ConnmanTestMsg connman{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>(), - *g_setup->m_node.addrman, - *g_setup->m_node.netgroupman, + addr_man, + netgroupman, Params(), fuzzed_data_provider.ConsumeBool()}; @@ -92,13 +111,13 @@ FUZZ_TARGET(connman, .init = initialize_connman) }, [&] { auto max_addresses = fuzzed_data_provider.ConsumeIntegral<size_t>(); - auto max_pct = fuzzed_data_provider.ConsumeIntegral<size_t>(); + auto max_pct = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 100); auto filtered = fuzzed_data_provider.ConsumeBool(); (void)connman.GetAddresses(max_addresses, max_pct, /*network=*/std::nullopt, filtered); }, [&] { auto max_addresses = fuzzed_data_provider.ConsumeIntegral<size_t>(); - auto max_pct = fuzzed_data_provider.ConsumeIntegral<size_t>(); + auto max_pct = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 100); (void)connman.GetAddresses(/*requestor=*/random_node, max_addresses, max_pct); }, [&] { @@ -112,7 +131,7 @@ FUZZ_TARGET(connman, .init = initialize_connman) }, [&] { CSerializedNetMsg serialized_net_msg; - serialized_net_msg.m_type = fuzzed_data_provider.ConsumeRandomLengthString(CMessageHeader::COMMAND_SIZE); + serialized_net_msg.m_type = fuzzed_data_provider.ConsumeRandomLengthString(CMessageHeader::MESSAGE_TYPE_SIZE); serialized_net_msg.data = ConsumeRandomLengthByteVector(fuzzed_data_provider); connman.PushMessage(&random_node, std::move(serialized_net_msg)); }, @@ -140,6 +159,7 @@ FUZZ_TARGET(connman, .init = initialize_connman) (void)connman.GetTotalBytesSent(); (void)connman.GetTryNewOutboundPeer(); (void)connman.GetUseAddrmanOutgoing(); + (void)connman.ASMapHealthCheck(); connman.ClearTestNodes(); } diff --git a/src/test/fuzz/crypto_chacha20poly1305.cpp b/src/test/fuzz/crypto_chacha20poly1305.cpp index 5e62e6f3df..0700ba7fb6 100644 --- a/src/test/fuzz/crypto_chacha20poly1305.cpp +++ b/src/test/fuzz/crypto_chacha20poly1305.cpp @@ -39,7 +39,7 @@ FUZZ_TARGET(crypto_aeadchacha20poly1305) // data). InsecureRandomContext rng(provider.ConsumeIntegral<uint64_t>()); - LIMITED_WHILE(provider.ConsumeBool(), 10000) + LIMITED_WHILE(provider.ConsumeBool(), 100) { // Mode: // - Bit 0: whether to use single-plain Encrypt/Decrypt; otherwise use a split at prefix. diff --git a/src/test/fuzz/crypto_common.cpp b/src/test/fuzz/crypto_common.cpp index 8e07dfedb9..5a76d4e1a9 100644 --- a/src/test/fuzz/crypto_common.cpp +++ b/src/test/fuzz/crypto_common.cpp @@ -35,6 +35,10 @@ FUZZ_TARGET(crypto_common) WriteLE64(writele64_arr.data(), random_u64); assert(ReadLE64(writele64_arr.data()) == random_u64); + std::array<uint8_t, 2> writebe16_arr; + WriteBE16(writebe16_arr.data(), random_u16); + assert(ReadBE16(writebe16_arr.data()) == random_u16); + std::array<uint8_t, 4> writebe32_arr; WriteBE32(writebe32_arr.data(), random_u32); assert(ReadBE32(writebe32_arr.data()) == random_u32); diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 6623edcf99..270d546c39 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -263,7 +263,7 @@ FUZZ_TARGET(service_deserialize, .init = initialize_deserialize) FUZZ_TARGET_DESERIALIZE(messageheader_deserialize, { CMessageHeader mh; DeserializeFromFuzzingInput(buffer, mh); - (void)mh.IsCommandValid(); + (void)mh.IsMessageTypeValid(); }) FUZZ_TARGET(address_deserialize, .init = initialize_deserialize) { diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index bba2dd8e3a..e4e4723c74 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -6,6 +6,7 @@ #include <netaddress.h> #include <netbase.h> +#include <test/fuzz/util/check_globals.h> #include <test/util/random.h> #include <test/util/setup_common.h> #include <util/check.h> @@ -35,8 +36,6 @@ __AFL_FUZZ_INIT(); const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; -const std::function<std::string()> G_TEST_GET_FULL_NAME{}; - /** * A copy of the command line arguments that start with `--`. * First `LLVMFuzzerInitialize()` is called, which saves the arguments to `g_args`. @@ -80,6 +79,15 @@ void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, static std::string_view g_fuzz_target; static const TypeTestOneInput* g_test_one_input{nullptr}; +inline void test_one_input(FuzzBufferType buffer) +{ + CheckGlobals check{}; + (*Assert(g_test_one_input))(buffer); +} + +const std::function<std::string()> G_TEST_GET_FULL_NAME{[]{ + return std::string{g_fuzz_target}; +}}; #if defined(__clang__) && defined(__linux__) extern "C" void __llvm_profile_reset_counters(void) __attribute__((weak)); @@ -154,6 +162,10 @@ void initialize() std::cerr << "No fuzz target compiled for " << g_fuzz_target << "." << std::endl; std::exit(EXIT_FAILURE); } + if constexpr (!G_FUZZING) { + std::cerr << "Must compile with -DBUILD_FOR_FUZZING=ON to execute a fuzz target." << std::endl; + std::exit(EXIT_FAILURE); + } Assert(!g_test_one_input); g_test_one_input = &it->second.test_one_input; it->second.opts.init(); @@ -205,7 +217,6 @@ void signal_handler(int signal) // This function is used by libFuzzer extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - static const auto& test_one_input = *Assert(g_test_one_input); test_one_input({data, size}); return 0; } @@ -222,7 +233,6 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) int main(int argc, char** argv) { initialize(); - static const auto& test_one_input = *Assert(g_test_one_input); #ifdef __AFL_LOOP // Enable AFL persistent mode. Requires compilation using afl-clang-fast++. // See fuzzing.md for details. diff --git a/src/test/fuzz/golomb_rice.cpp b/src/test/fuzz/golomb_rice.cpp index 92e49445ba..066c168652 100644 --- a/src/test/fuzz/golomb_rice.cpp +++ b/src/test/fuzz/golomb_rice.cpp @@ -8,6 +8,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/util/random.h> #include <util/bytevectorhash.h> #include <util/golombrice.h> @@ -42,6 +43,7 @@ std::vector<uint64_t> BuildHashedSet(const std::unordered_set<std::vector<uint8_ FUZZ_TARGET(golomb_rice) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); std::vector<uint8_t> golomb_rice_data; std::vector<uint64_t> encoded_deltas; diff --git a/src/test/fuzz/headerssync.cpp b/src/test/fuzz/headerssync.cpp index 1aa878bd6d..8135b0d4ea 100644 --- a/src/test/fuzz/headerssync.cpp +++ b/src/test/fuzz/headerssync.cpp @@ -48,6 +48,7 @@ public: FUZZ_TARGET(headers_sync_state, .init = initialize_headers_sync_state_fuzz) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); auto mock_time{ConsumeTime(fuzzed_data_provider)}; diff --git a/src/test/fuzz/i2p.cpp b/src/test/fuzz/i2p.cpp index 51517187a0..b8024f7b1c 100644 --- a/src/test/fuzz/i2p.cpp +++ b/src/test/fuzz/i2p.cpp @@ -21,6 +21,7 @@ void initialize_i2p() FUZZ_TARGET(i2p, .init = initialize_i2p) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index e27b2065d5..b9e3154106 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -69,7 +69,7 @@ FUZZ_TARGET(integer, .init = initialize_integer) const bool b = fuzzed_data_provider.ConsumeBool(); const Consensus::Params& consensus_params = Params().GetConsensus(); - (void)CheckProofOfWork(u256, u32, consensus_params); + (void)CheckProofOfWorkImpl(u256, u32, consensus_params); if (u64 <= MAX_MONEY) { const uint64_t compressed_money_amount = CompressAmount(u64); assert(u64 == DecompressAmount(compressed_money_amount)); diff --git a/src/test/fuzz/key.cpp b/src/test/fuzz/key.cpp index 82973803f8..c7ff2f3a28 100644 --- a/src/test/fuzz/key.cpp +++ b/src/test/fuzz/key.cpp @@ -18,6 +18,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/util/random.h> #include <util/chaintype.h> #include <util/strencodings.h> @@ -38,6 +39,7 @@ void initialize_key() FUZZ_TARGET(key, .init = initialize_key) { + SeedRandomStateForTest(SeedRand::ZEROS); const CKey key = [&] { CKey k; k.Set(buffer.begin(), buffer.end(), true); diff --git a/src/test/fuzz/mini_miner.cpp b/src/test/fuzz/mini_miner.cpp index 51de4d0166..a5bccd103d 100644 --- a/src/test/fuzz/mini_miner.cpp +++ b/src/test/fuzz/mini_miner.cpp @@ -34,6 +34,7 @@ void initialize_miner() // Test that the MiniMiner can run with various outpoints and feerates. FUZZ_TARGET(mini_miner, .init = initialize_miner) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; bilingual_str error; CTxMemPool pool{CTxMemPool::Options{}, error}; @@ -59,7 +60,7 @@ FUZZ_TARGET(mini_miner, .init = initialize_miner) TestMemPoolEntryHelper entry; const CAmount fee{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)}; assert(MoneyRange(fee)); - pool.addUnchecked(entry.Fee(fee).FromTx(tx)); + AddToMempool(pool, entry.Fee(fee).FromTx(tx)); // All outputs are available to spend for (uint32_t n{0}; n < num_outputs; ++n) { @@ -112,6 +113,7 @@ FUZZ_TARGET(mini_miner, .init = initialize_miner) // Test that MiniMiner and BlockAssembler build the same block given the same transactions and constraints. FUZZ_TARGET(mini_miner_selection, .init = initialize_miner) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; bilingual_str error; CTxMemPool pool{CTxMemPool::Options{}, error}; @@ -154,7 +156,7 @@ FUZZ_TARGET(mini_miner_selection, .init = initialize_miner) TestMemPoolEntryHelper entry; const CAmount fee{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)}; assert(MoneyRange(fee)); - pool.addUnchecked(entry.Fee(fee).FromTx(tx)); + AddToMempool(pool, entry.Fee(fee).FromTx(tx)); transactions.push_back(tx); } std::vector<COutPoint> outpoints; @@ -174,15 +176,15 @@ FUZZ_TARGET(mini_miner_selection, .init = initialize_miner) miner_options.blockMinFeeRate = target_feerate; miner_options.nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; miner_options.test_block_validity = false; + miner_options.coinbase_output_script = CScript() << OP_0; node::BlockAssembler miner{g_setup->m_node.chainman->ActiveChainstate(), &pool, miner_options}; node::MiniMiner mini_miner{pool, outpoints}; assert(mini_miner.IsReadyToCalculate()); - CScript spk_placeholder = CScript() << OP_0; // Use BlockAssembler as oracle. BlockAssembler and MiniMiner should select the same // transactions, stopping once packages do not meet target_feerate. - const auto blocktemplate{miner.CreateNewBlock(spk_placeholder)}; + const auto blocktemplate{miner.CreateNewBlock()}; mini_miner.BuildMockTemplate(target_feerate); assert(!mini_miner.IsReadyToCalculate()); auto mock_template_txids = mini_miner.GetMockTemplateTxids(); diff --git a/src/test/fuzz/miniscript.cpp b/src/test/fuzz/miniscript.cpp index 5b9e168856..60d096bb5a 100644 --- a/src/test/fuzz/miniscript.cpp +++ b/src/test/fuzz/miniscript.cpp @@ -22,7 +22,7 @@ using NodeRef = miniscript::NodeRef<CPubKey>; using Node = miniscript::Node<CPubKey>; using Type = miniscript::Type; using MsCtx = miniscript::MiniscriptContext; -using miniscript::operator"" _mst; +using miniscript::operator""_mst; //! Some pre-computed data for more efficient string roundtrips and to simulate challenges. struct TestData { diff --git a/src/test/fuzz/netaddress.cpp b/src/test/fuzz/netaddress.cpp index 4803cdccad..28a2026a35 100644 --- a/src/test/fuzz/netaddress.cpp +++ b/src/test/fuzz/netaddress.cpp @@ -6,6 +6,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util/net.h> +#include <test/util/random.h> #include <cassert> #include <cstdint> @@ -13,6 +14,7 @@ FUZZ_TARGET(netaddress) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const CNetAddr net_addr = ConsumeNetAddr(fuzzed_data_provider); diff --git a/src/test/fuzz/p2p_handshake.cpp b/src/test/fuzz/p2p_handshake.cpp index 6c1ed11d45..d608efd87a 100644 --- a/src/test/fuzz/p2p_handshake.cpp +++ b/src/test/fuzz/p2p_handshake.cpp @@ -39,6 +39,7 @@ void initialize() FUZZ_TARGET(p2p_handshake, .init = ::initialize) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); ConnmanTestMsg& connman = static_cast<ConnmanTestMsg&>(*g_setup->m_node.connman); diff --git a/src/test/fuzz/p2p_headers_presync.cpp b/src/test/fuzz/p2p_headers_presync.cpp new file mode 100644 index 0000000000..873eb2b1cc --- /dev/null +++ b/src/test/fuzz/p2p_headers_presync.cpp @@ -0,0 +1,235 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <arith_uint256.h> +#include <blockencodings.h> +#include <net.h> +#include <net_processing.h> +#include <netmessagemaker.h> +#include <node/peerman_args.h> +#include <pow.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/net.h> +#include <test/util/script.h> +#include <test/util/setup_common.h> +#include <uint256.h> +#include <validation.h> + +namespace { +constexpr uint32_t FUZZ_MAX_HEADERS_RESULTS{16}; + +class HeadersSyncSetup : public TestingSetup +{ + std::vector<CNode*> m_connections; + +public: + HeadersSyncSetup(const ChainType chain_type, TestOpts opts) : TestingSetup(chain_type, opts) + { + PeerManager::Options peerman_opts; + node::ApplyArgsManOptions(*m_node.args, peerman_opts); + peerman_opts.max_headers_result = FUZZ_MAX_HEADERS_RESULTS; + m_node.peerman = PeerManager::make(*m_node.connman, *m_node.addrman, + m_node.banman.get(), *m_node.chainman, + *m_node.mempool, *m_node.warnings, peerman_opts); + + CConnman::Options options; + options.m_msgproc = m_node.peerman.get(); + m_node.connman->Init(options); + } + + void ResetAndInitialize() EXCLUSIVE_LOCKS_REQUIRED(NetEventsInterface::g_msgproc_mutex); + void SendMessage(FuzzedDataProvider& fuzzed_data_provider, CSerializedNetMsg&& msg) + EXCLUSIVE_LOCKS_REQUIRED(NetEventsInterface::g_msgproc_mutex); +}; + +void HeadersSyncSetup::ResetAndInitialize() +{ + m_connections.clear(); + auto& connman = static_cast<ConnmanTestMsg&>(*m_node.connman); + connman.StopNodes(); + + NodeId id{0}; + std::vector<ConnectionType> conn_types = { + ConnectionType::OUTBOUND_FULL_RELAY, + ConnectionType::BLOCK_RELAY, + ConnectionType::INBOUND + }; + + for (auto conn_type : conn_types) { + CAddress addr{}; + m_connections.push_back(new CNode(id++, nullptr, addr, 0, 0, addr, "", conn_type, false)); + CNode& p2p_node = *m_connections.back(); + + connman.Handshake( + /*node=*/p2p_node, + /*successfully_connected=*/true, + /*remote_services=*/ServiceFlags(NODE_NETWORK | NODE_WITNESS), + /*local_services=*/ServiceFlags(NODE_NETWORK | NODE_WITNESS), + /*version=*/PROTOCOL_VERSION, + /*relay_txs=*/true); + + connman.AddTestNode(p2p_node); + } +} + +void HeadersSyncSetup::SendMessage(FuzzedDataProvider& fuzzed_data_provider, CSerializedNetMsg&& msg) +{ + auto& connman = static_cast<ConnmanTestMsg&>(*m_node.connman); + CNode& connection = *PickValue(fuzzed_data_provider, m_connections); + + connman.FlushSendBuffer(connection); + (void)connman.ReceiveMsgFrom(connection, std::move(msg)); + connection.fPauseSend = false; + try { + connman.ProcessMessagesOnce(connection); + } catch (const std::ios_base::failure&) { + } + m_node.peerman->SendMessages(&connection); +} + +CBlockHeader ConsumeHeader(FuzzedDataProvider& fuzzed_data_provider, const uint256& prev_hash, uint32_t prev_nbits) +{ + CBlockHeader header; + header.nNonce = 0; + // Either use the previous difficulty or let the fuzzer choose. The upper target in the + // range comes from the bits value of the genesis block, which is 0x1d00ffff. The lower + // target comes from the bits value of mainnet block 840000, which is 0x17034219. + // Calling lower_target.SetCompact(0x17034219) and upper_target.SetCompact(0x1d00ffff) + // should return the values below. + // + // RPC commands to verify: + // getblockheader 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f + // getblockheader 0000000000000000000320283a032748cef8227873ff4872689bf23f1cda83a5 + if (fuzzed_data_provider.ConsumeBool()) { + header.nBits = prev_nbits; + } else { + arith_uint256 lower_target = UintToArith256(uint256{"0000000000000000000342190000000000000000000000000000000000000000"}); + arith_uint256 upper_target = UintToArith256(uint256{"00000000ffff0000000000000000000000000000000000000000000000000000"}); + arith_uint256 target = ConsumeArithUInt256InRange(fuzzed_data_provider, lower_target, upper_target); + header.nBits = target.GetCompact(); + } + header.nTime = ConsumeTime(fuzzed_data_provider); + header.hashPrevBlock = prev_hash; + header.nVersion = fuzzed_data_provider.ConsumeIntegral<int32_t>(); + return header; +} + +CBlock ConsumeBlock(FuzzedDataProvider& fuzzed_data_provider, const uint256& prev_hash, uint32_t prev_nbits) +{ + auto header = ConsumeHeader(fuzzed_data_provider, prev_hash, prev_nbits); + // In order to reach the headers acceptance logic, the block is + // constructed in a way that will pass the mutation checks. + CBlock block{header}; + CMutableTransaction tx; + tx.vin.resize(1); + tx.vout.resize(1); + tx.vout[0].nValue = 0; + tx.vin[0].scriptSig.resize(2); + block.vtx.push_back(MakeTransactionRef(tx)); + block.hashMerkleRoot = block.vtx[0]->GetHash(); + return block; +} + +void FinalizeHeader(CBlockHeader& header, const ChainstateManager& chainman) +{ + while (!CheckProofOfWork(header.GetHash(), header.nBits, chainman.GetParams().GetConsensus())) { + ++(header.nNonce); + } +} + +// Global setup works for this test as state modification (specifically in the +// block index) would indicate a bug. +HeadersSyncSetup* g_testing_setup; + +void initialize() +{ + static auto setup = MakeNoLogFileContext<HeadersSyncSetup>(ChainType::MAIN, {.extra_args = {"-checkpoints=0"}}); + g_testing_setup = setup.get(); +} +} // namespace + +FUZZ_TARGET(p2p_headers_presync, .init = initialize) +{ + SeedRandomStateForTest(SeedRand::ZEROS); + ChainstateManager& chainman = *g_testing_setup->m_node.chainman; + + LOCK(NetEventsInterface::g_msgproc_mutex); + + g_testing_setup->ResetAndInitialize(); + + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + CBlockHeader base{chainman.GetParams().GenesisBlock()}; + SetMockTime(base.nTime); + + // The chain is just a single block, so this is equal to 1 + size_t original_index_size{WITH_LOCK(cs_main, return chainman.m_blockman.m_block_index.size())}; + arith_uint256 total_work{WITH_LOCK(cs_main, return chainman.m_best_header->nChainWork)}; + + std::vector<CBlockHeader> all_headers; + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) + { + auto finalized_block = [&]() { + CBlock block = ConsumeBlock(fuzzed_data_provider, base.GetHash(), base.nBits); + FinalizeHeader(block, chainman); + return block; + }; + + // Send low-work headers, compact blocks, and blocks + CallOneOf( + fuzzed_data_provider, + [&]() NO_THREAD_SAFETY_ANALYSIS { + // Send FUZZ_MAX_HEADERS_RESULTS headers + std::vector<CBlock> headers; + headers.resize(FUZZ_MAX_HEADERS_RESULTS); + for (CBlock& header : headers) { + header = ConsumeHeader(fuzzed_data_provider, base.GetHash(), base.nBits); + FinalizeHeader(header, chainman); + base = header; + } + + all_headers.insert(all_headers.end(), headers.begin(), headers.end()); + + auto headers_msg = NetMsg::Make(NetMsgType::HEADERS, TX_WITH_WITNESS(headers)); + g_testing_setup->SendMessage(fuzzed_data_provider, std::move(headers_msg)); + }, + [&]() NO_THREAD_SAFETY_ANALYSIS { + // Send a compact block + auto block = finalized_block(); + CBlockHeaderAndShortTxIDs cmpct_block{block, fuzzed_data_provider.ConsumeIntegral<uint64_t>()}; + + all_headers.push_back(block); + + auto headers_msg = NetMsg::Make(NetMsgType::CMPCTBLOCK, TX_WITH_WITNESS(cmpct_block)); + g_testing_setup->SendMessage(fuzzed_data_provider, std::move(headers_msg)); + }, + [&]() NO_THREAD_SAFETY_ANALYSIS { + // Send a block + auto block = finalized_block(); + + all_headers.push_back(block); + + auto headers_msg = NetMsg::Make(NetMsgType::BLOCK, TX_WITH_WITNESS(block)); + g_testing_setup->SendMessage(fuzzed_data_provider, std::move(headers_msg)); + }); + } + + // This is a conservative overestimate, as base is only moved forward when sending headers. In theory, + // the longest chain generated by this test is 1600 (FUZZ_MAX_HEADERS_RESULTS * 100) headers. In that case, + // this variable will accurately reflect the chain's total work. + total_work += CalculateClaimedHeadersWork(all_headers); + + // This test should never create a chain with more work than MinimumChainWork. + assert(total_work < chainman.MinimumChainWork()); + + // The headers/blocks sent in this test should never be stored, as the chains don't have the work required + // to meet the anti-DoS work threshold. So, if at any point the block index grew in size, then there's a bug + // in the headers pre-sync logic. + assert(WITH_LOCK(cs_main, return chainman.m_blockman.m_block_index.size()) == original_index_size); + + g_testing_setup->m_node.validation_signals->SyncWithValidationInterfaceQueue(); +} diff --git a/src/test/fuzz/p2p_transport_serialization.cpp b/src/test/fuzz/p2p_transport_serialization.cpp index cf3ef45c0a..e377e23894 100644 --- a/src/test/fuzz/p2p_transport_serialization.cpp +++ b/src/test/fuzz/p2p_transport_serialization.cpp @@ -81,7 +81,7 @@ FUZZ_TARGET(p2p_transport_serialization, .init = initialize_p2p_transport_serial const std::chrono::microseconds m_time{std::numeric_limits<int64_t>::max()}; bool reject_message{false}; CNetMessage msg = recv_transport.GetReceivedMessage(m_time, reject_message); - assert(msg.m_type.size() <= CMessageHeader::COMMAND_SIZE); + assert(msg.m_type.size() <= CMessageHeader::MESSAGE_TYPE_SIZE); assert(msg.m_raw_message_size <= mutable_msg_bytes.size()); assert(msg.m_raw_message_size == CMessageHeader::HEADER_SIZE + msg.m_message_size); assert(msg.m_time == m_time); @@ -139,9 +139,9 @@ void SimulationTest(Transport& initiator, Transport& responder, R& rng, FuzzedDa // If v is 0xFF, construct a valid (but possibly unknown) message type from the fuzz // data. std::string ret; - while (ret.size() < CMessageHeader::COMMAND_SIZE) { + while (ret.size() < CMessageHeader::MESSAGE_TYPE_SIZE) { char c = provider.ConsumeIntegral<char>(); - // Match the allowed characters in CMessageHeader::IsCommandValid(). Any other + // Match the allowed characters in CMessageHeader::IsMessageTypeValid(). Any other // character is interpreted as end. if (c < ' ' || c > 0x7E) break; ret += c; diff --git a/src/test/fuzz/package_eval.cpp b/src/test/fuzz/package_eval.cpp index 652c7a7609..8e3d84a9e6 100644 --- a/src/test/fuzz/package_eval.cpp +++ b/src/test/fuzz/package_eval.cpp @@ -21,6 +21,7 @@ #include <validation.h> #include <validationinterface.h> +using node::BlockAssembler; using node::NodeContext; namespace { @@ -42,8 +43,11 @@ void initialize_tx_pool() static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); g_setup = testing_setup.get(); + BlockAssembler::Options options; + options.coinbase_output_script = P2WSH_EMPTY; + for (int i = 0; i < 2 * COINBASE_MATURITY; ++i) { - COutPoint prevout{MineBlock(g_setup->m_node, P2WSH_EMPTY)}; + COutPoint prevout{MineBlock(g_setup->m_node, options)}; if (i < COINBASE_MATURITY) { // Remember the txids to avoid expensive disk access later on g_outpoints_coinbase_init_mature.push_back(prevout); @@ -137,8 +141,213 @@ std::unique_ptr<CTxMemPool> MakeMempool(FuzzedDataProvider& fuzzed_data_provider return mempool; } +std::unique_ptr<CTxMemPool> MakeEphemeralMempool(const NodeContext& node) +{ + // Take the default options for tests... + CTxMemPool::Options mempool_opts{MemPoolOptionsForTest(node)}; + + mempool_opts.check_ratio = 1; + + // Require standardness rules otherwise ephemeral dust is no-op + mempool_opts.require_standard = true; + + // And set minrelay to 0 to allow ephemeral parent tx even with non-TRUC + mempool_opts.min_relay_feerate = CFeeRate(0); + + bilingual_str error; + // ...and construct a CTxMemPool from it + auto mempool{std::make_unique<CTxMemPool>(std::move(mempool_opts), error)}; + Assert(error.empty()); + return mempool; +} + +// Scan mempool for a tx that has spent dust and return a +// prevout of the child that isn't the dusty parent itself. +// This is used to double-spend the child out of the mempool, +// leaving the parent childless. +// This assumes CheckMempoolEphemeralInvariants has passed for tx_pool. +std::optional<COutPoint> GetChildEvictingPrevout(const CTxMemPool& tx_pool) +{ + LOCK(tx_pool.cs); + for (const auto& tx_info : tx_pool.infoAll()) { + const auto& entry = *Assert(tx_pool.GetEntry(tx_info.tx->GetHash())); + std::vector<uint32_t> dust_indexes{GetDust(*tx_info.tx, tx_pool.m_opts.dust_relay_feerate)}; + if (!dust_indexes.empty()) { + const auto& children = entry.GetMemPoolChildrenConst(); + if (!children.empty()) { + Assert(children.size() == 1); + // Find an input that doesn't spend from parent's txid + const auto& only_child = children.begin()->get().GetTx(); + for (const auto& tx_input : only_child.vin) { + if (tx_input.prevout.hash != tx_info.tx->GetHash()) { + return tx_input.prevout; + } + } + } + } + } + + return std::nullopt; +} + +FUZZ_TARGET(ephemeral_package_eval, .init = initialize_tx_pool) +{ + SeedRandomStateForTest(SeedRand::ZEROS); + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const auto& node = g_setup->m_node; + auto& chainstate{static_cast<DummyChainState&>(node.chainman->ActiveChainstate())}; + + MockTime(fuzzed_data_provider, chainstate); + + // All RBF-spendable outpoints outside of the unsubmitted package + std::set<COutPoint> mempool_outpoints; + std::map<COutPoint, CAmount> outpoints_value; + for (const auto& outpoint : g_outpoints_coinbase_init_mature) { + Assert(mempool_outpoints.insert(outpoint).second); + outpoints_value[outpoint] = 50 * COIN; + } + + auto outpoints_updater = std::make_shared<OutpointsUpdater>(mempool_outpoints); + node.validation_signals->RegisterSharedValidationInterface(outpoints_updater); + + auto tx_pool_{MakeEphemeralMempool(node)}; + MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(tx_pool_.get()); + + chainstate.SetMempool(&tx_pool); + + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 300) + { + Assert(!mempool_outpoints.empty()); + + std::vector<CTransactionRef> txs; + + // Find something we may want to double-spend with two input single tx + std::optional<COutPoint> outpoint_to_rbf{fuzzed_data_provider.ConsumeBool() ? GetChildEvictingPrevout(tx_pool) : std::nullopt}; + + // Make small packages + const auto num_txs = outpoint_to_rbf ? 1 : (size_t) fuzzed_data_provider.ConsumeIntegralInRange<int>(1, 4); + + std::set<COutPoint> package_outpoints; + while (txs.size() < num_txs) { + // Create transaction to add to the mempool + txs.emplace_back([&] { + CMutableTransaction tx_mut; + tx_mut.version = CTransaction::CURRENT_VERSION; + tx_mut.nLockTime = 0; + // Last transaction in a package needs to be a child of parents to get further in validation + // so the last transaction to be generated(in a >1 package) must spend all package-made outputs + // Note that this test currently only spends package outputs in last transaction. + bool last_tx = num_txs > 1 && txs.size() == num_txs - 1; + const auto num_in = outpoint_to_rbf ? 2 : + last_tx ? fuzzed_data_provider.ConsumeIntegralInRange<int>(package_outpoints.size()/2 + 1, package_outpoints.size()) : + fuzzed_data_provider.ConsumeIntegralInRange<int>(1, 4); + const auto num_out = outpoint_to_rbf ? 1 : fuzzed_data_provider.ConsumeIntegralInRange<int>(1, 4); + + auto& outpoints = last_tx ? package_outpoints : mempool_outpoints; + + Assert((int)outpoints.size() >= num_in && num_in > 0); + + CAmount amount_in{0}; + for (int i = 0; i < num_in; ++i) { + // Pop random outpoint. We erase them to avoid double-spending + // while in this loop, but later add them back (unless last_tx). + auto pop = outpoints.begin(); + std::advance(pop, fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, outpoints.size() - 1)); + auto outpoint = *pop; + + if (i == 0 && outpoint_to_rbf) { + outpoint = *outpoint_to_rbf; + outpoints.erase(outpoint); + } else { + outpoints.erase(pop); + } + // no need to update or erase from outpoints_value + amount_in += outpoints_value.at(outpoint); + + // Create input + CTxIn in; + in.prevout = outpoint; + in.scriptWitness.stack = P2WSH_EMPTY_TRUE_STACK; + + tx_mut.vin.push_back(in); + } + + const auto amount_fee = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(0, amount_in); + const auto amount_out = (amount_in - amount_fee) / num_out; + for (int i = 0; i < num_out; ++i) { + tx_mut.vout.emplace_back(amount_out, P2WSH_EMPTY); + } + + // Note output amounts can naturally drop to dust on their own. + if (!outpoint_to_rbf && fuzzed_data_provider.ConsumeBool()) { + uint32_t dust_index = fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(0, num_out); + tx_mut.vout.insert(tx_mut.vout.begin() + dust_index, CTxOut(0, P2WSH_EMPTY)); + } + + auto tx = MakeTransactionRef(tx_mut); + // Restore previously removed outpoints, except in-package outpoints (to allow RBF) + if (!last_tx) { + for (const auto& in : tx->vin) { + Assert(outpoints.insert(in.prevout).second); + } + // Cache the in-package outpoints being made + for (size_t i = 0; i < tx->vout.size(); ++i) { + package_outpoints.emplace(tx->GetHash(), i); + } + } + // We need newly-created values for the duration of this run + for (size_t i = 0; i < tx->vout.size(); ++i) { + outpoints_value[COutPoint(tx->GetHash(), i)] = tx->vout[i].nValue; + } + return tx; + }()); + } + + if (fuzzed_data_provider.ConsumeBool()) { + const auto& txid = fuzzed_data_provider.ConsumeBool() ? + txs.back()->GetHash() : + PickValue(fuzzed_data_provider, mempool_outpoints).hash; + const auto delta = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(-50 * COIN, +50 * COIN); + // We only prioritise out of mempool transactions since PrioritiseTransaction doesn't + // filter for ephemeral dust + if (tx_pool.exists(GenTxid::Txid(txid))) { + const auto tx_info{tx_pool.info(GenTxid::Txid(txid))}; + if (GetDust(*tx_info.tx, tx_pool.m_opts.dust_relay_feerate).empty()) { + tx_pool.PrioritiseTransaction(txid.ToUint256(), delta); + } + } + } + + auto single_submit = txs.size() == 1; + + const auto result_package = WITH_LOCK(::cs_main, + return ProcessNewPackage(chainstate, tx_pool, txs, /*test_accept=*/single_submit, /*client_maxfeerate=*/{})); + + const auto res = WITH_LOCK(::cs_main, return AcceptToMemoryPool(chainstate, txs.back(), GetTime(), + /*bypass_limits=*/fuzzed_data_provider.ConsumeBool(), /*test_accept=*/!single_submit)); + + if (!single_submit && result_package.m_state.GetResult() != PackageValidationResult::PCKG_POLICY) { + // We don't know anything about the validity since transactions were randomly generated, so + // just use result_package.m_state here. This makes the expect_valid check meaningless, but + // we can still verify that the contents of m_tx_results are consistent with m_state. + const bool expect_valid{result_package.m_state.IsValid()}; + Assert(!CheckPackageMempoolAcceptResult(txs, result_package, expect_valid, &tx_pool)); + } + + node.validation_signals->SyncWithValidationInterfaceQueue(); + + CheckMempoolEphemeralInvariants(tx_pool); + } + + node.validation_signals->UnregisterSharedValidationInterface(outpoints_updater); + + WITH_LOCK(::cs_main, tx_pool.check(chainstate.CoinsTip(), chainstate.m_chain.Height() + 1)); +} + + FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const auto& node = g_setup->m_node; auto& chainstate{static_cast<DummyChainState&>(node.chainman->ActiveChainstate())}; @@ -161,7 +370,7 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) chainstate.SetMempool(&tx_pool); - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 300) + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 300) { Assert(!mempool_outpoints.empty()); @@ -171,18 +380,15 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) const auto num_txs = (size_t) fuzzed_data_provider.ConsumeIntegralInRange<int>(1, 26); std::set<COutPoint> package_outpoints; while (txs.size() < num_txs) { - - // Last transaction in a package needs to be a child of parents to get further in validation - // so the last transaction to be generated(in a >1 package) must spend all package-made outputs - // Note that this test currently only spends package outputs in last transaction. - bool last_tx = num_txs > 1 && txs.size() == num_txs - 1; - // Create transaction to add to the mempool - const CTransactionRef tx = [&] { + txs.emplace_back([&] { CMutableTransaction tx_mut; tx_mut.version = fuzzed_data_provider.ConsumeBool() ? TRUC_VERSION : CTransaction::CURRENT_VERSION; tx_mut.nLockTime = fuzzed_data_provider.ConsumeBool() ? 0 : fuzzed_data_provider.ConsumeIntegral<uint32_t>(); - // Last tx will sweep all outpoints in package + // Last transaction in a package needs to be a child of parents to get further in validation + // so the last transaction to be generated(in a >1 package) must spend all package-made outputs + // Note that this test currently only spends package outputs in last transaction. + bool last_tx = num_txs > 1 && txs.size() == num_txs - 1; const auto num_in = last_tx ? package_outpoints.size() : fuzzed_data_provider.ConsumeIntegralInRange<int>(1, mempool_outpoints.size()); auto num_out = fuzzed_data_provider.ConsumeIntegralInRange<int>(1, mempool_outpoints.size() * 2); @@ -192,7 +398,8 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) CAmount amount_in{0}; for (size_t i = 0; i < num_in; ++i) { - // Pop random outpoint + // Pop random outpoint. We erase them to avoid double-spending + // while in this loop, but later add them back (unless last_tx). auto pop = outpoints.begin(); std::advance(pop, fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, outpoints.size() - 1)); const auto outpoint = *pop; @@ -255,8 +462,7 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) outpoints_value[COutPoint(tx->GetHash(), i)] = tx->vout[i].nValue; } return tx; - }(); - txs.push_back(tx); + }()); } if (fuzzed_data_provider.ConsumeBool()) { @@ -321,6 +527,11 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) } CheckMempoolTRUCInvariants(tx_pool); + + // Dust checks only make sense when dust is enforced + if (tx_pool.m_opts.require_standard) { + CheckMempoolEphemeralInvariants(tx_pool); + } } node.validation_signals->UnregisterSharedValidationInterface(outpoints_updater); diff --git a/src/wallet/test/fuzz/parse_iso8601.cpp b/src/test/fuzz/parse_iso8601.cpp index c1bafc1073..7e51f57905 100644 --- a/src/wallet/test/fuzz/parse_iso8601.cpp +++ b/src/test/fuzz/parse_iso8601.cpp @@ -1,11 +1,10 @@ -// Copyright (c) 2019-2022 The Bitcoin Core developers +// Copyright (c) 2019-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <util/time.h> -#include <wallet/rpc/util.h> #include <cassert> #include <cstdint> @@ -21,14 +20,8 @@ FUZZ_TARGET(parse_iso8601) const std::string iso8601_datetime = FormatISO8601DateTime(random_time); (void)FormatISO8601Date(random_time); - const int64_t parsed_time_1 = wallet::ParseISO8601DateTime(iso8601_datetime); - if (random_time >= 0) { - assert(parsed_time_1 >= 0); - if (iso8601_datetime.length() == 20) { - assert(parsed_time_1 == random_time); - } - } + const int64_t parsed_time_1{ParseISO8601DateTime(iso8601_datetime).value()}; + assert(parsed_time_1 == random_time); - const int64_t parsed_time_2 = wallet::ParseISO8601DateTime(random_string); - assert(parsed_time_2 >= 0); + (void)ParseISO8601DateTime(random_string); } diff --git a/src/test/fuzz/partially_downloaded_block.cpp b/src/test/fuzz/partially_downloaded_block.cpp index 77952cab9e..8a42807be8 100644 --- a/src/test/fuzz/partially_downloaded_block.cpp +++ b/src/test/fuzz/partially_downloaded_block.cpp @@ -44,6 +44,7 @@ PartiallyDownloadedBlock::CheckBlockFn FuzzedCheckBlock(std::optional<BlockValid FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; auto block{ConsumeDeserializable<CBlock>(fuzzed_data_provider, TX_WITH_WITNESS)}; @@ -78,7 +79,7 @@ FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) if (add_to_mempool && !pool.exists(GenTxid::Txid(tx->GetHash()))) { LOCK2(cs_main, pool.cs); - pool.addUnchecked(ConsumeTxMemPoolEntry(fuzzed_data_provider, *tx)); + AddToMempool(pool, ConsumeTxMemPoolEntry(fuzzed_data_provider, *tx)); available.insert(i); } } @@ -114,7 +115,6 @@ FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) fuzzed_data_provider.PickValueInArray( {BlockValidationResult::BLOCK_RESULT_UNSET, BlockValidationResult::BLOCK_CONSENSUS, - BlockValidationResult::BLOCK_RECENT_CONSENSUS_CHANGE, BlockValidationResult::BLOCK_CACHED_INVALID, BlockValidationResult::BLOCK_INVALID_HEADER, BlockValidationResult::BLOCK_MUTATED, diff --git a/src/test/fuzz/pow.cpp b/src/test/fuzz/pow.cpp index 05cdb740e4..dba999ce4f 100644 --- a/src/test/fuzz/pow.cpp +++ b/src/test/fuzz/pow.cpp @@ -80,7 +80,7 @@ FUZZ_TARGET(pow, .init = initialize_pow) { const std::optional<uint256> hash = ConsumeDeserializable<uint256>(fuzzed_data_provider); if (hash) { - (void)CheckProofOfWork(*hash, fuzzed_data_provider.ConsumeIntegral<unsigned int>(), consensus_params); + (void)CheckProofOfWorkImpl(*hash, fuzzed_data_provider.ConsumeIntegral<unsigned int>(), consensus_params); } } } diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 6373eac1c3..4bd38a1ac6 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -45,13 +45,14 @@ void initialize_process_message() {.extra_args = {"-txreconciliation"}}); g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { - MineBlock(g_setup->m_node, CScript() << OP_TRUE); + MineBlock(g_setup->m_node, {}); } g_setup->m_node.validation_signals->SyncWithValidationInterfaceQueue(); } FUZZ_TARGET(process_message, .init = initialize_process_message) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); ConnmanTestMsg& connman = *static_cast<ConnmanTestMsg*>(g_setup->m_node.connman.get()); @@ -61,7 +62,7 @@ FUZZ_TARGET(process_message, .init = initialize_process_message) LOCK(NetEventsInterface::g_msgproc_mutex); - const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; + const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::MESSAGE_TYPE_SIZE).c_str()}; if (!LIMIT_TO_MESSAGE_TYPE.empty() && random_message_type != LIMIT_TO_MESSAGE_TYPE) { return; } diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 62f38967a3..0688868c02 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -35,13 +35,14 @@ void initialize_process_messages() {.extra_args = {"-txreconciliation"}}); g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { - MineBlock(g_setup->m_node, CScript() << OP_TRUE); + MineBlock(g_setup->m_node, {}); } g_setup->m_node.validation_signals->SyncWithValidationInterfaceQueue(); } FUZZ_TARGET(process_messages, .init = initialize_process_messages) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); ConnmanTestMsg& connman = *static_cast<ConnmanTestMsg*>(g_setup->m_node.connman.get()); @@ -64,7 +65,7 @@ FUZZ_TARGET(process_messages, .init = initialize_process_messages) LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 30) { - const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; + const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::MESSAGE_TYPE_SIZE).c_str()}; const auto mock_time = ConsumeTime(fuzzed_data_provider); SetMockTime(mock_time); diff --git a/src/test/fuzz/protocol.cpp b/src/test/fuzz/protocol.cpp index 572181366b..71132f38f8 100644 --- a/src/test/fuzz/protocol.cpp +++ b/src/test/fuzz/protocol.cpp @@ -20,7 +20,7 @@ FUZZ_TARGET(protocol) return; } try { - (void)inv->GetCommand(); + (void)inv->GetMessageType(); } catch (const std::out_of_range&) { } (void)inv->ToString(); diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp index e0692600bb..37fa159c33 100644 --- a/src/test/fuzz/psbt.cpp +++ b/src/test/fuzz/psbt.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/fuzz/FuzzedDataProvider.h> -#include <test/fuzz/fuzz.h> - #include <node/psbt.h> #include <psbt.h> #include <pubkey.h> #include <script/script.h> #include <streams.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/util/random.h> #include <util/check.h> #include <cstdint> @@ -23,6 +23,7 @@ using node::PSBTInputAnalysis; FUZZ_TARGET(psbt) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; PartiallySignedTransaction psbt_mut; std::string error; diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index eb981352ec..3e5b361186 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -51,6 +51,7 @@ void initialize_package_rbf() FUZZ_TARGET(rbf, .init = initialize_rbf) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); @@ -73,12 +74,16 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) mtx->vin[0].prevout = COutPoint{another_tx.GetHash(), 0}; } LOCK2(cs_main, pool.cs); - pool.addUnchecked(ConsumeTxMemPoolEntry(fuzzed_data_provider, another_tx)); + if (!pool.GetIter(another_tx.GetHash())) { + AddToMempool(pool, ConsumeTxMemPoolEntry(fuzzed_data_provider, another_tx)); + } } const CTransaction tx{*mtx}; if (fuzzed_data_provider.ConsumeBool()) { LOCK2(cs_main, pool.cs); - pool.addUnchecked(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx)); + if (!pool.GetIter(tx.GetHash())) { + AddToMempool(pool, ConsumeTxMemPoolEntry(fuzzed_data_provider, tx)); + } } { LOCK(pool.cs); @@ -88,6 +93,7 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); @@ -104,10 +110,18 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) std::vector<CTransaction> mempool_txs; size_t iter{0}; - int32_t replacement_vsize = fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(1, 1000000); - // Keep track of the total vsize of CTxMemPoolEntry's being added to the mempool to avoid overflow // Add replacement_vsize since this is added to new diagram during RBF check + std::optional<CMutableTransaction> replacement_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); + if (!replacement_tx) { + return; + } + assert(iter <= g_outpoints.size()); + replacement_tx->vin.resize(1); + replacement_tx->vin[0].prevout = g_outpoints[iter++]; + CTransaction replacement_tx_final{*replacement_tx}; + auto replacement_entry = ConsumeTxMemPoolEntry(fuzzed_data_provider, replacement_tx_final); + int32_t replacement_vsize = replacement_entry.GetTxSize(); int64_t running_vsize_total{replacement_vsize}; LOCK2(cs_main, pool.cs); @@ -129,7 +143,8 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) mempool_txs.pop_back(); break; } - pool.addUnchecked(parent_entry); + assert(!pool.GetIter(parent_entry.GetTx().GetHash())); + AddToMempool(pool, parent_entry); if (fuzzed_data_provider.ConsumeBool()) { child.vin[0].prevout = COutPoint{mempool_txs.back().GetHash(), 0}; } @@ -141,7 +156,9 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) mempool_txs.pop_back(); break; } - pool.addUnchecked(child_entry); + if (!pool.GetIter(child_entry.GetTx().GetHash())) { + AddToMempool(pool, child_entry); + } if (fuzzed_data_provider.ConsumeBool()) { pool.PrioritiseTransaction(mempool_txs.back().GetHash().ToUint256(), fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(-100000, 100000)); @@ -162,9 +179,17 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) pool.CalculateDescendants(txiter, all_conflicts); } - // Calculate the chunks for a replacement. CAmount replacement_fees = ConsumeMoney(fuzzed_data_provider); - auto calc_results{pool.CalculateChunksForRBF(replacement_fees, replacement_vsize, direct_conflicts, all_conflicts)}; + auto changeset = pool.GetChangeSet(); + for (auto& txiter : all_conflicts) { + changeset->StageRemoval(txiter); + } + changeset->StageAddition(replacement_entry.GetSharedTx(), replacement_fees, + replacement_entry.GetTime().count(), replacement_entry.GetHeight(), + replacement_entry.GetSequence(), replacement_entry.GetSpendsCoinbase(), + replacement_entry.GetSigOpCost(), replacement_entry.GetLockPoints()); + // Calculate the chunks for a replacement. + auto calc_results{changeset->CalculateChunksForRBF()}; if (calc_results.has_value()) { // Sanity checks on the chunks. @@ -192,7 +217,7 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) } // If internals report error, wrapper should too - auto err_tuple{ImprovesFeerateDiagram(pool, direct_conflicts, all_conflicts, replacement_fees, replacement_vsize)}; + auto err_tuple{ImprovesFeerateDiagram(*changeset)}; if (!calc_results.has_value()) { assert(err_tuple.value().first == DiagramCheckError::UNCALCULABLE); } else { diff --git a/src/test/fuzz/rolling_bloom_filter.cpp b/src/test/fuzz/rolling_bloom_filter.cpp index 9c18ad49cb..1bea0ba9da 100644 --- a/src/test/fuzz/rolling_bloom_filter.cpp +++ b/src/test/fuzz/rolling_bloom_filter.cpp @@ -6,6 +6,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/util/random.h> #include <uint256.h> #include <cassert> @@ -16,6 +17,7 @@ FUZZ_TARGET(rolling_bloom_filter) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CRollingBloomFilter rolling_bloom_filter{ diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 9122617e46..7bf90b9b77 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -130,6 +130,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "getchaintxstats", "getconnectioncount", "getdeploymentinfo", + "getdescriptoractivity", "getdescriptorinfo", "getdifficulty", "getindexinfo", @@ -143,6 +144,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "getnetworkhashps", "getnetworkinfo", "getnodeaddresses", + "getorphantxs", "getpeerinfo", "getprioritisedtransactions", "getrawaddrman", @@ -363,6 +365,7 @@ void initialize_rpc() FUZZ_TARGET(rpc, .init = initialize_rpc) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; bool good_data{true}; SetMockTime(ConsumeTime(fuzzed_data_provider)); diff --git a/src/test/fuzz/script_sigcache.cpp b/src/test/fuzz/script_sigcache.cpp index 7f73c3706c..702a056d07 100644 --- a/src/test/fuzz/script_sigcache.cpp +++ b/src/test/fuzz/script_sigcache.cpp @@ -25,6 +25,7 @@ void initialize_script_sigcache() FUZZ_TARGET(script_sigcache, .init = initialize_script_sigcache) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const auto max_sigcache_bytes{fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, DEFAULT_SIGNATURE_CACHE_BYTES)}; diff --git a/src/test/fuzz/script_sign.cpp b/src/test/fuzz/script_sign.cpp index 7725ee699c..9fa5e0b7d8 100644 --- a/src/test/fuzz/script_sign.cpp +++ b/src/test/fuzz/script_sign.cpp @@ -13,6 +13,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/util/transaction_utils.h> #include <util/chaintype.h> #include <util/translation.h> diff --git a/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp b/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp index ae0c8479cb..c6e31e4dc7 100644 --- a/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp +++ b/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp @@ -7,6 +7,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/util/random.h> #include <cstdint> #include <vector> @@ -16,6 +17,7 @@ int ecdsa_signature_parse_der_lax(secp256k1_ecdsa_signature* sig, const unsigned FUZZ_TARGET(secp256k1_ecdsa_signature_parse_der_lax) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; const std::vector<uint8_t> signature_bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); if (signature_bytes.data() == nullptr) { diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index 2a043f7458..c9eb11222f 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -14,6 +14,7 @@ #include <primitives/transaction.h> #include <streams.h> #include <test/fuzz/fuzz.h> +#include <test/util/random.h> #include <univalue.h> #include <util/chaintype.h> #include <util/rbf.h> @@ -28,6 +29,7 @@ void initialize_transaction() FUZZ_TARGET(transaction, .init = initialize_transaction) { + SeedRandomStateForTest(SeedRand::ZEROS); DataStream ds{buffer}; bool valid_tx = true; const CTransaction tx = [&] { diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 64861311db..a697ee9d83 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -45,8 +45,11 @@ void initialize_tx_pool() static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); g_setup = testing_setup.get(); + BlockAssembler::Options options; + options.coinbase_output_script = P2WSH_OP_TRUE; + for (int i = 0; i < 2 * COINBASE_MATURITY; ++i) { - COutPoint prevout{MineBlock(g_setup->m_node, P2WSH_OP_TRUE)}; + COutPoint prevout{MineBlock(g_setup->m_node, options)}; // Remember the txids to avoid expensive disk access later on auto& outpoints = i < COINBASE_MATURITY ? g_outpoints_coinbase_init_mature : @@ -98,7 +101,7 @@ void Finish(FuzzedDataProvider& fuzzed_data_provider, MockedTxPool& tx_pool, Cha options.nBlockMaxWeight = fuzzed_data_provider.ConsumeIntegralInRange(0U, MAX_BLOCK_WEIGHT); options.blockMinFeeRate = CFeeRate{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; auto assembler = BlockAssembler{chainstate, &tx_pool, options}; - auto block_template = assembler.CreateNewBlock(CScript{} << OP_TRUE); + auto block_template = assembler.CreateNewBlock(); Assert(block_template->block.vtx.size() >= 1); } const auto info_all = tx_pool.infoAll(); @@ -187,6 +190,7 @@ void CheckATMPInvariants(const MempoolAcceptResult& res, bool txid_in_mempool, b FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const auto& node = g_setup->m_node; auto& chainstate{static_cast<DummyChainState&>(node.chainman->ActiveChainstate())}; @@ -214,9 +218,8 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) // Helper to query an amount const CCoinsViewMemPool amount_view{WITH_LOCK(::cs_main, return &chainstate.CoinsTip()), tx_pool}; const auto GetAmount = [&](const COutPoint& outpoint) { - Coin c; - Assert(amount_view.GetCoin(outpoint, c)); - return c.out.nValue; + auto coin{amount_view.GetCoin(outpoint).value()}; + return coin.out.nValue; }; LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 300) @@ -366,6 +369,7 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) FUZZ_TARGET(tx_pool, .init = initialize_tx_pool) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const auto& node = g_setup->m_node; auto& chainstate{static_cast<DummyChainState&>(node.chainman->ActiveChainstate())}; diff --git a/src/test/fuzz/txdownloadman.cpp b/src/test/fuzz/txdownloadman.cpp new file mode 100644 index 0000000000..4917e8b405 --- /dev/null +++ b/src/test/fuzz/txdownloadman.cpp @@ -0,0 +1,447 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/validation.h> +#include <node/context.h> +#include <node/mempool_args.h> +#include <node/miner.h> +#include <node/txdownloadman.h> +#include <node/txdownloadman_impl.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/fuzz/util/mempool.h> +#include <test/util/mining.h> +#include <test/util/script.h> +#include <test/util/setup_common.h> +#include <test/util/txmempool.h> +#include <util/hasher.h> +#include <util/rbf.h> +#include <txmempool.h> +#include <validation.h> +#include <validationinterface.h> + +namespace { + +const TestingSetup* g_setup; + +constexpr size_t NUM_COINS{50}; +COutPoint COINS[NUM_COINS]; + +static TxValidationResult TESTED_TX_RESULTS[] = { + // Skip TX_RESULT_UNSET + TxValidationResult::TX_CONSENSUS, + TxValidationResult::TX_INPUTS_NOT_STANDARD, + TxValidationResult::TX_NOT_STANDARD, + TxValidationResult::TX_MISSING_INPUTS, + TxValidationResult::TX_PREMATURE_SPEND, + TxValidationResult::TX_WITNESS_MUTATED, + TxValidationResult::TX_WITNESS_STRIPPED, + TxValidationResult::TX_CONFLICT, + TxValidationResult::TX_MEMPOOL_POLICY, + // Skip TX_NO_MEMPOOL + TxValidationResult::TX_RECONSIDERABLE, + TxValidationResult::TX_UNKNOWN, +}; + +// Precomputed transactions. Some may conflict with each other. +std::vector<CTransactionRef> TRANSACTIONS; + +// Limit the total number of peers because we don't expect coverage to change much with lots more peers. +constexpr int NUM_PEERS = 16; + +// Precomputed random durations (positive and negative, each ~exponentially distributed). +std::chrono::microseconds TIME_SKIPS[128]; + +static CTransactionRef MakeTransactionSpending(const std::vector<COutPoint>& outpoints, size_t num_outputs, bool add_witness) +{ + CMutableTransaction tx; + // If no outpoints are given, create a random one. + for (const auto& outpoint : outpoints) { + tx.vin.emplace_back(outpoint); + } + if (add_witness) { + tx.vin[0].scriptWitness.stack.push_back({1}); + } + for (size_t o = 0; o < num_outputs; ++o) tx.vout.emplace_back(CENT, P2WSH_OP_TRUE); + return MakeTransactionRef(tx); +} +static std::vector<COutPoint> PickCoins(FuzzedDataProvider& fuzzed_data_provider) +{ + std::vector<COutPoint> ret; + ret.push_back(fuzzed_data_provider.PickValueInArray(COINS)); + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10) { + ret.push_back(fuzzed_data_provider.PickValueInArray(COINS)); + } + return ret; +} + +void initialize() +{ + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); + for (uint32_t i = 0; i < uint32_t{NUM_COINS}; ++i) { + COINS[i] = COutPoint{Txid::FromUint256((HashWriter() << i).GetHash()), i}; + } + size_t outpoints_index = 0; + // 2 transactions same txid different witness + { + auto tx1{MakeTransactionSpending({COINS[outpoints_index]}, /*num_outputs=*/5, /*add_witness=*/false)}; + auto tx2{MakeTransactionSpending({COINS[outpoints_index]}, /*num_outputs=*/5, /*add_witness=*/true)}; + Assert(tx1->GetHash() == tx2->GetHash()); + TRANSACTIONS.emplace_back(tx1); + TRANSACTIONS.emplace_back(tx2); + outpoints_index += 1; + } + // 2 parents 1 child + { + auto tx_parent_1{MakeTransactionSpending({COINS[outpoints_index++]}, /*num_outputs=*/1, /*add_witness=*/true)}; + TRANSACTIONS.emplace_back(tx_parent_1); + auto tx_parent_2{MakeTransactionSpending({COINS[outpoints_index++]}, /*num_outputs=*/1, /*add_witness=*/false)}; + TRANSACTIONS.emplace_back(tx_parent_2); + TRANSACTIONS.emplace_back(MakeTransactionSpending({COutPoint{tx_parent_1->GetHash(), 0}, COutPoint{tx_parent_2->GetHash(), 0}}, + /*num_outputs=*/1, /*add_witness=*/true)); + } + // 1 parent 2 children + { + auto tx_parent{MakeTransactionSpending({COINS[outpoints_index++]}, /*num_outputs=*/2, /*add_witness=*/true)}; + TRANSACTIONS.emplace_back(tx_parent); + TRANSACTIONS.emplace_back(MakeTransactionSpending({COutPoint{tx_parent->GetHash(), 0}}, + /*num_outputs=*/1, /*add_witness=*/true)); + TRANSACTIONS.emplace_back(MakeTransactionSpending({COutPoint{tx_parent->GetHash(), 1}}, + /*num_outputs=*/1, /*add_witness=*/true)); + } + // chain of 5 segwit + { + COutPoint& last_outpoint = COINS[outpoints_index++]; + for (auto i{0}; i < 5; ++i) { + auto tx{MakeTransactionSpending({last_outpoint}, /*num_outputs=*/1, /*add_witness=*/true)}; + TRANSACTIONS.emplace_back(tx); + last_outpoint = COutPoint{tx->GetHash(), 0}; + } + } + // chain of 5 non-segwit + { + COutPoint& last_outpoint = COINS[outpoints_index++]; + for (auto i{0}; i < 5; ++i) { + auto tx{MakeTransactionSpending({last_outpoint}, /*num_outputs=*/1, /*add_witness=*/false)}; + TRANSACTIONS.emplace_back(tx); + last_outpoint = COutPoint{tx->GetHash(), 0}; + } + } + // Also create a loose tx for each outpoint. Some of these transactions conflict with the above + // or have the same txid. + for (const auto& outpoint : COINS) { + TRANSACTIONS.emplace_back(MakeTransactionSpending({outpoint}, /*num_outputs=*/1, /*add_witness=*/true)); + } + + // Create random-looking time jumps + int i = 0; + // TIME_SKIPS[N] for N=0..15 is just N microseconds. + for (; i < 16; ++i) { + TIME_SKIPS[i] = std::chrono::microseconds{i}; + } + // TIME_SKIPS[N] for N=16..127 has randomly-looking but roughly exponentially increasing values up to + // 198.416453 seconds. + for (; i < 128; ++i) { + int diff_bits = ((i - 10) * 2) / 9; + uint64_t diff = 1 + (CSipHasher(0, 0).Write(i).Finalize() >> (64 - diff_bits)); + TIME_SKIPS[i] = TIME_SKIPS[i - 1] + std::chrono::microseconds{diff}; + } +} + +void CheckPackageToValidate(const node::PackageToValidate& package_to_validate, NodeId peer) +{ + Assert(package_to_validate.m_senders.size() == 2); + Assert(package_to_validate.m_senders.front() == peer); + Assert(package_to_validate.m_senders.back() < NUM_PEERS); + + // Package is a 1p1c + const auto& package = package_to_validate.m_txns; + Assert(IsChildWithParents(package)); + Assert(package.size() == 2); +} + +FUZZ_TARGET(txdownloadman, .init = initialize) +{ + SeedRandomStateForTest(SeedRand::ZEROS); + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + // Initialize txdownloadman + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + const auto max_orphan_count = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, 300); + FastRandomContext det_rand{true}; + node::TxDownloadManager txdownloadman{node::TxDownloadOptions{pool, det_rand, max_orphan_count, true}}; + + std::chrono::microseconds time{244466666}; + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + NodeId rand_peer = fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, NUM_PEERS - 1); + + // Transaction can be one of the premade ones or a randomly generated one + auto rand_tx = fuzzed_data_provider.ConsumeBool() ? + MakeTransactionSpending(PickCoins(fuzzed_data_provider), + /*num_outputs=*/fuzzed_data_provider.ConsumeIntegralInRange(1, 500), + /*add_witness=*/fuzzed_data_provider.ConsumeBool()) : + TRANSACTIONS.at(fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(0, TRANSACTIONS.size() - 1)); + + CallOneOf( + fuzzed_data_provider, + [&] { + node::TxDownloadConnectionInfo info{ + .m_preferred = fuzzed_data_provider.ConsumeBool(), + .m_relay_permissions = fuzzed_data_provider.ConsumeBool(), + .m_wtxid_relay = fuzzed_data_provider.ConsumeBool() + }; + txdownloadman.ConnectedPeer(rand_peer, info); + }, + [&] { + txdownloadman.DisconnectedPeer(rand_peer); + txdownloadman.CheckIsEmpty(rand_peer); + }, + [&] { + txdownloadman.ActiveTipChange(); + }, + [&] { + CBlock block; + block.vtx.push_back(rand_tx); + txdownloadman.BlockConnected(std::make_shared<CBlock>(block)); + }, + [&] { + txdownloadman.BlockDisconnected(); + }, + [&] { + txdownloadman.MempoolAcceptedTx(rand_tx); + }, + [&] { + TxValidationState state; + state.Invalid(fuzzed_data_provider.PickValueInArray(TESTED_TX_RESULTS), ""); + bool first_time_failure{fuzzed_data_provider.ConsumeBool()}; + + node::RejectedTxTodo todo = txdownloadman.MempoolRejectedTx(rand_tx, state, rand_peer, first_time_failure); + Assert(first_time_failure || !todo.m_should_add_extra_compact_tx); + }, + [&] { + GenTxid gtxid = fuzzed_data_provider.ConsumeBool() ? + GenTxid::Txid(rand_tx->GetHash()) : + GenTxid::Wtxid(rand_tx->GetWitnessHash()); + txdownloadman.AddTxAnnouncement(rand_peer, gtxid, time, /*p2p_inv=*/fuzzed_data_provider.ConsumeBool()); + }, + [&] { + txdownloadman.GetRequestsToSend(rand_peer, time); + }, + [&] { + txdownloadman.ReceivedTx(rand_peer, rand_tx); + const auto& [should_validate, maybe_package] = txdownloadman.ReceivedTx(rand_peer, rand_tx); + // The only possible results should be: + // - Don't validate the tx, no package. + // - Don't validate the tx, package. + // - Validate the tx, no package. + // The only combination that doesn't make sense is validate both tx and package. + Assert(!(should_validate && maybe_package.has_value())); + if (maybe_package.has_value()) CheckPackageToValidate(*maybe_package, rand_peer); + }, + [&] { + txdownloadman.ReceivedNotFound(rand_peer, {rand_tx->GetWitnessHash()}); + }, + [&] { + const bool expect_work{txdownloadman.HaveMoreWork(rand_peer)}; + const auto ptx = txdownloadman.GetTxToReconsider(rand_peer); + // expect_work=true doesn't necessarily mean the next item from the workset isn't a + // nullptr, as the transaction could have been removed from orphanage without being + // removed from the peer's workset. + if (ptx) { + // However, if there was a non-null tx in the workset, HaveMoreWork should have + // returned true. + Assert(expect_work); + } + } + ); + // Jump forwards or backwards + auto time_skip = fuzzed_data_provider.PickValueInArray(TIME_SKIPS); + if (fuzzed_data_provider.ConsumeBool()) time_skip *= -1; + time += time_skip; + } + // Disconnect everybody, check that all data structures are empty. + for (NodeId nodeid = 0; nodeid < NUM_PEERS; ++nodeid) { + txdownloadman.DisconnectedPeer(nodeid); + txdownloadman.CheckIsEmpty(nodeid); + } + txdownloadman.CheckIsEmpty(); +} + +// Give node 0 relay permissions, and nobody else. This helps us remember who is a RelayPermissions +// peer without tracking anything (this is only for the txdownload_impl target). +static bool HasRelayPermissions(NodeId peer) { return peer == 0; } + +static void CheckInvariants(const node::TxDownloadManagerImpl& txdownload_impl, size_t max_orphan_count) +{ + const TxOrphanage& orphanage = txdownload_impl.m_orphanage; + + // Orphanage usage should never exceed what is allowed + Assert(orphanage.Size() <= max_orphan_count); + + // We should never have more than the maximum in-flight requests out for a peer. + for (NodeId peer = 0; peer < NUM_PEERS; ++peer) { + if (!HasRelayPermissions(peer)) { + Assert(txdownload_impl.m_txrequest.Count(peer) <= node::MAX_PEER_TX_ANNOUNCEMENTS); + } + } + txdownload_impl.m_txrequest.SanityCheck(); +} + +FUZZ_TARGET(txdownloadman_impl, .init = initialize) +{ + SeedRandomStateForTest(SeedRand::ZEROS); + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + // Initialize a TxDownloadManagerImpl + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + const auto max_orphan_count = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, 300); + FastRandomContext det_rand{true}; + node::TxDownloadManagerImpl txdownload_impl{node::TxDownloadOptions{pool, det_rand, max_orphan_count, true}}; + + std::chrono::microseconds time{244466666}; + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + NodeId rand_peer = fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, NUM_PEERS - 1); + + // Transaction can be one of the premade ones or a randomly generated one + auto rand_tx = fuzzed_data_provider.ConsumeBool() ? + MakeTransactionSpending(PickCoins(fuzzed_data_provider), + /*num_outputs=*/fuzzed_data_provider.ConsumeIntegralInRange(1, 500), + /*add_witness=*/fuzzed_data_provider.ConsumeBool()) : + TRANSACTIONS.at(fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(0, TRANSACTIONS.size() - 1)); + + CallOneOf( + fuzzed_data_provider, + [&] { + node::TxDownloadConnectionInfo info{ + .m_preferred = fuzzed_data_provider.ConsumeBool(), + .m_relay_permissions = HasRelayPermissions(rand_peer), + .m_wtxid_relay = fuzzed_data_provider.ConsumeBool() + }; + txdownload_impl.ConnectedPeer(rand_peer, info); + }, + [&] { + txdownload_impl.DisconnectedPeer(rand_peer); + txdownload_impl.CheckIsEmpty(rand_peer); + }, + [&] { + txdownload_impl.ActiveTipChange(); + // After a block update, nothing should be in the rejection caches + for (const auto& tx : TRANSACTIONS) { + Assert(!txdownload_impl.RecentRejectsFilter().contains(tx->GetWitnessHash().ToUint256())); + Assert(!txdownload_impl.RecentRejectsFilter().contains(tx->GetHash().ToUint256())); + Assert(!txdownload_impl.RecentRejectsReconsiderableFilter().contains(tx->GetWitnessHash().ToUint256())); + Assert(!txdownload_impl.RecentRejectsReconsiderableFilter().contains(tx->GetHash().ToUint256())); + } + }, + [&] { + CBlock block; + block.vtx.push_back(rand_tx); + txdownload_impl.BlockConnected(std::make_shared<CBlock>(block)); + // Block transactions must be removed from orphanage + Assert(!txdownload_impl.m_orphanage.HaveTx(rand_tx->GetWitnessHash())); + }, + [&] { + txdownload_impl.BlockDisconnected(); + Assert(!txdownload_impl.RecentConfirmedTransactionsFilter().contains(rand_tx->GetWitnessHash().ToUint256())); + Assert(!txdownload_impl.RecentConfirmedTransactionsFilter().contains(rand_tx->GetHash().ToUint256())); + }, + [&] { + txdownload_impl.MempoolAcceptedTx(rand_tx); + }, + [&] { + TxValidationState state; + state.Invalid(fuzzed_data_provider.PickValueInArray(TESTED_TX_RESULTS), ""); + bool first_time_failure{fuzzed_data_provider.ConsumeBool()}; + + bool reject_contains_wtxid{txdownload_impl.RecentRejectsFilter().contains(rand_tx->GetWitnessHash().ToUint256())}; + + node::RejectedTxTodo todo = txdownload_impl.MempoolRejectedTx(rand_tx, state, rand_peer, first_time_failure); + Assert(first_time_failure || !todo.m_should_add_extra_compact_tx); + if (!reject_contains_wtxid) Assert(todo.m_unique_parents.size() <= rand_tx->vin.size()); + }, + [&] { + GenTxid gtxid = fuzzed_data_provider.ConsumeBool() ? + GenTxid::Txid(rand_tx->GetHash()) : + GenTxid::Wtxid(rand_tx->GetWitnessHash()); + txdownload_impl.AddTxAnnouncement(rand_peer, gtxid, time, /*p2p_inv=*/fuzzed_data_provider.ConsumeBool()); + }, + [&] { + const auto getdata_requests = txdownload_impl.GetRequestsToSend(rand_peer, time); + // TxDownloadManager should not be telling us to request things we already have. + // Exclude m_lazy_recent_rejects_reconsiderable because it may request low-feerate parent of orphan. + for (const auto& gtxid : getdata_requests) { + Assert(!txdownload_impl.AlreadyHaveTx(gtxid, /*include_reconsiderable=*/false)); + } + }, + [&] { + const auto& [should_validate, maybe_package] = txdownload_impl.ReceivedTx(rand_peer, rand_tx); + // The only possible results should be: + // - Don't validate the tx, no package. + // - Don't validate the tx, package. + // - Validate the tx, no package. + // The only combination that doesn't make sense is validate both tx and package. + Assert(!(should_validate && maybe_package.has_value())); + if (should_validate) { + Assert(!txdownload_impl.AlreadyHaveTx(GenTxid::Wtxid(rand_tx->GetWitnessHash()), /*include_reconsiderable=*/true)); + } + if (maybe_package.has_value()) { + CheckPackageToValidate(*maybe_package, rand_peer); + + const auto& package = maybe_package->m_txns; + // Parent is in m_lazy_recent_rejects_reconsiderable and child is in m_orphanage + Assert(txdownload_impl.RecentRejectsReconsiderableFilter().contains(rand_tx->GetWitnessHash().ToUint256())); + Assert(txdownload_impl.m_orphanage.HaveTx(maybe_package->m_txns.back()->GetWitnessHash())); + // Package has not been rejected + Assert(!txdownload_impl.RecentRejectsReconsiderableFilter().contains(GetPackageHash(package))); + // Neither is in m_lazy_recent_rejects + Assert(!txdownload_impl.RecentRejectsFilter().contains(package.front()->GetWitnessHash().ToUint256())); + Assert(!txdownload_impl.RecentRejectsFilter().contains(package.back()->GetWitnessHash().ToUint256())); + } + }, + [&] { + txdownload_impl.ReceivedNotFound(rand_peer, {rand_tx->GetWitnessHash()}); + }, + [&] { + const bool expect_work{txdownload_impl.HaveMoreWork(rand_peer)}; + const auto ptx{txdownload_impl.GetTxToReconsider(rand_peer)}; + // expect_work=true doesn't necessarily mean the next item from the workset isn't a + // nullptr, as the transaction could have been removed from orphanage without being + // removed from the peer's workset. + if (ptx) { + // However, if there was a non-null tx in the workset, HaveMoreWork should have + // returned true. + Assert(expect_work); + Assert(txdownload_impl.AlreadyHaveTx(GenTxid::Wtxid(ptx->GetWitnessHash()), /*include_reconsiderable=*/false)); + // Presumably we have validated this tx. Use "missing inputs" to keep it in the + // orphanage longer. Later iterations might call MempoolAcceptedTx or + // MempoolRejectedTx with a different error. + TxValidationState state_missing_inputs; + state_missing_inputs.Invalid(TxValidationResult::TX_MISSING_INPUTS, ""); + txdownload_impl.MempoolRejectedTx(ptx, state_missing_inputs, rand_peer, fuzzed_data_provider.ConsumeBool()); + } + } + ); + + auto time_skip = fuzzed_data_provider.PickValueInArray(TIME_SKIPS); + if (fuzzed_data_provider.ConsumeBool()) time_skip *= -1; + time += time_skip; + CheckInvariants(txdownload_impl, max_orphan_count); + } + // Disconnect everybody, check that all data structures are empty. + for (NodeId nodeid = 0; nodeid < NUM_PEERS; ++nodeid) { + txdownload_impl.DisconnectedPeer(nodeid); + txdownload_impl.CheckIsEmpty(nodeid); + } + txdownload_impl.CheckIsEmpty(); +} + +} // namespace diff --git a/src/test/fuzz/txorphan.cpp b/src/test/fuzz/txorphan.cpp index 5129c05a39..31af1afff5 100644 --- a/src/test/fuzz/txorphan.cpp +++ b/src/test/fuzz/txorphan.cpp @@ -38,6 +38,7 @@ FUZZ_TARGET(txorphan, .init = initialize_orphanage) TxOrphanage orphanage; std::vector<COutPoint> outpoints; // Duplicates are tolerated + outpoints.reserve(200'000); // initial outpoints used to construct transactions later for (uint8_t i = 0; i < 4; i++) { @@ -55,12 +56,14 @@ FUZZ_TARGET(txorphan, .init = initialize_orphanage) const auto num_out = fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(1, 256); // pick outpoints from outpoints as input. We allow input duplicates on purpose, given we are not // running any transaction validation logic before adding transactions to the orphanage + tx_mut.vin.reserve(num_in); for (uint32_t i = 0; i < num_in; i++) { auto& prevout = PickValue(fuzzed_data_provider, outpoints); // try making transactions unique by setting a random nSequence, but allow duplicate transactions if they happen tx_mut.vin.emplace_back(prevout, CScript{}, fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(0, CTxIn::SEQUENCE_FINAL)); } // output amount will not affect txorphanage + tx_mut.vout.reserve(num_out); for (uint32_t i = 0; i < num_out; i++) { tx_mut.vout.emplace_back(CAmount{0}, CScript{}); } diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index 425b9559a7..d700e5ac97 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022 The Bitcoin Core developers +// Copyright (c) 2021-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -34,8 +34,8 @@ CAmount ConsumeMoney(FuzzedDataProvider& fuzzed_data_provider, const std::option int64_t ConsumeTime(FuzzedDataProvider& fuzzed_data_provider, const std::optional<int64_t>& min, const std::optional<int64_t>& max) noexcept { // Avoid t=0 (1970-01-01T00:00:00Z) since SetMockTime(0) disables mocktime. - static const int64_t time_min{946684801}; // 2000-01-01T00:00:01Z - static const int64_t time_max{4133980799}; // 2100-12-31T23:59:59Z + static const int64_t time_min{ParseISO8601DateTime("2000-01-01T00:00:01Z").value()}; + static const int64_t time_max{ParseISO8601DateTime("2100-12-31T23:59:59Z").value()}; return fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(min.value_or(time_min), max.value_or(time_max)); } diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index de09745730..38be59fb64 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -79,6 +79,7 @@ template<typename B = uint8_t> { const size_t n_elements = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, max_vector_size); std::vector<std::string> r; + r.reserve(n_elements); for (size_t i = 0; i < n_elements; ++i) { r.push_back(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length)); } @@ -90,6 +91,7 @@ template <typename T> { const size_t n_elements = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, max_vector_size); std::vector<T> r; + r.reserve(n_elements); for (size_t i = 0; i < n_elements; ++i) { r.push_back(fuzzed_data_provider.ConsumeIntegral<T>()); } @@ -180,6 +182,22 @@ template <typename WeakEnumType, size_t size> return UintToArith256(ConsumeUInt256(fuzzed_data_provider)); } +[[nodiscard]] inline arith_uint256 ConsumeArithUInt256InRange(FuzzedDataProvider& fuzzed_data_provider, const arith_uint256& min, const arith_uint256& max) noexcept +{ + assert(min <= max); + const arith_uint256 range = max - min; + const arith_uint256 value = ConsumeArithUInt256(fuzzed_data_provider); + arith_uint256 result = value; + // Avoid division by 0, in case range + 1 results in overflow. + if (range != ~arith_uint256(0)) { + const arith_uint256 quotient = value / (range + 1); + result = value - (quotient * (range + 1)); + } + result += min; + assert(result >= min && result <= max); + return result; +} + [[nodiscard]] std::map<COutPoint, Coin> ConsumeCoins(FuzzedDataProvider& fuzzed_data_provider) noexcept; [[nodiscard]] CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept; diff --git a/src/test/fuzz/util/CMakeLists.txt b/src/test/fuzz/util/CMakeLists.txt index f73a1a83c2..878286b0f4 100644 --- a/src/test/fuzz/util/CMakeLists.txt +++ b/src/test/fuzz/util/CMakeLists.txt @@ -3,6 +3,7 @@ # file COPYING or https://opensource.org/license/mit/. add_library(test_fuzz STATIC EXCLUDE_FROM_ALL + check_globals.cpp descriptor.cpp mempool.cpp net.cpp diff --git a/src/test/fuzz/util/check_globals.cpp b/src/test/fuzz/util/check_globals.cpp new file mode 100644 index 0000000000..fbc5a55598 --- /dev/null +++ b/src/test/fuzz/util/check_globals.cpp @@ -0,0 +1,41 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/util/check_globals.h> + +#include <test/util/random.h> + +#include <iostream> +#include <memory> +#include <optional> +#include <string> + +struct CheckGlobalsImpl { + CheckGlobalsImpl() + { + g_used_g_prng = false; + g_seeded_g_prng_zero = false; + } + ~CheckGlobalsImpl() + { + if (g_used_g_prng && !g_seeded_g_prng_zero) { + std::cerr << "\n\n" + "The current fuzz target used the global random state.\n\n" + + "This is acceptable, but requires the fuzz target to call \n" + "SeedRandomStateForTest(SeedRand::ZEROS) in the first line \n" + "of the FUZZ_TARGET function.\n\n" + + "An alternative solution would be to avoid any use of globals.\n\n" + + "Without a solution, fuzz instability and non-determinism can lead \n" + "to non-reproducible bugs or inefficient fuzzing.\n\n" + << std::endl; + std::abort(); // Abort, because AFL may try to recover from a std::exit + } + } +}; + +CheckGlobals::CheckGlobals() : m_impl(std::make_unique<CheckGlobalsImpl>()) {} +CheckGlobals::~CheckGlobals() = default; diff --git a/src/test/fuzz/util/check_globals.h b/src/test/fuzz/util/check_globals.h new file mode 100644 index 0000000000..79f247535a --- /dev/null +++ b/src/test/fuzz/util/check_globals.h @@ -0,0 +1,19 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_FUZZ_UTIL_CHECK_GLOBALS_H +#define BITCOIN_TEST_FUZZ_UTIL_CHECK_GLOBALS_H + +#include <memory> +#include <optional> +#include <string> + +struct CheckGlobalsImpl; +struct CheckGlobals { + CheckGlobals(); + ~CheckGlobals(); + std::unique_ptr<CheckGlobalsImpl> m_impl; +}; + +#endif // BITCOIN_TEST_FUZZ_UTIL_CHECK_GLOBALS_H diff --git a/src/test/fuzz/util/net.h b/src/test/fuzz/util/net.h index 1a5902329e..cc73cdff4b 100644 --- a/src/test/fuzz/util/net.h +++ b/src/test/fuzz/util/net.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_TEST_FUZZ_UTIL_NET_H #define BITCOIN_TEST_FUZZ_UTIL_NET_H +#include <addrman.h> +#include <addrman_impl.h> #include <net.h> #include <net_permissions.h> #include <netaddress.h> @@ -15,6 +17,7 @@ #include <test/fuzz/util.h> #include <test/util/net.h> #include <threadsafety.h> +#include <util/asmap.h> #include <util/sock.h> #include <chrono> @@ -34,6 +37,108 @@ */ CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider, FastRandomContext* rand = nullptr) noexcept; +class AddrManDeterministic : public AddrMan +{ +public: + explicit AddrManDeterministic(const NetGroupManager& netgroupman, FuzzedDataProvider& fuzzed_data_provider, int32_t check_ratio) + : AddrMan(netgroupman, /*deterministic=*/true, check_ratio) + { + WITH_LOCK(m_impl->cs, m_impl->insecure_rand.Reseed(ConsumeUInt256(fuzzed_data_provider))); + } + + /** + * Compare with another AddrMan. + * This compares: + * - the values in `mapInfo` (the keys aka ids are ignored) + * - vvNew entries refer to the same addresses + * - vvTried entries refer to the same addresses + */ + bool operator==(const AddrManDeterministic& other) const + { + LOCK2(m_impl->cs, other.m_impl->cs); + + if (m_impl->mapInfo.size() != other.m_impl->mapInfo.size() || m_impl->nNew != other.m_impl->nNew || + m_impl->nTried != other.m_impl->nTried) { + return false; + } + + // Check that all values in `mapInfo` are equal to all values in `other.mapInfo`. + // Keys may be different. + + auto addrinfo_hasher = [](const AddrInfo& a) { + CSipHasher hasher(0, 0); + auto addr_key = a.GetKey(); + auto source_key = a.source.GetAddrBytes(); + hasher.Write(TicksSinceEpoch<std::chrono::seconds>(a.m_last_success)); + hasher.Write(a.nAttempts); + hasher.Write(a.nRefCount); + hasher.Write(a.fInTried); + hasher.Write(a.GetNetwork()); + hasher.Write(a.source.GetNetwork()); + hasher.Write(addr_key.size()); + hasher.Write(source_key.size()); + hasher.Write(addr_key); + hasher.Write(source_key); + return (size_t)hasher.Finalize(); + }; + + auto addrinfo_eq = [](const AddrInfo& lhs, const AddrInfo& rhs) { + return std::tie(static_cast<const CService&>(lhs), lhs.source, lhs.m_last_success, lhs.nAttempts, lhs.nRefCount, lhs.fInTried) == + std::tie(static_cast<const CService&>(rhs), rhs.source, rhs.m_last_success, rhs.nAttempts, rhs.nRefCount, rhs.fInTried); + }; + + using Addresses = std::unordered_set<AddrInfo, decltype(addrinfo_hasher), decltype(addrinfo_eq)>; + + const size_t num_addresses{m_impl->mapInfo.size()}; + + Addresses addresses{num_addresses, addrinfo_hasher, addrinfo_eq}; + for (const auto& [id, addr] : m_impl->mapInfo) { + addresses.insert(addr); + } + + Addresses other_addresses{num_addresses, addrinfo_hasher, addrinfo_eq}; + for (const auto& [id, addr] : other.m_impl->mapInfo) { + other_addresses.insert(addr); + } + + if (addresses != other_addresses) { + return false; + } + + auto IdsReferToSameAddress = [&](nid_type id, nid_type other_id) EXCLUSIVE_LOCKS_REQUIRED(m_impl->cs, other.m_impl->cs) { + if (id == -1 && other_id == -1) { + return true; + } + if ((id == -1 && other_id != -1) || (id != -1 && other_id == -1)) { + return false; + } + return m_impl->mapInfo.at(id) == other.m_impl->mapInfo.at(other_id); + }; + + // Check that `vvNew` contains the same addresses as `other.vvNew`. Notice - `vvNew[i][j]` + // contains just an id and the address is to be found in `mapInfo.at(id)`. The ids + // themselves may differ between `vvNew` and `other.vvNew`. + for (size_t i = 0; i < ADDRMAN_NEW_BUCKET_COUNT; ++i) { + for (size_t j = 0; j < ADDRMAN_BUCKET_SIZE; ++j) { + if (!IdsReferToSameAddress(m_impl->vvNew[i][j], other.m_impl->vvNew[i][j])) { + return false; + } + } + } + + // Same for `vvTried`. + for (size_t i = 0; i < ADDRMAN_TRIED_BUCKET_COUNT; ++i) { + for (size_t j = 0; j < ADDRMAN_BUCKET_SIZE; ++j) { + if (!IdsReferToSameAddress(m_impl->vvTried[i][j], other.m_impl->vvTried[i][j])) { + return false; + } + } + } + + return true; + } +}; + class FuzzedSock : public Sock { FuzzedDataProvider& m_fuzzed_data_provider; @@ -93,6 +198,13 @@ public: return FuzzedSock{fuzzed_data_provider}; } +[[nodiscard]] inline NetGroupManager ConsumeNetGroupManager(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); + if (!SanityCheckASMap(asmap, 128)) asmap.clear(); + return NetGroupManager(asmap); +} + inline CSubNet ConsumeSubNet(FuzzedDataProvider& fuzzed_data_provider) noexcept { return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<uint8_t>()}; diff --git a/src/test/fuzz/util/wallet.h b/src/test/fuzz/util/wallet.h new file mode 100644 index 0000000000..8b55b7a985 --- /dev/null +++ b/src/test/fuzz/util/wallet.h @@ -0,0 +1,134 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_FUZZ_UTIL_WALLET_H +#define BITCOIN_TEST_FUZZ_UTIL_WALLET_H + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <policy/policy.h> +#include <wallet/coincontrol.h> +#include <wallet/fees.h> +#include <wallet/spend.h> +#include <wallet/test/util.h> +#include <wallet/wallet.h> + +namespace wallet { + +/** + * Wraps a descriptor wallet for fuzzing. + */ +struct FuzzedWallet { + std::shared_ptr<CWallet> wallet; + FuzzedWallet(interfaces::Chain& chain, const std::string& name, const std::string& seed_insecure) + { + wallet = std::make_shared<CWallet>(&chain, name, CreateMockableWalletDatabase()); + { + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + auto height{*Assert(chain.getHeight())}; + wallet->SetLastBlockProcessed(height, chain.getBlockHash(height)); + } + wallet->m_keypool_size = 1; // Avoid timeout in TopUp() + assert(wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); + ImportDescriptors(seed_insecure); + } + void ImportDescriptors(const std::string& seed_insecure) + { + const std::vector<std::string> DESCS{ + "pkh(%s/%s/*)", + "sh(wpkh(%s/%s/*))", + "tr(%s/%s/*)", + "wpkh(%s/%s/*)", + }; + + for (const std::string& desc_fmt : DESCS) { + for (bool internal : {true, false}) { + const auto descriptor{(strprintf)(desc_fmt, "[5aa9973a/66h/4h/2h]" + seed_insecure, int{internal})}; + + FlatSigningProvider keys; + std::string error; + auto parsed_desc = std::move(Parse(descriptor, keys, error, /*require_checksum=*/false).at(0)); + assert(parsed_desc); + assert(error.empty()); + assert(parsed_desc->IsRange()); + assert(parsed_desc->IsSingleType()); + assert(!keys.keys.empty()); + WalletDescriptor w_desc{std::move(parsed_desc), /*creation_time=*/0, /*range_start=*/0, /*range_end=*/1, /*next_index=*/0}; + assert(!wallet->GetDescriptorScriptPubKeyMan(w_desc)); + LOCK(wallet->cs_wallet); + auto spk_manager{wallet->AddWalletDescriptor(w_desc, keys, /*label=*/"", internal)}; + assert(spk_manager); + wallet->AddActiveScriptPubKeyMan(spk_manager->GetID(), *Assert(w_desc.descriptor->GetOutputType()), internal); + } + } + } + CTxDestination GetDestination(FuzzedDataProvider& fuzzed_data_provider) + { + auto type{fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES)}; + if (fuzzed_data_provider.ConsumeBool()) { + return *Assert(wallet->GetNewDestination(type, "")); + } else { + return *Assert(wallet->GetNewChangeDestination(type)); + } + } + CScript GetScriptPubKey(FuzzedDataProvider& fuzzed_data_provider) { return GetScriptForDestination(GetDestination(fuzzed_data_provider)); } + void FundTx(FuzzedDataProvider& fuzzed_data_provider, CMutableTransaction tx) + { + // The fee of "tx" is 0, so this is the total input and output amount + const CAmount total_amt{ + std::accumulate(tx.vout.begin(), tx.vout.end(), CAmount{}, [](CAmount t, const CTxOut& out) { return t + out.nValue; })}; + const uint32_t tx_size(GetVirtualTransactionSize(CTransaction{tx})); + std::set<int> subtract_fee_from_outputs; + if (fuzzed_data_provider.ConsumeBool()) { + for (size_t i{}; i < tx.vout.size(); ++i) { + if (fuzzed_data_provider.ConsumeBool()) { + subtract_fee_from_outputs.insert(i); + } + } + } + std::vector<CRecipient> recipients; + for (size_t idx = 0; idx < tx.vout.size(); idx++) { + const CTxOut& tx_out = tx.vout[idx]; + CTxDestination dest; + ExtractDestination(tx_out.scriptPubKey, dest); + CRecipient recipient = {dest, tx_out.nValue, subtract_fee_from_outputs.count(idx) == 1}; + recipients.push_back(recipient); + } + CCoinControl coin_control; + coin_control.m_allow_other_inputs = fuzzed_data_provider.ConsumeBool(); + CallOneOf( + fuzzed_data_provider, [&] { coin_control.destChange = GetDestination(fuzzed_data_provider); }, + [&] { coin_control.m_change_type.emplace(fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES)); }, + [&] { /* no op (leave uninitialized) */ }); + coin_control.fAllowWatchOnly = fuzzed_data_provider.ConsumeBool(); + coin_control.m_include_unsafe_inputs = fuzzed_data_provider.ConsumeBool(); + { + auto& r{coin_control.m_signal_bip125_rbf}; + CallOneOf( + fuzzed_data_provider, [&] { r = true; }, [&] { r = false; }, [&] { r = std::nullopt; }); + } + coin_control.m_feerate = CFeeRate{ + // A fee of this range should cover all cases + fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(0, 2 * total_amt), + tx_size, + }; + if (fuzzed_data_provider.ConsumeBool()) { + *coin_control.m_feerate += GetMinimumFeeRate(*wallet, coin_control, nullptr); + } + coin_control.fOverrideFeeRate = fuzzed_data_provider.ConsumeBool(); + // Add solving data (m_external_provider and SelectExternal)? + + int change_position{fuzzed_data_provider.ConsumeIntegralInRange<int>(-1, tx.vout.size() - 1)}; + bilingual_str error; + // Clear tx.vout since it is not meant to be used now that we are passing outputs directly. + // This sets us up for a future PR to completely remove tx from the function signature in favor of passing inputs directly + tx.vout.clear(); + (void)FundTransaction(*wallet, tx, recipients, change_position, /*lockUnspents=*/false, coin_control); + } +}; +} + +#endif // BITCOIN_TEST_FUZZ_UTIL_WALLET_H diff --git a/src/test/fuzz/utxo_snapshot.cpp b/src/test/fuzz/utxo_snapshot.cpp index 1241bba8be..9fe922cfaf 100644 --- a/src/test/fuzz/utxo_snapshot.cpp +++ b/src/test/fuzz/utxo_snapshot.cpp @@ -70,6 +70,7 @@ void initialize_chain() template <bool INVALID> void utxo_snapshot_fuzz(FuzzBufferType buffer) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); auto& setup{*g_setup}; bool dirty_chainman{false}; // Re-use the global chainman, but reset it when it is dirty diff --git a/src/test/fuzz/utxo_total_supply.cpp b/src/test/fuzz/utxo_total_supply.cpp index b0f1a1251a..84d82f71e2 100644 --- a/src/test/fuzz/utxo_total_supply.cpp +++ b/src/test/fuzz/utxo_total_supply.cpp @@ -17,8 +17,11 @@ #include <util/chaintype.h> #include <validation.h> +using node::BlockAssembler; + FUZZ_TARGET(utxo_total_supply) { + SeedRandomStateForTest(SeedRand::ZEROS); /** The testing setup that creates a chainman only (no chainstate) */ ChainTestingSetup test_setup{ ChainType::REGTEST, @@ -36,9 +39,11 @@ FUZZ_TARGET(utxo_total_supply) LOCK(chainman.GetMutex()); return chainman.ActiveHeight(); }; + BlockAssembler::Options options; + options.coinbase_output_script = CScript() << OP_FALSE; const auto PrepareNextBlock = [&]() { // Use OP_FALSE to avoid BIP30 check from hitting early - auto block = PrepareBlock(node, CScript{} << OP_FALSE); + auto block = PrepareBlock(node, options); // Replace OP_FALSE with OP_TRUE { CMutableTransaction tx{*block->vtx.back()}; diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index c70d9ddf1e..4f90138892 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -38,6 +38,7 @@ void initialize_validation_load_mempool() FUZZ_TARGET(validation_load_mempool, .init = initialize_validation_load_mempool) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; diff --git a/src/test/ipc_test.capnp b/src/test/ipc_test.capnp index 55a3dc2683..46cd08b94a 100644 --- a/src/test/ipc_test.capnp +++ b/src/test/ipc_test.capnp @@ -9,10 +9,15 @@ $Cxx.namespace("gen"); using Proxy = import "/mp/proxy.capnp"; $Proxy.include("test/ipc_test.h"); -$Proxy.includeTypes("ipc/capnp/common-types.h"); +$Proxy.includeTypes("test/ipc_test_types.h"); + +using Mining = import "../ipc/capnp/mining.capnp"; interface FooInterface $Proxy.wrap("FooImplementation") { add @0 (a :Int32, b :Int32) -> (result :Int32); passOutPoint @1 (arg :Data) -> (result :Data); passUniValue @2 (arg :Text) -> (result :Text); + passTransaction @3 (arg :Data) -> (result :Data); + passVectorChar @4 (arg :Data) -> (result :Data); + passBlockState @5 (arg :Mining.BlockValidationState) -> (result :Mining.BlockValidationState); } diff --git a/src/test/ipc_test.cpp b/src/test/ipc_test.cpp index e6de6e3e47..af37434980 100644 --- a/src/test/ipc_test.cpp +++ b/src/test/ipc_test.cpp @@ -12,6 +12,7 @@ #include <test/ipc_test.capnp.proxy.h> #include <test/ipc_test.h> #include <tinyformat.h> +#include <validation.h> #include <future> #include <thread> @@ -61,7 +62,7 @@ void IpcPipeTest() auto connection_client = std::make_unique<mp::Connection>(loop, kj::mv(pipe.ends[0])); auto foo_client = std::make_unique<mp::ProxyClient<gen::FooInterface>>( - connection_client->m_rpc_system.bootstrap(mp::ServerVatId().vat_id).castAs<gen::FooInterface>(), + connection_client->m_rpc_system->bootstrap(mp::ServerVatId().vat_id).castAs<gen::FooInterface>(), connection_client.get(), /* destroy_connection= */ false); foo_promise.set_value(std::move(foo_client)); disconnect_client = [&] { loop.sync([&] { connection_client.reset(); }); }; @@ -88,6 +89,38 @@ void IpcPipeTest() UniValue uni2{foo->passUniValue(uni1)}; BOOST_CHECK_EQUAL(uni1.write(), uni2.write()); + CMutableTransaction mtx; + mtx.version = 2; + mtx.nLockTime = 3; + mtx.vin.emplace_back(txout1); + mtx.vout.emplace_back(COIN, CScript()); + CTransactionRef tx1{MakeTransactionRef(mtx)}; + CTransactionRef tx2{foo->passTransaction(tx1)}; + BOOST_CHECK(*Assert(tx1) == *Assert(tx2)); + + std::vector<char> vec1{'H', 'e', 'l', 'l', 'o'}; + std::vector<char> vec2{foo->passVectorChar(vec1)}; + BOOST_CHECK_EQUAL(std::string_view(vec1.begin(), vec1.end()), std::string_view(vec2.begin(), vec2.end())); + + BlockValidationState bs1; + bs1.Invalid(BlockValidationResult::BLOCK_CHECKPOINT, "reject reason", "debug message"); + BlockValidationState bs2{foo->passBlockState(bs1)}; + BOOST_CHECK_EQUAL(bs1.IsValid(), bs2.IsValid()); + BOOST_CHECK_EQUAL(bs1.IsError(), bs2.IsError()); + BOOST_CHECK_EQUAL(bs1.IsInvalid(), bs2.IsInvalid()); + BOOST_CHECK_EQUAL(static_cast<int>(bs1.GetResult()), static_cast<int>(bs2.GetResult())); + BOOST_CHECK_EQUAL(bs1.GetRejectReason(), bs2.GetRejectReason()); + BOOST_CHECK_EQUAL(bs1.GetDebugMessage(), bs2.GetDebugMessage()); + + BlockValidationState bs3; + BlockValidationState bs4{foo->passBlockState(bs3)}; + BOOST_CHECK_EQUAL(bs3.IsValid(), bs4.IsValid()); + BOOST_CHECK_EQUAL(bs3.IsError(), bs4.IsError()); + BOOST_CHECK_EQUAL(bs3.IsInvalid(), bs4.IsInvalid()); + BOOST_CHECK_EQUAL(static_cast<int>(bs3.GetResult()), static_cast<int>(bs4.GetResult())); + BOOST_CHECK_EQUAL(bs3.GetRejectReason(), bs4.GetRejectReason()); + BOOST_CHECK_EQUAL(bs3.GetDebugMessage(), bs4.GetDebugMessage()); + // Test cleanup: disconnect pipe and join thread disconnect_client(); thread.join(); diff --git a/src/test/ipc_test.h b/src/test/ipc_test.h index 2453bfa23c..2d215a20f1 100644 --- a/src/test/ipc_test.h +++ b/src/test/ipc_test.h @@ -8,6 +8,7 @@ #include <primitives/transaction.h> #include <univalue.h> #include <util/fs.h> +#include <validation.h> class FooImplementation { @@ -15,6 +16,9 @@ public: int add(int a, int b) { return a + b; } COutPoint passOutPoint(COutPoint o) { return o; } UniValue passUniValue(UniValue v) { return v; } + CTransactionRef passTransaction(CTransactionRef t) { return t; } + std::vector<char> passVectorChar(std::vector<char> v) { return v; } + BlockValidationState passBlockState(BlockValidationState s) { return s; } }; void IpcPipeTest(); diff --git a/src/test/ipc_test_types.h b/src/test/ipc_test_types.h new file mode 100644 index 0000000000..b1d4829aa7 --- /dev/null +++ b/src/test/ipc_test_types.h @@ -0,0 +1,12 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_IPC_TEST_TYPES_H +#define BITCOIN_TEST_IPC_TEST_TYPES_H + +#include <ipc/capnp/common-types.h> +#include <ipc/capnp/mining-types.h> +#include <test/ipc_test.capnp.h> + +#endif // BITCOIN_TEST_IPC_TEST_TYPES_H diff --git a/src/test/logging_tests.cpp b/src/test/logging_tests.cpp index a6fd44e395..77ec81e597 100644 --- a/src/test/logging_tests.cpp +++ b/src/test/logging_tests.cpp @@ -83,15 +83,15 @@ BOOST_AUTO_TEST_CASE(logging_timer) BOOST_CHECK_EQUAL(micro_timer.LogMsg("msg").substr(0, result_prefix.size()), result_prefix); } -BOOST_FIXTURE_TEST_CASE(logging_LogPrintf_, LogSetup) +BOOST_FIXTURE_TEST_CASE(logging_LogPrintStr, LogSetup) { LogInstance().m_log_sourcelocations = true; - LogPrintf_("fn1", "src1", 1, BCLog::LogFlags::NET, BCLog::Level::Debug, "foo1: %s\n", "bar1"); - LogPrintf_("fn2", "src2", 2, BCLog::LogFlags::NET, BCLog::Level::Info, "foo2: %s\n", "bar2"); - LogPrintf_("fn3", "src3", 3, BCLog::LogFlags::ALL, BCLog::Level::Debug, "foo3: %s\n", "bar3"); - LogPrintf_("fn4", "src4", 4, BCLog::LogFlags::ALL, BCLog::Level::Info, "foo4: %s\n", "bar4"); - LogPrintf_("fn5", "src5", 5, BCLog::LogFlags::NONE, BCLog::Level::Debug, "foo5: %s\n", "bar5"); - LogPrintf_("fn6", "src6", 6, BCLog::LogFlags::NONE, BCLog::Level::Info, "foo6: %s\n", "bar6"); + LogInstance().LogPrintStr("foo1: bar1", "fn1", "src1", 1, BCLog::LogFlags::NET, BCLog::Level::Debug); + LogInstance().LogPrintStr("foo2: bar2", "fn2", "src2", 2, BCLog::LogFlags::NET, BCLog::Level::Info); + LogInstance().LogPrintStr("foo3: bar3", "fn3", "src3", 3, BCLog::LogFlags::ALL, BCLog::Level::Debug); + LogInstance().LogPrintStr("foo4: bar4", "fn4", "src4", 4, BCLog::LogFlags::ALL, BCLog::Level::Info); + LogInstance().LogPrintStr("foo5: bar5", "fn5", "src5", 5, BCLog::LogFlags::NONE, BCLog::Level::Debug); + LogInstance().LogPrintStr("foo6: bar6", "fn6", "src6", 6, BCLog::LogFlags::NONE, BCLog::Level::Info); std::ifstream file{tmp_log_path}; std::vector<std::string> log_lines; for (std::string log; std::getline(file, log);) { @@ -133,11 +133,11 @@ BOOST_FIXTURE_TEST_CASE(logging_LogPrintMacrosDeprecated, LogSetup) BOOST_FIXTURE_TEST_CASE(logging_LogPrintMacros, LogSetup) { - LogTrace(BCLog::NET, "foo6: %s\n", "bar6"); // not logged - LogDebug(BCLog::NET, "foo7: %s\n", "bar7"); - LogInfo("foo8: %s\n", "bar8"); - LogWarning("foo9: %s\n", "bar9"); - LogError("foo10: %s\n", "bar10"); + LogTrace(BCLog::NET, "foo6: %s", "bar6"); // not logged + LogDebug(BCLog::NET, "foo7: %s", "bar7"); + LogInfo("foo8: %s", "bar8"); + LogWarning("foo9: %s", "bar9"); + LogError("foo10: %s", "bar10"); std::ifstream file{tmp_log_path}; std::vector<std::string> log_lines; for (std::string log; std::getline(file, log);) { diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 9f8d434213..f0094dce59 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -72,17 +72,17 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) BOOST_CHECK_EQUAL(testPool.size(), poolSize); // Just the parent: - testPool.addUnchecked(entry.FromTx(txParent)); + AddToMempool(testPool, entry.FromTx(txParent)); poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txParent), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize - 1); // Parent, children, grandchildren: - testPool.addUnchecked(entry.FromTx(txParent)); + AddToMempool(testPool, entry.FromTx(txParent)); for (int i = 0; i < 3; i++) { - testPool.addUnchecked(entry.FromTx(txChild[i])); - testPool.addUnchecked(entry.FromTx(txGrandChild[i])); + AddToMempool(testPool, entry.FromTx(txChild[i])); + AddToMempool(testPool, entry.FromTx(txGrandChild[i])); } // Remove Child[0], GrandChild[0] should be removed: poolSize = testPool.size(); @@ -104,8 +104,8 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) // Add children and grandchildren, but NOT the parent (simulate the parent being in a block) for (int i = 0; i < 3; i++) { - testPool.addUnchecked(entry.FromTx(txChild[i])); - testPool.addUnchecked(entry.FromTx(txGrandChild[i])); + AddToMempool(testPool, entry.FromTx(txChild[i])); + AddToMempool(testPool, entry.FromTx(txGrandChild[i])); } // Now remove the parent, as might happen if a block-re-org occurs but the parent cannot be // put into the mempool (maybe because it is non-standard): @@ -137,28 +137,28 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; - pool.addUnchecked(entry.Fee(10000LL).FromTx(tx1)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(tx1)); /* highest fee */ CMutableTransaction tx2 = CMutableTransaction(); tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx2.vout[0].nValue = 2 * COIN; - pool.addUnchecked(entry.Fee(20000LL).FromTx(tx2)); + AddToMempool(pool, entry.Fee(20000LL).FromTx(tx2)); /* lowest fee */ CMutableTransaction tx3 = CMutableTransaction(); tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx3.vout[0].nValue = 5 * COIN; - pool.addUnchecked(entry.Fee(0LL).FromTx(tx3)); + AddToMempool(pool, entry.Fee(0LL).FromTx(tx3)); /* 2nd highest fee */ CMutableTransaction tx4 = CMutableTransaction(); tx4.vout.resize(1); tx4.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx4.vout[0].nValue = 6 * COIN; - pool.addUnchecked(entry.Fee(15000LL).FromTx(tx4)); + AddToMempool(pool, entry.Fee(15000LL).FromTx(tx4)); /* equal fee rate to tx1, but newer */ CMutableTransaction tx5 = CMutableTransaction(); @@ -166,7 +166,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx5.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx5.vout[0].nValue = 11 * COIN; entry.time = NodeSeconds{1s}; - pool.addUnchecked(entry.Fee(10000LL).FromTx(tx5)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(tx5)); BOOST_CHECK_EQUAL(pool.size(), 5U); std::vector<std::string> sortedOrder; @@ -184,7 +184,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx6.vout.resize(1); tx6.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx6.vout[0].nValue = 20 * COIN; - pool.addUnchecked(entry.Fee(0LL).FromTx(tx6)); + AddToMempool(pool, entry.Fee(0LL).FromTx(tx6)); BOOST_CHECK_EQUAL(pool.size(), 6U); // Check that at this point, tx6 is sorted low sortedOrder.insert(sortedOrder.begin(), tx6.GetHash().ToString()); @@ -208,7 +208,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_CHECK(*ancestors_calculated == setAncestors); } - pool.addUnchecked(entry.FromTx(tx7), setAncestors); + AddToMempool(pool, entry.FromTx(tx7)); BOOST_CHECK_EQUAL(pool.size(), 7U); // Now tx6 should be sorted higher (high fee child): tx7, tx6, tx2, ... @@ -226,7 +226,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx8.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx8.vout[0].nValue = 10 * COIN; setAncestors.insert(pool.GetIter(tx7.GetHash()).value()); - pool.addUnchecked(entry.Fee(0LL).Time(NodeSeconds{2s}).FromTx(tx8), setAncestors); + AddToMempool(pool, entry.Fee(0LL).Time(NodeSeconds{2s}).FromTx(tx8)); // Now tx8 should be sorted low, but tx6/tx both high sortedOrder.insert(sortedOrder.begin(), tx8.GetHash().ToString()); @@ -240,7 +240,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx9.vout.resize(1); tx9.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx9.vout[0].nValue = 1 * COIN; - pool.addUnchecked(entry.Fee(0LL).Time(NodeSeconds{3s}).FromTx(tx9), setAncestors); + AddToMempool(pool, entry.Fee(0LL).Time(NodeSeconds{3s}).FromTx(tx9)); // tx9 should be sorted low BOOST_CHECK_EQUAL(pool.size(), 9U); @@ -268,7 +268,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_CHECK(*ancestors_calculated == setAncestors); } - pool.addUnchecked(entry.FromTx(tx10), setAncestors); + AddToMempool(pool, entry.FromTx(tx10)); /** * tx8 and tx9 should both now be sorted higher @@ -313,14 +313,14 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; - pool.addUnchecked(entry.Fee(10000LL).FromTx(tx1)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(tx1)); /* highest fee */ CMutableTransaction tx2 = CMutableTransaction(); tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx2.vout[0].nValue = 2 * COIN; - pool.addUnchecked(entry.Fee(20000LL).FromTx(tx2)); + AddToMempool(pool, entry.Fee(20000LL).FromTx(tx2)); uint64_t tx2Size = GetVirtualTransactionSize(CTransaction(tx2)); /* lowest fee */ @@ -328,21 +328,21 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx3.vout[0].nValue = 5 * COIN; - pool.addUnchecked(entry.Fee(0LL).FromTx(tx3)); + AddToMempool(pool, entry.Fee(0LL).FromTx(tx3)); /* 2nd highest fee */ CMutableTransaction tx4 = CMutableTransaction(); tx4.vout.resize(1); tx4.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx4.vout[0].nValue = 6 * COIN; - pool.addUnchecked(entry.Fee(15000LL).FromTx(tx4)); + AddToMempool(pool, entry.Fee(15000LL).FromTx(tx4)); /* equal fee rate to tx1, but newer */ CMutableTransaction tx5 = CMutableTransaction(); tx5.vout.resize(1); tx5.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx5.vout[0].nValue = 11 * COIN; - pool.addUnchecked(entry.Fee(10000LL).FromTx(tx5)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(tx5)); BOOST_CHECK_EQUAL(pool.size(), 5U); std::vector<std::string> sortedOrder; @@ -371,7 +371,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) tx6.vout[0].nValue = 20 * COIN; uint64_t tx6Size = GetVirtualTransactionSize(CTransaction(tx6)); - pool.addUnchecked(entry.Fee(0LL).FromTx(tx6)); + AddToMempool(pool, entry.Fee(0LL).FromTx(tx6)); BOOST_CHECK_EQUAL(pool.size(), 6U); // Ties are broken by hash if (tx3.GetHash() < tx6.GetHash()) @@ -393,7 +393,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) /* set the fee to just below tx2's feerate when including ancestor */ CAmount fee = (20000/tx2Size)*(tx7Size + tx6Size) - 1; - pool.addUnchecked(entry.Fee(fee).FromTx(tx7)); + AddToMempool(pool, entry.Fee(fee).FromTx(tx7)); BOOST_CHECK_EQUAL(pool.size(), 7U); sortedOrder.insert(sortedOrder.begin()+1, tx7.GetHash().ToString()); CheckSort<ancestor_score>(pool, sortedOrder); @@ -425,7 +425,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) // Check that we sort by min(feerate, ancestor_feerate): // set the fee so that the ancestor feerate is above tx1/5, // but the transaction's own feerate is lower - pool.addUnchecked(entry.Fee(5000LL).FromTx(tx8)); + AddToMempool(pool, entry.Fee(5000LL).FromTx(tx8)); sortedOrder.insert(sortedOrder.end()-1, tx8.GetHash().ToString()); CheckSort<ancestor_score>(pool, sortedOrder); } @@ -443,7 +443,7 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; - pool.addUnchecked(entry.Fee(10000LL).FromTx(tx1)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(tx1)); CMutableTransaction tx2 = CMutableTransaction(); tx2.vin.resize(1); @@ -451,7 +451,7 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_2 << OP_EQUAL; tx2.vout[0].nValue = 10 * COIN; - pool.addUnchecked(entry.Fee(5000LL).FromTx(tx2)); + AddToMempool(pool, entry.Fee(5000LL).FromTx(tx2)); pool.TrimToSize(pool.DynamicMemoryUsage()); // should do nothing BOOST_CHECK(pool.exists(GenTxid::Txid(tx1.GetHash()))); @@ -461,7 +461,7 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) BOOST_CHECK(pool.exists(GenTxid::Txid(tx1.GetHash()))); BOOST_CHECK(!pool.exists(GenTxid::Txid(tx2.GetHash()))); - pool.addUnchecked(entry.FromTx(tx2)); + AddToMempool(pool, entry.FromTx(tx2)); CMutableTransaction tx3 = CMutableTransaction(); tx3.vin.resize(1); tx3.vin[0].prevout = COutPoint(tx2.GetHash(), 0); @@ -469,7 +469,7 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_3 << OP_EQUAL; tx3.vout[0].nValue = 10 * COIN; - pool.addUnchecked(entry.Fee(20000LL).FromTx(tx3)); + AddToMempool(pool, entry.Fee(20000LL).FromTx(tx3)); pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); // tx3 should pay for tx2 (CPFP) BOOST_CHECK(!pool.exists(GenTxid::Txid(tx1.GetHash()))); @@ -532,10 +532,10 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) tx7.vout[1].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[1].nValue = 10 * COIN; - pool.addUnchecked(entry.Fee(7000LL).FromTx(tx4)); - pool.addUnchecked(entry.Fee(1000LL).FromTx(tx5)); - pool.addUnchecked(entry.Fee(1100LL).FromTx(tx6)); - pool.addUnchecked(entry.Fee(9000LL).FromTx(tx7)); + AddToMempool(pool, entry.Fee(7000LL).FromTx(tx4)); + AddToMempool(pool, entry.Fee(1000LL).FromTx(tx5)); + AddToMempool(pool, entry.Fee(1100LL).FromTx(tx6)); + AddToMempool(pool, entry.Fee(9000LL).FromTx(tx7)); // we only require this to remove, at max, 2 txn, because it's not clear what we're really optimizing for aside from that pool.TrimToSize(pool.DynamicMemoryUsage() - 1); @@ -544,8 +544,8 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) BOOST_CHECK(!pool.exists(GenTxid::Txid(tx7.GetHash()))); if (!pool.exists(GenTxid::Txid(tx5.GetHash()))) - pool.addUnchecked(entry.Fee(1000LL).FromTx(tx5)); - pool.addUnchecked(entry.Fee(9000LL).FromTx(tx7)); + AddToMempool(pool, entry.Fee(1000LL).FromTx(tx5)); + AddToMempool(pool, entry.Fee(9000LL).FromTx(tx7)); pool.TrimToSize(pool.DynamicMemoryUsage() / 2); // should maximize mempool size by only removing 5/7 BOOST_CHECK(pool.exists(GenTxid::Txid(tx4.GetHash()))); @@ -553,8 +553,8 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) BOOST_CHECK(pool.exists(GenTxid::Txid(tx6.GetHash()))); BOOST_CHECK(!pool.exists(GenTxid::Txid(tx7.GetHash()))); - pool.addUnchecked(entry.Fee(1000LL).FromTx(tx5)); - pool.addUnchecked(entry.Fee(9000LL).FromTx(tx7)); + AddToMempool(pool, entry.Fee(1000LL).FromTx(tx5)); + AddToMempool(pool, entry.Fee(9000LL).FromTx(tx7)); std::vector<CTransactionRef> vtx; SetMockTime(42); @@ -613,7 +613,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) // [tx1] // CTransactionRef tx1 = make_tx(/*output_values=*/{10 * COIN}); - pool.addUnchecked(entry.Fee(10000LL).FromTx(tx1)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(tx1)); // Ancestors / descendants should be 1 / 1 (itself / itself) pool.GetTransactionAncestry(tx1->GetHash(), ancestors, descendants); @@ -625,7 +625,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) // [tx1].0 <- [tx2] // CTransactionRef tx2 = make_tx(/*output_values=*/{495 * CENT, 5 * COIN}, /*inputs=*/{tx1}); - pool.addUnchecked(entry.Fee(10000LL).FromTx(tx2)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(tx2)); // Ancestors / descendants should be: // transaction ancestors descendants @@ -644,7 +644,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) // [tx1].0 <- [tx2].0 <- [tx3] // CTransactionRef tx3 = make_tx(/*output_values=*/{290 * CENT, 200 * CENT}, /*inputs=*/{tx2}); - pool.addUnchecked(entry.Fee(10000LL).FromTx(tx3)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(tx3)); // Ancestors / descendants should be: // transaction ancestors descendants @@ -669,7 +669,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) // \---1 <- [tx4] // CTransactionRef tx4 = make_tx(/*output_values=*/{290 * CENT, 250 * CENT}, /*inputs=*/{tx2}, /*input_indices=*/{1}); - pool.addUnchecked(entry.Fee(10000LL).FromTx(tx4)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(tx4)); // Ancestors / descendants should be: // transaction ancestors descendants @@ -706,13 +706,13 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) CTransactionRef& tyi = *ty[i]; tyi = make_tx(/*output_values=*/{v}, /*inputs=*/i > 0 ? std::vector<CTransactionRef>{*ty[i - 1]} : std::vector<CTransactionRef>{}); v -= 50 * CENT; - pool.addUnchecked(entry.Fee(10000LL).FromTx(tyi)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(tyi)); pool.GetTransactionAncestry(tyi->GetHash(), ancestors, descendants); BOOST_CHECK_EQUAL(ancestors, i+1); BOOST_CHECK_EQUAL(descendants, i+1); } CTransactionRef ty6 = make_tx(/*output_values=*/{5 * COIN}, /*inputs=*/{tx3, ty5}); - pool.addUnchecked(entry.Fee(10000LL).FromTx(ty6)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(ty6)); // Ancestors / descendants should be: // transaction ancestors descendants @@ -778,10 +778,10 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTestsDiamond) tb = make_tx(/*output_values=*/{5 * COIN, 3 * COIN}, /*inputs=*/ {ta}); tc = make_tx(/*output_values=*/{2 * COIN}, /*inputs=*/{tb}, /*input_indices=*/{1}); td = make_tx(/*output_values=*/{6 * COIN}, /*inputs=*/{tb, tc}, /*input_indices=*/{0, 0}); - pool.addUnchecked(entry.Fee(10000LL).FromTx(ta)); - pool.addUnchecked(entry.Fee(10000LL).FromTx(tb)); - pool.addUnchecked(entry.Fee(10000LL).FromTx(tc)); - pool.addUnchecked(entry.Fee(10000LL).FromTx(td)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(ta)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(tb)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(tc)); + AddToMempool(pool, entry.Fee(10000LL).FromTx(td)); // Ancestors / descendants should be: // transaction ancestors descendants diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp index 70308cb29a..f8cb0094fc 100644 --- a/src/test/merkle_tests.cpp +++ b/src/test/merkle_tests.cpp @@ -23,110 +23,6 @@ static uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vecto return hash; } -/* This implements a constant-space merkle root/path calculator, limited to 2^32 leaves. */ -static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot, bool* pmutated, uint32_t branchpos, std::vector<uint256>* pbranch) { - if (pbranch) pbranch->clear(); - if (leaves.size() == 0) { - if (pmutated) *pmutated = false; - if (proot) *proot = uint256(); - return; - } - bool mutated = false; - // count is the number of leaves processed so far. - uint32_t count = 0; - // inner is an array of eagerly computed subtree hashes, indexed by tree - // level (0 being the leaves). - // For example, when count is 25 (11001 in binary), inner[4] is the hash of - // the first 16 leaves, inner[3] of the next 8 leaves, and inner[0] equal to - // the last leaf. The other inner entries are undefined. - uint256 inner[32]; - // Which position in inner is a hash that depends on the matching leaf. - int matchlevel = -1; - // First process all leaves into 'inner' values. - while (count < leaves.size()) { - uint256 h = leaves[count]; - bool matchh = count == branchpos; - count++; - int level; - // For each of the lower bits in count that are 0, do 1 step. Each - // corresponds to an inner value that existed before processing the - // current leaf, and each needs a hash to combine it. - for (level = 0; !(count & ((uint32_t{1}) << level)); level++) { - if (pbranch) { - if (matchh) { - pbranch->push_back(inner[level]); - } else if (matchlevel == level) { - pbranch->push_back(h); - matchh = true; - } - } - mutated |= (inner[level] == h); - h = Hash(inner[level], h); - } - // Store the resulting hash at inner position level. - inner[level] = h; - if (matchh) { - matchlevel = level; - } - } - // Do a final 'sweep' over the rightmost branch of the tree to process - // odd levels, and reduce everything to a single top value. - // Level is the level (counted from the bottom) up to which we've sweeped. - int level = 0; - // As long as bit number level in count is zero, skip it. It means there - // is nothing left at this level. - while (!(count & ((uint32_t{1}) << level))) { - level++; - } - uint256 h = inner[level]; - bool matchh = matchlevel == level; - while (count != ((uint32_t{1}) << level)) { - // If we reach this point, h is an inner value that is not the top. - // We combine it with itself (Bitcoin's special rule for odd levels in - // the tree) to produce a higher level one. - if (pbranch && matchh) { - pbranch->push_back(h); - } - h = Hash(h, h); - // Increment count to the value it would have if two entries at this - // level had existed. - count += ((uint32_t{1}) << level); - level++; - // And propagate the result upwards accordingly. - while (!(count & ((uint32_t{1}) << level))) { - if (pbranch) { - if (matchh) { - pbranch->push_back(inner[level]); - } else if (matchlevel == level) { - pbranch->push_back(h); - matchh = true; - } - } - h = Hash(inner[level], h); - level++; - } - } - // Return result. - if (pmutated) *pmutated = mutated; - if (proot) *proot = h; -} - -static std::vector<uint256> ComputeMerkleBranch(const std::vector<uint256>& leaves, uint32_t position) { - std::vector<uint256> ret; - MerkleComputation(leaves, nullptr, nullptr, position, &ret); - return ret; -} - -static std::vector<uint256> BlockMerkleBranch(const CBlock& block, uint32_t position) -{ - std::vector<uint256> leaves; - leaves.resize(block.vtx.size()); - for (size_t s = 0; s < block.vtx.size(); s++) { - leaves[s] = block.vtx[s]->GetHash(); - } - return ComputeMerkleBranch(leaves, position); -} - // Older version of the merkle root computation code, for comparison. static uint256 BlockBuildMerkleTree(const CBlock& block, bool* fMutated, std::vector<uint256>& vMerkleTree) { @@ -239,7 +135,7 @@ BOOST_AUTO_TEST_CASE(merkle_test) if (ntx > 16) { mtx = m_rng.randrange(ntx); } - std::vector<uint256> newBranch = BlockMerkleBranch(block, mtx); + std::vector<uint256> newBranch = TransactionMerklePath(block, mtx); std::vector<uint256> oldBranch = BlockGetMerkleBranch(block, merkleTree, mtx); BOOST_CHECK(oldBranch == newBranch); BOOST_CHECK(ComputeMerkleRootFromBranch(block.vtx[mtx]->GetHash(), newBranch, mtx) == oldRoot); diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 9f35690460..d48f9cca3f 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -8,6 +8,7 @@ #include <consensus/consensus.h> #include <consensus/merkle.h> #include <consensus/tx_verify.h> +#include <interfaces/mining.h> #include <node/miner.h> #include <policy/policy.h> #include <test/util/random.h> @@ -28,8 +29,9 @@ #include <boost/test/unit_test.hpp> using namespace util::hex_literals; +using interfaces::BlockTemplate; +using interfaces::Mining; using node::BlockAssembler; -using node::CBlockTemplate; namespace miner_tests { struct MinerTestingSetup : public TestingSetup { @@ -54,7 +56,10 @@ struct MinerTestingSetup : public TestingSetup { Assert(error.empty()); return *m_node.mempool; } - BlockAssembler AssemblerForTest(CTxMemPool& tx_mempool); + std::unique_ptr<Mining> MakeMining() + { + return interfaces::MakeMining(m_node); + } }; } // namespace miner_tests @@ -62,15 +67,6 @@ BOOST_FIXTURE_TEST_SUITE(miner_tests, MinerTestingSetup) static CFeeRate blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); -BlockAssembler MinerTestingSetup::AssemblerForTest(CTxMemPool& tx_mempool) -{ - BlockAssembler::Options options; - - options.nBlockMaxWeight = MAX_BLOCK_WEIGHT; - options.blockMinFeeRate = blockMinFeeRate; - return BlockAssembler{m_node.chainman->ActiveChainstate(), &tx_mempool, options}; -} - constexpr static struct { unsigned char extranonce; unsigned int nonce; @@ -108,6 +104,10 @@ static std::unique_ptr<CBlockIndex> CreateBlockIndex(int nHeight, CBlockIndex* a void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) { CTxMemPool& tx_mempool{MakeMempool()}; + auto mining{MakeMining()}; + BlockAssembler::Options options; + options.coinbase_output_script = scriptPubKey; + LOCK(tx_mempool.cs); // Test the ancestor feerate transaction selection. TestMemPoolEntryHelper entry; @@ -123,31 +123,33 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const tx.vout[0].nValue = 5000000000LL - 1000; // This tx has a low fee: 1000 satoshis Txid hashParentTx = tx.GetHash(); // save this txid for later use - tx_mempool.addUnchecked(entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); // This tx has a medium fee: 10000 satoshis tx.vin[0].prevout.hash = txFirst[1]->GetHash(); tx.vout[0].nValue = 5000000000LL - 10000; Txid hashMediumFeeTx = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(10000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(10000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); // This tx has a high fee, but depends on the first transaction tx.vin[0].prevout.hash = hashParentTx; tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 50k satoshi fee Txid hashHighFeeTx = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(50000).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(50000).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); - std::unique_ptr<CBlockTemplate> pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); - BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 4U); - BOOST_CHECK(pblocktemplate->block.vtx[1]->GetHash() == hashParentTx); - BOOST_CHECK(pblocktemplate->block.vtx[2]->GetHash() == hashHighFeeTx); - BOOST_CHECK(pblocktemplate->block.vtx[3]->GetHash() == hashMediumFeeTx); + std::unique_ptr<BlockTemplate> block_template = mining->createNewBlock(options); + BOOST_REQUIRE(block_template); + CBlock block{block_template->getBlock()}; + BOOST_REQUIRE_EQUAL(block.vtx.size(), 4U); + BOOST_CHECK(block.vtx[1]->GetHash() == hashParentTx); + BOOST_CHECK(block.vtx[2]->GetHash() == hashHighFeeTx); + BOOST_CHECK(block.vtx[3]->GetHash() == hashMediumFeeTx); // Test that a package below the block min tx fee doesn't get included tx.vin[0].prevout.hash = hashHighFeeTx; tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 0 fee Txid hashFreeTx = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(0).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(0).FromTx(tx)); size_t freeTxSize = ::GetSerializeSize(TX_WITH_WITNESS(tx)); // Calculate a fee on child transaction that will put the package just @@ -157,12 +159,14 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const tx.vin[0].prevout.hash = hashFreeTx; tx.vout[0].nValue = 5000000000LL - 1000 - 50000 - feeToUse; Txid hashLowFeeTx = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(feeToUse).FromTx(tx)); - pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); + AddToMempool(tx_mempool, entry.Fee(feeToUse).FromTx(tx)); + block_template = mining->createNewBlock(options); + BOOST_REQUIRE(block_template); + block = block_template->getBlock(); // Verify that the free tx and the low fee tx didn't get selected - for (size_t i=0; i<pblocktemplate->block.vtx.size(); ++i) { - BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashFreeTx); - BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashLowFeeTx); + for (size_t i=0; i<block.vtx.size(); ++i) { + BOOST_CHECK(block.vtx[i]->GetHash() != hashFreeTx); + BOOST_CHECK(block.vtx[i]->GetHash() != hashLowFeeTx); } // Test that packages above the min relay fee do get included, even if one @@ -171,11 +175,13 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const tx_mempool.removeRecursive(CTransaction(tx), MemPoolRemovalReason::REPLACED); tx.vout[0].nValue -= 2; // Now we should be just over the min relay fee hashLowFeeTx = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(feeToUse + 2).FromTx(tx)); - pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); - BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 6U); - BOOST_CHECK(pblocktemplate->block.vtx[4]->GetHash() == hashFreeTx); - BOOST_CHECK(pblocktemplate->block.vtx[5]->GetHash() == hashLowFeeTx); + AddToMempool(tx_mempool, entry.Fee(feeToUse + 2).FromTx(tx)); + block_template = mining->createNewBlock(options); + BOOST_REQUIRE(block_template); + block = block_template->getBlock(); + BOOST_REQUIRE_EQUAL(block.vtx.size(), 6U); + BOOST_CHECK(block.vtx[4]->GetHash() == hashFreeTx); + BOOST_CHECK(block.vtx[5]->GetHash() == hashLowFeeTx); // Test that transaction selection properly updates ancestor fee // calculations as ancestor transactions get included in a block. @@ -185,7 +191,7 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const tx.vout[0].nValue = 5000000000LL - 100000000; tx.vout[1].nValue = 100000000; // 1BTC output Txid hashFreeTx2 = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(0).SpendsCoinbase(true).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(0).SpendsCoinbase(true).FromTx(tx)); // This tx can't be mined by itself tx.vin[0].prevout.hash = hashFreeTx2; @@ -193,23 +199,27 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const feeToUse = blockMinFeeRate.GetFee(freeTxSize); tx.vout[0].nValue = 5000000000LL - 100000000 - feeToUse; Txid hashLowFeeTx2 = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(feeToUse).SpendsCoinbase(false).FromTx(tx)); - pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); + AddToMempool(tx_mempool, entry.Fee(feeToUse).SpendsCoinbase(false).FromTx(tx)); + block_template = mining->createNewBlock(options); + BOOST_REQUIRE(block_template); + block = block_template->getBlock(); // Verify that this tx isn't selected. - for (size_t i=0; i<pblocktemplate->block.vtx.size(); ++i) { - BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashFreeTx2); - BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashLowFeeTx2); + for (size_t i=0; i<block.vtx.size(); ++i) { + BOOST_CHECK(block.vtx[i]->GetHash() != hashFreeTx2); + BOOST_CHECK(block.vtx[i]->GetHash() != hashLowFeeTx2); } // This tx will be mineable, and should cause hashLowFeeTx2 to be selected // as well. tx.vin[0].prevout.n = 1; tx.vout[0].nValue = 100000000 - 10000; // 10k satoshi fee - tx_mempool.addUnchecked(entry.Fee(10000).FromTx(tx)); - pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); - BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 9U); - BOOST_CHECK(pblocktemplate->block.vtx[8]->GetHash() == hashLowFeeTx2); + AddToMempool(tx_mempool, entry.Fee(10000).FromTx(tx)); + block_template = mining->createNewBlock(options); + BOOST_REQUIRE(block_template); + block = block_template->getBlock(); + BOOST_REQUIRE_EQUAL(block.vtx.size(), 9U); + BOOST_CHECK(block.vtx[8]->GetHash() == hashLowFeeTx2); } void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst, int baseheight) @@ -225,13 +235,20 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: const CAmount HIGHFEE = COIN; const CAmount HIGHERFEE = 4 * COIN; + auto mining{MakeMining()}; + BOOST_REQUIRE(mining); + + BlockAssembler::Options options; + options.coinbase_output_script = scriptPubKey; + { CTxMemPool& tx_mempool{MakeMempool()}; LOCK(tx_mempool.cs); // Just to make sure we can still make simple blocks - auto pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); - BOOST_CHECK(pblocktemplate); + auto block_template{mining->createNewBlock(options)}; + BOOST_REQUIRE(block_template); + CBlock block{block_template->getBlock()}; // block sigops > limit: 1000 CHECKMULTISIG + 1 tx.vin.resize(1); @@ -246,11 +263,11 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: hash = tx.GetHash(); bool spendsCoinbase = i == 0; // only first tx spends coinbase // If we don't set the # of sig ops in the CTxMemPoolEntry, template creation fails - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); tx.vin[0].prevout.hash = hash; } - BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-blk-sigops")); + BOOST_CHECK_EXCEPTION(mining->createNewBlock(options), std::runtime_error, HasReason("bad-blk-sigops")); } { @@ -264,10 +281,10 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: hash = tx.GetHash(); bool spendsCoinbase = i == 0; // only first tx spends coinbase // If we do set the # of sig ops in the CTxMemPoolEntry, template creation passes - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(spendsCoinbase).SigOpsCost(80).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(spendsCoinbase).SigOpsCost(80).FromTx(tx)); tx.vin[0].prevout.hash = hash; } - BOOST_CHECK(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); + BOOST_REQUIRE(mining->createNewBlock(options)); } { @@ -288,10 +305,10 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vout[0].nValue -= LOWFEE; hash = tx.GetHash(); bool spendsCoinbase = i == 0; // only first tx spends coinbase - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); tx.vin[0].prevout.hash = hash; } - BOOST_CHECK(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); + BOOST_REQUIRE(mining->createNewBlock(options)); } { @@ -300,8 +317,8 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: // orphan in tx_mempool, template creation fails hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).FromTx(tx)); - BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); + AddToMempool(tx_mempool, entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).FromTx(tx)); + BOOST_CHECK_EXCEPTION(mining->createNewBlock(options), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); } { @@ -313,7 +330,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vin[0].prevout.hash = txFirst[1]->GetHash(); tx.vout[0].nValue = BLOCKSUBSIDY - HIGHFEE; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(HIGHFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); tx.vin[0].prevout.hash = hash; tx.vin.resize(2); tx.vin[1].scriptSig = CScript() << OP_1; @@ -321,8 +338,8 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vin[1].prevout.n = 0; tx.vout[0].nValue = tx.vout[0].nValue + BLOCKSUBSIDY - HIGHERFEE; // First txn output + fresh coinbase - new txn fee hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(HIGHERFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); + AddToMempool(tx_mempool, entry.Fee(HIGHERFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); + BOOST_REQUIRE(mining->createNewBlock(options)); } { @@ -336,9 +353,9 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vout[0].nValue = 0; hash = tx.GetHash(); // give it a fee so it'll get mined - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); // Should throw bad-cb-multiple - BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-cb-multiple")); + BOOST_CHECK_EXCEPTION(mining->createNewBlock(options), std::runtime_error, HasReason("bad-cb-multiple")); } { @@ -351,11 +368,11 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vout[0].nValue = BLOCKSUBSIDY - HIGHFEE; tx.vout[0].scriptPubKey = CScript() << OP_1; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(HIGHFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); tx.vout[0].scriptPubKey = CScript() << OP_2; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); + AddToMempool(tx_mempool, entry.Fee(HIGHFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); + BOOST_CHECK_EXCEPTION(mining->createNewBlock(options), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); } { @@ -375,7 +392,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: next->BuildSkip(); m_node.chainman->ActiveChain().SetTip(*next); } - BOOST_CHECK(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); + BOOST_REQUIRE(mining->createNewBlock(options)); // Extend to a 210000-long block chain. while (m_node.chainman->ActiveChain().Tip()->nHeight < 210000) { CBlockIndex* prev = m_node.chainman->ActiveChain().Tip(); @@ -387,7 +404,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: next->BuildSkip(); m_node.chainman->ActiveChain().SetTip(*next); } - BOOST_CHECK(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); + BOOST_REQUIRE(mining->createNewBlock(options)); // invalid p2sh txn in tx_mempool, template creation fails tx.vin[0].prevout.hash = txFirst[0]->GetHash(); @@ -397,14 +414,13 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: CScript script = CScript() << OP_0; tx.vout[0].scriptPubKey = GetScriptForDestination(ScriptHash(script)); hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); tx.vin[0].prevout.hash = hash; tx.vin[0].scriptSig = CScript() << std::vector<unsigned char>(script.begin(), script.end()); tx.vout[0].nValue -= LOWFEE; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); - // Should throw block-validation-failed - BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("block-validation-failed")); + AddToMempool(tx_mempool, entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); + BOOST_CHECK_EXCEPTION(mining->createNewBlock(options), std::runtime_error, HasReason("mandatory-script-verify-flag-failed")); // Delete the dummy blocks again. while (m_node.chainman->ActiveChain().Tip()->nHeight > nHeight) { @@ -439,7 +455,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vout[0].scriptPubKey = CScript() << OP_1; tx.nLockTime = 0; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(HIGHFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); BOOST_CHECK(CheckFinalTxAtTip(*Assert(m_node.chainman->ActiveChain().Tip()), CTransaction{tx})); // Locktime passes BOOST_CHECK(!TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks fail @@ -453,7 +469,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG | (((m_node.chainman->ActiveChain().Tip()->GetMedianTimePast()+1-m_node.chainman->ActiveChain()[1]->GetMedianTimePast()) >> CTxIn::SEQUENCE_LOCKTIME_GRANULARITY) + 1); // txFirst[1] is the 3rd block prevheights[0] = baseheight + 2; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Time(Now<NodeSeconds>()).FromTx(tx)); + AddToMempool(tx_mempool, entry.Time(Now<NodeSeconds>()).FromTx(tx)); BOOST_CHECK(CheckFinalTxAtTip(*Assert(m_node.chainman->ActiveChain().Tip()), CTransaction{tx})); // Locktime passes BOOST_CHECK(!TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks fail @@ -476,7 +492,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: prevheights[0] = baseheight + 3; tx.nLockTime = m_node.chainman->ActiveChain().Tip()->nHeight + 1; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Time(Now<NodeSeconds>()).FromTx(tx)); + AddToMempool(tx_mempool, entry.Time(Now<NodeSeconds>()).FromTx(tx)); BOOST_CHECK(!CheckFinalTxAtTip(*Assert(m_node.chainman->ActiveChain().Tip()), CTransaction{tx})); // Locktime fails BOOST_CHECK(TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), m_node.chainman->ActiveChain().Tip()->nHeight + 2, m_node.chainman->ActiveChain().Tip()->GetMedianTimePast())); // Locktime passes on 2nd block @@ -487,7 +503,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: prevheights.resize(1); prevheights[0] = baseheight + 4; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Time(Now<NodeSeconds>()).FromTx(tx)); + AddToMempool(tx_mempool, entry.Time(Now<NodeSeconds>()).FromTx(tx)); BOOST_CHECK(!CheckFinalTxAtTip(*Assert(m_node.chainman->ActiveChain().Tip()), CTransaction{tx})); // Locktime fails BOOST_CHECK(TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), m_node.chainman->ActiveChain().Tip()->nHeight + 2, m_node.chainman->ActiveChain().Tip()->GetMedianTimePast() + 1)); // Locktime passes 1 second later @@ -506,14 +522,15 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG | 1; BOOST_CHECK(!TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks fail - auto pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); - BOOST_CHECK(pblocktemplate); + auto block_template = mining->createNewBlock(options); + BOOST_REQUIRE(block_template); // None of the of the absolute height/time locked tx should have made // it into the template because we still check IsFinalTx in CreateNewBlock, // but relative locked txs will if inconsistently added to mempool. // For now these will still generate a valid template until BIP68 soft fork - BOOST_CHECK_EQUAL(pblocktemplate->block.vtx.size(), 3U); + CBlock block{block_template->getBlock()}; + BOOST_CHECK_EQUAL(block.vtx.size(), 3U); // However if we advance height by 1 and time by SEQUENCE_LOCK_TIME, all of them should be mined for (int i = 0; i < CBlockIndex::nMedianTimeSpan; ++i) { CBlockIndex* ancestor{Assert(m_node.chainman->ActiveChain().Tip()->GetAncestor(m_node.chainman->ActiveChain().Tip()->nHeight - i))}; @@ -522,12 +539,20 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: m_node.chainman->ActiveChain().Tip()->nHeight++; SetMockTime(m_node.chainman->ActiveChain().Tip()->GetMedianTimePast() + 1); - BOOST_CHECK(pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); - BOOST_CHECK_EQUAL(pblocktemplate->block.vtx.size(), 5U); + block_template = mining->createNewBlock(options); + BOOST_REQUIRE(block_template); + block = block_template->getBlock(); + BOOST_CHECK_EQUAL(block.vtx.size(), 5U); } void MinerTestingSetup::TestPrioritisedMining(const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) { + auto mining{MakeMining()}; + BOOST_REQUIRE(mining); + + BlockAssembler::Options options; + options.coinbase_output_script = scriptPubKey; + CTxMemPool& tx_mempool{MakeMempool()}; LOCK(tx_mempool.cs); @@ -542,7 +567,7 @@ void MinerTestingSetup::TestPrioritisedMining(const CScript& scriptPubKey, const tx.vout.resize(1); tx.vout[0].nValue = 5000000000LL; // 0 fee uint256 hashFreePrioritisedTx = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(0).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(0).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); tx_mempool.PrioritiseTransaction(hashFreePrioritisedTx, 5 * COIN); tx.vin[0].prevout.hash = txFirst[1]->GetHash(); @@ -550,20 +575,20 @@ void MinerTestingSetup::TestPrioritisedMining(const CScript& scriptPubKey, const tx.vout[0].nValue = 5000000000LL - 1000; // This tx has a low fee: 1000 satoshis Txid hashParentTx = tx.GetHash(); // save this txid for later use - tx_mempool.addUnchecked(entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); // This tx has a medium fee: 10000 satoshis tx.vin[0].prevout.hash = txFirst[2]->GetHash(); tx.vout[0].nValue = 5000000000LL - 10000; Txid hashMediumFeeTx = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(10000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(10000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); tx_mempool.PrioritiseTransaction(hashMediumFeeTx, -5 * COIN); // This tx also has a low fee, but is prioritised tx.vin[0].prevout.hash = hashParentTx; tx.vout[0].nValue = 5000000000LL - 1000 - 1000; // 1000 satoshi fee Txid hashPrioritsedChild = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); tx_mempool.PrioritiseTransaction(hashPrioritsedChild, 2 * COIN); // Test that transaction selection properly updates ancestor fee calculations as prioritised @@ -575,45 +600,48 @@ void MinerTestingSetup::TestPrioritisedMining(const CScript& scriptPubKey, const tx.vin[0].prevout.hash = txFirst[3]->GetHash(); tx.vout[0].nValue = 5000000000LL; // 0 fee Txid hashFreeParent = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(0).SpendsCoinbase(true).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(0).SpendsCoinbase(true).FromTx(tx)); tx_mempool.PrioritiseTransaction(hashFreeParent, 10 * COIN); tx.vin[0].prevout.hash = hashFreeParent; tx.vout[0].nValue = 5000000000LL; // 0 fee Txid hashFreeChild = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(0).SpendsCoinbase(false).FromTx(tx)); + AddToMempool(tx_mempool, entry.Fee(0).SpendsCoinbase(false).FromTx(tx)); tx_mempool.PrioritiseTransaction(hashFreeChild, 1 * COIN); tx.vin[0].prevout.hash = hashFreeChild; tx.vout[0].nValue = 5000000000LL; // 0 fee Txid hashFreeGrandchild = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(0).SpendsCoinbase(false).FromTx(tx)); - - auto pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); - BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 6U); - BOOST_CHECK(pblocktemplate->block.vtx[1]->GetHash() == hashFreeParent); - BOOST_CHECK(pblocktemplate->block.vtx[2]->GetHash() == hashFreePrioritisedTx); - BOOST_CHECK(pblocktemplate->block.vtx[3]->GetHash() == hashParentTx); - BOOST_CHECK(pblocktemplate->block.vtx[4]->GetHash() == hashPrioritsedChild); - BOOST_CHECK(pblocktemplate->block.vtx[5]->GetHash() == hashFreeChild); - for (size_t i=0; i<pblocktemplate->block.vtx.size(); ++i) { + AddToMempool(tx_mempool, entry.Fee(0).SpendsCoinbase(false).FromTx(tx)); + + auto block_template = mining->createNewBlock(options); + BOOST_REQUIRE(block_template); + CBlock block{block_template->getBlock()}; + BOOST_REQUIRE_EQUAL(block.vtx.size(), 6U); + BOOST_CHECK(block.vtx[1]->GetHash() == hashFreeParent); + BOOST_CHECK(block.vtx[2]->GetHash() == hashFreePrioritisedTx); + BOOST_CHECK(block.vtx[3]->GetHash() == hashParentTx); + BOOST_CHECK(block.vtx[4]->GetHash() == hashPrioritsedChild); + BOOST_CHECK(block.vtx[5]->GetHash() == hashFreeChild); + for (size_t i=0; i<block.vtx.size(); ++i) { // The FreeParent and FreeChild's prioritisations should not impact the child. - BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashFreeGrandchild); + BOOST_CHECK(block.vtx[i]->GetHash() != hashFreeGrandchild); // De-prioritised transaction should not be included. - BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashMediumFeeTx); + BOOST_CHECK(block.vtx[i]->GetHash() != hashMediumFeeTx); } } // NOTE: These tests rely on CreateNewBlock doing its own self-validation! BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) { - // Note that by default, these tests run with size accounting enabled. - CScript scriptPubKey = CScript() << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f"_hex_v_u8 << OP_CHECKSIG; - std::unique_ptr<CBlockTemplate> pblocktemplate; + auto mining{MakeMining()}; + BOOST_REQUIRE(mining); - CTxMemPool& tx_mempool{*m_node.mempool}; - // Simple block creation, nothing special yet: - BOOST_CHECK(pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); + // Note that by default, these tests run with size accounting enabled. + CScript scriptPubKey = CScript() << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f"_hex << OP_CHECKSIG; + BlockAssembler::Options options; + options.coinbase_output_script = scriptPubKey; + std::unique_ptr<BlockTemplate> block_template; // We can't make transactions until we have inputs // Therefore, load 110 blocks :) @@ -621,27 +649,48 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) int baseheight = 0; std::vector<CTransactionRef> txFirst; for (const auto& bi : BLOCKINFO) { - CBlock *pblock = &pblocktemplate->block; // pointer for convenience + const int current_height{mining->getTip()->height}; + + // Simple block creation, nothing special yet: + block_template = mining->createNewBlock(options); + BOOST_REQUIRE(block_template); + + CBlock block{block_template->getBlock()}; + CMutableTransaction txCoinbase(*block.vtx[0]); { LOCK(cs_main); - pblock->nVersion = VERSIONBITS_TOP_BITS; - pblock->nTime = m_node.chainman->ActiveChain().Tip()->GetMedianTimePast()+1; - CMutableTransaction txCoinbase(*pblock->vtx[0]); + block.nVersion = VERSIONBITS_TOP_BITS; + block.nTime = Assert(m_node.chainman)->ActiveChain().Tip()->GetMedianTimePast()+1; txCoinbase.version = 1; - txCoinbase.vin[0].scriptSig = CScript{} << (m_node.chainman->ActiveChain().Height() + 1) << bi.extranonce; + txCoinbase.vin[0].scriptSig = CScript{} << (current_height + 1) << bi.extranonce; txCoinbase.vout.resize(1); // Ignore the (optional) segwit commitment added by CreateNewBlock (as the hardcoded nonces don't account for this) txCoinbase.vout[0].scriptPubKey = CScript(); - pblock->vtx[0] = MakeTransactionRef(std::move(txCoinbase)); + block.vtx[0] = MakeTransactionRef(txCoinbase); if (txFirst.size() == 0) - baseheight = m_node.chainman->ActiveChain().Height(); + baseheight = current_height; if (txFirst.size() < 4) - txFirst.push_back(pblock->vtx[0]); - pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); - pblock->nNonce = bi.nonce; + txFirst.push_back(block.vtx[0]); + block.hashMerkleRoot = BlockMerkleRoot(block); + block.nNonce = bi.nonce; + } + std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); + // Alternate calls between Chainman's ProcessNewBlock and submitSolution + // via the Mining interface. The former is used by net_processing as well + // as the submitblock RPC. + if (current_height % 2 == 0) { + BOOST_REQUIRE(Assert(m_node.chainman)->ProcessNewBlock(shared_pblock, /*force_processing=*/true, /*min_pow_checked=*/true, nullptr)); + } else { + BOOST_REQUIRE(block_template->submitSolution(block.nVersion, block.nTime, block.nNonce, MakeTransactionRef(txCoinbase))); + } + { + LOCK(cs_main); + // The above calls don't guarantee the tip is actually updated, so + // we explictly check this. + auto maybe_new_tip{Assert(m_node.chainman)->ActiveChain().Tip()}; + BOOST_REQUIRE_EQUAL(maybe_new_tip->GetBlockHash(), block.GetHash()); } - std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); - BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlock(shared_pblock, true, true, nullptr)); - pblock->hashPrevBlock = pblock->GetHash(); + // This just adds coverage + mining->waitTipChanged(block.hashPrevBlock); } LOCK(cs_main); diff --git a/src/test/miniminer_tests.cpp b/src/test/miniminer_tests.cpp index 5b007997b9..d1fec479a0 100644 --- a/src/test/miniminer_tests.cpp +++ b/src/test/miniminer_tests.cpp @@ -84,7 +84,7 @@ BOOST_FIXTURE_TEST_CASE(miniminer_negative, TestChain100Setup) const CAmount negative_modified_fees{positive_base_fee + negative_fee_delta}; BOOST_CHECK(negative_modified_fees < 0); const auto tx_mod_negative = make_tx({COutPoint{m_coinbase_txns[4]->GetHash(), 0}}, /*num_outputs=*/1); - pool.addUnchecked(entry.Fee(positive_base_fee).FromTx(tx_mod_negative)); + AddToMempool(pool, entry.Fee(positive_base_fee).FromTx(tx_mod_negative)); pool.PrioritiseTransaction(tx_mod_negative->GetHash(), negative_fee_delta); const COutPoint only_outpoint{tx_mod_negative->GetHash(), 0}; @@ -114,21 +114,21 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) // Create a parent tx0 and child tx1 with normal fees: const auto tx0 = make_tx({COutPoint{m_coinbase_txns[0]->GetHash(), 0}}, /*num_outputs=*/2); - pool.addUnchecked(entry.Fee(med_fee).FromTx(tx0)); + AddToMempool(pool, entry.Fee(med_fee).FromTx(tx0)); const auto tx1 = make_tx({COutPoint{tx0->GetHash(), 0}}, /*num_outputs=*/1); - pool.addUnchecked(entry.Fee(med_fee).FromTx(tx1)); + AddToMempool(pool, entry.Fee(med_fee).FromTx(tx1)); // Create a low-feerate parent tx2 and high-feerate child tx3 (cpfp) const auto tx2 = make_tx({COutPoint{m_coinbase_txns[1]->GetHash(), 0}}, /*num_outputs=*/2); - pool.addUnchecked(entry.Fee(low_fee).FromTx(tx2)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(tx2)); const auto tx3 = make_tx({COutPoint{tx2->GetHash(), 0}}, /*num_outputs=*/1); - pool.addUnchecked(entry.Fee(high_fee).FromTx(tx3)); + AddToMempool(pool, entry.Fee(high_fee).FromTx(tx3)); // Create a parent tx4 and child tx5 where both have low fees const auto tx4 = make_tx({COutPoint{m_coinbase_txns[2]->GetHash(), 0}}, /*num_outputs=*/2); - pool.addUnchecked(entry.Fee(low_fee).FromTx(tx4)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(tx4)); const auto tx5 = make_tx({COutPoint{tx4->GetHash(), 0}}, /*num_outputs=*/1); - pool.addUnchecked(entry.Fee(low_fee).FromTx(tx5)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(tx5)); const CAmount tx5_delta{CENT/100}; // Make tx5's modified fee much higher than its base fee. This should cause it to pass // the fee-related checks despite being low-feerate. @@ -137,9 +137,9 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) // Create a high-feerate parent tx6, low-feerate child tx7 const auto tx6 = make_tx({COutPoint{m_coinbase_txns[3]->GetHash(), 0}}, /*num_outputs=*/2); - pool.addUnchecked(entry.Fee(high_fee).FromTx(tx6)); + AddToMempool(pool, entry.Fee(high_fee).FromTx(tx6)); const auto tx7 = make_tx({COutPoint{tx6->GetHash(), 0}}, /*num_outputs=*/1); - pool.addUnchecked(entry.Fee(low_fee).FromTx(tx7)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(tx7)); std::vector<COutPoint> all_unspent_outpoints({ COutPoint{tx0->GetHash(), 1}, @@ -405,23 +405,23 @@ BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) // Create 3 parents of different feerates, and 1 child spending outputs from all 3 parents. const auto tx0 = make_tx({COutPoint{m_coinbase_txns[0]->GetHash(), 0}}, /*num_outputs=*/2); - pool.addUnchecked(entry.Fee(low_fee).FromTx(tx0)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(tx0)); const auto tx1 = make_tx({COutPoint{m_coinbase_txns[1]->GetHash(), 0}}, /*num_outputs=*/2); - pool.addUnchecked(entry.Fee(med_fee).FromTx(tx1)); + AddToMempool(pool, entry.Fee(med_fee).FromTx(tx1)); const auto tx2 = make_tx({COutPoint{m_coinbase_txns[2]->GetHash(), 0}}, /*num_outputs=*/2); - pool.addUnchecked(entry.Fee(high_fee).FromTx(tx2)); + AddToMempool(pool, entry.Fee(high_fee).FromTx(tx2)); const auto tx3 = make_tx({COutPoint{tx0->GetHash(), 0}, COutPoint{tx1->GetHash(), 0}, COutPoint{tx2->GetHash(), 0}}, /*num_outputs=*/3); - pool.addUnchecked(entry.Fee(high_fee).FromTx(tx3)); + AddToMempool(pool, entry.Fee(high_fee).FromTx(tx3)); // Create 1 grandparent and 1 parent, then 2 children. const auto tx4 = make_tx({COutPoint{m_coinbase_txns[3]->GetHash(), 0}}, /*num_outputs=*/2); - pool.addUnchecked(entry.Fee(high_fee).FromTx(tx4)); + AddToMempool(pool, entry.Fee(high_fee).FromTx(tx4)); const auto tx5 = make_tx({COutPoint{tx4->GetHash(), 0}}, /*num_outputs=*/3); - pool.addUnchecked(entry.Fee(low_fee).FromTx(tx5)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(tx5)); const auto tx6 = make_tx({COutPoint{tx5->GetHash(), 0}}, /*num_outputs=*/2); - pool.addUnchecked(entry.Fee(med_fee).FromTx(tx6)); + AddToMempool(pool, entry.Fee(med_fee).FromTx(tx6)); const auto tx7 = make_tx({COutPoint{tx5->GetHash(), 1}}, /*num_outputs=*/2); - pool.addUnchecked(entry.Fee(high_fee).FromTx(tx7)); + AddToMempool(pool, entry.Fee(high_fee).FromTx(tx7)); std::vector<CTransactionRef> all_transactions{tx0, tx1, tx2, tx3, tx4, tx5, tx6, tx7}; std::vector<int64_t> tx_vsizes; @@ -604,7 +604,7 @@ BOOST_FIXTURE_TEST_CASE(calculate_cluster, TestChain100Setup) auto& lasttx = m_coinbase_txns[0]; for (auto i{0}; i < 500; ++i) { const auto tx = make_tx({COutPoint{lasttx->GetHash(), 0}}, /*num_outputs=*/1); - pool.addUnchecked(entry.Fee(CENT).FromTx(tx)); + AddToMempool(pool, entry.Fee(CENT).FromTx(tx)); chain_txids.push_back(tx->GetHash()); lasttx = tx; } @@ -616,7 +616,7 @@ BOOST_FIXTURE_TEST_CASE(calculate_cluster, TestChain100Setup) // GatherClusters stops at 500 transactions. const auto tx_501 = make_tx({COutPoint{lasttx->GetHash(), 0}}, /*num_outputs=*/1); - pool.addUnchecked(entry.Fee(CENT).FromTx(tx_501)); + AddToMempool(pool, entry.Fee(CENT).FromTx(tx_501)); const auto cluster_501 = pool.GatherClusters({tx_501->GetHash()}); BOOST_CHECK_EQUAL(cluster_501.size(), 0); @@ -629,12 +629,12 @@ BOOST_FIXTURE_TEST_CASE(calculate_cluster, TestChain100Setup) std::vector<Txid> zigzag_txids; for (auto p{0}; p < 50; ++p) { const auto txp = make_tx({COutPoint{Txid::FromUint256(GetRandHash()), 0}}, /*num_outputs=*/2); - pool.addUnchecked(entry.Fee(CENT).FromTx(txp)); + AddToMempool(pool, entry.Fee(CENT).FromTx(txp)); zigzag_txids.push_back(txp->GetHash()); } for (auto c{0}; c < 49; ++c) { const auto txc = make_tx({COutPoint{zigzag_txids[c], 1}, COutPoint{zigzag_txids[c+1], 0}}, /*num_outputs=*/1); - pool.addUnchecked(entry.Fee(CENT).FromTx(txc)); + AddToMempool(pool, entry.Fee(CENT).FromTx(txc)); zigzag_txids.push_back(txc->GetHash()); } const auto vec_iters_zigzag = pool.GetIterVec(convert_to_uint256_vec(zigzag_txids)); diff --git a/src/test/miniscript_tests.cpp b/src/test/miniscript_tests.cpp index ba3507d49d..077ef498b2 100644 --- a/src/test/miniscript_tests.cpp +++ b/src/test/miniscript_tests.cpp @@ -293,7 +293,7 @@ public: using Fragment = miniscript::Fragment; using NodeRef = miniscript::NodeRef<CPubKey>; -using miniscript::operator"" _mst; +using miniscript::operator""_mst; using Node = miniscript::Node<CPubKey>; /** Compute all challenges (pubkeys, hashes, timelocks) that occur in a given Miniscript. */ diff --git a/src/test/multisig_tests.cpp b/src/test/multisig_tests.cpp index 7a3e8e3a47..29a73d03d2 100644 --- a/src/test/multisig_tests.cpp +++ b/src/test/multisig_tests.cpp @@ -10,6 +10,7 @@ #include <script/sign.h> #include <script/signingprovider.h> #include <test/util/setup_common.h> +#include <test/util/transaction_utils.h> #include <tinyformat.h> #include <uint256.h> diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 4c72d03ab1..384b1d7cc9 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -1281,7 +1281,7 @@ public: void ReceiveMessage(const std::string& m_type, Span<const uint8_t> payload) { auto ret = ReceivePacket(); - BOOST_REQUIRE(ret.size() == payload.size() + 1 + CMessageHeader::COMMAND_SIZE); + BOOST_REQUIRE(ret.size() == payload.size() + 1 + CMessageHeader::MESSAGE_TYPE_SIZE); BOOST_CHECK(ret[0] == 0); for (unsigned i = 0; i < 12; ++i) { if (i < m_type.size()) { @@ -1290,7 +1290,7 @@ public: BOOST_CHECK(ret[1 + i] == 0); } } - BOOST_CHECK(std::ranges::equal(Span{ret}.subspan(1 + CMessageHeader::COMMAND_SIZE), payload)); + BOOST_CHECK(std::ranges::equal(Span{ret}.subspan(1 + CMessageHeader::MESSAGE_TYPE_SIZE), payload)); } /** Schedule an encrypted packet with specified message type and payload to be sent to @@ -1298,9 +1298,9 @@ public: void SendMessage(std::string mtype, Span<const uint8_t> payload) { // Construct contents consisting of 0x00 + 12-byte message type + payload. - std::vector<uint8_t> contents(1 + CMessageHeader::COMMAND_SIZE + payload.size()); + std::vector<uint8_t> contents(1 + CMessageHeader::MESSAGE_TYPE_SIZE + payload.size()); std::copy(mtype.begin(), mtype.end(), reinterpret_cast<char*>(contents.data() + 1)); - std::copy(payload.begin(), payload.end(), contents.begin() + 1 + CMessageHeader::COMMAND_SIZE); + std::copy(payload.begin(), payload.end(), contents.begin() + 1 + CMessageHeader::MESSAGE_TYPE_SIZE); // Send a packet with that as contents. SendPacket(contents); } @@ -1459,7 +1459,7 @@ BOOST_AUTO_TEST_CASE(v2transport_test) auto msg_data_2 = m_rng.randbytes<uint8_t>(m_rng.randrange(1000)); tester.SendMessage(uint8_t(13), msg_data_2); // headers short id // Send invalidly-encoded message - tester.SendMessage(std::string("blocktxn\x00\x00\x00a", CMessageHeader::COMMAND_SIZE), {}); + tester.SendMessage(std::string("blocktxn\x00\x00\x00a", CMessageHeader::MESSAGE_TYPE_SIZE), {}); tester.SendMessage("foobar", {}); // test receiving unknown message type tester.AddMessage("barfoo", {}); // test sending unknown message type ret = tester.Interact(); diff --git a/src/test/peerman_tests.cpp b/src/test/peerman_tests.cpp index 6de373eef2..64b13fa3cc 100644 --- a/src/test/peerman_tests.cpp +++ b/src/test/peerman_tests.cpp @@ -20,8 +20,7 @@ static void mineBlock(const node::NodeContext& node, std::chrono::seconds block_ { auto curr_time = GetTime<std::chrono::seconds>(); SetMockTime(block_time); // update time so the block is created with it - node::BlockAssembler::Options options; - CBlock block = node::BlockAssembler{node.chainman->ActiveChainstate(), nullptr, options}.CreateNewBlock(CScript() << OP_TRUE)->block; + CBlock block = node::BlockAssembler{node.chainman->ActiveChainstate(), nullptr, {}}.CreateNewBlock()->block; while (!CheckProofOfWork(block.GetHash(), block.nBits, node.chainman->GetConsensus())) ++block.nNonce; block.fChecked = true; // little speedup SetMockTime(curr_time); // process block at current time diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 83977c1c89..f767cd4287 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -63,9 +63,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) tx.vin[0].prevout.n = 10000*blocknum+100*j+k; // make transaction unique { LOCK2(cs_main, mpool.cs); - mpool.addUnchecked(entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); + AddToMempool(mpool, entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); // Since TransactionAddedToMempool callbacks are generated in ATMP, - // not addUnchecked, we cheat and create one manually here + // not AddToMempool, we cheat and create one manually here const int64_t virtual_size = GetVirtualTransactionSize(*MakeTransactionRef(tx)); const NewMempoolTransactionInfo tx_info{NewMempoolTransactionInfo(MakeTransactionRef(tx), feeV[j], @@ -164,9 +164,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) tx.vin[0].prevout.n = 10000*blocknum+100*j+k; { LOCK2(cs_main, mpool.cs); - mpool.addUnchecked(entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); + AddToMempool(mpool, entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); // Since TransactionAddedToMempool callbacks are generated in ATMP, - // not addUnchecked, we cheat and create one manually here + // not AddToMempool, we cheat and create one manually here const int64_t virtual_size = GetVirtualTransactionSize(*MakeTransactionRef(tx)); const NewMempoolTransactionInfo tx_info{NewMempoolTransactionInfo(MakeTransactionRef(tx), feeV[j], @@ -228,9 +228,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) tx.vin[0].prevout.n = 10000*blocknum+100*j+k; { LOCK2(cs_main, mpool.cs); - mpool.addUnchecked(entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); + AddToMempool(mpool, entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); // Since TransactionAddedToMempool callbacks are generated in ATMP, - // not addUnchecked, we cheat and create one manually here + // not AddToMempool, we cheat and create one manually here const int64_t virtual_size = GetVirtualTransactionSize(*MakeTransactionRef(tx)); const NewMempoolTransactionInfo tx_info{NewMempoolTransactionInfo(MakeTransactionRef(tx), feeV[j], diff --git a/src/test/rbf_tests.cpp b/src/test/rbf_tests.cpp index 19e45c550a..c6d06c937c 100644 --- a/src/test/rbf_tests.cpp +++ b/src/test/rbf_tests.cpp @@ -78,7 +78,7 @@ static CTransactionRef add_descendants(const CTransactionRef& tx, int32_t num_de auto tx_to_spend = tx; for (int32_t i{0}; i < num_descendants; ++i) { auto next_tx = make_tx(/*inputs=*/{tx_to_spend}, /*output_values=*/{(50 - i) * CENT}); - pool.addUnchecked(entry.FromTx(next_tx)); + AddToMempool(pool, entry.FromTx(next_tx)); tx_to_spend = next_tx; } // Return last created tx @@ -93,7 +93,7 @@ static CTransactionRef add_descendant_to_parents(const std::vector<CTransactionR TestMemPoolEntryHelper entry; // Assumes this isn't already spent in mempool auto child_tx = make_tx(/*inputs=*/parents, /*output_values=*/{50 * CENT}); - pool.addUnchecked(entry.FromTx(child_tx)); + AddToMempool(pool, entry.FromTx(child_tx)); // Return last created tx return child_tx; } @@ -107,8 +107,8 @@ static std::pair<CTransactionRef, CTransactionRef> add_children_to_parent(const TestMemPoolEntryHelper entry; // Assumes this isn't already spent in mempool auto children_tx = make_two_siblings(/*parent=*/parent, /*output_values=*/{50 * CENT}); - pool.addUnchecked(entry.FromTx(children_tx.first)); - pool.addUnchecked(entry.FromTx(children_tx.second)); + AddToMempool(pool, entry.FromTx(children_tx.first)); + AddToMempool(pool, entry.FromTx(children_tx.second)); return children_tx; } @@ -124,46 +124,46 @@ BOOST_FIXTURE_TEST_CASE(rbf_helper_functions, TestChain100Setup) // Create a parent tx1 and child tx2 with normal fees: const auto tx1 = make_tx(/*inputs=*/ {m_coinbase_txns[0]}, /*output_values=*/ {10 * COIN}); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx1)); + AddToMempool(pool, entry.Fee(normal_fee).FromTx(tx1)); const auto tx2 = make_tx(/*inputs=*/ {tx1}, /*output_values=*/ {995 * CENT}); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx2)); + AddToMempool(pool, entry.Fee(normal_fee).FromTx(tx2)); // Create a low-feerate parent tx3 and high-feerate child tx4 (cpfp) const auto tx3 = make_tx(/*inputs=*/ {m_coinbase_txns[1]}, /*output_values=*/ {1099 * CENT}); - pool.addUnchecked(entry.Fee(low_fee).FromTx(tx3)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(tx3)); const auto tx4 = make_tx(/*inputs=*/ {tx3}, /*output_values=*/ {999 * CENT}); - pool.addUnchecked(entry.Fee(high_fee).FromTx(tx4)); + AddToMempool(pool, entry.Fee(high_fee).FromTx(tx4)); // Create a parent tx5 and child tx6 where both have very low fees const auto tx5 = make_tx(/*inputs=*/ {m_coinbase_txns[2]}, /*output_values=*/ {1099 * CENT}); - pool.addUnchecked(entry.Fee(low_fee).FromTx(tx5)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(tx5)); const auto tx6 = make_tx(/*inputs=*/ {tx5}, /*output_values=*/ {1098 * CENT}); - pool.addUnchecked(entry.Fee(low_fee).FromTx(tx6)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(tx6)); // Make tx6's modified fee much higher than its base fee. This should cause it to pass // the fee-related checks despite being low-feerate. pool.PrioritiseTransaction(tx6->GetHash(), 1 * COIN); // Two independent high-feerate transactions, tx7 and tx8 const auto tx7 = make_tx(/*inputs=*/ {m_coinbase_txns[3]}, /*output_values=*/ {999 * CENT}); - pool.addUnchecked(entry.Fee(high_fee).FromTx(tx7)); + AddToMempool(pool, entry.Fee(high_fee).FromTx(tx7)); const auto tx8 = make_tx(/*inputs=*/ {m_coinbase_txns[4]}, /*output_values=*/ {999 * CENT}); - pool.addUnchecked(entry.Fee(high_fee).FromTx(tx8)); + AddToMempool(pool, entry.Fee(high_fee).FromTx(tx8)); // Normal txs, will chain txns right before CheckConflictTopology test const auto tx9 = make_tx(/*inputs=*/ {m_coinbase_txns[5]}, /*output_values=*/ {995 * CENT}); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx9)); + AddToMempool(pool, entry.Fee(normal_fee).FromTx(tx9)); const auto tx10 = make_tx(/*inputs=*/ {m_coinbase_txns[6]}, /*output_values=*/ {995 * CENT}); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx10)); + AddToMempool(pool, entry.Fee(normal_fee).FromTx(tx10)); // Will make these two parents of single child const auto tx11 = make_tx(/*inputs=*/ {m_coinbase_txns[7]}, /*output_values=*/ {995 * CENT}); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx11)); + AddToMempool(pool, entry.Fee(normal_fee).FromTx(tx11)); const auto tx12 = make_tx(/*inputs=*/ {m_coinbase_txns[8]}, /*output_values=*/ {995 * CENT}); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx12)); + AddToMempool(pool, entry.Fee(normal_fee).FromTx(tx12)); // Will make two children of this single parent const auto tx13 = make_tx(/*inputs=*/ {m_coinbase_txns[9]}, /*output_values=*/ {995 * CENT, 995 * CENT}); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx13)); + AddToMempool(pool, entry.Fee(normal_fee).FromTx(tx13)); const auto entry1_normal = pool.GetIter(tx1->GetHash()).value(); const auto entry2_normal = pool.GetIter(tx2->GetHash()).value(); @@ -362,48 +362,86 @@ BOOST_FIXTURE_TEST_CASE(improves_feerate, TestChain100Setup) const CAmount normal_fee{CENT/10}; // low feerate parent with normal feerate child - const auto tx1 = make_tx(/*inputs=*/ {m_coinbase_txns[0]}, /*output_values=*/ {10 * COIN}); - pool.addUnchecked(entry.Fee(low_fee).FromTx(tx1)); + const auto tx1 = make_tx(/*inputs=*/ {m_coinbase_txns[0], m_coinbase_txns[1]}, /*output_values=*/ {10 * COIN}); + AddToMempool(pool, entry.Fee(low_fee).FromTx(tx1)); const auto tx2 = make_tx(/*inputs=*/ {tx1}, /*output_values=*/ {995 * CENT}); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx2)); + AddToMempool(pool, entry.Fee(normal_fee).FromTx(tx2)); const auto entry1 = pool.GetIter(tx1->GetHash()).value(); const auto tx1_fee = entry1->GetModifiedFee(); - const auto tx1_size = entry1->GetTxSize(); const auto entry2 = pool.GetIter(tx2->GetHash()).value(); const auto tx2_fee = entry2->GetModifiedFee(); - const auto tx2_size = entry2->GetTxSize(); + + // conflicting transactions + const auto tx1_conflict = make_tx(/*inputs=*/ {m_coinbase_txns[0], m_coinbase_txns[2]}, /*output_values=*/ {10 * COIN}); + const auto tx3 = make_tx(/*inputs=*/ {tx1_conflict}, /*output_values=*/ {995 * CENT}); + auto entry3 = entry.FromTx(tx3); // Now test ImprovesFeerateDiagram with various levels of "package rbf" feerates // It doesn't improve itself - const auto res1 = ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee, tx1_size + tx2_size); + auto changeset = pool.GetChangeSet(); + changeset->StageRemoval(entry1); + changeset->StageRemoval(entry2); + changeset->StageAddition(tx1_conflict, tx1_fee, 0, 1, 0, false, 4, LockPoints()); + changeset->StageAddition(tx3, tx2_fee, 0, 1, 0, false, 4, LockPoints()); + const auto res1 = ImprovesFeerateDiagram(*changeset); BOOST_CHECK(res1.has_value()); BOOST_CHECK(res1.value().first == DiagramCheckError::FAILURE); BOOST_CHECK(res1.value().second == "insufficient feerate: does not improve feerate diagram"); // With one more satoshi it does - BOOST_CHECK(ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee + 1, tx1_size + tx2_size) == std::nullopt); - + changeset.reset(); + changeset = pool.GetChangeSet(); + changeset->StageRemoval(entry1); + changeset->StageRemoval(entry2); + changeset->StageAddition(tx1_conflict, tx1_fee+1, 0, 1, 0, false, 4, LockPoints()); + changeset->StageAddition(tx3, tx2_fee, 0, 1, 0, false, 4, LockPoints()); + BOOST_CHECK(ImprovesFeerateDiagram(*changeset) == std::nullopt); + + changeset.reset(); // With prioritisation of in-mempool conflicts, it affects the results of the comparison using the same args as just above pool.PrioritiseTransaction(entry1->GetSharedTx()->GetHash(), /*nFeeDelta=*/1); - const auto res2 = ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee + 1, tx1_size + tx2_size); + changeset = pool.GetChangeSet(); + changeset->StageRemoval(entry1); + changeset->StageRemoval(entry2); + changeset->StageAddition(tx1_conflict, tx1_fee+1, 0, 1, 0, false, 4, LockPoints()); + changeset->StageAddition(tx3, tx2_fee, 0, 1, 0, false, 4, LockPoints()); + const auto res2 = ImprovesFeerateDiagram(*changeset); BOOST_CHECK(res2.has_value()); BOOST_CHECK(res2.value().first == DiagramCheckError::FAILURE); BOOST_CHECK(res2.value().second == "insufficient feerate: does not improve feerate diagram"); + changeset.reset(); + pool.PrioritiseTransaction(entry1->GetSharedTx()->GetHash(), /*nFeeDelta=*/-1); - // With one less vB it does - BOOST_CHECK(ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee, tx1_size + tx2_size - 1) == std::nullopt); + // With fewer vbytes it does + CMutableTransaction tx4{entry3.GetTx()}; + tx4.vin[0].scriptWitness = CScriptWitness(); // Clear out the witness, to reduce size + auto entry4 = entry.FromTx(MakeTransactionRef(tx4)); + changeset = pool.GetChangeSet(); + changeset->StageRemoval(entry1); + changeset->StageRemoval(entry2); + changeset->StageAddition(tx1_conflict, tx1_fee, 0, 1, 0, false, 4, LockPoints()); + changeset->StageAddition(entry4.GetSharedTx(), tx2_fee, 0, 1, 0, false, 4, LockPoints()); + BOOST_CHECK(ImprovesFeerateDiagram(*changeset) == std::nullopt); + changeset.reset(); // Adding a grandchild makes the cluster size 3, which is uncalculable - const auto tx3 = make_tx(/*inputs=*/ {tx2}, /*output_values=*/ {995 * CENT}); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx3)); - const auto res3 = ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee + 1, tx1_size + tx2_size); + const auto tx5 = make_tx(/*inputs=*/ {tx2}, /*output_values=*/ {995 * CENT}); + AddToMempool(pool, entry.Fee(normal_fee).FromTx(tx5)); + const auto entry5 = pool.GetIter(tx5->GetHash()).value(); + + changeset = pool.GetChangeSet(); + changeset->StageRemoval(entry1); + changeset->StageRemoval(entry2); + changeset->StageRemoval(entry5); + changeset->StageAddition(tx1_conflict, tx1_fee, 0, 1, 0, false, 4, LockPoints()); + changeset->StageAddition(entry4.GetSharedTx(), tx2_fee + entry5->GetModifiedFee() + 1, 0, 1, 0, false, 4, LockPoints()); + const auto res3 = ImprovesFeerateDiagram(*changeset); BOOST_CHECK(res3.has_value()); BOOST_CHECK(res3.value().first == DiagramCheckError::UNCALCULABLE); - BOOST_CHECK(res3.value().second == strprintf("%s has 2 descendants, max 1 allowed", tx1->GetHash().GetHex())); - + BOOST_CHECK(res3.value().second == strprintf("%s has 2 ancestors, max 1 allowed", tx5->GetHash().GetHex())); } BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) @@ -419,24 +457,33 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) // low -> high -> medium fee transactions that would result in two chunks together since they // are all same size const auto low_tx = make_tx(/*inputs=*/ {m_coinbase_txns[0]}, /*output_values=*/ {10 * COIN}); - pool.addUnchecked(entry.Fee(low_fee).FromTx(low_tx)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(low_tx)); const auto entry_low = pool.GetIter(low_tx->GetHash()).value(); const auto low_size = entry_low->GetTxSize(); + const auto replacement_tx = make_tx(/*inputs=*/ {m_coinbase_txns[0]}, /*output_values=*/ {9 * COIN}); + auto entry_replacement = entry.FromTx(replacement_tx); + // Replacement of size 1 { - const auto replace_one{pool.CalculateChunksForRBF(/*replacement_fees=*/0, /*replacement_vsize=*/1, {entry_low}, {entry_low})}; + auto changeset = pool.GetChangeSet(); + changeset->StageRemoval(entry_low); + changeset->StageAddition(replacement_tx, 0, 0, 1, 0, false, 4, LockPoints()); + const auto replace_one{changeset->CalculateChunksForRBF()}; BOOST_CHECK(replace_one.has_value()); std::vector<FeeFrac> expected_old_chunks{{low_fee, low_size}}; BOOST_CHECK(replace_one->first == expected_old_chunks); - std::vector<FeeFrac> expected_new_chunks{{0, 1}}; + std::vector<FeeFrac> expected_new_chunks{{0, int32_t(entry_replacement.GetTxSize())}}; BOOST_CHECK(replace_one->second == expected_new_chunks); } // Non-zero replacement fee/size { - const auto replace_one_fee{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_low}, {entry_low})}; + auto changeset = pool.GetChangeSet(); + changeset->StageRemoval(entry_low); + changeset->StageAddition(replacement_tx, high_fee, 0, 1, 0, false, 4, LockPoints()); + const auto replace_one_fee{changeset->CalculateChunksForRBF()}; BOOST_CHECK(replace_one_fee.has_value()); std::vector<FeeFrac> expected_old_diagram{{low_fee, low_size}}; BOOST_CHECK(replace_one_fee->first == expected_old_diagram); @@ -446,12 +493,16 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) // Add a second transaction to the cluster that will make a single chunk, to be evicted in the RBF const auto high_tx = make_tx(/*inputs=*/ {low_tx}, /*output_values=*/ {995 * CENT}); - pool.addUnchecked(entry.Fee(high_fee).FromTx(high_tx)); + AddToMempool(pool, entry.Fee(high_fee).FromTx(high_tx)); const auto entry_high = pool.GetIter(high_tx->GetHash()).value(); const auto high_size = entry_high->GetTxSize(); { - const auto replace_single_chunk{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_low}, {entry_low, entry_high})}; + auto changeset = pool.GetChangeSet(); + changeset->StageRemoval(entry_low); + changeset->StageRemoval(entry_high); + changeset->StageAddition(replacement_tx, high_fee, 0, 1, 0, false, 4, LockPoints()); + const auto replace_single_chunk{changeset->CalculateChunksForRBF()}; BOOST_CHECK(replace_single_chunk.has_value()); std::vector<FeeFrac> expected_old_chunks{{low_fee + high_fee, low_size + high_size}}; BOOST_CHECK(replace_single_chunk->first == expected_old_chunks); @@ -461,7 +512,10 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) // Conflict with the 2nd tx, resulting in new diagram with three entries { - const auto replace_cpfp_child{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_high}, {entry_high})}; + auto changeset = pool.GetChangeSet(); + changeset->StageRemoval(entry_high); + changeset->StageAddition(replacement_tx, high_fee, 0, 1, 0, false, 4, LockPoints()); + const auto replace_cpfp_child{changeset->CalculateChunksForRBF()}; BOOST_CHECK(replace_cpfp_child.has_value()); std::vector<FeeFrac> expected_old_chunks{{low_fee + high_fee, low_size + high_size}}; BOOST_CHECK(replace_cpfp_child->first == expected_old_chunks); @@ -471,29 +525,37 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) // third transaction causes the topology check to fail const auto normal_tx = make_tx(/*inputs=*/ {high_tx}, /*output_values=*/ {995 * CENT}); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(normal_tx)); + AddToMempool(pool, entry.Fee(normal_fee).FromTx(normal_tx)); const auto entry_normal = pool.GetIter(normal_tx->GetHash()).value(); - const auto normal_size = entry_normal->GetTxSize(); { - const auto replace_too_large{pool.CalculateChunksForRBF(/*replacement_fees=*/normal_fee, /*replacement_vsize=*/normal_size, {entry_low}, {entry_low, entry_high, entry_normal})}; + auto changeset = pool.GetChangeSet(); + changeset->StageRemoval(entry_low); + changeset->StageRemoval(entry_high); + changeset->StageRemoval(entry_normal); + changeset->StageAddition(replacement_tx, high_fee, 0, 1, 0, false, 4, LockPoints()); + const auto replace_too_large{changeset->CalculateChunksForRBF()}; BOOST_CHECK(!replace_too_large.has_value()); - BOOST_CHECK_EQUAL(util::ErrorString(replace_too_large).original, strprintf("%s has 2 descendants, max 1 allowed", low_tx->GetHash().GetHex())); + BOOST_CHECK_EQUAL(util::ErrorString(replace_too_large).original, strprintf("%s has 2 ancestors, max 1 allowed", normal_tx->GetHash().GetHex())); } // Make a size 2 cluster that is itself two chunks; evict both txns const auto high_tx_2 = make_tx(/*inputs=*/ {m_coinbase_txns[1]}, /*output_values=*/ {10 * COIN}); - pool.addUnchecked(entry.Fee(high_fee).FromTx(high_tx_2)); + AddToMempool(pool, entry.Fee(high_fee).FromTx(high_tx_2)); const auto entry_high_2 = pool.GetIter(high_tx_2->GetHash()).value(); const auto high_size_2 = entry_high_2->GetTxSize(); const auto low_tx_2 = make_tx(/*inputs=*/ {high_tx_2}, /*output_values=*/ {9 * COIN}); - pool.addUnchecked(entry.Fee(low_fee).FromTx(low_tx_2)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(low_tx_2)); const auto entry_low_2 = pool.GetIter(low_tx_2->GetHash()).value(); const auto low_size_2 = entry_low_2->GetTxSize(); { - const auto replace_two_chunks_single_cluster{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_high_2}, {entry_high_2, entry_low_2})}; + auto changeset = pool.GetChangeSet(); + changeset->StageRemoval(entry_high_2); + changeset->StageRemoval(entry_low_2); + changeset->StageAddition(replacement_tx, high_fee, 0, 1, 0, false, 4, LockPoints()); + const auto replace_two_chunks_single_cluster{changeset->CalculateChunksForRBF()}; BOOST_CHECK(replace_two_chunks_single_cluster.has_value()); std::vector<FeeFrac> expected_old_chunks{{high_fee, high_size_2}, {low_fee, low_size_2}}; BOOST_CHECK(replace_two_chunks_single_cluster->first == expected_old_chunks); @@ -503,19 +565,24 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) // You can have more than two direct conflicts if the there are multiple affected clusters, all of size 2 or less const auto conflict_1 = make_tx(/*inputs=*/ {m_coinbase_txns[2]}, /*output_values=*/ {10 * COIN}); - pool.addUnchecked(entry.Fee(low_fee).FromTx(conflict_1)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(conflict_1)); const auto conflict_1_entry = pool.GetIter(conflict_1->GetHash()).value(); const auto conflict_2 = make_tx(/*inputs=*/ {m_coinbase_txns[3]}, /*output_values=*/ {10 * COIN}); - pool.addUnchecked(entry.Fee(low_fee).FromTx(conflict_2)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(conflict_2)); const auto conflict_2_entry = pool.GetIter(conflict_2->GetHash()).value(); const auto conflict_3 = make_tx(/*inputs=*/ {m_coinbase_txns[4]}, /*output_values=*/ {10 * COIN}); - pool.addUnchecked(entry.Fee(low_fee).FromTx(conflict_3)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(conflict_3)); const auto conflict_3_entry = pool.GetIter(conflict_3->GetHash()).value(); { - const auto replace_multiple_clusters{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry})}; + auto changeset = pool.GetChangeSet(); + changeset->StageRemoval(conflict_1_entry); + changeset->StageRemoval(conflict_2_entry); + changeset->StageRemoval(conflict_3_entry); + changeset->StageAddition(replacement_tx, high_fee, 0, 1, 0, false, 4, LockPoints()); + const auto replace_multiple_clusters{changeset->CalculateChunksForRBF()}; BOOST_CHECK(replace_multiple_clusters.has_value()); BOOST_CHECK(replace_multiple_clusters->first.size() == 3); BOOST_CHECK(replace_multiple_clusters->second.size() == 1); @@ -523,11 +590,17 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) // Add a child transaction to conflict_1 and make it cluster size 2, two chunks due to same feerate const auto conflict_1_child = make_tx(/*inputs=*/{conflict_1}, /*output_values=*/ {995 * CENT}); - pool.addUnchecked(entry.Fee(low_fee).FromTx(conflict_1_child)); + AddToMempool(pool, entry.Fee(low_fee).FromTx(conflict_1_child)); const auto conflict_1_child_entry = pool.GetIter(conflict_1_child->GetHash()).value(); { - const auto replace_multiple_clusters_2{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry, conflict_1_child_entry})}; + auto changeset = pool.GetChangeSet(); + changeset->StageRemoval(conflict_1_entry); + changeset->StageRemoval(conflict_2_entry); + changeset->StageRemoval(conflict_3_entry); + changeset->StageRemoval(conflict_1_child_entry); + changeset->StageAddition(replacement_tx, high_fee, 0, 1, 0, false, 4, LockPoints()); + const auto replace_multiple_clusters_2{changeset->CalculateChunksForRBF()}; BOOST_CHECK(replace_multiple_clusters_2.has_value()); BOOST_CHECK(replace_multiple_clusters_2->first.size() == 4); @@ -536,14 +609,21 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) // Add another descendant to conflict_1, making the cluster size > 2 should fail at this point. const auto conflict_1_grand_child = make_tx(/*inputs=*/{conflict_1_child}, /*output_values=*/ {995 * CENT}); - pool.addUnchecked(entry.Fee(high_fee).FromTx(conflict_1_grand_child)); + AddToMempool(pool, entry.Fee(high_fee).FromTx(conflict_1_grand_child)); const auto conflict_1_grand_child_entry = pool.GetIter(conflict_1_child->GetHash()).value(); { - const auto replace_cluster_size_3{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry, conflict_1_child_entry, conflict_1_grand_child_entry})}; + auto changeset = pool.GetChangeSet(); + changeset->StageRemoval(conflict_1_entry); + changeset->StageRemoval(conflict_2_entry); + changeset->StageRemoval(conflict_3_entry); + changeset->StageRemoval(conflict_1_child_entry); + changeset->StageRemoval(conflict_1_grand_child_entry); + changeset->StageAddition(replacement_tx, high_fee, 0, 1, 0, false, 4, LockPoints()); + const auto replace_cluster_size_3{changeset->CalculateChunksForRBF()}; BOOST_CHECK(!replace_cluster_size_3.has_value()); - BOOST_CHECK_EQUAL(util::ErrorString(replace_cluster_size_3).original, strprintf("%s has 2 descendants, max 1 allowed", conflict_1->GetHash().GetHex())); + BOOST_CHECK_EQUAL(util::ErrorString(replace_cluster_size_3).original, strprintf("%s has both ancestor and descendant, exceeding cluster limit of 2", conflict_1_child->GetHash().GetHex())); } } diff --git a/src/test/result_tests.cpp b/src/test/result_tests.cpp index 6a23a7b895..4284864422 100644 --- a/src/test/result_tests.cpp +++ b/src/test/result_tests.cpp @@ -42,7 +42,7 @@ util::Result<int> IntFn(int i, bool success) util::Result<bilingual_str> StrFn(bilingual_str s, bool success) { if (success) return s; - return util::Error{strprintf(Untranslated("str %s error."), s.original)}; + return util::Error{Untranslated(strprintf("str %s error.", s.original))}; } util::Result<NoCopy> NoCopyFn(int i, bool success) @@ -63,7 +63,8 @@ void ExpectSuccess(const util::Result<T>& result, const bilingual_str& str, Args { ExpectResult(result, true, str); BOOST_CHECK_EQUAL(result.has_value(), true); - BOOST_CHECK_EQUAL(result.value(), T{std::forward<Args>(args)...}); + T expected{std::forward<Args>(args)...}; + BOOST_CHECK_EQUAL(result.value(), expected); BOOST_CHECK_EQUAL(&result.value(), &*result); } diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 5089f3e8e3..87303d7417 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -428,6 +428,7 @@ BOOST_AUTO_TEST_CASE(rpc_getblockstats_calculate_percentiles_by_weight) { int64_t total_weight = 200; std::vector<std::pair<CAmount, int64_t>> feerates; + feerates.reserve(200); CAmount result[NUM_GETBLOCKSTATS_PERCENTILES] = { 0 }; for (int64_t i = 0; i < 100; i++) { diff --git a/src/test/script_p2sh_tests.cpp b/src/test/script_p2sh_tests.cpp index f91203cc48..bb408b7b0f 100644 --- a/src/test/script_p2sh_tests.cpp +++ b/src/test/script_p2sh_tests.cpp @@ -11,6 +11,7 @@ #include <script/sign.h> #include <script/signingprovider.h> #include <test/util/setup_common.h> +#include <test/util/transaction_utils.h> #include <validation.h> #include <vector> @@ -120,7 +121,7 @@ BOOST_AUTO_TEST_CASE(sign) { CScript sigSave = txTo[i].vin[0].scriptSig; txTo[i].vin[0].scriptSig = txTo[j].vin[0].scriptSig; - bool sigOK = CScriptCheck(txFrom.vout[txTo[i].vin[0].prevout.n], CTransaction(txTo[i]), signature_cache, 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)(); + bool sigOK = !CScriptCheck(txFrom.vout[txTo[i].vin[0].prevout.n], CTransaction(txTo[i]), signature_cache, 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)().has_value(); if (i == j) BOOST_CHECK_MESSAGE(sigOK, strprintf("VerifySignature %d %d", i, j)); else diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 0e2a1631ce..8753ddee37 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -123,7 +123,6 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript ScriptError err; const CTransaction txCredit{BuildCreditingTransaction(scriptPubKey, nValue)}; CMutableTransaction tx = BuildSpendingTransaction(scriptSig, scriptWitness, txCredit); - CMutableTransaction tx2 = tx; BOOST_CHECK_MESSAGE(VerifyScript(scriptSig, scriptPubKey, &scriptWitness, flags, MutableTransactionSignatureChecker(&tx, 0, txCredit.vout[0].nValue, MissingDataBehavior::ASSERT_FAIL), &err) == expect, message); BOOST_CHECK_MESSAGE(err == scriptError, FormatScriptError(err) + " where " + FormatScriptError((ScriptError_t)scriptError) + " expected: " + message); @@ -1368,6 +1367,13 @@ static CScript ScriptFromHex(const std::string& str) return ToScript(*Assert(TryParseHex(str))); } +BOOST_AUTO_TEST_CASE(script_byte_array_u8_vector_equivalence) +{ + const CScript scriptPubKey1 = CScript() << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f"_hex_v_u8 << OP_CHECKSIG; + const CScript scriptPubKey2 = CScript() << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f"_hex << OP_CHECKSIG; + BOOST_CHECK(scriptPubKey1 == scriptPubKey2); +} + BOOST_AUTO_TEST_CASE(script_FindAndDelete) { // Exercise the FindAndDelete functionality @@ -1421,7 +1427,7 @@ BOOST_AUTO_TEST_CASE(script_FindAndDelete) // prefix, leaving 02ff03 which is push-two-bytes: s = ToScript("0302ff030302ff03"_hex); d = ToScript("03"_hex); - expect = CScript() << "ff03"_hex_v_u8 << "ff03"_hex_v_u8; + expect = CScript() << "ff03"_hex << "ff03"_hex; BOOST_CHECK_EQUAL(FindAndDelete(s, d), 2); BOOST_CHECK(s == expect); @@ -1694,9 +1700,8 @@ BOOST_AUTO_TEST_CASE(bip341_keypath_test_vectors) BOOST_CHECK_EQUAL(HexStr(sighash), input["intermediary"]["sigHash"].get_str()); // To verify the sigmsg, hash the expected sigmsg, and compare it with the (expected) sighash. - BOOST_CHECK_EQUAL(HexStr((HashWriter{HASHER_TAPSIGHASH} << Span{ParseHex(input["intermediary"]["sigMsg"].get_str())}).GetSHA256()), input["intermediary"]["sigHash"].get_str()); + BOOST_CHECK_EQUAL(HexStr((HashWriter{HASHER_TAPSIGHASH} << std::span<const uint8_t>{ParseHex(input["intermediary"]["sigMsg"].get_str())}).GetSHA256()), input["intermediary"]["sigHash"].get_str()); } - } } diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index b28e1b4196..7e403694e2 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -304,7 +304,7 @@ public: if (s.template GetParams<BaseFormat>().m_base_format == BaseFormat::RAW) { s << m_base_data; } else { - s << Span{HexStr(Span{&m_base_data, 1})}; + s << std::span<const char>{HexStr(Span{&m_base_data, 1})}; } } diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 9217f05945..777122df6d 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -261,7 +261,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) for (uint8_t j = 0; j < 40; ++j) { file << j; } - std::rewind(file.Get()); + file.seek(0, SEEK_SET); // The buffer size (second arg) must be greater than the rewind // amount (third arg). @@ -391,7 +391,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_skip) for (uint8_t j = 0; j < 40; ++j) { file << j; } - std::rewind(file.Get()); + file.seek(0, SEEK_SET); // The buffer is 25 bytes, allow rewinding 10 bytes. BufferedFile bf{file, 25, 10}; @@ -444,7 +444,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) for (uint8_t i = 0; i < fileSize; ++i) { file << i; } - std::rewind(file.Get()); + file.seek(0, SEEK_SET); size_t bufSize = m_rng.randrange(300) + 1; size_t rewindSize = m_rng.randrange(bufSize); diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index bf50a61327..a5d9be07d5 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. // -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <test/util/setup_common.h> #include <common/run_command.h> #include <univalue.h> @@ -47,7 +47,7 @@ BOOST_AUTO_TEST_CASE(run_command) } { // Return non-zero exit code, with error message for stderr - const std::string command{"python3 -c 'import sys; print(\"err\", file=sys.stderr); sys.exit(2)'"}; + const std::string command{"sh -c 'echo err 1>&2 && false'"}; const std::string expected{"err"}; BOOST_CHECK_EXCEPTION(RunCommandParseJSON(command), std::runtime_error, [&](const std::runtime_error& e) { const std::string what(e.what()); diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 462abd5222..8beeec4991 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -588,7 +588,7 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) control.Add(std::move(vChecks)); } - bool controlCheck = control.Wait(); + bool controlCheck = !control.Complete().has_value(); assert(controlCheck); } @@ -813,6 +813,13 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) // Check dust with default relay fee: CAmount nDustThreshold = 182 * g_dust.GetFeePerK() / 1000; BOOST_CHECK_EQUAL(nDustThreshold, 546); + + // Add dust outputs up to allowed maximum, still standard! + for (size_t i{0}; i < MAX_DUST_OUTPUTS_PER_TX; ++i) { + t.vout.emplace_back(0, t.vout[0].scriptPubKey); + CheckIsStandard(t); + } + // dust: t.vout[0].nValue = nDustThreshold - 1; CheckIsNotStandard(t, "dust"); @@ -852,24 +859,24 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) CheckIsNotStandard(t, "scriptpubkey"); // MAX_OP_RETURN_RELAY-byte TxoutType::NULL_DATA (standard) - t.vout[0].scriptPubKey = CScript() << OP_RETURN << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"_hex_v_u8; + t.vout[0].scriptPubKey = CScript() << OP_RETURN << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"_hex; BOOST_CHECK_EQUAL(MAX_OP_RETURN_RELAY, t.vout[0].scriptPubKey.size()); CheckIsStandard(t); // MAX_OP_RETURN_RELAY+1-byte TxoutType::NULL_DATA (non-standard) - t.vout[0].scriptPubKey = CScript() << OP_RETURN << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3800"_hex_v_u8; + t.vout[0].scriptPubKey = CScript() << OP_RETURN << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3800"_hex; BOOST_CHECK_EQUAL(MAX_OP_RETURN_RELAY + 1, t.vout[0].scriptPubKey.size()); CheckIsNotStandard(t, "scriptpubkey"); // Data payload can be encoded in any way... - t.vout[0].scriptPubKey = CScript() << OP_RETURN << ""_hex_v_u8; + t.vout[0].scriptPubKey = CScript() << OP_RETURN << ""_hex; CheckIsStandard(t); - t.vout[0].scriptPubKey = CScript() << OP_RETURN << "00"_hex_v_u8 << "01"_hex_v_u8; + t.vout[0].scriptPubKey = CScript() << OP_RETURN << "00"_hex << "01"_hex; CheckIsStandard(t); // OP_RESERVED *is* considered to be a PUSHDATA type opcode by IsPushOnly()! - t.vout[0].scriptPubKey = CScript() << OP_RETURN << OP_RESERVED << -1 << 0 << "01"_hex_v_u8 << 2 << 3 << 4 << 5 << 6 << 7 << 8 << 9 << 10 << 11 << 12 << 13 << 14 << 15 << 16; + t.vout[0].scriptPubKey = CScript() << OP_RETURN << OP_RESERVED << -1 << 0 << "01"_hex << 2 << 3 << 4 << 5 << 6 << 7 << 8 << 9 << 10 << 11 << 12 << 13 << 14 << 15 << 16; CheckIsStandard(t); - t.vout[0].scriptPubKey = CScript() << OP_RETURN << 0 << "01"_hex_v_u8 << 2 << "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"_hex_v_u8; + t.vout[0].scriptPubKey = CScript() << OP_RETURN << 0 << "01"_hex << 2 << "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"_hex; CheckIsStandard(t); // ...so long as it only contains PUSHDATA's @@ -883,13 +890,13 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) // Only one TxoutType::NULL_DATA permitted in all cases t.vout.resize(2); - t.vout[0].scriptPubKey = CScript() << OP_RETURN << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"_hex_v_u8; + t.vout[0].scriptPubKey = CScript() << OP_RETURN << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"_hex; t.vout[0].nValue = 0; - t.vout[1].scriptPubKey = CScript() << OP_RETURN << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"_hex_v_u8; + t.vout[1].scriptPubKey = CScript() << OP_RETURN << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"_hex; t.vout[1].nValue = 0; CheckIsNotStandard(t, "multi-op-return"); - t.vout[0].scriptPubKey = CScript() << OP_RETURN << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"_hex_v_u8; + t.vout[0].scriptPubKey = CScript() << OP_RETURN << "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"_hex; t.vout[1].scriptPubKey = CScript() << OP_RETURN; CheckIsNotStandard(t, "multi-op-return"); @@ -969,6 +976,10 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) CheckIsNotStandard(t, "bare-multisig"); g_bare_multi = DEFAULT_PERMIT_BAREMULTISIG; + // Add dust outputs up to allowed maximum + assert(t.vout.size() == 1); + t.vout.insert(t.vout.end(), MAX_DUST_OUTPUTS_PER_TX, {0, t.vout[0].scriptPubKey}); + // Check compressed P2PK outputs dust threshold (must have leading 02 or 03) t.vout[0].scriptPubKey = CScript() << std::vector<unsigned char>(33, 0x02) << OP_CHECKSIG; t.vout[0].nValue = 576; diff --git a/src/test/txdownload_tests.cpp b/src/test/txdownload_tests.cpp new file mode 100644 index 0000000000..6d277b5629 --- /dev/null +++ b/src/test/txdownload_tests.cpp @@ -0,0 +1,336 @@ +// Copyright (c) 2011-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <addresstype.h> +#include <consensus/validation.h> +#include <net_processing.h> +#include <node/txdownloadman_impl.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <test/util/random.h> +#include <test/util/setup_common.h> +#include <validation.h> + +#include <array> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(txdownload_tests, TestingSetup) + +struct Behaviors { + bool m_txid_in_rejects; + bool m_wtxid_in_rejects; + bool m_txid_in_rejects_recon; + bool m_wtxid_in_rejects_recon; + bool m_keep_for_compact; + bool m_ignore_inv_txid; + bool m_ignore_inv_wtxid; + + // Constructor. We are passing and casting ints because they are more readable in a table (see expected_behaviors). + Behaviors(bool txid_rejects, bool wtxid_rejects, bool txid_recon, bool wtxid_recon, bool keep, bool txid_inv, bool wtxid_inv) : + m_txid_in_rejects(txid_rejects), + m_wtxid_in_rejects(wtxid_rejects), + m_txid_in_rejects_recon(txid_recon), + m_wtxid_in_rejects_recon(wtxid_recon), + m_keep_for_compact(keep), + m_ignore_inv_txid(txid_inv), + m_ignore_inv_wtxid(wtxid_inv) + {} + + void CheckEqual(const Behaviors& other, bool segwit) + { + BOOST_CHECK_EQUAL(other.m_wtxid_in_rejects, m_wtxid_in_rejects); + BOOST_CHECK_EQUAL(other.m_wtxid_in_rejects_recon, m_wtxid_in_rejects_recon); + BOOST_CHECK_EQUAL(other.m_keep_for_compact, m_keep_for_compact); + BOOST_CHECK_EQUAL(other.m_ignore_inv_wtxid, m_ignore_inv_wtxid); + + // false negatives for nonsegwit transactions, since txid == wtxid. + if (segwit) { + BOOST_CHECK_EQUAL(other.m_txid_in_rejects, m_txid_in_rejects); + BOOST_CHECK_EQUAL(other.m_txid_in_rejects_recon, m_txid_in_rejects_recon); + BOOST_CHECK_EQUAL(other.m_ignore_inv_txid, m_ignore_inv_txid); + } + } +}; + +// Map from failure reason to expected behavior for a segwit tx that fails +// Txid and Wtxid are assumed to be different here. For a nonsegwit transaction, use the wtxid results. +static std::map<TxValidationResult, Behaviors> expected_behaviors{ + {TxValidationResult::TX_CONSENSUS, {/*txid_rejects*/0,/*wtxid_rejects*/1,/*txid_recon*/0,/*wtxid_recon*/0,/*keep*/1,/*txid_inv*/0,/*wtxid_inv*/1}}, + {TxValidationResult::TX_INPUTS_NOT_STANDARD, { 1, 1, 0, 0, 1, 1, 1}}, + {TxValidationResult::TX_NOT_STANDARD, { 0, 1, 0, 0, 1, 0, 1}}, + {TxValidationResult::TX_MISSING_INPUTS, { 0, 0, 0, 0, 1, 0, 1}}, + {TxValidationResult::TX_PREMATURE_SPEND, { 0, 1, 0, 0, 1, 0, 1}}, + {TxValidationResult::TX_WITNESS_MUTATED, { 0, 1, 0, 0, 1, 0, 1}}, + {TxValidationResult::TX_WITNESS_STRIPPED, { 0, 0, 0, 0, 0, 0, 0}}, + {TxValidationResult::TX_CONFLICT, { 0, 1, 0, 0, 1, 0, 1}}, + {TxValidationResult::TX_MEMPOOL_POLICY, { 0, 1, 0, 0, 1, 0, 1}}, + {TxValidationResult::TX_NO_MEMPOOL, { 0, 1, 0, 0, 1, 0, 1}}, + {TxValidationResult::TX_RECONSIDERABLE, { 0, 0, 0, 1, 1, 0, 1}}, + {TxValidationResult::TX_UNKNOWN, { 0, 1, 0, 0, 1, 0, 1}}, +}; + +static bool CheckOrphanBehavior(node::TxDownloadManagerImpl& txdownload_impl, const CTransactionRef& tx, const node::RejectedTxTodo& ret, std::string& err_msg, + bool expect_orphan, bool expect_keep, unsigned int expected_parents) +{ + // Missing inputs can never result in a PackageToValidate. + if (ret.m_package_to_validate.has_value()) { + err_msg = strprintf("returned a PackageToValidate on missing inputs"); + return false; + } + + if (expect_orphan != txdownload_impl.m_orphanage.HaveTx(tx->GetWitnessHash())) { + err_msg = strprintf("unexpectedly %s tx in orphanage", expect_orphan ? "did not find" : "found"); + return false; + } + if (expect_keep != ret.m_should_add_extra_compact_tx) { + err_msg = strprintf("unexpectedly returned %s add to vExtraTxnForCompact", expect_keep ? "should not" : "should"); + return false; + } + if (expected_parents != ret.m_unique_parents.size()) { + err_msg = strprintf("expected %u unique_parents, got %u", expected_parents, ret.m_unique_parents.size()); + return false; + } + return true; +} + +static CTransactionRef CreatePlaceholderTx(bool segwit) +{ + // Each tx returned from here spends the previous one. + static Txid prevout_hash{}; + + CMutableTransaction mtx; + mtx.vin.emplace_back(prevout_hash, 0); + // This makes txid != wtxid + if (segwit) mtx.vin[0].scriptWitness.stack.push_back({1}); + mtx.vout.emplace_back(CENT, CScript()); + auto ptx = MakeTransactionRef(mtx); + prevout_hash = ptx->GetHash(); + return ptx; +} + +BOOST_FIXTURE_TEST_CASE(tx_rejection_types, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + FastRandomContext det_rand{true}; + node::TxDownloadOptions DEFAULT_OPTS{pool, det_rand, DEFAULT_MAX_ORPHAN_TRANSACTIONS, true}; + + // A new TxDownloadManagerImpl is created for each tx so we can just reuse the same one. + TxValidationState state; + NodeId nodeid{0}; + std::chrono::microseconds now{GetTime()}; + node::TxDownloadConnectionInfo connection_info{/*m_preferred=*/false, /*m_relay_permissions=*/false, /*m_wtxid_relay=*/true}; + + for (const auto segwit_parent : {true, false}) { + for (const auto segwit_child : {true, false}) { + const auto ptx_parent = CreatePlaceholderTx(segwit_parent); + const auto ptx_child = CreatePlaceholderTx(segwit_child); + const auto& parent_txid = ptx_parent->GetHash().ToUint256(); + const auto& parent_wtxid = ptx_parent->GetWitnessHash().ToUint256(); + const auto& child_txid = ptx_child->GetHash().ToUint256(); + const auto& child_wtxid = ptx_child->GetWitnessHash().ToUint256(); + + for (const auto& [result, expected_behavior] : expected_behaviors) { + node::TxDownloadManagerImpl txdownload_impl{DEFAULT_OPTS}; + txdownload_impl.ConnectedPeer(nodeid, connection_info); + // Parent failure + state.Invalid(result, ""); + const auto& [keep, unique_txids, package_to_validate] = txdownload_impl.MempoolRejectedTx(ptx_parent, state, nodeid, /*first_time_failure=*/true); + + // No distinction between txid and wtxid caching for nonsegwit transactions, so only test these specific + // behaviors for segwit transactions. + Behaviors actual_behavior{ + /*txid_rejects=*/txdownload_impl.RecentRejectsFilter().contains(parent_txid), + /*wtxid_rejects=*/txdownload_impl.RecentRejectsFilter().contains(parent_wtxid), + /*txid_recon=*/txdownload_impl.RecentRejectsReconsiderableFilter().contains(parent_txid), + /*wtxid_recon=*/txdownload_impl.RecentRejectsReconsiderableFilter().contains(parent_wtxid), + /*keep=*/keep, + /*txid_inv=*/txdownload_impl.AddTxAnnouncement(nodeid, GenTxid::Txid(parent_txid), now, /*p2p_inv=*/true), + /*wtxid_inv=*/txdownload_impl.AddTxAnnouncement(nodeid, GenTxid::Wtxid(parent_wtxid), now, /*p2p_inv=*/true), + }; + BOOST_TEST_MESSAGE("Testing behavior for " << result << (segwit_parent ? " segwit " : " nonsegwit")); + actual_behavior.CheckEqual(expected_behavior, /*segwit=*/segwit_parent); + + // Later, a child of this transaction fails for missing inputs + state.Invalid(TxValidationResult::TX_MISSING_INPUTS, ""); + txdownload_impl.MempoolRejectedTx(ptx_child, state, nodeid, /*first_time_failure=*/true); + + // If parent (by txid) was rejected, child is too. + const bool parent_txid_rejected{segwit_parent ? expected_behavior.m_txid_in_rejects : expected_behavior.m_wtxid_in_rejects}; + BOOST_CHECK_EQUAL(parent_txid_rejected, txdownload_impl.RecentRejectsFilter().contains(child_txid)); + BOOST_CHECK_EQUAL(parent_txid_rejected, txdownload_impl.RecentRejectsFilter().contains(child_wtxid)); + + // Unless rejected, the child should be in orphanage. + BOOST_CHECK_EQUAL(!parent_txid_rejected, txdownload_impl.m_orphanage.HaveTx(ptx_child->GetWitnessHash())); + } + } + } +} + +BOOST_FIXTURE_TEST_CASE(handle_missing_inputs, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + FastRandomContext det_rand{true}; + node::TxDownloadOptions DEFAULT_OPTS{pool, det_rand, DEFAULT_MAX_ORPHAN_TRANSACTIONS, true}; + NodeId nodeid{1}; + node::TxDownloadConnectionInfo DEFAULT_CONN{/*m_preferred=*/false, /*m_relay_permissions=*/false, /*m_wtxid_relay=*/true}; + + // We need mature coinbases + mineBlocks(20); + + // Transactions with missing inputs are treated differently depending on how much we know about + // their parents. + CKey wallet_key = GenerateRandomKey(); + CScript destination = GetScriptForDestination(PKHash(wallet_key.GetPubKey())); + // Amount for spending coinbase in a 1-in-1-out tx, at depth n, each time deducting 1000 from the amount as fees. + CAmount amount_depth_1{50 * COIN - 1000}; + CAmount amount_depth_2{amount_depth_1 - 1000}; + // Amount for spending coinbase in a 1-in-2-out tx, deducting 1000 in fees + CAmount amount_split_half{25 * COIN - 500}; + int test_chain_height{100}; + + TxValidationState state_orphan; + state_orphan.Invalid(TxValidationResult::TX_MISSING_INPUTS, ""); + + // Transactions are not all submitted to mempool. Conserve the number of m_coinbase_txns we + // consume, and only increment this index number when we would conflict with an existing + // mempool transaction. + size_t coinbase_idx{0}; + + for (int decisions = 0; decisions < (1 << 4); ++decisions) { + auto mtx_single_parent = CreateValidMempoolTransaction(m_coinbase_txns[coinbase_idx], /*input_vout=*/0, test_chain_height, coinbaseKey, destination, amount_depth_1, /*submit=*/false); + auto single_parent = MakeTransactionRef(mtx_single_parent); + + auto mtx_orphan = CreateValidMempoolTransaction(single_parent, /*input_vout=*/0, test_chain_height, wallet_key, destination, amount_depth_2, /*submit=*/false); + auto orphan = MakeTransactionRef(mtx_orphan); + + node::TxDownloadManagerImpl txdownload_impl{DEFAULT_OPTS}; + txdownload_impl.ConnectedPeer(nodeid, DEFAULT_CONN); + + // Each bit of decisions tells us whether the parent is in a particular cache. + // It is definitely possible for a transaction to be in multiple caches. For example, it + // may have both a low feerate and found to violate some mempool policy when validated + // in a 1p1c. + const bool parent_recent_rej(decisions & 1); + const bool parent_recent_rej_recon((decisions >> 1) & 1); + const bool parent_recent_conf((decisions >> 2) & 1); + const bool parent_in_mempool((decisions >> 3) & 1); + + if (parent_recent_rej) txdownload_impl.RecentRejectsFilter().insert(single_parent->GetHash().ToUint256()); + if (parent_recent_rej_recon) txdownload_impl.RecentRejectsReconsiderableFilter().insert(single_parent->GetHash().ToUint256()); + if (parent_recent_conf) txdownload_impl.RecentConfirmedTransactionsFilter().insert(single_parent->GetHash().ToUint256()); + if (parent_in_mempool) { + const auto mempool_result = WITH_LOCK(::cs_main, return m_node.chainman->ProcessTransaction(single_parent)); + BOOST_CHECK(mempool_result.m_result_type == MempoolAcceptResult::ResultType::VALID); + coinbase_idx += 1; + assert(coinbase_idx < m_coinbase_txns.size()); + } + + // Whether or not the transaction is added as an orphan depends solely on whether or not + // it's in RecentRejectsFilter. Specifically, the parent is allowed to be in + // RecentRejectsReconsiderableFilter, but it cannot be in RecentRejectsFilter. + const bool expect_keep_orphan = !parent_recent_rej; + const auto ret_1p1c = txdownload_impl.MempoolRejectedTx(orphan, state_orphan, nodeid, /*first_time_failure=*/true); + std::string err_msg; + const bool ok = CheckOrphanBehavior(txdownload_impl, orphan, ret_1p1c, err_msg, + /*expect_orphan=*/expect_keep_orphan, /*expect_keep=*/true, /*expected_parents=*/expect_keep_orphan ? 1 : 0); + BOOST_CHECK_MESSAGE(ok, err_msg); + } + + // Orphan with multiple parents + { + std::vector<CTransactionRef> parents; + std::vector<COutPoint> outpoints; + int32_t num_parents{24}; + for (int32_t i = 0; i < num_parents; ++i) { + assert(coinbase_idx < m_coinbase_txns.size()); + auto mtx_parent = CreateValidMempoolTransaction(m_coinbase_txns[coinbase_idx++], /*input_vout=*/0, test_chain_height, + coinbaseKey, destination, amount_depth_1 + i, /*submit=*/false); + auto ptx_parent = MakeTransactionRef(mtx_parent); + parents.emplace_back(ptx_parent); + outpoints.emplace_back(ptx_parent->GetHash(), 0); + } + + // Send all coins to 1 output. + auto mtx_orphan = CreateValidMempoolTransaction(parents, outpoints, test_chain_height, {wallet_key}, {{amount_depth_2 * num_parents, destination}}, /*submit=*/false); + auto orphan = MakeTransactionRef(mtx_orphan); + + // 1 parent in RecentRejectsReconsiderableFilter, the rest are unknown + { + node::TxDownloadManagerImpl txdownload_impl{DEFAULT_OPTS}; + txdownload_impl.ConnectedPeer(nodeid, DEFAULT_CONN); + + txdownload_impl.RecentRejectsReconsiderableFilter().insert(parents[0]->GetHash().ToUint256()); + const auto ret_1p1c_parent_reconsiderable = txdownload_impl.MempoolRejectedTx(orphan, state_orphan, nodeid, /*first_time_failure=*/true); + std::string err_msg; + const bool ok = CheckOrphanBehavior(txdownload_impl, orphan, ret_1p1c_parent_reconsiderable, err_msg, + /*expect_orphan=*/true, /*expect_keep=*/true, /*expected_parents=*/num_parents); + BOOST_CHECK_MESSAGE(ok, err_msg); + } + + // 1 parent in RecentRejectsReconsiderableFilter, the rest are confirmed + { + node::TxDownloadManagerImpl txdownload_impl{DEFAULT_OPTS}; + txdownload_impl.ConnectedPeer(nodeid, DEFAULT_CONN); + + txdownload_impl.RecentRejectsReconsiderableFilter().insert(parents[0]->GetHash().ToUint256()); + for (int32_t i = 1; i < num_parents; ++i) { + txdownload_impl.RecentConfirmedTransactionsFilter().insert(parents[i]->GetHash().ToUint256()); + } + + const auto ret_1recon_conf = txdownload_impl.MempoolRejectedTx(orphan, state_orphan, nodeid, /*first_time_failure=*/true); + std::string err_msg; + const bool ok = CheckOrphanBehavior(txdownload_impl, orphan, ret_1recon_conf, err_msg, + /*expect_orphan=*/true, /*expect_keep=*/true, /*expected_parents=*/num_parents); + BOOST_CHECK_MESSAGE(ok, err_msg); + } + + // 1 parent in RecentRejectsReconsiderableFilter, 1 other in {RecentRejectsReconsiderableFilter, RecentRejectsFilter} + for (int i = 0; i < 2; ++i) { + node::TxDownloadManagerImpl txdownload_impl{DEFAULT_OPTS}; + txdownload_impl.ConnectedPeer(nodeid, DEFAULT_CONN); + + txdownload_impl.RecentRejectsReconsiderableFilter().insert(parents[1]->GetHash().ToUint256()); + + // Doesn't really matter which parent + auto& alreadyhave_parent = parents[0]; + if (i == 0) { + txdownload_impl.RecentRejectsReconsiderableFilter().insert(alreadyhave_parent->GetHash().ToUint256()); + } else if (i == 1) { + txdownload_impl.RecentRejectsFilter().insert(alreadyhave_parent->GetHash().ToUint256()); + } + + const auto ret_2_problems = txdownload_impl.MempoolRejectedTx(orphan, state_orphan, nodeid, /*first_time_failure=*/true); + std::string err_msg; + const bool ok = CheckOrphanBehavior(txdownload_impl, orphan, ret_2_problems, err_msg, + /*expect_orphan=*/false, /*expect_keep=*/true, /*expected_parents=*/0); + BOOST_CHECK_MESSAGE(ok, err_msg); + } + } + + // Orphan with multiple inputs spending from a single parent + { + assert(coinbase_idx < m_coinbase_txns.size()); + auto parent_2outputs = MakeTransactionRef(CreateValidMempoolTransaction({m_coinbase_txns[coinbase_idx]}, {{m_coinbase_txns[coinbase_idx]->GetHash(), 0}}, test_chain_height, {coinbaseKey}, + {{amount_split_half, destination}, {amount_split_half, destination}}, /*submit=*/false)); + + auto orphan = MakeTransactionRef(CreateValidMempoolTransaction({parent_2outputs}, {{parent_2outputs->GetHash(), 0}, {parent_2outputs->GetHash(), 1}}, + test_chain_height, {wallet_key}, {{amount_depth_2, destination}}, /*submit=*/false)); + // Parent is in RecentRejectsReconsiderableFilter. Inputs will find it twice, but this + // should only counts as 1 parent in the filter. + { + node::TxDownloadManagerImpl txdownload_impl{DEFAULT_OPTS}; + txdownload_impl.ConnectedPeer(nodeid, DEFAULT_CONN); + + txdownload_impl.RecentRejectsReconsiderableFilter().insert(parent_2outputs->GetHash().ToUint256()); + const auto ret_1p1c_2reconsiderable = txdownload_impl.MempoolRejectedTx(orphan, state_orphan, nodeid, /*first_time_failure=*/true); + std::string err_msg; + const bool ok = CheckOrphanBehavior(txdownload_impl, orphan, ret_1p1c_2reconsiderable, err_msg, + /*expect_orphan=*/true, /*expect_keep=*/true, /*expected_parents=*/1); + BOOST_CHECK_MESSAGE(ok, err_msg); + } + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 5a32b02ad9..9ee5387830 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -33,7 +33,7 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) BOOST_REQUIRE(txindex.StartBackgroundSync()); // Allow tx index to catch up with the block index. - IndexWaitSynced(txindex, *Assert(m_node.shutdown)); + IndexWaitSynced(txindex, *Assert(m_node.shutdown_signal)); // Check that txindex excludes genesis block transactions. const CBlock& genesis_block = Params().GenesisBlock(); diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index ea211aedf3..3c36b78260 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -283,6 +283,8 @@ BOOST_AUTO_TEST_CASE(noncontextual_package_tests) BOOST_CHECK(GetPackageHash({tx_parent}) != GetPackageHash({tx_child})); BOOST_CHECK(GetPackageHash({tx_child, tx_child}) != GetPackageHash({tx_child})); BOOST_CHECK(GetPackageHash({tx_child, tx_parent}) != GetPackageHash({tx_child, tx_child})); + BOOST_CHECK(!IsChildWithParents({})); + BOOST_CHECK(!IsChildWithParentsTree({})); } // 24 Parents and 1 Child @@ -492,6 +494,97 @@ BOOST_AUTO_TEST_CASE(package_submission_tests) } } +// Tests for packages containing a single transaction +BOOST_AUTO_TEST_CASE(package_single_tx) +{ + // Mine blocks to mature coinbases. + mineBlocks(3); + LOCK(cs_main); + auto expected_pool_size{m_node.mempool->size()}; + + const CAmount high_fee{1000}; + + // No unconfirmed parents + CKey single_key = GenerateRandomKey(); + CScript single_locking_script = GetScriptForDestination(PKHash(single_key.GetPubKey())); + auto mtx_single = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/single_locking_script, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/false); + CTransactionRef tx_single = MakeTransactionRef(mtx_single); + Package package_tx_single{tx_single}; + const auto result_single_tx = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, + package_tx_single, /*test_accept=*/false, /*client_maxfeerate=*/{}); + expected_pool_size += 1; + BOOST_CHECK_MESSAGE(result_single_tx.m_state.IsValid(), + "Package validation unexpectedly failed: " << result_single_tx.m_state.ToString()); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + + // Parent and Child. Both submitted by themselves through the ProcessNewPackage interface. + CKey parent_key = GenerateRandomKey(); + CScript parent_locking_script = GetScriptForDestination(WitnessV0KeyHash(parent_key.GetPubKey())); + auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[1], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/parent_locking_script, + /*output_amount=*/CAmount(50 * COIN) - high_fee, /*submit=*/false); + CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); + Package package_just_parent{tx_parent}; + const auto result_just_parent = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_just_parent, /*test_accept=*/false, /*client_maxfeerate=*/{}); + if (auto err_parent_child{CheckPackageMempoolAcceptResult(package_just_parent, result_just_parent, /*expect_valid=*/true, nullptr)}) { + BOOST_ERROR(err_parent_child.value()); + } else { + auto it_parent = result_just_parent.m_tx_results.find(tx_parent->GetWitnessHash()); + BOOST_CHECK_MESSAGE(it_parent->second.m_state.IsValid(), it_parent->second.m_state.ToString()); + BOOST_CHECK(it_parent->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_parent)) == high_fee); + BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().size(), 1); + BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().front(), tx_parent->GetWitnessHash()); + } + expected_pool_size += 1; + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + + CKey child_key = GenerateRandomKey(); + CScript child_locking_script = GetScriptForDestination(WitnessV0KeyHash(child_key.GetPubKey())); + auto mtx_child = CreateValidMempoolTransaction(/*input_transaction=*/tx_parent, /*input_vout=*/0, + /*input_height=*/101, /*input_signing_key=*/parent_key, + /*output_destination=*/child_locking_script, + /*output_amount=*/CAmount(50 * COIN) - 2 * high_fee, /*submit=*/false); + CTransactionRef tx_child = MakeTransactionRef(mtx_child); + Package package_just_child{tx_child}; + const auto result_just_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_just_child, /*test_accept=*/false, /*client_maxfeerate=*/{}); + if (auto err_parent_child{CheckPackageMempoolAcceptResult(package_just_child, result_just_child, /*expect_valid=*/true, nullptr)}) { + BOOST_ERROR(err_parent_child.value()); + } else { + auto it_child = result_just_child.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK_MESSAGE(it_child->second.m_state.IsValid(), it_child->second.m_state.ToString()); + BOOST_CHECK(it_child->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_child)) == high_fee); + BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().size(), 1); + BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().front(), tx_child->GetWitnessHash()); + } + expected_pool_size += 1; + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + + // Too-low fee to RBF tx_single + auto mtx_single_low_fee = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/single_locking_script, + /*output_amount=*/CAmount(49 * COIN - 1), /*submit=*/false); + CTransactionRef tx_single_low_fee = MakeTransactionRef(mtx_single_low_fee); + Package package_tx_single_low_fee{tx_single_low_fee}; + const auto result_single_tx_low_fee = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, + package_tx_single_low_fee, /*test_accept=*/false, /*client_maxfeerate=*/{}); + + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + + BOOST_CHECK(!result_single_tx_low_fee.m_state.IsValid()); + BOOST_CHECK_EQUAL(result_single_tx_low_fee.m_state.GetResult(), PackageValidationResult::PCKG_TX); + auto it_low_fee = result_single_tx_low_fee.m_tx_results.find(tx_single_low_fee->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_low_fee->second.m_state.GetResult(), TxValidationResult::TX_RECONSIDERABLE); + if (auto err_single{CheckPackageMempoolAcceptResult(package_tx_single_low_fee, result_single_tx_low_fee, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_single.value()); + } + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); +} + // Tests for packages containing transactions that have same-txid-different-witness equivalents in // the mempool. BOOST_AUTO_TEST_CASE(package_witness_swap_tests) @@ -1078,7 +1171,25 @@ BOOST_AUTO_TEST_CASE(package_rbf_tests) BOOST_CHECK_EQUAL(it_child_3->second.m_effective_feerate.value().GetFee(package3_total_vsize), 199 + 1300); BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - } + // Finally, check that we can prioritise tx_child_1 to get package1 into the mempool. + // It should not be possible to resubmit package1 and get it in without prioritisation. + const auto submit4 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package1, false, std::nullopt); + if (auto err_4{CheckPackageMempoolAcceptResult(package1, submit4, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_4.value()); + } + m_node.mempool->PrioritiseTransaction(tx_child_1->GetHash(), 1363); + const auto submit5 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package1, false, std::nullopt); + if (auto err_5{CheckPackageMempoolAcceptResult(package1, submit5, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_5.value()); + } + it_parent_1 = submit5.m_tx_results.find(tx_parent_1->GetWitnessHash()); + it_child_1 = submit5.m_tx_results.find(tx_child_1->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent_1->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(it_child_1->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + LOCK(m_node.mempool->cs); + BOOST_CHECK(m_node.mempool->GetIter(tx_parent_1->GetHash()).has_value()); + BOOST_CHECK(m_node.mempool->GetIter(tx_child_1->GetHash()).has_value()); + } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txrequest_tests.cpp b/src/test/txrequest_tests.cpp index 6bb732ba69..d73692d8f9 100644 --- a/src/test/txrequest_tests.cpp +++ b/src/test/txrequest_tests.cpp @@ -178,10 +178,10 @@ public: size_t real_total = runner.txrequest.Count(peer); size_t real_candidates = runner.txrequest.CountCandidates(peer); size_t real_inflight = runner.txrequest.CountInFlight(peer); - BOOST_CHECK_MESSAGE(real_total == total, strprintf("[" + comment + "] total %i (%i expected)", real_total, total)); - BOOST_CHECK_MESSAGE(real_inflight == inflight, strprintf("[" + comment + "] inflight %i (%i expected)", real_inflight, inflight)); - BOOST_CHECK_MESSAGE(real_candidates == candidates, strprintf("[" + comment + "] candidates %i (%i expected)", real_candidates, candidates)); - BOOST_CHECK_MESSAGE(ret == expected, "[" + comment + "] mismatching requestables"); + BOOST_CHECK_MESSAGE(real_total == total, strprintf("[%s] total %i (%i expected)", comment, real_total, total)); + BOOST_CHECK_MESSAGE(real_inflight == inflight, strprintf("[%s] inflight %i (%i expected)", comment, real_inflight, inflight)); + BOOST_CHECK_MESSAGE(real_candidates == candidates, strprintf("[%s] candidates %i (%i expected)", comment, real_candidates, candidates)); + BOOST_CHECK_MESSAGE(ret == expected, strprintf("[%s] mismatching requestables", comment)); }); } diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index 97b27ef370..54d94b3ce4 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -6,6 +6,7 @@ #include <key_io.h> #include <policy/packages.h> #include <policy/policy.h> +#include <policy/ephemeral_policy.h> #include <policy/truc_policy.h> #include <primitives/transaction.h> #include <random.h> @@ -89,6 +90,188 @@ static inline CTransactionRef make_tx(const std::vector<COutPoint>& inputs, int3 return MakeTransactionRef(mtx); } +static constexpr auto NUM_EPHEMERAL_TX_OUTPUTS = 3; +static constexpr auto EPHEMERAL_DUST_INDEX = NUM_EPHEMERAL_TX_OUTPUTS - 1; + +// Same as make_tx but adds 2 normal outputs and 0-value dust to end of vout +static inline CTransactionRef make_ephemeral_tx(const std::vector<COutPoint>& inputs, int32_t version) +{ + CMutableTransaction mtx = CMutableTransaction{}; + mtx.version = version; + mtx.vin.resize(inputs.size()); + for (size_t i{0}; i < inputs.size(); ++i) { + mtx.vin[i].prevout = inputs[i]; + } + mtx.vout.resize(NUM_EPHEMERAL_TX_OUTPUTS); + for (auto i{0}; i < NUM_EPHEMERAL_TX_OUTPUTS; ++i) { + mtx.vout[i].scriptPubKey = CScript() << OP_TRUE; + mtx.vout[i].nValue = (i == EPHEMERAL_DUST_INDEX) ? 0 : 10000; + } + return MakeTransactionRef(mtx); +} + +BOOST_FIXTURE_TEST_CASE(ephemeral_tests, RegTestingSetup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(cs_main, pool.cs); + TestMemPoolEntryHelper entry; + CTxMemPool::setEntries empty_ancestors; + + TxValidationState child_state; + Txid child_txid; + + // Arbitrary non-0 feerate for these tests + CFeeRate dustrelay(DUST_RELAY_TX_FEE); + + // Basic transaction with dust + auto grandparent_tx_1 = make_ephemeral_tx(random_outpoints(1), /*version=*/2); + const auto dust_txid = grandparent_tx_1->GetHash(); + + // Child transaction spending dust + auto dust_spend = make_tx({COutPoint{dust_txid, EPHEMERAL_DUST_INDEX}}, /*version=*/2); + + // We first start with nothing "in the mempool", using package checks + + // Trivial single transaction with no dust + BOOST_CHECK(CheckEphemeralSpends({dust_spend}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + // Now with dust, ok because the tx has no dusty parents + BOOST_CHECK(CheckEphemeralSpends({grandparent_tx_1}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + // Dust checks pass + BOOST_CHECK(CheckEphemeralSpends({grandparent_tx_1, dust_spend}, CFeeRate(0), pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + BOOST_CHECK(CheckEphemeralSpends({grandparent_tx_1, dust_spend}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + auto dust_non_spend = make_tx({COutPoint{dust_txid, EPHEMERAL_DUST_INDEX - 1}}, /*version=*/2); + + // Child spending non-dust only from parent should be disallowed even if dust otherwise spent + const auto dust_non_spend_txid{dust_non_spend->GetHash()}; + BOOST_CHECK(!CheckEphemeralSpends({grandparent_tx_1, dust_non_spend, dust_spend}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(!child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, dust_non_spend_txid); + child_state = TxValidationState(); + child_txid = Txid(); + + BOOST_CHECK(!CheckEphemeralSpends({grandparent_tx_1, dust_spend, dust_non_spend}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(!child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, dust_non_spend_txid); + child_state = TxValidationState(); + child_txid = Txid(); + + BOOST_CHECK(!CheckEphemeralSpends({grandparent_tx_1, dust_non_spend}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(!child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, dust_non_spend_txid); + child_state = TxValidationState(); + child_txid = Txid(); + + auto grandparent_tx_2 = make_ephemeral_tx(random_outpoints(1), /*version=*/2); + const auto dust_txid_2 = grandparent_tx_2->GetHash(); + + // Spend dust from one but not another is ok, as long as second grandparent has no child + BOOST_CHECK(CheckEphemeralSpends({grandparent_tx_1, grandparent_tx_2, dust_spend}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + auto dust_non_spend_both_parents = make_tx({COutPoint{dust_txid, EPHEMERAL_DUST_INDEX}, COutPoint{dust_txid_2, EPHEMERAL_DUST_INDEX - 1}}, /*version=*/2); + // But if we spend from the parent, it must spend dust + BOOST_CHECK(!CheckEphemeralSpends({grandparent_tx_1, grandparent_tx_2, dust_non_spend_both_parents}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(!child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, dust_non_spend_both_parents->GetHash()); + child_state = TxValidationState(); + child_txid = Txid(); + + auto dust_spend_both_parents = make_tx({COutPoint{dust_txid, EPHEMERAL_DUST_INDEX}, COutPoint{dust_txid_2, EPHEMERAL_DUST_INDEX}}, /*version=*/2); + BOOST_CHECK(CheckEphemeralSpends({grandparent_tx_1, grandparent_tx_2, dust_spend_both_parents}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + // Spending other outputs is also correct, as long as the dusty one is spent + const std::vector<COutPoint> all_outpoints{COutPoint(dust_txid, 0), COutPoint(dust_txid, 1), COutPoint(dust_txid, 2), + COutPoint(dust_txid_2, 0), COutPoint(dust_txid_2, 1), COutPoint(dust_txid_2, 2)}; + auto dust_spend_all_outpoints = make_tx(all_outpoints, /*version=*/2); + BOOST_CHECK(CheckEphemeralSpends({grandparent_tx_1, grandparent_tx_2, dust_spend_all_outpoints}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + // 2 grandparents with dust <- 1 dust-spending parent with dust <- child with no dust + auto parent_with_dust = make_ephemeral_tx({COutPoint{dust_txid, EPHEMERAL_DUST_INDEX}, COutPoint{dust_txid_2, EPHEMERAL_DUST_INDEX}}, /*version=*/2); + // Ok for parent to have dust + BOOST_CHECK(CheckEphemeralSpends({grandparent_tx_1, grandparent_tx_2, parent_with_dust}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + auto child_no_dust = make_tx({COutPoint{parent_with_dust->GetHash(), EPHEMERAL_DUST_INDEX}}, /*version=*/2); + BOOST_CHECK(CheckEphemeralSpends({grandparent_tx_1, grandparent_tx_2, parent_with_dust, child_no_dust}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + // 2 grandparents with dust <- 1 dust-spending parent with dust <- child with dust + auto child_with_dust = make_ephemeral_tx({COutPoint{parent_with_dust->GetHash(), EPHEMERAL_DUST_INDEX}}, /*version=*/2); + BOOST_CHECK(CheckEphemeralSpends({grandparent_tx_1, grandparent_tx_2, parent_with_dust, child_with_dust}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + // Tests with parents in mempool + + // Nothing in mempool, this should pass for any transaction + BOOST_CHECK(CheckEphemeralSpends({grandparent_tx_1}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + // Add first grandparent to mempool and fetch entry + AddToMempool(pool, entry.FromTx(grandparent_tx_1)); + + // Ignores ancestors that aren't direct parents + BOOST_CHECK(CheckEphemeralSpends({child_no_dust}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + // Valid spend of dust with grandparent in mempool + BOOST_CHECK(CheckEphemeralSpends({parent_with_dust}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + // Second grandparent in same package + BOOST_CHECK(CheckEphemeralSpends({parent_with_dust, grandparent_tx_2}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + // Order in package doesn't matter + BOOST_CHECK(CheckEphemeralSpends({grandparent_tx_2, parent_with_dust}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + // Add second grandparent to mempool + AddToMempool(pool, entry.FromTx(grandparent_tx_2)); + + // Only spends single dust out of two direct parents + BOOST_CHECK(!CheckEphemeralSpends({dust_non_spend_both_parents}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(!child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, dust_non_spend_both_parents->GetHash()); + child_state = TxValidationState(); + child_txid = Txid(); + + // Spends both parents' dust + BOOST_CHECK(CheckEphemeralSpends({parent_with_dust}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); + + // Now add dusty parent to mempool + AddToMempool(pool, entry.FromTx(parent_with_dust)); + + // Passes dust checks even with non-parent ancestors + BOOST_CHECK(CheckEphemeralSpends({child_no_dust}, dustrelay, pool, child_state, child_txid)); + BOOST_CHECK(child_state.IsValid()); + BOOST_CHECK_EQUAL(child_txid, Txid()); +} + BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) { // Test TRUC policy helper functions @@ -99,9 +282,9 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) CTxMemPool::setEntries empty_ancestors; auto mempool_tx_v3 = make_tx(random_outpoints(1), /*version=*/3); - pool.addUnchecked(entry.FromTx(mempool_tx_v3)); + AddToMempool(pool, entry.FromTx(mempool_tx_v3)); auto mempool_tx_v2 = make_tx(random_outpoints(1), /*version=*/2); - pool.addUnchecked(entry.FromTx(mempool_tx_v2)); + AddToMempool(pool, entry.FromTx(mempool_tx_v2)); // Default values. CTxMemPool::Limits m_limits{}; @@ -211,7 +394,7 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) package_multi_parents.emplace_back(mempool_tx_v3); for (size_t i{0}; i < 2; ++i) { auto mempool_tx = make_tx(random_outpoints(i + 1), /*version=*/3); - pool.addUnchecked(entry.FromTx(mempool_tx)); + AddToMempool(pool, entry.FromTx(mempool_tx)); mempool_outpoints.emplace_back(mempool_tx->GetHash(), 0); package_multi_parents.emplace_back(mempool_tx); } @@ -236,7 +419,7 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) auto last_outpoint{random_outpoints(1)[0]}; for (size_t i{0}; i < 2; ++i) { auto mempool_tx = make_tx({last_outpoint}, /*version=*/3); - pool.addUnchecked(entry.FromTx(mempool_tx)); + AddToMempool(pool, entry.FromTx(mempool_tx)); last_outpoint = COutPoint{mempool_tx->GetHash(), 0}; package_multi_gen.emplace_back(mempool_tx); if (i == 1) middle_tx = mempool_tx; @@ -323,7 +506,7 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) BOOST_CHECK(GetTransactionWeight(*tx_mempool_v3_child) <= TRUC_CHILD_MAX_VSIZE * WITNESS_SCALE_FACTOR); auto ancestors{pool.CalculateMemPoolAncestors(entry.FromTx(tx_mempool_v3_child), m_limits)}; BOOST_CHECK(SingleTRUCChecks(tx_mempool_v3_child, *ancestors, empty_conflicts_set, GetVirtualTransactionSize(*tx_mempool_v3_child)) == std::nullopt); - pool.addUnchecked(entry.FromTx(tx_mempool_v3_child)); + AddToMempool(pool, entry.FromTx(tx_mempool_v3_child)); Package package_v3_1p1c{mempool_tx_v3, tx_mempool_v3_child}; BOOST_CHECK(PackageTRUCChecks(tx_mempool_v3_child, GetVirtualTransactionSize(*tx_mempool_v3_child), package_v3_1p1c, empty_ancestors) == std::nullopt); @@ -351,7 +534,7 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) expected_error_str); // Configuration where parent already has 2 other children in mempool (no sibling eviction allowed). This may happen as the result of a reorg. - pool.addUnchecked(entry.FromTx(tx_v3_child2)); + AddToMempool(pool, entry.FromTx(tx_v3_child2)); auto tx_v3_child3 = make_tx({COutPoint{mempool_tx_v3->GetHash(), 24}}, /*version=*/3); auto entry_mempool_parent = pool.GetIter(mempool_tx_v3->GetHash().ToUint256()).value(); BOOST_CHECK_EQUAL(entry_mempool_parent->GetCountWithDescendants(), 3); @@ -370,9 +553,9 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) auto tx_mempool_nibling = make_tx({COutPoint{tx_mempool_sibling->GetHash(), 0}}, /*version=*/3); auto tx_to_submit = make_tx({COutPoint{tx_mempool_grandparent->GetHash(), 1}}, /*version=*/3); - pool.addUnchecked(entry.FromTx(tx_mempool_grandparent)); - pool.addUnchecked(entry.FromTx(tx_mempool_sibling)); - pool.addUnchecked(entry.FromTx(tx_mempool_nibling)); + AddToMempool(pool, entry.FromTx(tx_mempool_grandparent)); + AddToMempool(pool, entry.FromTx(tx_mempool_sibling)); + AddToMempool(pool, entry.FromTx(tx_mempool_nibling)); auto ancestors_3gen{pool.CalculateMemPoolAncestors(entry.FromTx(tx_to_submit), m_limits)}; const auto expected_error_str{strprintf("tx %s (wtxid=%s) would exceed descendant count limit", diff --git a/src/test/util/cluster_linearize.h b/src/test/util/cluster_linearize.h index 9477d2ed41..871aa9d74e 100644 --- a/src/test/util/cluster_linearize.h +++ b/src/test/util/cluster_linearize.h @@ -27,7 +27,7 @@ using TestBitSet = BitSet<32>; template<typename SetType> bool IsAcyclic(const DepGraph<SetType>& depgraph) noexcept { - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + for (ClusterIndex i : depgraph.Positions()) { if ((depgraph.Ancestors(i) & depgraph.Descendants(i)) != SetType::Singleton(i)) { return false; } @@ -57,11 +57,14 @@ bool IsAcyclic(const DepGraph<SetType>& depgraph) noexcept * by parent relations that were serialized before it). * - The various insertion positions in the cluster, from the very end of the cluster, to the * front. + * - The appending of 1, 2, 3, ... holes at the end of the cluster, followed by appending the new + * transaction. * - * Let's say you have a 7-transaction cluster, consisting of transactions F,A,C,B,G,E,D, but - * serialized in order A,B,C,D,E,F,G, because that happens to be a topological ordering. By the - * time G gets serialized, what has been serialized already represents the cluster F,A,C,B,E,D (in - * that order). G has B and E as direct parents, and E depends on C. + * Let's say you have a 7-transaction cluster, consisting of transactions F,A,C,B,_,G,E,_,D + * (where _ represent holes; unused positions within the DepGraph) but serialized in order + * A,B,C,D,E,F,G, because that happens to be a topological ordering. By the time G gets serialized, + * what has been serialized already represents the cluster F,A,C,B,_,E,_,D (in that order). G has B + * and E as direct parents, and E depends on C. * * In this case, the possibilities are, in order: * - [ ] the dependency G->F @@ -71,17 +74,23 @@ bool IsAcyclic(const DepGraph<SetType>& depgraph) noexcept * - [ ] the dependency G->A * - [ ] put G at the end of the cluster * - [ ] put G before D + * - [ ] put G before the hole before D * - [X] put G before E + * - [ ] put G before the hole before E * - [ ] put G before B * - [ ] put G before C * - [ ] put G before A * - [ ] put G before F + * - [ ] add 1 hole at the end of the cluster, followed by G + * - [ ] add 2 holes at the end of the cluster, followed by G + * - [ ] add ... * - * The skip values in this case are 1 (G->F), 1 (G->D), 3 (G->A, G at end, G before D). No skip - * after 3 is needed (or permitted), because there can only be one position for G. Also note that - * G->C is not included in the list of possibilities, as it is implied by the included G->E and - * E->C that came before it. On deserialization, if the last skip value was 8 or larger (putting - * G before the beginning of the cluster), it is interpreted as wrapping around back to the end. + * The skip values in this case are 1 (G->F), 1 (G->D), 4 (G->A, G at end, G before D, G before + * hole). No skip after 4 is needed (or permitted), because there can only be one position for G. + * Also note that G->C is not included in the list of possibilities, as it is implied by the + * included G->E and E->C that came before it. On deserialization, if the last skip value was 8 or + * larger (putting G before the beginning of the cluster), it is interpreted as wrapping around + * back to the end. * * * Rationale: @@ -102,7 +111,7 @@ bool IsAcyclic(const DepGraph<SetType>& depgraph) noexcept struct DepGraphFormatter { /** Convert x>=0 to 2x (even), x<0 to -2x-1 (odd). */ - static uint64_t SignedToUnsigned(int64_t x) noexcept + [[maybe_unused]] static uint64_t SignedToUnsigned(int64_t x) noexcept { if (x < 0) { return 2 * uint64_t(-(x + 1)) + 1; @@ -112,7 +121,7 @@ struct DepGraphFormatter } /** Convert even x to x/2 (>=0), odd x to -(x/2)-1 (<0). */ - static int64_t UnsignedToSigned(uint64_t x) noexcept + [[maybe_unused]] static int64_t UnsignedToSigned(uint64_t x) noexcept { if (x & 1) { return -int64_t(x / 2) - 1; @@ -125,18 +134,18 @@ struct DepGraphFormatter static void Ser(Stream& s, const DepGraph<SetType>& depgraph) { /** Construct a topological order to serialize the transactions in. */ - std::vector<ClusterIndex> topo_order(depgraph.TxCount()); - std::iota(topo_order.begin(), topo_order.end(), ClusterIndex{0}); + std::vector<ClusterIndex> topo_order; + topo_order.reserve(depgraph.TxCount()); + for (auto i : depgraph.Positions()) topo_order.push_back(i); std::sort(topo_order.begin(), topo_order.end(), [&](ClusterIndex a, ClusterIndex b) { auto anc_a = depgraph.Ancestors(a).Count(), anc_b = depgraph.Ancestors(b).Count(); if (anc_a != anc_b) return anc_a < anc_b; return a < b; }); - /** Which transactions the deserializer already knows when it has deserialized what has - * been serialized here so far, and in what order. */ - std::vector<ClusterIndex> rebuilt_order; - rebuilt_order.reserve(depgraph.TxCount()); + /** Which positions (incl. holes) the deserializer already knows when it has deserialized + * what has been serialized here so far. */ + SetType done; // Loop over the transactions in topological order. for (ClusterIndex topo_idx = 0; topo_idx < topo_order.size(); ++topo_idx) { @@ -166,14 +175,20 @@ struct DepGraphFormatter } } // Write position information. - ClusterIndex insert_distance = 0; - while (insert_distance < rebuilt_order.size()) { - // Loop to find how far from the end in rebuilt_order to insert. - if (idx > *(rebuilt_order.end() - 1 - insert_distance)) break; - ++insert_distance; + auto add_holes = SetType::Fill(idx) - done - depgraph.Positions(); + if (add_holes.None()) { + // The new transaction is to be inserted N positions back from the end of the + // cluster. Emit N to indicate that that many insertion choices are skipped. + auto skips = (done - SetType::Fill(idx)).Count(); + s << VARINT(diff + skips); + } else { + // The new transaction is to be appended at the end of the cluster, after N holes. + // Emit current_cluster_size + N, to indicate all insertion choices are skipped, + // plus N possibilities for the number of holes. + s << VARINT(diff + done.Count() + add_holes.Count()); + done |= add_holes; } - rebuilt_order.insert(rebuilt_order.end() - insert_distance, idx); - s << VARINT(diff + insert_distance); + done.Set(idx); } // Output a final 0 to denote the end of the graph. @@ -186,13 +201,19 @@ struct DepGraphFormatter /** The dependency graph which we deserialize into first, with transactions in * topological serialization order, not original cluster order. */ DepGraph<SetType> topo_depgraph; - /** Mapping from cluster order to serialization order, used later to reconstruct the + /** Mapping from serialization order to cluster order, used later to reconstruct the * cluster order. */ std::vector<ClusterIndex> reordering; + /** How big the entries vector in the reconstructed depgraph will be (including holes). */ + ClusterIndex total_size{0}; // Read transactions in topological order. - try { - while (true) { + while (true) { + FeeFrac new_feerate; //!< The new transaction's fee and size. + SetType new_ancestors; //!< The new transaction's ancestors (excluding itself). + uint64_t diff{0}; //!< How many potential parents/insertions we have to skip. + bool read_error{false}; + try { // Read size. Size 0 signifies the end of the DepGraph. int32_t size; s >> VARINT_MODE(size, VarIntMode::NONNEGATIVE_SIGNED); @@ -204,21 +225,18 @@ struct DepGraphFormatter s >> VARINT(coded_fee); coded_fee &= 0xFFFFFFFFFFFFF; // Enough for fee between -21M...21M BTC. static_assert(0xFFFFFFFFFFFFF > uint64_t{2} * 21000000 * 100000000); - auto fee = UnsignedToSigned(coded_fee); - // Extend topo_depgraph with the new transaction (at the end). - auto topo_idx = topo_depgraph.AddTransaction({fee, size}); - reordering.push_back(topo_idx); + new_feerate = {UnsignedToSigned(coded_fee), size}; // Read dependency information. - uint64_t diff = 0; //!< How many potential parents we have to skip. + auto topo_idx = reordering.size(); s >> VARINT(diff); for (ClusterIndex dep_dist = 0; dep_dist < topo_idx; ++dep_dist) { /** Which topo_depgraph index we are currently considering as parent of topo_idx. */ ClusterIndex dep_topo_idx = topo_idx - 1 - dep_dist; // Ignore transactions which are already known ancestors of topo_idx. - if (topo_depgraph.Descendants(dep_topo_idx)[topo_idx]) continue; + if (new_ancestors[dep_topo_idx]) continue; if (diff == 0) { // When the skip counter has reached 0, add an actual dependency. - topo_depgraph.AddDependency(dep_topo_idx, topo_idx); + new_ancestors |= topo_depgraph.Ancestors(dep_topo_idx); // And read the number of skips after it. s >> VARINT(diff); } else { @@ -226,31 +244,52 @@ struct DepGraphFormatter --diff; } } - // If we reach this point, we can interpret the remaining skip value as how far from the - // end of reordering topo_idx should be placed (wrapping around), so move it to its - // correct location. The preliminary reordering.push_back(topo_idx) above was to make - // sure that if a deserialization exception occurs, topo_idx still appears somewhere. - reordering.pop_back(); - reordering.insert(reordering.end() - (diff % (reordering.size() + 1)), topo_idx); + } catch (const std::ios_base::failure&) { + // Continue even if a read error was encountered. + read_error = true; } - } catch (const std::ios_base::failure&) {} - - // Construct the original cluster order depgraph. - depgraph = {}; - // Add transactions to depgraph in the original cluster order. - for (auto topo_idx : reordering) { - depgraph.AddTransaction(topo_depgraph.FeeRate(topo_idx)); - } - // Translate dependencies from topological to cluster order. - for (ClusterIndex idx = 0; idx < reordering.size(); ++idx) { - ClusterIndex topo_idx = reordering[idx]; - for (ClusterIndex dep_idx = 0; dep_idx < reordering.size(); ++dep_idx) { - ClusterIndex dep_topo_idx = reordering[dep_idx]; - if (topo_depgraph.Ancestors(topo_idx)[dep_topo_idx]) { - depgraph.AddDependency(dep_idx, idx); + // Construct a new transaction whenever we made it past the new_feerate construction. + if (new_feerate.IsEmpty()) break; + assert(reordering.size() < SetType::Size()); + auto topo_idx = topo_depgraph.AddTransaction(new_feerate); + topo_depgraph.AddDependencies(new_ancestors, topo_idx); + if (total_size < SetType::Size()) { + // Normal case. + diff %= SetType::Size(); + if (diff <= total_size) { + // Insert the new transaction at distance diff back from the end. + for (auto& pos : reordering) { + pos += (pos >= total_size - diff); + } + reordering.push_back(total_size++ - diff); + } else { + // Append diff - total_size holes at the end, plus the new transaction. + total_size = diff; + reordering.push_back(total_size++); + } + } else { + // In case total_size == SetType::Size, it is not possible to insert the new + // transaction without exceeding SetType's size. Instead, interpret diff as an + // index into the holes, and overwrite a position there. This branch is never used + // when deserializing the output of the serializer, but gives meaning to otherwise + // invalid input. + diff %= (SetType::Size() - reordering.size()); + SetType holes = SetType::Fill(SetType::Size()); + for (auto pos : reordering) holes.Reset(pos); + for (auto pos : holes) { + if (diff == 0) { + reordering.push_back(pos); + break; + } + --diff; } } + // Stop if a read error was encountered during deserialization. + if (read_error) break; } + + // Construct the original cluster order depgraph. + depgraph = DepGraph(topo_depgraph, reordering, total_size); } }; @@ -258,8 +297,19 @@ struct DepGraphFormatter template<typename SetType> void SanityCheck(const DepGraph<SetType>& depgraph) { + // Verify Positions and PositionRange consistency. + ClusterIndex num_positions{0}; + ClusterIndex position_range{0}; + for (ClusterIndex i : depgraph.Positions()) { + ++num_positions; + position_range = i + 1; + } + assert(num_positions == depgraph.TxCount()); + assert(position_range == depgraph.PositionRange()); + assert(position_range >= num_positions); + assert(position_range <= SetType::Size()); // Consistency check between ancestors internally. - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + for (ClusterIndex i : depgraph.Positions()) { // Transactions include themselves as ancestors. assert(depgraph.Ancestors(i)[i]); // If a is an ancestor of b, then b's ancestors must include all of a's ancestors. @@ -268,13 +318,27 @@ void SanityCheck(const DepGraph<SetType>& depgraph) } } // Consistency check between ancestors and descendants. - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { - for (ClusterIndex j = 0; j < depgraph.TxCount(); ++j) { + for (ClusterIndex i : depgraph.Positions()) { + for (ClusterIndex j : depgraph.Positions()) { assert(depgraph.Ancestors(i)[j] == depgraph.Descendants(j)[i]); } + // No transaction is a parent or child of itself. + auto parents = depgraph.GetReducedParents(i); + auto children = depgraph.GetReducedChildren(i); + assert(!parents[i]); + assert(!children[i]); + // Parents of a transaction do not have ancestors inside those parents (except itself). + // Note that even the transaction itself may be missing (if it is part of a cycle). + for (auto parent : parents) { + assert((depgraph.Ancestors(parent) & parents).IsSubsetOf(SetType::Singleton(parent))); + } + // Similar for children and descendants. + for (auto child : children) { + assert((depgraph.Descendants(child) & children).IsSubsetOf(SetType::Singleton(child))); + } } - // If DepGraph is acyclic, serialize + deserialize must roundtrip. if (IsAcyclic(depgraph)) { + // If DepGraph is acyclic, serialize + deserialize must roundtrip. std::vector<unsigned char> ser; VectorWriter writer(ser, 0); writer << Using<DepGraphFormatter>(depgraph); @@ -292,42 +356,36 @@ void SanityCheck(const DepGraph<SetType>& depgraph) reader >> Using<DepGraphFormatter>(decoded_depgraph); assert(depgraph == decoded_depgraph); assert(reader.empty()); - } -} -/** Verify that a DepGraph corresponds to the information in a cluster. */ -template<typename SetType> -void VerifyDepGraphFromCluster(const Cluster<SetType>& cluster, const DepGraph<SetType>& depgraph) -{ - // Sanity check the depgraph, which includes a check for correspondence between ancestors and - // descendants, so it suffices to check just ancestors below. - SanityCheck(depgraph); - // Verify transaction count. - assert(cluster.size() == depgraph.TxCount()); - // Verify feerates. - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { - assert(depgraph.FeeRate(i) == cluster[i].first); - } - // Verify ancestors. - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { - // Start with the transaction having itself as ancestor. - auto ancestors = SetType::Singleton(i); - // Add parents of ancestors to the set of ancestors until it stops changing. - while (true) { - const auto old_ancestors = ancestors; - for (auto ancestor : ancestors) { - ancestors |= cluster[ancestor].second; - } - if (old_ancestors == ancestors) break; + // In acyclic graphs, the union of parents with parents of parents etc. yields the + // full ancestor set (and similar for children and descendants). + std::vector<SetType> parents(depgraph.PositionRange()), children(depgraph.PositionRange()); + for (ClusterIndex i : depgraph.Positions()) { + parents[i] = depgraph.GetReducedParents(i); + children[i] = depgraph.GetReducedChildren(i); } - // Compare against depgraph. - assert(depgraph.Ancestors(i) == ancestors); - // Some additional sanity tests: - // - Every transaction has itself as ancestor. - assert(ancestors[i]); - // - Every transaction has its direct parents as ancestors. - for (auto parent : cluster[i].second) { - assert(ancestors[parent]); + for (auto i : depgraph.Positions()) { + // Initialize the set of ancestors with just the current transaction itself. + SetType ancestors = SetType::Singleton(i); + // Iteratively add parents of all transactions in the ancestor set to itself. + while (true) { + const auto old_ancestors = ancestors; + for (auto j : ancestors) ancestors |= parents[j]; + // Stop when no more changes are being made. + if (old_ancestors == ancestors) break; + } + assert(ancestors == depgraph.Ancestors(i)); + + // Initialize the set of descendants with just the current transaction itself. + SetType descendants = SetType::Singleton(i); + // Iteratively add children of all transactions in the descendant set to itself. + while (true) { + const auto old_descendants = descendants; + for (auto j : descendants) descendants |= children[j]; + // Stop when no more changes are being made. + if (old_descendants == descendants) break; + } + assert(descendants == depgraph.Descendants(i)); } } } @@ -341,7 +399,7 @@ void SanityCheck(const DepGraph<SetType>& depgraph, Span<const ClusterIndex> lin TestBitSet done; for (auto i : linearization) { // Check transaction position is in range. - assert(i < depgraph.TxCount()); + assert(depgraph.Positions()[i]); // Check topology and lack of duplicates. assert((depgraph.Ancestors(i) - done) == TestBitSet::Singleton(i)); done.Set(i); diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index ad7a38d3fe..04925792dc 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -24,9 +24,10 @@ COutPoint generatetoaddress(const NodeContext& node, const std::string& address) { const auto dest = DecodeDestination(address); assert(IsValidDestination(dest)); - const auto coinbase_script = GetScriptForDestination(dest); + BlockAssembler::Options assembler_options; + assembler_options.coinbase_output_script = GetScriptForDestination(dest); - return MineBlock(node, coinbase_script); + return MineBlock(node, assembler_options); } std::vector<std::shared_ptr<CBlock>> CreateBlockChain(size_t total_height, const CChainParams& params) @@ -60,9 +61,9 @@ std::vector<std::shared_ptr<CBlock>> CreateBlockChain(size_t total_height, const return ret; } -COutPoint MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) +COutPoint MineBlock(const NodeContext& node, const node::BlockAssembler::Options& assembler_options) { - auto block = PrepareBlock(node, coinbase_scriptPubKey); + auto block = PrepareBlock(node, assembler_options); auto valid = MineBlock(node, block); assert(!valid.IsNull()); return valid; @@ -108,12 +109,12 @@ COutPoint MineBlock(const NodeContext& node, std::shared_ptr<CBlock>& block) return {}; } -std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey, +std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const BlockAssembler::Options& assembler_options) { auto block = std::make_shared<CBlock>( BlockAssembler{Assert(node.chainman)->ActiveChainstate(), Assert(node.mempool.get()), assembler_options} - .CreateNewBlock(coinbase_scriptPubKey) + .CreateNewBlock() ->block); LOCK(cs_main); @@ -125,6 +126,7 @@ std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coi std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) { BlockAssembler::Options assembler_options; + assembler_options.coinbase_output_script = coinbase_scriptPubKey; ApplyArgsManOptions(*node.args, assembler_options); - return PrepareBlock(node, coinbase_scriptPubKey, assembler_options); + return PrepareBlock(node, assembler_options); } diff --git a/src/test/util/mining.h b/src/test/util/mining.h index 3f071257f1..9c6e29b4d3 100644 --- a/src/test/util/mining.h +++ b/src/test/util/mining.h @@ -23,7 +23,8 @@ struct NodeContext; std::vector<std::shared_ptr<CBlock>> CreateBlockChain(size_t total_height, const CChainParams& params); /** Returns the generated coin */ -COutPoint MineBlock(const node::NodeContext&, const CScript& coinbase_scriptPubKey); +COutPoint MineBlock(const node::NodeContext&, + const node::BlockAssembler::Options& assembler_options); /** * Returns the generated coin (or Null if the block was invalid). @@ -32,8 +33,8 @@ COutPoint MineBlock(const node::NodeContext&, const CScript& coinbase_scriptPubK COutPoint MineBlock(const node::NodeContext&, std::shared_ptr<CBlock>& block); /** Prepare a block to be mined */ -std::shared_ptr<CBlock> PrepareBlock(const node::NodeContext&, const CScript& coinbase_scriptPubKey); -std::shared_ptr<CBlock> PrepareBlock(const node::NodeContext& node, const CScript& coinbase_scriptPubKey, +std::shared_ptr<CBlock> PrepareBlock(const node::NodeContext&); +std::shared_ptr<CBlock> PrepareBlock(const node::NodeContext& node, const node::BlockAssembler::Options& assembler_options); /** RPC-like helper function, returns the generated coin */ diff --git a/src/test/util/net.h b/src/test/util/net.h index 043e317bf0..d3aefda4f0 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -12,6 +12,7 @@ #include <netaddress.h> #include <node/connection_types.h> #include <node/eviction.h> +#include <span.h> #include <sync.h> #include <util/sock.h> @@ -28,9 +29,6 @@ class FastRandomContext; -template <typename C> -class Span; - struct ConnmanTestMsg : public CConnman { using CConnman::CConnman; diff --git a/src/test/util/random.cpp b/src/test/util/random.cpp index 32d785e45d..770848f708 100644 --- a/src/test/util/random.cpp +++ b/src/test/util/random.cpp @@ -12,6 +12,8 @@ #include <cstdlib> #include <iostream> +std::atomic<bool> g_seeded_g_prng_zero{false}; + extern void MakeRandDeterministicDANGEROUS(const uint256& seed) noexcept; void SeedRandomStateForTest(SeedRand seedtype) @@ -36,6 +38,10 @@ void SeedRandomStateForTest(SeedRand seedtype) return GetRandHash(); }(); + g_seeded_g_prng_zero = seedtype == SeedRand::ZEROS; + if constexpr (G_FUZZING) { + Assert(g_seeded_g_prng_zero); // Only SeedRandomStateForTest(SeedRand::ZEROS) is allowed in fuzz tests + } const uint256& seed{seedtype == SeedRand::FIXED_SEED ? ctx_seed : uint256::ZERO}; LogInfo("Setting random seed for current tests to %s=%s\n", RANDOM_CTX_SEED, seed.GetHex()); MakeRandDeterministicDANGEROUS(seed); diff --git a/src/test/util/random.h b/src/test/util/random.h index 441150e666..47bc7a18f9 100644 --- a/src/test/util/random.h +++ b/src/test/util/random.h @@ -9,6 +9,7 @@ #include <random.h> #include <uint256.h> +#include <atomic> #include <cstdint> enum class SeedRand { @@ -27,6 +28,9 @@ enum class SeedRand { /** Seed the global RNG state for testing and log the seed value. This affects all randomness, except GetStrongRandBytes(). */ void SeedRandomStateForTest(SeedRand seed); +extern std::atomic<bool> g_seeded_g_prng_zero; +extern std::atomic<bool> g_used_g_prng; + template <RandomNumberGenerator Rng> inline CAmount RandMoney(Rng&& rng) { diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index fa89ceb332..3f8c6f41ba 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -77,6 +77,11 @@ const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; constexpr inline auto TEST_DIR_PATH_ELEMENT{"test_common bitcoin"}; // Includes a space to catch possible path escape issues. /** Random context to get unique temp data dirs. Separate from m_rng, which can be seeded from a const env var */ static FastRandomContext g_rng_temp_path; +static const bool g_rng_temp_path_init{[] { + // Must be initialized before any SeedRandomForTest + (void)g_rng_temp_path.rand64(); + return true; +}()}; struct NetworkSetup { @@ -87,8 +92,7 @@ struct NetworkSetup }; static NetworkSetup g_networksetup_instance; -/** Register test-only arguments */ -static void SetupUnitTestArgs(ArgsManager& argsman) +void SetupCommonTestArgs(ArgsManager& argsman) { argsman.AddArg("-testdatadir", strprintf("Custom data directory (default: %s<random_string>)", fs::PathToString(fs::temp_directory_path() / TEST_DIR_PATH_ELEMENT / "")), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); @@ -104,7 +108,11 @@ static void ExitFailure(std::string_view str_err) BasicTestingSetup::BasicTestingSetup(const ChainType chainType, TestOpts opts) : m_args{} { - m_node.shutdown = &m_interrupt; + if constexpr (!G_FUZZING) { + SeedRandomForTest(SeedRand::FIXED_SEED); + } + m_node.shutdown_signal = &m_interrupt; + m_node.shutdown_request = [this]{ return m_interrupt(); }; m_node.args = &gArgs; std::vector<const char*> arguments = Cat( { @@ -126,7 +134,7 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, TestOpts opts) gArgs.ClearPathCache(); { SetupServerArgs(*m_node.args); - SetupUnitTestArgs(*m_node.args); + SetupCommonTestArgs(*m_node.args); std::string error; if (!m_node.args->ParseParameters(arguments.size(), arguments.data(), error)) { m_node.args->ClearArgs(); @@ -134,14 +142,14 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, TestOpts opts) } } - // Use randomly chosen seed for deterministic PRNG, so that (by default) test - // data directories use a random name that doesn't overlap with other tests. - SeedRandomForTest(SeedRand::FIXED_SEED); - + const std::string test_name{G_TEST_GET_FULL_NAME ? G_TEST_GET_FULL_NAME() : ""}; if (!m_node.args->IsArgSet("-testdatadir")) { - // By default, the data directory has a random name - const auto rand_str{g_rng_temp_path.rand256().ToString()}; - m_path_root = fs::temp_directory_path() / TEST_DIR_PATH_ELEMENT / rand_str; + // To avoid colliding with a leftover prior datadir, and to allow + // tests, such as the fuzz tests to run in several processes at the + // same time, add a random element to the path. Keep it small enough to + // avoid a MAX_PATH violation on Windows. + const auto rand{HexStr(g_rng_temp_path.randbytes(10))}; + m_path_root = fs::temp_directory_path() / TEST_DIR_PATH_ELEMENT / test_name / rand; TryCreateDirectories(m_path_root); } else { // Custom data directory @@ -150,8 +158,7 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, TestOpts opts) if (root_dir.empty()) ExitFailure("-testdatadir argument is empty, please specify a path"); root_dir = fs::absolute(root_dir); - const std::string test_path{G_TEST_GET_FULL_NAME ? G_TEST_GET_FULL_NAME() : ""}; - m_path_lock = root_dir / TEST_DIR_PATH_ELEMENT / fs::PathFromString(test_path); + m_path_lock = root_dir / TEST_DIR_PATH_ELEMENT / fs::PathFromString(test_name); m_path_root = m_path_lock / "datadir"; // Try to obtain the lock; if unsuccessful don't disturb the existing test. @@ -224,7 +231,7 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts) m_cache_sizes = CalculateCacheSizes(m_args); - m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)); + m_node.notifications = std::make_unique<KernelNotifications>(Assert(m_node.shutdown_request), m_node.exit_status, *Assert(m_node.warnings)); m_make_chainman = [this, &chainparams, opts] { Assert(!m_node.chainman); @@ -245,7 +252,7 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts) .blocks_dir = m_args.GetBlocksDirPath(), .notifications = chainman_opts.notifications, }; - m_node.chainman = std::make_unique<ChainstateManager>(*Assert(m_node.shutdown), chainman_opts, blockman_opts); + m_node.chainman = std::make_unique<ChainstateManager>(*Assert(m_node.shutdown_signal), chainman_opts, blockman_opts); LOCK(m_node.chainman->GetMutex()); m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<BlockTreeDB>(DBParams{ .path = m_args.GetDataDirNet() / "blocks" / "index", @@ -371,7 +378,8 @@ CBlock TestChain100Setup::CreateBlock( Chainstate& chainstate) { BlockAssembler::Options options; - CBlock block = BlockAssembler{chainstate, nullptr, options}.CreateNewBlock(scriptPubKey)->block; + options.coinbase_output_script = scriptPubKey; + CBlock block = BlockAssembler{chainstate, nullptr, options}.CreateNewBlock()->block; Assert(block.vtx.size() == 1); for (const CMutableTransaction& tx : txns) { @@ -432,9 +440,8 @@ std::pair<CMutableTransaction, CAmount> TestChain100Setup::CreateValidTransactio std::map<COutPoint, Coin> input_coins; CAmount inputs_amount{0}; for (const auto& outpoint_to_spend : inputs) { - // - Use GetCoin to properly populate utxo_to_spend, - Coin utxo_to_spend; - assert(coins_cache.GetCoin(outpoint_to_spend, utxo_to_spend)); + // Use GetCoin to properly populate utxo_to_spend + auto utxo_to_spend{coins_cache.GetCoin(outpoint_to_spend).value()}; input_coins.insert({outpoint_to_spend, utxo_to_spend}); inputs_amount += utxo_to_spend.out.nValue; } @@ -539,9 +546,11 @@ std::vector<CTransactionRef> TestChain100Setup::PopulateMempool(FastRandomContex if (submit) { LOCK2(cs_main, m_node.mempool->cs); LockPoints lp; - m_node.mempool->addUnchecked(CTxMemPoolEntry(ptx, /*fee=*/(total_in - num_outputs * amount_per_output), - /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, - /*spends_coinbase=*/false, /*sigops_cost=*/4, lp)); + auto changeset = m_node.mempool->GetChangeSet(); + changeset->StageAddition(ptx, /*fee=*/(total_in - num_outputs * amount_per_output), + /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, + /*spends_coinbase=*/false, /*sigops_cost=*/4, lp); + changeset->Apply(); } --num_transactions; } @@ -569,9 +578,13 @@ void TestChain100Setup::MockMempoolMinFee(const CFeeRate& target_feerate) // The new mempool min feerate is equal to the removed package's feerate + incremental feerate. const auto tx_fee = target_feerate.GetFee(GetVirtualTransactionSize(*tx)) - m_node.mempool->m_opts.incremental_relay_feerate.GetFee(GetVirtualTransactionSize(*tx)); - m_node.mempool->addUnchecked(CTxMemPoolEntry(tx, /*fee=*/tx_fee, - /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, - /*spends_coinbase=*/true, /*sigops_cost=*/1, lp)); + { + auto changeset = m_node.mempool->GetChangeSet(); + changeset->StageAddition(tx, /*fee=*/tx_fee, + /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, + /*spends_coinbase=*/true, /*sigops_cost=*/1, lp); + changeset->Apply(); + } m_node.mempool->TrimToSize(0); assert(m_node.mempool->GetMinFee() == target_feerate); } diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 30d4280fa5..b0f7bdade2 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -45,6 +45,9 @@ extern const std::function<std::string()> G_TEST_GET_FULL_NAME; static constexpr CAmount CENT{1000000}; +/** Register common test args. Shared across binaries that rely on the test framework. */ +void SetupCommonTestArgs(ArgsManager& argsman); + struct TestOpts { std::vector<const char*> extra_args{}; bool coins_db_in_memory{true}; @@ -75,6 +78,23 @@ struct BasicTestingSetup { fs::path m_path_root; fs::path m_path_lock; bool m_has_custom_datadir{false}; + /** @brief Test-specific arguments and settings. + * + * This member is intended to be the primary source of settings for code + * being tested by unit tests. It exists to make tests more self-contained + * and reduce reliance on global state. + * + * Usage guidelines: + * 1. Prefer using m_args where possible in test code. + * 2. If m_args is not accessible, use m_node.args as a fallback. + * 3. Avoid direct references to gArgs in test code. + * + * Note: Currently, m_node.args points to gArgs for backwards + * compatibility. In the future, it will point to m_args to further isolate + * test environments. + * + * @see https://github.com/bitcoin/bitcoin/issues/25055 for additional context. + */ ArgsManager m_args; }; @@ -274,11 +294,9 @@ std::ostream& operator<<(std::ostream& os, const uint256& num); class HasReason { public: - explicit HasReason(const std::string& reason) : m_reason(reason) {} - bool operator()(const std::exception& e) const - { - return std::string(e.what()).find(m_reason) != std::string::npos; - }; + explicit HasReason(std::string_view reason) : m_reason(reason) {} + bool operator()(std::string_view s) const { return s.find(m_reason) != std::string_view::npos; } + bool operator()(const std::exception& e) const { return (*this)(e.what()); } private: const std::string m_reason; diff --git a/src/test/util/transaction_utils.cpp b/src/test/util/transaction_utils.cpp index 5727da4444..a588e61944 100644 --- a/src/test/util/transaction_utils.cpp +++ b/src/test/util/transaction_utils.cpp @@ -90,3 +90,24 @@ void BulkTransaction(CMutableTransaction& tx, int32_t target_weight) assert(GetTransactionWeight(CTransaction(tx)) >= target_weight); assert(GetTransactionWeight(CTransaction(tx)) <= target_weight + 3); } + +bool SignSignature(const SigningProvider &provider, const CScript& fromPubKey, CMutableTransaction& txTo, unsigned int nIn, const CAmount& amount, int nHashType, SignatureData& sig_data) +{ + assert(nIn < txTo.vin.size()); + + MutableTransactionSignatureCreator creator(txTo, nIn, amount, nHashType); + + bool ret = ProduceSignature(provider, creator, fromPubKey, sig_data); + UpdateInput(txTo.vin.at(nIn), sig_data); + return ret; +} + +bool SignSignature(const SigningProvider &provider, const CTransaction& txFrom, CMutableTransaction& txTo, unsigned int nIn, int nHashType, SignatureData& sig_data) +{ + assert(nIn < txTo.vin.size()); + const CTxIn& txin = txTo.vin[nIn]; + assert(txin.prevout.n < txFrom.vout.size()); + const CTxOut& txout = txFrom.vout[txin.prevout.n]; + + return SignSignature(provider, txout.scriptPubKey, txTo, nIn, txout.nValue, nHashType, sig_data); +} diff --git a/src/test/util/transaction_utils.h b/src/test/util/transaction_utils.h index 80f2d1acbf..4a18ab6ab4 100644 --- a/src/test/util/transaction_utils.h +++ b/src/test/util/transaction_utils.h @@ -6,6 +6,7 @@ #define BITCOIN_TEST_UTIL_TRANSACTION_UTILS_H #include <primitives/transaction.h> +#include <script/sign.h> #include <array> @@ -30,4 +31,23 @@ std::vector<CMutableTransaction> SetupDummyInputs(FillableSigningProvider& keyst // by appending a single output with padded output script void BulkTransaction(CMutableTransaction& tx, int32_t target_weight); +/** + * Produce a satisfying script (scriptSig or witness). + * + * @param provider Utility containing the information necessary to solve a script. + * @param fromPubKey The script to produce a satisfaction for. + * @param txTo The spending transaction. + * @param nIn The index of the input in `txTo` referring the output being spent. + * @param amount The value of the output being spent. + * @param nHashType Signature hash type. + * @param sig_data Additional data provided to solve a script. Filled with the resulting satisfying + * script and whether the satisfaction is complete. + * + * @return True if the produced script is entirely satisfying `fromPubKey`. + **/ +bool SignSignature(const SigningProvider &provider, const CScript& fromPubKey, CMutableTransaction& txTo, + unsigned int nIn, const CAmount& amount, int nHashType, SignatureData& sig_data); +bool SignSignature(const SigningProvider &provider, const CTransaction& txFrom, CMutableTransaction& txTo, + unsigned int nIn, int nHashType, SignatureData& sig_data); + #endif // BITCOIN_TEST_UTIL_TRANSACTION_UTILS_H diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp index 9d6b4810d0..f1ca33bec7 100644 --- a/src/test/util/txmempool.cpp +++ b/src/test/util/txmempool.cpp @@ -141,6 +141,43 @@ std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, return std::nullopt; } +void CheckMempoolEphemeralInvariants(const CTxMemPool& tx_pool) +{ + LOCK(tx_pool.cs); + for (const auto& tx_info : tx_pool.infoAll()) { + const auto& entry = *Assert(tx_pool.GetEntry(tx_info.tx->GetHash())); + + std::vector<uint32_t> dust_indexes = GetDust(*tx_info.tx, tx_pool.m_opts.dust_relay_feerate); + + Assert(dust_indexes.size() < 2); + + if (dust_indexes.empty()) continue; + + // Transaction must have no base fee + Assert(entry.GetFee() == 0 && entry.GetModifiedFee() == 0); + + // Transaction has single dust; make sure it's swept or will not be mined + const auto& children = entry.GetMemPoolChildrenConst(); + + // Multiple children should never happen as non-dust-spending child + // can get mined as package + Assert(children.size() < 2); + + if (children.empty()) { + // No children and no fees; modified fees aside won't get mined so it's fine + // Happens naturally if child spend is RBF cycled away. + continue; + } + + // Only-child should be spending the dust + const auto& only_child = children.begin()->get().GetTx(); + COutPoint dust_outpoint{tx_info.tx->GetHash(), dust_indexes[0]}; + Assert(std::any_of(only_child.vin.begin(), only_child.vin.end(), [&dust_outpoint](const CTxIn& txin) { + return txin.prevout == dust_outpoint; + })); + } +} + void CheckMempoolTRUCInvariants(const CTxMemPool& tx_pool) { LOCK(tx_pool.cs); @@ -171,3 +208,13 @@ void CheckMempoolTRUCInvariants(const CTxMemPool& tx_pool) } } } + +void AddToMempool(CTxMemPool& tx_pool, const CTxMemPoolEntry& entry) +{ + LOCK2(cs_main, tx_pool.cs); + auto changeset = tx_pool.GetChangeSet(); + changeset->StageAddition(entry.GetSharedTx(), entry.GetFee(), + entry.GetTime().count(), entry.GetHeight(), entry.GetSequence(), + entry.GetSpendsCoinbase(), entry.GetSigOpCost(), entry.GetLockPoints()); + changeset->Apply(); +} diff --git a/src/test/util/txmempool.h b/src/test/util/txmempool.h index 6d41fdf87f..36caad2ae1 100644 --- a/src/test/util/txmempool.h +++ b/src/test/util/txmempool.h @@ -47,6 +47,13 @@ std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, bool expect_valid, const CTxMemPool* mempool); +/** Check that we never get into a state where an ephemeral dust + * transaction would be mined without the spend of the dust + * also being mined. This assumes standardness checks are being + * enforced. +*/ +void CheckMempoolEphemeralInvariants(const CTxMemPool& tx_pool); + /** For every transaction in tx_pool, check TRUC invariants: * - a TRUC tx's ancestor count must be within TRUC_ANCESTOR_LIMIT * - a TRUC tx's descendant count must be within TRUC_DESCENDANT_LIMIT @@ -56,4 +63,8 @@ std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, * */ void CheckMempoolTRUCInvariants(const CTxMemPool& tx_pool); +/** One-line wrapper for creating a mempool changeset with a single transaction + * and applying it. */ +void AddToMempool(CTxMemPool& tx_pool, const CTxMemPoolEntry& entry); + #endif // BITCOIN_TEST_UTIL_TXMEMPOOL_H diff --git a/src/test/util_string_tests.cpp b/src/test/util_string_tests.cpp index 8b974ffe6f..340c650fe3 100644 --- a/src/test/util_string_tests.cpp +++ b/src/test/util_string_tests.cpp @@ -5,25 +5,39 @@ #include <util/string.h> #include <boost/test/unit_test.hpp> +#include <test/util/setup_common.h> using namespace util; +using util::detail::CheckNumFormatSpecifiers; BOOST_AUTO_TEST_SUITE(util_string_tests) +template <unsigned NumArgs> +void TfmFormatZeroes(const std::string& fmt) +{ + std::apply([&](auto... args) { + (void)tfm::format(fmt, args...); + }, std::array<int, NumArgs>{}); +} + // Helper to allow compile-time sanity checks while providing the number of // args directly. Normally PassFmt<sizeof...(Args)> would be used. template <unsigned NumArgs> -inline void PassFmt(util::ConstevalFormatString<NumArgs> fmt) +void PassFmt(ConstevalFormatString<NumArgs> fmt) { - // This was already executed at compile-time, but is executed again at run-time to avoid -Wunused. - decltype(fmt)::Detail_CheckNumFormatSpecifiers(fmt.fmt); + // Execute compile-time check again at run-time to get code coverage stats + BOOST_CHECK_NO_THROW(CheckNumFormatSpecifiers<NumArgs>(fmt.fmt)); + + // If ConstevalFormatString didn't throw above, make sure tinyformat doesn't + // throw either for the same format string and parameter count combination. + // Proves that we have some extent of protection from runtime errors + // (tinyformat may still throw for some type mismatches). + BOOST_CHECK_NO_THROW(TfmFormatZeroes<NumArgs>(fmt.fmt)); } template <unsigned WrongNumArgs> -inline void FailFmtWithError(std::string_view wrong_fmt, std::string_view error) +void FailFmtWithError(const char* wrong_fmt, std::string_view error) { - using ErrType = const char*; - auto check_throw{[error](const ErrType& str) { return str == error; }}; - BOOST_CHECK_EXCEPTION(util::ConstevalFormatString<WrongNumArgs>::Detail_CheckNumFormatSpecifiers(wrong_fmt), ErrType, check_throw); + BOOST_CHECK_EXCEPTION(CheckNumFormatSpecifiers<WrongNumArgs>(wrong_fmt), const char*, HasReason{error}); } BOOST_AUTO_TEST_CASE(ConstevalFormatString_NumSpec) @@ -31,6 +45,7 @@ BOOST_AUTO_TEST_CASE(ConstevalFormatString_NumSpec) PassFmt<0>(""); PassFmt<0>("%%"); PassFmt<1>("%s"); + PassFmt<1>("%c"); PassFmt<0>("%%s"); PassFmt<0>("s%%"); PassFmt<1>("%%%s"); @@ -45,6 +60,8 @@ BOOST_AUTO_TEST_CASE(ConstevalFormatString_NumSpec) PassFmt<1>("%+2s"); PassFmt<1>("%.6i"); PassFmt<1>("%5.2f"); + PassFmt<1>("%5.f"); + PassFmt<1>("%.f"); PassFmt<1>("%#x"); PassFmt<1>("%1$5i"); PassFmt<1>("%1$-5i"); @@ -55,15 +72,27 @@ BOOST_AUTO_TEST_CASE(ConstevalFormatString_NumSpec) PassFmt<1>("%_"); PassFmt<1>("%\n"); - // The `*` specifier behavior is unsupported and can lead to runtime - // errors when used in a ConstevalFormatString. Please refer to the - // note in the ConstevalFormatString docs. - PassFmt<1>("%*c"); - PassFmt<2>("%2$*3$d"); - PassFmt<1>("%.*f"); + PassFmt<2>("%*c"); + PassFmt<2>("%+*c"); + PassFmt<2>("%.*f"); + PassFmt<3>("%*.*f"); + PassFmt<3>("%2$*3$d"); + PassFmt<3>("%2$*3$.9d"); + PassFmt<3>("%2$.*3$d"); + PassFmt<3>("%2$9.*3$d"); + PassFmt<3>("%2$+9.*3$d"); + PassFmt<4>("%3$*2$.*4$f"); + + // Make sure multiple flag characters "- 0+" are accepted + PassFmt<3>("'%- 0+*.*f'"); + PassFmt<3>("'%1$- 0+*3$.*2$f'"); auto err_mix{"Format specifiers must be all positional or all non-positional!"}; FailFmtWithError<1>("%s%1$s", err_mix); + FailFmtWithError<2>("%2$*d", err_mix); + FailFmtWithError<2>("%*2$d", err_mix); + FailFmtWithError<2>("%.*3$d", err_mix); + FailFmtWithError<2>("%2$.*d", err_mix); auto err_num{"Format specifier count must match the argument count!"}; FailFmtWithError<1>("", err_num); @@ -71,16 +100,50 @@ BOOST_AUTO_TEST_CASE(ConstevalFormatString_NumSpec) FailFmtWithError<2>("%s", err_num); FailFmtWithError<0>("%1$s", err_num); FailFmtWithError<2>("%1$s", err_num); + FailFmtWithError<1>("%*c", err_num); auto err_0_pos{"Positional format specifier must have position of at least 1"}; FailFmtWithError<1>("%$s", err_0_pos); FailFmtWithError<1>("%$", err_0_pos); FailFmtWithError<0>("%0$", err_0_pos); FailFmtWithError<0>("%0$s", err_0_pos); + FailFmtWithError<2>("%2$*$d", err_0_pos); + FailFmtWithError<2>("%2$*0$d", err_0_pos); + FailFmtWithError<3>("%3$*2$.*$f", err_0_pos); + FailFmtWithError<3>("%3$*2$.*0$f", err_0_pos); auto err_term{"Format specifier incorrectly terminated by end of string"}; FailFmtWithError<1>("%", err_term); + FailFmtWithError<1>("%9", err_term); + FailFmtWithError<1>("%9.", err_term); + FailFmtWithError<1>("%9.9", err_term); + FailFmtWithError<1>("%*", err_term); + FailFmtWithError<1>("%+*", err_term); + FailFmtWithError<1>("%.*", err_term); + FailFmtWithError<1>("%9.*", err_term); FailFmtWithError<1>("%1$", err_term); + FailFmtWithError<1>("%1$9", err_term); + FailFmtWithError<2>("%1$*2$", err_term); + FailFmtWithError<2>("%1$.*2$", err_term); + FailFmtWithError<2>("%1$9.*2$", err_term); + + // Non-parity between tinyformat and ConstevalFormatString. + // tinyformat throws but ConstevalFormatString does not. + BOOST_CHECK_EXCEPTION(tfm::format(ConstevalFormatString<1>{"%n"}, 0), tfm::format_error, + HasReason{"tinyformat: %n conversion spec not supported"}); + BOOST_CHECK_EXCEPTION(tfm::format(ConstevalFormatString<2>{"%*s"}, "hi", "hi"), tfm::format_error, + HasReason{"tinyformat: Cannot convert from argument type to integer for use as variable width or precision"}); + BOOST_CHECK_EXCEPTION(tfm::format(ConstevalFormatString<2>{"%.*s"}, "hi", "hi"), tfm::format_error, + HasReason{"tinyformat: Cannot convert from argument type to integer for use as variable width or precision"}); + + // Ensure that tinyformat throws if format string contains wrong number + // of specifiers. PassFmt relies on this to verify tinyformat successfully + // formats the strings, and will need to be updated if tinyformat is changed + // not to throw on failure. + BOOST_CHECK_EXCEPTION(TfmFormatZeroes<2>("%s"), tfm::format_error, + HasReason{"tinyformat: Not enough conversion specifiers in format string"}); + BOOST_CHECK_EXCEPTION(TfmFormatZeroes<1>("%s %s"), tfm::format_error, + HasReason{"tinyformat: Too many conversion specifiers in format string"}); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 1624fb8b5b..3f6e5b1b66 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2022 The Bitcoin Core developers +// Copyright (c) 2011-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -323,20 +323,65 @@ BOOST_AUTO_TEST_CASE(util_TrimString) BOOST_CHECK_EQUAL(TrimStringView(std::string("\x05\x04\x03\x02\x01\x00", 6), std::string("\x05\x04\x03\x02\x01\x00", 6)), ""); } +BOOST_AUTO_TEST_CASE(util_ParseISO8601DateTime) +{ + BOOST_CHECK_EQUAL(ParseISO8601DateTime("1969-12-31T23:59:59Z").value(), -1); + BOOST_CHECK_EQUAL(ParseISO8601DateTime("1970-01-01T00:00:00Z").value(), 0); + BOOST_CHECK_EQUAL(ParseISO8601DateTime("1970-01-01T00:00:01Z").value(), 1); + BOOST_CHECK_EQUAL(ParseISO8601DateTime("2000-01-01T00:00:01Z").value(), 946684801); + BOOST_CHECK_EQUAL(ParseISO8601DateTime("2011-09-30T23:36:17Z").value(), 1317425777); + BOOST_CHECK_EQUAL(ParseISO8601DateTime("2100-12-31T23:59:59Z").value(), 4133980799); + BOOST_CHECK_EQUAL(ParseISO8601DateTime("9999-12-31T23:59:59Z").value(), 253402300799); + + // Accept edge-cases, where the time overflows. They are not produced by + // FormatISO8601DateTime, so this can be changed in the future, if needed. + // For now, keep compatibility with the previous implementation. + BOOST_CHECK_EQUAL(ParseISO8601DateTime("2000-01-01T99:00:00Z").value(), 947041200); + BOOST_CHECK_EQUAL(ParseISO8601DateTime("2000-01-01T00:99:00Z").value(), 946690740); + BOOST_CHECK_EQUAL(ParseISO8601DateTime("2000-01-01T00:00:99Z").value(), 946684899); + BOOST_CHECK_EQUAL(ParseISO8601DateTime("2000-01-01T99:99:99Z").value(), 947047239); + + // Reject date overflows. + BOOST_CHECK(!ParseISO8601DateTime("2000-99-01T00:00:00Z")); + BOOST_CHECK(!ParseISO8601DateTime("2000-01-99T00:00:00Z")); + + // Reject out-of-range years + BOOST_CHECK(!ParseISO8601DateTime("32768-12-31T23:59:59Z")); + BOOST_CHECK(!ParseISO8601DateTime("32767-12-31T23:59:59Z")); + BOOST_CHECK(!ParseISO8601DateTime("32767-12-31T00:00:00Z")); + BOOST_CHECK(!ParseISO8601DateTime("999-12-31T00:00:00Z")); + + // Reject invalid format + const std::string valid{"2000-01-01T00:00:01Z"}; + BOOST_CHECK(ParseISO8601DateTime(valid).has_value()); + for (auto mut{0U}; mut < valid.size(); ++mut) { + std::string invalid{valid}; + invalid[mut] = 'a'; + BOOST_CHECK(!ParseISO8601DateTime(invalid)); + } +} + BOOST_AUTO_TEST_CASE(util_FormatISO8601DateTime) { BOOST_CHECK_EQUAL(FormatISO8601DateTime(971890963199), "32767-12-31T23:59:59Z"); BOOST_CHECK_EQUAL(FormatISO8601DateTime(971890876800), "32767-12-31T00:00:00Z"); - BOOST_CHECK_EQUAL(FormatISO8601DateTime(1317425777), "2011-09-30T23:36:17Z"); + + BOOST_CHECK_EQUAL(FormatISO8601DateTime(-1), "1969-12-31T23:59:59Z"); BOOST_CHECK_EQUAL(FormatISO8601DateTime(0), "1970-01-01T00:00:00Z"); + BOOST_CHECK_EQUAL(FormatISO8601DateTime(1), "1970-01-01T00:00:01Z"); + BOOST_CHECK_EQUAL(FormatISO8601DateTime(946684801), "2000-01-01T00:00:01Z"); + BOOST_CHECK_EQUAL(FormatISO8601DateTime(1317425777), "2011-09-30T23:36:17Z"); + BOOST_CHECK_EQUAL(FormatISO8601DateTime(4133980799), "2100-12-31T23:59:59Z"); + BOOST_CHECK_EQUAL(FormatISO8601DateTime(253402300799), "9999-12-31T23:59:59Z"); } BOOST_AUTO_TEST_CASE(util_FormatISO8601Date) { BOOST_CHECK_EQUAL(FormatISO8601Date(971890963199), "32767-12-31"); BOOST_CHECK_EQUAL(FormatISO8601Date(971890876800), "32767-12-31"); - BOOST_CHECK_EQUAL(FormatISO8601Date(1317425777), "2011-09-30"); + BOOST_CHECK_EQUAL(FormatISO8601Date(0), "1970-01-01"); + BOOST_CHECK_EQUAL(FormatISO8601Date(1317425777), "2011-09-30"); } BOOST_AUTO_TEST_CASE(util_FormatMoney) diff --git a/src/test/util_trace_tests.cpp b/src/test/util_trace_tests.cpp new file mode 100644 index 0000000000..06fbcf3b15 --- /dev/null +++ b/src/test/util_trace_tests.cpp @@ -0,0 +1,58 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +#include <util/trace.h> + +TRACEPOINT_SEMAPHORE(test, zero_args); +TRACEPOINT_SEMAPHORE(test, one_arg); +TRACEPOINT_SEMAPHORE(test, six_args); +TRACEPOINT_SEMAPHORE(test, twelve_args); +TRACEPOINT_SEMAPHORE(test, check_if_attached); +TRACEPOINT_SEMAPHORE(test, expensive_section); + +BOOST_FIXTURE_TEST_SUITE(util_trace_tests, BasicTestingSetup) + +// Tests the TRACEPOINT macro and that we can compile tracepoints with 0 to 12 args. +BOOST_AUTO_TEST_CASE(test_tracepoints) +{ + TRACEPOINT(test, zero_args); + TRACEPOINT(test, one_arg, 1); + TRACEPOINT(test, six_args, 1, 2, 3, 4, 5, 6); + TRACEPOINT(test, twelve_args, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12); + BOOST_CHECK(true); +} + +int fail_test_if_executed() +{ + BOOST_CHECK(false); + return 0; +} + +BOOST_AUTO_TEST_CASE(test_tracepoint_check_if_attached) +{ + // TRACEPOINT should check if we are attaching to the tracepoint and only then + // process arguments. This means, only if we are attached to the + // `test:check_if_attached` tracepoint, fail_test_if_executed() is executed. + // Since we don't attach to the tracepoint when running the test, it succeeds. + TRACEPOINT(test, check_if_attached, fail_test_if_executed()); + BOOST_CHECK(true); +} + +BOOST_AUTO_TEST_CASE(test_tracepoint_manual_tracepoint_active_check) +{ + // We should be able to use the TRACEPOINT_ACTIVE() macro to only + // execute an 'expensive' code section if we are attached to the + // tracepoint. + if (TRACEPOINT_ACTIVE(test, expensive_section)) { + BOOST_CHECK(false); // expensive_function() + TRACEPOINT(test, expensive_section); + } + BOOST_CHECK(true); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index f5c4204c55..5c1f195868 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -66,7 +66,8 @@ std::shared_ptr<CBlock> MinerTestingSetup::Block(const uint256& prev_hash) static uint64_t time = Params().GenesisBlock().nTime; BlockAssembler::Options options; - auto ptemplate = BlockAssembler{m_node.chainman->ActiveChainstate(), m_node.mempool.get(), options}.CreateNewBlock(CScript{} << i++ << OP_TRUE); + options.coinbase_output_script = CScript{} << i++ << OP_TRUE; + auto ptemplate = BlockAssembler{m_node.chainman->ActiveChainstate(), m_node.mempool.get(), options}.CreateNewBlock(); auto pblock = std::make_shared<CBlock>(ptemplate->block); pblock->hashPrevBlock = prev_hash; pblock->nTime = ++time; @@ -181,7 +182,7 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) bool ignored; FastRandomContext insecure; for (int i = 0; i < 1000; i++) { - auto block = blocks[insecure.randrange(blocks.size() - 1)]; + const auto& block = blocks[insecure.randrange(blocks.size() - 1)]; Assert(m_node.chainman)->ProcessNewBlock(block, true, true, &ignored); } @@ -331,7 +332,8 @@ BOOST_AUTO_TEST_CASE(witness_commitment_index) CScript pubKey; pubKey << 1 << OP_TRUE; BlockAssembler::Options options; - auto ptemplate = BlockAssembler{m_node.chainman->ActiveChainstate(), m_node.mempool.get(), options}.CreateNewBlock(pubKey); + options.coinbase_output_script = pubKey; + auto ptemplate = BlockAssembler{m_node.chainman->ActiveChainstate(), m_node.mempool.get(), options}.CreateNewBlock(); CBlock pblock = ptemplate->block; CTxOut witness; diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 30c5982b17..bf8cd819f2 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -4,6 +4,7 @@ // #include <chainparams.h> #include <consensus/validation.h> +#include <node/kernel_notifications.h> #include <random.h> #include <rpc/blockchain.h> #include <sync.h> @@ -69,14 +70,19 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) { ChainstateManager& chainman = *Assert(m_node.chainman); - uint256 curr_tip = ::g_best_block; + const auto get_notify_tip{[&]() { + LOCK(m_node.notifications->m_tip_block_mutex); + BOOST_REQUIRE(m_node.notifications->TipBlock()); + return *m_node.notifications->TipBlock(); + }}; + uint256 curr_tip = get_notify_tip(); // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can // be found. mineBlocks(10); // After adding some blocks to the tip, best block should have changed. - BOOST_CHECK(::g_best_block != curr_tip); + BOOST_CHECK(get_notify_tip() != curr_tip); // Grab block 1 from disk; we'll add it to the background chain later. std::shared_ptr<CBlock> pblockone = std::make_shared<CBlock>(); @@ -91,15 +97,15 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) // Ensure our active chain is the snapshot chainstate. BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.IsSnapshotActive())); - curr_tip = ::g_best_block; + curr_tip = get_notify_tip(); // Mine a new block on top of the activated snapshot chainstate. mineBlocks(1); // Defined in TestChain100Setup. // After adding some blocks to the snapshot tip, best block should have changed. - BOOST_CHECK(::g_best_block != curr_tip); + BOOST_CHECK(get_notify_tip() != curr_tip); - curr_tip = ::g_best_block; + curr_tip = get_notify_tip(); BOOST_CHECK_EQUAL(chainman.GetAll().size(), 2); @@ -135,10 +141,10 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) // Ensure tip is as expected BOOST_CHECK_EQUAL(background_cs.m_chain.Tip()->GetBlockHash(), pblockone->GetHash()); - // g_best_block should be unchanged after adding a block to the background + // get_notify_tip() should be unchanged after adding a block to the background // validation chain. BOOST_CHECK(block_added); - BOOST_CHECK_EQUAL(curr_tip, ::g_best_block); + BOOST_CHECK_EQUAL(curr_tip, get_notify_tip()); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index b4fcfbd853..6c2a825e64 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -382,7 +382,7 @@ struct SnapshotTestSetup : TestChain100Setup { LOCK(::cs_main); chainman.ResetChainstates(); BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0); - m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)); + m_node.notifications = std::make_unique<KernelNotifications>(Assert(m_node.shutdown_request), m_node.exit_status, *Assert(m_node.warnings)); const ChainstateManager::Options chainman_opts{ .chainparams = ::Params(), .datadir = chainman.m_options.datadir, @@ -397,7 +397,7 @@ struct SnapshotTestSetup : TestChain100Setup { // For robustness, ensure the old manager is destroyed before creating a // new one. m_node.chainman.reset(); - m_node.chainman = std::make_unique<ChainstateManager>(*Assert(m_node.shutdown), chainman_opts, blockman_opts); + m_node.chainman = std::make_unique<ChainstateManager>(*Assert(m_node.shutdown_signal), chainman_opts, blockman_opts); } return *Assert(m_node.chainman); } diff --git a/src/tinyformat.h b/src/tinyformat.h index f536306375..0d0e9149cc 100644 --- a/src/tinyformat.h +++ b/src/tinyformat.h @@ -145,6 +145,7 @@ namespace tfm = tinyformat; #include <iostream> #include <sstream> #include <stdexcept> // Added for Bitcoin Core +#include <util/string.h> // Added for Bitcoin Core #ifndef TINYFORMAT_ASSERT # include <cassert> @@ -178,6 +179,18 @@ namespace tfm = tinyformat; namespace tinyformat { +// Added for Bitcoin Core. Wrapper for checking format strings at compile time. +// Unlike ConstevalFormatString this supports std::string for runtime string +// formatting without compile time checks. +template <unsigned num_params> +struct FormatStringCheck { + consteval FormatStringCheck(const char* str) : fmt{util::ConstevalFormatString<num_params>{str}.fmt} {} + FormatStringCheck(const std::string& str) : fmt{str.c_str()} {} + FormatStringCheck(util::ConstevalFormatString<num_params> str) : fmt{str.fmt} {} + operator const char*() { return fmt; } + const char* fmt; +}; + // Added for Bitcoin Core class format_error: public std::runtime_error { @@ -1056,7 +1069,7 @@ inline void vformat(std::ostream& out, const char* fmt, FormatListRef list) /// Format list of arguments to the stream according to given format string. template<typename... Args> -void format(std::ostream& out, const char* fmt, const Args&... args) +void format(std::ostream& out, FormatStringCheck<sizeof...(Args)> fmt, const Args&... args) { vformat(out, fmt, makeFormatList(args...)); } @@ -1064,7 +1077,7 @@ void format(std::ostream& out, const char* fmt, const Args&... args) /// Format list of arguments according to the given format string and return /// the result as a string. template<typename... Args> -std::string format(const char* fmt, const Args&... args) +std::string format(FormatStringCheck<sizeof...(Args)> fmt, const Args&... args) { std::ostringstream oss; format(oss, fmt, args...); @@ -1073,13 +1086,13 @@ std::string format(const char* fmt, const Args&... args) /// Format list of arguments to std::cout, according to the given format string template<typename... Args> -void printf(const char* fmt, const Args&... args) +void printf(FormatStringCheck<sizeof...(Args)> fmt, const Args&... args) { format(std::cout, fmt, args...); } template<typename... Args> -void printfln(const char* fmt, const Args&... args) +void printfln(FormatStringCheck<sizeof...(Args)> fmt, const Args&... args) { format(std::cout, fmt, args...); std::cout << '\n'; @@ -1145,15 +1158,6 @@ TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_FORMAT_FUNCS) #endif -// Added for Bitcoin Core -template<typename... Args> -std::string format(const std::string &fmt, const Args&... args) -{ - std::ostringstream oss; - format(oss, fmt.c_str(), args...); - return oss.str(); -} - } // namespace tinyformat // Added for Bitcoin Core: diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 58fc1bdf2a..407d6085aa 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -356,7 +356,7 @@ void TorController::get_socks_cb(TorControlConnection& _conn, const TorControlRe std::string socks_location; if (reply.code == 250) { for (const auto& line : reply.lines) { - if (0 == line.compare(0, 20, "net/listeners/socks=")) { + if (line.starts_with("net/listeners/socks=")) { const std::string port_list_str = line.substr(20); std::vector<std::string> port_list = SplitString(port_list_str, ' '); @@ -367,7 +367,7 @@ void TorController::get_socks_cb(TorControlConnection& _conn, const TorControlRe if (portstr.empty()) continue; } socks_location = portstr; - if (0 == portstr.compare(0, 10, "127.0.0.1:")) { + if (portstr.starts_with("127.0.0.1:")) { // Prefer localhost - ignore other ports break; } @@ -711,9 +711,9 @@ void StopTorControl() } } -CService DefaultOnionServiceTarget() +CService DefaultOnionServiceTarget(uint16_t port) { struct in_addr onion_service_target; onion_service_target.s_addr = htonl(INADDR_LOOPBACK); - return {onion_service_target, BaseParams().OnionServiceTargetPort()}; + return {onion_service_target, port}; } diff --git a/src/torcontrol.h b/src/torcontrol.h index 4a0eef223e..0b66201cf1 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -27,7 +27,7 @@ void StartTorControl(CService onion_service_target); void InterruptTorControl(); void StopTorControl(); -CService DefaultOnionServiceTarget(); +CService DefaultOnionServiceTarget(uint16_t port); /** Reply from Tor, can be single or multi-line */ class TorControlReply diff --git a/src/txdb.cpp b/src/txdb.cpp index 9b43a2b03e..1622039d63 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -65,8 +65,10 @@ void CCoinsViewDB::ResizeCache(size_t new_cache_size) } } -bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { - return m_db->Read(CoinEntry(&outpoint), coin); +std::optional<Coin> CCoinsViewDB::GetCoin(const COutPoint& outpoint) const +{ + if (Coin coin; m_db->Read(CoinEntry(&outpoint), coin)) return coin; + return std::nullopt; } bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const { diff --git a/src/txdb.h b/src/txdb.h index e0acb09e98..565b060dbe 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -25,8 +25,6 @@ class uint256; static const int64_t nDefaultDbCache = 450; //! -dbbatchsize default (bytes) static const int64_t nDefaultDbBatchSize = 16 << 20; -//! max. -dbcache (MiB) -static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 16384 : 1024; //! min. -dbcache (MiB) static const int64_t nMinDbCache = 4; //! Max memory allocated to block tree DB specific cache, if no -txindex (MiB) @@ -59,7 +57,7 @@ protected: public: explicit CCoinsViewDB(DBParams db_params, CoinsViewOptions options); - bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + std::optional<Coin> GetCoin(const COutPoint& outpoint) const override; bool HaveCoin(const COutPoint &outpoint) const override; uint256 GetBestBlock() const override; std::vector<uint256> GetHeadBlocks() const override; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index f8f5ec0360..3a5a3fb306 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -34,6 +34,9 @@ #include <string_view> #include <utility> +TRACEPOINT_SEMAPHORE(mempool, added); +TRACEPOINT_SEMAPHORE(mempool, removed); + bool TestLockPointValidity(CChain& active_chain, const LockPoints& lp) { AssertLockHeld(cs_main); @@ -354,7 +357,7 @@ void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, b // of ancestors reachable via GetMemPoolParents()/GetMemPoolChildren() // will be the same as the set of ancestors whose packages include this // transaction, because when we add a new transaction to the mempool in - // addUnchecked(), we assume it has no children, and in the case of a + // addNewTransaction(), we assume it has no children, and in the case of a // reorg where that assumption is false, the in-mempool children aren't // linked to the in-block tx's until UpdateTransactionsFromBlock() is // called. @@ -429,21 +432,51 @@ void CTxMemPool::AddTransactionsUpdated(unsigned int n) nTransactionsUpdated += n; } -void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAncestors) +void CTxMemPool::Apply(ChangeSet* changeset) { - // Add to memory pool without checking anything. - // Used by AcceptToMemoryPool(), which DOES do - // all the appropriate checks. - indexed_transaction_set::iterator newit = mapTx.emplace(CTxMemPoolEntry::ExplicitCopy, entry).first; + AssertLockHeld(cs); + RemoveStaged(changeset->m_to_remove, false, MemPoolRemovalReason::REPLACED); + + for (size_t i=0; i<changeset->m_entry_vec.size(); ++i) { + auto tx_entry = changeset->m_entry_vec[i]; + std::optional<CTxMemPool::setEntries> ancestors; + if (i == 0) { + // Note: ChangeSet::CalculateMemPoolAncestors() will return a + // cached value if mempool ancestors for this transaction were + // previously calculated. + // We can only use a cached ancestor calculation for the first + // transaction in a package, because in-package parents won't be + // present in the cached ancestor sets of in-package children. + // We pass in Limits::NoLimits() to ensure that this function won't fail + // (we're going to be applying this set of transactions whether or + // not the mempool policy limits are being respected). + ancestors = *Assume(changeset->CalculateMemPoolAncestors(tx_entry, Limits::NoLimits())); + } + // First splice this entry into mapTx. + auto node_handle = changeset->m_to_add.extract(tx_entry); + auto result = mapTx.insert(std::move(node_handle)); - // Update transaction for any feeDelta created by PrioritiseTransaction - CAmount delta{0}; - ApplyDelta(entry.GetTx().GetHash(), delta); - // The following call to UpdateModifiedFee assumes no previous fee modifications - Assume(entry.GetFee() == entry.GetModifiedFee()); - if (delta) { - mapTx.modify(newit, [&delta](CTxMemPoolEntry& e) { e.UpdateModifiedFee(delta); }); + Assume(result.inserted); + txiter it = result.position; + + // Now update the entry for ancestors/descendants. + if (ancestors.has_value()) { + addNewTransaction(it, *ancestors); + } else { + addNewTransaction(it); + } } +} + +void CTxMemPool::addNewTransaction(CTxMemPool::txiter it) +{ + auto ancestors{AssumeCalculateMemPoolAncestors(__func__, *it, Limits::NoLimits())}; + return addNewTransaction(it, ancestors); +} + +void CTxMemPool::addNewTransaction(CTxMemPool::txiter newit, CTxMemPool::setEntries& setAncestors) +{ + const CTxMemPoolEntry& entry = *newit; // Update cachedInnerUsage to include contained transaction's usage. // (When we update the entry for in-mempool parents, memory usage will be @@ -465,7 +498,7 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAnces // Update ancestors with information about this tx for (const auto& pit : GetIterSet(setParentTransactions)) { - UpdateParent(newit, pit, true); + UpdateParent(newit, pit, true); } UpdateAncestorsOf(true, newit, setAncestors); UpdateEntryForAncestors(newit, setAncestors); @@ -477,7 +510,7 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAnces txns_randomized.emplace_back(newit->GetSharedTx()); newit->idx_randomized = txns_randomized.size() - 1; - TRACE3(mempool, added, + TRACEPOINT(mempool, added, entry.GetTx().GetHash().data(), entry.GetTxSize(), entry.GetFee() @@ -497,7 +530,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) // notification. m_opts.signals->TransactionRemovedFromMempool(it->GetSharedTx(), reason, mempool_sequence); } - TRACE5(mempool, removed, + TRACEPOINT(mempool, removed, it->GetTx().GetHash().data(), RemovalReasonToString(reason).c_str(), it->GetTxSize(), @@ -563,6 +596,7 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, MemPoolRemovalReaso { // Remove transaction from memory pool AssertLockHeld(cs); + Assume(!m_have_changeset); setEntries txToRemove; txiter origit = mapTx.find(origTx.GetHash()); if (origit != mapTx.end()) { @@ -594,6 +628,7 @@ void CTxMemPool::removeForReorg(CChain& chain, std::function<bool(txiter)> check // Remove transactions spending a coinbase which are now immature and no-longer-final transactions AssertLockHeld(cs); AssertLockHeld(::cs_main); + Assume(!m_have_changeset); setEntries txToRemove; for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { @@ -632,6 +667,7 @@ void CTxMemPool::removeConflicts(const CTransaction &tx) void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigned int nBlockHeight) { AssertLockHeld(cs); + Assume(!m_have_changeset); std::vector<RemovedMempoolTransactionInfo> txs_removed_for_block; txs_removed_for_block.reserve(vtx.size()); for (const auto& tx : vtx) @@ -988,12 +1024,12 @@ bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } -bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { +std::optional<Coin> CCoinsViewMemPool::GetCoin(const COutPoint& outpoint) const +{ // Check to see if the inputs are made available by another tx in the package. // These Coins would not be available in the underlying CoinsView. if (auto it = m_temp_added.find(outpoint); it != m_temp_added.end()) { - coin = it->second; - return true; + return it->second; } // If an entry in the mempool exists, always return that one, as it's guaranteed to never @@ -1002,14 +1038,13 @@ bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { CTransactionRef ptx = mempool.get(outpoint.hash); if (ptx) { if (outpoint.n < ptx->vout.size()) { - coin = Coin(ptx->vout[outpoint.n], MEMPOOL_HEIGHT, false); + Coin coin(ptx->vout[outpoint.n], MEMPOOL_HEIGHT, false); m_non_base_coins.emplace(outpoint); - return true; - } else { - return false; + return coin; } + return std::nullopt; } - return base->GetCoin(outpoint, coin); + return base->GetCoin(outpoint); } void CCoinsViewMemPool::PackageAddTransaction(const CTransactionRef& tx) @@ -1051,6 +1086,7 @@ void CTxMemPool::RemoveStaged(setEntries &stage, bool updateDescendants, MemPool int CTxMemPool::Expire(std::chrono::seconds time) { AssertLockHeld(cs); + Assume(!m_have_changeset); indexed_transaction_set::index<entry_time>::type::iterator it = mapTx.get<entry_time>().begin(); setEntries toremove; while (it != mapTx.get<entry_time>().end() && it->GetTime() < time) { @@ -1065,12 +1101,6 @@ int CTxMemPool::Expire(std::chrono::seconds time) return stage.size(); } -void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry) -{ - auto ancestors{AssumeCalculateMemPoolAncestors(__func__, entry, Limits::NoLimits())}; - return addUnchecked(entry, ancestors); -} - void CTxMemPool::UpdateChild(txiter entry, txiter child, bool add) { AssertLockHeld(cs); @@ -1127,6 +1157,7 @@ void CTxMemPool::trackPackageRemoved(const CFeeRate& rate) { void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<COutPoint>* pvNoSpendsRemaining) { AssertLockHeld(cs); + Assume(!m_have_changeset); unsigned nTxnRemoved = 0; CFeeRate maxFeeRateRemoved(0); @@ -1281,11 +1312,15 @@ std::optional<std::string> CTxMemPool::CheckConflictTopology(const setEntries& d return std::nullopt; } -util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CTxMemPool::CalculateChunksForRBF(CAmount replacement_fees, int64_t replacement_vsize, const setEntries& direct_conflicts, const setEntries& all_conflicts) +util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CTxMemPool::ChangeSet::CalculateChunksForRBF() { - Assume(replacement_vsize > 0); + LOCK(m_pool->cs); + FeeFrac replacement_feerate{0, 0}; + for (auto it : m_entry_vec) { + replacement_feerate += {it->GetModifiedFee(), it->GetTxSize()}; + } - auto err_string{CheckConflictTopology(direct_conflicts)}; + auto err_string{m_pool->CheckConflictTopology(m_to_remove)}; if (err_string.has_value()) { // Unsupported topology for calculating a feerate diagram return util::Error{Untranslated(err_string.value())}; @@ -1307,7 +1342,7 @@ util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CTxMemPool:: // they have a strict topology of 1 or two connected transactions. // OLD: Compute existing chunks from all affected clusters - for (auto txiter : all_conflicts) { + for (auto txiter : m_to_remove) { // Does this transaction have descendants? if (txiter->GetCountWithDescendants() > 1) { // Consider this tx when we consider the descendant. @@ -1347,13 +1382,13 @@ util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CTxMemPool:: */ // OLD - CON: Add any parents of direct conflicts that are not conflicted themselves - for (auto direct_conflict : direct_conflicts) { + for (auto direct_conflict : m_to_remove) { // If a direct conflict has an ancestor that is not in all_conflicts, // it can be affected by the replacement of the child. if (direct_conflict->GetMemPoolParentsConst().size() > 0) { // Grab the parent. const CTxMemPoolEntry& parent = direct_conflict->GetMemPoolParentsConst().begin()->get(); - if (!all_conflicts.count(mapTx.iterator_to(parent))) { + if (!m_to_remove.contains(m_pool->mapTx.iterator_to(parent))) { // This transaction would be left over, so add to the NEW // diagram. new_chunks.emplace_back(parent.GetModifiedFee(), parent.GetTxSize()); @@ -1361,9 +1396,32 @@ util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CTxMemPool:: } } // + CNK: Add the proposed chunk itself - new_chunks.emplace_back(replacement_fees, int32_t(replacement_vsize)); + new_chunks.emplace_back(replacement_feerate); // No topology restrictions post-chunking; sort std::sort(new_chunks.begin(), new_chunks.end(), std::greater()); return std::make_pair(old_chunks, new_chunks); } + +CTxMemPool::ChangeSet::TxHandle CTxMemPool::ChangeSet::StageAddition(const CTransactionRef& tx, const CAmount fee, int64_t time, unsigned int entry_height, uint64_t entry_sequence, bool spends_coinbase, int64_t sigops_cost, LockPoints lp) +{ + LOCK(m_pool->cs); + Assume(m_to_add.find(tx->GetHash()) == m_to_add.end()); + auto newit = m_to_add.emplace(tx, fee, time, entry_height, entry_sequence, spends_coinbase, sigops_cost, lp).first; + CAmount delta{0}; + m_pool->ApplyDelta(tx->GetHash(), delta); + if (delta) m_to_add.modify(newit, [&delta](CTxMemPoolEntry& e) { e.UpdateModifiedFee(delta); }); + + m_entry_vec.push_back(newit); + return newit; +} + +void CTxMemPool::ChangeSet::Apply() +{ + LOCK(m_pool->cs); + m_pool->Apply(this); + m_to_add.clear(); + m_to_remove.clear(); + m_entry_vec.clear(); + m_ancestors.clear(); +} diff --git a/src/txmempool.h b/src/txmempool.h index d0cb41a078..10acb2aa22 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -260,13 +260,13 @@ struct TxMempoolInfo * * Usually when a new transaction is added to the mempool, it has no in-mempool * children (because any such children would be an orphan). So in - * addUnchecked(), we: - * - update a new entry's setMemPoolParents to include all in-mempool parents - * - update the new entry's direct parents to include the new tx as a child + * addNewTransaction(), we: + * - update a new entry's m_parents to include all in-mempool parents + * - update each of those parent entries to include the new tx as a child * - update all ancestors of the transaction to include the new tx's size/fee * * When a transaction is removed from the mempool, we must: - * - update all in-mempool parents to not track the tx in setMemPoolChildren + * - update all in-mempool parents to not track the tx in their m_children * - update all ancestors to not include the tx's size/fees in descendant state * - update all in-mempool children to not include it as a parent * @@ -283,7 +283,7 @@ struct TxMempoolInfo * unreachable from just looking at transactions in the mempool (the linking * transactions may also be in the disconnected block, waiting to be added). * Because of this, there's not much benefit in trying to search for in-mempool - * children in addUnchecked(). Instead, in the special case of transactions + * children in addNewTransaction(). Instead, in the special case of transactions * being added from a disconnected block, we require the caller to clean up the * state, to account for in-mempool, out-of-block descendants for all the * in-block transactions by calling UpdateTransactionsFromBlock(). Note that @@ -453,15 +453,6 @@ public: */ void check(const CCoinsViewCache& active_coins_tip, int64_t spendheight) const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - // addUnchecked must updated state for all ancestors of a given transaction, - // to track size/count of descendant transactions. First version of - // addUnchecked can be used to have it call CalculateMemPoolAncestors(), and - // then invoke the second version. - // Note that addUnchecked is ONLY called from ATMP outside of tests - // and any other callers may break wallet's in-mempool tracking (due to - // lack of CValidationInterface::TransactionAddedToMempool callbacks). - void addUnchecked(const CTxMemPoolEntry& entry) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); - void addUnchecked(const CTxMemPoolEntry& entry, setEntries& setAncestors) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); void removeRecursive(const CTransaction& tx, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); /** After reorg, filter the entries that would no longer be valid in the next block, and update @@ -519,15 +510,6 @@ public: * don't actually exist in the mempool, returns an empty vector. */ std::vector<txiter> GetIterVec(const std::vector<uint256>& txids) const EXCLUSIVE_LOCKS_REQUIRED(cs); - /** Remove a set of transactions from the mempool. - * If a transaction is in this set, then all in-mempool descendants must - * also be in the set, unless this transaction is being removed for being - * in a block. - * Set updateDescendants to true when removing a tx that was in a block, so - * that any in-mempool descendants have their ancestor state updated. - */ - void RemoveStaged(setEntries& stage, bool updateDescendants, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); - /** UpdateTransactionsFromBlock is called when adding transactions from a * disconnected block back to the mempool, new mempool entries may have * children in the mempool (which is generally not the case when otherwise @@ -724,29 +706,21 @@ public: return m_sequence_number; } - /** - * Calculate the sorted chunks for the old and new mempool relating to the - * clusters that would be affected by a potential replacement transaction. - * (replacement_fees, replacement_vsize) values are gathered from a - * proposed set of replacement transactions that are considered as a single - * chunk, and represent their complete cluster. In other words, they have no - * in-mempool ancestors. - * - * @param[in] replacement_fees Package fees - * @param[in] replacement_vsize Package size (must be greater than 0) - * @param[in] direct_conflicts All transactions that would be removed directly by - * having a conflicting input with a proposed transaction - * @param[in] all_conflicts All transactions that would be removed - * @return old and new diagram pair respectively, or an error string if the conflicts don't match a calculable topology - */ - util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CalculateChunksForRBF(CAmount replacement_fees, int64_t replacement_vsize, const setEntries& direct_conflicts, const setEntries& all_conflicts) EXCLUSIVE_LOCKS_REQUIRED(cs); - /* Check that all direct conflicts are in a cluster size of two or less. Each * direct conflict may be in a separate cluster. */ std::optional<std::string> CheckConflictTopology(const setEntries& direct_conflicts); private: + /** Remove a set of transactions from the mempool. + * If a transaction is in this set, then all in-mempool descendants must + * also be in the set, unless this transaction is being removed for being + * in a block. + * Set updateDescendants to true when removing a tx that was in a block, so + * that any in-mempool descendants have their ancestor state updated. + */ + void RemoveStaged(setEntries& stage, bool updateDescendants, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); + /** UpdateForDescendants is used by UpdateTransactionsFromBlock to update * the descendants for a single transaction that has been added to the * mempool but may have child transactions in the mempool, eg during a @@ -791,7 +765,7 @@ private: /** Before calling removeUnchecked for a given transaction, * UpdateForRemoveFromMempool must be called on the entire (dependent) set * of transactions being removed at the same time. We use each - * CTxMemPoolEntry's setMemPoolParents in order to walk ancestors of a + * CTxMemPoolEntry's m_parents in order to walk ancestors of a * given transaction that is removed, so we can't remove intermediate * transactions in a chain before we've updated all the state for the * removal. @@ -816,6 +790,122 @@ public: assert(m_epoch.guarded()); // verify guard even when it==nullopt return !it || visited(*it); } + + /* + * CTxMemPool::ChangeSet: + * + * This class is used for all mempool additions and associated removals (eg + * due to rbf). Removals that don't need to be evaluated for acceptance, + * such as removing transactions that appear in a block, or due to reorg, + * or removals related to mempool limiting or expiry do not need to use + * this. + * + * Callers can interleave calls to StageAddition()/StageRemoval(), and + * removals may be invoked in any order, but additions must be done in a + * topological order in the case of transaction packages (ie, parents must + * be added before children). + * + * CalculateChunksForRBF() can be used to calculate the feerate diagram of + * the proposed set of new transactions and compare with the existing + * mempool. + * + * CalculateMemPoolAncestors() calculates the in-mempool (not including + * what is in the change set itself) ancestors of a given transaction. + * + * Apply() will apply the removals and additions that are staged into the + * mempool. + * + * Only one changeset may exist at a time. While a changeset is + * outstanding, no removals or additions may be made directly to the + * mempool. + */ + class ChangeSet { + public: + explicit ChangeSet(CTxMemPool* pool) : m_pool(pool) {} + ~ChangeSet() EXCLUSIVE_LOCKS_REQUIRED(m_pool->cs) { m_pool->m_have_changeset = false; } + + ChangeSet(const ChangeSet&) = delete; + ChangeSet& operator=(const ChangeSet&) = delete; + + using TxHandle = CTxMemPool::txiter; + + TxHandle StageAddition(const CTransactionRef& tx, const CAmount fee, int64_t time, unsigned int entry_height, uint64_t entry_sequence, bool spends_coinbase, int64_t sigops_cost, LockPoints lp); + void StageRemoval(CTxMemPool::txiter it) { m_to_remove.insert(it); } + + const CTxMemPool::setEntries& GetRemovals() const { return m_to_remove; } + + util::Result<CTxMemPool::setEntries> CalculateMemPoolAncestors(TxHandle tx, const Limits& limits) + { + // Look up transaction in our cache first + auto it = m_ancestors.find(tx); + if (it != m_ancestors.end()) return it->second; + + // If not found, try to have the mempool calculate it, and cache + // for later. + LOCK(m_pool->cs); + auto ret{m_pool->CalculateMemPoolAncestors(*tx, limits)}; + if (ret) m_ancestors.try_emplace(tx, *ret); + return ret; + } + + std::vector<CTransactionRef> GetAddedTxns() const { + std::vector<CTransactionRef> ret; + ret.reserve(m_entry_vec.size()); + for (const auto& entry : m_entry_vec) { + ret.emplace_back(entry->GetSharedTx()); + } + return ret; + } + + /** + * Calculate the sorted chunks for the old and new mempool relating to the + * clusters that would be affected by a potential replacement transaction. + * + * @return old and new diagram pair respectively, or an error string if the conflicts don't match a calculable topology + */ + util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CalculateChunksForRBF(); + + size_t GetTxCount() const { return m_entry_vec.size(); } + const CTransaction& GetAddedTxn(size_t index) const { return m_entry_vec.at(index)->GetTx(); } + + void Apply() EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + private: + CTxMemPool* m_pool; + CTxMemPool::indexed_transaction_set m_to_add; + std::vector<CTxMemPool::txiter> m_entry_vec; // track the added transactions' insertion order + // map from the m_to_add index to the ancestors for the transaction + std::map<CTxMemPool::txiter, CTxMemPool::setEntries, CompareIteratorByHash> m_ancestors; + CTxMemPool::setEntries m_to_remove; + + friend class CTxMemPool; + }; + + std::unique_ptr<ChangeSet> GetChangeSet() EXCLUSIVE_LOCKS_REQUIRED(cs) { + Assume(!m_have_changeset); + m_have_changeset = true; + return std::make_unique<ChangeSet>(this); + } + + bool m_have_changeset GUARDED_BY(cs){false}; + + friend class CTxMemPool::ChangeSet; + +private: + // Apply the given changeset to the mempool, by removing transactions in + // the to_remove set and adding transactions in the to_add set. + void Apply(CTxMemPool::ChangeSet* changeset) EXCLUSIVE_LOCKS_REQUIRED(cs); + + // addNewTransaction must update state for all ancestors of a given transaction, + // to track size/count of descendant transactions. First version of + // addNewTransaction can be used to have it call CalculateMemPoolAncestors(), and + // then invoke the second version. + // Note that addNewTransaction is ONLY called (via Apply()) from ATMP + // outside of tests and any other callers may break wallet's in-mempool + // tracking (due to lack of CValidationInterface::TransactionAddedToMempool + // callbacks). + void addNewTransaction(CTxMemPool::txiter it) EXCLUSIVE_LOCKS_REQUIRED(cs); + void addNewTransaction(CTxMemPool::txiter it, CTxMemPool::setEntries& setAncestors) EXCLUSIVE_LOCKS_REQUIRED(cs); }; /** @@ -851,7 +941,7 @@ public: CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn); /** GetCoin, returning whether it exists and is not spent. Also updates m_non_base_coins if the * coin is not fetched from base. */ - bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + std::optional<Coin> GetCoin(const COutPoint& outpoint) const override; /** Add the coins created by this transaction. These coins are only temporarily stored in * m_temp_added and cannot be flushed to the back end. Only used for package validation. */ void PackageAddTransaction(const CTransactionRef& tx); diff --git a/src/txorphanage.cpp b/src/txorphanage.cpp index 35a215c88a..ba4ba6c3b6 100644 --- a/src/txorphanage.cpp +++ b/src/txorphanage.cpp @@ -33,7 +33,7 @@ bool TxOrphanage::AddTx(const CTransactionRef& tx, NodeId peer) return false; } - auto ret = m_orphans.emplace(wtxid, OrphanTx{tx, peer, Now<NodeSeconds>() + ORPHAN_TX_EXPIRE_TIME, m_orphan_list.size()}); + auto ret = m_orphans.emplace(wtxid, OrphanTx{{tx, peer, Now<NodeSeconds>() + ORPHAN_TX_EXPIRE_TIME}, m_orphan_list.size()}); assert(ret.second); m_orphan_list.push_back(ret.first); for (const CTxIn& txin : tx->vin) { @@ -277,3 +277,13 @@ std::vector<std::pair<CTransactionRef, NodeId>> TxOrphanage::GetChildrenFromDiff } return children_found; } + +std::vector<TxOrphanage::OrphanTxBase> TxOrphanage::GetOrphanTransactions() const +{ + std::vector<OrphanTxBase> ret; + ret.reserve(m_orphans.size()); + for (auto const& o : m_orphans) { + ret.push_back({o.second.tx, o.second.fromPeer, o.second.nTimeExpire}); + } + return ret; +} diff --git a/src/txorphanage.h b/src/txorphanage.h index 2c53d1d40f..b1fae3fa00 100644 --- a/src/txorphanage.h +++ b/src/txorphanage.h @@ -67,16 +67,22 @@ public: std::vector<std::pair<CTransactionRef, NodeId>> GetChildrenFromDifferentPeer(const CTransactionRef& parent, NodeId nodeid) const; /** Return how many entries exist in the orphange */ - size_t Size() + size_t Size() const { return m_orphans.size(); } -protected: - struct OrphanTx { + /** Allows providing orphan information externally */ + struct OrphanTxBase { CTransactionRef tx; NodeId fromPeer; NodeSeconds nTimeExpire; + }; + + std::vector<OrphanTxBase> GetOrphanTransactions() const; + +protected: + struct OrphanTx : public OrphanTxBase { size_t list_pos; }; diff --git a/src/uint256.h b/src/uint256.h index 8223787041..708f0c2ff1 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -69,16 +69,27 @@ public: /** @name Hex representation * - * The reverse-byte hex representation is a convenient way to view the blob - * as a number, because it is consistent with the way the base_uint class - * converts blobs to numbers. + * The hex representation used by GetHex(), ToString(), FromHex() and + * SetHexDeprecated() is unusual, since it shows bytes of the base_blob in + * reverse order. For example, a 4-byte blob {0x12, 0x34, 0x56, 0x78} is + * represented as "78563412" instead of the more typical "12345678" + * representation that would be shown in a hex editor or used by typical + * byte-array / hex conversion functions like python's bytes.hex() and + * bytes.fromhex(). + * + * The nice thing about the reverse-byte representation, even though it is + * unusual, is that if a blob contains an arithmetic number in little endian + * format (with least significant bytes first, and most significant bytes + * last), the GetHex() output will match the way the number would normally + * be written in base-16 (with most significant digits first and least + * significant digits last). + * + * This means, for example, that ArithToUint256(num).GetHex() can be used to + * display an arith_uint256 num value as a number, because + * ArithToUint256() converts the number to a blob in little-endian format, + * so the arith_uint256 class doesn't need to have its own number parsing + * and formatting functions. * - * @note base_uint treats the blob as an array of bytes with the numerically - * least significant byte first and the most significant byte last. Because - * numbers are typically written with the most significant digit first and - * the least significant digit last, the reverse hex display of the blob - * corresponds to the same numeric value that base_uint interprets from the - * blob. * @{*/ std::string GetHex() const; /** Unlike FromHex this accepts any invalid input, thus it is fragile and deprecated! diff --git a/src/univalue/CMakeLists.txt b/src/univalue/CMakeLists.txt index 96733fe077..c31e82cadc 100644 --- a/src/univalue/CMakeLists.txt +++ b/src/univalue/CMakeLists.txt @@ -15,10 +15,119 @@ target_include_directories(univalue target_link_libraries(univalue PRIVATE core_interface) if(BUILD_TESTS) - add_executable(unitester test/unitester.cpp) - target_compile_definitions(unitester - PRIVATE - JSON_TEST_SRC=\"${CMAKE_CURRENT_SOURCE_DIR}/test\" + include(GenerateHeaders) + generate_header_from_json(test/fail1.json) + generate_header_from_json(test/fail10.json) + generate_header_from_json(test/fail11.json) + generate_header_from_json(test/fail12.json) + generate_header_from_json(test/fail13.json) + generate_header_from_json(test/fail14.json) + generate_header_from_json(test/fail15.json) + generate_header_from_json(test/fail16.json) + generate_header_from_json(test/fail17.json) + generate_header_from_json(test/fail18.json) + generate_header_from_json(test/fail19.json) + generate_header_from_json(test/fail2.json) + generate_header_from_json(test/fail20.json) + generate_header_from_json(test/fail21.json) + generate_header_from_json(test/fail22.json) + generate_header_from_json(test/fail23.json) + generate_header_from_json(test/fail24.json) + generate_header_from_json(test/fail25.json) + generate_header_from_json(test/fail26.json) + generate_header_from_json(test/fail27.json) + generate_header_from_json(test/fail28.json) + generate_header_from_json(test/fail29.json) + generate_header_from_json(test/fail3.json) + generate_header_from_json(test/fail30.json) + generate_header_from_json(test/fail31.json) + generate_header_from_json(test/fail32.json) + generate_header_from_json(test/fail33.json) + generate_header_from_json(test/fail34.json) + generate_header_from_json(test/fail35.json) + generate_header_from_json(test/fail36.json) + generate_header_from_json(test/fail37.json) + generate_header_from_json(test/fail38.json) + generate_header_from_json(test/fail39.json) + generate_header_from_json(test/fail4.json) + generate_header_from_json(test/fail40.json) + generate_header_from_json(test/fail41.json) + generate_header_from_json(test/fail42.json) + generate_header_from_json(test/fail44.json) + generate_header_from_json(test/fail45.json) + generate_header_from_json(test/fail5.json) + generate_header_from_json(test/fail6.json) + generate_header_from_json(test/fail7.json) + generate_header_from_json(test/fail8.json) + generate_header_from_json(test/fail9.json) + generate_header_from_json(test/pass1.json) + generate_header_from_json(test/pass2.json) + generate_header_from_json(test/pass3.json) + generate_header_from_json(test/pass4.json) + generate_header_from_json(test/round1.json) + generate_header_from_json(test/round2.json) + generate_header_from_json(test/round3.json) + generate_header_from_json(test/round4.json) + generate_header_from_json(test/round5.json) + generate_header_from_json(test/round6.json) + generate_header_from_json(test/round7.json) + add_executable(unitester + ${CMAKE_CURRENT_BINARY_DIR}/test/fail1.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail10.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail11.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail12.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail13.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail14.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail15.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail16.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail17.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail18.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail19.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail2.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail20.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail21.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail22.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail23.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail24.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail25.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail26.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail27.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail28.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail29.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail3.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail30.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail31.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail32.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail33.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail34.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail35.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail36.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail37.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail38.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail39.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail4.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail40.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail41.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail42.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail44.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail45.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail5.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail6.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail7.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail8.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/fail9.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/pass1.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/pass2.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/pass3.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/pass4.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/round1.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/round2.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/round3.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/round4.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/round5.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/round6.json.h + ${CMAKE_CURRENT_BINARY_DIR}/test/round7.json.h + test/unitester.cpp ) target_link_libraries(unitester PRIVATE diff --git a/src/univalue/test/fail18.json b/src/univalue/test/fail18.json index edac92716f..a72bec7e18 100644 --- a/src/univalue/test/fail18.json +++ b/src/univalue/test/fail18.json @@ -1 +1 @@ -[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]
\ No newline at end of file +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] diff --git a/src/univalue/test/pass2.json b/src/univalue/test/pass2.json index d3c63c7ad8..624387bf33 100644 --- a/src/univalue/test/pass2.json +++ b/src/univalue/test/pass2.json @@ -1 +1 @@ -[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]]
\ No newline at end of file +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] diff --git a/src/univalue/test/unitester.cpp b/src/univalue/test/unitester.cpp index d27e647e5c..e517e64188 100644 --- a/src/univalue/test/unitester.cpp +++ b/src/univalue/test/unitester.cpp @@ -1,130 +1,159 @@ // Copyright 2014 BitPay Inc. -// Distributed under the MIT/X11 software license, see the accompanying +// Copyright (c) 2015-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying // file COPYING or https://opensource.org/licenses/mit-license.php. #include <univalue.h> +#include <univalue/test/fail1.json.h> +#include <univalue/test/fail10.json.h> +#include <univalue/test/fail11.json.h> +#include <univalue/test/fail12.json.h> +#include <univalue/test/fail13.json.h> +#include <univalue/test/fail14.json.h> +#include <univalue/test/fail15.json.h> +#include <univalue/test/fail16.json.h> +#include <univalue/test/fail17.json.h> +#include <univalue/test/fail18.json.h> +#include <univalue/test/fail19.json.h> +#include <univalue/test/fail2.json.h> +#include <univalue/test/fail20.json.h> +#include <univalue/test/fail21.json.h> +#include <univalue/test/fail22.json.h> +#include <univalue/test/fail23.json.h> +#include <univalue/test/fail24.json.h> +#include <univalue/test/fail25.json.h> +#include <univalue/test/fail26.json.h> +#include <univalue/test/fail27.json.h> +#include <univalue/test/fail28.json.h> +#include <univalue/test/fail29.json.h> +#include <univalue/test/fail3.json.h> +#include <univalue/test/fail30.json.h> +#include <univalue/test/fail31.json.h> +#include <univalue/test/fail32.json.h> +#include <univalue/test/fail33.json.h> +#include <univalue/test/fail34.json.h> +#include <univalue/test/fail35.json.h> +#include <univalue/test/fail36.json.h> +#include <univalue/test/fail37.json.h> +#include <univalue/test/fail38.json.h> +#include <univalue/test/fail39.json.h> +#include <univalue/test/fail4.json.h> +#include <univalue/test/fail40.json.h> +#include <univalue/test/fail41.json.h> +#include <univalue/test/fail42.json.h> +#include <univalue/test/fail44.json.h> +#include <univalue/test/fail45.json.h> +#include <univalue/test/fail5.json.h> +#include <univalue/test/fail6.json.h> +#include <univalue/test/fail7.json.h> +#include <univalue/test/fail8.json.h> +#include <univalue/test/fail9.json.h> +#include <univalue/test/pass1.json.h> +#include <univalue/test/pass2.json.h> +#include <univalue/test/pass3.json.h> +#include <univalue/test/pass4.json.h> +#include <univalue/test/round1.json.h> +#include <univalue/test/round2.json.h> +#include <univalue/test/round3.json.h> +#include <univalue/test/round4.json.h> +#include <univalue/test/round5.json.h> +#include <univalue/test/round6.json.h> +#include <univalue/test/round7.json.h> + +#include <array> #include <cassert> #include <cstdio> #include <string> -#ifndef JSON_TEST_SRC -#error JSON_TEST_SRC must point to test source directory -#endif - -std::string srcdir(JSON_TEST_SRC); - static std::string rtrim(std::string s) { - s.erase(s.find_last_not_of(" \n\r\t")+1); + s.erase(s.find_last_not_of(" \n\r\t") + 1); return s; } static void runtest(std::string filename, const std::string& jdata) { - std::string prefix = filename.substr(0, 4); - - bool wantPass = (prefix == "pass") || (prefix == "roun"); - bool wantFail = (prefix == "fail"); - bool wantRoundTrip = (prefix == "roun"); - assert(wantPass || wantFail); - - UniValue val; - bool testResult = val.read(jdata); - - if (wantPass) { - assert(testResult == true); - } else { - assert(testResult == false); - } - - if (wantRoundTrip) { - std::string odata = val.write(0, 0); - assert(odata == rtrim(jdata)); - } -} - -static void runtest_file(const char *filename_) -{ - std::string basename(filename_); - std::string filename = srcdir + "/" + basename; - FILE *f = fopen(filename.c_str(), "r"); - assert(f != nullptr); + std::string prefix = filename.substr(0, 4); - std::string jdata; + bool wantPass = (prefix == "pass") || (prefix == "roun"); + bool wantFail = (prefix == "fail"); + bool wantRoundTrip = (prefix == "roun"); + assert(wantPass || wantFail); - char buf[4096]; - while (!feof(f)) { - int bread = fread(buf, 1, sizeof(buf), f); - assert(!ferror(f)); - - std::string s(buf, bread); - jdata += s; - } + UniValue val; + bool testResult = val.read(jdata); - assert(!ferror(f)); - fclose(f); + if (wantPass) { + assert(testResult == true); + } else { + assert(testResult == false); + } - runtest(basename, jdata); + if (wantRoundTrip) { + std::string odata = val.write(0, 0); + assert(odata == rtrim(jdata)); + } } -static const char *filenames[] = { - "fail10.json", - "fail11.json", - "fail12.json", - "fail13.json", - "fail14.json", - "fail15.json", - "fail16.json", - "fail17.json", - //"fail18.json", // investigate - "fail19.json", - "fail1.json", - "fail20.json", - "fail21.json", - "fail22.json", - "fail23.json", - "fail24.json", - "fail25.json", - "fail26.json", - "fail27.json", - "fail28.json", - "fail29.json", - "fail2.json", - "fail30.json", - "fail31.json", - "fail32.json", - "fail33.json", - "fail34.json", - "fail35.json", - "fail36.json", - "fail37.json", - "fail38.json", // invalid unicode: only first half of surrogate pair - "fail39.json", // invalid unicode: only second half of surrogate pair - "fail40.json", // invalid unicode: broken UTF-8 - "fail41.json", // invalid unicode: unfinished UTF-8 - "fail42.json", // valid json with garbage following a nul byte - "fail44.json", // unterminated string - "fail45.json", // nested beyond max depth - "fail3.json", - "fail4.json", // extra comma - "fail5.json", - "fail6.json", - "fail7.json", - "fail8.json", - "fail9.json", // extra comma - "pass1.json", - "pass2.json", - "pass3.json", - "pass4.json", - "round1.json", // round-trip test - "round2.json", // unicode - "round3.json", // bare string - "round4.json", // bare number - "round5.json", // bare true - "round6.json", // bare false - "round7.json", // bare null -}; +#define TEST_FILE(name) {#name, json_tests::name} +inline constexpr std::array tests{std::to_array<std::tuple<std::string_view, std::string_view>>({ + TEST_FILE(fail1), + TEST_FILE(fail10), + TEST_FILE(fail11), + TEST_FILE(fail12), + TEST_FILE(fail13), + TEST_FILE(fail14), + TEST_FILE(fail15), + TEST_FILE(fail16), + TEST_FILE(fail17), + TEST_FILE(fail18), + TEST_FILE(fail19), + TEST_FILE(fail2), + TEST_FILE(fail20), + TEST_FILE(fail21), + TEST_FILE(fail22), + TEST_FILE(fail23), + TEST_FILE(fail24), + TEST_FILE(fail25), + TEST_FILE(fail26), + TEST_FILE(fail27), + TEST_FILE(fail28), + TEST_FILE(fail29), + TEST_FILE(fail3), + TEST_FILE(fail30), + TEST_FILE(fail31), + TEST_FILE(fail32), + TEST_FILE(fail33), + TEST_FILE(fail34), + TEST_FILE(fail35), + TEST_FILE(fail36), + TEST_FILE(fail37), + TEST_FILE(fail38), // invalid unicode: only first half of surrogate pair + TEST_FILE(fail39), // invalid unicode: only second half of surrogate pair + TEST_FILE(fail4), // extra comma + TEST_FILE(fail40), // invalid unicode: broken UTF-8 + TEST_FILE(fail41), // invalid unicode: unfinished UTF-8 + TEST_FILE(fail42), // valid json with garbage following a nul byte + TEST_FILE(fail44), // unterminated string + TEST_FILE(fail45), // nested beyond max depth + TEST_FILE(fail5), + TEST_FILE(fail6), + TEST_FILE(fail7), + TEST_FILE(fail8), + TEST_FILE(fail9), // extra comma + TEST_FILE(pass1), + TEST_FILE(pass2), + TEST_FILE(pass3), + TEST_FILE(pass4), + TEST_FILE(round1), // round-trip test + TEST_FILE(round2), // unicode + TEST_FILE(round3), // bare string + TEST_FILE(round4), // bare number + TEST_FILE(round5), // bare true + TEST_FILE(round6), // bare false + TEST_FILE(round7), // bare null +})}; // Test \u handling void unescape_unicode_test() @@ -156,10 +185,10 @@ void no_nul_test() assert(val.read({buf + 3, 7})); } -int main (int argc, char *argv[]) +int main(int argc, char* argv[]) { - for (const auto& f: filenames) { - runtest_file(f); + for (const auto& [file, json] : tests) { + runtest(std::string{file}, std::string{json}); } unescape_unicode_test(); @@ -167,4 +196,3 @@ int main (int argc, char *argv[]) return 0; } - diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 26c6271f9b..4999dbf13f 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -42,4 +42,5 @@ target_link_libraries(bitcoin_util bitcoin_clientversion bitcoin_crypto $<$<PLATFORM_ID:Windows>:ws2_32> + $<$<PLATFORM_ID:Windows>:iphlpapi> ) diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index f50cd8a28c..04b0673c49 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -203,10 +203,10 @@ std::vector<bool> DecodeAsmap(fs::path path) LogPrintf("Failed to open asmap file from disk\n"); return bits; } - fseek(filestr, 0, SEEK_END); - int length = ftell(filestr); + file.seek(0, SEEK_END); + int length = file.tell(); LogPrintf("Opened asmap file %s (%d bytes) from disk\n", fs::quoted(fs::PathToString(path)), length); - fseek(filestr, 0, SEEK_SET); + file.seek(0, SEEK_SET); uint8_t cur_byte; for (int i = 0; i < length; ++i) { file >> cur_byte; diff --git a/src/util/bitset.h b/src/util/bitset.h index 6f9e808c37..17ebc709eb 100644 --- a/src/util/bitset.h +++ b/src/util/bitset.h @@ -366,7 +366,7 @@ public: if (count) { unsigned i = 0; while (count > LIMB_BITS) { - ret.m_val[i++] = ~I{0}; + ret.m_val[i++] = I(~I{0}); count -= LIMB_BITS; } ret.m_val[i] = I(~I{0}) >> (LIMB_BITS - count); diff --git a/src/util/check.cpp b/src/util/check.cpp index eb3885832b..1430c0e8e2 100644 --- a/src/util/check.cpp +++ b/src/util/check.cpp @@ -4,7 +4,7 @@ #include <util/check.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <clientversion.h> #include <tinyformat.h> @@ -19,7 +19,7 @@ std::string StrFormatInternalBug(std::string_view msg, std::string_view file, in return strprintf("Internal bug detected: %s\n%s:%d (%s)\n" "%s %s\n" "Please report this issue here: %s\n", - msg, file, line, func, PACKAGE_NAME, FormatFullVersion(), PACKAGE_BUGREPORT); + msg, file, line, func, CLIENT_NAME, FormatFullVersion(), CLIENT_BUGREPORT); } NonFatalCheckError::NonFatalCheckError(std::string_view msg, std::string_view file, int line, std::string_view func) diff --git a/src/util/check.h b/src/util/check.h index a02a1de8dc..efc78915a9 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -13,6 +13,14 @@ #include <string_view> #include <utility> +constexpr bool G_FUZZING{ +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + true +#else + false +#endif +}; + std::string StrFormatInternalBug(std::string_view msg, std::string_view file, int line, std::string_view func); class NonFatalCheckError : public std::runtime_error @@ -40,11 +48,11 @@ void assertion_fail(std::string_view file, int line, std::string_view func, std: /** Helper for Assert()/Assume() */ template <bool IS_ASSERT, typename T> -T&& inline_assertion_check(LIFETIMEBOUND T&& val, [[maybe_unused]] const char* file, [[maybe_unused]] int line, [[maybe_unused]] const char* func, [[maybe_unused]] const char* assertion) +constexpr T&& inline_assertion_check(LIFETIMEBOUND T&& val, [[maybe_unused]] const char* file, [[maybe_unused]] int line, [[maybe_unused]] const char* func, [[maybe_unused]] const char* assertion) { - if constexpr (IS_ASSERT + if (IS_ASSERT || std::is_constant_evaluated() || G_FUZZING #ifdef ABORT_ON_FAILED_ASSUME - || true + || true #endif ) { if (!val) { diff --git a/src/util/feefrac.h b/src/util/feefrac.h index 9772162010..161322b50a 100644 --- a/src/util/feefrac.h +++ b/src/util/feefrac.h @@ -64,13 +64,13 @@ struct FeeFrac int32_t size; /** Construct an IsEmpty() FeeFrac. */ - inline FeeFrac() noexcept : fee{0}, size{0} {} + constexpr inline FeeFrac() noexcept : fee{0}, size{0} {} /** Construct a FeeFrac with specified fee and size. */ - inline FeeFrac(int64_t f, int32_t s) noexcept : fee{f}, size{s} {} + constexpr inline FeeFrac(int64_t f, int32_t s) noexcept : fee{f}, size{s} {} - inline FeeFrac(const FeeFrac&) noexcept = default; - inline FeeFrac& operator=(const FeeFrac&) noexcept = default; + constexpr inline FeeFrac(const FeeFrac&) noexcept = default; + constexpr inline FeeFrac& operator=(const FeeFrac&) noexcept = default; /** Check if this is empty (size and fee are 0). */ bool inline IsEmpty() const noexcept { diff --git a/src/util/fs_helpers.cpp b/src/util/fs_helpers.cpp index 41c8fe3b8f..4d06afe144 100644 --- a/src/util/fs_helpers.cpp +++ b/src/util/fs_helpers.cpp @@ -5,7 +5,7 @@ #include <util/fs_helpers.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <logging.h> #include <sync.h> @@ -22,7 +22,7 @@ #include <utility> #ifndef WIN32 -// for posix_fallocate, in configure.ac we check if it is present after this +// for posix_fallocate, in cmake/introspection.cmake we check if it is present after this #ifdef __linux__ #ifdef _POSIX_C_SOURCE @@ -117,7 +117,7 @@ bool FileCommit(FILE* file) LogPrintf("FlushFileBuffers failed: %s\n", Win32ErrorString(GetLastError())); return false; } -#elif defined(MAC_OSX) && defined(F_FULLFSYNC) +#elif defined(__APPLE__) && defined(F_FULLFSYNC) if (fcntl(fileno(file), F_FULLFSYNC, 0) == -1) { // Manpage says "value other than -1" is returned on success LogPrintf("fcntl F_FULLFSYNC failed: %s\n", SysErrorString(errno)); return false; @@ -195,7 +195,7 @@ void AllocateFileRange(FILE* file, unsigned int offset, unsigned int length) nFileSize.u.HighPart = nEndPos >> 32; SetFilePointerEx(hFile, nFileSize, 0, FILE_BEGIN); SetEndOfFile(hFile); -#elif defined(MAC_OSX) +#elif defined(__APPLE__) // OSX specific version // NOTE: Contrary to other OS versions, the OSX version assumes that // NOTE: offset is the size of the file. diff --git a/src/util/hasher.h b/src/util/hasher.h index 3ad6d5bb94..e4594c7dda 100644 --- a/src/util/hasher.h +++ b/src/util/hasher.h @@ -8,13 +8,12 @@ #include <crypto/common.h> #include <crypto/siphash.h> #include <primitives/transaction.h> +#include <span.h> #include <uint256.h> #include <cstdint> #include <cstring> -template <typename C> class Span; - class SaltedTxidHasher { private: diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 1543de03ab..d0809162fa 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -377,6 +377,24 @@ consteval uint8_t ConstevalHexDigit(const char c) throw "Only lowercase hex digits are allowed, for consistency"; } +namespace detail { +template <size_t N> +struct Hex { + std::array<std::byte, N / 2> bytes{}; + consteval Hex(const char (&hex_str)[N]) + // 2 hex digits required per byte + implicit null terminator + requires(N % 2 == 1) + { + if (hex_str[N - 1]) throw "null terminator required"; + for (std::size_t i = 0; i < bytes.size(); ++i) { + bytes[i] = static_cast<std::byte>( + (ConstevalHexDigit(hex_str[2 * i]) << 4) | + ConstevalHexDigit(hex_str[2 * i + 1])); + } + } +}; +} // namespace detail + /** * ""_hex is a compile-time user-defined literal returning a * `std::array<std::byte>`, equivalent to ParseHex(). Variants provided: @@ -407,25 +425,6 @@ consteval uint8_t ConstevalHexDigit(const char c) * time/runtime barrier. */ inline namespace hex_literals { -namespace detail { - -template <size_t N> -struct Hex { - std::array<std::byte, N / 2> bytes{}; - consteval Hex(const char (&hex_str)[N]) - // 2 hex digits required per byte + implicit null terminator - requires(N % 2 == 1) - { - if (hex_str[N - 1]) throw "null terminator required"; - for (std::size_t i = 0; i < bytes.size(); ++i) { - bytes[i] = static_cast<std::byte>( - (ConstevalHexDigit(hex_str[2 * i]) << 4) | - ConstevalHexDigit(hex_str[2 * i + 1])); - } - } -}; - -} // namespace detail template <util::detail::Hex str> constexpr auto operator""_hex() { return str.bytes; } diff --git a/src/util/string.h b/src/util/string.h index c5183d6c80..b523e6ef4e 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -6,7 +6,6 @@ #define BITCOIN_UTIL_STRING_H #include <span.h> -#include <tinyformat.h> #include <array> #include <cstdint> @@ -18,65 +17,81 @@ #include <vector> namespace util { -/** - * @brief A wrapper for a compile-time partially validated format string - * - * This struct can be used to enforce partial compile-time validation of format - * strings, to reduce the likelihood of tinyformat throwing exceptions at - * run-time. Validation is partial to try and prevent the most common errors - * while avoiding re-implementing the entire parsing logic. - * - * @note Counting of `*` dynamic width and precision fields (such as `%*c`, - * `%2$*3$d`, `%.*f`) is not implemented to minimize code complexity as long as - * they are not used in the codebase. Usage of these fields is not counted and - * can lead to run-time exceptions. Code wanting to use the `*` specifier can - * side-step this struct and call tinyformat directly. - */ +namespace detail { template <unsigned num_params> -struct ConstevalFormatString { - const char* const fmt; - consteval ConstevalFormatString(const char* str) : fmt{str} { Detail_CheckNumFormatSpecifiers(fmt); } - constexpr static void Detail_CheckNumFormatSpecifiers(std::string_view str) - { - unsigned count_normal{0}; // Number of "normal" specifiers, like %s - unsigned count_pos{0}; // Max number in positional specifier, like %8$s - for (auto it{str.begin()}; it < str.end();) { - if (*it != '%') { - ++it; - continue; - } - - if (++it >= str.end()) throw "Format specifier incorrectly terminated by end of string"; - if (*it == '%') { - // Percent escape: %% - ++it; - continue; - } +constexpr static void CheckNumFormatSpecifiers(const char* str) +{ + unsigned count_normal{0}; // Number of "normal" specifiers, like %s + unsigned count_pos{0}; // Max number in positional specifier, like %8$s + for (auto it{str}; *it != '\0'; ++it) { + if (*it != '%' || *++it == '%') continue; // Skip escaped %% + auto add_arg = [&] { unsigned maybe_num{0}; while ('0' <= *it && *it <= '9') { maybe_num *= 10; maybe_num += *it - '0'; ++it; - }; + } if (*it == '$') { + ++it; // Positional specifier, like %8$s if (maybe_num == 0) throw "Positional format specifier must have position of at least 1"; count_pos = std::max(count_pos, maybe_num); - if (++it >= str.end()) throw "Format specifier incorrectly terminated by end of string"; } else { // Non-positional specifier, like %s ++count_normal; + } + }; + + // Increase argument count and consume positional specifier, if present. + add_arg(); + + // Consume flags. + while (*it == '#' || *it == '0' || *it == '-' || *it == ' ' || *it == '+') ++it; + + auto parse_size = [&] { + if (*it == '*') { ++it; + add_arg(); + } else { + while ('0' <= *it && *it <= '9') ++it; } - // The remainder "[flags][width][.precision][length]type" of the - // specifier is not checked. Parsing continues with the next '%'. + }; + + // Consume dynamic or static width value. + parse_size(); + + // Consume dynamic or static precision value. + if (*it == '.') { + ++it; + parse_size(); } - if (count_normal && count_pos) throw "Format specifiers must be all positional or all non-positional!"; - unsigned count{count_normal | count_pos}; - if (num_params != count) throw "Format specifier count must match the argument count!"; + + if (*it == '\0') throw "Format specifier incorrectly terminated by end of string"; + + // Length and type in "[flags][width][.precision][length]type" + // is not checked. Parsing continues with the next '%'. } + if (count_normal && count_pos) throw "Format specifiers must be all positional or all non-positional!"; + unsigned count{count_normal | count_pos}; + if (num_params != count) throw "Format specifier count must match the argument count!"; +} +} // namespace detail + +/** + * @brief A wrapper for a compile-time partially validated format string + * + * This struct can be used to enforce partial compile-time validation of format + * strings, to reduce the likelihood of tinyformat throwing exceptions at + * run-time. Validation is partial to try and prevent the most common errors + * while avoiding re-implementing the entire parsing logic. + */ +template <unsigned num_params> +struct ConstevalFormatString { + const char* const fmt; + consteval ConstevalFormatString(const char* str) : fmt{str} { detail::CheckNumFormatSpecifiers<num_params>(fmt); } }; void ReplaceAll(std::string& in_out, const std::string& search, const std::string& substitute); @@ -235,12 +250,4 @@ template <typename T1, size_t PREFIX_LEN> } } // namespace util -namespace tinyformat { -template <typename... Args> -std::string format(util::ConstevalFormatString<sizeof...(Args)> fmt, const Args&... args) -{ - return format(fmt.fmt, args...); -} -} // namespace tinyformat - #endif // BITCOIN_UTIL_STRING_H diff --git a/src/util/syserror.cpp b/src/util/syserror.cpp index 6f3a724483..a902826f8e 100644 --- a/src/util/syserror.cpp +++ b/src/util/syserror.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <tinyformat.h> #include <util/syserror.h> diff --git a/src/util/threadnames.cpp b/src/util/threadnames.cpp index 0249de37e3..032572fb53 100644 --- a/src/util/threadnames.cpp +++ b/src/util/threadnames.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <cstring> #include <string> @@ -29,7 +29,7 @@ static void SetThreadName(const char* name) ::prctl(PR_SET_NAME, name, 0, 0, 0); #elif (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) pthread_set_name_np(pthread_self(), name); -#elif defined(MAC_OSX) +#elif defined(__APPLE__) pthread_setname_np(name); #else // Prevent warnings for unused parameters... diff --git a/src/util/time.cpp b/src/util/time.cpp index f08eb5300a..e20f30a474 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,10 +8,13 @@ #include <compat/compat.h> #include <tinyformat.h> #include <util/check.h> +#include <util/strencodings.h> #include <atomic> #include <chrono> +#include <optional> #include <string> +#include <string_view> #include <thread> void UninterruptibleSleep(const std::chrono::microseconds& n) { std::this_thread::sleep_for(n); } @@ -60,6 +63,30 @@ std::string FormatISO8601Date(int64_t nTime) return strprintf("%04i-%02u-%02u", signed{ymd.year()}, unsigned{ymd.month()}, unsigned{ymd.day()}); } +std::optional<int64_t> ParseISO8601DateTime(std::string_view str) +{ + constexpr auto FMT_SIZE{std::string_view{"2000-01-01T01:01:01Z"}.size()}; + if (str.size() != FMT_SIZE || str[4] != '-' || str[7] != '-' || str[10] != 'T' || str[13] != ':' || str[16] != ':' || str[19] != 'Z') { + return {}; + } + const auto year{ToIntegral<uint16_t>(str.substr(0, 4))}; + const auto month{ToIntegral<uint8_t>(str.substr(5, 2))}; + const auto day{ToIntegral<uint8_t>(str.substr(8, 2))}; + const auto hour{ToIntegral<uint8_t>(str.substr(11, 2))}; + const auto min{ToIntegral<uint8_t>(str.substr(14, 2))}; + const auto sec{ToIntegral<uint8_t>(str.substr(17, 2))}; + if (!year || !month || !day || !hour || !min || !sec) { + return {}; + } + const std::chrono::year_month_day ymd{std::chrono::year{*year}, std::chrono::month{*month}, std::chrono::day{*day}}; + if (!ymd.ok()) { + return {}; + } + const auto time{std::chrono::hours{*hour} + std::chrono::minutes{*min} + std::chrono::seconds{*sec}}; + const auto tp{std::chrono::sys_days{ymd} + time}; + return int64_t{TicksSinceEpoch<std::chrono::seconds>(tp)}; +} + struct timeval MillisToTimeval(int64_t nTimeout) { struct timeval timeout; diff --git a/src/util/time.h b/src/util/time.h index 108560e0e0..27cbe50581 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,9 @@ #include <chrono> // IWYU pragma: export #include <cstdint> +#include <optional> #include <string> +#include <string_view> using namespace std::chrono_literals; @@ -105,6 +107,7 @@ T GetTime() */ std::string FormatISO8601DateTime(int64_t nTime); std::string FormatISO8601Date(int64_t nTime); +std::optional<int64_t> ParseISO8601DateTime(std::string_view str); /** * Convert milliseconds to a struct timeval for e.g. select. diff --git a/src/util/tokenpipe.cpp b/src/util/tokenpipe.cpp index 16fbb664ea..9425c62ebf 100644 --- a/src/util/tokenpipe.cpp +++ b/src/util/tokenpipe.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/tokenpipe.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #ifndef WIN32 diff --git a/src/util/trace.h b/src/util/trace.h index d9ed65e3aa..3deefeade3 100644 --- a/src/util/trace.h +++ b/src/util/trace.h @@ -5,43 +5,50 @@ #ifndef BITCOIN_UTIL_TRACE_H #define BITCOIN_UTIL_TRACE_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #ifdef ENABLE_TRACING +// Setting SDT_USE_VARIADIC lets systemtap (sys/sdt.h) know that we want to use +// the optional variadic macros to define tracepoints. +#define SDT_USE_VARIADIC 1 + +// Setting _SDT_HAS_SEMAPHORES let's systemtap (sys/sdt.h) know that we want to +// use the optional semaphore feature for our tracepoints. This feature allows +// us to check if something is attached to a tracepoint. We only want to prepare +// some potentially expensive tracepoint arguments, if the tracepoint is being +// used. Here, an expensive argument preparation could, for example, be +// calculating a hash or serialization of a data structure. +#define _SDT_HAS_SEMAPHORES 1 + +// Used to define a counting semaphore for a tracepoint. This semaphore is +// automatically incremented by tracing frameworks (bpftrace, bcc, libbpf, ...) +// upon attaching to the tracepoint and decremented when detaching. This needs +// to be a global variable. It's placed in the '.probes' ELF section. +#define TRACEPOINT_SEMAPHORE(context, event) \ + unsigned short context##_##event##_semaphore __attribute__((section(".probes"))) + #include <sys/sdt.h> -#define TRACE(context, event) DTRACE_PROBE(context, event) -#define TRACE1(context, event, a) DTRACE_PROBE1(context, event, a) -#define TRACE2(context, event, a, b) DTRACE_PROBE2(context, event, a, b) -#define TRACE3(context, event, a, b, c) DTRACE_PROBE3(context, event, a, b, c) -#define TRACE4(context, event, a, b, c, d) DTRACE_PROBE4(context, event, a, b, c, d) -#define TRACE5(context, event, a, b, c, d, e) DTRACE_PROBE5(context, event, a, b, c, d, e) -#define TRACE6(context, event, a, b, c, d, e, f) DTRACE_PROBE6(context, event, a, b, c, d, e, f) -#define TRACE7(context, event, a, b, c, d, e, f, g) DTRACE_PROBE7(context, event, a, b, c, d, e, f, g) -#define TRACE8(context, event, a, b, c, d, e, f, g, h) DTRACE_PROBE8(context, event, a, b, c, d, e, f, g, h) -#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) DTRACE_PROBE9(context, event, a, b, c, d, e, f, g, h, i) -#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) DTRACE_PROBE10(context, event, a, b, c, d, e, f, g, h, i, j) -#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) DTRACE_PROBE11(context, event, a, b, c, d, e, f, g, h, i, j, k) -#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) DTRACE_PROBE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) +// Returns true if something is attached to the tracepoint. +#define TRACEPOINT_ACTIVE(context, event) (context##_##event##_semaphore > 0) + +// A USDT tracepoint with one to twelve arguments. It's checked that the +// tracepoint is active before preparing its arguments. +#define TRACEPOINT(context, event, ...) \ + do { \ + if (TRACEPOINT_ACTIVE(context, event)) { \ + STAP_PROBEV(context, event __VA_OPT__(, ) __VA_ARGS__); \ + } \ + } while(0) #else -#define TRACE(context, event) -#define TRACE1(context, event, a) -#define TRACE2(context, event, a, b) -#define TRACE3(context, event, a, b, c) -#define TRACE4(context, event, a, b, c, d) -#define TRACE5(context, event, a, b, c, d, e) -#define TRACE6(context, event, a, b, c, d, e, f) -#define TRACE7(context, event, a, b, c, d, e, f, g) -#define TRACE8(context, event, a, b, c, d, e, f, g, h) -#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) -#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) -#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) -#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) - -#endif +#define TRACEPOINT_SEMAPHORE(context, event) +#define TRACEPOINT_ACTIVE(context, event) false +#define TRACEPOINT(context, ...) + +#endif // ENABLE_TRACING #endif // BITCOIN_UTIL_TRACE_H diff --git a/src/util/translation.h b/src/util/translation.h index 6effe102f9..7c734a1766 100644 --- a/src/util/translation.h +++ b/src/util/translation.h @@ -10,6 +10,9 @@ #include <functional> #include <string> +/** Translate a message to the native language of the user. */ +const extern std::function<std::string(const char*)> G_TRANSLATION_FUN; + /** * Bilingual messages: * - in GUI: user's native language + untranslated (i.e. English) @@ -64,9 +67,6 @@ bilingual_str format(const bilingual_str& fmt, const Args&... args) } } // namespace tinyformat -/** Translate a message to the native language of the user. */ -const extern std::function<std::string(const char*)> G_TRANSLATION_FUN; - struct ConstevalStringLiteral { const char* const lit; consteval ConstevalStringLiteral(const char* str) : lit{str} {} diff --git a/src/validation.cpp b/src/validation.cpp index 8e4ea8eda2..3e8a7cf520 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <validation.h> @@ -32,6 +32,7 @@ #include <logging/timer.h> #include <node/blockstorage.h> #include <node/utxo_snapshot.h> +#include <policy/ephemeral_policy.h> #include <policy/policy.h> #include <policy/rbf.h> #include <policy/settings.h> @@ -87,6 +88,8 @@ using node::CBlockIndexHeightOnlyComparator; using node::CBlockIndexWorkComparator; using node::SnapshotMetadata; +/** Size threshold for warning about slow UTXO set flush to disk. */ +static constexpr size_t WARN_FLUSH_COINS_SIZE = 1 << 30; // 1 GiB /** Time to wait between writing blocks/block index to disk. */ static constexpr std::chrono::hours DATABASE_WRITE_INTERVAL{1}; /** Time to wait between flushing chainstate to disk. */ @@ -108,9 +111,10 @@ const std::vector<std::string> CHECKLEVEL_DOC { * */ static constexpr int PRUNE_LOCK_BUFFER{10}; -GlobalMutex g_best_block_mutex; -std::condition_variable g_best_block_cv; -uint256 g_best_block; +TRACEPOINT_SEMAPHORE(validation, block_connected); +TRACEPOINT_SEMAPHORE(utxocache, flush); +TRACEPOINT_SEMAPHORE(mempool, replaced); +TRACEPOINT_SEMAPHORE(mempool, rejected); const CBlockIndex* Chainstate::FindForkInGlobalIndex(const CBlockLocator& locator) const { @@ -180,18 +184,14 @@ std::optional<std::vector<int>> CalculatePrevHeights( std::vector<int> prev_heights; prev_heights.resize(tx.vin.size()); for (size_t i = 0; i < tx.vin.size(); ++i) { - const CTxIn& txin = tx.vin[i]; - Coin coin; - if (!coins.GetCoin(txin.prevout, coin)) { + if (auto coin{coins.GetCoin(tx.vin[i].prevout)}) { + prev_heights[i] = coin->nHeight == MEMPOOL_HEIGHT + ? tip.nHeight + 1 // Assume all mempool transaction confirm in the next block. + : coin->nHeight; + } else { LogPrintf("ERROR: %s: Missing input %d in transaction \'%s\'\n", __func__, i, tx.GetHash().GetHex()); return std::nullopt; } - if (coin.nHeight == MEMPOOL_HEIGHT) { - // Assume all mempool transaction confirm in the next block. - prev_heights[i] = tip.nHeight + 1; - } else { - prev_heights[i] = coin.nHeight; - } } return prev_heights; } @@ -326,7 +326,7 @@ void Chainstate::MaybeUpdateMempoolForReorg( } } - // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have + // AcceptToMemoryPool/addNewTransaction all assume that new mempool entries have // no in-mempool children, which is generally not true when adding // previously-confirmed transactions back to the mempool. // UpdateTransactionsFromBlock finds descendants of any transactions in @@ -635,10 +635,9 @@ private: CTxMemPool::setEntries m_iters_conflicting; /** All mempool ancestors of this transaction. */ CTxMemPool::setEntries m_ancestors; - /** Mempool entry constructed for this transaction. Constructed in PreChecks() but not - * inserted into the mempool until Finalize(). */ - std::unique_ptr<CTxMemPoolEntry> m_entry; - /** Whether RBF-related data structures (m_conflicts, m_iters_conflicting, m_all_conflicting, + /* Handle to the tx in the changeset */ + CTxMemPool::ChangeSet::TxHandle m_tx_handle; + /** Whether RBF-related data structures (m_conflicts, m_iters_conflicting, * m_replaced_transactions) include a sibling in addition to txns with conflicting inputs. */ bool m_sibling_eviction{false}; @@ -691,9 +690,7 @@ private: bool ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Try to add the transaction to the mempool, removing any conflicts first. - // Returns true if the transaction is in the mempool after any size - // limiting is performed, false otherwise. - bool Finalize(const ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + void FinalizeSubpackage(const ATMPArgs& args) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Submit all transactions to the mempool and call ConsensusScriptChecks to add to the script // cache - should only be called after successful validation of all transactions in the package. @@ -744,10 +741,10 @@ private: /** Whether the transaction(s) would replace any mempool transactions and/or evict any siblings. * If so, RBF rules apply. */ bool m_rbf{false}; - /** All directly conflicting mempool transactions and their descendants. */ - CTxMemPool::setEntries m_all_conflicts; /** Mempool transactions that were replaced. */ std::list<CTransactionRef> m_replaced_transactions; + /* Changeset representing adding transactions and removing their conflicts. */ + std::unique_ptr<CTxMemPool::ChangeSet> m_changeset; /** Total modified fees of mempool transactions being replaced. */ CAmount m_conflicting_fees{0}; @@ -782,7 +779,6 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Alias what we need out of ws TxValidationState& state = ws.m_state; - std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; if (!CheckTransaction(tx, state)) { return false; // state filled in by CheckTransaction @@ -824,30 +820,10 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) const CTransaction* ptxConflicting = m_pool.GetConflictTx(txin.prevout); if (ptxConflicting) { if (!args.m_allow_replacement) { - // Transaction conflicts with a mempool tx, but we're not allowing replacements. + // Transaction conflicts with a mempool tx, but we're not allowing replacements in this context. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "bip125-replacement-disallowed"); } - if (!ws.m_conflicts.count(ptxConflicting->GetHash())) - { - // Transactions that don't explicitly signal replaceability are - // *not* replaceable with the current logic, even if one of their - // unconfirmed ancestors signals replaceability. This diverges - // from BIP125's inherited signaling description (see CVE-2021-31876). - // Applications relying on first-seen mempool behavior should - // check all unconfirmed ancestors; otherwise an opt-in ancestor - // might be replaced, causing removal of this descendant. - // - // All TRUC transactions are considered replaceable. - // - // Replaceability signaling of the original transactions may be - // ignored due to node setting. - const bool allow_rbf{m_pool.m_opts.full_rbf || SignalsOptInRBF(*ptxConflicting) || ptxConflicting->version == TRUC_VERSION}; - if (!allow_rbf) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "txn-mempool-conflict"); - } - - ws.m_conflicts.insert(ptxConflicting->GetHash()); - } + ws.m_conflicts.insert(ptxConflicting->GetHash()); } } @@ -913,10 +889,6 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) int64_t nSigOpsCost = GetTransactionSigOpCost(tx, m_view, STANDARD_SCRIPT_VERIFY_FLAGS); - // ws.m_modified_fees includes any fee deltas from PrioritiseTransaction - ws.m_modified_fees = ws.m_base_fees; - m_pool.ApplyDelta(hash, ws.m_modified_fees); - // Keep track of transactions that spend a coinbase, which we re-scan // during reorgs to ensure COINBASE_MATURITY is still met. bool fSpendsCoinbase = false; @@ -931,9 +903,22 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Set entry_sequence to 0 when bypass_limits is used; this allows txs from a block // reorg to be marked earlier than any child txs that were already in the mempool. const uint64_t entry_sequence = bypass_limits ? 0 : m_pool.GetSequence(); - entry.reset(new CTxMemPoolEntry(ptx, ws.m_base_fees, nAcceptTime, m_active_chainstate.m_chain.Height(), entry_sequence, - fSpendsCoinbase, nSigOpsCost, lock_points.value())); - ws.m_vsize = entry->GetTxSize(); + if (!m_subpackage.m_changeset) { + m_subpackage.m_changeset = m_pool.GetChangeSet(); + } + ws.m_tx_handle = m_subpackage.m_changeset->StageAddition(ptx, ws.m_base_fees, nAcceptTime, m_active_chainstate.m_chain.Height(), entry_sequence, fSpendsCoinbase, nSigOpsCost, lock_points.value()); + + // ws.m_modified_fees includes any fee deltas from PrioritiseTransaction + ws.m_modified_fees = ws.m_tx_handle->GetModifiedFee(); + + ws.m_vsize = ws.m_tx_handle->GetTxSize(); + + // Enforces 0-fee for dust transactions, no incentive to be mined alone + if (m_pool.m_opts.require_standard) { + if (!PreCheckEphemeralTx(*ptx, m_pool.m_opts.dust_relay_feerate, ws.m_base_fees, ws.m_modified_fees, state)) { + return false; // state filled in by PreCheckEphemeralTx + } + } if (nSigOpsCost > MAX_STANDARD_TX_SIGOPS_COST) return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "bad-txns-too-many-sigops", @@ -998,7 +983,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) maybe_rbf_limits.descendant_size_vbytes += conflict->GetSizeWithDescendants(); } - if (auto ancestors{m_pool.CalculateMemPoolAncestors(*entry, maybe_rbf_limits)}) { + if (auto ancestors{m_subpackage.m_changeset->CalculateMemPoolAncestors(ws.m_tx_handle, maybe_rbf_limits)}) { ws.m_ancestors = std::move(*ancestors); } else { // If CalculateMemPoolAncestors fails second time, we want the original error string. @@ -1030,7 +1015,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) if (ws.m_vsize > EXTRA_DESCENDANT_TX_SIZE_LIMIT || ws.m_ptx->version == TRUC_VERSION) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", error_message); } - if (auto ancestors_retry{m_pool.CalculateMemPoolAncestors(*entry, cpfp_carve_out_limits)}) { + if (auto ancestors_retry{m_subpackage.m_changeset->CalculateMemPoolAncestors(ws.m_tx_handle, cpfp_carve_out_limits)}) { ws.m_ancestors = std::move(*ancestors_retry); } else { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", error_message); @@ -1104,13 +1089,15 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) strprintf("insufficient fee%s", ws.m_sibling_eviction ? " (including sibling eviction)" : ""), *err_string); } + CTxMemPool::setEntries all_conflicts; + // Calculate all conflicting entries and enforce Rule #5. - if (const auto err_string{GetEntriesForConflicts(tx, m_pool, ws.m_iters_conflicting, m_subpackage.m_all_conflicts)}) { + if (const auto err_string{GetEntriesForConflicts(tx, m_pool, ws.m_iters_conflicting, all_conflicts)}) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, strprintf("too many potential replacements%s", ws.m_sibling_eviction ? " (including sibling eviction)" : ""), *err_string); } // Enforce Rule #2. - if (const auto err_string{HasNoNewUnconfirmed(tx, m_pool, m_subpackage.m_all_conflicts)}) { + if (const auto err_string{HasNoNewUnconfirmed(tx, m_pool, all_conflicts)}) { // Sibling eviction is only done for TRUC transactions, which cannot have multiple ancestors. Assume(!ws.m_sibling_eviction); return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, @@ -1119,7 +1106,7 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) // Check if it's economically rational to mine this transaction rather than the ones it // replaces and pays for its own relay fees. Enforce Rules #3 and #4. - for (CTxMemPool::txiter it : m_subpackage.m_all_conflicts) { + for (CTxMemPool::txiter it : all_conflicts) { m_subpackage.m_conflicting_fees += it->GetModifiedFee(); m_subpackage.m_conflicting_size += it->GetTxSize(); } @@ -1129,6 +1116,11 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, strprintf("insufficient fee%s", ws.m_sibling_eviction ? " (including sibling eviction)" : ""), *err_string); } + + // Add all the to-be-removed transactions to the changeset. + for (auto it : all_conflicts) { + m_subpackage.m_changeset->StageRemoval(it); + } return true; } @@ -1182,13 +1174,16 @@ bool MemPoolAccept::PackageMempoolChecks(const std::vector<CTransactionRef>& txn // Don't consider replacements that would cause us to remove a large number of mempool entries. // This limit is not increased in a package RBF. Use the aggregate number of transactions. + CTxMemPool::setEntries all_conflicts; if (const auto err_string{GetEntriesForConflicts(*child_ws.m_ptx, m_pool, direct_conflict_iters, - m_subpackage.m_all_conflicts)}) { + all_conflicts)}) { return package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package RBF failed: too many potential replacements", *err_string); } - for (CTxMemPool::txiter it : m_subpackage.m_all_conflicts) { + + for (CTxMemPool::txiter it : all_conflicts) { + m_subpackage.m_changeset->StageRemoval(it); m_subpackage.m_conflicting_fees += it->GetModifiedFee(); m_subpackage.m_conflicting_size += it->GetTxSize(); } @@ -1215,14 +1210,15 @@ bool MemPoolAccept::PackageMempoolChecks(const std::vector<CTransactionRef>& txn // Check if it's economically rational to mine this package rather than the ones it replaces. // This takes the place of ReplacementChecks()'s PaysMoreThanConflicts() in the package RBF setting. - if (const auto err_tup{ImprovesFeerateDiagram(m_pool, direct_conflict_iters, m_subpackage.m_all_conflicts, m_subpackage.m_total_modified_fees, m_subpackage.m_total_vsize)}) { + if (const auto err_tup{ImprovesFeerateDiagram(*m_subpackage.m_changeset)}) { return package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package RBF failed: " + err_tup.value().second, ""); } - LogDebug(BCLog::TXPACKAGES, "package RBF checks passed: parent %s (wtxid=%s), child %s (wtxid=%s)\n", + LogDebug(BCLog::TXPACKAGES, "package RBF checks passed: parent %s (wtxid=%s), child %s (wtxid=%s), package hash (%s)\n", txns.front()->GetHash().ToString(), txns.front()->GetWitnessHash().ToString(), - txns.back()->GetHash().ToString(), txns.back()->GetWitnessHash().ToString()); + txns.back()->GetHash().ToString(), txns.back()->GetWitnessHash().ToString(), + GetPackageHash(txns).ToString()); return true; @@ -1289,58 +1285,55 @@ bool MemPoolAccept::ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws) return true; } -bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) +void MemPoolAccept::FinalizeSubpackage(const ATMPArgs& args) { AssertLockHeld(cs_main); AssertLockHeld(m_pool.cs); - const CTransaction& tx = *ws.m_ptx; - const uint256& hash = ws.m_hash; - TxValidationState& state = ws.m_state; - const bool bypass_limits = args.m_bypass_limits; - std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; - if (!m_subpackage.m_all_conflicts.empty()) Assume(args.m_allow_replacement); + if (!m_subpackage.m_changeset->GetRemovals().empty()) Assume(args.m_allow_replacement); // Remove conflicting transactions from the mempool - for (CTxMemPool::txiter it : m_subpackage.m_all_conflicts) + for (CTxMemPool::txiter it : m_subpackage.m_changeset->GetRemovals()) { - LogDebug(BCLog::MEMPOOL, "replacing mempool tx %s (wtxid=%s, fees=%s, vsize=%s). New tx %s (wtxid=%s, fees=%s, vsize=%s)\n", - it->GetTx().GetHash().ToString(), - it->GetTx().GetWitnessHash().ToString(), - it->GetFee(), - it->GetTxSize(), - hash.ToString(), - tx.GetWitnessHash().ToString(), - entry->GetFee(), - entry->GetTxSize()); - TRACE7(mempool, replaced, + std::string log_string = strprintf("replacing mempool tx %s (wtxid=%s, fees=%s, vsize=%s). ", + it->GetTx().GetHash().ToString(), + it->GetTx().GetWitnessHash().ToString(), + it->GetFee(), + it->GetTxSize()); + FeeFrac feerate{m_subpackage.m_total_modified_fees, int32_t(m_subpackage.m_total_vsize)}; + uint256 tx_or_package_hash{}; + const bool replaced_with_tx{m_subpackage.m_changeset->GetTxCount() == 1}; + if (replaced_with_tx) { + const CTransaction& tx = m_subpackage.m_changeset->GetAddedTxn(0); + tx_or_package_hash = tx.GetHash(); + log_string += strprintf("New tx %s (wtxid=%s, fees=%s, vsize=%s)", + tx.GetHash().ToString(), + tx.GetWitnessHash().ToString(), + feerate.fee, + feerate.size); + } else { + tx_or_package_hash = GetPackageHash(m_subpackage.m_changeset->GetAddedTxns()); + log_string += strprintf("New package %s with %lu txs, fees=%s, vsize=%s", + tx_or_package_hash.ToString(), + m_subpackage.m_changeset->GetTxCount(), + feerate.fee, + feerate.size); + + } + LogDebug(BCLog::MEMPOOL, "%s\n", log_string); + TRACEPOINT(mempool, replaced, it->GetTx().GetHash().data(), it->GetTxSize(), it->GetFee(), std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(it->GetTime()).count(), - hash.data(), - entry->GetTxSize(), - entry->GetFee() + tx_or_package_hash.data(), + feerate.size, + feerate.fee, + replaced_with_tx ); m_subpackage.m_replaced_transactions.push_back(it->GetSharedTx()); } - m_pool.RemoveStaged(m_subpackage.m_all_conflicts, false, MemPoolRemovalReason::REPLACED); - // Don't attempt to process the same conflicts repeatedly during subpackage evaluation: - // they no longer exist on subsequent calls to Finalize() post-RemoveStaged - m_subpackage.m_all_conflicts.clear(); - // Store transaction in memory - m_pool.addUnchecked(*entry, ws.m_ancestors); - - // trim mempool and check if tx was trimmed - // If we are validating a package, don't trim here because we could evict a previous transaction - // in the package. LimitMempoolSize() should be called at the very end to make sure the mempool - // is still within limits and package submission happens atomically. - if (!args.m_package_submission && !bypass_limits) { - LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip()); - if (!m_pool.exists(GenTxid::Txid(hash))) - // The tx no longer meets our (new) mempool minimum feerate but could be reconsidered in a package. - return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "mempool full"); - } - return true; + m_subpackage.m_changeset->Apply(); + m_subpackage.m_changeset.reset(); } bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& workspaces, @@ -1355,6 +1348,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& return !m_pool.exists(GenTxid::Txid(ws.m_ptx->GetHash())); })); bool all_submitted = true; + FinalizeSubpackage(args); // ConsensusScriptChecks adds to the script cache and is therefore consensus-critical; // CheckInputsFromMempoolAndCache asserts that transactions only spend coins available from the // mempool or UTXO set. Submit each transaction to the mempool immediately after calling @@ -1368,36 +1362,19 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& package_state.Invalid(PackageValidationResult::PCKG_MEMPOOL_ERROR, strprintf("BUG! PolicyScriptChecks succeeded but ConsensusScriptChecks failed: %s", ws.m_ptx->GetHash().ToString())); - } - - // Re-calculate mempool ancestors to call addUnchecked(). They may have changed since the - // last calculation done in PreChecks, since package ancestors have already been submitted. - { - auto ancestors{m_pool.CalculateMemPoolAncestors(*ws.m_entry, m_pool.m_opts.limits)}; - if(!ancestors) { - results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); - // Since PreChecks() and PackageMempoolChecks() both enforce limits, this should never fail. - Assume(false); - all_submitted = false; - package_state.Invalid(PackageValidationResult::PCKG_MEMPOOL_ERROR, - strprintf("BUG! Mempool ancestors or descendants were underestimated: %s", - ws.m_ptx->GetHash().ToString())); - } - ws.m_ancestors = std::move(ancestors).value_or(ws.m_ancestors); - } - // If we call LimitMempoolSize() for each individual Finalize(), the mempool will not take - // the transaction's descendant feerate into account because it hasn't seen them yet. Also, - // we risk evicting a transaction that a subsequent package transaction depends on. Instead, - // allow the mempool to temporarily bypass limits, the maximum package size) while - // submitting transactions individually and then trim at the very end. - if (!Finalize(args, ws)) { - results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); - // Since LimitMempoolSize() won't be called, this should never fail. - Assume(false); - all_submitted = false; - package_state.Invalid(PackageValidationResult::PCKG_MEMPOOL_ERROR, - strprintf("BUG! Adding to mempool failed: %s", ws.m_ptx->GetHash().ToString())); - } + // Remove the transaction from the mempool. + if (!m_subpackage.m_changeset) m_subpackage.m_changeset = m_pool.GetChangeSet(); + m_subpackage.m_changeset->StageRemoval(m_pool.GetIter(ws.m_ptx->GetHash()).value()); + } + } + if (!all_submitted) { + Assume(m_subpackage.m_changeset); + // This code should be unreachable; it's here as belt-and-suspenders + // to try to ensure we have no consensus-invalid transactions in the + // mempool. + m_subpackage.m_changeset->Apply(); + m_subpackage.m_changeset.reset(); + return false; } std::vector<Wtxid> all_package_wtxids; @@ -1414,6 +1391,8 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& // Add successful results. The returned results may change later if LimitMempoolSize() evicts them. for (Workspace& ws : workspaces) { + auto iter = m_pool.GetIter(ws.m_ptx->GetHash()); + Assume(iter.has_value()); const auto effective_feerate = args.m_package_feerates ? ws.m_package_feerate : CFeeRate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; const auto effective_feerate_wtxids = args.m_package_feerates ? all_package_wtxids : @@ -1424,7 +1403,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& if (!m_pool.m_opts.signals) continue; const CTransaction& tx = *ws.m_ptx; const auto tx_info = NewMempoolTransactionInfo(ws.m_ptx, ws.m_base_fees, - ws.m_vsize, ws.m_entry->GetHeight(), + ws.m_vsize, (*iter)->GetHeight(), args.m_bypass_limits, args.m_package_submission, IsCurrentForFeeEstimation(m_active_chainstate), m_pool.HasNoInputsOf(tx)); @@ -1449,12 +1428,22 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef return MempoolAcceptResult::Failure(ws.m_state); } + m_subpackage.m_total_vsize = ws.m_vsize; + m_subpackage.m_total_modified_fees = ws.m_modified_fees; + // Individual modified feerate exceeded caller-defined max; abort if (args.m_client_maxfeerate && CFeeRate(ws.m_modified_fees, ws.m_vsize) > args.m_client_maxfeerate.value()) { ws.m_state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "max feerate exceeded", ""); return MempoolAcceptResult::Failure(ws.m_state); } + if (m_pool.m_opts.require_standard) { + Txid dummy_txid; + if (!CheckEphemeralSpends(/*package=*/{ptx}, m_pool.m_opts.dust_relay_feerate, m_pool, ws.m_state, dummy_txid)) { + return MempoolAcceptResult::Failure(ws.m_state); + } + } + if (m_subpackage.m_rbf && !ReplacementChecks(ws)) { if (ws.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE) { // Failed for incentives-based fee reasons. Provide the effective feerate and which tx was included. @@ -1476,17 +1465,24 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef ws.m_base_fees, effective_feerate, single_wtxid); } - if (!Finalize(args, ws)) { - // The only possible failure reason is fee-related (mempool full). - // Failed for fee reasons. Provide the effective feerate and which txns were included. - Assume(ws.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE); - return MempoolAcceptResult::FeeFailure(ws.m_state, CFeeRate(ws.m_modified_fees, ws.m_vsize), {ws.m_ptx->GetWitnessHash()}); + FinalizeSubpackage(args); + + // Limit the mempool, if appropriate. + if (!args.m_package_submission && !args.m_bypass_limits) { + LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip()); + if (!m_pool.exists(GenTxid::Txid(ws.m_hash))) { + // The tx no longer meets our (new) mempool minimum feerate but could be reconsidered in a package. + ws.m_state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "mempool full"); + return MempoolAcceptResult::FeeFailure(ws.m_state, CFeeRate(ws.m_modified_fees, ws.m_vsize), {ws.m_ptx->GetWitnessHash()}); + } } if (m_pool.m_opts.signals) { const CTransaction& tx = *ws.m_ptx; + auto iter = m_pool.GetIter(tx.GetHash()); + Assume(iter.has_value()); const auto tx_info = NewMempoolTransactionInfo(ws.m_ptx, ws.m_base_fees, - ws.m_vsize, ws.m_entry->GetHeight(), + ws.m_vsize, (*iter)->GetHeight(), args.m_bypass_limits, args.m_package_submission, IsCurrentForFeeEstimation(m_active_chainstate), m_pool.HasNoInputsOf(tx)); @@ -1593,6 +1589,17 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: return PackageMempoolAcceptResult(package_state, std::move(results)); } + // Now that we've bounded the resulting possible ancestry count, check package for dust spends + if (m_pool.m_opts.require_standard) { + TxValidationState child_state; + Txid child_txid; + if (!CheckEphemeralSpends(txns, m_pool.m_opts.dust_relay_feerate, m_pool, child_state, child_txid)) { + package_state.Invalid(PackageValidationResult::PCKG_TX, "unspent-dust"); + results.emplace(child_txid, MempoolAcceptResult::Failure(child_state)); + return PackageMempoolAcceptResult(package_state, std::move(results)); + } + } + for (Workspace& ws : workspaces) { ws.m_package_feerate = package_feerate; if (!PolicyScriptChecks(args, ws)) { @@ -1680,10 +1687,14 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptSubPackage(const std::vector<CTr PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, ATMPArgs& args) { + Assert(!package.empty()); AssertLockHeld(cs_main); // Used if returning a PackageMempoolAcceptResult directly from this function. PackageValidationState package_state_quit_early; + // There are two topologies we are able to handle through this function: + // (1) A single transaction + // (2) A child-with-unconfirmed-parents package. // Check that the package is well-formed. If it isn't, we won't try to validate any of the // transactions and thus won't return any MempoolAcceptResults, just a package-wide error. @@ -1692,48 +1703,50 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, return PackageMempoolAcceptResult(package_state_quit_early, {}); } - // All transactions in the package must be a parent of the last transaction. This is just an - // opportunity for us to fail fast on a context-free check without taking the mempool lock. - if (!IsChildWithParents(package)) { - package_state_quit_early.Invalid(PackageValidationResult::PCKG_POLICY, "package-not-child-with-parents"); - return PackageMempoolAcceptResult(package_state_quit_early, {}); - } - - // IsChildWithParents() guarantees the package is > 1 transactions. - assert(package.size() > 1); - // The package must be 1 child with all of its unconfirmed parents. The package is expected to - // be sorted, so the last transaction is the child. - const auto& child = package.back(); - std::unordered_set<uint256, SaltedTxidHasher> unconfirmed_parent_txids; - std::transform(package.cbegin(), package.cend() - 1, - std::inserter(unconfirmed_parent_txids, unconfirmed_parent_txids.end()), - [](const auto& tx) { return tx->GetHash(); }); - - // All child inputs must refer to a preceding package transaction or a confirmed UTXO. The only - // way to verify this is to look up the child's inputs in our current coins view (not including - // mempool), and enforce that all parents not present in the package be available at chain tip. - // Since this check can bring new coins into the coins cache, keep track of these coins and - // uncache them if we don't end up submitting this package to the mempool. - const CCoinsViewCache& coins_tip_cache = m_active_chainstate.CoinsTip(); - for (const auto& input : child->vin) { - if (!coins_tip_cache.HaveCoinInCache(input.prevout)) { - args.m_coins_to_uncache.push_back(input.prevout); - } - } - // Using the MemPoolAccept m_view cache allows us to look up these same coins faster later. - // This should be connecting directly to CoinsTip, not to m_viewmempool, because we specifically - // require inputs to be confirmed if they aren't in the package. - m_view.SetBackend(m_active_chainstate.CoinsTip()); - const auto package_or_confirmed = [this, &unconfirmed_parent_txids](const auto& input) { - return unconfirmed_parent_txids.count(input.prevout.hash) > 0 || m_view.HaveCoin(input.prevout); - }; - if (!std::all_of(child->vin.cbegin(), child->vin.cend(), package_or_confirmed)) { - package_state_quit_early.Invalid(PackageValidationResult::PCKG_POLICY, "package-not-child-with-unconfirmed-parents"); - return PackageMempoolAcceptResult(package_state_quit_early, {}); + if (package.size() > 1) { + // All transactions in the package must be a parent of the last transaction. This is just an + // opportunity for us to fail fast on a context-free check without taking the mempool lock. + if (!IsChildWithParents(package)) { + package_state_quit_early.Invalid(PackageValidationResult::PCKG_POLICY, "package-not-child-with-parents"); + return PackageMempoolAcceptResult(package_state_quit_early, {}); + } + + // IsChildWithParents() guarantees the package is > 1 transactions. + assert(package.size() > 1); + // The package must be 1 child with all of its unconfirmed parents. The package is expected to + // be sorted, so the last transaction is the child. + const auto& child = package.back(); + std::unordered_set<uint256, SaltedTxidHasher> unconfirmed_parent_txids; + std::transform(package.cbegin(), package.cend() - 1, + std::inserter(unconfirmed_parent_txids, unconfirmed_parent_txids.end()), + [](const auto& tx) { return tx->GetHash(); }); + + // All child inputs must refer to a preceding package transaction or a confirmed UTXO. The only + // way to verify this is to look up the child's inputs in our current coins view (not including + // mempool), and enforce that all parents not present in the package be available at chain tip. + // Since this check can bring new coins into the coins cache, keep track of these coins and + // uncache them if we don't end up submitting this package to the mempool. + const CCoinsViewCache& coins_tip_cache = m_active_chainstate.CoinsTip(); + for (const auto& input : child->vin) { + if (!coins_tip_cache.HaveCoinInCache(input.prevout)) { + args.m_coins_to_uncache.push_back(input.prevout); + } + } + // Using the MemPoolAccept m_view cache allows us to look up these same coins faster later. + // This should be connecting directly to CoinsTip, not to m_viewmempool, because we specifically + // require inputs to be confirmed if they aren't in the package. + m_view.SetBackend(m_active_chainstate.CoinsTip()); + const auto package_or_confirmed = [this, &unconfirmed_parent_txids](const auto& input) { + return unconfirmed_parent_txids.count(input.prevout.hash) > 0 || m_view.HaveCoin(input.prevout); + }; + if (!std::all_of(child->vin.cbegin(), child->vin.cend(), package_or_confirmed)) { + package_state_quit_early.Invalid(PackageValidationResult::PCKG_POLICY, "package-not-child-with-unconfirmed-parents"); + return PackageMempoolAcceptResult(package_state_quit_early, {}); + } + // Protect against bugs where we pull more inputs from disk that miss being added to + // coins_to_uncache. The backend will be connected again when needed in PreChecks. + m_view.SetBackend(m_dummy); } - // Protect against bugs where we pull more inputs from disk that miss being added to - // coins_to_uncache. The backend will be connected again when needed in PreChecks. - m_view.SetBackend(m_dummy); LOCK(m_pool.cs); // Stores results from which we will create the returned PackageMempoolAcceptResult. @@ -1743,6 +1756,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // this transaction. "Nonfinal" because if a transaction fails by itself but succeeds later // (i.e. when evaluated with a fee-bumping child), the result in this map may be discarded. std::map<uint256, MempoolAcceptResult> individual_results_nonfinal; + // Tracks whether we think package submission could result in successful entry to the mempool bool quit_early{false}; std::vector<CTransactionRef> txns_package_eval; for (const auto& tx : package) { @@ -1784,8 +1798,9 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // in package validation, because its fees should only be "used" once. assert(m_pool.exists(GenTxid::Wtxid(wtxid))); results_final.emplace(wtxid, single_res); - } else if (single_res.m_state.GetResult() != TxValidationResult::TX_RECONSIDERABLE && - single_res.m_state.GetResult() != TxValidationResult::TX_MISSING_INPUTS) { + } else if (package.size() == 1 || // If there is only one transaction, no need to retry it "as a package" + (single_res.m_state.GetResult() != TxValidationResult::TX_RECONSIDERABLE && + single_res.m_state.GetResult() != TxValidationResult::TX_MISSING_INPUTS)) { // Package validation policy only differs from individual policy in its evaluation // of feerate. For example, if a transaction fails here due to violation of a // consensus rule, the result will not change when it is submitted as part of a @@ -1809,6 +1824,11 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, AcceptSubPackage(txns_package_eval, args); PackageValidationState& package_state_final = multi_submission_result.m_state; + // This is invoked by AcceptSubPackage() already, so this is just here for + // clarity (since it's not permitted to invoke LimitMempoolSize() while a + // changeset is outstanding). + ClearSubPackageState(); + // Make sure we haven't exceeded max mempool size. // Package transactions that were submitted to mempool or already in mempool may be evicted. LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip()); @@ -1874,7 +1894,7 @@ MempoolAcceptResult AcceptToMemoryPool(Chainstate& active_chainstate, const CTra for (const COutPoint& hashTx : coins_to_uncache) active_chainstate.CoinsTip().Uncache(hashTx); - TRACE2(mempool, rejected, + TRACEPOINT(mempool, rejected, tx->GetHash().data(), result.m_state.GetRejectReason().c_str() ); @@ -1976,6 +1996,8 @@ void Chainstate::InitCoinsDB( .obfuscate = true, .options = m_chainman.m_options.coins_db}, m_chainman.m_options.coins_view); + + m_coinsdb_cache_size_bytes = cache_size_bytes; } void Chainstate::InitCoinsCache(size_t cache_size_bytes) @@ -2024,7 +2046,8 @@ void Chainstate::CheckForkWarningConditions() // Before we get past initial download, we cannot reliably alert about forks // (we assume we don't get stuck on a fork before finishing our initial sync) - if (m_chainman.IsInitialBlockDownload()) { + // Also not applicable to the background chainstate + if (m_chainman.IsInitialBlockDownload() || this->GetRole() == ChainstateRole::BACKGROUND) { return; } @@ -2045,8 +2068,9 @@ void Chainstate::InvalidChainFound(CBlockIndex* pindexNew) if (!m_chainman.m_best_invalid || pindexNew->nChainWork > m_chainman.m_best_invalid->nChainWork) { m_chainman.m_best_invalid = pindexNew; } + SetBlockFailureFlags(pindexNew); if (m_chainman.m_best_header != nullptr && m_chainman.m_best_header->GetAncestor(pindexNew->nHeight) == pindexNew) { - m_chainman.m_best_header = m_chain.Tip(); + m_chainman.RecalculateBestHeader(); } LogPrintf("%s: invalid block=%s height=%d log2_work=%f date=%s\n", __func__, @@ -2089,10 +2113,16 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund AddCoins(inputs, tx, nHeight); } -bool CScriptCheck::operator()() { +std::optional<std::pair<ScriptError, std::string>> CScriptCheck::operator()() { const CScript &scriptSig = ptxTo->vin[nIn].scriptSig; const CScriptWitness *witness = &ptxTo->vin[nIn].scriptWitness; - return VerifyScript(scriptSig, m_tx_out.scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, m_tx_out.nValue, cacheStore, *m_signature_cache, *txdata), &error); + ScriptError error{SCRIPT_ERR_UNKNOWN_ERROR}; + if (VerifyScript(scriptSig, m_tx_out.scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, m_tx_out.nValue, cacheStore, *m_signature_cache, *txdata), &error)) { + return std::nullopt; + } else { + auto debug_str = strprintf("input %i of %s (wtxid %s), spending %s:%i", nIn, ptxTo->GetHash().ToString(), ptxTo->GetWitnessHash().ToString(), ptxTo->vin[nIn].prevout.hash.ToString(), ptxTo->vin[nIn].prevout.n); + return std::make_pair(error, std::move(debug_str)); + } } ValidationCache::ValidationCache(const size_t script_execution_cache_bytes, const size_t signature_cache_bytes) @@ -2181,7 +2211,7 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, CScriptCheck check(txdata.m_spent_outputs[i], tx, validation_cache.m_signature_cache, i, flags, cacheSigStore, &txdata); if (pvChecks) { pvChecks->emplace_back(std::move(check)); - } else if (!check()) { + } else if (auto result = check(); result.has_value()) { if (flags & STANDARD_NOT_MANDATORY_VERIFY_FLAGS) { // Check whether the failure was caused by a // non-mandatory script verification check, such as @@ -2193,19 +2223,23 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, // data providers. CScriptCheck check2(txdata.m_spent_outputs[i], tx, validation_cache.m_signature_cache, i, flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheSigStore, &txdata); - if (check2()) - return state.Invalid(TxValidationResult::TX_NOT_STANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); + auto mandatory_result = check2(); + if (!mandatory_result.has_value()) { + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(result->first)), result->second); + } else { + // If the second check failed, it failed due to a mandatory script verification + // flag, but the first check might have failed on a non-mandatory script + // verification flag. + // + // Avoid reporting a mandatory script check failure with a non-mandatory error + // string by reporting the error from the second check. + result = mandatory_result; + } } + // MANDATORY flag failures correspond to - // TxValidationResult::TX_CONSENSUS. Because CONSENSUS - // failures are the most serious case of validation - // failures, we may need to consider using - // RECENT_CONSENSUS_CHANGE for any script failure that - // could be due to non-upgraded nodes which we may want to - // support, to avoid splitting the network (but this - // depends on the details of how net_processing handles - // such errors). - return state.Invalid(TxValidationResult::TX_CONSENSUS, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); + // TxValidationResult::TX_CONSENSUS. + return state.Invalid(TxValidationResult::TX_CONSENSUS, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(result->first)), result->second); } } @@ -2571,8 +2605,8 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, for (const auto& tx : block.vtx) { for (size_t o = 0; o < tx->vout.size(); o++) { if (view.HaveCoin(COutPoint(tx->GetHash(), o))) { - LogPrintf("ERROR: ConnectBlock(): tried to overwrite transaction\n"); - return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-BIP30"); + state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-BIP30", + "tried to overwrite transaction"); } } } @@ -2611,6 +2645,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, blockundo.vtxundo.reserve(block.vtx.size() - 1); for (unsigned int i = 0; i < block.vtx.size(); i++) { + if (!state.IsValid()) break; const CTransaction &tx = *(block.vtx[i]); nInputs += tx.vin.size(); @@ -2622,14 +2657,15 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, if (!Consensus::CheckTxInputs(tx, tx_state, view, pindex->nHeight, txfee)) { // Any transaction validation failure in ConnectBlock is a block consensus failure state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, - tx_state.GetRejectReason(), tx_state.GetDebugMessage()); - LogError("%s: Consensus::CheckTxInputs: %s, %s\n", __func__, tx.GetHash().ToString(), state.ToString()); - return false; + tx_state.GetRejectReason(), + tx_state.GetDebugMessage() + " in transaction " + tx.GetHash().ToString()); + break; } nFees += txfee; if (!MoneyRange(nFees)) { - LogPrintf("ERROR: %s: accumulated fee in the block out of range.\n", __func__); - return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-accumulated-fee-outofrange"); + state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-accumulated-fee-outofrange", + "accumulated fee in the block out of range"); + break; } // Check that transaction is BIP68 final @@ -2641,8 +2677,9 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, } if (!SequenceLocks(tx, nLockTimeFlags, prevheights, *pindex)) { - LogPrintf("ERROR: %s: contains a non-BIP68-final transaction\n", __func__); - return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-nonfinal"); + state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-nonfinal", + "contains a non-BIP68-final transaction " + tx.GetHash().ToString()); + break; } } @@ -2652,8 +2689,8 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // * witness (when witness enabled in flags and excludes coinbase) nSigOpsCost += GetTransactionSigOpCost(tx, view, flags); if (nSigOpsCost > MAX_BLOCK_SIGOPS_COST) { - LogPrintf("ERROR: ConnectBlock(): too many sigops\n"); - return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-sigops"); + state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-sigops", "too many sigops"); + break; } if (!tx.IsCoinBase()) @@ -2665,9 +2702,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // Any transaction validation failure in ConnectBlock is a block consensus failure state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, tx_state.GetRejectReason(), tx_state.GetDebugMessage()); - LogError("ConnectBlock(): CheckInputScripts on %s failed with %s\n", - tx.GetHash().ToString(), state.ToString()); - return false; + break; } control.Add(std::move(vChecks)); } @@ -2687,14 +2722,18 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, Ticks<MillisecondsDouble>(m_chainman.time_connect) / m_chainman.num_blocks_total); CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, params.GetConsensus()); - if (block.vtx[0]->GetValueOut() > blockReward) { - LogPrintf("ERROR: ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)\n", block.vtx[0]->GetValueOut(), blockReward); - return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-amount"); + if (block.vtx[0]->GetValueOut() > blockReward && state.IsValid()) { + state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-amount", + strprintf("coinbase pays too much (actual=%d vs limit=%d)", block.vtx[0]->GetValueOut(), blockReward)); } - if (!control.Wait()) { - LogPrintf("ERROR: %s: CheckQueue failed\n", __func__); - return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "block-validation-failed"); + auto parallel_result = control.Complete(); + if (parallel_result.has_value() && state.IsValid()) { + state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(parallel_result->first)), parallel_result->second); + } + if (!state.IsValid()) { + LogInfo("Block validation error: %s", state.ToString()); + return false; } const auto time_4{SteadyClock::now()}; m_chainman.time_verify += time_4 - time_2; @@ -2704,8 +2743,9 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, Ticks<SecondsDouble>(m_chainman.time_verify), Ticks<MillisecondsDouble>(m_chainman.time_verify) / m_chainman.num_blocks_total); - if (fJustCheck) + if (fJustCheck) { return true; + } if (!m_blockman.WriteUndoDataForBlock(blockundo, state, *pindex)) { return false; @@ -2733,13 +2773,13 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, Ticks<SecondsDouble>(m_chainman.time_index), Ticks<MillisecondsDouble>(m_chainman.time_index) / m_chainman.num_blocks_total); - TRACE6(validation, block_connected, + TRACEPOINT(validation, block_connected, block_hash.data(), pindex->nHeight, block.vtx.size(), nInputs, nSigOpsCost, - time_5 - time_start // in microseconds (µs) + Ticks<std::chrono::nanoseconds>(time_5 - time_start) ); return true; @@ -2891,8 +2931,9 @@ bool Chainstate::FlushStateToDisk( } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush && !CoinsTip().GetBestBlock().IsNull()) { - LOG_TIME_MILLIS_WITH_CATEGORY(strprintf("write coins cache to disk (%d coins, %.2fkB)", - coins_count, coins_mem_usage / 1000), BCLog::BENCH); + if (coins_mem_usage >= WARN_FLUSH_COINS_SIZE) LogWarning("Flushing large (%d GiB) UTXO set to disk, it may take several minutes", coins_mem_usage >> 30); + LOG_TIME_MILLIS_WITH_CATEGORY(strprintf("write coins cache to disk (%d coins, %.2fKiB)", + coins_count, coins_mem_usage >> 10), BCLog::BENCH); // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written @@ -2909,7 +2950,7 @@ bool Chainstate::FlushStateToDisk( } m_last_flush = nNow; full_flush_completed = true; - TRACE5(utxocache, flush, + TRACEPOINT(utxocache, flush, int64_t{Ticks<std::chrono::microseconds>(SteadyClock::now() - nNow)}, (uint32_t)mode, (uint64_t)coins_count, @@ -2945,9 +2986,9 @@ void Chainstate::PruneAndFlush() } static void UpdateTipLog( + const ChainstateManager& chainman, const CCoinsViewCache& coins_tip, const CBlockIndex* tip, - const CChainParams& params, const std::string& func_name, const std::string& prefix, const std::string& warning_messages) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) @@ -2959,7 +3000,7 @@ static void UpdateTipLog( tip->GetBlockHash().ToString(), tip->nHeight, tip->nVersion, log(tip->nChainWork.getdouble()) / log(2.0), tip->m_chain_tx_count, FormatISO8601DateTime(tip->GetBlockTime()), - GuessVerificationProgress(params.TxData(), tip), + chainman.GuessVerificationProgress(tip), coins_tip.DynamicMemoryUsage() * (1.0 / (1 << 20)), coins_tip.GetCacheSize(), !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages) : ""); @@ -2970,15 +3011,13 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) AssertLockHeld(::cs_main); const auto& coins_tip = this->CoinsTip(); - const CChainParams& params{m_chainman.GetParams()}; - // The remainder of the function isn't relevant if we are not acting on // the active chainstate, so return if need be. if (this != &m_chainman.ActiveChainstate()) { // Only log every so often so that we don't bury log messages at the tip. constexpr int BACKGROUND_LOG_INTERVAL = 2000; if (pindexNew->nHeight % BACKGROUND_LOG_INTERVAL == 0) { - UpdateTipLog(coins_tip, pindexNew, params, __func__, "[background validation] ", ""); + UpdateTipLog(m_chainman, coins_tip, pindexNew, __func__, "[background validation] ", ""); } return; } @@ -2988,18 +3027,12 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) m_mempool->AddTransactionsUpdated(1); } - { - LOCK(g_best_block_mutex); - g_best_block = pindexNew->GetBlockHash(); - g_best_block_cv.notify_all(); - } - std::vector<bilingual_str> warning_messages; if (!m_chainman.IsInitialBlockDownload()) { const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(m_chainman, bit); - ThresholdState state = checker.GetStateFor(pindex, params.GetConsensus(), m_chainman.m_warningcache.at(bit)); + ThresholdState state = checker.GetStateFor(pindex, m_chainman.GetConsensus(), m_chainman.m_warningcache.at(bit)); if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { const bilingual_str warning = strprintf(_("Unknown new rules activated (versionbit %i)"), bit); if (state == ThresholdState::ACTIVE) { @@ -3010,7 +3043,7 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) } } } - UpdateTipLog(coins_tip, pindexNew, params, __func__, "", + UpdateTipLog(m_chainman, coins_tip, pindexNew, __func__, "", util::Join(warning_messages, Untranslated(", ")).original); } @@ -3456,7 +3489,7 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< // chainstate past the snapshot base block. if (WITH_LOCK(::cs_main, return m_disabled)) { LogPrintf("m_disabled is set - this chainstate should not be in operation. " - "Please report this as a bug. %s\n", PACKAGE_BUGREPORT); + "Please report this as a bug. %s\n", CLIENT_BUGREPORT); return false; } @@ -3496,6 +3529,10 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< bool fInvalidFound = false; std::shared_ptr<const CBlock> nullBlockPtr; + // BlockConnected signals must be sent for the original role; + // in case snapshot validation is completed during ActivateBestChainStep, the + // result of GetRole() changes from BACKGROUND to NORMAL. + const ChainstateRole chainstate_role{this->GetRole()}; if (!ActivateBestChainStep(state, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : nullBlockPtr, fInvalidFound, connectTrace)) { // A system error occurred return false; @@ -3511,7 +3548,7 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< for (const PerBlockConnectTrace& trace : connectTrace.GetBlocksConnected()) { assert(trace.pblock && trace.pindex); if (m_chainman.m_options.signals) { - m_chainman.m_options.signals->BlockConnected(this->GetRole(), trace.pblock, trace.pindex); + m_chainman.m_options.signals->BlockConnected(chainstate_role, trace.pblock, trace.pindex); } } @@ -3542,7 +3579,6 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< m_chainman.m_options.signals->UpdatedBlockTip(pindexNewTip, pindexFork, still_in_ibd); } - // Always notify the UI if a new block tip was connected if (kernel::IsInterrupted(m_chainman.GetNotifications().blockTip(GetSynchronizationState(still_in_ibd, m_chainman.m_blockman.m_blockfiles_indexed), *pindexNewTip))) { // Just breaking and returning success for now. This could // be changed to bubble up the kernel::Interrupted value to @@ -3785,6 +3821,17 @@ bool Chainstate::InvalidateBlock(BlockValidationState& state, CBlockIndex* pinde return true; } +void Chainstate::SetBlockFailureFlags(CBlockIndex* invalid_block) +{ + AssertLockHeld(cs_main); + + for (auto& [_, block_index] : m_blockman.m_block_index) { + if (block_index.GetAncestor(invalid_block->nHeight) == invalid_block && !(block_index.nStatus & BLOCK_FAILED_MASK)) { + block_index.nStatus |= BLOCK_FAILED_CHILD; + } + } +} + void Chainstate::ResetBlockFailureFlags(CBlockIndex *pindex) { AssertLockHeld(cs_main); @@ -3856,7 +3903,7 @@ void ChainstateManager::ReceivedBlockTransactions(const CBlock& block, CBlockInd if (!Assume(pindexNew->m_chain_tx_count == 0 || pindexNew->m_chain_tx_count == prev_tx_sum(*pindexNew) || pindexNew == GetSnapshotBaseBlock())) { LogWarning("Internal bug detected: block %d has unexpected m_chain_tx_count %i that should be %i (%s %s). Please report this issue here: %s\n", - pindexNew->nHeight, pindexNew->m_chain_tx_count, prev_tx_sum(*pindexNew), PACKAGE_NAME, FormatFullVersion(), PACKAGE_BUGREPORT); + pindexNew->nHeight, pindexNew->m_chain_tx_count, prev_tx_sum(*pindexNew), CLIENT_NAME, FormatFullVersion(), CLIENT_BUGREPORT); pindexNew->m_chain_tx_count = 0; } pindexNew->nFile = pos.nFile; @@ -3884,7 +3931,7 @@ void ChainstateManager::ReceivedBlockTransactions(const CBlock& block, CBlockInd // incorrect hardcoded AssumeutxoData::m_chain_tx_count value. if (!Assume(pindex->m_chain_tx_count == 0 || pindex->m_chain_tx_count == prev_tx_sum(*pindex))) { LogWarning("Internal bug detected: block %d has unexpected m_chain_tx_count %i that should be %i (%s %s). Please report this issue here: %s\n", - pindex->nHeight, pindex->m_chain_tx_count, prev_tx_sum(*pindex), PACKAGE_NAME, FormatFullVersion(), PACKAGE_BUGREPORT); + pindex->nHeight, pindex->m_chain_tx_count, prev_tx_sum(*pindex), CLIENT_NAME, FormatFullVersion(), CLIENT_BUGREPORT); } pindex->m_chain_tx_count = prev_tx_sum(*pindex); pindex->nSequenceId = nBlockSequenceId++; @@ -4289,7 +4336,7 @@ bool ChainstateManager::AcceptBlockHeader(const CBlockHeader& block, BlockValida *ppindex = pindex; if (pindex->nStatus & BLOCK_FAILED_MASK) { LogDebug(BCLog::VALIDATION, "%s: block %s is marked invalid\n", __func__, hash.ToString()); - return state.Invalid(BlockValidationResult::BLOCK_CACHED_INVALID, "duplicate"); + return state.Invalid(BlockValidationResult::BLOCK_CACHED_INVALID, "duplicate-invalid"); } return true; } @@ -4678,7 +4725,14 @@ bool Chainstate::LoadChainTip() tip->GetBlockHash().ToString(), m_chain.Height(), FormatISO8601DateTime(tip->GetBlockTime()), - GuessVerificationProgress(m_chainman.GetParams().TxData(), tip)); + m_chainman.GuessVerificationProgress(tip)); + + // Ensure KernelNotifications m_tip_block is set even if no new block arrives. + if (this->GetRole() != ChainstateRole::BACKGROUND) { + // Ignoring return value for now. + (void)m_chainman.GetNotifications().blockTip(GetSynchronizationState(/*init=*/true, m_chainman.m_blockman.m_blockfiles_indexed), *pindex); + } + return true; } @@ -5562,13 +5616,16 @@ bool Chainstate::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) //! Guess how far we are in the verification process at the given block index //! require cs_main if pindex has not been validated yet (because m_chain_tx_count might be unset) -double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pindex) { - if (pindex == nullptr) +double ChainstateManager::GuessVerificationProgress(const CBlockIndex* pindex) const +{ + const ChainTxData& data{GetParams().TxData()}; + if (pindex == nullptr) { return 0.0; + } if (!Assume(pindex->m_chain_tx_count > 0)) { LogWarning("Internal bug detected: block %d has unset m_chain_tx_count (%s %s). Please report this issue here: %s\n", - pindex->nHeight, PACKAGE_NAME, FormatFullVersion(), PACKAGE_BUGREPORT); + pindex->nHeight, CLIENT_NAME, FormatFullVersion(), CLIENT_BUGREPORT); return 0.0; } @@ -5677,20 +5734,20 @@ util::Result<CBlockIndex*> ChainstateManager::ActivateSnapshot( if (!GetParams().AssumeutxoForBlockhash(base_blockhash).has_value()) { auto available_heights = GetParams().GetAvailableSnapshotHeights(); std::string heights_formatted = util::Join(available_heights, ", ", [&](const auto& i) { return util::ToString(i); }); - return util::Error{strprintf(Untranslated("assumeutxo block hash in snapshot metadata not recognized (hash: %s). The following snapshot heights are available: %s"), + return util::Error{Untranslated(strprintf("assumeutxo block hash in snapshot metadata not recognized (hash: %s). The following snapshot heights are available: %s", base_blockhash.ToString(), - heights_formatted)}; + heights_formatted))}; } snapshot_start_block = m_blockman.LookupBlockIndex(base_blockhash); if (!snapshot_start_block) { - return util::Error{strprintf(Untranslated("The base block header (%s) must appear in the headers chain. Make sure all headers are syncing, and call loadtxoutset again"), - base_blockhash.ToString())}; + return util::Error{Untranslated(strprintf("The base block header (%s) must appear in the headers chain. Make sure all headers are syncing, and call loadtxoutset again", + base_blockhash.ToString()))}; } bool start_block_invalid = snapshot_start_block->nStatus & BLOCK_FAILED_MASK; if (start_block_invalid) { - return util::Error{strprintf(Untranslated("The base block header (%s) is part of an invalid chain"), base_blockhash.ToString())}; + return util::Error{Untranslated(strprintf("The base block header (%s) is part of an invalid chain", base_blockhash.ToString()))}; } if (!m_best_header || m_best_header->GetAncestor(snapshot_start_block->nHeight) != snapshot_start_block) { @@ -5769,7 +5826,7 @@ util::Result<CBlockIndex*> ChainstateManager::ActivateSnapshot( if (auto res{this->PopulateAndValidateSnapshot(*snapshot_chainstate, coins_file, metadata)}; !res) { LOCK(::cs_main); - return cleanup_bad_snapshot(strprintf(Untranslated("Population failed: %s"), util::ErrorString(res))); + return cleanup_bad_snapshot(Untranslated(strprintf("Population failed: %s", util::ErrorString(res).original))); } LOCK(::cs_main); // cs_main required for rest of snapshot activation. @@ -5850,16 +5907,16 @@ util::Result<void> ChainstateManager::PopulateAndValidateSnapshot( if (!snapshot_start_block) { // Needed for ComputeUTXOStats to determine the // height and to avoid a crash when base_blockhash.IsNull() - return util::Error{strprintf(Untranslated("Did not find snapshot start blockheader %s"), - base_blockhash.ToString())}; + return util::Error{Untranslated(strprintf("Did not find snapshot start blockheader %s", + base_blockhash.ToString()))}; } int base_height = snapshot_start_block->nHeight; const auto& maybe_au_data = GetParams().AssumeutxoForHeight(base_height); if (!maybe_au_data) { - return util::Error{strprintf(Untranslated("Assumeutxo height in snapshot metadata not recognized " - "(%d) - refusing to load snapshot"), base_height)}; + return util::Error{Untranslated(strprintf("Assumeutxo height in snapshot metadata not recognized " + "(%d) - refusing to load snapshot", base_height))}; } const AssumeutxoData& au_data = *maybe_au_data; @@ -5897,12 +5954,12 @@ util::Result<void> ChainstateManager::PopulateAndValidateSnapshot( if (coin.nHeight > base_height || outpoint.n >= std::numeric_limits<decltype(outpoint.n)>::max() // Avoid integer wrap-around in coinstats.cpp:ApplyHash ) { - return util::Error{strprintf(Untranslated("Bad snapshot data after deserializing %d coins"), - coins_count - coins_left)}; + return util::Error{Untranslated(strprintf("Bad snapshot data after deserializing %d coins", + coins_count - coins_left))}; } if (!MoneyRange(coin.out.nValue)) { - return util::Error{strprintf(Untranslated("Bad snapshot data after deserializing %d coins - bad tx out value"), - coins_count - coins_left)}; + return util::Error{Untranslated(strprintf("Bad snapshot data after deserializing %d coins - bad tx out value", + coins_count - coins_left))}; } coins_cache.EmplaceCoinInternalDANGER(std::move(outpoint), std::move(coin)); @@ -5940,8 +5997,8 @@ util::Result<void> ChainstateManager::PopulateAndValidateSnapshot( } } } catch (const std::ios_base::failure&) { - return util::Error{strprintf(Untranslated("Bad snapshot format or truncated snapshot after deserializing %d coins"), - coins_processed)}; + return util::Error{Untranslated(strprintf("Bad snapshot format or truncated snapshot after deserializing %d coins", + coins_processed))}; } } @@ -5961,8 +6018,8 @@ util::Result<void> ChainstateManager::PopulateAndValidateSnapshot( out_of_coins = true; } if (!out_of_coins) { - return util::Error{strprintf(Untranslated("Bad snapshot - coins left over after deserializing %d coins"), - coins_count)}; + return util::Error{Untranslated(strprintf("Bad snapshot - coins left over after deserializing %d coins", + coins_count))}; } LogPrintf("[snapshot] loaded %d (%.2f MB) coins from snapshot %s\n", @@ -5993,8 +6050,8 @@ util::Result<void> ChainstateManager::PopulateAndValidateSnapshot( // Assert that the deserialized chainstate contents match the expected assumeutxo value. if (AssumeutxoHash{maybe_stats->hashSerialized} != au_data.hash_serialized) { - return util::Error{strprintf(Untranslated("Bad snapshot content hash: expected %s, got %s"), - au_data.hash_serialized.ToString(), maybe_stats->hashSerialized.ToString())}; + return util::Error{Untranslated(strprintf("Bad snapshot content hash: expected %s, got %s", + au_data.hash_serialized.ToString(), maybe_stats->hashSerialized.ToString()))}; } snapshot_chainstate.m_chain.SetTip(*snapshot_start_block); @@ -6014,7 +6071,7 @@ util::Result<void> ChainstateManager::PopulateAndValidateSnapshot( index = snapshot_chainstate.m_chain[i]; // Fake BLOCK_OPT_WITNESS so that Chainstate::NeedsRedownload() - // won't ask to rewind the entire assumed-valid chain on startup. + // won't ask for -reindex on startup. if (DeploymentActiveAt(*index, *this, Consensus::DEPLOYMENT_SEGWIT)) { index->nStatus |= BLOCK_OPT_WITNESS; } @@ -6087,7 +6144,7 @@ SnapshotCompletionResult ChainstateManager::MaybeCompleteSnapshotValidation() "Please report this incident to %s, including how you obtained the snapshot. " "The invalid snapshot chainstate will be left on disk in case it is " "helpful in diagnosing the issue that caused this error."), - PACKAGE_NAME, snapshot_tip_height, snapshot_base_height, snapshot_base_height, PACKAGE_BUGREPORT + CLIENT_NAME, snapshot_tip_height, snapshot_base_height, snapshot_base_height, CLIENT_BUGREPORT ); LogError("[snapshot] !!! %s\n", user_error.original); @@ -6100,7 +6157,7 @@ SnapshotCompletionResult ChainstateManager::MaybeCompleteSnapshotValidation() auto rename_result = m_snapshot_chainstate->InvalidateCoinsDBOnDisk(); if (!rename_result) { - user_error = strprintf(Untranslated("%s\n%s"), user_error, util::ErrorString(rename_result)); + user_error += Untranslated("\n") + util::ErrorString(rename_result); } GetNotifications().fatalError(user_error); @@ -6252,7 +6309,7 @@ static ChainstateManager::Options&& Flatten(ChainstateManager::Options&& opts) } ChainstateManager::ChainstateManager(const util::SignalInterrupt& interrupt, Options options, node::BlockManager::Options blockman_options) - : m_script_check_queue{/*batch_size=*/128, options.worker_threads_num}, + : m_script_check_queue{/*batch_size=*/128, std::clamp(options.worker_threads_num, 0, MAX_SCRIPTCHECK_THREADS)}, m_interrupt{interrupt}, m_options{Flatten(std::move(options))}, m_blockman{interrupt, std::move(blockman_options)}, @@ -6396,6 +6453,17 @@ std::optional<int> ChainstateManager::GetSnapshotBaseHeight() const return base ? std::make_optional(base->nHeight) : std::nullopt; } +void ChainstateManager::RecalculateBestHeader() +{ + AssertLockHeld(cs_main); + m_best_header = ActiveChain().Tip(); + for (auto& entry : m_blockman.m_block_index) { + if (!(entry.second.nStatus & BLOCK_FAILED_MASK) && m_best_header->nChainWork < entry.second.nChainWork) { + m_best_header = &entry.second; + } + } +} + bool ChainstateManager::ValidatedSnapshotCleanup() { AssertLockHeld(::cs_main); diff --git a/src/validation.h b/src/validation.h index 059ae52bdf..e2ff5925c5 100644 --- a/src/validation.h +++ b/src/validation.h @@ -42,7 +42,6 @@ #include <span> #include <stdint.h> #include <string> -#include <thread> #include <type_traits> #include <utility> #include <vector> @@ -79,6 +78,9 @@ static constexpr int DEFAULT_CHECKLEVEL{3}; // Setting the target to >= 550 MiB will make it likely we can respect the target. static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; +/** Maximum number of dedicated script-checking threads allowed */ +static constexpr int MAX_SCRIPTCHECK_THREADS{15}; + /** Current sync state passed to tip changed callbacks. */ enum class SynchronizationState { INIT_REINDEX, @@ -86,11 +88,6 @@ enum class SynchronizationState { POST_INIT }; -extern GlobalMutex g_best_block_mutex; -extern std::condition_variable g_best_block_cv; -/** Used to notify getblocktemplate RPC of new tips. */ -extern uint256 g_best_block; - /** Documentation for argument 'checklevel'. */ extern const std::vector<std::string> CHECKLEVEL_DOC; @@ -98,9 +95,6 @@ CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); bool FatalError(kernel::Notifications& notifications, BlockValidationState& state, const bilingual_str& message); -/** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ -double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pindex); - /** Prune block files up to a given height */ void PruneBlockFilesManual(Chainstate& active_chainstate, int nManualPruneHeight); @@ -341,7 +335,6 @@ private: unsigned int nIn; unsigned int nFlags; bool cacheStore; - ScriptError error{SCRIPT_ERR_UNKNOWN_ERROR}; PrecomputedTransactionData *txdata; SignatureCache* m_signature_cache; @@ -354,9 +347,7 @@ public: CScriptCheck(CScriptCheck&&) = default; CScriptCheck& operator=(CScriptCheck&&) = default; - bool operator()(); - - ScriptError GetScriptError() const { return error; } + std::optional<std::pair<ScriptError, std::string>> operator()(); }; // CScriptCheck is used a lot in std::vector, make sure that's efficient @@ -735,6 +726,9 @@ public: EXCLUSIVE_LOCKS_REQUIRED(!m_chainstate_mutex) LOCKS_EXCLUDED(::cs_main); + /** Set invalidity status to all descendants of a block */ + void SetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** Remove invalidity status from a block and its descendants. */ void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -1008,7 +1002,6 @@ public: const util::SignalInterrupt& m_interrupt; const Options m_options; - std::thread m_thread_load; //! A single BlockManager instance is shared across each constructed //! chainstate to avoid duplicating block metadata. node::BlockManager m_blockman; @@ -1155,6 +1148,9 @@ public: /** Check whether we are doing an initial block download (synchronizing from disk or network) */ bool IsInitialBlockDownload() const; + /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ + double GuessVerificationProgress(const CBlockIndex* pindex) const; + /** * Import blocks from an external file * @@ -1321,6 +1317,11 @@ public: //! nullopt. std::optional<int> GetSnapshotBaseHeight() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + //! If, due to invalidation / reconsideration of blocks, the previous + //! best header is no longer valid / guaranteed to be the most-work + //! header in our block-index not known to be invalid, recalculate it. + void RecalculateBestHeader() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + CCheckQueue<CScriptCheck>& GetCheckQueue() { return m_script_check_queue; } ~ChainstateManager(); diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index e8ff1d78e3..da2685d771 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -9,6 +9,7 @@ #include <consensus/validation.h> #include <kernel/chain.h> #include <kernel/mempool_entry.h> +#include <kernel/mempool_removal_reason.h> #include <logging.h> #include <primitives/block.h> #include <primitives/transaction.h> @@ -19,8 +20,6 @@ #include <unordered_map> #include <utility> -std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept; - /** * ValidationSignalsImpl manages a list of shared_ptr<CValidationInterface> callbacks. * diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index af0c78f0d9..f3fe8a19c1 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -208,6 +208,7 @@ public: bool TxnBegin() override; bool TxnCommit() override; bool TxnAbort() override; + bool HasActiveTxn() override { return activeTxn != nullptr; } DbTxn* txn() const { return activeTxn; } }; diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index cee558088f..6e6d7e053b 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -478,7 +478,6 @@ util::Result<SelectionResult> CoinGrinder(std::vector<OutputGroup>& utxo_pool, c // Neither adding to the current selection nor exploring the omission branch of the last selected UTXO can // find any solutions. Redirect to exploring the Omission branch of the penultimate selected UTXO (i.e. // set `next_utxo` to one after the penultimate selected, then deselect the last two selected UTXOs) - should_cut = false; deselect_last(); should_shift = true; } diff --git a/src/wallet/db.h b/src/wallet/db.h index 049af8dd19..e8790006a4 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -122,6 +122,7 @@ public: virtual bool TxnBegin() = 0; virtual bool TxnCommit() = 0; virtual bool TxnAbort() = 0; + virtual bool HasActiveTxn() = 0; }; /** An instance of this class represents one database. diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 3184d0f3b0..b875461c9f 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -24,29 +24,29 @@ namespace wallet { static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWalletTx& wtx, bool require_mine, std::vector<bilingual_str>& errors) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { if (wallet.HasWalletSpend(wtx.tx)) { - errors.push_back(Untranslated("Transaction has descendants in the wallet")); + errors.emplace_back(Untranslated("Transaction has descendants in the wallet")); return feebumper::Result::INVALID_PARAMETER; } { if (wallet.chain().hasDescendantsInMempool(wtx.GetHash())) { - errors.push_back(Untranslated("Transaction has descendants in the mempool")); + errors.emplace_back(Untranslated("Transaction has descendants in the mempool")); return feebumper::Result::INVALID_PARAMETER; } } if (wallet.GetTxDepthInMainChain(wtx) != 0) { - errors.push_back(Untranslated("Transaction has been mined, or is conflicted with a mined transaction")); + errors.emplace_back(Untranslated("Transaction has been mined, or is conflicted with a mined transaction")); return feebumper::Result::WALLET_ERROR; } if (!SignalsOptInRBF(*wtx.tx)) { - errors.push_back(Untranslated("Transaction is not BIP 125 replaceable")); + errors.emplace_back(Untranslated("Transaction is not BIP 125 replaceable")); return feebumper::Result::WALLET_ERROR; } if (wtx.mapValue.count("replaced_by_txid")) { - errors.push_back(strprintf(Untranslated("Cannot bump transaction %s which was already bumped by transaction %s"), wtx.GetHash().ToString(), wtx.mapValue.at("replaced_by_txid"))); + errors.push_back(Untranslated(strprintf("Cannot bump transaction %s which was already bumped by transaction %s", wtx.GetHash().ToString(), wtx.mapValue.at("replaced_by_txid")))); return feebumper::Result::WALLET_ERROR; } @@ -55,7 +55,7 @@ static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWallet // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) isminefilter filter = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; if (!AllInputsMine(wallet, *wtx.tx, filter)) { - errors.push_back(Untranslated("Transaction contains inputs that don't belong to this wallet")); + errors.emplace_back(Untranslated("Transaction contains inputs that don't belong to this wallet")); return feebumper::Result::WALLET_ERROR; } } @@ -74,10 +74,10 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CMutableTrans CFeeRate minMempoolFeeRate = wallet.chain().mempoolMinFee(); if (newFeerate.GetFeePerK() < minMempoolFeeRate.GetFeePerK()) { - errors.push_back(strprintf( - Untranslated("New fee rate (%s) is lower than the minimum fee rate (%s) to get into the mempool -- "), + errors.push_back(Untranslated( + strprintf("New fee rate (%s) is lower than the minimum fee rate (%s) to get into the mempool -- ", FormatMoney(newFeerate.GetFeePerK()), - FormatMoney(minMempoolFeeRate.GetFeePerK()))); + FormatMoney(minMempoolFeeRate.GetFeePerK())))); return feebumper::Result::WALLET_ERROR; } @@ -89,7 +89,7 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CMutableTrans std::optional<CAmount> combined_bump_fee = wallet.chain().calculateCombinedBumpFee(reused_inputs, newFeerate); if (!combined_bump_fee.has_value()) { - errors.push_back(strprintf(Untranslated("Failed to calculate bump fees, because unconfirmed UTXOs depend on enormous cluster of unconfirmed transactions."))); + errors.push_back(Untranslated(strprintf("Failed to calculate bump fees, because unconfirmed UTXOs depend on enormous cluster of unconfirmed transactions."))); } CAmount new_total_fee = newFeerate.GetFee(maxTxSize) + combined_bump_fee.value(); @@ -99,23 +99,23 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CMutableTrans CAmount minTotalFee = old_fee + incrementalRelayFee.GetFee(maxTxSize); if (new_total_fee < minTotalFee) { - errors.push_back(strprintf(Untranslated("Insufficient total fee %s, must be at least %s (oldFee %s + incrementalFee %s)"), - FormatMoney(new_total_fee), FormatMoney(minTotalFee), FormatMoney(old_fee), FormatMoney(incrementalRelayFee.GetFee(maxTxSize)))); + errors.push_back(Untranslated(strprintf("Insufficient total fee %s, must be at least %s (oldFee %s + incrementalFee %s)", + FormatMoney(new_total_fee), FormatMoney(minTotalFee), FormatMoney(old_fee), FormatMoney(incrementalRelayFee.GetFee(maxTxSize))))); return feebumper::Result::INVALID_PARAMETER; } CAmount requiredFee = GetRequiredFee(wallet, maxTxSize); if (new_total_fee < requiredFee) { - errors.push_back(strprintf(Untranslated("Insufficient total fee (cannot be less than required fee %s)"), - FormatMoney(requiredFee))); + errors.push_back(Untranslated(strprintf("Insufficient total fee (cannot be less than required fee %s)", + FormatMoney(requiredFee)))); return feebumper::Result::INVALID_PARAMETER; } // Check that in all cases the new fee doesn't violate maxTxFee const CAmount max_tx_fee = wallet.m_default_max_tx_fee; if (new_total_fee > max_tx_fee) { - errors.push_back(strprintf(Untranslated("Specified or calculated fee %s is too high (cannot be higher than -maxtxfee %s)"), - FormatMoney(new_total_fee), FormatMoney(max_tx_fee))); + errors.push_back(Untranslated(strprintf("Specified or calculated fee %s is too high (cannot be higher than -maxtxfee %s)", + FormatMoney(new_total_fee), FormatMoney(max_tx_fee)))); return feebumper::Result::WALLET_ERROR; } @@ -167,7 +167,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo { // For now, cannot specify both new outputs to use and an output index to send change if (!outputs.empty() && original_change_index.has_value()) { - errors.push_back(Untranslated("The options 'outputs' and 'original_change_index' are incompatible. You can only either specify a new set of outputs, or designate a change output to be recycled.")); + errors.emplace_back(Untranslated("The options 'outputs' and 'original_change_index' are incompatible. You can only either specify a new set of outputs, or designate a change output to be recycled.")); return Result::INVALID_PARAMETER; } @@ -178,14 +178,14 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo errors.clear(); auto it = wallet.mapWallet.find(txid); if (it == wallet.mapWallet.end()) { - errors.push_back(Untranslated("Invalid or non-wallet transaction id")); + errors.emplace_back(Untranslated("Invalid or non-wallet transaction id")); return Result::INVALID_ADDRESS_OR_KEY; } const CWalletTx& wtx = it->second; // Make sure that original_change_index is valid if (original_change_index.has_value() && original_change_index.value() >= wtx.tx->vout.size()) { - errors.push_back(Untranslated("Change position is out of range")); + errors.emplace_back(Untranslated("Change position is out of range")); return Result::INVALID_PARAMETER; } @@ -201,7 +201,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo for (const CTxIn& txin : wtx.tx->vin) { const Coin& coin = coins.at(txin.prevout); if (coin.out.IsNull()) { - errors.push_back(Untranslated(strprintf("%s:%u is already spent", txin.prevout.hash.GetHex(), txin.prevout.n))); + errors.emplace_back(Untranslated(strprintf("%s:%u is already spent", txin.prevout.hash.GetHex(), txin.prevout.n))); return Result::MISC_ERROR; } PreselectedInput& preset_txin = new_coin_control.Select(txin.prevout); @@ -319,7 +319,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo auto res = CreateTransaction(wallet, recipients, /*change_pos=*/std::nullopt, new_coin_control, false); if (!res) { - errors.push_back(Untranslated("Unable to create transaction.") + Untranslated(" ") + util::ErrorString(res)); + errors.emplace_back(Untranslated("Unable to create transaction.") + Untranslated(" ") + util::ErrorString(res)); return Result::WALLET_ERROR; } @@ -361,7 +361,7 @@ Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransacti } auto it = txid.IsNull() ? wallet.mapWallet.end() : wallet.mapWallet.find(txid); if (it == wallet.mapWallet.end()) { - errors.push_back(Untranslated("Invalid or non-wallet transaction id")); + errors.emplace_back(Untranslated("Invalid or non-wallet transaction id")); return Result::MISC_ERROR; } const CWalletTx& oldWtx = it->second; @@ -382,7 +382,7 @@ Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransacti // mark the original tx as bumped bumped_txid = tx->GetHash(); if (!wallet.MarkReplaced(oldWtx.GetHash(), bumped_txid)) { - errors.push_back(Untranslated("Created new bumpfee transaction but could not mark the original transaction as replaced")); + errors.emplace_back(Untranslated("Created new bumpfee transaction but could not mark the original transaction as replaced")); } return Result::OK; } diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 14d22bb54e..cfd09a2e10 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <common/args.h> #include <init.h> diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 81f5e30082..2b5c021cda 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -69,7 +69,7 @@ bool VerifyWallets(WalletContext& context) // Pass write=false because no need to write file and probably // better not to. If unnamed wallet needs to be added next startup // and the setting is empty, this code will just run again. - chain.overwriteRwSetting("wallet", wallets, /*write=*/false); + chain.overwriteRwSetting("wallet", std::move(wallets), interfaces::SettingsAction::SKIP_WRITE); } } diff --git a/src/wallet/migrate.h b/src/wallet/migrate.h index 58c8c0adf4..16eadeb019 100644 --- a/src/wallet/migrate.h +++ b/src/wallet/migrate.h @@ -115,6 +115,7 @@ public: bool TxnBegin() override { return false; } bool TxnCommit() override { return false; } bool TxnAbort() override { return false; } + bool HasActiveTxn() override { return false; } }; //! Return object giving access to Berkeley Read Only database at specified path. diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index 838d062108..1c2951deee 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <core_io.h> #include <key_io.h> diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 20d09b1d9a..bfd7249c04 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -1,8 +1,8 @@ -// Copyright (c) 2009-2022 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chain.h> #include <clientversion.h> @@ -534,7 +534,7 @@ RPCHelpMan importwallet() // Use uiInterface.ShowProgress instead of pwallet.ShowProgress because pwallet.ShowProgress has a cancel button tied to AbortRescan which // we don't want for this progress bar showing the import progress. uiInterface.ShowProgress does not have a cancel button. - pwallet->chain().showProgress(strprintf("%s " + _("Importing…").translated, pwallet->GetDisplayName()), 0, false); // show progress dialog in GUI + pwallet->chain().showProgress(strprintf("%s %s", pwallet->GetDisplayName(), _("Importing…").translated), 0, false); // show progress dialog in GUI std::vector<std::tuple<CKey, int64_t, bool, std::string>> keys; std::vector<std::pair<CScript, int64_t>> scripts; while (file.good()) { @@ -549,7 +549,7 @@ RPCHelpMan importwallet() continue; CKey key = DecodeSecret(vstr[0]); if (key.IsValid()) { - int64_t nTime = ParseISO8601DateTime(vstr[1]); + int64_t nTime{ParseISO8601DateTime(vstr[1]).value_or(0)}; std::string strLabel; bool fLabel = true; for (unsigned int nStr = 2; nStr < vstr.size(); nStr++) { @@ -569,7 +569,7 @@ RPCHelpMan importwallet() } else if(IsHex(vstr[0])) { std::vector<unsigned char> vData(ParseHex(vstr[0])); CScript script = CScript(vData.begin(), vData.end()); - int64_t birth_time = ParseISO8601DateTime(vstr[1]); + int64_t birth_time{ParseISO8601DateTime(vstr[1]).value_or(0)}; if (birth_time > 0) nTimeBegin = std::min(nTimeBegin, birth_time); scripts.emplace_back(script, birth_time); } @@ -757,7 +757,7 @@ RPCHelpMan dumpwallet() std::sort(vKeyBirth.begin(), vKeyBirth.end()); // produce output - file << strprintf("# Wallet dump created by %s %s\n", PACKAGE_NAME, FormatFullVersion()); + file << strprintf("# Wallet dump created by %s %s\n", CLIENT_NAME, FormatFullVersion()); file << strprintf("# * Created on %s\n", FormatISO8601DateTime(GetTime())); file << strprintf("# * Best block at time of backup was %i (%s),\n", wallet.GetLastBlockHeight(), wallet.GetLastBlockHash().ToString()); file << strprintf("# mined on %s\n", FormatISO8601DateTime(block_time)); diff --git a/src/wallet/rpc/util.cpp b/src/wallet/rpc/util.cpp index 67b5ae0fe2..414f0deeb2 100644 --- a/src/wallet/rpc/util.cpp +++ b/src/wallet/rpc/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2022 The Bitcoin Core developers +// Copyright (c) 2011-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,26 +14,10 @@ #include <string_view> #include <univalue.h> -#include <boost/date_time/posix_time/posix_time.hpp> - namespace wallet { static const std::string WALLET_ENDPOINT_BASE = "/wallet/"; const std::string HELP_REQUIRING_PASSPHRASE{"\nRequires wallet passphrase to be set with walletpassphrase call if wallet is encrypted.\n"}; -int64_t ParseISO8601DateTime(const std::string& str) -{ - static const boost::posix_time::ptime epoch = boost::posix_time::from_time_t(0); - static const std::locale loc(std::locale::classic(), - new boost::posix_time::time_input_facet("%Y-%m-%dT%H:%M:%SZ")); - std::istringstream iss(str); - iss.imbue(loc); - boost::posix_time::ptime ptime(boost::date_time::not_a_date_time); - iss >> ptime; - if (ptime.is_not_a_date_time() || epoch > ptime) - return 0; - return (ptime - epoch).total_seconds(); -} - bool GetAvoidReuseFlag(const CWallet& wallet, const UniValue& param) { bool can_avoid_reuse = wallet.IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE); bool avoid_reuse = param.isNull() ? can_avoid_reuse : param.get_bool(); @@ -91,7 +75,7 @@ std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& reques RPC_WALLET_NOT_FOUND, "No wallet is loaded. Load a wallet using loadwallet or create a new one with createwallet. (Note: A default wallet is no longer automatically created)"); } throw JSONRPCError(RPC_WALLET_NOT_SPECIFIED, - "Wallet file not specified (must request wallet RPC through /wallet/<filename> uri-path)."); + "Multiple wallets are loaded. Please select which wallet to use by requesting the RPC through the /wallet/<walletname> URI path."); } void EnsureWalletIsUnlocked(const CWallet& wallet) diff --git a/src/wallet/rpc/util.h b/src/wallet/rpc/util.h index 2fdba04352..002d0355e5 100644 --- a/src/wallet/rpc/util.h +++ b/src/wallet/rpc/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 The Bitcoin Core developers +// Copyright (c) 2017-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -51,7 +51,6 @@ std::string LabelFromValue(const UniValue& value); void PushParentDescriptors(const CWallet& wallet, const CScript& script_pubkey, UniValue& entry); void HandleWalletError(const std::shared_ptr<CWallet> wallet, DatabaseStatus& status, bilingual_str& error); -int64_t ParseISO8601DateTime(const std::string& str); void AppendLastProcessedBlock(UniValue& entry, const CWallet& wallet) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); } // namespace wallet diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index 39582b3f6a..5140ac8c05 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <core_io.h> #include <key_io.h> diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index 04c02b0dcc..b924239073 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -44,6 +44,7 @@ public: bool TxnBegin() override { return true; } bool TxnCommit() override { return true; } bool TxnAbort() override { return true; } + bool HasActiveTxn() override { return false; } }; /** A dummy WalletDatabase that does nothing and never fails. Only used by salvage. @@ -99,7 +100,7 @@ bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bil newFilename.c_str(), DB_AUTO_COMMIT); if (result != 0) { - error = strprintf(Untranslated("Failed to rename %s to %s"), filename, newFilename); + error = Untranslated(strprintf("Failed to rename %s to %s", filename, newFilename)); return false; } @@ -116,10 +117,10 @@ bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bil Db db(env->dbenv.get(), 0); result = db.verify(newFilename.c_str(), nullptr, &strDump, DB_SALVAGE | DB_AGGRESSIVE); if (result == DB_VERIFY_BAD) { - warnings.push_back(Untranslated("Salvage: Database salvage found errors, all data may not be recoverable.")); + warnings.emplace_back(Untranslated("Salvage: Database salvage found errors, all data may not be recoverable.")); } if (result != 0 && result != DB_VERIFY_BAD) { - error = strprintf(Untranslated("Salvage: Database salvage failed with result %d."), result); + error = Untranslated(strprintf("Salvage: Database salvage failed with result %d.", result)); return false; } @@ -143,7 +144,7 @@ bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bil break; getline(strDump, valueHex); if (valueHex == DATA_END) { - warnings.push_back(Untranslated("Salvage: WARNING: Number of keys in data does not match number of values.")); + warnings.emplace_back(Untranslated("Salvage: WARNING: Number of keys in data does not match number of values.")); break; } salvagedData.emplace_back(ParseHex(keyHex), ParseHex(valueHex)); @@ -152,7 +153,7 @@ bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bil bool fSuccess; if (keyHex != DATA_END) { - warnings.push_back(Untranslated("Salvage: WARNING: Unexpected end of file while reading salvage output.")); + warnings.emplace_back(Untranslated("Salvage: WARNING: Unexpected end of file while reading salvage output.")); fSuccess = false; } else { fSuccess = (result == 0); @@ -160,7 +161,7 @@ bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bil if (salvagedData.empty()) { - error = strprintf(Untranslated("Salvage(aggressive) found no records in %s."), newFilename); + error = Untranslated(strprintf("Salvage(aggressive) found no records in %s.", newFilename)); return false; } @@ -172,7 +173,7 @@ bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bil DB_CREATE, // Flags 0); if (ret > 0) { - error = strprintf(Untranslated("Cannot create database file %s"), filename); + error = Untranslated(strprintf("Cannot create database file %s", filename)); pdbCopy->close(0); return false; } @@ -203,7 +204,7 @@ bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bil if (!fReadOK) { - warnings.push_back(strprintf(Untranslated("WARNING: WalletBatch::Recover skipping %s: %s"), strType, strErr)); + warnings.push_back(Untranslated(strprintf("WARNING: WalletBatch::Recover skipping %s: %s", strType, strErr))); continue; } Dbt datKey(row.first.data(), row.first.size()); diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 46ec5dc1ac..23e2257b1e 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -1799,7 +1799,7 @@ std::optional<MigrationData> LegacyDataSPKM::MigrateToDescriptor() keyid_it++; continue; } - if (m_hd_chain.seed_id == meta.hd_seed_id || m_inactive_hd_chains.count(meta.hd_seed_id) > 0) { + if (!meta.hd_seed_id.IsNull() && (m_hd_chain.seed_id == meta.hd_seed_id || m_inactive_hd_chains.count(meta.hd_seed_id) > 0)) { keyid_it = keyids.erase(keyid_it); continue; } @@ -1807,6 +1807,12 @@ std::optional<MigrationData> LegacyDataSPKM::MigrateToDescriptor() keyid_it++; } + WalletBatch batch(m_storage.GetDatabase()); + if (!batch.TxnBegin()) { + LogPrintf("Error generating descriptors for migration, cannot initialize db transaction\n"); + return std::nullopt; + } + // keyids is now all non-HD keys. Each key will have its own combo descriptor for (const CKeyID& keyid : keyids) { CKey key; @@ -1837,8 +1843,8 @@ std::optional<MigrationData> LegacyDataSPKM::MigrateToDescriptor() // Make the DescriptorScriptPubKeyMan and get the scriptPubKeys auto desc_spk_man = std::make_unique<DescriptorScriptPubKeyMan>(m_storage, w_desc, /*keypool_size=*/0); - desc_spk_man->AddDescriptorKey(key, key.GetPubKey()); - desc_spk_man->TopUp(); + WITH_LOCK(desc_spk_man->cs_desc_man, desc_spk_man->AddDescriptorKeyWithDB(batch, key, key.GetPubKey())); + desc_spk_man->TopUpWithDB(batch); auto desc_spks = desc_spk_man->GetScriptPubKeys(); // Remove the scriptPubKeys from our current set @@ -1883,8 +1889,8 @@ std::optional<MigrationData> LegacyDataSPKM::MigrateToDescriptor() // Make the DescriptorScriptPubKeyMan and get the scriptPubKeys auto desc_spk_man = std::make_unique<DescriptorScriptPubKeyMan>(m_storage, w_desc, /*keypool_size=*/0); - desc_spk_man->AddDescriptorKey(master_key.key, master_key.key.GetPubKey()); - desc_spk_man->TopUp(); + WITH_LOCK(desc_spk_man->cs_desc_man, desc_spk_man->AddDescriptorKeyWithDB(batch, master_key.key, master_key.key.GetPubKey())); + desc_spk_man->TopUpWithDB(batch); auto desc_spks = desc_spk_man->GetScriptPubKeys(); // Remove the scriptPubKeys from our current set @@ -1950,9 +1956,9 @@ std::optional<MigrationData> LegacyDataSPKM::MigrateToDescriptor() if (!GetKey(keyid, key)) { continue; } - desc_spk_man->AddDescriptorKey(key, key.GetPubKey()); + WITH_LOCK(desc_spk_man->cs_desc_man, desc_spk_man->AddDescriptorKeyWithDB(batch, key, key.GetPubKey())); } - desc_spk_man->TopUp(); + desc_spk_man->TopUpWithDB(batch); auto desc_spks_set = desc_spk_man->GetScriptPubKeys(); desc_spks.insert(desc_spks.end(), desc_spks_set.begin(), desc_spks_set.end()); @@ -2019,13 +2025,26 @@ std::optional<MigrationData> LegacyDataSPKM::MigrateToDescriptor() // Make sure that we have accounted for all scriptPubKeys assert(spks.size() == 0); + + // Finalize transaction + if (!batch.TxnCommit()) { + LogPrintf("Error generating descriptors for migration, cannot commit db transaction\n"); + return std::nullopt; + } + return out; } bool LegacyDataSPKM::DeleteRecords() { + return RunWithinTxn(m_storage.GetDatabase(), /*process_desc=*/"delete legacy records", [&](WalletBatch& batch){ + return DeleteRecordsWithDB(batch); + }); +} + +bool LegacyDataSPKM::DeleteRecordsWithDB(WalletBatch& batch) +{ LOCK(cs_KeyStore); - WalletBatch batch(m_storage.GetDatabase()); return batch.EraseRecords(DBKeys::LEGACY_TYPES); } diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index ba3562c638..d8b6c90178 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -254,9 +254,9 @@ public: /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ template <typename... Params> - void WalletLogPrintf(const char* fmt, Params... parameters) const + void WalletLogPrintf(util::ConstevalFormatString<sizeof...(Params)> wallet_fmt, const Params&... params) const { - LogPrintf(("%s " + std::string{fmt}).c_str(), m_storage.GetDisplayName(), parameters...); + LogInfo("%s %s", m_storage.GetDisplayName(), tfm::format(wallet_fmt, params...)); }; /** Watch-only address added */ @@ -366,8 +366,9 @@ public: /** Get the DescriptorScriptPubKeyMans (with private keys) that have the same scriptPubKeys as this LegacyScriptPubKeyMan. * Does not modify this ScriptPubKeyMan. */ std::optional<MigrationData> MigrateToDescriptor(); - /** Delete all the records ofthis LegacyScriptPubKeyMan from disk*/ + /** Delete all the records of this LegacyScriptPubKeyMan from disk*/ bool DeleteRecords(); + bool DeleteRecordsWithDB(WalletBatch& batch); }; // Implements the full legacy wallet behavior @@ -582,6 +583,7 @@ public: class DescriptorScriptPubKeyMan : public ScriptPubKeyMan { + friend class LegacyDataSPKM; private: using ScriptPubKeyMap = std::map<CScript, int32_t>; // Map of scripts to descriptor range index using PubKeyMap = std::map<CPubKey, int32_t>; // Map of pubkeys involved in scripts to descriptor range index diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 7abf7f59c0..f45db1c16f 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -35,6 +35,11 @@ using common::TransactionErrorString; using interfaces::FoundBlock; using node::TransactionError; +TRACEPOINT_SEMAPHORE(coin_selection, selected_coins); +TRACEPOINT_SEMAPHORE(coin_selection, normal_create_tx_internal); +TRACEPOINT_SEMAPHORE(coin_selection, attempting_aps_create_tx); +TRACEPOINT_SEMAPHORE(coin_selection, aps_create_tx_internal); + namespace wallet { static constexpr size_t OUTPUT_GROUP_MAX_ENTRIES{100}; @@ -1159,7 +1164,7 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( return util::Error{err.empty() ?_("Insufficient funds") : err}; } const SelectionResult& result = *select_coins_res; - TRACE5(coin_selection, selected_coins, + TRACEPOINT(coin_selection, selected_coins, wallet.GetName().c_str(), GetAlgorithmName(result.GetAlgo()).c_str(), result.GetTarget(), @@ -1167,6 +1172,7 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( result.GetSelectedValue()); // vouts to the payees + txNew.vout.reserve(vecSend.size() + 1); // + 1 because of possible later insert for (const auto& recipient : vecSend) { txNew.vout.emplace_back(recipient.nAmount, GetScriptForDestination(recipient.dest)); @@ -1217,6 +1223,7 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( // behavior." bool use_anti_fee_sniping = true; const uint32_t default_sequence{coin_control.m_signal_bip125_rbf.value_or(wallet.m_signal_rbf) ? MAX_BIP125_RBF_SEQUENCE : CTxIn::MAX_SEQUENCE_NONFINAL}; + txNew.vin.reserve(selected_coins.size()); for (const auto& coin : selected_coins) { std::optional<uint32_t> sequence = coin_control.GetSequence(coin->outpoint); if (sequence) { @@ -1378,7 +1385,7 @@ util::Result<CreatedTransactionResult> CreateTransaction( LOCK(wallet.cs_wallet); auto res = CreateTransactionInternal(wallet, vecSend, change_pos, coin_control, sign); - TRACE4(coin_selection, normal_create_tx_internal, + TRACEPOINT(coin_selection, normal_create_tx_internal, wallet.GetName().c_str(), bool(res), res ? res->fee : 0, @@ -1387,7 +1394,7 @@ util::Result<CreatedTransactionResult> CreateTransaction( const auto& txr_ungrouped = *res; // try with avoidpartialspends unless it's enabled already if (txr_ungrouped.fee > 0 /* 0 means non-functional fee rate estimation */ && wallet.m_max_aps_fee > -1 && !coin_control.m_avoid_partial_spends) { - TRACE1(coin_selection, attempting_aps_create_tx, wallet.GetName().c_str()); + TRACEPOINT(coin_selection, attempting_aps_create_tx, wallet.GetName().c_str()); CCoinControl tmp_cc = coin_control; tmp_cc.m_avoid_partial_spends = true; @@ -1399,7 +1406,7 @@ util::Result<CreatedTransactionResult> CreateTransaction( auto txr_grouped = CreateTransactionInternal(wallet, vecSend, change_pos, tmp_cc, sign); // if fee of this alternative one is within the range of the max fee, we use this one const bool use_aps{txr_grouped.has_value() ? (txr_grouped->fee <= txr_ungrouped.fee + wallet.m_max_aps_fee) : false}; - TRACE5(coin_selection, aps_create_tx_internal, + TRACEPOINT(coin_selection, aps_create_tx_internal, wallet.GetName().c_str(), use_aps, txr_grouped.has_value(), diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index f2110ea3f7..a8c9f8a8ab 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <wallet/sqlite.h> @@ -282,7 +282,7 @@ void SQLiteDatabase::Open() // Now begin a transaction to acquire the exclusive lock. This lock won't be released until we close because of the exclusive locking mode. int ret = sqlite3_exec(m_db, "BEGIN EXCLUSIVE TRANSACTION", nullptr, nullptr, nullptr); if (ret != SQLITE_OK) { - throw std::runtime_error("SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of " PACKAGE_NAME "?\n"); + throw std::runtime_error("SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of " CLIENT_NAME "?\n"); } ret = sqlite3_exec(m_db, "COMMIT", nullptr, nullptr, nullptr); if (ret != SQLITE_OK) { diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index 6b84f34366..78a3accf89 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -95,6 +95,7 @@ public: bool TxnBegin() override; bool TxnCommit() override; bool TxnAbort() override; + bool HasActiveTxn() override { return m_txn; } }; /** An instance of this class represents one SQLite3 database. diff --git a/src/wallet/test/CMakeLists.txt b/src/wallet/test/CMakeLists.txt index 7332674242..8b442b262b 100644 --- a/src/wallet/test/CMakeLists.txt +++ b/src/wallet/test/CMakeLists.txt @@ -14,7 +14,6 @@ target_sources(test_bitcoin init_tests.cpp ismine_tests.cpp psbt_wallet_tests.cpp - rpc_util_tests.cpp scriptpubkeyman_tests.cpp spend_tests.cpp wallet_crypto_tests.cpp diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 2fac356263..41951e84c8 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <boost/test/unit_test.hpp> @@ -28,15 +28,15 @@ inline std::ostream& operator<<(std::ostream& os, const std::pair<const Serializ { Span key{kv.first}, value{kv.second}; os << "(\"" << std::string_view{reinterpret_cast<const char*>(key.data()), key.size()} << "\", \"" - << std::string_view{reinterpret_cast<const char*>(key.data()), key.size()} << "\")"; + << std::string_view{reinterpret_cast<const char*>(value.data()), value.size()} << "\")"; return os; } namespace wallet { -static Span<const std::byte> StringBytes(std::string_view str) +inline std::span<const std::byte> StringBytes(std::string_view str) { - return AsBytes<const char>({str.data(), str.size()}); + return std::as_bytes(std::span{str}); } static SerializeData StringData(std::string_view str) diff --git a/src/wallet/test/fuzz/CMakeLists.txt b/src/wallet/test/fuzz/CMakeLists.txt index c30671db48..7b071e2f12 100644 --- a/src/wallet/test/fuzz/CMakeLists.txt +++ b/src/wallet/test/fuzz/CMakeLists.txt @@ -9,8 +9,8 @@ target_sources(fuzz crypter.cpp fees.cpp $<$<BOOL:${USE_SQLITE}>:${CMAKE_CURRENT_LIST_DIR}/notifications.cpp> - parse_iso8601.cpp $<$<BOOL:${USE_SQLITE}>:${CMAKE_CURRENT_LIST_DIR}/scriptpubkeyman.cpp> + spend.cpp wallet_bdb_parser.cpp ) target_link_libraries(fuzz bitcoin_wallet) diff --git a/src/wallet/test/fuzz/coinselection.cpp b/src/wallet/test/fuzz/coinselection.cpp index 31fa00c0a2..ed77befac2 100644 --- a/src/wallet/test/fuzz/coinselection.cpp +++ b/src/wallet/test/fuzz/coinselection.cpp @@ -218,6 +218,7 @@ FUZZ_TARGET(coin_grinder_is_optimal) FUZZ_TARGET(coinselection) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; std::vector<COutput> utxo_pool; diff --git a/src/wallet/test/fuzz/notifications.cpp b/src/wallet/test/fuzz/notifications.cpp index a7015f6685..7b85fb26fc 100644 --- a/src/wallet/test/fuzz/notifications.cpp +++ b/src/wallet/test/fuzz/notifications.cpp @@ -18,6 +18,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/wallet.h> #include <test/util/setup_common.h> #include <tinyformat.h> #include <uint256.h> @@ -53,133 +54,21 @@ void initialize_setup() g_setup = testing_setup.get(); } -void ImportDescriptors(CWallet& wallet, const std::string& seed_insecure) -{ - const std::vector<std::string> DESCS{ - "pkh(%s/%s/*)", - "sh(wpkh(%s/%s/*))", - "tr(%s/%s/*)", - "wpkh(%s/%s/*)", - }; - - for (const std::string& desc_fmt : DESCS) { - for (bool internal : {true, false}) { - const auto descriptor{(strprintf)(desc_fmt, "[5aa9973a/66h/4h/2h]" + seed_insecure, int{internal})}; - - FlatSigningProvider keys; - std::string error; - auto parsed_desc = std::move(Parse(descriptor, keys, error, /*require_checksum=*/false).at(0)); - assert(parsed_desc); - assert(error.empty()); - assert(parsed_desc->IsRange()); - assert(parsed_desc->IsSingleType()); - assert(!keys.keys.empty()); - WalletDescriptor w_desc{std::move(parsed_desc), /*creation_time=*/0, /*range_start=*/0, /*range_end=*/1, /*next_index=*/0}; - assert(!wallet.GetDescriptorScriptPubKeyMan(w_desc)); - LOCK(wallet.cs_wallet); - auto spk_manager{wallet.AddWalletDescriptor(w_desc, keys, /*label=*/"", internal)}; - assert(spk_manager); - wallet.AddActiveScriptPubKeyMan(spk_manager->GetID(), *Assert(w_desc.descriptor->GetOutputType()), internal); - } - } -} - -/** - * Wraps a descriptor wallet for fuzzing. - */ -struct FuzzedWallet { - std::shared_ptr<CWallet> wallet; - FuzzedWallet(const std::string& name, const std::string& seed_insecure) - { - auto& chain{*Assert(g_setup->m_node.chain)}; - wallet = std::make_shared<CWallet>(&chain, name, CreateMockableWalletDatabase()); - { - LOCK(wallet->cs_wallet); - wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); - auto height{*Assert(chain.getHeight())}; - wallet->SetLastBlockProcessed(height, chain.getBlockHash(height)); - } - wallet->m_keypool_size = 1; // Avoid timeout in TopUp() - assert(wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); - ImportDescriptors(*wallet, seed_insecure); - } - CTxDestination GetDestination(FuzzedDataProvider& fuzzed_data_provider) - { - auto type{fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES)}; - if (fuzzed_data_provider.ConsumeBool()) { - return *Assert(wallet->GetNewDestination(type, "")); - } else { - return *Assert(wallet->GetNewChangeDestination(type)); - } - } - CScript GetScriptPubKey(FuzzedDataProvider& fuzzed_data_provider) { return GetScriptForDestination(GetDestination(fuzzed_data_provider)); } - void FundTx(FuzzedDataProvider& fuzzed_data_provider, CMutableTransaction tx) - { - // The fee of "tx" is 0, so this is the total input and output amount - const CAmount total_amt{ - std::accumulate(tx.vout.begin(), tx.vout.end(), CAmount{}, [](CAmount t, const CTxOut& out) { return t + out.nValue; })}; - const uint32_t tx_size(GetVirtualTransactionSize(CTransaction{tx})); - std::set<int> subtract_fee_from_outputs; - if (fuzzed_data_provider.ConsumeBool()) { - for (size_t i{}; i < tx.vout.size(); ++i) { - if (fuzzed_data_provider.ConsumeBool()) { - subtract_fee_from_outputs.insert(i); - } - } - } - std::vector<CRecipient> recipients; - for (size_t idx = 0; idx < tx.vout.size(); idx++) { - const CTxOut& tx_out = tx.vout[idx]; - CTxDestination dest; - ExtractDestination(tx_out.scriptPubKey, dest); - CRecipient recipient = {dest, tx_out.nValue, subtract_fee_from_outputs.count(idx) == 1}; - recipients.push_back(recipient); - } - CCoinControl coin_control; - coin_control.m_allow_other_inputs = fuzzed_data_provider.ConsumeBool(); - CallOneOf( - fuzzed_data_provider, [&] { coin_control.destChange = GetDestination(fuzzed_data_provider); }, - [&] { coin_control.m_change_type.emplace(fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES)); }, - [&] { /* no op (leave uninitialized) */ }); - coin_control.fAllowWatchOnly = fuzzed_data_provider.ConsumeBool(); - coin_control.m_include_unsafe_inputs = fuzzed_data_provider.ConsumeBool(); - { - auto& r{coin_control.m_signal_bip125_rbf}; - CallOneOf( - fuzzed_data_provider, [&] { r = true; }, [&] { r = false; }, [&] { r = std::nullopt; }); - } - coin_control.m_feerate = CFeeRate{ - // A fee of this range should cover all cases - fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(0, 2 * total_amt), - tx_size, - }; - if (fuzzed_data_provider.ConsumeBool()) { - *coin_control.m_feerate += GetMinimumFeeRate(*wallet, coin_control, nullptr); - } - coin_control.fOverrideFeeRate = fuzzed_data_provider.ConsumeBool(); - // Add solving data (m_external_provider and SelectExternal)? - - int change_position{fuzzed_data_provider.ConsumeIntegralInRange<int>(-1, tx.vout.size() - 1)}; - bilingual_str error; - // Clear tx.vout since it is not meant to be used now that we are passing outputs directly. - // This sets us up for a future PR to completely remove tx from the function signature in favor of passing inputs directly - tx.vout.clear(); - (void)FundTransaction(*wallet, tx, recipients, change_position, /*lockUnspents=*/false, coin_control); - } -}; - FUZZ_TARGET(wallet_notifications, .init = initialize_setup) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; // The total amount, to be distributed to the wallets a and b in txs // without fee. Thus, the balance of the wallets should always equal the // total amount. const auto total_amount{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY / 100000)}; FuzzedWallet a{ + *g_setup->m_node.chain, "fuzzed_wallet_a", "tprv8ZgxMBicQKsPd1QwsGgzfu2pcPYbBosZhJknqreRHgsWx32nNEhMjGQX2cgFL8n6wz9xdDYwLcs78N4nsCo32cxEX8RBtwGsEGgybLiQJfk", }; FuzzedWallet b{ + *g_setup->m_node.chain, "fuzzed_wallet_b", "tprv8ZgxMBicQKsPfCunYTF18sEmEyjz8TfhGnZ3BoVAhkqLv7PLkQgmoG2Ecsp4JuqciWnkopuEwShit7st743fdmB9cMD4tznUkcs33vK51K9", }; @@ -196,7 +85,7 @@ FUZZ_TARGET(wallet_notifications, .init = initialize_setup) auto& [coins, block]{chain.back()}; coins.emplace(total_amount, COutPoint{Txid::FromUint256(uint256::ONE), 1}); } - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 200) + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 20) { CallOneOf( fuzzed_data_provider, diff --git a/src/wallet/test/fuzz/scriptpubkeyman.cpp b/src/wallet/test/fuzz/scriptpubkeyman.cpp index 091d42f6cf..b0d8628a91 100644 --- a/src/wallet/test/fuzz/scriptpubkeyman.cpp +++ b/src/wallet/test/fuzz/scriptpubkeyman.cpp @@ -85,6 +85,7 @@ static DescriptorScriptPubKeyMan* CreateDescriptor(WalletDescriptor& wallet_desc FUZZ_TARGET(scriptpubkeyman, .init = initialize_spkm) { + SeedRandomStateForTest(SeedRand::ZEROS); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; const auto& node{g_setup->m_node}; Chainstate& chainstate{node.chainman->ActiveChainstate()}; diff --git a/src/wallet/test/fuzz/spend.cpp b/src/wallet/test/fuzz/spend.cpp new file mode 100644 index 0000000000..c4c04bce4b --- /dev/null +++ b/src/wallet/test/fuzz/spend.cpp @@ -0,0 +1,102 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/fuzz/util/wallet.h> +#include <test/util/random.h> +#include <test/util/setup_common.h> +#include <wallet/coincontrol.h> +#include <wallet/context.h> +#include <wallet/spend.h> +#include <wallet/test/util.h> +#include <wallet/wallet.h> +#include <validation.h> +#include <addresstype.h> + +using util::ToString; + +namespace wallet { +namespace { +const TestingSetup* g_setup; + +void initialize_setup() +{ + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); +} + +FUZZ_TARGET(wallet_create_transaction, .init = initialize_setup) +{ + SeedRandomStateForTest(SeedRand::ZEROS); + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + const auto& node = g_setup->m_node; + Chainstate& chainstate{node.chainman->ActiveChainstate()}; + ArgsManager& args = *node.args; + args.ForceSetArg("-dustrelayfee", ToString(fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(0, MAX_MONEY))); + FuzzedWallet fuzzed_wallet{ + *g_setup->m_node.chain, + "fuzzed_wallet_a", + "tprv8ZgxMBicQKsPd1QwsGgzfu2pcPYbBosZhJknqreRHgsWx32nNEhMjGQX2cgFL8n6wz9xdDYwLcs78N4nsCo32cxEX8RBtwGsEGgybLiQJfk", + }; + + CCoinControl coin_control; + if (fuzzed_data_provider.ConsumeBool()) coin_control.m_version = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + coin_control.m_avoid_partial_spends = fuzzed_data_provider.ConsumeBool(); + coin_control.m_include_unsafe_inputs = fuzzed_data_provider.ConsumeBool(); + if (fuzzed_data_provider.ConsumeBool()) coin_control.m_confirm_target = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, 999'000); + coin_control.destChange = fuzzed_data_provider.ConsumeBool() ? fuzzed_wallet.GetDestination(fuzzed_data_provider) : ConsumeTxDestination(fuzzed_data_provider); + if (fuzzed_data_provider.ConsumeBool()) coin_control.m_change_type = fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES); + if (fuzzed_data_provider.ConsumeBool()) coin_control.m_feerate = CFeeRate(ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)); + coin_control.m_allow_other_inputs = fuzzed_data_provider.ConsumeBool(); + coin_control.m_locktime = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + coin_control.fOverrideFeeRate = fuzzed_data_provider.ConsumeBool(); + + int next_locktime{0}; + CAmount all_values{0}; + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + CMutableTransaction tx; + tx.nLockTime = next_locktime++; + tx.vout.resize(1); + CAmount n_value{ConsumeMoney(fuzzed_data_provider)}; + all_values += n_value; + if (all_values > MAX_MONEY) return; + tx.vout[0].nValue = n_value; + tx.vout[0].scriptPubKey = GetScriptForDestination(fuzzed_wallet.GetDestination(fuzzed_data_provider)); + LOCK(fuzzed_wallet.wallet->cs_wallet); + auto txid{tx.GetHash()}; + auto ret{fuzzed_wallet.wallet->mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)), TxStateConfirmed{chainstate.m_chain.Tip()->GetBlockHash(), chainstate.m_chain.Height(), /*index=*/0}))}; + assert(ret.second); + } + + std::vector<CRecipient> recipients; + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { + CTxDestination destination; + CallOneOf( + fuzzed_data_provider, + [&] { + destination = fuzzed_wallet.GetDestination(fuzzed_data_provider); + }, + [&] { + CScript script; + script << OP_RETURN; + destination = CNoDestination{script}; + }, + [&] { + destination = ConsumeTxDestination(fuzzed_data_provider); + } + ); + recipients.push_back({destination, + /*nAmount=*/ConsumeMoney(fuzzed_data_provider), + /*fSubtractFeeFromAmount=*/fuzzed_data_provider.ConsumeBool()}); + } + + std::optional<unsigned int> change_pos; + if (fuzzed_data_provider.ConsumeBool()) change_pos = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + (void)CreateTransaction(*fuzzed_wallet.wallet, recipients, change_pos, coin_control); +} +} // namespace +} // namespace wallet diff --git a/src/wallet/test/fuzz/wallet_bdb_parser.cpp b/src/wallet/test/fuzz/wallet_bdb_parser.cpp index 5ec24faede..6482b65d06 100644 --- a/src/wallet/test/fuzz/wallet_bdb_parser.cpp +++ b/src/wallet/test/fuzz/wallet_bdb_parser.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index 8deab74fac..f6688ed30a 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -684,7 +684,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); - scriptPubKey << OP_0 << "aabb"_hex_v_u8; + scriptPubKey << OP_0 << "aabb"_hex; result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); @@ -699,7 +699,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); - scriptPubKey << OP_16 << "aabb"_hex_v_u8; + scriptPubKey << OP_16 << "aabb"_hex; result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); diff --git a/src/wallet/test/rpc_util_tests.cpp b/src/wallet/test/rpc_util_tests.cpp deleted file mode 100644 index 32f6f5ab46..0000000000 --- a/src/wallet/test/rpc_util_tests.cpp +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) 2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <wallet/rpc/util.h> - -#include <boost/test/unit_test.hpp> - -namespace wallet { - -BOOST_AUTO_TEST_SUITE(wallet_util_tests) - -BOOST_AUTO_TEST_CASE(util_ParseISO8601DateTime) -{ - BOOST_CHECK_EQUAL(ParseISO8601DateTime("1970-01-01T00:00:00Z"), 0); - BOOST_CHECK_EQUAL(ParseISO8601DateTime("1960-01-01T00:00:00Z"), 0); - BOOST_CHECK_EQUAL(ParseISO8601DateTime("2000-01-01T00:00:01Z"), 946684801); - BOOST_CHECK_EQUAL(ParseISO8601DateTime("2011-09-30T23:36:17Z"), 1317425777); - BOOST_CHECK_EQUAL(ParseISO8601DateTime("2100-12-31T23:59:59Z"), 4133980799); -} - -BOOST_AUTO_TEST_SUITE_END() - -} // namespace wallet diff --git a/src/wallet/test/util.h b/src/wallet/test/util.h index fc7674e961..c8a89c0e64 100644 --- a/src/wallet/test/util.h +++ b/src/wallet/test/util.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_WALLET_TEST_UTIL_H #define BITCOIN_WALLET_TEST_UTIL_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <addresstype.h> #include <wallet/db.h> @@ -95,6 +95,7 @@ public: bool TxnBegin() override { return m_pass; } bool TxnCommit() override { return m_pass; } bool TxnAbort() override { return m_pass; } + bool HasActiveTxn() override { return false; } }; /** A WalletDatabase whose contents and return values can be modified as needed for testing diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 5a520cbfe9..b5de4b4b3d 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -334,12 +334,11 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // concurrently, ensuring no race conditions occur during either process. BOOST_FIXTURE_TEST_CASE(write_wallet_settings_concurrently, TestingSetup) { - WalletContext context; - context.chain = m_node.chain.get(); + auto chain = m_node.chain.get(); const auto NUM_WALLETS{5}; // Since we're counting the number of wallets, ensure we start without any. - BOOST_REQUIRE(context.chain->getRwSetting("wallet").isNull()); + BOOST_REQUIRE(chain->getRwSetting("wallet").isNull()); const auto& check_concurrent_wallet = [&](const auto& settings_function, int num_expected_wallets) { std::vector<std::thread> threads; @@ -347,19 +346,19 @@ BOOST_FIXTURE_TEST_CASE(write_wallet_settings_concurrently, TestingSetup) for (auto i{0}; i < NUM_WALLETS; ++i) threads.emplace_back(settings_function, i); for (auto& t : threads) t.join(); - auto wallets = context.chain->getRwSetting("wallet"); + auto wallets = chain->getRwSetting("wallet"); BOOST_CHECK_EQUAL(wallets.getValues().size(), num_expected_wallets); }; // Add NUM_WALLETS wallets concurrently, ensure we end up with NUM_WALLETS stored. - check_concurrent_wallet([&context](int i) { - Assert(AddWalletSetting(*context.chain, strprintf("wallet_%d", i))); + check_concurrent_wallet([&chain](int i) { + Assert(AddWalletSetting(*chain, strprintf("wallet_%d", i))); }, /*num_expected_wallets=*/NUM_WALLETS); // Remove NUM_WALLETS wallets concurrently, ensure we end up with 0 wallets. - check_concurrent_wallet([&context](int i) { - Assert(RemoveWalletSetting(*context.chain, strprintf("wallet_%d", i))); + check_concurrent_wallet([&chain](int i) { + Assert(RemoveWalletSetting(*chain, strprintf("wallet_%d", i))); }, /*num_expected_wallets=*/0); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 83e96adf07..b971be5ddd 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -5,7 +5,7 @@ #include <wallet/wallet.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <addresstype.h> #include <blockfilter.h> #include <chain.h> @@ -290,7 +290,7 @@ std::shared_ptr<CWallet> LoadWalletInternal(WalletContext& context, const std::s // Legacy wallets are being deprecated, warn if the loaded wallet is legacy if (!wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { - warnings.push_back(_("Wallet loaded successfully. The legacy wallet type is being deprecated and support for creating and opening legacy wallets will be removed in the future. Legacy wallets can be migrated to a descriptor wallet with migratewallet.")); + warnings.emplace_back(_("Wallet loaded successfully. The legacy wallet type is being deprecated and support for creating and opening legacy wallets will be removed in the future. Legacy wallets can be migrated to a descriptor wallet with migratewallet.")); } NotifyWalletLoaded(context, wallet); @@ -483,7 +483,7 @@ std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& // Legacy wallets are being deprecated, warn if a newly created wallet is legacy if (!(wallet_creation_flags & WALLET_FLAG_DESCRIPTORS)) { - warnings.push_back(_("Wallet created successfully. The legacy wallet type is being deprecated and support for creating and opening legacy wallets will be removed in the future.")); + warnings.emplace_back(_("Wallet created successfully. The legacy wallet type is being deprecated and support for creating and opening legacy wallets will be removed in the future.")); } status = DatabaseStatus::SUCCESS; @@ -519,7 +519,7 @@ std::shared_ptr<CWallet> RestoreWallet(WalletContext& context, const fs::path& b } catch (const std::exception& e) { assert(!wallet); if (!error.empty()) error += Untranslated("\n"); - error += strprintf(Untranslated("Unexpected exception: %s"), e.what()); + error += Untranslated(strprintf("Unexpected exception: %s", e.what())); } if (!wallet) { fs::remove_all(wallet_path); @@ -1702,9 +1702,15 @@ bool CWallet::CanGetAddresses(bool internal) const void CWallet::SetWalletFlag(uint64_t flags) { + WalletBatch batch(GetDatabase()); + return SetWalletFlagWithDB(batch, flags); +} + +void CWallet::SetWalletFlagWithDB(WalletBatch& batch, uint64_t flags) +{ LOCK(cs_wallet); m_wallet_flags |= flags; - if (!WalletBatch(GetDatabase()).WriteWalletFlags(m_wallet_flags)) + if (!batch.WriteWalletFlags(m_wallet_flags)) throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed"); } @@ -1891,7 +1897,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc fast_rescan_filter ? "fast variant using block filters" : "slow variant inspecting all blocks"); fAbortRescan = false; - ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if rescan required on startup (e.g. due to corruption) + ShowProgress(strprintf("%s %s", GetDisplayName(), _("Rescanning…").translated), 0); // show rescan progress in GUI as dialog or on splashscreen, if rescan required on startup (e.g. due to corruption) uint256 tip_hash = WITH_LOCK(cs_wallet, return GetLastBlockHash()); uint256 end_hash = tip_hash; if (max_height) chain().findAncestorByHeight(tip_hash, *max_height, FoundBlock().hash(end_hash)); @@ -1906,7 +1912,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc m_scanning_progress = 0; } if (block_height % 100 == 0 && progress_end - progress_begin > 0.0) { - ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), std::max(1, std::min(99, (int)(m_scanning_progress * 100)))); + ShowProgress(strprintf("%s %s", GetDisplayName(), _("Rescanning…").translated), std::max(1, std::min(99, (int)(m_scanning_progress * 100)))); } bool next_interval = reserver.now() >= current_time + INTERVAL_TIME; @@ -2003,7 +2009,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc WalletLogPrintf("Scanning current mempool transactions.\n"); WITH_LOCK(cs_wallet, chain().requestMempoolTransactions(*this)); } - ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), 100); // hide progress dialog in GUI + ShowProgress(strprintf("%s %s", GetDisplayName(), _("Rescanning…").translated), 100); // hide progress dialog in GUI if (block_height && fAbortRescan) { WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", block_height, progress_current); result.status = ScanResult::USER_ABORT; @@ -2382,8 +2388,21 @@ DBErrors CWallet::LoadWallet() util::Result<void> CWallet::RemoveTxs(std::vector<uint256>& txs_to_remove) { AssertLockHeld(cs_wallet); - WalletBatch batch(GetDatabase()); - if (!batch.TxnBegin()) return util::Error{_("Error starting db txn for wallet transactions removal")}; + bilingual_str str_err; // future: make RunWithinTxn return a util::Result + bool was_txn_committed = RunWithinTxn(GetDatabase(), /*process_desc=*/"remove transactions", [&](WalletBatch& batch) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { + util::Result<void> result{RemoveTxs(batch, txs_to_remove)}; + if (!result) str_err = util::ErrorString(result); + return result.has_value(); + }); + if (!str_err.empty()) return util::Error{str_err}; + if (!was_txn_committed) return util::Error{_("Error starting/committing db txn for wallet transactions removal process")}; + return {}; // all good +} + +util::Result<void> CWallet::RemoveTxs(WalletBatch& batch, std::vector<uint256>& txs_to_remove) +{ + AssertLockHeld(cs_wallet); + if (!batch.HasActiveTxn()) return util::Error{strprintf(_("The transactions removal process can only be executed within a db txn"))}; // Check for transaction existence and remove entries from disk using TxIterator = std::unordered_map<uint256, CWalletTx, SaltedTxidHasher>::const_iterator; @@ -2392,38 +2411,30 @@ util::Result<void> CWallet::RemoveTxs(std::vector<uint256>& txs_to_remove) for (const uint256& hash : txs_to_remove) { auto it_wtx = mapWallet.find(hash); if (it_wtx == mapWallet.end()) { - str_err = strprintf(_("Transaction %s does not belong to this wallet"), hash.GetHex()); - break; + return util::Error{strprintf(_("Transaction %s does not belong to this wallet"), hash.GetHex())}; } if (!batch.EraseTx(hash)) { - str_err = strprintf(_("Failure removing transaction: %s"), hash.GetHex()); - break; + return util::Error{strprintf(_("Failure removing transaction: %s"), hash.GetHex())}; } erased_txs.emplace_back(it_wtx); } - // Roll back removals in case of an error - if (!str_err.empty()) { - batch.TxnAbort(); - return util::Error{str_err}; - } - - // Dump changes to disk - if (!batch.TxnCommit()) return util::Error{_("Error committing db txn for wallet transactions removal")}; - - // Update the in-memory state and notify upper layers about the removals - for (const auto& it : erased_txs) { - const uint256 hash{it->first}; - wtxOrdered.erase(it->second.m_it_wtxOrdered); - for (const auto& txin : it->second.tx->vin) - mapTxSpends.erase(txin.prevout); - mapWallet.erase(it); - NotifyTransactionChanged(hash, CT_DELETED); - } + // Register callback to update the memory state only when the db txn is actually dumped to disk + batch.RegisterTxnListener({.on_commit=[&, erased_txs]() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { + // Update the in-memory state and notify upper layers about the removals + for (const auto& it : erased_txs) { + const uint256 hash{it->first}; + wtxOrdered.erase(it->second.m_it_wtxOrdered); + for (const auto& txin : it->second.tx->vin) + mapTxSpends.erase(txin.prevout); + mapWallet.erase(it); + NotifyTransactionChanged(hash, CT_DELETED); + } - MarkDirty(); + MarkDirty(); + }, .on_abort={}}); - return {}; // all good + return {}; } bool CWallet::SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& address, const std::string& strName, const std::optional<AddressPurpose>& new_purpose) @@ -2484,7 +2495,7 @@ bool CWallet::DelAddressBookWithDB(WalletBatch& batch, const CTxDestination& add // NOTE: This isn't a problem for sending addresses because they don't have any data that needs to be kept. // When adding new address data, it should be considered here whether to retain or delete it. if (IsMine(address)) { - WalletLogPrintf("%s called with IsMine address, NOT SUPPORTED. Please report this bug! %s\n", __func__, PACKAGE_BUGREPORT); + WalletLogPrintf("%s called with IsMine address, NOT SUPPORTED. Please report this bug! %s\n", __func__, CLIENT_BUGREPORT); return false; } // Delete data rows associated with this address @@ -2997,7 +3008,7 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri walletFile)); } else if (nLoadWalletRet == DBErrors::TOO_NEW) { - error = strprintf(_("Error loading %s: Wallet requires newer version of %s"), walletFile, PACKAGE_NAME); + error = strprintf(_("Error loading %s: Wallet requires newer version of %s"), walletFile, CLIENT_NAME); return nullptr; } else if (nLoadWalletRet == DBErrors::EXTERNAL_SIGNER_SUPPORT_REQUIRED) { @@ -3006,7 +3017,7 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri } else if (nLoadWalletRet == DBErrors::NEED_REWRITE) { - error = strprintf(_("Wallet needed to be rewritten: restart %s to complete"), PACKAGE_NAME); + error = strprintf(_("Wallet needed to be rewritten: restart %s to complete"), CLIENT_NAME); return nullptr; } else if (nLoadWalletRet == DBErrors::NEED_RESCAN) { warnings.push_back(strprintf(_("Error reading %s! Transaction data may be missing or incorrect." @@ -3410,6 +3421,14 @@ void CWallet::postInitProcess() bool CWallet::BackupWallet(const std::string& strDest) const { + if (m_chain) { + CBlockLocator loc; + WITH_LOCK(cs_wallet, chain().findBlock(m_last_block_processed, FoundBlock().locator(loc))); + if (!loc.IsNull()) { + WalletBatch batch(GetDatabase()); + batch.WriteBestBlock(loc); + } + } return GetDatabase().Backup(strDest); } @@ -3728,22 +3747,30 @@ DescriptorScriptPubKeyMan& CWallet::SetupDescriptorScriptPubKeyMan(WalletBatch& return *out; } -void CWallet::SetupDescriptorScriptPubKeyMans(const CExtKey& master_key) +void CWallet::SetupDescriptorScriptPubKeyMans(WalletBatch& batch, const CExtKey& master_key) { AssertLockHeld(cs_wallet); - - // Create single batch txn - WalletBatch batch(GetDatabase()); - if (!batch.TxnBegin()) throw std::runtime_error("Error: cannot create db transaction for descriptors setup"); - for (bool internal : {false, true}) { for (OutputType t : OUTPUT_TYPES) { SetupDescriptorScriptPubKeyMan(batch, master_key, t, internal); } } +} + +void CWallet::SetupOwnDescriptorScriptPubKeyMans(WalletBatch& batch) +{ + AssertLockHeld(cs_wallet); + assert(!IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)); + // Make a seed + CKey seed_key = GenerateRandomKey(); + CPubKey seed = seed_key.GetPubKey(); + assert(seed_key.VerifyPubKey(seed)); - // Ensure information is committed to disk - if (!batch.TxnCommit()) throw std::runtime_error("Error: cannot commit db transaction for descriptors setup"); + // Get the extended key + CExtKey master_key; + master_key.SetSeed(seed_key); + + SetupDescriptorScriptPubKeyMans(batch, master_key); } void CWallet::SetupDescriptorScriptPubKeyMans() @@ -3751,16 +3778,10 @@ void CWallet::SetupDescriptorScriptPubKeyMans() AssertLockHeld(cs_wallet); if (!IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { - // Make a seed - CKey seed_key = GenerateRandomKey(); - CPubKey seed = seed_key.GetPubKey(); - assert(seed_key.VerifyPubKey(seed)); - - // Get the extended key - CExtKey master_key; - master_key.SetSeed(seed_key); - - SetupDescriptorScriptPubKeyMans(master_key); + if (!RunWithinTxn(GetDatabase(), /*process_desc=*/"setup descriptors", [&](WalletBatch& batch) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet){ + SetupOwnDescriptorScriptPubKeyMans(batch); + return true; + })) throw std::runtime_error("Error: cannot process db transaction for descriptors setup"); } else { ExternalSigner signer = ExternalSignerScriptPubKeyMan::GetExternalSigner(); @@ -4047,15 +4068,14 @@ std::optional<MigrationData> CWallet::GetDescriptorsForLegacy(bilingual_str& err return res; } -bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error) +util::Result<void> CWallet::ApplyMigrationData(WalletBatch& local_wallet_batch, MigrationData& data) { AssertLockHeld(cs_wallet); LegacyDataSPKM* legacy_spkm = GetLegacyDataSPKM(); if (!Assume(legacy_spkm)) { // This shouldn't happen - error = Untranslated(STR_INTERNAL_BUG("Error: Legacy wallet data missing")); - return false; + return util::Error{Untranslated(STR_INTERNAL_BUG("Error: Legacy wallet data missing"))}; } // Get all invalid or non-watched scripts that will not be migrated @@ -4065,20 +4085,23 @@ bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error) if (ExtractDestination(script, dest)) not_migrated_dests.emplace(dest); } - Assume(!m_cached_spks.empty()); + // When the legacy wallet has no spendable scripts, the main wallet will be empty, leaving its script cache empty as well. + // The watch-only and/or solvable wallet(s) will contain the scripts in their respective caches. + if (!data.desc_spkms.empty()) Assume(!m_cached_spks.empty()); + if (!data.watch_descs.empty()) Assume(!data.watchonly_wallet->m_cached_spks.empty()); + if (!data.solvable_descs.empty()) Assume(!data.solvable_wallet->m_cached_spks.empty()); for (auto& desc_spkm : data.desc_spkms) { if (m_spk_managers.count(desc_spkm->GetID()) > 0) { - error = _("Error: Duplicate descriptors created during migration. Your wallet may be corrupted."); - return false; + return util::Error{_("Error: Duplicate descriptors created during migration. Your wallet may be corrupted.")}; } uint256 id = desc_spkm->GetID(); AddScriptPubKeyMan(id, std::move(desc_spkm)); } // Remove the LegacyScriptPubKeyMan from disk - if (!legacy_spkm->DeleteRecords()) { - return false; + if (!legacy_spkm->DeleteRecordsWithDB(local_wallet_batch)) { + return util::Error{_("Error: cannot remove legacy wallet records")}; } // Remove the LegacyScriptPubKeyMan from memory @@ -4087,22 +4110,21 @@ bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error) m_internal_spk_managers.clear(); // Setup new descriptors - SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + SetWalletFlagWithDB(local_wallet_batch, WALLET_FLAG_DESCRIPTORS); if (!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { // Use the existing master key if we have it if (data.master_key.key.IsValid()) { - SetupDescriptorScriptPubKeyMans(data.master_key); + SetupDescriptorScriptPubKeyMans(local_wallet_batch, data.master_key); } else { // Setup with a new seed if we don't. - SetupDescriptorScriptPubKeyMans(); + SetupOwnDescriptorScriptPubKeyMans(local_wallet_batch); } } // Get best block locator so that we can copy it to the watchonly and solvables CBlockLocator best_block_locator; - if (!WalletBatch(GetDatabase()).ReadBestBlock(best_block_locator)) { - error = _("Error: Unable to read wallet's best block locator record"); - return false; + if (!local_wallet_batch.ReadBestBlock(best_block_locator)) { + return util::Error{_("Error: Unable to read wallet's best block locator record")}; } // Check if the transactions in the wallet are still ours. Either they belong here, or they belong in the watchonly wallet. @@ -4111,21 +4133,23 @@ bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error) std::unique_ptr<WalletBatch> watchonly_batch; if (data.watchonly_wallet) { watchonly_batch = std::make_unique<WalletBatch>(data.watchonly_wallet->GetDatabase()); + if (!watchonly_batch->TxnBegin()) return util::Error{strprintf(_("Error: database transaction cannot be executed for wallet %s"), data.watchonly_wallet->GetName())}; // Copy the next tx order pos to the watchonly wallet LOCK(data.watchonly_wallet->cs_wallet); data.watchonly_wallet->nOrderPosNext = nOrderPosNext; watchonly_batch->WriteOrderPosNext(data.watchonly_wallet->nOrderPosNext); // Write the best block locator to avoid rescanning on reload if (!watchonly_batch->WriteBestBlock(best_block_locator)) { - error = _("Error: Unable to write watchonly wallet best block locator record"); - return false; + return util::Error{_("Error: Unable to write watchonly wallet best block locator record")}; } } + std::unique_ptr<WalletBatch> solvables_batch; if (data.solvable_wallet) { + solvables_batch = std::make_unique<WalletBatch>(data.solvable_wallet->GetDatabase()); + if (!solvables_batch->TxnBegin()) return util::Error{strprintf(_("Error: database transaction cannot be executed for wallet %s"), data.solvable_wallet->GetName())}; // Write the best block locator to avoid rescanning on reload - if (!WalletBatch(data.solvable_wallet->GetDatabase()).WriteBestBlock(best_block_locator)) { - error = _("Error: Unable to write solvable wallet best block locator record"); - return false; + if (!solvables_batch->WriteBestBlock(best_block_locator)) { + return util::Error{_("Error: Unable to write solvable wallet best block locator record")}; } } for (const auto& [_pos, wtx] : wtxOrdered) { @@ -4144,8 +4168,7 @@ bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error) ins_wtx.CopyFrom(to_copy_wtx); return true; })) { - error = strprintf(_("Error: Could not add watchonly tx %s to watchonly wallet"), wtx->GetHash().GetHex()); - return false; + return util::Error{strprintf(_("Error: Could not add watchonly tx %s to watchonly wallet"), wtx->GetHash().GetHex())}; } watchonly_batch->WriteTx(data.watchonly_wallet->mapWallet.at(hash)); // Mark as to remove from the migrated wallet only if it does not also belong to it @@ -4157,31 +4180,21 @@ bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error) } if (!is_mine) { // Both not ours and not in the watchonly wallet - error = strprintf(_("Error: Transaction %s in wallet cannot be identified to belong to migrated wallets"), wtx->GetHash().GetHex()); - return false; + return util::Error{strprintf(_("Error: Transaction %s in wallet cannot be identified to belong to migrated wallets"), wtx->GetHash().GetHex())}; } } - watchonly_batch.reset(); // Flush + // Do the removes if (txids_to_delete.size() > 0) { - if (auto res = RemoveTxs(txids_to_delete); !res) { - error = _("Error: Could not delete watchonly transactions. ") + util::ErrorString(res); - return false; + if (auto res = RemoveTxs(local_wallet_batch, txids_to_delete); !res) { + return util::Error{_("Error: Could not delete watchonly transactions. ") + util::ErrorString(res)}; } } // Pair external wallets with their corresponding db handler std::vector<std::pair<std::shared_ptr<CWallet>, std::unique_ptr<WalletBatch>>> wallets_vec; - for (const auto& ext_wallet : {data.watchonly_wallet, data.solvable_wallet}) { - if (!ext_wallet) continue; - - std::unique_ptr<WalletBatch> batch = std::make_unique<WalletBatch>(ext_wallet->GetDatabase()); - if (!batch->TxnBegin()) { - error = strprintf(_("Error: database transaction cannot be executed for wallet %s"), ext_wallet->GetName()); - return false; - } - wallets_vec.emplace_back(ext_wallet, std::move(batch)); - } + if (data.watchonly_wallet) wallets_vec.emplace_back(data.watchonly_wallet, std::move(watchonly_batch)); + if (data.solvable_wallet) wallets_vec.emplace_back(data.solvable_wallet, std::move(solvables_batch)); // Write address book entry to disk auto func_store_addr = [](WalletBatch& batch, const CTxDestination& dest, const CAddressBookData& entry) { @@ -4228,39 +4241,27 @@ bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error) continue; } - error = _("Error: Address book data in wallet cannot be identified to belong to migrated wallets"); - return false; + return util::Error{_("Error: Address book data in wallet cannot be identified to belong to migrated wallets")}; } } // Persist external wallets address book entries for (auto& [wallet, batch] : wallets_vec) { if (!batch->TxnCommit()) { - error = strprintf(_("Error: address book copy failed for wallet %s"), wallet->GetName()); - return false; + return util::Error{strprintf(_("Error: Unable to write data to disk for wallet %s"), wallet->GetName())}; } } // Remove the things to delete in this wallet - WalletBatch local_wallet_batch(GetDatabase()); - local_wallet_batch.TxnBegin(); if (dests_to_delete.size() > 0) { for (const auto& dest : dests_to_delete) { if (!DelAddressBookWithDB(local_wallet_batch, dest)) { - error = _("Error: Unable to remove watchonly address book data"); - return false; + return util::Error{_("Error: Unable to remove watchonly address book data")}; } } } - local_wallet_batch.TxnCommit(); - - // Connect the SPKM signals - ConnectScriptPubKeyManNotifiers(); - NotifyCanGetAddressesChanged(); - - WalletLogPrintf("Wallet migration complete.\n"); - return true; + return {}; // all good } bool CWallet::CanGrindR() const @@ -4371,10 +4372,14 @@ bool DoMigration(CWallet& wallet, WalletContext& context, bilingual_str& error, } // Add the descriptors to wallet, remove LegacyScriptPubKeyMan, and cleanup txs and address book data - if (!wallet.ApplyMigrationData(*data, error)) { - return false; - } - return true; + return RunWithinTxn(wallet.GetDatabase(), /*process_desc=*/"apply migration process", [&](WalletBatch& batch) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet){ + if (auto res_migration = wallet.ApplyMigrationData(batch, *data); !res_migration) { + error = util::ErrorString(res_migration); + return false; + } + wallet.WalletLogPrintf("Wallet migration complete.\n"); + return true; + }); } util::Result<MigrationResult> MigrateLegacyToDescriptor(const std::string& wallet_name, const SecureString& passphrase, WalletContext& context) @@ -4390,6 +4395,11 @@ util::Result<MigrationResult> MigrateLegacyToDescriptor(const std::string& walle return util::Error{_("Error: This wallet is already a descriptor wallet")}; } + // Flush chain state before unloading wallet + CBlockLocator locator; + WITH_LOCK(wallet->cs_wallet, context.chain->findBlock(wallet->GetLastBlockHash(), FoundBlock().locator(locator))); + if (!locator.IsNull()) wallet->chainStateFlushed(ChainstateRole::NORMAL, locator); + if (!RemoveWallet(context, wallet, /*load_on_start=*/std::nullopt, warnings)) { return util::Error{_("Unable to unload the wallet before migrating")}; } @@ -4401,6 +4411,9 @@ util::Result<MigrationResult> MigrateLegacyToDescriptor(const std::string& walle if (!wallet_path) { return util::Error{util::ErrorString(wallet_path)}; } + if (!fs::exists(*wallet_path)) { + return util::Error{_("Error: Wallet does not exist")}; + } if (!IsBDBFile(BDBDataFile(*wallet_path))) { return util::Error{_("Error: This wallet is already a descriptor wallet")}; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 485eed11fa..d869f031bb 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -422,6 +422,9 @@ private: // Same as 'AddActiveScriptPubKeyMan' but designed for use within a batch transaction context void AddActiveScriptPubKeyManWithDb(WalletBatch& batch, uint256 id, OutputType type, bool internal); + /** Store wallet flags */ + void SetWalletFlagWithDB(WalletBatch& batch, uint64_t flags); + //! Cache of descriptor ScriptPubKeys used for IsMine. Maps ScriptPubKey to set of spkms std::unordered_map<CScript, std::vector<ScriptPubKeyMan*>, SaltedSipHasher> m_cached_spks; @@ -791,6 +794,7 @@ public: /** Erases the provided transactions from the wallet. */ util::Result<void> RemoveTxs(std::vector<uint256>& txs_to_remove) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + util::Result<void> RemoveTxs(WalletBatch& batch, std::vector<uint256>& txs_to_remove) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool SetAddressBook(const CTxDestination& address, const std::string& strName, const std::optional<AddressPurpose>& purpose); @@ -927,9 +931,9 @@ public: /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ template <typename... Params> - void WalletLogPrintf(const char* fmt, Params... parameters) const + void WalletLogPrintf(util::ConstevalFormatString<sizeof...(Params)> wallet_fmt, const Params&... params) const { - LogPrintf(("%s " + std::string{fmt}).c_str(), GetDisplayName(), parameters...); + LogInfo("%s %s", GetDisplayName(), tfm::format(wallet_fmt, params...)); }; /** Upgrade the wallet */ @@ -1018,9 +1022,12 @@ public: //! Create new DescriptorScriptPubKeyMan and add it to the wallet DescriptorScriptPubKeyMan& SetupDescriptorScriptPubKeyMan(WalletBatch& batch, const CExtKey& master_key, const OutputType& output_type, bool internal) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Create new DescriptorScriptPubKeyMans and add them to the wallet - void SetupDescriptorScriptPubKeyMans(const CExtKey& master_key) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void SetupDescriptorScriptPubKeyMans(WalletBatch& batch, const CExtKey& master_key) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SetupDescriptorScriptPubKeyMans() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + //! Create new seed and default DescriptorScriptPubKeyMans for this wallet + void SetupOwnDescriptorScriptPubKeyMans(WalletBatch& batch) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + //! Return the DescriptorScriptPubKeyMan for a WalletDescriptor if it is already in the wallet DescriptorScriptPubKeyMan* GetDescriptorScriptPubKeyMan(const WalletDescriptor& desc) const; @@ -1044,7 +1051,7 @@ public: //! Adds the ScriptPubKeyMans given in MigrationData to this wallet, removes LegacyScriptPubKeyMan, //! and where needed, moves tx and address book entries to watchonly_wallet or solvable_wallet - bool ApplyMigrationData(MigrationData& data, bilingual_str& error) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + util::Result<void> ApplyMigrationData(WalletBatch& local_wallet_batch, MigrationData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Whether the (external) signer performs R-value signature grinding bool CanGrindR() const; diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 47b84f7e6a..368415da12 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <wallet/walletdb.h> @@ -1013,7 +1013,7 @@ static DBErrors LoadAddressBookRecords(CWallet* pwallet, DatabaseBatch& batch) E // "1" or "p" for present (which was written prior to // f5ba424cd44619d9b9be88b8593d69a7ba96db26). pwallet->LoadAddressPreviouslySpent(dest); - } else if (strKey.compare(0, 2, "rr") == 0) { + } else if (strKey.starts_with("rr")) { // Load "rr##" keys where ## is a decimal number, and strValue // is a serialized RecentRequestEntry object. pwallet->LoadAddressReceiveRequest(dest, strKey.substr(2), strValue); @@ -1335,10 +1335,8 @@ bool WalletBatch::WriteWalletFlags(const uint64_t flags) bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) { - return RunWithinTxn(*this, "erase records", [&types](WalletBatch& self) { - return std::all_of(types.begin(), types.end(), [&self](const std::string& type) { - return self.m_batch->ErasePrefix(DataStream() << type); - }); + return std::all_of(types.begin(), types.end(), [&](const std::string& type) { + return m_batch->ErasePrefix(DataStream() << type); }); } @@ -1349,12 +1347,34 @@ bool WalletBatch::TxnBegin() bool WalletBatch::TxnCommit() { - return m_batch->TxnCommit(); + bool res = m_batch->TxnCommit(); + if (res) { + for (const auto& listener : m_txn_listeners) { + listener.on_commit(); + } + // txn finished, clear listeners + m_txn_listeners.clear(); + } + return res; } bool WalletBatch::TxnAbort() { - return m_batch->TxnAbort(); + bool res = m_batch->TxnAbort(); + if (res) { + for (const auto& listener : m_txn_listeners) { + listener.on_abort(); + } + // txn finished, clear listeners + m_txn_listeners.clear(); + } + return res; +} + +void WalletBatch::RegisterTxnListener(const DbTxnListener& l) +{ + assert(m_batch->HasActiveTxn()); + m_txn_listeners.emplace_back(l); } std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error) diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index bffcc87202..32c3c29b5e 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -180,6 +180,11 @@ public: } }; +struct DbTxnListener +{ + std::function<void()> on_commit, on_abort; +}; + /** Access to the wallet database. * Opens the database and provides read and write access to it. Each read and write is its own transaction. * Multiple operation transactions can be started using TxnBegin() and committed using TxnCommit() @@ -292,9 +297,18 @@ public: bool TxnCommit(); //! Abort current transaction bool TxnAbort(); + bool HasActiveTxn() { return m_batch->HasActiveTxn(); } + + //! Registers db txn callback functions + void RegisterTxnListener(const DbTxnListener& l); + private: std::unique_ptr<DatabaseBatch> m_batch; WalletDatabase& m_database; + + // External functions listening to the current db txn outcome. + // Listeners are cleared at the end of the transaction. + std::vector<DbTxnListener> m_txn_listeners; }; /** diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 10785ad354..f7e7b74fc3 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <wallet/wallettool.h> @@ -75,10 +75,10 @@ static std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::pa name); } else if (load_wallet_ret == DBErrors::TOO_NEW) { tfm::format(std::cerr, "Error loading %s: Wallet requires newer version of %s", - name, PACKAGE_NAME); + name, CLIENT_NAME); return nullptr; } else if (load_wallet_ret == DBErrors::NEED_REWRITE) { - tfm::format(std::cerr, "Wallet needed to be rewritten: restart %s to complete", PACKAGE_NAME); + tfm::format(std::cerr, "Wallet needed to be rewritten: restart %s to complete", CLIENT_NAME); return nullptr; } else if (load_wallet_ret == DBErrors::NEED_RESCAN) { tfm::format(std::cerr, "Error reading %s! Some transaction data might be missing or" diff --git a/src/zmq/CMakeLists.txt b/src/zmq/CMakeLists.txt index 8ecb236b46..19ac722b7a 100644 --- a/src/zmq/CMakeLists.txt +++ b/src/zmq/CMakeLists.txt @@ -12,13 +12,10 @@ add_library(bitcoin_zmq STATIC EXCLUDE_FROM_ALL target_compile_definitions(bitcoin_zmq INTERFACE ENABLE_ZMQ=1 - PRIVATE - $<$<AND:$<PLATFORM_ID:Windows>,$<CXX_COMPILER_ID:GNU>>:ZMQ_STATIC> ) target_link_libraries(bitcoin_zmq PRIVATE core_interface univalue - $<TARGET_NAME_IF_EXISTS:libzmq> - $<TARGET_NAME_IF_EXISTS:PkgConfig::libzmq> + zeromq ) diff --git a/test/config.ini.in b/test/config.ini.in index 291599da45..3bf79ef25d 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -6,8 +6,8 @@ # test/*/test_runner.py and test/util/rpcauth-test.py [environment] -PACKAGE_NAME=@PACKAGE_NAME@ -PACKAGE_BUGREPORT=@PACKAGE_BUGREPORT@ +CLIENT_NAME=@CLIENT_NAME@ +CLIENT_BUGREPORT=@CLIENT_BUGREPORT@ SRCDIR=@abs_top_srcdir@ BUILDDIR=@abs_top_builddir@ EXEEXT=@EXEEXT@ diff --git a/test/functional/README.md b/test/functional/README.md index a4994f2e7c..a34bf1827c 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -10,7 +10,8 @@ that file and modify to fit your needs. #### Coverage -Running `test/functional/test_runner.py` with the `--coverage` argument tracks which RPCs are +Assuming the build directory is `build`, +running `build/test/functional/test_runner.py` with the `--coverage` argument tracks which RPCs are called by the tests and prints a report of uncovered RPCs in the summary. This can be used (along with the `--extended` argument) to find out which RPCs we don't have test cases for. diff --git a/test/functional/combine_logs.py b/test/functional/combine_logs.py index 33c81bde13..998cb20831 100755 --- a/test/functional/combine_logs.py +++ b/test/functional/combine_logs.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2021 The Bitcoin Core developers +# Copyright (c) 2017-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Combine logs from multiple bitcoin nodes as well as the test_framework log. @@ -79,14 +79,16 @@ def read_logs(tmp_dir): Delegates to generator function get_log_events() to provide individual log events for each of the input log files.""" - # Find out what the folder is called that holds the debug.log file - glob = pathlib.Path(tmp_dir).glob('node0/**/debug.log') - path = next(glob, None) - if path: - assert next(glob, None) is None # more than one debug.log, should never happen - chain = re.search(r'node0/(.+?)/debug\.log$', path.as_posix()).group(1) # extract the chain name - else: - chain = 'regtest' # fallback to regtest (should only happen when none exists) + # Find out what the folder is called that holds node 0's debug.log file + debug_logs = list(pathlib.Path(tmp_dir).glob('node0/**/debug.log')) + match len(debug_logs): + case 0: + chain = 'regtest' # fallback to regtest + case 1: + chain = re.search(r'node0/(.+?)/debug\.log$', debug_logs[0].as_posix()).group(1) + case _: + raise RuntimeError('Max one debug.log is supported, found several:\n\t' + + '\n\t'.join(map(str, debug_logs))) files = [("test", "%s/test_framework.log" % tmp_dir)] for i in itertools.count(): diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index 2e4ca83bf0..d2d7202d86 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -263,6 +263,17 @@ def getDisabledOpcodeTemplate(opcode): 'valid_in_block' : True }) +class NonStandardAndInvalid(BadTxTemplate): + """A non-standard transaction which is also consensus-invalid should return the consensus error.""" + reject_reason = "mandatory-script-verify-flag-failed (OP_RETURN was encountered)" + expect_disconnect = True + valid_in_block = False + + def get_tx(self): + return create_tx_with_script( + self.spend_tx, 0, script_sig=b'\x00' * 3 + b'\xab\x6a', + amount=(self.spend_avail // 2)) + # Disabled opcode tx templates (CVE-2010-5137) DisabledOpcodeTemplates = [getDisabledOpcodeTemplate(opcode) for opcode in [ OP_CAT, diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py index c4e60fba12..62f9cee1df 100755 --- a/test/functional/feature_addrman.py +++ b/test/functional/feature_addrman.py @@ -54,7 +54,7 @@ class AddrmanTest(BitcoinTestFramework): peers_dat = os.path.join(self.nodes[0].chain_path, "peers.dat") init_error = lambda reason: ( f"Error: Invalid or corrupt peers.dat \\({reason}\\). If you believe this " - f"is a bug, please report it to {self.config['environment']['PACKAGE_BUGREPORT']}. " + f"is a bug, please report it to {self.config['environment']['CLIENT_BUGREPORT']}. " f'As a workaround, you can move the file \\("{re.escape(peers_dat)}"\\) out of the way \\(rename, ' "move, or delete\\) to have a new one created on the next start." ) diff --git a/test/functional/feature_anchors.py b/test/functional/feature_anchors.py index 5ca46c1366..154461e739 100755 --- a/test/functional/feature_anchors.py +++ b/test/functional/feature_anchors.py @@ -102,7 +102,7 @@ class AnchorsTest(BitcoinTestFramework): self.nodes[0].addconnection(ONION_ADDR, 'block-relay-only', v2transport=False) self.log.debug("Stop node") - with self.nodes[0].assert_debug_log([f"DumpAnchors: Flush 1 outbound block-relay-only peer addresses to anchors.dat"]): + with self.nodes[0].assert_debug_log(["DumpAnchors: Flush 1 outbound block-relay-only peer addresses to anchors.dat"]): self.stop_node(0) # Manually close keep_alive proxy connection onion_proxy.stop() diff --git a/test/functional/feature_assumeutxo.py b/test/functional/feature_assumeutxo.py index c91f254f11..8addb7e238 100755 --- a/test/functional/feature_assumeutxo.py +++ b/test/functional/feature_assumeutxo.py @@ -9,7 +9,6 @@ to a hash that has been compiled into bitcoind. The assumeutxo value generated and used here is committed to in `CRegTestParams::m_assumeutxo_data` in `src/kernel/chainparams.cpp`. """ -import time from shutil import rmtree from dataclasses import dataclass @@ -31,6 +30,7 @@ from test_framework.util import ( assert_approx, assert_equal, assert_raises_rpc_error, + ensure_for, sha256sum_file, try_rpc, ) @@ -165,8 +165,8 @@ class AssumeutxoTest(BitcoinTestFramework): with self.nodes[0].assert_debug_log([log_msg]): self.nodes[0].assert_start_raises_init_error(expected_msg=error_msg) - expected_error_msg = f"Error: A fatal internal error occurred, see debug.log for details: Assumeutxo data not found for the given blockhash '7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a'." - error_details = f"Assumeutxo data not found for the given blockhash" + expected_error_msg = "Error: A fatal internal error occurred, see debug.log for details: Assumeutxo data not found for the given blockhash '7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a'." + error_details = "Assumeutxo data not found for the given blockhash" expected_error(log_msg=error_details, error_msg=expected_error_msg) # resurrect node again @@ -305,15 +305,14 @@ class AssumeutxoTest(BitcoinTestFramework): # If it does request such blocks, the snapshot_node will ignore requests it cannot fulfill, causing the ibd_node # to stall. This stall could last for up to 10 min, ultimately resulting in an abrupt disconnection due to the # ibd_node's perceived unresponsiveness. - time.sleep(3) # Sleep here because we can't detect when a node avoids requesting blocks from other peer. - assert_equal(len(ibd_node.getpeerinfo()[0]['inflight']), 0) + ensure_for(duration=3, f=lambda: len(ibd_node.getpeerinfo()[0]['inflight']) == 0) # Now disconnect nodes and finish background chain sync self.disconnect_nodes(ibd_node.index, snapshot_node.index) self.connect_nodes(snapshot_node.index, miner.index) self.sync_blocks(nodes=(miner, snapshot_node)) # Check the base snapshot block was stored and ensure node signals full-node service support - self.wait_until(lambda: not try_rpc(-1, "Block not found", snapshot_node.getblock, snapshot_block_hash)) + self.wait_until(lambda: not try_rpc(-1, "Block not available (not fully downloaded)", snapshot_node.getblock, snapshot_block_hash)) self.wait_until(lambda: 'NETWORK' in snapshot_node.getnetworkinfo()['localservicesnames']) # Now that the snapshot_node is synced, verify the ibd_node can sync from it @@ -418,7 +417,7 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(n0.getblockchaininfo()["blocks"], FINAL_HEIGHT) - self.log.info(f"Check that dumptxoutset works for past block heights") + self.log.info("Check that dumptxoutset works for past block heights") # rollback defaults to the snapshot base height dump_output2 = n0.dumptxoutset('utxos2.dat', "rollback") check_dump_output(dump_output2) @@ -485,7 +484,7 @@ class AssumeutxoTest(BitcoinTestFramework): # find coinbase output at snapshot height on node0 and scan for it on node1, # where the block is not available, but the snapshot was loaded successfully coinbase_tx = n0.getblock(snapshot_hash, verbosity=2)['tx'][0] - assert_raises_rpc_error(-1, "Block not found on disk", n1.getblock, snapshot_hash) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", n1.getblock, snapshot_hash) coinbase_output_descriptor = coinbase_tx['vout'][0]['scriptPubKey']['desc'] scan_result = n1.scantxoutset('start', [coinbase_output_descriptor]) assert_equal(scan_result['success'], True) @@ -557,7 +556,7 @@ class AssumeutxoTest(BitcoinTestFramework): self.log.info("Submit a spending transaction for a snapshot chainstate coin to the mempool") # spend the coinbase output of the first block that is not available on node1 spend_coin_blockhash = n1.getblockhash(START_HEIGHT + 1) - assert_raises_rpc_error(-1, "Block not found on disk", n1.getblock, spend_coin_blockhash) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", n1.getblock, spend_coin_blockhash) prev_tx = n0.getblock(spend_coin_blockhash, 3)['tx'][0] prevout = {"txid": prev_tx['txid'], "vout": 0, "scriptPubKey": prev_tx['vout'][0]['scriptPubKey']['hex']} privkey = n0.get_deterministic_priv_key().key diff --git a/test/functional/feature_blocksxor.py b/test/functional/feature_blocksxor.py index 7698a66ec4..9824bf9715 100755 --- a/test/functional/feature_blocksxor.py +++ b/test/functional/feature_blocksxor.py @@ -31,7 +31,7 @@ class BlocksXORTest(BitcoinTestFramework): node = self.nodes[0] wallet = MiniWallet(node) for _ in range(5): - wallet.send_self_transfer(from_node=node, target_weight=80000) + wallet.send_self_transfer(from_node=node, target_vsize=20000) self.generate(wallet, 1) block_files = list(node.blocks_path.glob('blk[0-9][0-9][0-9][0-9][0-9].dat')) diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index dc01f8fa8f..60b3fb4e20 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -87,7 +87,6 @@ class BIP65Test(BitcoinTestFramework): self.noban_tx_relay = True self.extra_args = [[ f'-testactivationheight=cltv@{CLTV_HEIGHT}', - '-par=1', # Use only one script thread to get the exact reject reason for testing '-acceptnonstdtxn=1', # cltv_invalidate is nonstandard ]] self.setup_clean_chain = True @@ -149,6 +148,10 @@ class BIP65Test(BitcoinTestFramework): # create and test one invalid tx per CLTV failure reason (5 in total) for i in range(5): spendtx = wallet.create_self_transfer()['tx'] + assert_equal(len(spendtx.vin), 1) + coin = spendtx.vin[0] + coin_txid = format(coin.prevout.hash, '064x') + coin_vout = coin.prevout.n cltv_invalidate(spendtx, i) expected_cltv_reject_reason = [ @@ -160,12 +163,15 @@ class BIP65Test(BitcoinTestFramework): ][i] # First we show that this tx is valid except for CLTV by getting it # rejected from the mempool for exactly that reason. + spendtx_txid = spendtx.hash + spendtx_wtxid = spendtx.getwtxid() assert_equal( [{ - 'txid': spendtx.hash, - 'wtxid': spendtx.getwtxid(), + 'txid': spendtx_txid, + 'wtxid': spendtx_wtxid, 'allowed': False, 'reject-reason': expected_cltv_reject_reason, + 'reject-details': expected_cltv_reject_reason + f", input 0 of {spendtx_txid} (wtxid {spendtx_wtxid}), spending {coin_txid}:{coin_vout}" }], self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0), ) @@ -175,7 +181,7 @@ class BIP65Test(BitcoinTestFramework): block.hashMerkleRoot = block.calc_merkle_root() block.solve() - with self.nodes[0].assert_debug_log(expected_msgs=[f'CheckInputScripts on {block.vtx[-1].hash} failed with {expected_cltv_reject_reason}']): + with self.nodes[0].assert_debug_log(expected_msgs=[f'Block validation error: {expected_cltv_reject_reason}']): peer.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) peer.sync_with_ping() diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index 44c7edf962..31847b3fbf 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2022 The Bitcoin Core developers +# Copyright (c) 2017-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various command line arguments and configuration file parameters.""" @@ -27,13 +27,78 @@ class ConfArgsTest(BitcoinTestFramework): self.wallet_names = [] self.disable_autoconnect = False + # Overridden to avoid attempt to sync not yet started nodes. + def setup_network(self): + self.setup_nodes() + + # Overridden to not start nodes automatically - doing so is the + # responsibility of each test function. + def setup_nodes(self): + self.add_nodes(self.num_nodes, self.extra_args) + # Ensure a log file exists as TestNode.assert_debug_log() expects it. + self.nodes[0].debug_log_path.parent.mkdir() + self.nodes[0].debug_log_path.touch() + + def test_dir_config(self): + self.log.info('Error should be emitted if config file is a directory') + conf_path = self.nodes[0].datadir_path / 'bitcoin.conf' + os.rename(conf_path, conf_path.with_suffix('.confbkp')) + conf_path.mkdir() + self.stop_node(0) + self.nodes[0].assert_start_raises_init_error( + extra_args=['-regtest'], + expected_msg=f'Error: Error reading configuration file: Config file "{conf_path}" is a directory.', + ) + conf_path.rmdir() + os.rename(conf_path.with_suffix('.confbkp'), conf_path) + + self.log.debug('Verifying includeconf directive pointing to directory is caught') + with open(conf_path, 'a', encoding='utf-8') as conf: + conf.write(f'includeconf={self.nodes[0].datadir_path}\n') + self.nodes[0].assert_start_raises_init_error( + extra_args=['-regtest'], + expected_msg=f'Error: Error reading configuration file: Included config file "{self.nodes[0].datadir_path}" is a directory.', + ) + + self.nodes[0].replace_in_config([(f'includeconf={self.nodes[0].datadir_path}', '')]) + + def test_negated_config(self): + self.log.info('Disabling configuration via -noconf') + + conf_path = self.nodes[0].datadir_path / 'bitcoin.conf' + with open(conf_path, encoding='utf-8') as conf: + settings = [f'-{line.rstrip()}' for line in conf if len(line) > 1 and line[0] != '['] + os.rename(conf_path, conf_path.with_suffix('.confbkp')) + + self.log.debug('Verifying garbage in config can be detected') + with open(conf_path, 'a', encoding='utf-8') as conf: + conf.write('garbage\n') + self.nodes[0].assert_start_raises_init_error( + extra_args=['-regtest'], + expected_msg='Error: Error reading configuration file: parse error on line 1: garbage', + ) + + self.log.debug('Verifying that disabling of the config file means garbage inside of it does ' \ + 'not prevent the node from starting, and message about existing config file is logged') + ignored_file_message = [f'[InitConfig] Data directory "{self.nodes[0].datadir_path}" contains a "bitcoin.conf" file which is explicitly ignored using -noconf.'] + with self.nodes[0].assert_debug_log(timeout=60, expected_msgs=ignored_file_message): + self.start_node(0, extra_args=settings + ['-noconf']) + self.stop_node(0) + + self.log.debug('Verifying no message appears when removing config file') + os.remove(conf_path) + with self.nodes[0].assert_debug_log(timeout=60, expected_msgs=[], unexpected_msgs=ignored_file_message): + self.start_node(0, extra_args=settings + ['-noconf']) + self.stop_node(0) + + os.rename(conf_path.with_suffix('.confbkp'), conf_path) + def test_config_file_parser(self): self.log.info('Test config file parser') - self.stop_node(0) # Check that startup fails if conf= is set in bitcoin.conf or in an included conf file bad_conf_file_path = self.nodes[0].datadir_path / "bitcoin_bad.conf" - util.write_config(bad_conf_file_path, n=0, chain='', extra_config=f'conf=some.conf\n') + util.write_config(bad_conf_file_path, n=0, chain='', extra_config='conf=some.conf\n') conf_in_config_file_err = 'Error: Error reading configuration file: conf cannot be set in the configuration file; use includeconf= if you want to include additional config files' self.nodes[0].assert_start_raises_init_error( extra_args=[f'-conf={bad_conf_file_path}'], @@ -162,12 +227,11 @@ class ConfArgsTest(BitcoinTestFramework): ) def test_log_buffer(self): - self.stop_node(0) with self.nodes[0].assert_debug_log(expected_msgs=['Warning: parsed potentially confusing double-negative -connect=0\n']): self.start_node(0, extra_args=['-noconnect=0']) + self.stop_node(0) def test_args_log(self): - self.stop_node(0) self.log.info('Test config args logging') with self.nodes[0].assert_debug_log( expected_msgs=[ @@ -196,10 +260,10 @@ class ConfArgsTest(BitcoinTestFramework): '-rpcuser=secret-rpcuser', '-torpassword=secret-torpassword', ]) + self.stop_node(0) def test_networkactive(self): self.log.info('Test -networkactive option') - self.stop_node(0) with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: true\n']): self.start_node(0) @@ -222,16 +286,12 @@ class ConfArgsTest(BitcoinTestFramework): self.stop_node(0) with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: false\n']): self.start_node(0, extra_args=['-nonetworkactive=1']) + self.stop_node(0) def test_seed_peers(self): self.log.info('Test seed peers') default_data_dir = self.nodes[0].datadir_path peer_dat = default_data_dir / 'peers.dat' - # Only regtest has no fixed seeds. To avoid connections to random - # nodes, regtest is the only network where it is safe to enable - # -fixedseeds in tests - util.assert_equal(self.nodes[0].getblockchaininfo()['chain'],'regtest') - self.stop_node(0) # No peers.dat exists and -dnsseed=1 # We expect the node will use DNS Seeds, but Regtest mode does not have @@ -248,6 +308,12 @@ class ConfArgsTest(BitcoinTestFramework): timeout=10, ): self.start_node(0, extra_args=['-dnsseed=1', '-fixedseeds=1', f'-mocktime={start}']) + + # Only regtest has no fixed seeds. To avoid connections to random + # nodes, regtest is the only network where it is safe to enable + # -fixedseeds in tests + util.assert_equal(self.nodes[0].getblockchaininfo()['chain'],'regtest') + with self.nodes[0].assert_debug_log(expected_msgs=[ "Adding fixed seeds as 60 seconds have passed and addrman is empty", ]): @@ -294,13 +360,13 @@ class ConfArgsTest(BitcoinTestFramework): "Adding fixed seeds as 60 seconds have passed and addrman is empty", ]): self.nodes[0].setmocktime(start + 65) + self.stop_node(0) def test_connect_with_seednode(self): self.log.info('Test -connect with -seednode') seednode_ignored = ['-seednode is ignored when -connect is used\n'] dnsseed_ignored = ['-dnsseed is ignored when -connect is used and -proxy is specified\n'] addcon_thread_started = ['addcon thread start\n'] - self.stop_node(0) # When -connect is supplied, expanding addrman via getaddr calls to ADDR_FETCH(-seednode) # nodes is irrelevant and -seednode is ignored. @@ -325,6 +391,7 @@ class ConfArgsTest(BitcoinTestFramework): with self.nodes[0].assert_debug_log(expected_msgs=addcon_thread_started, unexpected_msgs=seednode_ignored): self.restart_node(0, extra_args=[connect_arg, '-seednode=fakeaddress2']) + self.stop_node(0) def test_ignored_conf(self): self.log.info('Test error is triggered when the datadir in use contains a bitcoin.conf file that would be ignored ' @@ -423,6 +490,8 @@ class ConfArgsTest(BitcoinTestFramework): self.test_networkactive() self.test_connect_with_seednode() + self.test_dir_config() + self.test_negated_config() self.test_config_file_parser() self.test_config_file_log() self.test_invalid_command_line_options() diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index df02bcc6ad..d5b6d0dba9 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -99,7 +99,6 @@ class BIP68_112_113Test(BitcoinTestFramework): self.noban_tx_relay = True self.extra_args = [[ f'-testactivationheight=csv@{CSV_ACTIVATION_HEIGHT}', - '-par=1', # Use only one script thread to get the exact reject reason for testing ]] self.supports_cli = False diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 48b0b745c6..0c3b0f1224 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -51,7 +51,6 @@ class BIP66Test(BitcoinTestFramework): self.noban_tx_relay = True self.extra_args = [[ f'-testactivationheight=dersig@{DERSIG_HEIGHT}', - '-par=1', # Use only one script thread to get the exact log msg for testing ]] self.setup_clean_chain = True self.rpc_timeout = 240 @@ -110,18 +109,23 @@ class BIP66Test(BitcoinTestFramework): self.log.info("Test that transactions with non-DER signatures cannot appear in a block") block.nVersion = 4 - spendtx = self.create_tx(self.coinbase_txids[1]) + coin_txid = self.coinbase_txids[1] + spendtx = self.create_tx(coin_txid) unDERify(spendtx) spendtx.rehash() # First we show that this tx is valid except for DERSIG by getting it # rejected from the mempool for exactly that reason. + spendtx_txid = spendtx.hash + spendtx_wtxid = spendtx.getwtxid() assert_equal( [{ - 'txid': spendtx.hash, - 'wtxid': spendtx.getwtxid(), + 'txid': spendtx_txid, + 'wtxid': spendtx_wtxid, 'allowed': False, 'reject-reason': 'mandatory-script-verify-flag-failed (Non-canonical DER signature)', + 'reject-details': 'mandatory-script-verify-flag-failed (Non-canonical DER signature), ' + + f"input 0 of {spendtx_txid} (wtxid {spendtx_wtxid}), spending {coin_txid}:0" }], self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0), ) @@ -131,7 +135,7 @@ class BIP66Test(BitcoinTestFramework): block.hashMerkleRoot = block.calc_merkle_root() block.solve() - with self.nodes[0].assert_debug_log(expected_msgs=[f'CheckInputScripts on {block.vtx[-1].hash} failed with mandatory-script-verify-flag-failed (Non-canonical DER signature)']): + with self.nodes[0].assert_debug_log(expected_msgs=['Block validation error: mandatory-script-verify-flag-failed (Non-canonical DER signature)']): peer.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) peer.sync_with_ping() diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 83627ff5c2..974d8268a2 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -398,6 +398,7 @@ class EstimateFeeTest(BitcoinTestFramework): self.start_node(0) self.connect_nodes(0, 1) self.connect_nodes(0, 2) + self.sync_blocks() assert_equal(self.nodes[0].estimatesmartfee(1)["errors"], ["Insufficient data or no feerate found"]) def broadcast_and_mine(self, broadcaster, miner, feerate, count): diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index e71871114d..f56643c62e 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -7,7 +7,10 @@ import random import string from test_framework.test_framework import BitcoinTestFramework -from test_framework.test_node import ErrorMatch +from test_framework.test_node import ( + BITCOIN_PID_FILENAME_DEFAULT, + ErrorMatch, +) class FilelockTest(BitcoinTestFramework): def add_options(self, parser): @@ -27,13 +30,13 @@ class FilelockTest(BitcoinTestFramework): self.log.info(f"Using datadir {datadir}") self.log.info("Check that we can't start a second bitcoind instance using the same datadir") - expected_msg = f"Error: Cannot obtain a lock on data directory {datadir}. {self.config['environment']['PACKAGE_NAME']} is probably already running." + expected_msg = f"Error: Cannot obtain a lock on data directory {datadir}. {self.config['environment']['CLIENT_NAME']} is probably already running." self.nodes[1].assert_start_raises_init_error(extra_args=[f'-datadir={self.nodes[0].datadir_path}', '-noserver'], expected_msg=expected_msg) self.log.info("Check that cookie and PID file are not deleted when attempting to start a second bitcoind using the same datadir") cookie_file = datadir / ".cookie" assert cookie_file.exists() # should not be deleted during the second bitcoind instance shutdown - pid_file = datadir / "bitcoind.pid" + pid_file = datadir / BITCOIN_PID_FILENAME_DEFAULT assert pid_file.exists() if self.is_wallet_compiled(): @@ -43,7 +46,7 @@ class FilelockTest(BitcoinTestFramework): wallet_dir = self.nodes[0].wallets_path self.log.info("Check that we can't start a second bitcoind instance using the same wallet") if descriptors: - expected_msg = f"Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['PACKAGE_NAME']}?" + expected_msg = f"Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['CLIENT_NAME']}?" else: expected_msg = "Error: Error initializing wallet database environment" self.nodes[1].assert_start_raises_init_error(extra_args=[f'-walletdir={wallet_dir}', f'-wallet={wallet_name}', '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX) diff --git a/test/functional/feature_framework_miniwallet.py b/test/functional/feature_framework_miniwallet.py index d1aa24e7cd..464f18eb48 100755 --- a/test/functional/feature_framework_miniwallet.py +++ b/test/functional/feature_framework_miniwallet.py @@ -9,7 +9,7 @@ import string from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( - assert_greater_than_or_equal, + assert_equal, ) from test_framework.wallet import ( MiniWallet, @@ -22,21 +22,19 @@ class FeatureFrameworkMiniWalletTest(BitcoinTestFramework): self.num_nodes = 1 def test_tx_padding(self): - """Verify that MiniWallet's transaction padding (`target_weight` parameter) - works accurately enough (i.e. at most 3 WUs higher) with all modes.""" + """Verify that MiniWallet's transaction padding (`target_vsize` parameter) + works accurately with all modes.""" for mode_name, wallet in self.wallets: self.log.info(f"Test tx padding with MiniWallet mode {mode_name}...") utxo = wallet.get_utxo(mark_as_spent=False) - for target_weight in [1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 4000000, - 989, 2001, 4337, 13371, 23219, 49153, 102035, 223419, 3999989]: - tx = wallet.create_self_transfer(utxo_to_spend=utxo, target_weight=target_weight)["tx"] - self.log.debug(f"-> target weight: {target_weight}, actual weight: {tx.get_weight()}") - assert_greater_than_or_equal(tx.get_weight(), target_weight) - assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) + for target_vsize in [250, 500, 1250, 2500, 5000, 12500, 25000, 50000, 1000000, + 248, 501, 1085, 3343, 5805, 12289, 25509, 55855, 999998]: + tx = wallet.create_self_transfer(utxo_to_spend=utxo, target_vsize=target_vsize)["tx"] + assert_equal(tx.get_vsize(), target_vsize) def test_wallet_tagging(self): """Verify that tagged wallet instances are able to send funds.""" - self.log.info(f"Test tagged wallet instances...") + self.log.info("Test tagged wallet instances...") node = self.nodes[0] untagged_wallet = self.wallets[0][1] for i in range(10): diff --git a/test/functional/feature_index_prune.py b/test/functional/feature_index_prune.py index 030bf51ed8..191c6466df 100755 --- a/test/functional/feature_index_prune.py +++ b/test/functional/feature_index_prune.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -# Copyright (c) 2020-2022 The Bitcoin Core developers +# Copyright (c) 2020-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test indices in conjunction with prune.""" +import concurrent.futures import os from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -19,9 +20,25 @@ class FeatureIndexPruneTest(BitcoinTestFramework): ["-fastprune", "-prune=1", "-blockfilterindex=1"], ["-fastprune", "-prune=1", "-coinstatsindex=1"], ["-fastprune", "-prune=1", "-blockfilterindex=1", "-coinstatsindex=1"], - [] + [], ] + def setup_network(self): + self.setup_nodes() # No P2P connection, so that linear_sync works + + def linear_sync(self, node_from, *, height_from=None): + # Linear sync over RPC, because P2P sync may not be linear + to_height = node_from.getblockcount() + if height_from is None: + height_from = min([n.getblockcount() for n in self.nodes]) + 1 + with concurrent.futures.ThreadPoolExecutor(max_workers=self.num_nodes) as rpc_threads: + for i in range(height_from, to_height + 1): + b = node_from.getblock(blockhash=node_from.getblockhash(i), verbosity=0) + list(rpc_threads.map(lambda n: n.submitblock(b), self.nodes)) + + def generate(self, node, num_blocks, sync_fun=None): + return super().generate(node, num_blocks, sync_fun=sync_fun or (lambda: self.linear_sync(node))) + def sync_index(self, height): expected_filter = { 'basic block filter index': {'synced': True, 'best_block_height': height}, @@ -36,22 +53,9 @@ class FeatureIndexPruneTest(BitcoinTestFramework): expected = {**expected_filter, **expected_stats} self.wait_until(lambda: self.nodes[2].getindexinfo() == expected) - def reconnect_nodes(self): - self.connect_nodes(0,1) - self.connect_nodes(0,2) - self.connect_nodes(0,3) - - def mine_batches(self, blocks): - n = blocks // 250 - for _ in range(n): - self.generate(self.nodes[0], 250) - self.generate(self.nodes[0], blocks % 250) - self.sync_blocks() - def restart_without_indices(self): for i in range(3): self.restart_node(i, extra_args=["-fastprune", "-prune=1"]) - self.reconnect_nodes() def run_test(self): filter_nodes = [self.nodes[0], self.nodes[2]] @@ -65,7 +69,7 @@ class FeatureIndexPruneTest(BitcoinTestFramework): for node in stats_nodes: assert node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash'] - self.mine_batches(500) + self.generate(self.nodes[0], 500) self.sync_index(height=700) self.log.info("prune some blocks") @@ -104,7 +108,7 @@ class FeatureIndexPruneTest(BitcoinTestFramework): msg = "Querying specific block heights requires coinstatsindex" assert_raises_rpc_error(-8, msg, node.gettxoutsetinfo, "muhash", height_hash) - self.mine_batches(749) + self.generate(self.nodes[0], 749) self.log.info("prune exactly up to the indices best blocks while the indices are disabled") for i in range(3): @@ -118,7 +122,7 @@ class FeatureIndexPruneTest(BitcoinTestFramework): self.log.info("prune further than the indices best blocks while the indices are disabled") self.restart_without_indices() - self.mine_batches(1000) + self.generate(self.nodes[0], 1000) for i in range(3): pruneheight_3 = self.nodes[i].pruneblockchain(2000) @@ -134,12 +138,10 @@ class FeatureIndexPruneTest(BitcoinTestFramework): self.log.info("make sure the nodes start again with the indices and an additional -reindex arg") for i in range(3): - restart_args = self.extra_args[i]+["-reindex"] + restart_args = self.extra_args[i] + ["-reindex"] self.restart_node(i, extra_args=restart_args) - # The nodes need to be reconnected to the non-pruning node upon restart, otherwise they will be stuck - self.connect_nodes(i, 3) - self.sync_blocks(timeout=300) + self.linear_sync(self.nodes[3]) self.sync_index(height=2500) for node in self.nodes[:2]: @@ -150,8 +152,7 @@ class FeatureIndexPruneTest(BitcoinTestFramework): self.log.info("ensure that prune locks don't prevent indices from failing in a reorg scenario") with self.nodes[0].assert_debug_log(['basic block filter index prune lock moved back to 2480']): self.nodes[3].invalidateblock(self.nodes[0].getblockhash(2480)) - self.generate(self.nodes[3], 30) - self.sync_blocks() + self.generate(self.nodes[3], 30, sync_fun=lambda: self.linear_sync(self.nodes[3], height_from=2480)) if __name__ == '__main__': diff --git a/test/functional/feature_init.py b/test/functional/feature_init.py index 659d33684e..4a2f7ecf42 100755 --- a/test/functional/feature_init.py +++ b/test/functional/feature_init.py @@ -2,17 +2,20 @@ # Copyright (c) 2021-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Stress tests related to node initialization.""" +"""Tests related to node initialization.""" from pathlib import Path import platform import shutil from test_framework.test_framework import BitcoinTestFramework, SkipTest -from test_framework.test_node import ErrorMatch +from test_framework.test_node import ( + BITCOIN_PID_FILENAME_DEFAULT, + ErrorMatch, +) from test_framework.util import assert_equal -class InitStressTest(BitcoinTestFramework): +class InitTest(BitcoinTestFramework): """ Ensure that initialization can be interrupted at a number of points and not impair subsequent starts. @@ -25,7 +28,7 @@ class InitStressTest(BitcoinTestFramework): self.setup_clean_chain = False self.num_nodes = 1 - def run_test(self): + def init_stress_test(self): """ - test terminating initialization after seeing a certain log line. - test removing certain essential files to test startup error paths. @@ -97,13 +100,13 @@ class InitStressTest(BitcoinTestFramework): files_to_delete = { 'blocks/index/*.ldb': 'Error opening block database.', - 'chainstate/*.ldb': 'Error opening block database.', + 'chainstate/*.ldb': 'Error opening coins database.', 'blocks/blk*.dat': 'Error loading block database.', } files_to_perturb = { 'blocks/index/*.ldb': 'Error loading block database.', - 'chainstate/*.ldb': 'Error opening block database.', + 'chainstate/*.ldb': 'Error opening coins database.', 'blocks/blk*.dat': 'Corrupted block database detected.', } @@ -147,6 +150,31 @@ class InitStressTest(BitcoinTestFramework): shutil.move(node.chain_path / "blocks_bak", node.chain_path / "blocks") shutil.move(node.chain_path / "chainstate_bak", node.chain_path / "chainstate") + def init_pid_test(self): + BITCOIN_PID_FILENAME_CUSTOM = "my_fancy_bitcoin_pid_file.foobar" + + self.log.info("Test specifying custom pid file via -pid command line option") + custom_pidfile_relative = BITCOIN_PID_FILENAME_CUSTOM + self.log.info(f"-> path relative to datadir ({custom_pidfile_relative})") + self.restart_node(0, [f"-pid={custom_pidfile_relative}"]) + datadir = self.nodes[0].chain_path + assert not (datadir / BITCOIN_PID_FILENAME_DEFAULT).exists() + assert (datadir / custom_pidfile_relative).exists() + self.stop_node(0) + assert not (datadir / custom_pidfile_relative).exists() + + custom_pidfile_absolute = Path(self.options.tmpdir) / BITCOIN_PID_FILENAME_CUSTOM + self.log.info(f"-> absolute path ({custom_pidfile_absolute})") + self.restart_node(0, [f"-pid={custom_pidfile_absolute}"]) + assert not (datadir / BITCOIN_PID_FILENAME_DEFAULT).exists() + assert custom_pidfile_absolute.exists() + self.stop_node(0) + assert not custom_pidfile_absolute.exists() + + def run_test(self): + self.init_pid_test() + self.init_stress_test() + if __name__ == '__main__': - InitStressTest(__file__).main() + InitTest(__file__).main() diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py index 1519c132b9..0e6ec14c77 100755 --- a/test/functional/feature_loadblock.py +++ b/test/functional/feature_loadblock.py @@ -51,8 +51,8 @@ class LoadblockTest(BitcoinTestFramework): cfg.write(f"port={node_url.port}\n") cfg.write(f"host={node_url.hostname}\n") cfg.write(f"output_file={bootstrap_file}\n") - cfg.write(f"max_height=100\n") - cfg.write(f"netmagic=fabfb5da\n") + cfg.write("max_height=100\n") + cfg.write("netmagic=fabfb5da\n") cfg.write(f"input={blocks_dir}\n") cfg.write(f"genesis={genesis_block}\n") cfg.write(f"hashlist={hash_list.name}\n") diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 136cdd024d..ad5a7e4831 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -122,7 +122,7 @@ class MaxUploadTest(BitcoinTestFramework): assert_equal(len(self.nodes[0].getpeerinfo()), 3) # At most a couple more tries should succeed (depending on how long # the test has been running so far). - with self.nodes[0].assert_debug_log(expected_msgs=["historical block serving limit reached, disconnect peer"]): + with self.nodes[0].assert_debug_log(expected_msgs=["historical block serving limit reached, disconnecting peer=0"]): for _ in range(3): p2p_conns[0].send_message(getdata_request) p2p_conns[0].wait_for_disconnect() @@ -147,7 +147,7 @@ class MaxUploadTest(BitcoinTestFramework): # But if p2p_conns[1] tries for an old block, it gets disconnected too. getdata_request.inv = [CInv(MSG_BLOCK, big_old_block)] - with self.nodes[0].assert_debug_log(expected_msgs=["historical block serving limit reached, disconnect peer"]): + with self.nodes[0].assert_debug_log(expected_msgs=["historical block serving limit reached, disconnecting peer=1"]): p2p_conns[1].send_message(getdata_request) p2p_conns[1].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 1) @@ -197,7 +197,7 @@ class MaxUploadTest(BitcoinTestFramework): assert_equal(peer_info[0]['permissions'], ['download']) self.log.info("Peer gets disconnected for a mempool request after limit is reached") - with self.nodes[0].assert_debug_log(expected_msgs=["mempool request with bandwidth limit reached, disconnect peer"]): + with self.nodes[0].assert_debug_log(expected_msgs=["mempool request with bandwidth limit reached, disconnecting peer=0"]): peer.send_message(msg_mempool()) peer.wait_for_disconnect() diff --git a/test/functional/feature_minchainwork.py b/test/functional/feature_minchainwork.py index 34228f6f38..d6654ef8ed 100755 --- a/test/functional/feature_minchainwork.py +++ b/test/functional/feature_minchainwork.py @@ -19,7 +19,10 @@ import time from test_framework.p2p import P2PInterface, msg_getheaders from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal +from test_framework.util import ( + assert_equal, + ensure_for, +) # 2 hashes required per regtest block (with no difficulty adjustment) REGTEST_WORK_PER_BLOCK = 2 @@ -58,18 +61,14 @@ class MinimumChainWorkTest(BitcoinTestFramework): hashes = self.generate(self.nodes[0], num_blocks_to_generate, sync_fun=self.no_op) self.log.info(f"Node0 current chain work: {self.nodes[0].getblockheader(hashes[-1])['chainwork']}") - - # Sleep a few seconds and verify that node2 didn't get any new blocks - # or headers. We sleep, rather than sync_blocks(node0, node1) because - # it's reasonable either way for node1 to get the blocks, or not get - # them (since they're below node1's minchainwork). - time.sleep(3) - self.log.info("Verifying node 2 has no more blocks than before") self.log.info(f"Blockcounts: {[n.getblockcount() for n in self.nodes]}") # Node2 shouldn't have any new headers yet, because node1 should not # have relayed anything. - assert_equal(len(self.nodes[2].getchaintips()), 1) + # We wait 3 seconds, rather than sync_blocks(node0, node1) because + # it's reasonable either way for node1 to get the blocks, or not get + # them (since they're below node1's minchainwork). + ensure_for(duration=3, f=lambda: len(self.nodes[2].getchaintips()) == 1) assert_equal(self.nodes[2].getchaintips()[0]['height'], 0) assert self.nodes[1].getbestblockhash() != self.nodes[0].getbestblockhash() @@ -81,8 +80,7 @@ class MinimumChainWorkTest(BitcoinTestFramework): msg.locator.vHave = [int(self.nodes[2].getbestblockhash(), 16)] msg.hashstop = 0 peer.send_and_ping(msg) - time.sleep(5) - assert "headers" not in peer.last_message or len(peer.last_message["headers"].headers) == 0 + ensure_for(duration=5, f=lambda: "headers" not in peer.last_message or len(peer.last_message["headers"].headers) == 0) self.log.info("Generating one more block") self.generate(self.nodes[0], 1) diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index a53f78c13d..885bc4855b 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -57,7 +57,6 @@ class NULLDUMMYTest(BitcoinTestFramework): self.extra_args = [[ f'-testactivationheight=segwit@{COINBASE_MATURITY + 5}', '-addresstype=legacy', - '-par=1', # Use only one script thread to get the exact reject reason for testing ]] def create_transaction(self, *, txid, input_details=None, addr, amount, privkey): diff --git a/test/functional/feature_port.py b/test/functional/feature_port.py new file mode 100755 index 0000000000..2746d7d79c --- /dev/null +++ b/test/functional/feature_port.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test the -port option and its interactions with +-bind. +""" + +from test_framework.test_framework import ( + BitcoinTestFramework, +) +from test_framework.util import ( + p2p_port, +) + + +class PortTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + # Avoid any -bind= on the command line. + self.bind_to_localhost_only = False + self.num_nodes = 1 + + def run_test(self): + node = self.nodes[0] + node.has_explicit_bind = True + port1 = p2p_port(self.num_nodes) + port2 = p2p_port(self.num_nodes + 5) + + self.log.info("When starting with -port, bitcoind binds to it and uses port + 1 for an onion bind") + with node.assert_debug_log(expected_msgs=[f'Bound to 0.0.0.0:{port1}', f'Bound to 127.0.0.1:{port1 + 1}']): + self.restart_node(0, extra_args=["-listen", f"-port={port1}"]) + + self.log.info("When specifying -port multiple times, only the last one is taken") + with node.assert_debug_log(expected_msgs=[f'Bound to 0.0.0.0:{port2}', f'Bound to 127.0.0.1:{port2 + 1}'], unexpected_msgs=[f'Bound to 0.0.0.0:{port1}']): + self.restart_node(0, extra_args=["-listen", f"-port={port1}", f"-port={port2}"]) + + self.log.info("When specifying ports with both -port and -bind, the one from -port is ignored") + with node.assert_debug_log(expected_msgs=[f'Bound to 0.0.0.0:{port2}'], unexpected_msgs=[f'Bound to 0.0.0.0:{port1}']): + self.restart_node(0, extra_args=["-listen", f"-port={port1}", f"-bind=0.0.0.0:{port2}"]) + + self.log.info("When -bind specifies no port, the values from -port and -bind are combined") + with self.nodes[0].assert_debug_log(expected_msgs=[f'Bound to 0.0.0.0:{port1}']): + self.restart_node(0, extra_args=["-listen", f"-port={port1}", "-bind=0.0.0.0"]) + + self.log.info("When an onion bind specifies no port, the value from -port, incremented by 1, is taken") + with self.nodes[0].assert_debug_log(expected_msgs=[f'Bound to 127.0.0.1:{port1 + 1}']): + self.restart_node(0, extra_args=["-listen", f"-port={port1}", "-bind=127.0.0.1=onion"]) + + self.log.info("Invalid values for -port raise errors") + self.stop_node(0) + node.extra_args = ["-listen", "-port=65536"] + node.assert_start_raises_init_error(expected_msg="Error: Invalid port specified in -port: '65536'") + node.extra_args = ["-listen", "-port=0"] + node.assert_start_raises_init_error(expected_msg="Error: Invalid port specified in -port: '0'") + + +if __name__ == '__main__': + PortTest(__file__).main() diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index b660b96935..ef2ecfab9e 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -9,7 +9,6 @@ from decimal import Decimal from test_framework.messages import ( MAX_BIP125_RBF_SEQUENCE, COIN, - SEQUENCE_FINAL, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -26,18 +25,15 @@ class ReplaceByFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - # both nodes disable full-rbf to test BIP125 signaling self.extra_args = [ [ - "-mempoolfullrbf=0", "-limitancestorcount=50", "-limitancestorsize=101", "-limitdescendantcount=200", "-limitdescendantsize=101", ], - # second node has default mempool parameters, besides mempoolfullrbf being disabled + # second node has default mempool parameters [ - "-mempoolfullrbf=0", ], ] self.supports_cli = False @@ -69,18 +65,12 @@ class ReplaceByFeeTest(BitcoinTestFramework): self.log.info("Running test too many replacements using default mempool params...") self.test_too_many_replacements_with_default_mempool_params() - self.log.info("Running test opt-in...") - self.test_opt_in() - self.log.info("Running test RPC...") self.test_rpc() self.log.info("Running test prioritised transactions...") self.test_prioritised_transactions() - self.log.info("Running test no inherited signaling...") - self.test_no_inherited_signaling() - self.log.info("Running test replacement relay fee...") self.test_replacement_relay_fee() @@ -113,14 +103,22 @@ class ReplaceByFeeTest(BitcoinTestFramework): """Simple doublespend""" # we use MiniWallet to create a transaction template with inputs correctly set, # and modify the output (amount, scriptPubKey) according to our needs - tx = self.wallet.create_self_transfer()["tx"] + tx = self.wallet.create_self_transfer(fee_rate=Decimal("0.003"))["tx"] tx1a_txid = self.nodes[0].sendrawtransaction(tx.serialize().hex()) # Should fail because we haven't changed the fee tx.vout[0].scriptPubKey[-1] ^= 1 + tx.rehash() + tx_hex = tx.serialize().hex() # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx.serialize().hex(), 0) + reject_reason = "insufficient fee" + reject_details = f"{reject_reason}, rejecting replacement {tx.hash}; new feerate 0.00300000 BTC/kvB <= old feerate 0.00300000 BTC/kvB" + res = self.nodes[0].testmempoolaccept(rawtxs=[tx_hex])[0] + assert_equal(res["reject-reason"], reject_reason) + assert_equal(res["reject-details"], reject_details) + assert_raises_rpc_error(-26, f"{reject_details}", self.nodes[0].sendrawtransaction, tx_hex, 0) + # Extra 0.1 BTC fee tx.vout[0].nValue -= int(0.1 * COIN) @@ -164,7 +162,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): dbl_tx_hex = dbl_tx.serialize().hex() # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) + reject_reason = "insufficient fee" + reject_details = f"{reject_reason}, rejecting replacement {dbl_tx.hash}, less fees than conflicting txs; 3.00 < 4.00" + res = self.nodes[0].testmempoolaccept(rawtxs=[dbl_tx_hex])[0] + assert_equal(res["reject-reason"], reject_reason) + assert_equal(res["reject-details"], reject_details) + assert_raises_rpc_error(-26, f"{reject_details}", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) + + # Accepted with sufficient fee dbl_tx.vout[0].nValue = int(0.1 * COIN) @@ -283,22 +288,30 @@ class ReplaceByFeeTest(BitcoinTestFramework): utxo1 = self.make_utxo(self.nodes[0], int(1.2 * COIN)) utxo2 = self.make_utxo(self.nodes[0], 3 * COIN) - tx1a_utxo = self.wallet.send_self_transfer( + tx1a = self.wallet.send_self_transfer( from_node=self.nodes[0], utxo_to_spend=utxo1, sequence=0, fee=Decimal("0.1"), - )["new_utxo"] + ) + tx1a_utxo = tx1a["new_utxo"] # Direct spend an output of the transaction we're replacing. - tx2_hex = self.wallet.create_self_transfer_multi( + tx2 = self.wallet.create_self_transfer_multi( utxos_to_spend=[utxo1, utxo2, tx1a_utxo], sequence=0, amount_per_output=int(COIN * tx1a_utxo["value"]), - )["hex"] + )["tx"] + tx2_hex = tx2.serialize().hex() # This will raise an exception - assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, 0) + reject_reason = "bad-txns-spends-conflicting-tx" + reject_details = f"{reject_reason}, {tx2.hash} spends conflicting transaction {tx1a['tx'].hash}" + res = self.nodes[0].testmempoolaccept(rawtxs=[tx2_hex])[0] + assert_equal(res["reject-reason"], reject_reason) + assert_equal(res["reject-details"], reject_details) + assert_raises_rpc_error(-26, f"{reject_details}", self.nodes[0].sendrawtransaction, tx2_hex, 0) + # Spend tx1a's output to test the indirect case. tx1b_utxo = self.wallet.send_self_transfer( @@ -329,14 +342,21 @@ class ReplaceByFeeTest(BitcoinTestFramework): fee=Decimal("0.1"), ) - tx2_hex = self.wallet.create_self_transfer_multi( + tx2 = self.wallet.create_self_transfer_multi( utxos_to_spend=[confirmed_utxo, unconfirmed_utxo], sequence=0, amount_per_output=1 * COIN, - )["hex"] + )["tx"] + tx2_hex = tx2.serialize().hex() # This will raise an exception - assert_raises_rpc_error(-26, "replacement-adds-unconfirmed", self.nodes[0].sendrawtransaction, tx2_hex, 0) + reject_reason = "replacement-adds-unconfirmed" + reject_details = f"{reject_reason}, replacement {tx2.hash} adds unconfirmed input, idx 1" + res = self.nodes[0].testmempoolaccept(rawtxs=[tx2_hex])[0] + assert_equal(res["reject-reason"], reject_reason) + assert_equal(res["reject-details"], reject_details) + assert_raises_rpc_error(-26, f"{reject_details}", self.nodes[0].sendrawtransaction, tx2_hex, 0) + def test_too_many_replacements(self): """Replacements that evict too many transactions are rejected""" @@ -378,7 +398,13 @@ class ReplaceByFeeTest(BitcoinTestFramework): double_tx_hex = double_tx.serialize().hex() # This will raise an exception - assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, double_tx_hex, 0) + reject_reason = "too many potential replacements" + reject_details = f"{reject_reason}, rejecting replacement {double_tx.hash}; too many potential replacements ({MAX_REPLACEMENT_LIMIT + 1} > {MAX_REPLACEMENT_LIMIT})" + res = self.nodes[0].testmempoolaccept(rawtxs=[double_tx_hex])[0] + assert_equal(res["reject-reason"], reject_reason) + assert_equal(res["reject-details"], reject_details) + assert_raises_rpc_error(-26, f"{reject_details}", self.nodes[0].sendrawtransaction, double_tx_hex, 0) + # If we remove an input, it should pass double_tx.vin.pop() @@ -426,14 +452,12 @@ class ReplaceByFeeTest(BitcoinTestFramework): for graph_num in range(num_tx_graphs): root_utxos.append(wallet.get_utxo()) - optin_parent_tx = wallet.send_self_transfer_multi( + parent_tx = wallet.send_self_transfer_multi( from_node=normal_node, - sequence=MAX_BIP125_RBF_SEQUENCE, utxos_to_spend=[root_utxos[graph_num]], num_outputs=txs_per_graph, ) - assert_equal(True, normal_node.getmempoolentry(optin_parent_tx['txid'])['bip125-replaceable']) - new_utxos = optin_parent_tx['new_utxos'] + new_utxos = parent_tx['new_utxos'] for utxo in new_utxos: # Create spends for each output from the "root" of this graph. @@ -470,81 +494,6 @@ class ReplaceByFeeTest(BitcoinTestFramework): self.generate(normal_node, 1) self.wallet.rescan_utxos() - def test_opt_in(self): - """Replacing should only work if orig tx opted in""" - tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - - # Create a non-opting in transaction - tx1a_utxo = self.wallet.send_self_transfer( - from_node=self.nodes[0], - utxo_to_spend=tx0_outpoint, - sequence=SEQUENCE_FINAL, - fee=Decimal("0.1"), - )["new_utxo"] - - # This transaction isn't shown as replaceable - assert_equal(self.nodes[0].getmempoolentry(tx1a_utxo["txid"])['bip125-replaceable'], False) - - # Shouldn't be able to double-spend - tx1b_hex = self.wallet.create_self_transfer( - utxo_to_spend=tx0_outpoint, - sequence=0, - fee=Decimal("0.2"), - )["hex"] - - # This will raise an exception - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx1b_hex, 0) - - tx1_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - - # Create a different non-opting in transaction - tx2a_utxo = self.wallet.send_self_transfer( - from_node=self.nodes[0], - utxo_to_spend=tx1_outpoint, - sequence=0xfffffffe, - fee=Decimal("0.1"), - )["new_utxo"] - - # Still shouldn't be able to double-spend - tx2b_hex = self.wallet.create_self_transfer( - utxo_to_spend=tx1_outpoint, - sequence=0, - fee=Decimal("0.2"), - )["hex"] - - # This will raise an exception - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx2b_hex, 0) - - # Now create a new transaction that spends from tx1a and tx2a - # opt-in on one of the inputs - # Transaction should be replaceable on either input - - tx3a_txid = self.wallet.send_self_transfer_multi( - from_node=self.nodes[0], - utxos_to_spend=[tx1a_utxo, tx2a_utxo], - sequence=[SEQUENCE_FINAL, 0xfffffffd], - fee_per_output=int(0.1 * COIN), - )["txid"] - - # This transaction is shown as replaceable - assert_equal(self.nodes[0].getmempoolentry(tx3a_txid)['bip125-replaceable'], True) - - self.wallet.send_self_transfer( - from_node=self.nodes[0], - utxo_to_spend=tx1a_utxo, - sequence=0, - fee=Decimal("0.4"), - ) - - # If tx3b was accepted, tx3c won't look like a replacement, - # but make sure it is accepted anyway - self.wallet.send_self_transfer( - from_node=self.nodes[0], - utxo_to_spend=tx2a_utxo, - sequence=0, - fee=Decimal("0.4"), - ) - def test_prioritised_transactions(self): # Ensure that fee deltas used via prioritisetransaction are # correctly used by replacement logic @@ -629,69 +578,6 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert_equal(json0["vin"][0]["sequence"], 4294967293) assert_equal(json1["vin"][0]["sequence"], 4294967294) - def test_no_inherited_signaling(self): - confirmed_utxo = self.wallet.get_utxo() - - # Create an explicitly opt-in parent transaction - optin_parent_tx = self.wallet.send_self_transfer( - from_node=self.nodes[0], - utxo_to_spend=confirmed_utxo, - sequence=MAX_BIP125_RBF_SEQUENCE, - fee_rate=Decimal('0.01'), - ) - assert_equal(True, self.nodes[0].getmempoolentry(optin_parent_tx['txid'])['bip125-replaceable']) - - replacement_parent_tx = self.wallet.create_self_transfer( - utxo_to_spend=confirmed_utxo, - sequence=MAX_BIP125_RBF_SEQUENCE, - fee_rate=Decimal('0.02'), - ) - - # Test if parent tx can be replaced. - res = self.nodes[0].testmempoolaccept(rawtxs=[replacement_parent_tx['hex']])[0] - - # Parent can be replaced. - assert_equal(res['allowed'], True) - - # Create an opt-out child tx spending the opt-in parent - parent_utxo = self.wallet.get_utxo(txid=optin_parent_tx['txid']) - optout_child_tx = self.wallet.send_self_transfer( - from_node=self.nodes[0], - utxo_to_spend=parent_utxo, - sequence=SEQUENCE_FINAL, - fee_rate=Decimal('0.01'), - ) - - # Reports true due to inheritance - assert_equal(True, self.nodes[0].getmempoolentry(optout_child_tx['txid'])['bip125-replaceable']) - - replacement_child_tx = self.wallet.create_self_transfer( - utxo_to_spend=parent_utxo, - sequence=SEQUENCE_FINAL, - fee_rate=Decimal('0.02'), - ) - - # Broadcast replacement child tx - # BIP 125 : - # 1. The original transactions signal replaceability explicitly or through inheritance as described in the above - # Summary section. - # The original transaction (`optout_child_tx`) doesn't signal RBF but its parent (`optin_parent_tx`) does. - # The replacement transaction (`replacement_child_tx`) should be able to replace the original transaction. - # See CVE-2021-31876 for further explanations. - assert_equal(True, self.nodes[0].getmempoolentry(optin_parent_tx['txid'])['bip125-replaceable']) - assert_raises_rpc_error(-26, 'txn-mempool-conflict', self.nodes[0].sendrawtransaction, replacement_child_tx["hex"], 0) - - self.log.info('Check that the child tx can still be replaced (via a tx that also replaces the parent)') - replacement_parent_tx = self.wallet.send_self_transfer( - from_node=self.nodes[0], - utxo_to_spend=confirmed_utxo, - sequence=SEQUENCE_FINAL, - fee_rate=Decimal('0.03'), - ) - # Check that child is removed and update wallet utxo state - assert_raises_rpc_error(-5, 'Transaction not in mempool', self.nodes[0].getmempoolentry, optout_child_tx['txid']) - self.wallet.get_utxo(txid=optout_child_tx['txid']) - def test_replacement_relay_fee(self): tx = self.wallet.send_self_transfer(from_node=self.nodes[0])['tx'] @@ -702,12 +588,12 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx.serialize().hex()) def test_fullrbf(self): + # BIP125 signaling is not respected confirmed_utxo = self.make_utxo(self.nodes[0], int(2 * COIN)) - self.restart_node(0, extra_args=["-mempoolfullrbf=1"]) assert self.nodes[0].getmempoolinfo()["fullrbf"] - # Create an explicitly opt-out transaction + # Create an explicitly opt-out BIP125 transaction, which will be ignored optout_tx = self.wallet.send_self_transfer( from_node=self.nodes[0], utxo_to_spend=confirmed_utxo, @@ -718,7 +604,6 @@ class ReplaceByFeeTest(BitcoinTestFramework): conflicting_tx = self.wallet.create_self_transfer( utxo_to_spend=confirmed_utxo, - sequence=SEQUENCE_FINAL, fee_rate=Decimal('0.02'), ) diff --git a/test/functional/feature_settings.py b/test/functional/feature_settings.py index a7294944bf..e9148b7923 100755 --- a/test/functional/feature_settings.py +++ b/test/functional/feature_settings.py @@ -46,7 +46,7 @@ class SettingsTest(BitcoinTestFramework): # Assert default settings file was created self.stop_node(0) - default_settings = {"_warning_": f"This file is automatically generated and updated by {self.config['environment']['PACKAGE_NAME']}. Please do not edit this file while the node is running, as any changes might be ignored or overwritten."} + default_settings = {"_warning_": f"This file is automatically generated and updated by {self.config['environment']['CLIENT_NAME']}. Please do not edit this file while the node is running, as any changes might be ignored or overwritten."} with settings.open() as fp: assert_equal(json.load(fp), default_settings) diff --git a/test/functional/feature_signet.py b/test/functional/feature_signet.py index b648266cae..63091b3ee8 100755 --- a/test/functional/feature_signet.py +++ b/test/functional/feature_signet.py @@ -9,6 +9,8 @@ from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal +SIGNET_DEFAULT_CHALLENGE = '512103ad5e0edad18cb1f0fc0d28a3d4f1f3e445640337489abb10404f2d1e086be430210359ef5021964fe22d6f8e05b2463c9540ce96883fe3b278760f048f5189f2e6c452ae' + signet_blocks = [ '00000020f61eee3b63a380a477a063af32b2bbc97c9ff9f01f2c4225e973988108000000f575c83235984e7dc4afc1f30944c170462e84437ab6f2d52e16878a79e4678bd1914d5fae77031eccf4070001010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025151feffffff0200f2052a010000001600149243f727dd5343293eb83174324019ec16c2630f0000000000000000776a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf94c4fecc7daa2490047304402205e423a8754336ca99dbe16509b877ef1bf98d008836c725005b3c787c41ebe46022047246e4467ad7cc7f1ad98662afcaf14c115e0095a227c7b05c5182591c23e7e01000120000000000000000000000000000000000000000000000000000000000000000000000000', '00000020533b53ded9bff4adc94101d32400a144c54edc5ed492a3b26c63b2d686000000b38fef50592017cfafbcab88eb3d9cf50b2c801711cad8299495d26df5e54812e7914d5fae77031ecfdd0b0001010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025251feffffff0200f2052a01000000160014fd09839740f0e0b4fc6d5e2527e4022aa9b89dfa0000000000000000776a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf94c4fecc7daa24900473044022031d64a1692cdad1fc0ced69838169fe19ae01be524d831b95fcf5ea4e6541c3c02204f9dea0801df8b4d0cd0857c62ab35c6c25cc47c930630dc7fe723531daa3e9b01000120000000000000000000000000000000000000000000000000000000000000000000000000', @@ -22,21 +24,31 @@ signet_blocks = [ '00000020a868e8514be5e46dabd6a122132f423f36a43b716a40c394e2a8d063e1010000f4c6c717e99d800c699c25a2006a75a0c5c09f432a936f385e6fce139cdbd1a5e9964d5fae77031e7d026e0001010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025a51feffffff0200f2052a01000000160014aaa671c82b138e3b8f510cd801e5f2bd0aa305940000000000000000776a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf94c4fecc7daa24900473044022042309f4c3c7a1a2ac8c24f890f962df1c0086cec10be0868087cfc427520cb2702201dafee8911c269b7e786e242045bb57cef3f5b0f177010c6159abae42f646cc501000120000000000000000000000000000000000000000000000000000000000000000000000000', ] +class SignetParams: + def __init__(self, challenge=None): + if challenge is None: + self.challenge = SIGNET_DEFAULT_CHALLENGE + self.shared_args = [] + else: + self.challenge = challenge + self.shared_args = [f"-signetchallenge={challenge}"] class SignetBasicTest(BitcoinTestFramework): def set_test_params(self): self.chain = "signet" self.num_nodes = 6 self.setup_clean_chain = True - shared_args1 = ["-signetchallenge=51"] # OP_TRUE - shared_args2 = [] # default challenge - # we use the exact same challenge except we do it as a 2-of-2, which means it should fail - shared_args3 = ["-signetchallenge=522103ad5e0edad18cb1f0fc0d28a3d4f1f3e445640337489abb10404f2d1e086be430210359ef5021964fe22d6f8e05b2463c9540ce96883fe3b278760f048f5189f2e6c452ae"] + self.signets = [ + SignetParams(challenge='51'), # OP_TRUE + SignetParams(), # default challenge + # default challenge as a 2-of-2, which means it should fail + SignetParams(challenge='522103ad5e0edad18cb1f0fc0d28a3d4f1f3e445640337489abb10404f2d1e086be430210359ef5021964fe22d6f8e05b2463c9540ce96883fe3b278760f048f5189f2e6c452ae') + ] self.extra_args = [ - shared_args1, shared_args1, - shared_args2, shared_args2, - shared_args3, shared_args3, + self.signets[0].shared_args, self.signets[0].shared_args, + self.signets[1].shared_args, self.signets[1].shared_args, + self.signets[2].shared_args, self.signets[2].shared_args, ] def setup_network(self): @@ -50,14 +62,28 @@ class SignetBasicTest(BitcoinTestFramework): def run_test(self): self.log.info("basic tests using OP_TRUE challenge") + self.log.info('getblockchaininfo') + def check_getblockchaininfo(node_idx, signet_idx): + blockchain_info = self.nodes[node_idx].getblockchaininfo() + assert_equal(blockchain_info['chain'], 'signet') + assert_equal(blockchain_info['signet_challenge'], self.signets[signet_idx].challenge) + check_getblockchaininfo(node_idx=1, signet_idx=0) + check_getblockchaininfo(node_idx=2, signet_idx=1) + check_getblockchaininfo(node_idx=5, signet_idx=2) + self.log.info('getmininginfo') - mining_info = self.nodes[0].getmininginfo() - assert_equal(mining_info['blocks'], 0) - assert_equal(mining_info['chain'], 'signet') - assert 'currentblocktx' not in mining_info - assert 'currentblockweight' not in mining_info - assert_equal(mining_info['networkhashps'], Decimal('0')) - assert_equal(mining_info['pooledtx'], 0) + def check_getmininginfo(node_idx, signet_idx): + mining_info = self.nodes[node_idx].getmininginfo() + assert_equal(mining_info['blocks'], 0) + assert_equal(mining_info['chain'], 'signet') + assert 'currentblocktx' not in mining_info + assert 'currentblockweight' not in mining_info + assert_equal(mining_info['networkhashps'], Decimal('0')) + assert_equal(mining_info['pooledtx'], 0) + assert_equal(mining_info['signet_challenge'], self.signets[signet_idx].challenge) + check_getmininginfo(node_idx=0, signet_idx=0) + check_getmininginfo(node_idx=3, signet_idx=1) + check_getmininginfo(node_idx=4, signet_idx=2) self.generate(self.nodes[0], 1, sync_fun=self.no_op) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 443c1c33da..daf11a9ae6 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -1285,7 +1285,6 @@ class TaprootTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True - self.extra_args = [["-par=1"]] def block_submit(self, node, txs, msg, err_msg, cb_pubkey=None, fees=0, sigops_weight=0, witness=False, accept=False): diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 30bf97185a..134b4e566b 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -30,7 +30,12 @@ JSON_PARSING_ERROR = 'error: Error parsing JSON: foo' BLOCKS_VALUE_OF_ZERO = 'error: the first argument (number of blocks to generate, default: 1) must be an integer value greater than zero' TOO_MANY_ARGS = 'error: too many arguments (maximum 2 for nblocks and maxtries)' WALLET_NOT_LOADED = 'Requested wallet does not exist or is not loaded' -WALLET_NOT_SPECIFIED = 'Wallet file not specified' +WALLET_NOT_SPECIFIED = ( + "Multiple wallets are loaded. Please select which wallet to use by requesting the RPC " + "through the /wallet/<walletname> URI path. Or for the CLI, specify the \"-rpcwallet=<walletname>\" " + "option before the command (run \"bitcoin-cli -h\" for help or \"bitcoin-cli listwallets\" to see " + "which wallets are currently loaded)." +) def cli_get_info_string_to_dict(cli_get_info_string): @@ -159,6 +164,9 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Test connecting with non-existing RPC cookie file") assert_raises_process_error(1, "Could not locate RPC credentials", self.nodes[0].cli('-rpccookiefile=does-not-exist', '-rpcpassword=').echo) + self.log.info("Test connecting without RPC cookie file and with password arg") + assert_equal(BLOCKS, self.nodes[0].cli('-norpccookiefile', f'-rpcuser={user}', f'-rpcpassword={password}').getblockcount()) + self.log.info("Test -getinfo with arguments fails") assert_raises_process_error(1, "-getinfo takes no arguments", self.nodes[0].cli('-getinfo').help) @@ -331,6 +339,10 @@ class TestBitcoinCli(BitcoinTestFramework): n4 = 10 blocks = self.nodes[0].getblockcount() + self.log.info('Test -generate -rpcwallet=<filename> raise RPC error') + wallet2_path = f'-rpcwallet={self.nodes[0].wallets_path / wallets[2] / self.wallet_data_filename}' + assert_raises_rpc_error(-18, WALLET_NOT_LOADED, self.nodes[0].cli(wallet2_path, '-generate').echo) + self.log.info('Test -generate -rpcwallet with no args') generate = self.nodes[0].cli(rpcwallet2, '-generate').send_cli() assert_equal(set(generate.keys()), {'address', 'blocks'}) @@ -366,7 +378,7 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Test -version with node stopped") self.stop_node(0) cli_response = self.nodes[0].cli('-version').send_cli() - assert f"{self.config['environment']['PACKAGE_NAME']} RPC client version" in cli_response + assert f"{self.config['environment']['CLIENT_NAME']} RPC client version" in cli_response self.log.info("Test -rpcwait option successfully waits for RPC connection") self.nodes[0].start() # start node without RPC connection diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index ba6e960476..ed117ae08b 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -209,7 +209,7 @@ class RESTTest (BitcoinTestFramework): self.test_rest_request(f"/getutxos/{spending[0]}-+1", ret_type=RetType.OBJ, status=400) self.test_rest_request(f"/getutxos/{spending[0]}--1", ret_type=RetType.OBJ, status=400) self.test_rest_request(f"/getutxos/{spending[0]}aa-1234", ret_type=RetType.OBJ, status=400) - self.test_rest_request(f"/getutxos/aa-1234", ret_type=RetType.OBJ, status=400) + self.test_rest_request("/getutxos/aa-1234", ret_type=RetType.OBJ, status=400) # Test limits long_uri = '/'.join([f"{txid}-{n_}" for n_ in range(20)]) diff --git a/test/functional/interface_usdt_mempool.py b/test/functional/interface_usdt_mempool.py index a088278665..44882f642e 100755 --- a/test/functional/interface_usdt_mempool.py +++ b/test/functional/interface_usdt_mempool.py @@ -7,6 +7,7 @@ See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-mempool """ +import ctypes from decimal import Decimal # Test will be skipped if we don't have bcc installed @@ -63,6 +64,7 @@ struct replaced_event u8 replacement_hash[HASH_LENGTH]; s32 replacement_vsize; s64 replacement_fee; + bool replaced_by_transaction; }; // BPF perf buffer to push the data to user space. @@ -115,6 +117,7 @@ int trace_replaced(struct pt_regs *ctx) { bpf_usdt_readarg_p(5, ctx, &replaced.replacement_hash, HASH_LENGTH); bpf_usdt_readarg(6, ctx, &replaced.replacement_vsize); bpf_usdt_readarg(7, ctx, &replaced.replacement_fee); + bpf_usdt_readarg(8, ctx, &replaced.replaced_by_transaction); replaced_events.perf_submit(ctx, &replaced, sizeof(replaced)); return 0; @@ -123,6 +126,19 @@ int trace_replaced(struct pt_regs *ctx) { """ +class MempoolReplaced(ctypes.Structure): + _fields_ = [ + ("replaced_hash", ctypes.c_ubyte * 32), + ("replaced_vsize", ctypes.c_int32), + ("replaced_fee", ctypes.c_int64), + ("replaced_entry_time", ctypes.c_uint64), + ("replacement_hash", ctypes.c_ubyte * 32), + ("replacement_vsize", ctypes.c_int32), + ("replacement_fee", ctypes.c_int64), + ("replaced_by_transaction", ctypes.c_bool), + ] + + class MempoolTracepointTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -230,7 +246,8 @@ class MempoolTracepointTest(BitcoinTestFramework): bpf = BPF(text=MEMPOOL_TRACEPOINTS_PROGRAM, usdt_contexts=[ctx], debug=0, cflags=["-Wno-error=implicit-function-declaration"]) def handle_replaced_event(_, data, __): - events.append(bpf["replaced_events"].event(data)) + event = ctypes.cast(data, ctypes.POINTER(MempoolReplaced)).contents + events.append(event) bpf["replaced_events"].open_perf_buffer(handle_replaced_event) @@ -261,6 +278,7 @@ class MempoolTracepointTest(BitcoinTestFramework): assert_equal(bytes(event.replacement_hash)[::-1].hex(), replacement_tx["txid"]) assert_equal(event.replacement_vsize, replacement_tx["tx"].get_vsize()) assert_equal(event.replacement_fee, replacement_fee) + assert_equal(event.replaced_by_transaction, True) bpf.cleanup() self.generate(self.wallet, 1) diff --git a/test/functional/interface_usdt_net.py b/test/functional/interface_usdt_net.py index 7b55259b63..5468ddf858 100755 --- a/test/functional/interface_usdt_net.py +++ b/test/functional/interface_usdt_net.py @@ -40,7 +40,8 @@ net_tracepoints_program = """ MAX_MSG_TYPE_LENGTH, MAX_MSG_DATA_LENGTH ) + """ -#define MIN(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; }) +// A min() macro. Prefixed with _TRACEPOINT_TEST to avoid collision with other MIN macros. +#define _TRACEPOINT_TEST_MIN(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; }) struct p2p_message { @@ -60,7 +61,7 @@ int trace_inbound_message(struct pt_regs *ctx) { bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); bpf_usdt_readarg(5, ctx, &msg.msg_size); - bpf_usdt_readarg_p(6, ctx, &msg.msg, MIN(msg.msg_size, MAX_MSG_DATA_LENGTH)); + bpf_usdt_readarg_p(6, ctx, &msg.msg, _TRACEPOINT_TEST_MIN(msg.msg_size, MAX_MSG_DATA_LENGTH)); inbound_messages.perf_submit(ctx, &msg, sizeof(msg)); return 0; } @@ -73,7 +74,7 @@ int trace_outbound_message(struct pt_regs *ctx) { bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); bpf_usdt_readarg(5, ctx, &msg.msg_size); - bpf_usdt_readarg_p(6, ctx, &msg.msg, MIN(msg.msg_size, MAX_MSG_DATA_LENGTH)); + bpf_usdt_readarg_p(6, ctx, &msg.msg, _TRACEPOINT_TEST_MIN(msg.msg_size, MAX_MSG_DATA_LENGTH)); outbound_messages.perf_submit(ctx, &msg, sizeof(msg)); return 0; }; diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py index ad98a3a162..1617c580f3 100755 --- a/test/functional/interface_usdt_utxocache.py +++ b/test/functional/interface_usdt_utxocache.py @@ -393,7 +393,7 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0, cflags=["-Wno-error=implicit-function-declaration"]) bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush) - self.log.info(f"prune blockchain to trigger a flush for pruning") + self.log.info("prune blockchain to trigger a flush for pruning") expected_flushes.append({"mode": "NONE", "for_prune": True, "size": 0}) self.nodes[0].pruneblockchain(315) @@ -401,7 +401,7 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): bpf.cleanup() self.log.info( - f"check that we don't expect additional flushes and that the handle_* function succeeded") + "check that we don't expect additional flushes and that the handle_* function succeeded") assert_equal(0, len(expected_flushes)) assert_equal(EXPECTED_HANDLE_FLUSH_SUCCESS, handle_flush_succeeds) diff --git a/test/functional/interface_usdt_validation.py b/test/functional/interface_usdt_validation.py index 9a37b96ada..8a98a452de 100755 --- a/test/functional/interface_usdt_validation.py +++ b/test/functional/interface_usdt_validation.py @@ -8,6 +8,7 @@ """ import ctypes +import time # Test will be skipped if we don't have bcc installed try: @@ -105,10 +106,12 @@ class ValidationTracepointTest(BitcoinTestFramework): handle_blockconnected) self.log.info(f"mine {BLOCKS_EXPECTED} blocks") - block_hashes = self.generatetoaddress( - self.nodes[0], BLOCKS_EXPECTED, ADDRESS_BCRT1_UNSPENDABLE) - for block_hash in block_hashes: - expected_blocks[block_hash] = self.nodes[0].getblock(block_hash, 2) + generatetoaddress_duration = dict() + for _ in range(BLOCKS_EXPECTED): + start = time.time() + hash = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] + generatetoaddress_duration[hash] = (time.time() - start) * 1e9 # in nanoseconds + expected_blocks[hash] = self.nodes[0].getblock(hash, 2) bpf.perf_buffer_poll(timeout=200) @@ -123,6 +126,10 @@ class ValidationTracepointTest(BitcoinTestFramework): assert_equal(0, event.sigops) # no sigops in coinbase tx # only plausibility checks assert event.duration > 0 + # generatetoaddress (mining and connecting) takes longer than + # connecting the block. In case the duration unit is off, we'll + # detect it with this assert. + assert event.duration < generatetoaddress_duration[block_hash] del expected_blocks[block_hash] assert_equal(BLOCKS_EXPECTED, len(events)) assert_equal(0, len(expected_blocks)) diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index b960f40ccc..96e701a7fd 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -6,7 +6,6 @@ import os import struct import tempfile -from time import sleep from io import BytesIO from test_framework.address import ( @@ -27,6 +26,7 @@ from test_framework.messages import ( from test_framework.util import ( assert_equal, assert_raises_rpc_error, + ensure_for, p2p_port, ) from test_framework.wallet import ( @@ -394,11 +394,10 @@ class ZMQTest (BitcoinTestFramework): block_count = self.nodes[0].getblockcount() best_hash = self.nodes[0].getbestblockhash() self.nodes[0].invalidateblock(best_hash) - sleep(2) # Bit of room to make sure transaction things happened # Make sure getrawmempool mempool_sequence results aren't "queued" but immediately reflective # of the time they were gathered. - assert self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"] > seq_num + ensure_for(duration=2, f=lambda: self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"] > seq_num) assert_equal((best_hash, "D", None), seq.receive_sequence()) assert_equal((rbf_txid, "A", seq_num), seq.receive_sequence()) @@ -489,13 +488,8 @@ class ZMQTest (BitcoinTestFramework): mempool_snapshot = self.nodes[0].getrawmempool(mempool_sequence=True) mempool_view = set(mempool_snapshot["txids"]) get_raw_seq = mempool_snapshot["mempool_sequence"] - assert_equal(get_raw_seq, 6) - # Snapshot may be too old compared to zmq message we read off latest - while zmq_mem_seq >= get_raw_seq: - sleep(2) - mempool_snapshot = self.nodes[0].getrawmempool(mempool_sequence=True) - mempool_view = set(mempool_snapshot["txids"]) - get_raw_seq = mempool_snapshot["mempool_sequence"] + assert_equal(get_raw_seq, num_txs + 1) + assert zmq_mem_seq < get_raw_seq # Things continue to happen in the "interim" while waiting for snapshot results # We have node 0 do all these to avoid p2p races with RBF announcements diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 4d08575255..27ecc3b4a8 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -55,7 +55,6 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [[ '-txindex','-permitbaremultisig=0', - '-mempoolfullrbf=0', ]] * self.num_nodes self.supports_cli = False @@ -68,6 +67,8 @@ class MempoolAcceptanceTest(BitcoinTestFramework): if "fees" in r: r["fees"].pop("effective-feerate") r["fees"].pop("effective-includes") + if "reject-details" in r: + r.pop("reject-details") assert_equal(result_expected, result_test) assert_equal(self.nodes[0].getmempoolinfo()['size'], self.mempool_size) # Must not change mempool state @@ -155,25 +156,13 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A transaction that replaces a mempool transaction') tx = tx_from_hex(raw_tx_0) tx.vout[0].nValue -= int(fee * COIN) # Double the fee - tx.vin[0].nSequence = MAX_BIP125_RBF_SEQUENCE + 1 # Now, opt out of RBF raw_tx_0 = tx.serialize().hex() txid_0 = tx.rehash() self.check_mempool_result( result_expected=[{'txid': txid_0, 'allowed': True, 'vsize': tx.get_vsize(), 'fees': {'base': (2 * fee)}}], rawtxs=[raw_tx_0], ) - - self.log.info('A transaction that conflicts with an unconfirmed tx') - # Send the transaction that replaces the mempool transaction and opts out of replaceability node.sendrawtransaction(hexstring=tx.serialize().hex(), maxfeerate=0) - # take original raw_tx_0 - tx = tx_from_hex(raw_tx_0) - tx.vout[0].nValue -= int(4 * fee * COIN) # Set more fee - self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'txn-mempool-conflict'}], - rawtxs=[tx.serialize().hex()], - maxfeerate=0, - ) self.log.info('A transaction with missing inputs, that never existed') tx = tx_from_hex(raw_tx_0) diff --git a/test/functional/mempool_accept_wtxid.py b/test/functional/mempool_accept_wtxid.py index e00e7b1ae4..f74d00e37c 100755 --- a/test/functional/mempool_accept_wtxid.py +++ b/test/functional/mempool_accept_wtxid.py @@ -100,13 +100,15 @@ class MempoolWtxidTest(BitcoinTestFramework): "txid": child_one_txid, "wtxid": child_one_wtxid, "allowed": False, - "reject-reason": "txn-already-in-mempool" + "reject-reason": "txn-already-in-mempool", + "reject-details": "txn-already-in-mempool" }]) assert_equal(node.testmempoolaccept([child_two.serialize().hex()])[0], { "txid": child_two_txid, "wtxid": child_two_wtxid, "allowed": False, - "reject-reason": "txn-same-nonwitness-data-in-mempool" + "reject-reason": "txn-same-nonwitness-data-in-mempool", + "reject-details": "txn-same-nonwitness-data-in-mempool" }) # sendrawtransaction will not throw but quits early when the exact same transaction is already in mempool diff --git a/test/functional/mempool_datacarrier.py b/test/functional/mempool_datacarrier.py index ed6ad8461a..48e636caa2 100755 --- a/test/functional/mempool_datacarrier.py +++ b/test/functional/mempool_datacarrier.py @@ -26,7 +26,7 @@ class DataCarrierTest(BitcoinTestFramework): [], ["-datacarrier=0"], ["-datacarrier=1", f"-datacarriersize={MAX_OP_RETURN_RELAY - 1}"], - ["-datacarrier=1", f"-datacarriersize=2"], + ["-datacarrier=1", "-datacarriersize=2"], ] def test_null_data_transaction(self, node: TestNode, data, success: bool) -> None: diff --git a/test/functional/mempool_dust.py b/test/functional/mempool_dust.py index 1ea03a8a9e..937e77fbd4 100755 --- a/test/functional/mempool_dust.py +++ b/test/functional/mempool_dust.py @@ -71,9 +71,39 @@ class DustRelayFeeTest(BitcoinTestFramework): # finally send the transaction to avoid running out of MiniWallet UTXOs self.wallet.sendrawtransaction(from_node=node, tx_hex=tx_good_hex) + def test_dustrelay(self): + self.log.info("Test that small outputs are acceptable when dust relay rate is set to 0 that would otherwise trigger ephemeral dust rules") + + self.restart_node(0, extra_args=["-dustrelayfee=0"]) + + assert_equal(self.nodes[0].getrawmempool(), []) + + # Create two dust outputs. Transaction has zero fees. both dust outputs are unspent, and would have failed individual checks. + # The amount is 1 satoshi because create_self_transfer_multi disallows 0. + dusty_tx = self.wallet.create_self_transfer_multi(fee_per_output=1000, amount_per_output=1, num_outputs=2) + dust_txid = self.nodes[0].sendrawtransaction(hexstring=dusty_tx["hex"], maxfeerate=0) + + assert_equal(self.nodes[0].getrawmempool(), [dust_txid]) + + # Spends one dust along with fee input, leave other dust unspent to check ephemeral dust checks aren't being enforced + sweep_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=[self.wallet.get_utxo(), dusty_tx["new_utxos"][0]]) + sweep_txid = self.nodes[0].sendrawtransaction(sweep_tx["hex"]) + + mempool_entries = self.nodes[0].getrawmempool() + assert dust_txid in mempool_entries + assert sweep_txid in mempool_entries + assert_equal(len(mempool_entries), 2) + + # Wipe extra arg to reset dust relay + self.restart_node(0, extra_args=[]) + + assert_equal(self.nodes[0].getrawmempool(), []) + def run_test(self): self.wallet = MiniWallet(self.nodes[0]) + self.test_dustrelay() + # prepare output scripts of each standard type _, uncompressed_pubkey = generate_keypair(compressed=False) _, pubkey = generate_keypair(compressed=True) diff --git a/test/functional/mempool_ephemeral_dust.py b/test/functional/mempool_ephemeral_dust.py new file mode 100755 index 0000000000..e614a9e607 --- /dev/null +++ b/test/functional/mempool_ephemeral_dust.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from decimal import Decimal + +from test_framework.messages import ( + COIN, + CTxOut, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.mempool_util import assert_mempool_contents +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, +) +from test_framework.wallet import ( + MiniWallet, +) + +class EphemeralDustTest(BitcoinTestFramework): + def set_test_params(self): + # Mempools should match via 1P1C p2p relay + self.num_nodes = 2 + + # Don't test trickling logic + self.noban_tx_relay = True + + def add_output_to_create_multi_result(self, result, output_value=0): + """ Add output without changing absolute tx fee + """ + assert len(result["tx"].vout) > 0 + assert result["tx"].vout[0].nValue >= output_value + result["tx"].vout.append(CTxOut(output_value, result["tx"].vout[0].scriptPubKey)) + # Take value from first output + result["tx"].vout[0].nValue -= output_value + result["new_utxos"][0]["value"] = Decimal(result["tx"].vout[0].nValue) / COIN + new_txid = result["tx"].rehash() + result["txid"] = new_txid + result["wtxid"] = result["tx"].getwtxid() + result["hex"] = result["tx"].serialize().hex() + for new_utxo in result["new_utxos"]: + new_utxo["txid"] = new_txid + new_utxo["wtxid"] = result["tx"].getwtxid() + + result["new_utxos"].append({"txid": new_txid, "vout": len(result["tx"].vout) - 1, "value": Decimal(output_value) / COIN, "height": 0, "coinbase": False, "confirmations": 0}) + + def create_ephemeral_dust_package(self, *, tx_version, dust_tx_fee=0, dust_value=0, num_dust_outputs=1, extra_sponsors=None): + """Creates a 1P1C package containing ephemeral dust. By default, the parent transaction + is zero-fee and creates a single zero-value dust output, and all of its outputs are + spent by the child.""" + dusty_tx = self.wallet.create_self_transfer_multi(fee_per_output=dust_tx_fee, version=tx_version) + for _ in range(num_dust_outputs): + self.add_output_to_create_multi_result(dusty_tx, dust_value) + + extra_sponsors = extra_sponsors or [] + sweep_tx = self.wallet.create_self_transfer_multi( + utxos_to_spend=dusty_tx["new_utxos"] + extra_sponsors, + version=tx_version, + ) + + return dusty_tx, sweep_tx + + def run_test(self): + + node = self.nodes[0] + self.wallet = MiniWallet(node) + + self.test_normal_dust() + self.test_sponsor_cycle() + self.test_node_restart() + self.test_fee_having_parent() + self.test_multidust() + self.test_nonzero_dust() + self.test_non_truc() + self.test_unspent_ephemeral() + self.test_reorgs() + self.test_no_minrelay_fee() + + def test_normal_dust(self): + self.log.info("Create 0-value dusty output, show that it works inside truc when spent in package") + + assert_equal(self.nodes[0].getrawmempool(), []) + dusty_tx, sweep_tx = self.create_ephemeral_dust_package(tx_version=3) + + # Test doesn't work because lack of package feerates + test_res = self.nodes[0].testmempoolaccept([dusty_tx["hex"], sweep_tx["hex"]]) + assert not test_res[0]["allowed"] + assert_equal(test_res[0]["reject-reason"], "min relay fee not met") + + # And doesn't work on its own + assert_raises_rpc_error(-26, "min relay fee not met", self.nodes[0].sendrawtransaction, dusty_tx["hex"]) + + # If we add modified fees, it is still not allowed due to dust check + self.nodes[0].prioritisetransaction(txid=dusty_tx["txid"], dummy=0, fee_delta=COIN) + test_res = self.nodes[0].testmempoolaccept([dusty_tx["hex"]]) + assert not test_res[0]["allowed"] + assert_equal(test_res[0]["reject-reason"], "dust") + # Reset priority + self.nodes[0].prioritisetransaction(txid=dusty_tx["txid"], dummy=0, fee_delta=-COIN) + assert_equal(self.nodes[0].getprioritisedtransactions(), {}) + + # Package evaluation succeeds + res = self.nodes[0].submitpackage([dusty_tx["hex"], sweep_tx["hex"]]) + assert_equal(res["package_msg"], "success") + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"], sweep_tx["tx"]]) + + # Entry is denied when non-0-fee, either base or unmodified. + # If in-mempool, we're not allowed to prioritise due to detected dust output + assert_raises_rpc_error(-8, "Priority is not supported for transactions with dust outputs.", self.nodes[0].prioritisetransaction, dusty_tx["txid"], 0, 1) + assert_equal(self.nodes[0].getprioritisedtransactions(), {}) + + self.generate(self.nodes[0], 1) + assert_equal(self.nodes[0].getrawmempool(), []) + + def test_node_restart(self): + self.log.info("Test that an ephemeral package is rejected on restart due to individual evaluation") + + assert_equal(self.nodes[0].getrawmempool(), []) + dusty_tx, sweep_tx = self.create_ephemeral_dust_package(tx_version=3) + + res = self.nodes[0].submitpackage([dusty_tx["hex"], sweep_tx["hex"]]) + assert_equal(res["package_msg"], "success") + assert_equal(len(self.nodes[0].getrawmempool()), 2) + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"], sweep_tx["tx"]]) + + # Node restart; doesn't allow ephemeral transaction back in due to individual submission + # resulting in 0-fee. Supporting re-submission of CPFP packages on restart is desired but not + # yet implemented. + self.restart_node(0) + self.restart_node(1) + self.connect_nodes(0, 1) + assert_mempool_contents(self, self.nodes[0], expected=[]) + + def test_fee_having_parent(self): + self.log.info("Test that a transaction with ephemeral dust may not have non-0 base fee") + + assert_equal(self.nodes[0].getrawmempool(), []) + + sats_fee = 1 + dusty_tx, sweep_tx = self.create_ephemeral_dust_package(tx_version=3, dust_tx_fee=sats_fee) + assert_equal(int(COIN * dusty_tx["fee"]), sats_fee) # has fees + assert_greater_than(dusty_tx["tx"].vout[0].nValue, 330) # main output is not dust + assert_equal(dusty_tx["tx"].vout[1].nValue, 0) # added one is dust + + # When base fee is non-0, we report dust like usual + res = self.nodes[0].submitpackage([dusty_tx["hex"], sweep_tx["hex"]]) + assert_equal(res["package_msg"], "transaction failed") + assert_equal(res["tx-results"][dusty_tx["wtxid"]]["error"], "dust, tx with dust output must be 0-fee") + + # Priority is ignored: rejected even if modified fee is 0 + self.nodes[0].prioritisetransaction(txid=dusty_tx["txid"], dummy=0, fee_delta=-sats_fee) + self.nodes[1].prioritisetransaction(txid=dusty_tx["txid"], dummy=0, fee_delta=-sats_fee) + res = self.nodes[0].submitpackage([dusty_tx["hex"], sweep_tx["hex"]]) + assert_equal(res["package_msg"], "transaction failed") + assert_equal(res["tx-results"][dusty_tx["wtxid"]]["error"], "dust, tx with dust output must be 0-fee") + + # Will not be accepted if base fee is 0 with modified fee of non-0 + dusty_tx, sweep_tx = self.create_ephemeral_dust_package(tx_version=3) + + self.nodes[0].prioritisetransaction(txid=dusty_tx["txid"], dummy=0, fee_delta=1000) + self.nodes[1].prioritisetransaction(txid=dusty_tx["txid"], dummy=0, fee_delta=1000) + + # It's rejected submitted alone + test_res = self.nodes[0].testmempoolaccept([dusty_tx["hex"]]) + assert not test_res[0]["allowed"] + assert_equal(test_res[0]["reject-reason"], "dust") + + # Or as a package + res = self.nodes[0].submitpackage([dusty_tx["hex"], sweep_tx["hex"]]) + assert_equal(res["package_msg"], "transaction failed") + assert_equal(res["tx-results"][dusty_tx["wtxid"]]["error"], "dust, tx with dust output must be 0-fee") + + assert_mempool_contents(self, self.nodes[0], expected=[]) + + def test_multidust(self): + self.log.info("Test that a transaction with multiple ephemeral dusts is not allowed") + + assert_mempool_contents(self, self.nodes[0], expected=[]) + dusty_tx, sweep_tx = self.create_ephemeral_dust_package(tx_version=3, num_dust_outputs=2) + + res = self.nodes[0].submitpackage([dusty_tx["hex"], sweep_tx["hex"]]) + assert_equal(res["package_msg"], "transaction failed") + assert_equal(res["tx-results"][dusty_tx["wtxid"]]["error"], "dust") + assert_equal(self.nodes[0].getrawmempool(), []) + + def test_nonzero_dust(self): + self.log.info("Test that a single output of any satoshi amount is allowed, not checking spending") + + # We aren't checking spending, allow it in with no fee + self.restart_node(0, extra_args=["-minrelaytxfee=0"]) + self.restart_node(1, extra_args=["-minrelaytxfee=0"]) + self.connect_nodes(0, 1) + + # 330 is dust threshold for taproot outputs + for value in [1, 329, 330]: + assert_equal(self.nodes[0].getrawmempool(), []) + dusty_tx, _ = self.create_ephemeral_dust_package(tx_version=3, dust_value=value) + test_res = self.nodes[0].testmempoolaccept([dusty_tx["hex"]]) + assert test_res[0]["allowed"] + + self.restart_node(0, extra_args=[]) + self.restart_node(1, extra_args=[]) + self.connect_nodes(0, 1) + assert_mempool_contents(self, self.nodes[0], expected=[]) + + # N.B. If individual minrelay requirement is dropped, this test can be dropped + def test_non_truc(self): + self.log.info("Test that v2 dust-having transaction is rejected even if spent, because of min relay requirement") + + assert_equal(self.nodes[0].getrawmempool(), []) + dusty_tx, sweep_tx = self.create_ephemeral_dust_package(tx_version=2) + + res = self.nodes[0].submitpackage([dusty_tx["hex"], sweep_tx["hex"]]) + assert_equal(res["package_msg"], "transaction failed") + assert_equal(res["tx-results"][dusty_tx["wtxid"]]["error"], "min relay fee not met, 0 < 147") + + assert_equal(self.nodes[0].getrawmempool(), []) + + def test_unspent_ephemeral(self): + self.log.info("Test that spending from a tx with ephemeral outputs is only allowed if dust is spent as well") + + assert_equal(self.nodes[0].getrawmempool(), []) + dusty_tx, sweep_tx = self.create_ephemeral_dust_package(tx_version=3, dust_value=329) + + # Valid sweep we will RBF incorrectly by not spending dust as well + self.nodes[0].submitpackage([dusty_tx["hex"], sweep_tx["hex"]]) + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"], sweep_tx["tx"]]) + + # Doesn't spend in-mempool dust output from parent + unspent_sweep_tx = self.wallet.create_self_transfer_multi(fee_per_output=2000, utxos_to_spend=[dusty_tx["new_utxos"][0]], version=3) + assert_greater_than(unspent_sweep_tx["fee"], sweep_tx["fee"]) + res = self.nodes[0].submitpackage([dusty_tx["hex"], unspent_sweep_tx["hex"]]) + assert_equal(res["tx-results"][unspent_sweep_tx["wtxid"]]["error"], f"missing-ephemeral-spends, tx {unspent_sweep_tx['txid']} did not spend parent's ephemeral dust") + assert_raises_rpc_error(-26, f"missing-ephemeral-spends, tx {unspent_sweep_tx['txid']} did not spend parent's ephemeral dust", self.nodes[0].sendrawtransaction, unspent_sweep_tx["hex"]) + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"], sweep_tx["tx"]]) + + # Spend works with dust spent + sweep_tx_2 = self.wallet.create_self_transfer_multi(fee_per_output=2000, utxos_to_spend=dusty_tx["new_utxos"], version=3) + assert sweep_tx["hex"] != sweep_tx_2["hex"] + res = self.nodes[0].submitpackage([dusty_tx["hex"], sweep_tx_2["hex"]]) + assert_equal(res["package_msg"], "success") + + # Re-set and test again with nothing from package in mempool this time + self.generate(self.nodes[0], 1) + assert_equal(self.nodes[0].getrawmempool(), []) + + dusty_tx, _ = self.create_ephemeral_dust_package(tx_version=3, dust_value=329) + + # Spend non-dust only + unspent_sweep_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=[dusty_tx["new_utxos"][0]], version=3) + + res = self.nodes[0].submitpackage([dusty_tx["hex"], unspent_sweep_tx["hex"]]) + assert_equal(res["package_msg"], "unspent-dust") + + assert_equal(self.nodes[0].getrawmempool(), []) + + # Now spend dust only which should work + second_coin = self.wallet.get_utxo() # another fee-bringing coin + sweep_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=[dusty_tx["new_utxos"][1], second_coin], version=3) + + res = self.nodes[0].submitpackage([dusty_tx["hex"], sweep_tx["hex"]]) + assert_equal(res["package_msg"], "success") + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"], sweep_tx["tx"]]) + + self.generate(self.nodes[0], 1) + assert_mempool_contents(self, self.nodes[0], expected=[]) + + def test_sponsor_cycle(self): + self.log.info("Test that dust txn is not evicted when it becomes childless, but won't be mined") + + assert_equal(self.nodes[0].getrawmempool(), []) + sponsor_coin = self.wallet.get_utxo() + # Bring "fee" input that can be double-spend separately + dusty_tx, sweep_tx = self.create_ephemeral_dust_package(tx_version=3, extra_sponsors=[sponsor_coin]) + + res = self.nodes[0].submitpackage([dusty_tx["hex"], sweep_tx["hex"]]) + assert_equal(res["package_msg"], "success") + assert_equal(len(self.nodes[0].getrawmempool()), 2) + # sync to make sure unsponsor_tx hits second node's mempool after initial package + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"], sweep_tx["tx"]]) + + # Now we RBF away the child using the sponsor input only + unsponsor_tx = self.wallet.create_self_transfer_multi( + utxos_to_spend=[sponsor_coin], + num_outputs=1, + fee_per_output=2000, + version=3 + ) + self.nodes[0].sendrawtransaction(unsponsor_tx["hex"]) + + # Parent is now childless and fee-free, so will not be mined + entry_info = self.nodes[0].getmempoolentry(dusty_tx["txid"]) + assert_equal(entry_info["descendantcount"], 1) + assert_equal(entry_info["fees"]["descendant"], Decimal(0)) + + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"], unsponsor_tx["tx"]]) + + # Dust tx is not mined + self.generate(self.nodes[0], 1) + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"]]) + + # Create sweep that doesn't spend conflicting sponsor coin + sweep_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=dusty_tx["new_utxos"], version=3) + + # Can resweep + self.nodes[0].sendrawtransaction(sweep_tx["hex"]) + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"], sweep_tx["tx"]]) + + self.generate(self.nodes[0], 1) + assert_mempool_contents(self, self.nodes[0], expected=[]) + + def test_reorgs(self): + self.log.info("Test that reorgs breaking the truc topology doesn't cause issues") + + assert_equal(self.nodes[0].getrawmempool(), []) + + # Many shallow re-orgs confuse block gossiping making test less reliable otherwise + self.disconnect_nodes(0, 1) + + # Get dusty tx mined, then check that it makes it back into mempool on reorg + # due to bypass_limits allowing 0-fee individually + dusty_tx, _ = self.create_ephemeral_dust_package(tx_version=3) + assert_raises_rpc_error(-26, "min relay fee not met", self.nodes[0].sendrawtransaction, dusty_tx["hex"]) + + block_res = self.generateblock(self.nodes[0], self.wallet.get_address(), [dusty_tx["hex"]], sync_fun=self.no_op) + self.nodes[0].invalidateblock(block_res["hash"]) + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"]], sync=False) + + # Create a sweep that has dust of its own and leaves dusty_tx's dust unspent + sweep_tx = self.wallet.create_self_transfer_multi(fee_per_output=0, utxos_to_spend=[dusty_tx["new_utxos"][0]], version=3) + self.add_output_to_create_multi_result(sweep_tx) + assert_raises_rpc_error(-26, "min relay fee not met", self.nodes[0].sendrawtransaction, sweep_tx["hex"]) + + # Mine the sweep then re-org, the sweep will not make it back in due to spend checks + block_res = self.generateblock(self.nodes[0], self.wallet.get_address(), [dusty_tx["hex"], sweep_tx["hex"]], sync_fun=self.no_op) + self.nodes[0].invalidateblock(block_res["hash"]) + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"]], sync=False) + + # Should re-enter if dust is swept + sweep_tx_2 = self.wallet.create_self_transfer_multi(fee_per_output=0, utxos_to_spend=dusty_tx["new_utxos"], version=3) + self.add_output_to_create_multi_result(sweep_tx_2) + assert_raises_rpc_error(-26, "min relay fee not met", self.nodes[0].sendrawtransaction, sweep_tx_2["hex"]) + + reconsider_block_res = self.generateblock(self.nodes[0], self.wallet.get_address(), [dusty_tx["hex"], sweep_tx_2["hex"]], sync_fun=self.no_op) + self.nodes[0].invalidateblock(reconsider_block_res["hash"]) + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"], sweep_tx_2["tx"]], sync=False) + + # TRUC transactions restriction for ephemeral dust disallows further spends of ancestor chains + child_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=sweep_tx_2["new_utxos"], version=3) + assert_raises_rpc_error(-26, "TRUC-violation", self.nodes[0].sendrawtransaction, child_tx["hex"]) + + self.nodes[0].reconsiderblock(reconsider_block_res["hash"]) + assert_equal(self.nodes[0].getrawmempool(), []) + + self.log.info("Test that ephemeral dust tx with fees or multi dust don't enter mempool via reorg") + multi_dusty_tx, _ = self.create_ephemeral_dust_package(tx_version=3, num_dust_outputs=2) + block_res = self.generateblock(self.nodes[0], self.wallet.get_address(), [multi_dusty_tx["hex"]], sync_fun=self.no_op) + self.nodes[0].invalidateblock(block_res["hash"]) + assert_equal(self.nodes[0].getrawmempool(), []) + + # With fee and one dust + dusty_fee_tx, _ = self.create_ephemeral_dust_package(tx_version=3, dust_tx_fee=1) + block_res = self.generateblock(self.nodes[0], self.wallet.get_address(), [dusty_fee_tx["hex"]], sync_fun=self.no_op) + self.nodes[0].invalidateblock(block_res["hash"]) + assert_equal(self.nodes[0].getrawmempool(), []) + + # Re-connect and make sure we have same state still + self.connect_nodes(0, 1) + self.sync_all() + + # N.B. this extra_args can be removed post cluster mempool + def test_no_minrelay_fee(self): + self.log.info("Test that ephemeral dust works in non-TRUC contexts when there's no minrelay requirement") + + # Note: since minrelay is 0, it is not testing 1P1C relay + self.restart_node(0, extra_args=["-minrelaytxfee=0"]) + self.restart_node(1, extra_args=["-minrelaytxfee=0"]) + self.connect_nodes(0, 1) + + assert_equal(self.nodes[0].getrawmempool(), []) + dusty_tx, sweep_tx = self.create_ephemeral_dust_package(tx_version=2) + + self.nodes[0].submitpackage([dusty_tx["hex"], sweep_tx["hex"]]) + + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"], sweep_tx["tx"]]) + + # generate coins for next tests + self.generate(self.nodes[0], 1) + self.wallet.rescan_utxos() + assert_equal(self.nodes[0].getrawmempool(), []) + + self.log.info("Test batched ephemeral dust sweep") + dusty_txs = [] + for _ in range(24): + dusty_txs.append(self.wallet.create_self_transfer_multi(fee_per_output=0, version=2)) + self.add_output_to_create_multi_result(dusty_txs[-1]) + + all_parent_utxos = [utxo for tx in dusty_txs for utxo in tx["new_utxos"]] + + # Missing one dust spend from a single parent, child rejected + insufficient_sweep_tx = self.wallet.create_self_transfer_multi(fee_per_output=25000, utxos_to_spend=all_parent_utxos[:-1], version=2) + + res = self.nodes[0].submitpackage([dusty_tx["hex"] for dusty_tx in dusty_txs] + [insufficient_sweep_tx["hex"]]) + assert_equal(res['package_msg'], "transaction failed") + assert_equal(res['tx-results'][insufficient_sweep_tx['wtxid']]['error'], f"missing-ephemeral-spends, tx {insufficient_sweep_tx['txid']} did not spend parent's ephemeral dust") + # Everything got in except for insufficient spend + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"] for dusty_tx in dusty_txs]) + + # Next put some parents in mempool, but not others, and test unspent dust again with all parents spent + B_coin = self.wallet.get_utxo() # coin to cycle out CPFP + sweep_all_but_one_tx = self.wallet.create_self_transfer_multi(fee_per_output=20000, utxos_to_spend=all_parent_utxos[:-2] + [B_coin], version=2) + res = self.nodes[0].submitpackage([dusty_tx["hex"] for dusty_tx in dusty_txs[:-1]] + [sweep_all_but_one_tx["hex"]]) + assert_equal(res['package_msg'], "success") + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"] for dusty_tx in dusty_txs] + [sweep_all_but_one_tx["tx"]]) + + res = self.nodes[0].submitpackage([dusty_tx["hex"] for dusty_tx in dusty_txs] + [insufficient_sweep_tx["hex"]]) + assert_equal(res['package_msg'], "transaction failed") + assert_equal(res['tx-results'][insufficient_sweep_tx["wtxid"]]["error"], f"missing-ephemeral-spends, tx {insufficient_sweep_tx['txid']} did not spend parent's ephemeral dust") + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"] for dusty_tx in dusty_txs] + [sweep_all_but_one_tx["tx"]]) + + # Cycle out the partial sweep to avoid triggering package RBF behavior which limits package to no in-mempool ancestors + cancel_sweep = self.wallet.create_self_transfer_multi(fee_per_output=21000, utxos_to_spend=[B_coin], version=2) + self.nodes[0].sendrawtransaction(cancel_sweep["hex"]) + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"] for dusty_tx in dusty_txs] + [cancel_sweep["tx"]]) + + # Sweeps all dust, where all dusty txs are already in-mempool + sweep_tx = self.wallet.create_self_transfer_multi(fee_per_output=25000, utxos_to_spend=all_parent_utxos, version=2) + + # N.B. Since we have multiple parents these are not propagating via 1P1C relay. + # minrelay being zero allows them to propagate on their own. + res = self.nodes[0].submitpackage([dusty_tx["hex"] for dusty_tx in dusty_txs] + [sweep_tx["hex"]]) + assert_equal(res['package_msg'], "success") + assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"] for dusty_tx in dusty_txs] + [sweep_tx["tx"], cancel_sweep["tx"]]) + + self.generate(self.nodes[0], 1) + self.wallet.rescan_utxos() + assert_equal(self.nodes[0].getrawmempool(), []) + + # Other topology tests (e.g., grandparents and parents both with dust) require relaxation of submitpackage topology + + self.restart_node(0, extra_args=[]) + self.restart_node(1, extra_args=[]) + self.connect_nodes(0, 1) + + assert_equal(self.nodes[0].getrawmempool(), []) + +if __name__ == "__main__": + EphemeralDustTest(__file__).main() diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index 626928a49a..85d158d611 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -55,12 +55,12 @@ class MempoolLimitTest(BitcoinTestFramework): self.generate(node, 1) # tx_A needs to be RBF'd, set minfee at set size - A_weight = 1000 + A_vsize = 250 mempoolmin_feerate = node.getmempoolinfo()["mempoolminfee"] tx_A = self.wallet.send_self_transfer( from_node=node, fee_rate=mempoolmin_feerate, - target_weight=A_weight, + target_vsize=A_vsize, utxo_to_spend=rbf_utxo, confirmed_only=True ) @@ -68,15 +68,15 @@ class MempoolLimitTest(BitcoinTestFramework): # RBF's tx_A, is not yet submitted tx_B = self.wallet.create_self_transfer( fee=tx_A["fee"] * 4, - target_weight=A_weight, + target_vsize=A_vsize, utxo_to_spend=rbf_utxo, confirmed_only=True ) # Spends tx_B's output, too big for cpfp carveout (because that would also increase the descendant limit by 1) - non_cpfp_carveout_weight = 40001 # EXTRA_DESCENDANT_TX_SIZE_LIMIT + 1 + non_cpfp_carveout_vsize = 10001 # EXTRA_DESCENDANT_TX_SIZE_LIMIT + 1 tx_C = self.wallet.create_self_transfer( - target_weight=non_cpfp_carveout_weight, + target_vsize=non_cpfp_carveout_vsize, fee_rate=mempoolmin_feerate, utxo_to_spend=tx_B["new_utxo"], confirmed_only=True @@ -85,6 +85,100 @@ class MempoolLimitTest(BitcoinTestFramework): assert_equal(res["package_msg"], "transaction failed") assert "too-long-mempool-chain" in res["tx-results"][tx_C["wtxid"]]["error"] + def test_mid_package_eviction_success(self): + node = self.nodes[0] + self.log.info("Check a package where each parent passes the current mempoolminfee but a parent could be evicted before getting child's descendant feerate") + + # Clear mempool so it can be filled with minrelay txns + self.restart_node(0, extra_args=self.extra_args[0] + ["-persistmempool=0"]) + assert_equal(node.getrawmempool(), []) + + # Restarting the node resets mempool minimum feerate + assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) + assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) + + fill_mempool(self, node) + current_info = node.getmempoolinfo() + mempoolmin_feerate = current_info["mempoolminfee"] + + mempool_txids = node.getrawmempool() + mempool_entries = [node.getmempoolentry(entry) for entry in mempool_txids] + fees_btc_per_kvb = [entry["fees"]["base"] / (Decimal(entry["vsize"]) / 1000) for entry in mempool_entries] + mempool_entry_minrate = min(fees_btc_per_kvb) + mempool_entry_minrate = mempool_entry_minrate.quantize(Decimal("0.00000000")) + + # There is a gap, our parents will be minrate, with child bringing up descendant fee sufficiently to avoid + # eviction even though parents cause eviction on their own + assert_greater_than(mempool_entry_minrate, mempoolmin_feerate) + + package_hex = [] + # UTXOs to be spent by the ultimate child transaction + parent_utxos = [] + + # Series of parents that don't need CPFP and are submitted individually. Each one is large + # which means in aggregate they could trigger eviction, but child submission should result + # in them not being evicted + parent_vsize = 25000 + num_big_parents = 3 + # Need to be large enough to trigger eviction + # (note that the mempool usage of a tx is about three times its vsize) + assert_greater_than(parent_vsize * num_big_parents * 3, current_info["maxmempool"] - current_info["bytes"]) + + big_parent_txids = [] + big_parent_wtxids = [] + for i in range(num_big_parents): + # Last parent is higher feerate causing other parents to possibly + # be evicted if trimming was allowed, which would cause the package to end up failing + parent_feerate = mempoolmin_feerate + Decimal("0.00000001") if i == num_big_parents - 1 else mempoolmin_feerate + parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_vsize=parent_vsize, confirmed_only=True) + parent_utxos.append(parent["new_utxo"]) + package_hex.append(parent["hex"]) + big_parent_txids.append(parent["txid"]) + big_parent_wtxids.append(parent["wtxid"]) + # There is room for each of these transactions independently + assert node.testmempoolaccept([parent["hex"]])[0]["allowed"] + + # Create a child spending everything with an insane fee, bumping the package above mempool_entry_minrate + child = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_utxos, fee_per_output=10000000) + package_hex.append(child["hex"]) + + # Package should be submitted, temporarily exceeding maxmempool, but not evicted. + package_res = None + with node.assert_debug_log(expected_msgs=["rolling minimum fee bumped"]): + package_res = node.submitpackage(package=package_hex, maxfeerate=0) + + assert_equal(package_res["package_msg"], "success") + + # Ensure that intra-package trimming is not happening. + # Each transaction separately satisfies the current + # minfee and shouldn't need package evaluation to + # be included. If trimming of a parent were to happen, + # package evaluation would happen to reintrodce the evicted + # parent. + assert_equal(len(package_res["tx-results"]), len(big_parent_wtxids) + 1) + for wtxid in big_parent_wtxids + [child["wtxid"]]: + assert_equal(len(package_res["tx-results"][wtxid]["fees"]["effective-includes"]), 1) + + # Maximum size must never be exceeded. + assert_greater_than(node.getmempoolinfo()["maxmempool"], node.getmempoolinfo()["bytes"]) + + # Package found in mempool still + resulting_mempool_txids = node.getrawmempool() + assert child["txid"] in resulting_mempool_txids + for txid in big_parent_txids: + assert txid in resulting_mempool_txids + + # Check every evicted tx was higher feerate than parents which evicted it + eviction_set = set(mempool_txids) - set(resulting_mempool_txids) - set(big_parent_txids) + parent_entries = [node.getmempoolentry(entry) for entry in big_parent_txids] + max_parent_feerate = max([entry["fees"]["modified"] / (Decimal(entry["vsize"]) / 1000) for entry in parent_entries]) + for eviction in eviction_set: + assert eviction in mempool_txids + for txid, entry in zip(mempool_txids, mempool_entries): + if txid == eviction: + evicted_feerate_btc_per_kvb = entry["fees"]["modified"] / (Decimal(entry["vsize"]) / 1000) + assert_greater_than(evicted_feerate_btc_per_kvb, max_parent_feerate) + def test_mid_package_eviction(self): node = self.nodes[0] self.log.info("Check a package where each parent passes the current mempoolminfee but would cause eviction before package submission terminates") @@ -103,14 +197,14 @@ class MempoolLimitTest(BitcoinTestFramework): # UTXOs to be spent by the ultimate child transaction parent_utxos = [] - evicted_weight = 8000 + evicted_vsize = 2000 # Mempool transaction which is evicted due to being at the "bottom" of the mempool when the # mempool overflows and evicts by descendant score. It's important that the eviction doesn't # happen in the middle of package evaluation, as it can invalidate the coins cache. mempool_evicted_tx = self.wallet.send_self_transfer( from_node=node, fee_rate=mempoolmin_feerate, - target_weight=evicted_weight, + target_vsize=evicted_vsize, confirmed_only=True ) # Already in mempool when package is submitted. @@ -132,14 +226,16 @@ class MempoolLimitTest(BitcoinTestFramework): # Series of parents that don't need CPFP and are submitted individually. Each one is large and # high feerate, which means they should trigger eviction but not be evicted. - parent_weight = 100000 + parent_vsize = 25000 num_big_parents = 3 - assert_greater_than(parent_weight * num_big_parents, current_info["maxmempool"] - current_info["bytes"]) + # Need to be large enough to trigger eviction + # (note that the mempool usage of a tx is about three times its vsize) + assert_greater_than(parent_vsize * num_big_parents * 3, current_info["maxmempool"] - current_info["bytes"]) parent_feerate = 100 * mempoolmin_feerate big_parent_txids = [] for i in range(num_big_parents): - parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_weight=parent_weight, confirmed_only=True) + parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_vsize=parent_vsize, confirmed_only=True) parent_utxos.append(parent["new_utxo"]) package_hex.append(parent["hex"]) big_parent_txids.append(parent["txid"]) @@ -311,8 +407,9 @@ class MempoolLimitTest(BitcoinTestFramework): entry = node.getmempoolentry(txid) worst_feerate_btcvb = min(worst_feerate_btcvb, entry["fees"]["descendant"] / entry["descendantsize"]) # Needs to be large enough to trigger eviction - target_weight_each = 200000 - assert_greater_than(target_weight_each * 2, node.getmempoolinfo()["maxmempool"] - node.getmempoolinfo()["bytes"]) + # (note that the mempool usage of a tx is about three times its vsize) + target_vsize_each = 50000 + assert_greater_than(target_vsize_each * 2 * 3, node.getmempoolinfo()["maxmempool"] - node.getmempoolinfo()["bytes"]) # Should be a true CPFP: parent's feerate is just below mempool min feerate parent_feerate = mempoolmin_feerate - Decimal("0.000001") # 0.1 sats/vbyte below min feerate # Parent + child is above mempool minimum feerate @@ -320,8 +417,8 @@ class MempoolLimitTest(BitcoinTestFramework): # However, when eviction is triggered, these transactions should be at the bottom. # This assertion assumes parent and child are the same size. miniwallet.rescan_utxos() - tx_parent_just_below = miniwallet.create_self_transfer(fee_rate=parent_feerate, target_weight=target_weight_each) - tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee_rate=child_feerate, target_weight=target_weight_each) + tx_parent_just_below = miniwallet.create_self_transfer(fee_rate=parent_feerate, target_vsize=target_vsize_each) + tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee_rate=child_feerate, target_vsize=target_vsize_each) # This package ranks below the lowest descendant package in the mempool package_fee = tx_parent_just_below["fee"] + tx_child_just_above["fee"] package_vsize = tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize() @@ -336,6 +433,7 @@ class MempoolLimitTest(BitcoinTestFramework): self.stop_node(0) self.nodes[0].assert_start_raises_init_error(["-maxmempool=4"], "Error: -maxmempool must be at least 5 MB") + self.test_mid_package_eviction_success() self.test_mid_package_replacement() self.test_mid_package_eviction() self.test_rbf_carveout_disallowed() diff --git a/test/functional/mempool_package_limits.py b/test/functional/mempool_package_limits.py index 6e26a684e2..3290ff43c4 100755 --- a/test/functional/mempool_package_limits.py +++ b/test/functional/mempool_package_limits.py @@ -4,9 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for limiting mempool and package ancestors/descendants.""" from test_framework.blocktools import COINBASE_MATURITY -from test_framework.messages import ( - WITNESS_SCALE_FACTOR, -) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -290,19 +287,18 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): parent_utxos = [] target_vsize = 30_000 high_fee = 10 * target_vsize # 10 sats/vB - target_weight = target_vsize * WITNESS_SCALE_FACTOR self.log.info("Check that in-mempool and in-package ancestor size limits are calculated properly in packages") # Mempool transactions A and B for _ in range(2): - bulked_tx = self.wallet.create_self_transfer(target_weight=target_weight) + bulked_tx = self.wallet.create_self_transfer(target_vsize=target_vsize) self.wallet.sendrawtransaction(from_node=node, tx_hex=bulked_tx["hex"]) parent_utxos.append(bulked_tx["new_utxo"]) # Package transaction C - pc_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_utxos, fee_per_output=high_fee, target_weight=target_weight) + pc_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_utxos, fee_per_output=high_fee, target_vsize=target_vsize) # Package transaction D - pd_tx = self.wallet.create_self_transfer(utxo_to_spend=pc_tx["new_utxos"][0], target_weight=target_weight) + pd_tx = self.wallet.create_self_transfer(utxo_to_spend=pc_tx["new_utxos"][0], target_vsize=target_vsize) assert_equal(2, node.getmempoolinfo()["size"]) return [pc_tx["hex"], pd_tx["hex"]] @@ -321,20 +317,19 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): node = self.nodes[0] target_vsize = 21_000 high_fee = 10 * target_vsize # 10 sats/vB - target_weight = target_vsize * WITNESS_SCALE_FACTOR self.log.info("Check that in-mempool and in-package descendant sizes are calculated properly in packages") # Top parent in mempool, Ma - ma_tx = self.wallet.create_self_transfer_multi(num_outputs=2, fee_per_output=high_fee // 2, target_weight=target_weight) + ma_tx = self.wallet.create_self_transfer_multi(num_outputs=2, fee_per_output=high_fee // 2, target_vsize=target_vsize) self.wallet.sendrawtransaction(from_node=node, tx_hex=ma_tx["hex"]) package_hex = [] for j in range(2): # Two legs (left and right) # Mempool transaction (Mb and Mc) - mempool_tx = self.wallet.create_self_transfer(utxo_to_spend=ma_tx["new_utxos"][j], target_weight=target_weight) + mempool_tx = self.wallet.create_self_transfer(utxo_to_spend=ma_tx["new_utxos"][j], target_vsize=target_vsize) self.wallet.sendrawtransaction(from_node=node, tx_hex=mempool_tx["hex"]) # Package transaction (Pd and Pe) - package_tx = self.wallet.create_self_transfer(utxo_to_spend=mempool_tx["new_utxo"], target_weight=target_weight) + package_tx = self.wallet.create_self_transfer(utxo_to_spend=mempool_tx["new_utxo"], target_vsize=target_vsize) package_hex.append(package_tx["hex"]) assert_equal(3, node.getmempoolinfo()["size"]) diff --git a/test/functional/mempool_package_rbf.py b/test/functional/mempool_package_rbf.py index f4d57262f2..4dc6f8fe36 100755 --- a/test/functional/mempool_package_rbf.py +++ b/test/functional/mempool_package_rbf.py @@ -19,6 +19,7 @@ from test_framework.wallet import ( DEFAULT_FEE, MiniWallet, ) +from test_framework import mempool_util MAX_REPLACEMENT_CANDIDATES = 100 @@ -37,15 +38,7 @@ class PackageRBFTest(BitcoinTestFramework): ]] * self.num_nodes def assert_mempool_contents(self, expected=None): - """Assert that all transactions in expected are in the mempool, - and no additional ones exist. - """ - if not expected: - expected = [] - mempool = self.nodes[0].getrawmempool(verbose=False) - assert_equal(len(mempool), len(expected)) - for tx in expected: - assert tx.rehash() in mempool + mempool_util.assert_mempool_contents(self, self.nodes[0], expected, sync=False) def create_simple_package(self, parent_coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE, heavy_child=False): """Create a 1 parent 1 child package using the coin passed in as the parent's input. The @@ -226,7 +219,7 @@ class PackageRBFTest(BitcoinTestFramework): package_child = self.wallet.create_self_transfer(fee_rate=child_feerate, utxo_to_spend=package_parent["new_utxos"][0]) pkg_results = node.submitpackage([package_parent["hex"], package_child["hex"]], maxfeerate=0) - assert_equal(f"package RBF failed: too many potential replacements, rejecting replacement {package_child['tx'].rehash()}; too many potential replacements (102 > 100)\n", pkg_results["package_msg"]) + assert_equal(f"package RBF failed: too many potential replacements, rejecting replacement {package_child['tx'].rehash()}; too many potential replacements (102 > 100)", pkg_results["package_msg"]) self.assert_mempool_contents(expected=expected_txns) # Make singleton tx to conflict with in next batch @@ -241,7 +234,7 @@ class PackageRBFTest(BitcoinTestFramework): package_parent = self.wallet.create_self_transfer_multi(utxos_to_spend=double_spending_coins, fee_per_output=parent_fee_per_conflict) package_child = self.wallet.create_self_transfer(fee_rate=child_feerate, utxo_to_spend=package_parent["new_utxos"][0]) pkg_results = node.submitpackage([package_parent["hex"], package_child["hex"]], maxfeerate=0) - assert_equal(f"package RBF failed: too many potential replacements, rejecting replacement {package_child['tx'].rehash()}; too many potential replacements (101 > 100)\n", pkg_results["package_msg"]) + assert_equal(f"package RBF failed: too many potential replacements, rejecting replacement {package_child['tx'].rehash()}; too many potential replacements (101 > 100)", pkg_results["package_msg"]) self.assert_mempool_contents(expected=expected_txns) # Finally, evict MAX_REPLACEMENT_CANDIDATES @@ -391,11 +384,11 @@ class PackageRBFTest(BitcoinTestFramework): # Now make conflicting packages for each coin package_hex1, _package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex1) - assert_equal(f"package RBF failed: {parent1_result['tx'].rehash()} is not the only parent of child {child_result['tx'].rehash()}", package_result["package_msg"]) + assert_equal(f"package RBF failed: {child_result['tx'].rehash()} has 2 ancestors, max 1 allowed", package_result["package_msg"]) package_hex2, _package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex2) - assert_equal(f"package RBF failed: {parent2_result['tx'].rehash()} is not the only parent of child {child_result['tx'].rehash()}", package_result["package_msg"]) + assert_equal(f"package RBF failed: {child_result['tx'].rehash()} has 2 ancestors, max 1 allowed", package_result["package_msg"]) package_hex3, _package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex3) @@ -445,7 +438,7 @@ class PackageRBFTest(BitcoinTestFramework): # Now make conflicting packages for each coin package_hex1, _package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex1) - assert_equal(f"package RBF failed: {parent_result['tx'].rehash()} has 2 descendants, max 1 allowed", package_result["package_msg"]) + assert_equal(f"package RBF failed: {child2_result['tx'].rehash()} is not the only child of parent {parent_result['tx'].rehash()}", package_result["package_msg"]) package_hex2, _package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex2) @@ -554,7 +547,7 @@ class PackageRBFTest(BitcoinTestFramework): self.generate(node, 1) def test_child_conflicts_parent_mempool_ancestor(self): - fill_mempool(self, self.nodes[0]) + fill_mempool(self, self.nodes[0], tx_sync_fun=self.no_op) # Reset coins since we filled the mempool with current coins self.coins = self.wallet.get_utxos(mark_as_spent=False, confirmed_only=True) diff --git a/test/functional/mempool_truc.py b/test/functional/mempool_truc.py index 28f3256ef1..435b61f24f 100755 --- a/test/functional/mempool_truc.py +++ b/test/functional/mempool_truc.py @@ -4,10 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. from decimal import Decimal -from test_framework.messages import ( - MAX_BIP125_RBF_SEQUENCE, - WITNESS_SCALE_FACTOR, -) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -23,6 +19,7 @@ from test_framework.wallet import ( MAX_REPLACEMENT_CANDIDATES = 100 TRUC_MAX_VSIZE = 10000 +TRUC_CHILD_MAX_VSIZE = 1000 def cleanup(extra_args=None): def decorator(func): @@ -55,14 +52,14 @@ class MempoolTRUC(BitcoinTestFramework): def test_truc_max_vsize(self): node = self.nodes[0] self.log.info("Test TRUC-specific maximum transaction vsize") - tx_v3_heavy = self.wallet.create_self_transfer(target_weight=(TRUC_MAX_VSIZE + 1) * WITNESS_SCALE_FACTOR, version=3) + tx_v3_heavy = self.wallet.create_self_transfer(target_vsize=TRUC_MAX_VSIZE + 1, version=3) assert_greater_than_or_equal(tx_v3_heavy["tx"].get_vsize(), TRUC_MAX_VSIZE) expected_error_heavy = f"TRUC-violation, version=3 tx {tx_v3_heavy['txid']} (wtxid={tx_v3_heavy['wtxid']}) is too big" assert_raises_rpc_error(-26, expected_error_heavy, node.sendrawtransaction, tx_v3_heavy["hex"]) self.check_mempool([]) # Ensure we are hitting the TRUC-specific limit and not something else - tx_v2_heavy = self.wallet.send_self_transfer(from_node=node, target_weight=(TRUC_MAX_VSIZE + 1) * WITNESS_SCALE_FACTOR, version=2) + tx_v2_heavy = self.wallet.send_self_transfer(from_node=node, target_vsize=TRUC_MAX_VSIZE + 1, version=2) self.check_mempool([tx_v2_heavy["txid"]]) @cleanup(extra_args=["-datacarriersize=1000"]) @@ -73,10 +70,10 @@ class MempoolTRUC(BitcoinTestFramework): self.check_mempool([tx_v3_parent_normal["txid"]]) tx_v3_child_heavy = self.wallet.create_self_transfer( utxo_to_spend=tx_v3_parent_normal["new_utxo"], - target_weight=4004, + target_vsize=TRUC_CHILD_MAX_VSIZE + 1, version=3 ) - assert_greater_than_or_equal(tx_v3_child_heavy["tx"].get_vsize(), 1000) + assert_greater_than_or_equal(tx_v3_child_heavy["tx"].get_vsize(), TRUC_CHILD_MAX_VSIZE) expected_error_child_heavy = f"TRUC-violation, version=3 child tx {tx_v3_child_heavy['txid']} (wtxid={tx_v3_child_heavy['wtxid']}) is too big" assert_raises_rpc_error(-26, expected_error_child_heavy, node.sendrawtransaction, tx_v3_child_heavy["hex"]) self.check_mempool([tx_v3_parent_normal["txid"]]) @@ -88,20 +85,21 @@ class MempoolTRUC(BitcoinTestFramework): from_node=node, fee_rate=DEFAULT_FEE, utxo_to_spend=tx_v3_parent_normal["new_utxo"], - target_weight=3987, + target_vsize=TRUC_CHILD_MAX_VSIZE - 3, version=3 ) - assert_greater_than_or_equal(1000, tx_v3_child_almost_heavy["tx"].get_vsize()) + assert_greater_than_or_equal(TRUC_CHILD_MAX_VSIZE, tx_v3_child_almost_heavy["tx"].get_vsize()) self.check_mempool([tx_v3_parent_normal["txid"], tx_v3_child_almost_heavy["txid"]]) assert_equal(node.getmempoolentry(tx_v3_parent_normal["txid"])["descendantcount"], 2) tx_v3_child_almost_heavy_rbf = self.wallet.send_self_transfer( from_node=node, fee_rate=DEFAULT_FEE * 2, utxo_to_spend=tx_v3_parent_normal["new_utxo"], - target_weight=3500, + target_vsize=875, version=3 ) - assert_greater_than_or_equal(tx_v3_child_almost_heavy["tx"].get_vsize() + tx_v3_child_almost_heavy_rbf["tx"].get_vsize(), 1000) + assert_greater_than_or_equal(tx_v3_child_almost_heavy["tx"].get_vsize() + tx_v3_child_almost_heavy_rbf["tx"].get_vsize(), + TRUC_CHILD_MAX_VSIZE) self.check_mempool([tx_v3_parent_normal["txid"], tx_v3_child_almost_heavy_rbf["txid"]]) assert_equal(node.getmempoolentry(tx_v3_parent_normal["txid"])["descendantcount"], 2) @@ -114,7 +112,6 @@ class MempoolTRUC(BitcoinTestFramework): from_node=node, fee_rate=DEFAULT_FEE, utxo_to_spend=utxo_v3_bip125, - sequence=MAX_BIP125_RBF_SEQUENCE, version=3 ) self.check_mempool([tx_v3_bip125["txid"]]) @@ -162,30 +159,6 @@ class MempoolTRUC(BitcoinTestFramework): self.check_mempool([tx_v3_bip125_rbf_v2["txid"], tx_v3_parent["txid"], tx_v3_child["txid"]]) - @cleanup(extra_args=["-mempoolfullrbf=0"]) - def test_truc_bip125(self): - node = self.nodes[0] - self.log.info("Test TRUC transactions that don't signal BIP125 are replaceable") - assert_equal(node.getmempoolinfo()["fullrbf"], False) - utxo_v3_no_bip125 = self.wallet.get_utxo() - tx_v3_no_bip125 = self.wallet.send_self_transfer( - from_node=node, - fee_rate=DEFAULT_FEE, - utxo_to_spend=utxo_v3_no_bip125, - sequence=MAX_BIP125_RBF_SEQUENCE + 1, - version=3 - ) - - self.check_mempool([tx_v3_no_bip125["txid"]]) - assert not node.getmempoolentry(tx_v3_no_bip125["txid"])["bip125-replaceable"] - tx_v3_no_bip125_rbf = self.wallet.send_self_transfer( - from_node=node, - fee_rate=DEFAULT_FEE * 2, - utxo_to_spend=utxo_v3_no_bip125, - version=3 - ) - self.check_mempool([tx_v3_no_bip125_rbf["txid"]]) - @cleanup(extra_args=["-datacarriersize=40000"]) def test_truc_reorg(self): node = self.nodes[0] @@ -199,8 +172,8 @@ class MempoolTRUC(BitcoinTestFramework): self.check_mempool([]) tx_v2_from_v3 = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block["new_utxo"], version=2) tx_v3_from_v2 = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v2_block["new_utxo"], version=3) - tx_v3_child_large = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block2["new_utxo"], target_weight=5000, version=3) - assert_greater_than(node.getmempoolentry(tx_v3_child_large["txid"])["vsize"], 1000) + tx_v3_child_large = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block2["new_utxo"], target_vsize=1250, version=3) + assert_greater_than(node.getmempoolentry(tx_v3_child_large["txid"])["vsize"], TRUC_CHILD_MAX_VSIZE) self.check_mempool([tx_v2_from_v3["txid"], tx_v3_from_v2["txid"], tx_v3_child_large["txid"]]) node.invalidateblock(block[0]) self.check_mempool([tx_v3_block["txid"], tx_v2_block["txid"], tx_v3_block2["txid"], tx_v2_from_v3["txid"], tx_v3_from_v2["txid"], tx_v3_child_large["txid"]]) @@ -217,22 +190,22 @@ class MempoolTRUC(BitcoinTestFramework): """ node = self.nodes[0] self.log.info("Test that a decreased limitdescendantsize also applies to TRUC child") - parent_target_weight = 9990 * WITNESS_SCALE_FACTOR - child_target_weight = 500 * WITNESS_SCALE_FACTOR + parent_target_vsize = 9990 + child_target_vsize = 500 tx_v3_parent_large1 = self.wallet.send_self_transfer( from_node=node, - target_weight=parent_target_weight, + target_vsize=parent_target_vsize, version=3 ) tx_v3_child_large1 = self.wallet.create_self_transfer( utxo_to_spend=tx_v3_parent_large1["new_utxo"], - target_weight=child_target_weight, + target_vsize=child_target_vsize, version=3 ) # Parent and child are within v3 limits, but parent's 10kvB descendant limit is exceeded assert_greater_than_or_equal(TRUC_MAX_VSIZE, tx_v3_parent_large1["tx"].get_vsize()) - assert_greater_than_or_equal(1000, tx_v3_child_large1["tx"].get_vsize()) + assert_greater_than_or_equal(TRUC_CHILD_MAX_VSIZE, tx_v3_child_large1["tx"].get_vsize()) assert_greater_than(tx_v3_parent_large1["tx"].get_vsize() + tx_v3_child_large1["tx"].get_vsize(), 10000) assert_raises_rpc_error(-26, f"too-long-mempool-chain, exceeds descendant size limit for tx {tx_v3_parent_large1['txid']}", node.sendrawtransaction, tx_v3_child_large1["hex"]) @@ -244,21 +217,21 @@ class MempoolTRUC(BitcoinTestFramework): self.restart_node(0, extra_args=["-limitancestorsize=10", "-datacarriersize=40000"]) tx_v3_parent_large2 = self.wallet.send_self_transfer( from_node=node, - target_weight=parent_target_weight, + target_vsize=parent_target_vsize, version=3 ) tx_v3_child_large2 = self.wallet.create_self_transfer( utxo_to_spend=tx_v3_parent_large2["new_utxo"], - target_weight=child_target_weight, + target_vsize=child_target_vsize, version=3 ) # Parent and child are within TRUC limits assert_greater_than_or_equal(TRUC_MAX_VSIZE, tx_v3_parent_large2["tx"].get_vsize()) - assert_greater_than_or_equal(1000, tx_v3_child_large2["tx"].get_vsize()) + assert_greater_than_or_equal(TRUC_CHILD_MAX_VSIZE, tx_v3_child_large2["tx"].get_vsize()) assert_greater_than(tx_v3_parent_large2["tx"].get_vsize() + tx_v3_child_large2["tx"].get_vsize(), 10000) - assert_raises_rpc_error(-26, f"too-long-mempool-chain, exceeds ancestor size limit", node.sendrawtransaction, tx_v3_child_large2["hex"]) + assert_raises_rpc_error(-26, "too-long-mempool-chain, exceeds ancestor size limit", node.sendrawtransaction, tx_v3_child_large2["hex"]) self.check_mempool([tx_v3_parent_large2["txid"]]) @cleanup(extra_args=["-datacarriersize=1000"]) @@ -267,12 +240,12 @@ class MempoolTRUC(BitcoinTestFramework): node = self.nodes[0] tx_v3_parent_normal = self.wallet.create_self_transfer( fee_rate=0, - target_weight=4004, + target_vsize=1001, version=3 ) tx_v3_parent_2_normal = self.wallet.create_self_transfer( fee_rate=0, - target_weight=4004, + target_vsize=1001, version=3 ) tx_v3_child_multiparent = self.wallet.create_self_transfer_multi( @@ -282,7 +255,7 @@ class MempoolTRUC(BitcoinTestFramework): ) tx_v3_child_heavy = self.wallet.create_self_transfer_multi( utxos_to_spend=[tx_v3_parent_normal["new_utxo"]], - target_weight=4004, + target_vsize=TRUC_CHILD_MAX_VSIZE + 1, fee_per_output=10000, version=3 ) @@ -294,7 +267,7 @@ class MempoolTRUC(BitcoinTestFramework): self.check_mempool([]) result = node.submitpackage([tx_v3_parent_normal["hex"], tx_v3_child_heavy["hex"]]) - # tx_v3_child_heavy is heavy based on weight, not sigops. + # tx_v3_child_heavy is heavy based on vsize, not sigops. assert_equal(result['package_msg'], f"TRUC-violation, version=3 child tx {tx_v3_child_heavy['txid']} (wtxid={tx_v3_child_heavy['wtxid']}) is too big: {tx_v3_child_heavy['tx'].get_vsize()} > 1000 virtual bytes") self.check_mempool([]) @@ -416,7 +389,7 @@ class MempoolTRUC(BitcoinTestFramework): node = self.nodes[0] tx_v3_parent = self.wallet.create_self_transfer( fee_rate=0, - target_weight=4004, + target_vsize=1001, version=3 ) tx_v2_child = self.wallet.create_self_transfer_multi( @@ -630,7 +603,6 @@ class MempoolTRUC(BitcoinTestFramework): self.test_truc_max_vsize() self.test_truc_acceptance() self.test_truc_replacement() - self.test_truc_bip125() self.test_truc_reorg() self.test_nondefault_package_limits() self.test_truc_ancestors_package() diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index 7c96b4b570..b9692d2e21 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -5,11 +5,12 @@ """Test that the mempool ensures transaction delivery by periodically sending to peers until a GETDATA is received.""" -import time - from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal +from test_framework.util import ( + assert_equal, + ensure_for, +) from test_framework.wallet import MiniWallet MAX_INITIAL_BROADCAST_DELAY = 15 * 60 # 15 minutes in seconds @@ -83,8 +84,8 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): conn = node.add_p2p_connection(P2PTxInvStore()) node.mockscheduler(MAX_INITIAL_BROADCAST_DELAY) - time.sleep(2) # allow sufficient time for possibility of broadcast - assert_equal(len(conn.get_invs()), 0) + # allow sufficient time for possibility of broadcast + ensure_for(duration=2, f=lambda: len(conn.get_invs()) == 0) self.disconnect_nodes(0, 1) node.disconnect_p2ps() diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index aca71933ec..d367ec122d 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -56,7 +56,12 @@ def assert_template(node, block, expect, rehash=True): class MiningTest(BitcoinTestFramework): def set_test_params(self): - self.num_nodes = 2 + self.num_nodes = 3 + self.extra_args = [ + [], + [], + ["-fastprune", "-prune=1"] + ] self.setup_clean_chain = True self.supports_cli = False @@ -122,6 +127,7 @@ class MiningTest(BitcoinTestFramework): def test_timewarp(self): self.log.info("Test timewarp attack mitigation (BIP94)") node = self.nodes[0] + self.restart_node(0, extra_args=['-test=bip94']) self.log.info("Mine until the last block of the retarget period") blockchain_info = self.nodes[0].getblockchaininfo() @@ -168,6 +174,21 @@ class MiningTest(BitcoinTestFramework): bad_block.solve() node.submitheader(hexdata=CBlockHeader(bad_block).serialize().hex()) + def test_pruning(self): + self.log.info("Test that submitblock stores previously pruned block") + prune_node = self.nodes[2] + self.generate(prune_node, 400, sync_fun=self.no_op) + pruned_block = prune_node.getblock(prune_node.getblockhash(2), verbosity=0) + pruned_height = prune_node.pruneblockchain(400) + assert_greater_than_or_equal(pruned_height, 2) + pruned_blockhash = prune_node.getblockhash(2) + + assert_raises_rpc_error(-1, 'Block not available (pruned data)', prune_node.getblock, pruned_blockhash) + + result = prune_node.submitblock(pruned_block) + assert_equal(result, "inconclusive") + assert_equal(prune_node.getblock(pruned_blockhash, verbosity=0), pruned_block) + def run_test(self): node = self.nodes[0] self.wallet = MiniWallet(node) @@ -239,9 +260,19 @@ class MiningTest(BitcoinTestFramework): bad_block.vtx[0].rehash() assert_template(node, bad_block, 'bad-cb-missing') - self.log.info("submitblock: Test invalid coinbase transaction") - assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, CBlock().serialize().hex()) - assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, bad_block.serialize().hex()) + self.log.info("submitblock: Test bad input hash for coinbase transaction") + bad_block.solve() + assert_equal("bad-cb-missing", node.submitblock(hexdata=bad_block.serialize().hex())) + + self.log.info("submitblock: Test block with no transactions") + no_tx_block = copy.deepcopy(block) + no_tx_block.vtx.clear() + no_tx_block.hashMerkleRoot = 0 + no_tx_block.solve() + assert_equal("bad-blk-length", node.submitblock(hexdata=no_tx_block.serialize().hex())) + + self.log.info("submitblock: Test empty block") + assert_equal('high-hash', node.submitblock(hexdata=CBlock().serialize().hex())) self.log.info("getblocktemplate: Test truncated final transaction") assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, { @@ -376,6 +407,7 @@ class MiningTest(BitcoinTestFramework): self.test_blockmintxfee_parameter() self.test_timewarp() + self.test_pruning() if __name__ == '__main__': diff --git a/test/functional/p2p_1p1c_network.py b/test/functional/p2p_1p1c_network.py index f9e782f524..cdc4e1691d 100755 --- a/test/functional/p2p_1p1c_network.py +++ b/test/functional/p2p_1p1c_network.py @@ -49,9 +49,6 @@ class PackageRelayTest(BitcoinTestFramework): def raise_network_minfee(self): fill_mempool(self, self.nodes[0]) - self.log.debug("Wait for the network to sync mempools") - self.sync_mempools() - self.log.debug("Check that all nodes' mempool minimum feerates are above min relay feerate") for node in self.nodes: assert_equal(node.getmempoolinfo()['minrelaytxfee'], FEERATE_1SAT_VB) diff --git a/test/functional/p2p_addrv2_relay.py b/test/functional/p2p_addrv2_relay.py index d5ded926d3..8012137971 100755 --- a/test/functional/p2p_addrv2_relay.py +++ b/test/functional/p2p_addrv2_relay.py @@ -78,7 +78,7 @@ class AddrTest(BitcoinTestFramework): def run_test(self): self.log.info('Check disconnection when sending sendaddrv2 after verack') conn = self.nodes[0].add_p2p_connection(P2PInterface()) - with self.nodes[0].assert_debug_log(['sendaddrv2 received after verack from peer=0; disconnecting']): + with self.nodes[0].assert_debug_log(['sendaddrv2 received after verack, disconnecting peer=0']): conn.send_message(msg_sendaddrv2()) conn.wait_for_disconnect() diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 49cf26d425..ca36b2fbc0 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -733,12 +733,12 @@ class CompactBlocksTest(BitcoinTestFramework): # Now send the compact block with all transactions prefilled, and # verify that we don't get disconnected. comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=True) + comp_block.initialize_from_block(block, prefill_list=list(range(len(block.vtx))), use_witness=True) msg = msg_cmpctblock(comp_block.to_p2p()) test_node.send_and_ping(msg) # Check that the tip didn't advance - assert int(node.getbestblockhash(), 16) is not block.sha256 + assert int(node.getbestblockhash(), 16) != block.sha256 test_node.sync_with_ping() # Helper for enabling cb announcements diff --git a/test/functional/p2p_handshake.py b/test/functional/p2p_handshake.py index 18307a2824..4148790c19 100755 --- a/test/functional/p2p_handshake.py +++ b/test/functional/p2p_handshake.py @@ -86,7 +86,7 @@ class P2PHandshakeTest(BitcoinTestFramework): DESIRABLE_SERVICE_FLAGS_PRUNED, expect_disconnect=False) self.log.info("Check that feeler connections get disconnected immediately") - with node.assert_debug_log([f"feeler connection completed"]): + with node.assert_debug_log(["feeler connection completed"]): self.add_outbound_connection(node, "feeler", NODE_NONE, wait_for_disconnect=True) self.log.info("Check that connecting to ourself leads to immediate disconnect") diff --git a/test/functional/p2p_ibd_stalling.py b/test/functional/p2p_ibd_stalling.py index fa07873929..4348bf7ee9 100755 --- a/test/functional/p2p_ibd_stalling.py +++ b/test/functional/p2p_ibd_stalling.py @@ -74,6 +74,7 @@ class P2PIBDStallingTest(BitcoinTestFramework): self.log.info("Check that a staller does not get disconnected if the 1024 block lookahead buffer is filled") self.mocktime = int(time.time()) + 1 + node.setmocktime(self.mocktime) for id in range(NUM_PEERS): peers.append(node.add_outbound_p2p_connection(P2PStaller(stall_block), p2p_idx=id, connection_type="outbound-full-relay")) peers[-1].block_store = block_dict diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 241aefab24..ee8c6c16ca 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -165,7 +165,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): node.p2ps[0].send_txs_and_test([rejected_parent], node, success=False) self.log.info('Test that a peer disconnection causes erase its transactions from the orphan pool') - with node.assert_debug_log(['Erased 100 orphan transaction(s) from peer=25']): + with node.assert_debug_log(['Erased 100 orphan transaction(s) from peer=26']): self.reconnect_p2p(num_connections=1) self.log.info('Test that a transaction in the orphan pool is included in a new tip block causes erase this transaction from the orphan pool') diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index f800e815d8..7848cdaadb 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -172,7 +172,7 @@ class P2PLeakTest(BitcoinTestFramework): self.log.info('Check that old peers are disconnected') p2p_old_peer = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False) - with self.nodes[0].assert_debug_log(["using obsolete version 31799; disconnecting"]): + with self.nodes[0].assert_debug_log(["using obsolete version 31799, disconnecting peer=5"]): p2p_old_peer.send_message(self.create_old_version(31799)) p2p_old_peer.wait_for_disconnect() diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index df6e6a2e28..7788be6adb 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -102,10 +102,10 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): tip_height = pruned_node.getblockcount() limit_buffer = 2 # Prevent races by waiting for the tip to arrive first - self.wait_until(lambda: not try_rpc(-1, "Block not found", full_node.getblock, pruned_node.getbestblockhash())) + self.wait_until(lambda: not try_rpc(-1, "Block not available (not fully downloaded)", full_node.getblock, pruned_node.getbestblockhash())) for height in range(start_height_full_node + 1, tip_height + 1): if height <= tip_height - (NODE_NETWORK_LIMITED_MIN_BLOCKS - limit_buffer): - assert_raises_rpc_error(-1, "Block not found on disk", full_node.getblock, pruned_node.getblockhash(height)) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", full_node.getblock, pruned_node.getblockhash(height)) else: full_node.getblock(pruned_node.getblockhash(height)) # just assert it does not throw an exception diff --git a/test/functional/p2p_orphan_handling.py b/test/functional/p2p_orphan_handling.py index 22600bf8a4..0864deeb4f 100755 --- a/test/functional/p2p_orphan_handling.py +++ b/test/functional/p2p_orphan_handling.py @@ -5,6 +5,7 @@ import time +from test_framework.mempool_util import tx_in_orphanage from test_framework.messages import ( CInv, CTxInWitness, @@ -41,6 +42,8 @@ from test_framework.wallet import ( # for one peer and y seconds for another, use specific values instead. TXREQUEST_TIME_SKIP = NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY + OVERLOADED_PEER_TX_DELAY + 1 +DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100 + def cleanup(func): # Time to fastfoward (using setmocktime) in between subtests to ensure they do not interfere with # one another, in seconds. Equal to 12 hours, which is enough to expire anything that may exist @@ -207,9 +210,9 @@ class OrphanHandlingTest(BitcoinTestFramework): # Relay the child. It should not be accepted because it has missing inputs. # Its parent should not be requested because its hash (txid == wtxid) has been added to the rejection filter. - with node.assert_debug_log(['not keeping orphan with rejected parents {}'.format(child_nonsegwit["txid"])]): - self.relay_transaction(peer2, child_nonsegwit["tx"]) + self.relay_transaction(peer2, child_nonsegwit["tx"]) assert child_nonsegwit["txid"] not in node.getrawmempool() + assert not tx_in_orphanage(node, child_nonsegwit["tx"]) # No parents are requested. self.nodes[0].bumpmocktime(GETDATA_TX_INTERVAL) @@ -228,6 +231,7 @@ class OrphanHandlingTest(BitcoinTestFramework): # Relay the child. It should not be accepted because it has missing inputs. self.relay_transaction(peer2, child_low_fee["tx"]) assert child_low_fee["txid"] not in node.getrawmempool() + assert tx_in_orphanage(node, child_low_fee["tx"]) # The parent should be requested because even though the txid commits to the fee, it doesn't # commit to the feerate. Delayed because it's by txid and this is not a preferred relay peer. @@ -247,6 +251,7 @@ class OrphanHandlingTest(BitcoinTestFramework): # Relay the child. It should not be accepted because it has missing inputs. self.relay_transaction(peer2, child_invalid_witness["tx"]) assert child_invalid_witness["txid"] not in node.getrawmempool() + assert tx_in_orphanage(node, child_invalid_witness["tx"]) # The parent should be requested since the unstripped wtxid would differ. Delayed because # it's by txid and this is not a preferred relay peer. @@ -295,6 +300,7 @@ class OrphanHandlingTest(BitcoinTestFramework): self.relay_transaction(peer, orphan["tx"]) self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) peer.sync_with_ping() + assert tx_in_orphanage(node, orphan["tx"]) assert_equal(len(peer.last_message["getdata"].inv), 2) peer.wait_for_parent_requests([int(txid_conf_old, 16), int(missing_tx["txid"], 16)]) @@ -333,7 +339,7 @@ class OrphanHandlingTest(BitcoinTestFramework): # The wtxid and txid need to be the same for the node to recognize that the missing input # and in-flight request for inflight_parent_AB are the same transaction. - assert_equal(inflight_parent_AB["txid"], inflight_parent_AB["tx"].getwtxid()) + assert_equal(inflight_parent_AB["txid"], inflight_parent_AB["wtxid"]) # Announce inflight_parent_AB and wait for getdata peer_txrequest.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=int(inflight_parent_AB["tx"].getwtxid(), 16))])) @@ -344,6 +350,7 @@ class OrphanHandlingTest(BitcoinTestFramework): # Relay orphan child_A self.relay_transaction(peer_orphans, child_A["tx"]) self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) + assert tx_in_orphanage(node, child_A["tx"]) # There are 3 missing parents. missing_parent_A and missing_parent_AB should be requested. # But inflight_parent_AB should not, because there is already an in-flight request for it. peer_orphans.wait_for_parent_requests([int(missing_parent_A["txid"], 16), int(missing_parent_AB["txid"], 16)]) @@ -352,11 +359,16 @@ class OrphanHandlingTest(BitcoinTestFramework): # Relay orphan child_B self.relay_transaction(peer_orphans, child_B["tx"]) self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) + assert tx_in_orphanage(node, child_B["tx"]) # Only missing_parent_B should be requested. Not inflight_parent_AB or missing_parent_AB # because they are already being requested from peer_txrequest and peer_orphans respectively. peer_orphans.wait_for_parent_requests([int(missing_parent_B["txid"], 16)]) peer_orphans.assert_never_requested(int(inflight_parent_AB["txid"], 16)) + # But inflight_parent_AB will be requested eventually if original peer doesn't respond + node.bumpmocktime(GETDATA_TX_INTERVAL) + peer_orphans.wait_for_parent_requests([int(inflight_parent_AB["txid"], 16)]) + @cleanup def test_orphan_of_orphan(self): node = self.nodes[0] @@ -371,12 +383,14 @@ class OrphanHandlingTest(BitcoinTestFramework): # The node should put missing_parent_orphan into the orphanage and request missing_grandparent self.relay_transaction(peer, missing_parent_orphan["tx"]) self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) + assert tx_in_orphanage(node, missing_parent_orphan["tx"]) peer.wait_for_parent_requests([int(missing_grandparent["txid"], 16)]) # The node should put the orphan into the orphanage and request missing_parent, skipping # missing_parent_orphan because it already has it in the orphanage. self.relay_transaction(peer, orphan["tx"]) self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) + assert tx_in_orphanage(node, orphan["tx"]) peer.wait_for_parent_requests([int(missing_parent["txid"], 16)]) @cleanup @@ -396,18 +410,19 @@ class OrphanHandlingTest(BitcoinTestFramework): # Relay the parent. It should be rejected because it pays 0 fees. self.relay_transaction(peer1, parent_low_fee_nonsegwit["tx"]) + assert parent_low_fee_nonsegwit["txid"] not in node.getrawmempool() # Relay the child. It should be rejected for having missing parents, and this rejection is # cached by txid and wtxid. - with node.assert_debug_log(['not keeping orphan with rejected parents {}'.format(child["txid"])]): - self.relay_transaction(peer1, child["tx"]) + self.relay_transaction(peer1, child["tx"]) assert_equal(0, len(node.getrawmempool())) + assert not tx_in_orphanage(node, child["tx"]) peer1.assert_never_requested(parent_low_fee_nonsegwit["txid"]) # Grandchild should also not be kept in orphanage because its parent has been rejected. - with node.assert_debug_log(['not keeping orphan with rejected parents {}'.format(grandchild["txid"])]): - self.relay_transaction(peer2, grandchild["tx"]) + self.relay_transaction(peer2, grandchild["tx"]) assert_equal(0, len(node.getrawmempool())) + assert not tx_in_orphanage(node, grandchild["tx"]) peer2.assert_never_requested(child["txid"]) peer2.assert_never_requested(child["tx"].getwtxid()) @@ -435,6 +450,7 @@ class OrphanHandlingTest(BitcoinTestFramework): # 1. Fake orphan is received first. It is missing an input. bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit)) + assert tx_in_orphanage(node, tx_orphan_bad_wit) # 2. Node requests the missing parent by txid. parent_txid_int = int(tx_parent["txid"], 16) @@ -443,8 +459,9 @@ class OrphanHandlingTest(BitcoinTestFramework): # 3. Honest peer relays the real child, which is also missing parents and should be placed # in the orphanage. - with node.assert_debug_log(["missingorspent", "stored orphan tx"]): + with node.assert_debug_log(["missingorspent"]): honest_peer.send_and_ping(msg_tx(tx_child["tx"])) + assert tx_in_orphanage(node, tx_child["tx"]) # Time out the previous request for the parent (node will not request the same transaction # from multiple nodes at the same time) @@ -484,6 +501,7 @@ class OrphanHandlingTest(BitcoinTestFramework): # 1. Fake orphan is received first. It is missing an input. bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit)) + assert tx_in_orphanage(node, tx_orphan_bad_wit) # 2. Node requests missing tx_grandparent by txid. grandparent_txid_int = int(tx_grandparent["txid"], 16) @@ -493,16 +511,16 @@ class OrphanHandlingTest(BitcoinTestFramework): # 3. Honest peer relays the grandchild, which is missing a parent. The parent by txid already # exists in orphanage, but should be re-requested because the node shouldn't assume that the # witness data is the same. In this case, a same-txid-different-witness transaction exists! - with node.assert_debug_log(["stored orphan tx"]): - honest_peer.send_and_ping(msg_tx(tx_grandchild["tx"])) + honest_peer.send_and_ping(msg_tx(tx_grandchild["tx"])) + assert tx_in_orphanage(node, tx_grandchild["tx"]) middle_txid_int = int(tx_middle["txid"], 16) node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) honest_peer.wait_for_getdata([middle_txid_int]) # 4. Honest peer relays the real child, which is also missing parents and should be placed # in the orphanage. - with node.assert_debug_log(["stored orphan tx"]): - honest_peer.send_and_ping(msg_tx(tx_middle["tx"])) + honest_peer.send_and_ping(msg_tx(tx_middle["tx"])) + assert tx_in_orphanage(node, tx_middle["tx"]) assert_equal(len(node.getrawmempool()), 0) # 5. Honest peer sends tx_grandparent @@ -515,6 +533,7 @@ class OrphanHandlingTest(BitcoinTestFramework): assert tx_middle["txid"] in node_mempool assert tx_grandchild["txid"] in node_mempool assert_equal(node.getmempoolentry(tx_middle["txid"])["wtxid"], tx_middle["wtxid"]) + assert_equal(len(node.getorphantxs()), 0) @cleanup def test_orphan_txid_inv(self): @@ -533,6 +552,7 @@ class OrphanHandlingTest(BitcoinTestFramework): # 1. Fake orphan is received first. It is missing an input. bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit)) + assert tx_in_orphanage(node, tx_orphan_bad_wit) # 2. Node requests the missing parent by txid. parent_txid_int = int(tx_parent["txid"], 16) @@ -547,8 +567,8 @@ class OrphanHandlingTest(BitcoinTestFramework): # 4. The child is requested. Honest peer sends it. node.bumpmocktime(TXREQUEST_TIME_SKIP) honest_peer.wait_for_getdata([child_txid_int]) - with node.assert_debug_log(["stored orphan tx"]): - honest_peer.send_and_ping(msg_tx(tx_child["tx"])) + honest_peer.send_and_ping(msg_tx(tx_child["tx"])) + assert tx_in_orphanage(node, tx_child["tx"]) # 5. After first parent request times out, the node sends another one for the missing parent # of the real orphan child. @@ -565,6 +585,48 @@ class OrphanHandlingTest(BitcoinTestFramework): assert tx_parent["txid"] in node_mempool assert tx_child["txid"] in node_mempool assert_equal(node.getmempoolentry(tx_child["txid"])["wtxid"], tx_child["wtxid"]) + assert_equal(len(node.getorphantxs()), 0) + + @cleanup + def test_max_orphan_amount(self): + self.log.info("Check that we never exceed our storage limits for orphans") + + node = self.nodes[0] + self.generate(self.wallet, 1) + peer_1 = node.add_p2p_connection(P2PInterface()) + + self.log.info("Check that orphanage is empty on start of test") + assert len(node.getorphantxs()) == 0 + + self.log.info("Filling up orphanage with " + str(DEFAULT_MAX_ORPHAN_TRANSACTIONS) + "(DEFAULT_MAX_ORPHAN_TRANSACTIONS) orphans") + orphans = [] + parent_orphans = [] + for _ in range(DEFAULT_MAX_ORPHAN_TRANSACTIONS): + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + parent_orphans.append(tx_parent_1["tx"]) + orphans.append(tx_child_1["tx"]) + peer_1.send_message(msg_tx(tx_child_1["tx"])) + + peer_1.sync_with_ping() + orphanage = node.getorphantxs() + assert_equal(len(orphanage), DEFAULT_MAX_ORPHAN_TRANSACTIONS) + + for orphan in orphans: + assert tx_in_orphanage(node, orphan) + + self.log.info("Check that we do not add more than the max orphan amount") + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + peer_1.send_and_ping(msg_tx(tx_child_1["tx"])) + parent_orphans.append(tx_parent_1["tx"]) + orphanage = node.getorphantxs() + assert_equal(len(orphanage), DEFAULT_MAX_ORPHAN_TRANSACTIONS) + + self.log.info("Clearing the orphanage") + for index, parent_orphan in enumerate(parent_orphans): + peer_1.send_and_ping(msg_tx(parent_orphan)) + assert_equal(len(node.getorphantxs()),0) def run_test(self): @@ -582,6 +644,7 @@ class OrphanHandlingTest(BitcoinTestFramework): self.test_same_txid_orphan() self.test_same_txid_orphan_of_orphan() self.test_orphan_txid_inv() + self.test_max_orphan_amount() if __name__ == '__main__': diff --git a/test/functional/p2p_seednode.py b/test/functional/p2p_seednode.py index 6c510a6a0b..5a9fca3278 100755 --- a/test/functional/p2p_seednode.py +++ b/test/functional/p2p_seednode.py @@ -20,17 +20,18 @@ class P2PSeedNodes(BitcoinTestFramework): self.disable_autoconnect = False def test_no_seednode(self): - # Check that if no seednode is provided, the node proceeds as usual (without waiting) + self.log.info("Check that if no seednode is provided, the node proceeds as usual (without waiting)") with self.nodes[0].assert_debug_log(expected_msgs=[], unexpected_msgs=["Empty addrman, adding seednode", f"Couldn't connect to peers from addrman after {ADD_NEXT_SEEDNODE} seconds. Adding seednode"], timeout=ADD_NEXT_SEEDNODE): self.restart_node(0) def test_seednode_empty_addrman(self): seed_node = "0.0.0.1" - # Check that the seednode is added to m_addr_fetches on bootstrap on an empty addrman + self.log.info("Check that the seednode is immediately added on bootstrap on an empty addrman") with self.nodes[0].assert_debug_log(expected_msgs=[f"Empty addrman, adding seednode ({seed_node}) to addrfetch"], timeout=ADD_NEXT_SEEDNODE): self.restart_node(0, extra_args=[f'-seednode={seed_node}']) - def test_seednode_addrman_unreachable_peers(self): + def test_seednode_non_empty_addrman(self): + self.log.info("Check that if addrman is non-empty, seednodes are queried with a delay") seed_node = "0.0.0.2" node = self.nodes[0] # Fill the addrman with unreachable nodes @@ -39,17 +40,18 @@ class P2PSeedNodes(BitcoinTestFramework): port = 8333 + i node.addpeeraddress(ip, port) - # Restart the node so seednode is processed again + # Restart the node so seednode is processed again. Specify a non-working proxy to make sure no actual connections to random IPs are attempted. + with node.assert_debug_log(expected_msgs=["trying v1 connection"], timeout=ADD_NEXT_SEEDNODE): + self.restart_node(0, extra_args=[f'-seednode={seed_node}', '-proxy=127.0.0.1:1']) + with node.assert_debug_log(expected_msgs=[f"Couldn't connect to peers from addrman after {ADD_NEXT_SEEDNODE} seconds. Adding seednode ({seed_node}) to addrfetch"], unexpected_msgs=["Empty addrman, adding seednode"], timeout=ADD_NEXT_SEEDNODE * 1.5): - self.restart_node(0, extra_args=[f'-seednode={seed_node}']) node.setmocktime(int(time.time()) + ADD_NEXT_SEEDNODE + 1) def run_test(self): self.test_no_seednode() self.test_seednode_empty_addrman() - self.test_seednode_addrman_unreachable_peers() + self.test_seednode_non_empty_addrman() if __name__ == '__main__': P2PSeedNodes(__file__).main() - diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 9be53d2ab8..677a1120d6 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -5,7 +5,6 @@ """Test segwit transactions and blocks on P2P network.""" from decimal import Decimal import random -import time from test_framework.blocktools import ( WITNESS_COMMITMENT_HEADER, @@ -83,8 +82,9 @@ from test_framework.script_util import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - softfork_active, assert_raises_rpc_error, + ensure_for, + softfork_active, ) from test_framework.wallet import MiniWallet from test_framework.wallet_util import generate_keypair @@ -184,8 +184,7 @@ class TestP2PConn(P2PInterface): else: self.wait_for_getdata([tx.sha256]) else: - time.sleep(5) - assert not self.last_message.get("getdata") + ensure_for(duration=5, f=lambda: not self.last_message.get("getdata")) def announce_block_and_wait_for_getdata(self, block, use_header, timeout=60): with p2p_lock: @@ -215,6 +214,9 @@ class SegWitTest(BitcoinTestFramework): self.noban_tx_relay = True # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. self.extra_args = [ + # -par=1 should not affect validation outcome or logging/reported failures. It is kept + # here to exercise the code path still (as it is distinct for multithread script + # validation). ["-acceptnonstdtxn=1", f"-testactivationheight=segwit@{SEGWIT_HEIGHT}", "-par=1"], ["-acceptnonstdtxn=0", f"-testactivationheight=segwit@{SEGWIT_HEIGHT}"], ] @@ -508,10 +510,6 @@ class SegWitTest(BitcoinTestFramework): # When the block is serialized without witness, validation fails because the transaction is # invalid (transactions are always validated with SCRIPT_VERIFY_WITNESS so a segwit v0 transaction # without a witness is invalid). - # Note: The reject reason for this failure could be - # 'block-validation-failed' (if script check threads > 1) or - # 'mandatory-script-verify-flag-failed (Witness program was passed an - # empty witness)' (otherwise). test_witness_block(self.nodes[0], self.test_node, block, accepted=False, with_witness=False, reason='mandatory-script-verify-flag-failed (Witness program was passed an empty witness)') @@ -1016,7 +1014,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vout.append(CTxOut(tx.vout[0].nValue, CScript([OP_TRUE]))) tx2.wit.vtxinwit.extend([CTxInWitness(), CTxInWitness()]) tx2.wit.vtxinwit[0].scriptWitness.stack = [CScript([CScriptNum(1)]), CScript([CScriptNum(1)]), witness_script] - tx2.wit.vtxinwit[1].scriptWitness.stack = [CScript([OP_TRUE])] + tx2.wit.vtxinwit[1].scriptWitness.stack = [] block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx2]) diff --git a/test/functional/p2p_sendtxrcncl.py b/test/functional/p2p_sendtxrcncl.py index 2c7216b5ca..3f1fca5c78 100755 --- a/test/functional/p2p_sendtxrcncl.py +++ b/test/functional/p2p_sendtxrcncl.py @@ -168,7 +168,7 @@ class SendTxRcnclTest(BitcoinTestFramework): with self.nodes[0].assert_debug_log(["received: sendtxrcncl"]): peer.send_message(create_sendtxrcncl_msg()) self.log.info('second SENDTXRCNCL triggers a disconnect') - with self.nodes[0].assert_debug_log(["(sendtxrcncl received from already registered peer); disconnecting"]): + with self.nodes[0].assert_debug_log(["(sendtxrcncl received from already registered peer), disconnecting peer=0"]): peer.send_message(create_sendtxrcncl_msg()) peer.wait_for_disconnect() @@ -226,7 +226,7 @@ class SendTxRcnclTest(BitcoinTestFramework): self.log.info('SENDTXRCNCL if block-relay-only triggers a disconnect') peer = self.nodes[0].add_outbound_p2p_connection( PeerNoVerack(), wait_for_verack=False, p2p_idx=0, connection_type="block-relay-only") - with self.nodes[0].assert_debug_log(["we indicated no tx relay; disconnecting"]): + with self.nodes[0].assert_debug_log(["we indicated no tx relay, disconnecting peer=5"]): peer.send_message(create_sendtxrcncl_msg()) peer.wait_for_disconnect() diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index 1fd78e163b..ce01547887 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -84,15 +84,15 @@ class TimeoutsTest(BitcoinTestFramework): if self.options.v2transport: expected_timeout_logs = [ - "version handshake timeout peer=0", - "version handshake timeout peer=1", - "version handshake timeout peer=2", + "version handshake timeout, disconnecting peer=0", + "version handshake timeout, disconnecting peer=1", + "version handshake timeout, disconnecting peer=2", ] else: expected_timeout_logs = [ - "version handshake timeout peer=0", - "socket no message in first 3 seconds, 1 0 peer=1", - "socket no message in first 3 seconds, 0 0 peer=2", + "version handshake timeout, disconnecting peer=0", + "socket no message in first 3 seconds, never sent to peer, disconnecting peer=1", + "socket no message in first 3 seconds, never received from peer, never sent to peer, disconnecting peer=2", ] with self.nodes[0].assert_debug_log(expected_msgs=expected_timeout_logs): diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index efad4e7c0f..c69d6ff405 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -250,7 +250,7 @@ class TxDownloadTest(BitcoinTestFramework): def test_rejects_filter_reset(self): self.log.info('Check that rejected tx is not requested again') node = self.nodes[0] - fill_mempool(self, node) + fill_mempool(self, node, tx_sync_fun=self.no_op) self.wallet.rescan_utxos() mempoolminfee = node.getmempoolinfo()['mempoolminfee'] peer = node.add_p2p_connection(TestP2PConn()) diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index 835ecbf184..1430131a97 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -119,7 +119,7 @@ class AcceptBlockTest(BitcoinTestFramework): assert_equal(x['status'], "headers-only") tip_entry_found = True assert tip_entry_found - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_h1f.hash) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", self.nodes[0].getblock, block_h1f.hash) # 4. Send another two block that build on the fork. block_h2f = create_block(block_h1f.sha256, create_coinbase(2), block_time) @@ -191,7 +191,7 @@ class AcceptBlockTest(BitcoinTestFramework): # Blocks 1-287 should be accepted, block 288 should be ignored because it's too far ahead for x in all_blocks[:-1]: self.nodes[0].getblock(x.hash) - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[-1].hash) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", self.nodes[0].getblock, all_blocks[-1].hash) # 5. Test handling of unrequested block on the node that didn't process # Should still not be processed (even though it has a child that has more @@ -230,7 +230,7 @@ class AcceptBlockTest(BitcoinTestFramework): assert_equal(self.nodes[0].getblockcount(), 290) self.nodes[0].getblock(all_blocks[286].hash) assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[287].hash) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", self.nodes[0].getblock, all_blocks[287].hash) self.log.info("Successfully reorged to longer chain") # 8. Create a chain which is invalid at a height longer than the @@ -260,7 +260,7 @@ class AcceptBlockTest(BitcoinTestFramework): assert_equal(x['status'], "headers-only") tip_entry_found = True assert tip_entry_found - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_292.hash) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", self.nodes[0].getblock, block_292.hash) test_node.send_message(msg_block(block_289f)) test_node.send_and_ping(msg_block(block_290f)) diff --git a/test/functional/p2p_v2_misbehaving.py b/test/functional/p2p_v2_misbehaving.py index 0af96a4f8c..ee589010cd 100755 --- a/test/functional/p2p_v2_misbehaving.py +++ b/test/functional/p2p_v2_misbehaving.py @@ -151,7 +151,7 @@ class EncryptedP2PMisbehaving(BitcoinTestFramework): # Ensure that the bytes sent after 4 bytes network magic are actually received. self.wait_until(lambda: node0.getpeerinfo()[-1]["bytesrecv"] > 4) self.wait_until(lambda: node0.getpeerinfo()[-1]["bytessent"] > 0) - with node0.assert_debug_log(['V2 handshake timeout peer=0']): + with node0.assert_debug_log(['V2 handshake timeout, disconnecting peer=0']): node0.bumpmocktime(4) # `InactivityCheck()` triggers now peer1.wait_for_disconnect(timeout=1) self.log.info('successful disconnection since modified ellswift was sent as response') @@ -162,7 +162,7 @@ class EncryptedP2PMisbehaving(BitcoinTestFramework): expected_debug_message = [ [], # EARLY_KEY_RESPONSE ["V2 transport error: missing garbage terminator, peer=1"], # EXCESS_GARBAGE - ["V2 handshake timeout peer=3"], # WRONG_GARBAGE_TERMINATOR + ["V2 handshake timeout, disconnecting peer=3"], # WRONG_GARBAGE_TERMINATOR ["V2 transport error: packet decryption failure"], # WRONG_GARBAGE ["V2 transport error: packet decryption failure"], # SEND_NO_AAD [], # SEND_NON_EMPTY_VERSION_PACKET diff --git a/test/functional/rpc_bind.py b/test/functional/rpc_bind.py index 8c76c1f5f5..69afd45b9a 100755 --- a/test/functional/rpc_bind.py +++ b/test/functional/rpc_bind.py @@ -45,6 +45,19 @@ class RPCBindTest(BitcoinTestFramework): assert_equal(set(get_bind_addrs(pid)), set(expected)) self.stop_nodes() + def run_invalid_bind_test(self, allow_ips, addresses): + ''' + Attempt to start a node with requested rpcallowip and rpcbind + parameters, expecting that the node will fail. + ''' + self.log.info(f'Invalid bind test for {addresses}') + base_args = ['-disablewallet', '-nolisten'] + if allow_ips: + base_args += ['-rpcallowip=' + x for x in allow_ips] + init_error = 'Error: Invalid port specified in -rpcbind: ' + for addr in addresses: + self.nodes[0].assert_start_raises_init_error(base_args + [f'-rpcbind={addr}'], init_error + f"'{addr}'") + def run_allowip_test(self, allow_ips, rpchost, rpcport): ''' Start a node with rpcallow IP, and request getnetworkinfo @@ -84,6 +97,10 @@ class RPCBindTest(BitcoinTestFramework): if not self.options.run_nonloopback: self._run_loopback_tests() + if self.options.run_ipv4: + self.run_invalid_bind_test(['127.0.0.1'], ['127.0.0.1:notaport', '127.0.0.1:-18443', '127.0.0.1:0', '127.0.0.1:65536']) + if self.options.run_ipv6: + self.run_invalid_bind_test(['[::1]'], ['[::1]:notaport', '[::1]:-18443', '[::1]:0', '[::1]:65536']) if not self.options.run_ipv4 and not self.options.run_ipv6: self._run_nonloopback_tests() diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 98147237b1..f02e6914ef 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -32,14 +32,16 @@ from test_framework.blocktools import ( TIME_GENESIS_BLOCK, create_block, create_coinbase, + create_tx_with_script, ) from test_framework.messages import ( CBlockHeader, + COIN, from_hex, msg_block, ) from test_framework.p2p import P2PInterface -from test_framework.script import hash256 +from test_framework.script import hash256, OP_TRUE from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -88,6 +90,7 @@ class BlockchainTest(BitcoinTestFramework): self._test_getdifficulty() self._test_getnetworkhashps() self._test_stopatheight() + self._test_waitforblock() # also tests waitfornewblock self._test_waitforblockheight() self._test_getblock() self._test_getdeploymentinfo() @@ -505,6 +508,38 @@ class BlockchainTest(BitcoinTestFramework): self.start_node(0) assert_equal(self.nodes[0].getblockcount(), HEIGHT + 7) + def _test_waitforblock(self): + self.log.info("Test waitforblock and waitfornewblock") + node = self.nodes[0] + + current_height = node.getblock(node.getbestblockhash())['height'] + current_hash = node.getblock(node.getbestblockhash())['hash'] + + self.log.debug("Roll the chain back a few blocks and then reconsider it") + rollback_height = current_height - 100 + rollback_hash = node.getblockhash(rollback_height) + rollback_header = node.getblockheader(rollback_hash) + + node.invalidateblock(rollback_hash) + assert_equal(node.getblockcount(), rollback_height - 1) + + self.log.debug("waitforblock should return the same block after its timeout") + assert_equal(node.waitforblock(blockhash=current_hash, timeout=1)['hash'], rollback_header['previousblockhash']) + + node.reconsiderblock(rollback_hash) + # The chain has probably already been restored by the time reconsiderblock returns, + # but poll anyway. + self.wait_until(lambda: node.waitforblock(blockhash=current_hash, timeout=100)['hash'] == current_hash) + + # roll back again + node.invalidateblock(rollback_hash) + assert_equal(node.getblockcount(), rollback_height - 1) + + node.reconsiderblock(rollback_hash) + # The chain has probably already been restored by the time reconsiderblock returns, + # but poll anyway. + self.wait_until(lambda: node.waitfornewblock(timeout=100)['hash'] == current_hash) + def _test_waitforblockheight(self): self.log.info("Test waitforblockheight") node = self.nodes[0] @@ -556,12 +591,12 @@ class BlockchainTest(BitcoinTestFramework): block = node.getblock(blockhash, verbosity) assert_equal(blockhash, hash256(bytes.fromhex(block[:160]))[::-1].hex()) - def assert_fee_not_in_block(verbosity): - block = node.getblock(blockhash, verbosity) + def assert_fee_not_in_block(hash, verbosity): + block = node.getblock(hash, verbosity) assert 'fee' not in block['tx'][1] - def assert_fee_in_block(verbosity): - block = node.getblock(blockhash, verbosity) + def assert_fee_in_block(hash, verbosity): + block = node.getblock(hash, verbosity) tx = block['tx'][1] assert 'fee' in tx assert_equal(tx['fee'], tx['vsize'] * fee_per_byte) @@ -580,8 +615,8 @@ class BlockchainTest(BitcoinTestFramework): total_vout += vout["value"] assert_equal(total_vin, total_vout + tx["fee"]) - def assert_vin_does_not_contain_prevout(verbosity): - block = node.getblock(blockhash, verbosity) + def assert_vin_does_not_contain_prevout(hash, verbosity): + block = node.getblock(hash, verbosity) tx = block["tx"][1] if isinstance(tx, str): # In verbosity level 1, only the transaction hashes are written @@ -595,16 +630,16 @@ class BlockchainTest(BitcoinTestFramework): assert_hexblock_hashes(False) self.log.info("Test that getblock with verbosity 1 doesn't include fee") - assert_fee_not_in_block(1) - assert_fee_not_in_block(True) + assert_fee_not_in_block(blockhash, 1) + assert_fee_not_in_block(blockhash, True) self.log.info('Test that getblock with verbosity 2 and 3 includes expected fee') - assert_fee_in_block(2) - assert_fee_in_block(3) + assert_fee_in_block(blockhash, 2) + assert_fee_in_block(blockhash, 3) self.log.info("Test that getblock with verbosity 1 and 2 does not include prevout") - assert_vin_does_not_contain_prevout(1) - assert_vin_does_not_contain_prevout(2) + assert_vin_does_not_contain_prevout(blockhash, 1) + assert_vin_does_not_contain_prevout(blockhash, 2) self.log.info("Test that getblock with verbosity 3 includes prevout") assert_vin_contains_prevout(3) @@ -612,7 +647,7 @@ class BlockchainTest(BitcoinTestFramework): self.log.info("Test getblock with invalid verbosity type returns proper error message") assert_raises_rpc_error(-3, "JSON value of type string is not of expected type number", node.getblock, blockhash, "2") - self.log.info("Test that getblock with verbosity 2 and 3 still works with pruned Undo data") + self.log.info("Test that getblock doesn't work with deleted Undo data") def move_block_file(old, new): old_path = self.nodes[0].blocks_path / old @@ -622,10 +657,8 @@ class BlockchainTest(BitcoinTestFramework): # Move instead of deleting so we can restore chain state afterwards move_block_file('rev00000.dat', 'rev_wrong') - assert_fee_not_in_block(2) - assert_fee_not_in_block(3) - assert_vin_does_not_contain_prevout(2) - assert_vin_does_not_contain_prevout(3) + assert_raises_rpc_error(-32603, "Undo data expected but can't be read. This could be due to disk corruption or a conflict with a pruning event.", lambda: node.getblock(blockhash, 2)) + assert_raises_rpc_error(-32603, "Undo data expected but can't be read. This could be due to disk corruption or a conflict with a pruning event.", lambda: node.getblock(blockhash, 3)) # Restore chain state move_block_file('rev_wrong', 'rev00000.dat') @@ -633,6 +666,31 @@ class BlockchainTest(BitcoinTestFramework): assert 'previousblockhash' not in node.getblock(node.getblockhash(0)) assert 'nextblockhash' not in node.getblock(node.getbestblockhash()) + self.log.info("Test getblock when only header is known") + current_height = node.getblock(node.getbestblockhash())['height'] + block_time = node.getblock(node.getbestblockhash())['time'] + 1 + block = create_block(int(blockhash, 16), create_coinbase(current_height + 1, nValue=100), block_time) + block.solve() + node.submitheader(block.serialize().hex()) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", lambda: node.getblock(block.hash)) + + self.log.info("Test getblock when block data is available but undo data isn't") + # Submits a block building on the header-only block, so it can't be connected and has no undo data + tx = create_tx_with_script(block.vtx[0], 0, script_sig=bytes([OP_TRUE]), amount=50 * COIN) + block_noundo = create_block(block.sha256, create_coinbase(current_height + 2, nValue=100), block_time + 1, txlist=[tx]) + block_noundo.solve() + node.submitblock(block_noundo.serialize().hex()) + + assert_fee_not_in_block(block_noundo.hash, 2) + assert_fee_not_in_block(block_noundo.hash, 3) + assert_vin_does_not_contain_prevout(block_noundo.hash, 2) + assert_vin_does_not_contain_prevout(block_noundo.hash, 3) + + self.log.info("Test getblock when block is missing") + move_block_file('blk00000.dat', 'blk00000.dat.bak') + assert_raises_rpc_error(-1, "Block not found on disk", node.getblock, blockhash) + move_block_file('blk00000.dat.bak', 'blk00000.dat') + if __name__ == '__main__': BlockchainTest(__file__).main() diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index d95820bbf8..0692a1203d 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -194,13 +194,19 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "redeemScript/witnessScript does not match scriptPubKey", node2.signrawtransactionwithkey, rawtx, priv_keys[0:nsigs-1], [prevtx_err]) rawtx2 = node2.signrawtransactionwithkey(rawtx, priv_keys[0:nsigs - 1], prevtxs) - rawtx3 = node2.signrawtransactionwithkey(rawtx2["hex"], [priv_keys[-1]], prevtxs) - assert rawtx3['complete'] - - tx = node0.sendrawtransaction(rawtx3["hex"], 0) + assert_equal(rawtx2["complete"], False) + rawtx3 = node2.signrawtransactionwithkey(rawtx, [priv_keys[-1]], prevtxs) + assert_equal(rawtx3["complete"], False) + assert_raises_rpc_error(-22, "TX decode failed", node2.combinerawtransaction, [rawtx2['hex'], rawtx3['hex'] + "00"]) + assert_raises_rpc_error(-22, "Missing transactions", node2.combinerawtransaction, []) + combined_rawtx = node2.combinerawtransaction([rawtx2["hex"], rawtx3["hex"]]) + + tx = node0.sendrawtransaction(combined_rawtx, 0) blk = self.generate(node0, 1)[0] assert tx in node0.getblock(blk)["tx"] + assert_raises_rpc_error(-25, "Input not found or already spent", node2.combinerawtransaction, [rawtx2['hex'], rawtx3['hex']]) + # When the wallet is enabled, assert node2 sees the incoming amount if self.is_wallet_compiled(): assert_equal(node2.getbalances()['mine']['trusted'], node2_balance + outval) diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 3262c664ad..292744116b 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -19,6 +19,17 @@ class DumptxoutsetTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 + def check_expected_network(self, node, active): + rev_file = node.blocks_path / "rev00000.dat" + bogus_file = node.blocks_path / "bogus.dat" + rev_file.rename(bogus_file) + assert_raises_rpc_error( + -1, 'Could not roll back to requested height.', node.dumptxoutset, 'utxos.dat', rollback=99) + assert_equal(node.getnetworkinfo()['networkactive'], active) + + # Cleanup + bogus_file.rename(rev_file) + def run_test(self): """Test a trivial usage of the dumptxoutset RPC command.""" node = self.nodes[0] @@ -56,20 +67,18 @@ class DumptxoutsetTest(BitcoinTestFramework): assert_raises_rpc_error( -8, "Couldn't open file {}.incomplete for writing".format(invalid_path), node.dumptxoutset, invalid_path, "latest") - self.log.info(f"Test that dumptxoutset with unknown dump type fails") + self.log.info("Test that dumptxoutset with unknown dump type fails") assert_raises_rpc_error( -8, 'Invalid snapshot type "bogus" specified. Please specify "rollback" or "latest"', node.dumptxoutset, 'utxos.dat', "bogus") - self.log.info(f"Test that dumptxoutset failure does not leave the network activity suspended") - rev_file = node.blocks_path / "rev00000.dat" - bogus_file = node.blocks_path / "bogus.dat" - rev_file.rename(bogus_file) - assert_raises_rpc_error( - -1, 'Could not roll back to requested height.', node.dumptxoutset, 'utxos.dat', rollback=99) - assert_equal(node.getnetworkinfo()['networkactive'], True) + self.log.info("Test that dumptxoutset failure does not leave the network activity suspended when it was on previously") + self.check_expected_network(node, True) + + self.log.info("Test that dumptxoutset failure leaves the network activity suspended when it was off") + node.setnetworkactive(False) + self.check_expected_network(node, False) + node.setnetworkactive(True) - # Cleanup - bogus_file.rename(rev_file) if __name__ == '__main__': DumptxoutsetTest(__file__).main() diff --git a/test/functional/rpc_generate.py b/test/functional/rpc_generate.py index 68de900664..618bbb2e00 100755 --- a/test/functional/rpc_generate.py +++ b/test/functional/rpc_generate.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2020-2022 The Bitcoin Core developers +# Copyright (c) 2020-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test generate* RPCs.""" +from concurrent.futures import ThreadPoolExecutor + from test_framework.test_framework import BitcoinTestFramework from test_framework.wallet import MiniWallet from test_framework.util import ( @@ -83,11 +85,18 @@ class RPCGenerateTest(BitcoinTestFramework): txid = block['tx'][1] assert_equal(node.getrawtransaction(txid=txid, verbose=False, blockhash=hash), rawtx) + # Ensure that generateblock can be called concurrently by many threads. + self.log.info('Generate blocks in parallel') + generate_50_blocks = lambda n: [n.generateblock(output=address, transactions=[]) for _ in range(50)] + rpcs = [node.cli for _ in range(6)] + with ThreadPoolExecutor(max_workers=len(rpcs)) as threads: + list(threads.map(generate_50_blocks, rpcs)) + self.log.info('Fail to generate block with out of order txs') txid1 = miniwallet.send_self_transfer(from_node=node)['txid'] utxo1 = miniwallet.get_utxo(txid=txid1) rawtx2 = miniwallet.create_self_transfer(utxo_to_spend=utxo1)['hex'] - assert_raises_rpc_error(-25, 'testBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) + assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) self.log.info('Fail to generate block with txid not in mempool') missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index e309018516..62b3d664e0 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -58,7 +58,7 @@ class GetBlockFromPeerTest(BitcoinTestFramework): self.log.info("Node 0 should only have the header for node 1's block 3") x = next(filter(lambda x: x['hash'] == short_tip, self.nodes[0].getchaintips())) assert_equal(x['status'], "headers-only") - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, short_tip) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", self.nodes[0].getblock, short_tip) self.log.info("Fetch block from node 1") peers = self.nodes[0].getpeerinfo() diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index d1e4895eb6..002763201a 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -114,7 +114,7 @@ class GetblockstatsTest(BitcoinTestFramework): assert_equal(stats[self.max_stat_pos]['height'], self.start_height + self.max_stat_pos) for i in range(self.max_stat_pos+1): - self.log.info('Checking block %d\n' % (i)) + self.log.info('Checking block %d' % (i)) assert_equal(stats[i], self.expected_stats[i]) # Check selecting block by hash too @@ -182,5 +182,16 @@ class GetblockstatsTest(BitcoinTestFramework): assert_equal(tip_stats["utxo_increase_actual"], 4) assert_equal(tip_stats["utxo_size_inc_actual"], 300) + self.log.info("Test when only header is known") + block = self.generateblock(self.nodes[0], output="raw(55)", transactions=[], submit=False) + self.nodes[0].submitheader(block["hex"]) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", lambda: self.nodes[0].getblockstats(block['hash'])) + + self.log.info('Test when block is missing') + (self.nodes[0].blocks_path / 'blk00000.dat').rename(self.nodes[0].blocks_path / 'blk00000.dat.backup') + assert_raises_rpc_error(-1, 'Block not found on disk', self.nodes[0].getblockstats, hash_or_height=1) + (self.nodes[0].blocks_path / 'blk00000.dat.backup').rename(self.nodes[0].blocks_path / 'blk00000.dat') + + if __name__ == '__main__': GetblockstatsTest(__file__).main() diff --git a/test/functional/rpc_getchaintips.py b/test/functional/rpc_getchaintips.py index 1bd4413ce1..226e995307 100755 --- a/test/functional/rpc_getchaintips.py +++ b/test/functional/rpc_getchaintips.py @@ -10,6 +10,10 @@ - verify that getchaintips now returns two chain tips. """ +from test_framework.blocktools import ( + create_block, + create_coinbase, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal @@ -18,44 +22,73 @@ class GetChainTipsTest (BitcoinTestFramework): self.num_nodes = 4 def run_test(self): + self.log.info("Test getchaintips behavior with two chains of different length") tips = self.nodes[0].getchaintips() assert_equal(len(tips), 1) assert_equal(tips[0]['branchlen'], 0) assert_equal(tips[0]['height'], 200) assert_equal(tips[0]['status'], 'active') - # Split the network and build two chains of different lengths. + self.log.info("Split the network and build two chains of different lengths.") self.split_network() self.generate(self.nodes[0], 10, sync_fun=lambda: self.sync_all(self.nodes[:2])) self.generate(self.nodes[2], 20, sync_fun=lambda: self.sync_all(self.nodes[2:])) - tips = self.nodes[1].getchaintips () - assert_equal (len (tips), 1) + tips = self.nodes[1].getchaintips() + assert_equal(len(tips), 1) shortTip = tips[0] - assert_equal (shortTip['branchlen'], 0) - assert_equal (shortTip['height'], 210) - assert_equal (tips[0]['status'], 'active') + assert_equal(shortTip['branchlen'], 0) + assert_equal(shortTip['height'], 210) + assert_equal(tips[0]['status'], 'active') - tips = self.nodes[3].getchaintips () - assert_equal (len (tips), 1) + tips = self.nodes[3].getchaintips() + assert_equal(len(tips), 1) longTip = tips[0] - assert_equal (longTip['branchlen'], 0) - assert_equal (longTip['height'], 220) - assert_equal (tips[0]['status'], 'active') + assert_equal(longTip['branchlen'], 0) + assert_equal(longTip['height'], 220) + assert_equal(tips[0]['status'], 'active') - # Join the network halves and check that we now have two tips + self.log.info("Join the network halves and check that we now have two tips") # (at least at the nodes that previously had the short chain). - self.join_network () + self.join_network() - tips = self.nodes[0].getchaintips () - assert_equal (len (tips), 2) - assert_equal (tips[0], longTip) + tips = self.nodes[0].getchaintips() + assert_equal(len(tips), 2) + assert_equal(tips[0], longTip) - assert_equal (tips[1]['branchlen'], 10) - assert_equal (tips[1]['status'], 'valid-fork') + assert_equal(tips[1]['branchlen'], 10) + assert_equal(tips[1]['status'], 'valid-fork') tips[1]['branchlen'] = 0 tips[1]['status'] = 'active' - assert_equal (tips[1], shortTip) + assert_equal(tips[1], shortTip) + + self.log.info("Test getchaintips behavior with invalid blocks") + self.disconnect_nodes(0, 1) + n0 = self.nodes[0] + tip = int(n0.getbestblockhash(), 16) + start_height = self.nodes[0].getblockcount() + # Create invalid block (too high coinbase) + block_time = n0.getblock(n0.getbestblockhash())['time'] + 1 + invalid_block = create_block(tip, create_coinbase(start_height+1, nValue=100), block_time) + invalid_block.solve() + + block_time += 1 + block2 = create_block(invalid_block.sha256, create_coinbase(2), block_time, version=4) + block2.solve() + + self.log.info("Submit headers-only chain") + n0.submitheader(invalid_block.serialize().hex()) + n0.submitheader(block2.serialize().hex()) + tips = n0.getchaintips() + assert_equal(len(tips), 3) + assert_equal(tips[0]['status'], 'headers-only') + + self.log.info("Submit invalid block that invalidates the headers-only chain") + n0.submitblock(invalid_block.serialize().hex()) + tips = n0.getchaintips() + assert_equal(len(tips), 3) + assert_equal(tips[0]['status'], 'invalid') + if __name__ == '__main__': GetChainTipsTest(__file__).main() diff --git a/test/functional/rpc_getdescriptoractivity.py b/test/functional/rpc_getdescriptoractivity.py new file mode 100755 index 0000000000..a0dc43718b --- /dev/null +++ b/test/functional/rpc_getdescriptoractivity.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from decimal import Decimal + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_raises_rpc_error +from test_framework.messages import COIN +from test_framework.wallet import MiniWallet, MiniWalletMode, getnewdestination + + +class GetBlocksActivityTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + node = self.nodes[0] + wallet = MiniWallet(node) + node.setmocktime(node.getblockheader(node.getbestblockhash())['time']) + self.generate(wallet, 200) + + self.test_no_activity(node) + self.test_activity_in_block(node, wallet) + self.test_no_mempool_inclusion(node, wallet) + self.test_multiple_addresses(node, wallet) + self.test_invalid_blockhash(node, wallet) + self.test_invalid_descriptor(node, wallet) + self.test_confirmed_and_unconfirmed(node, wallet) + self.test_receive_then_spend(node, wallet) + self.test_no_address(node, wallet) + + def test_no_activity(self, node): + _, _, addr_1 = getnewdestination() + result = node.getdescriptoractivity([], [f"addr({addr_1})"], True) + assert_equal(len(result['activity']), 0) + + def test_activity_in_block(self, node, wallet): + _, spk_1, addr_1 = getnewdestination(address_type='bech32m') + txid = wallet.send_to(from_node=node, scriptPubKey=spk_1, amount=1 * COIN)['txid'] + blockhash = self.generate(node, 1)[0] + + # Test getdescriptoractivity with the specific blockhash + result = node.getdescriptoractivity([blockhash], [f"addr({addr_1})"], True) + assert_equal(list(result.keys()), ['activity']) + [activity] = result['activity'] + + for k, v in { + 'amount': Decimal('1.00000000'), + 'blockhash': blockhash, + 'height': 201, + 'txid': txid, + 'type': 'receive', + 'vout': 1, + }.items(): + assert_equal(activity[k], v) + + outspk = activity['output_spk'] + + assert_equal(outspk['asm'][:2], '1 ') + assert_equal(outspk['desc'].split('(')[0], 'rawtr') + assert_equal(outspk['hex'], spk_1.hex()) + assert_equal(outspk['address'], addr_1) + assert_equal(outspk['type'], 'witness_v1_taproot') + + + def test_no_mempool_inclusion(self, node, wallet): + _, spk_1, addr_1 = getnewdestination() + wallet.send_to(from_node=node, scriptPubKey=spk_1, amount=1 * COIN) + + _, spk_2, addr_2 = getnewdestination() + wallet.send_to( + from_node=node, scriptPubKey=spk_2, amount=1 * COIN) + + # Do not generate a block to keep the transaction in the mempool + + result = node.getdescriptoractivity([], [f"addr({addr_1})", f"addr({addr_2})"], False) + + assert_equal(len(result['activity']), 0) + + def test_multiple_addresses(self, node, wallet): + _, spk_1, addr_1 = getnewdestination() + _, spk_2, addr_2 = getnewdestination() + wallet.send_to(from_node=node, scriptPubKey=spk_1, amount=1 * COIN) + wallet.send_to(from_node=node, scriptPubKey=spk_2, amount=2 * COIN) + + blockhash = self.generate(node, 1)[0] + + result = node.getdescriptoractivity([blockhash], [f"addr({addr_1})", f"addr({addr_2})"], True) + + assert_equal(len(result['activity']), 2) + + # Duplicate address specification is fine. + assert_equal( + result, + node.getdescriptoractivity([blockhash], [ + f"addr({addr_1})", f"addr({addr_1})", f"addr({addr_2})"], True)) + + # Flipping descriptor order doesn't affect results. + result_flipped = node.getdescriptoractivity( + [blockhash], [f"addr({addr_2})", f"addr({addr_1})"], True) + assert_equal(result, result_flipped) + + [a1] = [a for a in result['activity'] if a['output_spk']['address'] == addr_1] + [a2] = [a for a in result['activity'] if a['output_spk']['address'] == addr_2] + + assert a1['blockhash'] == blockhash + assert a1['amount'] == 1.0 + + assert a2['blockhash'] == blockhash + assert a2['amount'] == 2.0 + + def test_invalid_blockhash(self, node, wallet): + self.generate(node, 20) # Generate to get more fees + + _, spk_1, addr_1 = getnewdestination() + wallet.send_to(from_node=node, scriptPubKey=spk_1, amount=1 * COIN) + + invalid_blockhash = "0000000000000000000000000000000000000000000000000000000000000000" + + assert_raises_rpc_error( + -5, "Block not found", + node.getdescriptoractivity, [invalid_blockhash], [f"addr({addr_1})"], True) + + def test_invalid_descriptor(self, node, wallet): + blockhash = self.generate(node, 1)[0] + _, _, addr_1 = getnewdestination() + + assert_raises_rpc_error( + -5, "is not a valid descriptor", + node.getdescriptoractivity, [blockhash], [f"addrx({addr_1})"], True) + + def test_confirmed_and_unconfirmed(self, node, wallet): + self.generate(node, 20) # Generate to get more fees + + _, spk_1, addr_1 = getnewdestination() + txid_1 = wallet.send_to( + from_node=node, scriptPubKey=spk_1, amount=1 * COIN)['txid'] + blockhash = self.generate(node, 1)[0] + + _, spk_2, to_addr = getnewdestination() + txid_2 = wallet.send_to( + from_node=node, scriptPubKey=spk_2, amount=1 * COIN)['txid'] + + result = node.getdescriptoractivity( + [blockhash], [f"addr({addr_1})", f"addr({to_addr})"], True) + + activity = result['activity'] + assert_equal(len(activity), 2) + + [confirmed] = [a for a in activity if a.get('blockhash') == blockhash] + assert confirmed['txid'] == txid_1 + assert confirmed['height'] == node.getblockchaininfo()['blocks'] + + [unconfirmed] = [a for a in activity if not a.get('blockhash')] + assert 'blockhash' not in unconfirmed + assert 'height' not in unconfirmed + + assert any(a['txid'] == txid_2 for a in activity if not a.get('blockhash')) + + def test_receive_then_spend(self, node, wallet): + """Also important because this tests multiple blockhashes.""" + self.generate(node, 20) # Generate to get more fees + + sent1 = wallet.send_self_transfer(from_node=node) + utxo = sent1['new_utxo'] + blockhash_1 = self.generate(node, 1)[0] + + sent2 = wallet.send_self_transfer(from_node=node, utxo_to_spend=utxo) + blockhash_2 = self.generate(node, 1)[0] + + result = node.getdescriptoractivity( + [blockhash_1, blockhash_2], [wallet.get_descriptor()], True) + + assert_equal(len(result['activity']), 4) + + assert result['activity'][1]['type'] == 'receive' + assert result['activity'][1]['txid'] == sent1['txid'] + assert result['activity'][1]['blockhash'] == blockhash_1 + + assert result['activity'][2]['type'] == 'spend' + assert result['activity'][2]['spend_txid'] == sent2['txid'] + assert result['activity'][2]['prevout_txid'] == sent1['txid'] + assert result['activity'][2]['blockhash'] == blockhash_2 + + # Test that reversing the blockorder yields the same result. + assert_equal(result, node.getdescriptoractivity( + [blockhash_1, blockhash_2], [wallet.get_descriptor()], True)) + + # Test that duplicating a blockhash yields the same result. + assert_equal(result, node.getdescriptoractivity( + [blockhash_1, blockhash_2, blockhash_2], [wallet.get_descriptor()], True)) + + def test_no_address(self, node, wallet): + raw_wallet = MiniWallet(self.nodes[0], mode=MiniWalletMode.RAW_P2PK) + self.generate(raw_wallet, 100) + + no_addr_tx = raw_wallet.send_self_transfer(from_node=node) + raw_desc = raw_wallet.get_descriptor() + + blockhash = self.generate(node, 1)[0] + + result = node.getdescriptoractivity([blockhash], [raw_desc], False) + + assert_equal(len(result['activity']), 2) + + a1 = result['activity'][0] + a2 = result['activity'][1] + + assert a1['type'] == "spend" + assert a1['blockhash'] == blockhash + # sPK lacks address. + assert_equal(list(a1['prevout_spk'].keys()), ['asm', 'desc', 'hex', 'type']) + assert a1['amount'] == no_addr_tx["fee"] + Decimal(no_addr_tx["tx"].vout[0].nValue) / COIN + + assert a2['type'] == "receive" + assert a2['blockhash'] == blockhash + # sPK lacks address. + assert_equal(list(a2['output_spk'].keys()), ['asm', 'desc', 'hex', 'type']) + assert a2['amount'] == Decimal(no_addr_tx["tx"].vout[0].nValue) / COIN + + +if __name__ == '__main__': + GetBlocksActivityTest(__file__).main() diff --git a/test/functional/rpc_invalidateblock.py b/test/functional/rpc_invalidateblock.py index db79d55259..e2eb033f09 100755 --- a/test/functional/rpc_invalidateblock.py +++ b/test/functional/rpc_invalidateblock.py @@ -6,6 +6,10 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR +from test_framework.blocktools import ( + create_block, + create_coinbase, +) from test_framework.util import ( assert_equal, assert_raises_rpc_error, @@ -35,12 +39,33 @@ class InvalidateTest(BitcoinTestFramework): self.connect_nodes(0, 1) self.sync_blocks(self.nodes[0:2]) assert_equal(self.nodes[0].getblockcount(), 6) - badhash = self.nodes[1].getblockhash(2) + + # Add a header to the tip of node 0 without submitting the block. This shouldn't + # affect results since this chain will be invalidated next. + tip = self.nodes[0].getbestblockhash() + block_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] + 1 + block = create_block(int(tip, 16), create_coinbase(self.nodes[0].getblockcount()), block_time, version=4) + block.solve() + self.nodes[0].submitheader(block.serialize().hex()) + assert_equal(self.nodes[0].getblockchaininfo()["headers"], self.nodes[0].getblockchaininfo()["blocks"] + 1) self.log.info("Invalidate block 2 on node 0 and verify we reorg to node 0's original chain") + badhash = self.nodes[1].getblockhash(2) + self.nodes[0].invalidateblock(badhash) + assert_equal(self.nodes[0].getblockcount(), 4) + assert_equal(self.nodes[0].getbestblockhash(), besthash_n0) + # Should report consistent blockchain info + assert_equal(self.nodes[0].getblockchaininfo()["headers"], self.nodes[0].getblockchaininfo()["blocks"]) + + self.log.info("Reconsider block 6 on node 0 again and verify that the best header is set correctly") + self.nodes[0].reconsiderblock(tip) + assert_equal(self.nodes[0].getblockchaininfo()["headers"], self.nodes[0].getblockchaininfo()["blocks"] + 1) + + self.log.info("Invalidate block 2 on node 0 and verify we reorg to node 0's original chain again") self.nodes[0].invalidateblock(badhash) assert_equal(self.nodes[0].getblockcount(), 4) assert_equal(self.nodes[0].getbestblockhash(), besthash_n0) + assert_equal(self.nodes[0].getblockchaininfo()["headers"], self.nodes[0].getblockchaininfo()["blocks"]) self.log.info("Make sure we won't reorg to a lower work chain:") self.connect_nodes(1, 2) @@ -83,6 +108,8 @@ class InvalidateTest(BitcoinTestFramework): self.nodes[1].reconsiderblock(blocks[-4]) # Should be back at the tip by now assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) + # Should report consistent blockchain info + assert_equal(self.nodes[1].getblockchaininfo()["headers"], self.nodes[1].getblockchaininfo()["blocks"]) self.log.info("Verify that invalidating an unknown block throws an error") assert_raises_rpc_error(-5, "Block not found", self.nodes[1].invalidateblock, "00" * 32) diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index b63059b1ee..41ecbbed22 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -63,6 +63,9 @@ class NetTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.extra_args = [["-minrelaytxfee=0.00001000"], ["-minrelaytxfee=0.00000500"]] + # Specify a non-working proxy to make sure no actual connections to public IPs are attempted + for args in self.extra_args: + args.append("-proxy=127.0.0.1:1") self.supports_cli = False def run_test(self): diff --git a/test/functional/rpc_orphans.py b/test/functional/rpc_orphans.py new file mode 100755 index 0000000000..4871166a39 --- /dev/null +++ b/test/functional/rpc_orphans.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Tests for orphan related RPCs.""" + +import time + +from test_framework.mempool_util import ( + ORPHAN_TX_EXPIRE_TIME, + tx_in_orphanage, +) +from test_framework.messages import msg_tx +from test_framework.p2p import P2PInterface +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet import MiniWallet + + +class OrphanRPCsTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.test_orphan_activity() + self.test_orphan_details() + self.test_misc() + + def test_orphan_activity(self): + self.log.info("Check that orphaned transactions are returned with getorphantxs") + node = self.nodes[0] + + self.log.info("Create two 1P1C packages, but only broadcast the children") + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + tx_parent_2 = self.wallet.create_self_transfer() + tx_child_2 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_2["new_utxo"]) + peer = node.add_p2p_connection(P2PInterface()) + peer.send_and_ping(msg_tx(tx_child_1["tx"])) + peer.send_and_ping(msg_tx(tx_child_2["tx"])) + + self.log.info("Check that neither parent is in the mempool") + assert_equal(node.getmempoolinfo()["size"], 0) + + orphanage = node.getorphantxs(verbosity=0) + self.log.info("Check the size of the orphanage") + assert_equal(len(orphanage), 2) + self.log.info("Check that undefined verbosity is disallowed") + assert_raises_rpc_error(-8, "Invalid verbosity value -1", node.getorphantxs, verbosity=-1) + assert_raises_rpc_error(-8, "Invalid verbosity value 3", node.getorphantxs, verbosity=3) + self.log.info("Check that both children are in the orphanage") + assert tx_in_orphanage(node, tx_child_1["tx"]) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Broadcast parent 1") + peer.send_and_ping(msg_tx(tx_parent_1["tx"])) + self.log.info("Check that parent 1 and child 1 are in the mempool") + raw_mempool = node.getrawmempool() + assert_equal(len(raw_mempool), 2) + assert tx_parent_1["txid"] in raw_mempool + assert tx_child_1["txid"] in raw_mempool + + self.log.info("Check that orphanage only contains child 2") + orphanage = node.getorphantxs() + assert_equal(len(orphanage), 1) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + peer.send_and_ping(msg_tx(tx_parent_2["tx"])) + self.log.info("Check that all parents and children are now in the mempool") + raw_mempool = node.getrawmempool() + assert_equal(len(raw_mempool), 4) + assert tx_parent_1["txid"] in raw_mempool + assert tx_child_1["txid"] in raw_mempool + assert tx_parent_2["txid"] in raw_mempool + assert tx_child_2["txid"] in raw_mempool + self.log.info("Check that the orphanage is empty") + assert_equal(len(node.getorphantxs()), 0) + + self.log.info("Confirm the transactions (clears mempool)") + self.generate(node, 1) + assert_equal(node.getmempoolinfo()["size"], 0) + + def test_orphan_details(self): + self.log.info("Check the transaction details returned from getorphantxs") + node = self.nodes[0] + + self.log.info("Create two orphans, from different peers") + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + tx_parent_2 = self.wallet.create_self_transfer() + tx_child_2 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_2["new_utxo"]) + peer_1 = node.add_p2p_connection(P2PInterface()) + peer_2 = node.add_p2p_connection(P2PInterface()) + entry_time = int(time.time()) + node.setmocktime(entry_time) + peer_1.send_and_ping(msg_tx(tx_child_1["tx"])) + peer_2.send_and_ping(msg_tx(tx_child_2["tx"])) + + orphanage = node.getorphantxs(verbosity=2) + assert tx_in_orphanage(node, tx_child_1["tx"]) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Check that orphan 1 and 2 were from different peers") + assert orphanage[0]["from"][0] != orphanage[1]["from"][0] + + self.log.info("Unorphan child 2") + peer_2.send_and_ping(msg_tx(tx_parent_2["tx"])) + assert not tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Checking orphan details") + orphanage = node.getorphantxs(verbosity=1) + assert_equal(len(node.getorphantxs()), 1) + orphan_1 = orphanage[0] + self.orphan_details_match(orphan_1, tx_child_1, verbosity=1) + self.log.info("Checking orphan entry/expiration times") + assert_equal(orphan_1["entry"], entry_time) + assert_equal(orphan_1["expiration"], entry_time + ORPHAN_TX_EXPIRE_TIME) + + self.log.info("Checking orphan details (verbosity 2)") + orphanage = node.getorphantxs(verbosity=2) + orphan_1 = orphanage[0] + self.orphan_details_match(orphan_1, tx_child_1, verbosity=2) + + def orphan_details_match(self, orphan, tx, verbosity): + self.log.info("Check txid/wtxid of orphan") + assert_equal(orphan["txid"], tx["txid"]) + assert_equal(orphan["wtxid"], tx["wtxid"]) + + self.log.info("Check the sizes of orphan") + assert_equal(orphan["bytes"], len(tx["tx"].serialize())) + assert_equal(orphan["vsize"], tx["tx"].get_vsize()) + assert_equal(orphan["weight"], tx["tx"].get_weight()) + + if verbosity == 2: + self.log.info("Check the transaction hex of orphan") + assert_equal(orphan["hex"], tx["hex"]) + + def test_misc(self): + node = self.nodes[0] + assert_raises_rpc_error(-3, "Verbosity was boolean but only integer allowed", node.getorphantxs, verbosity=True) + assert_raises_rpc_error(-3, "Verbosity was boolean but only integer allowed", node.getorphantxs, verbosity=False) + help_output = node.help() + self.log.info("Check that getorphantxs is a hidden RPC") + assert "getorphantxs" not in help_output + assert "unknown command: getorphantxs" not in node.help("getorphantxs") + + +if __name__ == '__main__': + OrphanRPCsTest(__file__).main() diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index ef2f66b3a0..a2f9210f94 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -110,17 +110,21 @@ class RPCPackagesTest(BitcoinTestFramework): self.assert_testres_equal(package_bad, testres_bad) self.log.info("Check testmempoolaccept tells us when some transactions completed validation successfully") - tx_bad_sig_hex = node.createrawtransaction([{"txid": coin["txid"], "vout": 0}], + tx_bad_sig_hex = node.createrawtransaction([{"txid": coin["txid"], "vout": coin["vout"]}], {address : coin["amount"] - Decimal("0.0001")}) tx_bad_sig = tx_from_hex(tx_bad_sig_hex) testres_bad_sig = node.testmempoolaccept(self.independent_txns_hex + [tx_bad_sig_hex]) # By the time the signature for the last transaction is checked, all the other transactions # have been fully validated, which is why the node returns full validation results for all # transactions here but empty results in other cases. + tx_bad_sig_txid = tx_bad_sig.rehash() + tx_bad_sig_wtxid = tx_bad_sig.getwtxid() assert_equal(testres_bad_sig, self.independent_txns_testres + [{ - "txid": tx_bad_sig.rehash(), - "wtxid": tx_bad_sig.getwtxid(), "allowed": False, - "reject-reason": "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)" + "txid": tx_bad_sig_txid, + "wtxid": tx_bad_sig_wtxid, "allowed": False, + "reject-reason": "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", + "reject-details": "mandatory-script-verify-flag-failed (Operation not valid with the current stack size), " + + f"input 0 of {tx_bad_sig_txid} (wtxid {tx_bad_sig_wtxid}), spending {coin['txid']}:{coin['vout']}" }]) self.log.info("Check testmempoolaccept reports txns in packages that exceed max feerate") @@ -304,7 +308,8 @@ class RPCPackagesTest(BitcoinTestFramework): assert testres_rbf_single[0]["allowed"] testres_rbf_package = self.independent_txns_testres_blank + [{ "txid": replacement_tx["txid"], "wtxid": replacement_tx["wtxid"], "allowed": False, - "reject-reason": "bip125-replacement-disallowed" + "reject-reason": "bip125-replacement-disallowed", + "reject-details": "bip125-replacement-disallowed" }] self.assert_testres_equal(self.independent_txns_hex + [replacement_tx["hex"]], testres_rbf_package) @@ -369,8 +374,16 @@ class RPCPackagesTest(BitcoinTestFramework): def test_submitpackage(self): node = self.nodes[0] - self.log.info("Submitpackage valid packages with 1 child and some number of parents") - for num_parents in [1, 2, 24]: + self.log.info("Submitpackage only allows valid hex inputs") + valid_tx_list = self.wallet.create_self_transfer_chain(chain_length=2) + hex_list = [valid_tx_list[0]["hex"][:-1] + 'X', valid_tx_list[1]["hex"]] + txid_list = [valid_tx_list[0]["txid"], valid_tx_list[1]["txid"]] + assert_raises_rpc_error(-22, "TX decode failed:", node.submitpackage, hex_list) + assert txid_list[0] not in node.getrawmempool() + assert txid_list[1] not in node.getrawmempool() + + self.log.info("Submitpackage valid packages with 1 child and some number of parents (or none)") + for num_parents in [0, 1, 2, 24]: self.test_submit_child_with_parents(num_parents, False) self.test_submit_child_with_parents(num_parents, True) @@ -381,10 +394,9 @@ class RPCPackagesTest(BitcoinTestFramework): assert_raises_rpc_error(-25, "package topology disallowed", node.submitpackage, chain_hex) assert_equal(legacy_pool, node.getrawmempool()) - assert_raises_rpc_error(-8, f"Array must contain between 2 and {MAX_PACKAGE_COUNT} transactions.", node.submitpackage, []) - assert_raises_rpc_error(-8, f"Array must contain between 2 and {MAX_PACKAGE_COUNT} transactions.", node.submitpackage, [chain_hex[0]] * 1) + assert_raises_rpc_error(-8, f"Array must contain between 1 and {MAX_PACKAGE_COUNT} transactions.", node.submitpackage, []) assert_raises_rpc_error( - -8, f"Array must contain between 2 and {MAX_PACKAGE_COUNT} transactions.", + -8, f"Array must contain between 1 and {MAX_PACKAGE_COUNT} transactions.", node.submitpackage, [chain_hex[0]] * (MAX_PACKAGE_COUNT + 1) ) diff --git a/test/functional/rpc_txoutproof.py b/test/functional/rpc_txoutproof.py index 387132b680..90572245d6 100755 --- a/test/functional/rpc_txoutproof.py +++ b/test/functional/rpc_txoutproof.py @@ -67,6 +67,10 @@ class MerkleBlockTest(BitcoinTestFramework): assert_equal(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid_spent], blockhash)), [txid_spent]) # We can't get the proof if we specify a non-existent block assert_raises_rpc_error(-5, "Block not found", self.nodes[0].gettxoutproof, [txid_spent], "0000000000000000000000000000000000000000000000000000000000000000") + # We can't get the proof if we only have the header of the specified block + block = self.generateblock(self.nodes[0], output="raw(55)", transactions=[], submit=False) + self.nodes[0].submitheader(block["hex"]) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", self.nodes[0].gettxoutproof, [txid_spent], block['hash']) # We can get the proof if the transaction is unspent assert_equal(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid_unspent])), [txid_unspent]) # We can get the proof if we provide a list of transactions and one of them is unspent. The ordering of the list should not matter. diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index 49eb64abad..75507877d5 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -22,13 +22,13 @@ import sys from typing import Optional -def call_with_auth(node, user, password): +def call_with_auth(node, user, password, method="getbestblockhash"): url = urllib.parse.urlparse(node.url) headers = {"Authorization": "Basic " + str_to_b64str('{}:{}'.format(user, password))} conn = http.client.HTTPConnection(url.hostname, url.port) conn.connect() - conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) + conn.request('POST', '/', f'{{"method": "{method}"}}', headers) resp = conn.getresponse() conn.close() return resp @@ -121,6 +121,25 @@ class HTTPBasicsTest(BitcoinTestFramework): for perm in ["owner", "group", "all"]: test_perm(perm) + def test_norpccookiefile(self, node0_cookie_path): + assert self.nodes[0].is_node_stopped(), "We expect previous test to stopped the node" + assert not node0_cookie_path.exists() + + self.log.info('Starting with -norpccookiefile') + # Start, but don't wait for RPC connection as TestNode.wait_for_rpc_connection() requires the cookie. + with self.nodes[0].busy_wait_for_debug_log([b'init message: Done loading']): + self.nodes[0].start(extra_args=["-norpccookiefile"]) + assert not node0_cookie_path.exists() + + self.log.info('Testing user/password authentication still works without cookie file') + assert_equal(200, call_with_auth(self.nodes[0], "rt", self.rtpassword).status) + # After confirming that we could log in, check that cookie file does not exist. + assert not node0_cookie_path.exists() + + # Need to shut down in slightly unorthodox way since cookie auth can't be used + assert_equal(200, call_with_auth(self.nodes[0], "rt", self.rtpassword, method="stop").status) + self.nodes[0].wait_until_stopped() + def run_test(self): self.conf_setup() self.log.info('Check correctness of the rpcauth config option') @@ -166,11 +185,19 @@ class HTTPBasicsTest(BitcoinTestFramework): self.stop_node(0) self.log.info('Check that failure to write cookie file will abort the node gracefully') - (self.nodes[0].chain_path / ".cookie.tmp").mkdir() + cookie_path = self.nodes[0].chain_path / ".cookie" + cookie_path_tmp = self.nodes[0].chain_path / ".cookie.tmp" + cookie_path_tmp.mkdir() self.nodes[0].assert_start_raises_init_error(expected_msg=init_error) + cookie_path_tmp.rmdir() + assert not cookie_path.exists() + self.restart_node(0) + assert cookie_path.exists() + self.stop_node(0) self.test_rpccookieperms() + self.test_norpccookiefile(cookie_path) if __name__ == '__main__': HTTPBasicsTest(__file__).main() diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index a357ae4d34..37fd5ae568 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -188,7 +188,12 @@ class AuthServiceProxy(): {'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)}, http_response.status) - responsedata = http_response.read().decode('utf8') + data = http_response.read() + try: + responsedata = data.decode('utf8') + except UnicodeDecodeError as e: + raise JSONRPCException({ + 'code': -342, 'message': f'Cannot decode response in utf8 format, content: {data}, exception: {e}'}) response = json.loads(responsedata, parse_float=decimal.Decimal) elapsed = time.time() - req_start_time if "error" in response and response["error"] is None: diff --git a/test/functional/test_framework/mempool_util.py b/test/functional/test_framework/mempool_util.py index 148cc935ed..0e9c821e2e 100644 --- a/test/functional/test_framework/mempool_util.py +++ b/test/functional/test_framework/mempool_util.py @@ -8,6 +8,7 @@ from decimal import Decimal from .blocktools import ( COINBASE_MATURITY, ) +from .messages import CTransaction from .util import ( assert_equal, assert_greater_than, @@ -18,15 +19,29 @@ from .wallet import ( MiniWallet, ) +ORPHAN_TX_EXPIRE_TIME = 1200 -def fill_mempool(test_framework, node): +def assert_mempool_contents(test_framework, node, expected=None, sync=True): + """Assert that all transactions in expected are in the mempool, + and no additional ones exist. 'expected' is an array of + CTransaction objects + """ + if sync: + test_framework.sync_mempools() + if not expected: + expected = [] + assert_equal(len(expected), len(set(expected))) + mempool = node.getrawmempool(verbose=False) + assert_equal(len(mempool), len(expected)) + for tx in expected: + assert tx.rehash() in mempool + + +def fill_mempool(test_framework, node, *, tx_sync_fun=None): """Fill mempool until eviction. Allows for simpler testing of scenarios with floating mempoolminfee > minrelay - Requires -datacarriersize=100000 and - -maxmempool=5. - It will not ensure mempools become synced as it - is based on a single node and assumes -minrelaytxfee + Requires -datacarriersize=100000 and -maxmempool=5 and assumes -minrelaytxfee is 1 sat/vbyte. To avoid unintentional tx dependencies, the mempool filling txs are created with a tagged ephemeral miniwallet instance. @@ -57,18 +72,25 @@ def fill_mempool(test_framework, node): tx_to_be_evicted_id = ephemeral_miniwallet.send_self_transfer( from_node=node, utxo_to_spend=confirmed_utxos.pop(0), fee_rate=relayfee)["txid"] + def send_batch(fee): + utxos = confirmed_utxos[:tx_batch_size] + create_lots_of_big_transactions(ephemeral_miniwallet, node, fee, tx_batch_size, txouts, utxos) + del confirmed_utxos[:tx_batch_size] + # Increase the tx fee rate to give the subsequent transactions a higher priority in the mempool # The tx has an approx. vsize of 65k, i.e. multiplying the previous fee rate (in sats/kvB) # by 130 should result in a fee that corresponds to 2x of that fee rate base_fee = relayfee * 130 + batch_fees = [(i + 1) * base_fee for i in range(num_of_batches)] test_framework.log.debug("Fill up the mempool with txs with higher fee rate") - with node.assert_debug_log(["rolling minimum fee bumped"]): - for batch_of_txid in range(num_of_batches): - fee = (batch_of_txid + 1) * base_fee - utxos = confirmed_utxos[:tx_batch_size] - create_lots_of_big_transactions(ephemeral_miniwallet, node, fee, tx_batch_size, txouts, utxos) - del confirmed_utxos[:tx_batch_size] + for fee in batch_fees[:-3]: + send_batch(fee) + tx_sync_fun() if tx_sync_fun else test_framework.sync_mempools() # sync before any eviction + assert_equal(node.getmempoolinfo()["mempoolminfee"], Decimal("0.00001000")) + for fee in batch_fees[-3:]: + send_batch(fee) + tx_sync_fun() if tx_sync_fun else test_framework.sync_mempools() # sync after all evictions test_framework.log.debug("The tx should be evicted by now") # The number of transactions created should be greater than the ones present in the mempool @@ -79,3 +101,8 @@ def fill_mempool(test_framework, node): test_framework.log.debug("Check that mempoolminfee is larger than minrelaytxfee") assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) + +def tx_in_orphanage(node, tx: CTransaction) -> bool: + """Returns true if the transaction is in the orphanage.""" + found = [o for o in node.getorphantxs(verbosity=1) if o["txid"] == tx.rehash() and o["wtxid"] == tx.getwtxid()] + return len(found) == 1 diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 1f566a1348..b4a5d9d5ef 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -327,7 +327,7 @@ class CAddress: elif self.net == self.NET_CJDNS: self.ip = socket.inet_ntop(socket.AF_INET6, addr_bytes) else: - raise Exception(f"Address type not supported") + raise Exception("Address type not supported") self.port = int.from_bytes(f.read(2), "big") @@ -354,7 +354,7 @@ class CAddress: elif self.net == self.NET_CJDNS: r += socket.inet_pton(socket.AF_INET6, self.ip) else: - raise Exception(f"Address type not supported") + raise Exception("Address type not supported") r += self.port.to_bytes(2, "big") return r diff --git a/test/functional/test_framework/netutil.py b/test/functional/test_framework/netutil.py index 08d41fe97f..f6acf926fa 100644 --- a/test/functional/test_framework/netutil.py +++ b/test/functional/test_framework/netutil.py @@ -167,3 +167,10 @@ def test_unix_socket(): return False else: return True + +def format_addr_port(addr, port): + '''Return either "addr:port" or "[addr]:port" based on whether addr looks like an IPv6 address.''' + if ":" in addr: + return f"[{addr}]:{port}" + else: + return f"{addr}:{port}" diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 4f1265eb54..523e1bd068 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2022 The Bitcoin Core developers +# Copyright (c) 2010-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test objects for interacting with a bitcoind node over the p2p protocol. @@ -188,6 +188,7 @@ class P2PConnection(asyncio.Protocol): self.on_connection_send_msg = None self.recvbuf = b"" self.magic_bytes = MAGIC_BYTES[net] + self.p2p_connected_to_node = dstport != 0 def peer_connect(self, dstaddr, dstport, *, net, timeout_factor, supports_v2_p2p): self.peer_connect_helper(dstaddr, dstport, net, timeout_factor) @@ -217,7 +218,12 @@ class P2PConnection(asyncio.Protocol): def connection_made(self, transport): """asyncio callback when a connection is opened.""" assert not self._transport - logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport)) + info = transport.get_extra_info("socket") + us = info.getsockname() + them = info.getpeername() + logger.debug(f"Connected: us={us[0]}:{us[1]}, them={them[0]}:{them[1]}") + self.dstaddr = them[0] + self.dstport = them[1] self._transport = transport # in an inbound connection to the TestNode with P2PConnection as the initiator, [TestNode <---- P2PConnection] # send the initial handshake immediately @@ -363,7 +369,7 @@ class P2PConnection(asyncio.Protocol): self.on_message(t) except Exception as e: if not self.reconnect: - logger.exception('Error reading message:', repr(e)) + logger.exception(f"Error reading message: {repr(e)}") raise def on_message(self, message): @@ -579,13 +585,13 @@ class P2PInterface(P2PConnection): # Connection helper methods - def wait_until(self, test_function_in, *, timeout=60, check_connected=True): + def wait_until(self, test_function_in, *, timeout=60, check_connected=True, check_interval=0.05): def test_function(): if check_connected: assert self.is_connected return test_function_in() - wait_until_helper_internal(test_function, timeout=timeout, lock=p2p_lock, timeout_factor=self.timeout_factor) + wait_until_helper_internal(test_function, timeout=timeout, lock=p2p_lock, timeout_factor=self.timeout_factor, check_interval=check_interval) def wait_for_connect(self, *, timeout=60): test_function = lambda: self.is_connected @@ -653,7 +659,7 @@ class P2PInterface(P2PConnection): def test_function(): last_getheaders = self.last_message.pop("getheaders", None) if block_hash is None: - return last_getheaders + return last_getheaders if last_getheaders is None: return False return block_hash == last_getheaders.locator.vHave[0] @@ -803,12 +809,13 @@ class P2PDataStore(P2PInterface): self.getdata_requests = [] def on_getdata(self, message): - """Check for the tx/block in our stores and if found, reply with an inv message.""" + """Check for the tx/block in our stores and if found, reply with MSG_TX or MSG_BLOCK.""" for inv in message.inv: self.getdata_requests.append(inv.hash) - if (inv.type & MSG_TYPE_MASK) == MSG_TX and inv.hash in self.tx_store.keys(): + invtype = inv.type & MSG_TYPE_MASK + if (invtype == MSG_TX or invtype == MSG_WTX) and inv.hash in self.tx_store.keys(): self.send_message(msg_tx(self.tx_store[inv.hash])) - elif (inv.type & MSG_TYPE_MASK) == MSG_BLOCK and inv.hash in self.block_store.keys(): + elif invtype == MSG_BLOCK and inv.hash in self.block_store.keys(): self.send_message(msg_block(self.block_store[inv.hash])) else: logger.debug('getdata message type {} received.'.format(hex(inv.type))) diff --git a/test/functional/test_framework/socks5.py b/test/functional/test_framework/socks5.py index 0ca06a7396..0cd16a3ff5 100644 --- a/test/functional/test_framework/socks5.py +++ b/test/functional/test_framework/socks5.py @@ -4,11 +4,16 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Dummy Socks5 server for testing.""" +import select import socket import threading import queue import logging +from .netutil import ( + format_addr_port +) + logger = logging.getLogger("TestFramework.socks5") # Protocol constants @@ -32,6 +37,42 @@ def recvall(s, n): n -= len(d) return rv +def sendall(s, data): + """Send all data to a socket, or fail.""" + sent = 0 + while sent < len(data): + _, wlist, _ = select.select([], [s], []) + if len(wlist) > 0: + n = s.send(data[sent:]) + if n == 0: + raise IOError('send() on socket returned 0') + sent += n + +def forward_sockets(a, b): + """Forward data received on socket a to socket b and vice versa, until EOF is received on one of the sockets.""" + # Mark as non-blocking so that we do not end up in a deadlock-like situation + # where we block and wait on data from `a` while there is data ready to be + # received on `b` and forwarded to `a`. And at the same time the application + # at `a` is not sending anything because it waits for the data from `b` to + # respond. + a.setblocking(False) + b.setblocking(False) + sockets = [a, b] + done = False + while not done: + rlist, _, xlist = select.select(sockets, [], sockets) + if len(xlist) > 0: + raise IOError('Exceptional condition on socket') + for s in rlist: + data = s.recv(4096) + if data is None or len(data) == 0: + done = True + break + if s == a: + sendall(b, data) + else: + sendall(a, data) + # Implementation classes class Socks5Configuration(): """Proxy configuration.""" @@ -41,6 +82,19 @@ class Socks5Configuration(): self.unauth = False # Support unauthenticated self.auth = False # Support authentication self.keep_alive = False # Do not automatically close connections + # This function is called whenever a new connection arrives to the proxy + # and it decides where the connection is redirected to. It is passed: + # - the address the client requested to connect to + # - the port the client requested to connect to + # It is supposed to return an object like: + # { + # "actual_to_addr": "127.0.0.1" + # "actual_to_port": 28276 + # } + # or None. + # If it returns an object then the connection is redirected to actual_to_addr:actual_to_port. + # If it returns None, or destinations_factory itself is None then the connection is closed. + self.destinations_factory = None class Socks5Command(): """Information about an incoming socks5 command.""" @@ -60,7 +114,7 @@ class Socks5Connection(): self.conn = conn def handle(self): - """Handle socks5 request according to RFC192.""" + """Handle socks5 request according to RFC1928.""" try: # Verify socks version ver = recvall(self.conn, 1)[0] @@ -117,6 +171,22 @@ class Socks5Connection(): cmdin = Socks5Command(cmd, atyp, addr, port, username, password) self.serv.queue.put(cmdin) logger.debug('Proxy: %s', cmdin) + + requested_to_addr = addr.decode("utf-8") + requested_to = format_addr_port(requested_to_addr, port) + + if self.serv.conf.destinations_factory is not None: + dest = self.serv.conf.destinations_factory(requested_to_addr, port) + if dest is not None: + logger.debug(f"Serving connection to {requested_to}, will redirect it to " + f"{dest['actual_to_addr']}:{dest['actual_to_port']} instead") + with socket.create_connection((dest["actual_to_addr"], dest["actual_to_port"])) as conn_to: + forward_sockets(self.conn, conn_to) + else: + logger.debug(f"Can't serve the connection to {requested_to}: the destinations factory returned None") + else: + logger.debug(f"Can't serve the connection to {requested_to}: no destinations factory") + # Fall through to disconnect except Exception as e: logger.exception("socks5 request handling failed.") @@ -124,6 +194,8 @@ class Socks5Connection(): finally: if not self.serv.keep_alive: self.conn.close() + else: + logger.debug("Keeping client connection alive") class Socks5Server(): def __init__(self, conf): diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 49212eb019..7e8c40cf16 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -129,7 +129,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): try: self.setup() - self.run_test() + if self.options.test_methods: + self.run_test_methods() + else: + self.run_test() + except JSONRPCException: self.log.exception("JSONRPC error") self.success = TestStatus.FAILED @@ -155,6 +159,13 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): exit_code = self.shutdown() sys.exit(exit_code) + def run_test_methods(self): + for method_name in self.options.test_methods: + self.log.info(f"Attempting to execute method: {method_name}") + method = getattr(self, method_name) + method() + self.log.info(f"Method '{method_name}' executed successfully.") + def parse_args(self, test_file): previous_releases_path = os.getenv("PREVIOUS_RELEASES_DIR") or os.getcwd() + "/releases" parser = argparse.ArgumentParser(usage="%(prog)s [options]") @@ -194,6 +205,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="use BIP324 v2 connections between all nodes by default") parser.add_argument("--v1transport", dest="v1transport", default=False, action="store_true", help="Explicitly use v1 transport (can be used to overwrite global --v2transport option)") + parser.add_argument("--test_methods", dest="test_methods", nargs='*', + help="Run specified test methods sequentially instead of the full test. Use only for methods that do not depend on any context set up in run_test or other methods.") self.add_options(parser) # Running TestShell in a Jupyter notebook causes an additional -f argument @@ -349,7 +362,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir)) self.log.error("") self.log.error("If this failure happened unexpectedly or intermittently, please file a bug and provide a link or upload of the combined log.") - self.log.error(self.config['environment']['PACKAGE_BUGREPORT']) + self.log.error(self.config['environment']['CLIENT_BUGREPORT']) self.log.error("") exit_code = TEST_EXIT_FAILED # Logging.shutdown will not remove stream- and filehandlers, so we must @@ -787,8 +800,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.sync_blocks(nodes) self.sync_mempools(nodes) - def wait_until(self, test_function, timeout=60): - return wait_until_helper_internal(test_function, timeout=timeout, timeout_factor=self.options.timeout_factor) + def wait_until(self, test_function, timeout=60, check_interval=0.05): + return wait_until_helper_internal(test_function, timeout=timeout, timeout_factor=self.options.timeout_factor, check_interval=check_interval) # Private helper methods. These should not be accessed by the subclass test scripts. diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 60ca9269a5..d896a421d4 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -48,6 +48,7 @@ BITCOIND_PROC_WAIT_TIMEOUT = 60 NUM_XOR_BYTES = 8 # The null blocks key (all 0s) NULL_BLK_XOR_KEY = bytes([0] * NUM_XOR_BYTES) +BITCOIN_PID_FILENAME_DEFAULT = "bitcoind.pid" class FailedToStartError(Exception): @@ -220,10 +221,9 @@ class TestNode(): extra_args = self.extra_args # If listening and no -bind is given, then bitcoind would bind P2P ports on - # 0.0.0.0:P and 127.0.0.1:18445 (for incoming Tor connections), where P is + # 0.0.0.0:P and 127.0.0.1:P+1 (for incoming Tor connections), where P is # a unique port chosen by the test framework and configured as port=P in - # bitcoin.conf. To avoid collisions on 127.0.0.1:18445, change it to - # 127.0.0.1:tor_port(). + # bitcoin.conf. To avoid collisions, change it to 127.0.0.1:tor_port(). will_listen = all(e != "-nolisten" and e != "-listen=0" for e in extra_args) has_explicit_bind = self.has_explicit_bind or any(e.startswith("-bind=") for e in extra_args) if will_listen and not has_explicit_bind: @@ -714,7 +714,6 @@ class TestNode(): if supports_v2_p2p is None: supports_v2_p2p = self.use_v2transport - p2p_conn.p2p_connected_to_node = True if self.use_v2transport: kwargs['services'] = kwargs.get('services', P2P_SERVICES) | NODE_P2P_V2 supports_v2_p2p = self.use_v2transport and supports_v2_p2p @@ -781,7 +780,6 @@ class TestNode(): self.log.debug("Connecting to %s:%d %s" % (address, port, connection_type)) self.addconnection('%s:%d' % (address, port), connection_type, advertise_v2_p2p) - p2p_conn.p2p_connected_to_node = False if supports_v2_p2p is None: supports_v2_p2p = self.use_v2transport if advertise_v2_p2p is None: @@ -838,8 +836,8 @@ class TestNode(): self.mocktime += seconds self.setmocktime(self.mocktime) - def wait_until(self, test_function, timeout=60): - return wait_until_helper_internal(test_function, timeout=timeout, timeout_factor=self.timeout_factor) + def wait_until(self, test_function, timeout=60, check_interval=0.05): + return wait_until_helper_internal(test_function, timeout=timeout, timeout_factor=self.timeout_factor, check_interval=check_interval) class TestNodeCLIAttr: diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index ce68de7eaa..14930ef671 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -268,7 +268,28 @@ def satoshi_round(amount: Union[int, float, str], *, rounding: str) -> Decimal: return Decimal(amount).quantize(SATOSHI_PRECISION, rounding=rounding) -def wait_until_helper_internal(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None, timeout_factor=1.0): +def ensure_for(*, duration, f, check_interval=0.2): + """Check if the predicate keeps returning True for duration. + + check_interval can be used to configure the wait time between checks. + Setting check_interval to 0 will allow to have two checks: one in the + beginning and one after duration. + """ + # If check_interval is 0 or negative or larger than duration, we fall back + # to checking once in the beginning and once at the end of duration + if check_interval <= 0 or check_interval > duration: + check_interval = duration + time_end = time.time() + duration + predicate_source = "''''\n" + inspect.getsource(f) + "'''" + while True: + if not f(): + raise AssertionError(f"Predicate {predicate_source} became false within {duration} seconds") + if time.time() > time_end: + return + time.sleep(check_interval) + + +def wait_until_helper_internal(predicate, *, timeout=60, lock=None, timeout_factor=1.0, check_interval=0.05): """Sleep until the predicate resolves to be True. Warning: Note that this method is not recommended to be used in tests as it is @@ -277,13 +298,10 @@ def wait_until_helper_internal(predicate, *, attempts=float('inf'), timeout=floa properly scaled. Furthermore, `wait_until()` from `P2PInterface` class in `p2p.py` has a preset lock. """ - if attempts == float('inf') and timeout == float('inf'): - timeout = 60 timeout = timeout * timeout_factor - attempt = 0 time_end = time.time() + timeout - while attempt < attempts and time.time() < time_end: + while time.time() < time_end: if lock: with lock: if predicate(): @@ -291,17 +309,12 @@ def wait_until_helper_internal(predicate, *, attempts=float('inf'), timeout=floa else: if predicate(): return - attempt += 1 - time.sleep(0.05) + time.sleep(check_interval) # Print the cause of the timeout predicate_source = "''''\n" + inspect.getsource(predicate) + "'''" logger.error("wait_until() failed. Predicate: {}".format(predicate_source)) - if attempt >= attempts: - raise AssertionError("Predicate {} not true after {} attempts".format(predicate_source, attempts)) - elif time.time() >= time_end: - raise AssertionError("Predicate {} not true after {} seconds".format(predicate_source, timeout)) - raise RuntimeError('Unreachable') + raise AssertionError("Predicate {} not true after {} seconds".format(predicate_source, timeout)) def sha256sum_file(filename): @@ -433,7 +446,6 @@ def write_config(config_path, *, n, chain, extra_config="", disable_autoconnect= # in tests. f.write("peertimeout=999999999\n") f.write("printtoconsole=0\n") - f.write("upnp=0\n") f.write("natpmp=0\n") f.write("shrinkdebugfile=0\n") f.write("deprecatedrpc=create_bdb\n") # Required to run the tests @@ -441,6 +453,17 @@ def write_config(config_path, *, n, chain, extra_config="", disable_autoconnect= f.write("unsafesqlitesync=1\n") if disable_autoconnect: f.write("connect=0\n") + # Limit max connections to mitigate test failures on some systems caused by the warning: + # "Warning: Reducing -maxconnections from <...> to <...> due to system limitations". + # The value is calculated as follows: + # available_fds = 256 // Same as FD_SETSIZE on NetBSD. + # MIN_CORE_FDS = 151 // Number of file descriptors required for core functionality. + # MAX_ADDNODE_CONNECTIONS = 8 // Maximum number of -addnode outgoing nodes. + # nBind == 3 // Maximum number of bound interfaces used in a test. + # + # min_required_fds = MIN_CORE_FDS + MAX_ADDNODE_CONNECTIONS + nBind = 151 + 8 + 3 = 162; + # nMaxConnections = available_fds - min_required_fds = 256 - 161 = 94; + f.write("maxconnections=94\n") f.write(extra_config) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index f3713f297e..1cef714705 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -7,7 +7,6 @@ from copy import deepcopy from decimal import Decimal from enum import Enum -import math from typing import ( Any, Optional, @@ -35,7 +34,6 @@ from test_framework.messages import ( CTxOut, hash256, ser_compact_size, - WITNESS_SCALE_FACTOR, ) from test_framework.script import ( CScript, @@ -119,20 +117,18 @@ class MiniWallet: def _create_utxo(self, *, txid, vout, value, height, coinbase, confirmations): return {"txid": txid, "vout": vout, "value": value, "height": height, "coinbase": coinbase, "confirmations": confirmations} - def _bulk_tx(self, tx, target_weight): - """Pad a transaction with extra outputs until it reaches a target weight (or higher). + def _bulk_tx(self, tx, target_vsize): + """Pad a transaction with extra outputs until it reaches a target vsize. returns the tx """ tx.vout.append(CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN]))) - # determine number of needed padding bytes by converting weight difference to vbytes - dummy_vbytes = (target_weight - tx.get_weight() + 3) // 4 + # determine number of needed padding bytes + dummy_vbytes = target_vsize - tx.get_vsize() # compensate for the increase of the compact-size encoded script length # (note that the length encoding of the unpadded output script needs one byte) dummy_vbytes -= len(ser_compact_size(dummy_vbytes)) - 1 tx.vout[-1].scriptPubKey = CScript([OP_RETURN] + [OP_1] * dummy_vbytes) - # Actual weight should be at most 3 higher than target weight - assert_greater_than_or_equal(tx.get_weight(), target_weight) - assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) + assert_equal(tx.get_vsize(), target_vsize) def get_balance(self): return sum(u['value'] for u in self._utxos) @@ -309,7 +305,7 @@ class MiniWallet: locktime=0, sequence=0, fee_per_output=1000, - target_weight=0, + target_vsize=0, confirmed_only=False, ): """ @@ -338,8 +334,8 @@ class MiniWallet: self.sign_tx(tx) - if target_weight: - self._bulk_tx(tx, target_weight) + if target_vsize: + self._bulk_tx(tx, target_vsize) txid = tx.rehash() return { @@ -364,7 +360,7 @@ class MiniWallet: fee_rate=Decimal("0.003"), fee=Decimal("0"), utxo_to_spend=None, - target_weight=0, + target_vsize=0, confirmed_only=False, **kwargs, ): @@ -379,20 +375,18 @@ class MiniWallet: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) else: assert False - if target_weight and not fee: # respect fee_rate if target weight is passed - # the actual weight might be off by 3 WUs, so calculate based on that (see self._bulk_tx) - max_actual_weight = target_weight + 3 - fee = get_fee(math.ceil(max_actual_weight / WITNESS_SCALE_FACTOR), fee_rate) + if target_vsize and not fee: # respect fee_rate if target vsize is passed + fee = get_fee(target_vsize, fee_rate) send_value = utxo_to_spend["value"] - (fee or (fee_rate * vsize / 1000)) # create tx tx = self.create_self_transfer_multi( utxos_to_spend=[utxo_to_spend], amount_per_output=int(COIN * send_value), - target_weight=target_weight, + target_vsize=target_vsize, **kwargs, ) - if not target_weight: + if not target_vsize: assert_equal(tx["tx"].get_vsize(), vsize) tx["new_utxo"] = tx.pop("new_utxos")[0] diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 3297a10699..85170b1045 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -51,23 +51,23 @@ except UnicodeDecodeError: CROSS = "x " CIRCLE = "o " -if platform.system() != 'Windows' or sys.getwindowsversion() >= (10, 0, 14393): #type:ignore - if platform.system() == 'Windows': - import ctypes - kernel32 = ctypes.windll.kernel32 # type: ignore - ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 - STD_OUTPUT_HANDLE = -11 - STD_ERROR_HANDLE = -12 - # Enable ascii color control to stdout - stdout = kernel32.GetStdHandle(STD_OUTPUT_HANDLE) - stdout_mode = ctypes.c_int32() - kernel32.GetConsoleMode(stdout, ctypes.byref(stdout_mode)) - kernel32.SetConsoleMode(stdout, stdout_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) - # Enable ascii color control to stderr - stderr = kernel32.GetStdHandle(STD_ERROR_HANDLE) - stderr_mode = ctypes.c_int32() - kernel32.GetConsoleMode(stderr, ctypes.byref(stderr_mode)) - kernel32.SetConsoleMode(stderr, stderr_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) +if platform.system() == 'Windows': + import ctypes + kernel32 = ctypes.windll.kernel32 # type: ignore + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 + STD_OUTPUT_HANDLE = -11 + STD_ERROR_HANDLE = -12 + # Enable ascii color control to stdout + stdout = kernel32.GetStdHandle(STD_OUTPUT_HANDLE) + stdout_mode = ctypes.c_int32() + kernel32.GetConsoleMode(stdout, ctypes.byref(stdout_mode)) + kernel32.SetConsoleMode(stdout, stdout_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) + # Enable ascii color control to stderr + stderr = kernel32.GetStdHandle(STD_ERROR_HANDLE) + stderr_mode = ctypes.c_int32() + kernel32.GetConsoleMode(stderr, ctypes.byref(stderr_mode)) + kernel32.SetConsoleMode(stderr, stderr_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) +else: # primitive formatting on supported # terminal via ANSI escape sequences: DEFAULT = ('\033[0m', '\033[0m') @@ -96,10 +96,17 @@ BASE_SCRIPTS = [ 'feature_fee_estimation.py', 'feature_taproot.py', 'feature_block.py', + 'mempool_ephemeral_dust.py', + 'wallet_conflicts.py --legacy-wallet', + 'wallet_conflicts.py --descriptors', + 'p2p_opportunistic_1p1c.py', + 'p2p_node_network_limited.py --v1transport', + 'p2p_node_network_limited.py --v2transport', # vv Tests less than 2m vv 'mining_getblocktemplate_longpoll.py', 'p2p_segwit.py', 'feature_maxuploadtarget.py', + 'feature_assumeutxo.py', 'mempool_updatefromblock.py', 'mempool_persist.py --descriptors', # vv Tests less than 60s vv @@ -143,6 +150,7 @@ BASE_SCRIPTS = [ 'p2p_feefilter.py', 'feature_csv_activation.py', 'p2p_sendheaders.py', + 'feature_config_args.py', 'wallet_listtransactions.py --legacy-wallet', 'wallet_listtransactions.py --descriptors', 'wallet_miniscript.py --descriptors', @@ -157,6 +165,7 @@ BASE_SCRIPTS = [ 'wallet_importmulti.py --legacy-wallet', 'mempool_limit.py', 'rpc_txoutproof.py', + 'rpc_orphans.py', 'wallet_listreceivedby.py --legacy-wallet', 'wallet_listreceivedby.py --descriptors', 'wallet_abandonconflict.py --legacy-wallet', @@ -195,7 +204,6 @@ BASE_SCRIPTS = [ 'rpc_getchaintips.py', 'rpc_misc.py', 'p2p_1p1c_network.py', - 'p2p_opportunistic_1p1c.py', 'interface_rest.py', 'mempool_spend_coinbase.py', 'wallet_avoid_mixing_output_types.py --descriptors', @@ -210,8 +218,6 @@ BASE_SCRIPTS = [ 'wallet_reindex.py --legacy-wallet', 'wallet_reindex.py --descriptors', 'wallet_reorgsrestore.py', - 'wallet_conflicts.py --legacy-wallet', - 'wallet_conflicts.py --descriptors', 'interface_http.py', 'interface_rpc.py', 'interface_usdt_coinselection.py', @@ -335,6 +341,7 @@ BASE_SCRIPTS = [ 'feature_minchainwork.py', 'rpc_estimatefee.py', 'rpc_getblockstats.py', + 'feature_port.py', 'feature_bind_port_externalip.py', 'wallet_create_tx.py --legacy-wallet', 'wallet_send.py --legacy-wallet', @@ -354,7 +361,6 @@ BASE_SCRIPTS = [ 'wallet_coinbase_category.py --descriptors', 'feature_filelock.py', 'feature_loadblock.py', - 'feature_assumeutxo.py', 'wallet_assumeutxo.py --descriptors', 'p2p_dos_header_tree.py', 'p2p_add_connections.py', @@ -375,6 +381,7 @@ BASE_SCRIPTS = [ 'rpc_deriveaddresses.py --usecli', 'p2p_ping.py', 'p2p_tx_privacy.py', + 'rpc_getdescriptoractivity.py', 'rpc_scanblocks.py', 'p2p_sendtxrcncl.py', 'rpc_scantxoutset.py', @@ -385,15 +392,12 @@ BASE_SCRIPTS = [ 'feature_coinstatsindex.py', 'wallet_orphanedreward.py', 'wallet_timelock.py', - 'p2p_node_network_limited.py --v1transport', - 'p2p_node_network_limited.py --v2transport', 'p2p_permissions.py', 'feature_blocksdir.py', 'wallet_startup.py', 'feature_remove_pruned_files_on_startup.py', 'p2p_i2p_ports.py', 'p2p_i2p_sessions.py', - 'feature_config_args.py', 'feature_presegwit_node_upgrade.py', 'feature_settings.py', 'rpc_getdescriptorinfo.py', @@ -446,8 +450,8 @@ def main(): help="Leave bitcoinds and test.* datadir on exit or error") parser.add_argument('--resultsfile', '-r', help='store test results (as CSV) to the provided file') - args, unknown_args = parser.parse_known_args() + fail_on_warn = args.ci if not args.ansi: global DEFAULT, BOLD, GREEN, RED DEFAULT = ("", "") @@ -488,7 +492,7 @@ def main(): if not enable_bitcoind: print("No functional tests to run.") - print("Rerun ./configure with --with-daemon and then make") + print("Re-compile with the -DBUILD_DAEMON=ON build option") sys.exit(1) # Build list of tests @@ -524,8 +528,12 @@ def main(): # Remove the test cases that the user has explicitly asked to exclude. # The user can specify a test case with or without the .py extension. if args.exclude: + def print_warning_missing_test(test_name): - print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], test_name)) + print("{}WARNING!{} Test '{}' not found in current test list. Check the --exclude list.".format(BOLD[1], BOLD[0], test_name)) + if fail_on_warn: + sys.exit(1) + def remove_tests(exclude_list): if not exclude_list: print_warning_missing_test(exclude_test) @@ -562,7 +570,7 @@ def main(): f"A minimum of {MIN_NO_CLEANUP_SPACE // (1024 * 1024 * 1024)} GB of free space is required.") passon_args.append("--nocleanup") - check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=args.ci) + check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=fail_on_warn) check_script_prefixes() if not args.keepcache: @@ -871,7 +879,6 @@ def check_script_list(*, src_dir, fail_on_warn): if len(missed_tests) != 0: print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests))) if fail_on_warn: - # On CI this warning is an error to prevent merging incomplete commits into master sys.exit(1) diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 784a769882..86292c0f0b 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -203,7 +203,7 @@ class ToolWalletTest(BitcoinTestFramework): locked_dir = self.nodes[0].wallets_path error = 'Error initializing wallet database environment "{}"!'.format(locked_dir) if self.options.descriptors: - error = f"SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['PACKAGE_NAME']}?" + error = f"SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['CLIENT_NAME']}?" self.assert_raises_tool_error( error, '-wallet=' + self.default_wallet_name, diff --git a/test/functional/wallet_assumeutxo.py b/test/functional/wallet_assumeutxo.py index a5025a64e7..ac904d6ce4 100755 --- a/test/functional/wallet_assumeutxo.py +++ b/test/functional/wallet_assumeutxo.py @@ -11,10 +11,13 @@ See feature_assumeutxo.py for background. - TODO: test loading a wallet (backup) on a pruned node """ +from test_framework.address import address_to_scriptpubkey from test_framework.test_framework import BitcoinTestFramework +from test_framework.messages import COIN from test_framework.util import ( assert_equal, assert_raises_rpc_error, + ensure_for, ) from test_framework.wallet import MiniWallet @@ -32,17 +35,18 @@ class AssumeutxoTest(BitcoinTestFramework): def set_test_params(self): """Use the pregenerated, deterministic chain up to height 199.""" - self.num_nodes = 2 + self.num_nodes = 3 self.rpc_timeout = 120 self.extra_args = [ [], [], + [], ] def setup_network(self): """Start with the nodes disconnected so that one can generate a snapshot including blocks the other hasn't yet seen.""" - self.add_nodes(2) + self.add_nodes(3) self.start_nodes(extra_args=self.extra_args) def run_test(self): @@ -55,6 +59,7 @@ class AssumeutxoTest(BitcoinTestFramework): """ n0 = self.nodes[0] n1 = self.nodes[1] + n2 = self.nodes[2] self.mini_wallet = MiniWallet(n0) @@ -62,8 +67,16 @@ class AssumeutxoTest(BitcoinTestFramework): for n in self.nodes: n.setmocktime(n.getblockheader(n.getbestblockhash())['time']) + # Create a wallet that we will create a backup for later (at snapshot height) n0.createwallet('w') w = n0.get_wallet_rpc("w") + w_address = w.getnewaddress() + + # Create another wallet and backup now (before snapshot height) + n0.createwallet('w2') + w2 = n0.get_wallet_rpc("w2") + w2_address = w2.getnewaddress() + w2.backupwallet("backup_w2.dat") # Generate a series of blocks that `n0` will have in the snapshot, # but that n1 doesn't yet see. In order for the snapshot to activate, @@ -78,12 +91,15 @@ class AssumeutxoTest(BitcoinTestFramework): # make n1 aware of the new header, but don't give it the block. n1.submitheader(newblock) + n2.submitheader(newblock) # Ensure everyone is seeing the same headers. for n in self.nodes: assert_equal(n.getblockchaininfo()[ "headers"], SNAPSHOT_BASE_HEIGHT) + # This backup is created at the snapshot height, so it's + # not part of the background sync anymore w.backupwallet("backup_w.dat") self.log.info("-- Testing assumeutxo") @@ -103,10 +119,17 @@ class AssumeutxoTest(BitcoinTestFramework): # Mine more blocks on top of the snapshot that n1 hasn't yet seen. This # will allow us to test n1's sync-to-tip on top of a snapshot. - self.generate(n0, nblocks=100, sync_fun=self.no_op) + w_skp = address_to_scriptpubkey(w_address) + w2_skp = address_to_scriptpubkey(w2_address) + for i in range(100): + if i % 3 == 0: + self.mini_wallet.send_to(from_node=n0, scriptPubKey=w_skp, amount=1 * COIN) + self.mini_wallet.send_to(from_node=n0, scriptPubKey=w2_skp, amount=10 * COIN) + self.generate(n0, nblocks=1, sync_fun=self.no_op) assert_equal(n0.getblockcount(), FINAL_HEIGHT) assert_equal(n1.getblockcount(), START_HEIGHT) + assert_equal(n2.getblockcount(), START_HEIGHT) assert_equal(n0.getblockchaininfo()["blocks"], FINAL_HEIGHT) @@ -126,8 +149,13 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(n1.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT) - self.log.info("Backup can't be loaded during background sync") - assert_raises_rpc_error(-4, "Wallet loading failed. Error loading wallet. Wallet requires blocks to be downloaded, and software does not currently support loading wallets while blocks are being downloaded out of order when using assumeutxo snapshots. Wallet should be able to load successfully after node sync reaches height 299", n1.restorewallet, "w", "backup_w.dat") + self.log.info("Backup from the snapshot height can be loaded during background sync") + n1.restorewallet("w", "backup_w.dat") + # Balance of w wallet is still still 0 because n1 has not synced yet + assert_equal(n1.getbalance(), 0) + + self.log.info("Backup from before the snapshot height can't be loaded during background sync") + assert_raises_rpc_error(-4, "Wallet loading failed. Error loading wallet. Wallet requires blocks to be downloaded, and software does not currently support loading wallets while blocks are being downloaded out of order when using assumeutxo snapshots. Wallet should be able to load successfully after node sync reaches height 299", n1.restorewallet, "w2", "backup_w2.dat") PAUSE_HEIGHT = FINAL_HEIGHT - 40 @@ -159,8 +187,22 @@ class AssumeutxoTest(BitcoinTestFramework): self.log.info("Ensuring background validation completes") self.wait_until(lambda: len(n1.getchainstates()['chainstates']) == 1) - self.log.info("Ensuring wallet can be restored from backup") - n1.restorewallet("w", "backup_w.dat") + self.log.info("Ensuring wallet can be restored from a backup that was created before the snapshot height") + n1.restorewallet("w2", "backup_w2.dat") + # Check balance of w2 wallet + assert_equal(n1.getbalance(), 340) + + # Check balance of w wallet after node is synced + n1.loadwallet("w") + w = n1.get_wallet_rpc("w") + assert_equal(w.getbalance(), 34) + + self.log.info("Check balance of a wallet that is active during snapshot completion") + n2.restorewallet("w", "backup_w.dat") + loaded = n2.loadtxoutset(dump_output['path']) + self.connect_nodes(0, 2) + self.wait_until(lambda: len(n2.getchainstates()['chainstates']) == 1) + ensure_for(duration=1, f=lambda: (n2.getbalance() == 34)) if __name__ == '__main__': diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index a639c34377..7c88f64dcf 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -140,6 +140,25 @@ class WalletBackupTest(BitcoinTestFramework): assert_raises_rpc_error(-36, error_message, node.restorewallet, wallet_name, backup_file) assert wallet_file.exists() + def test_pruned_wallet_backup(self): + self.log.info("Test loading backup on a pruned node when the backup was created close to the prune height of the restoring node") + node = self.nodes[3] + self.restart_node(3, ["-prune=1", "-fastprune=1"]) + # Ensure the chain tip is at height 214, because this test assume it is. + assert_equal(node.getchaintips()[0]["height"], 214) + # We need a few more blocks so we can actually get above an realistic + # minimal prune height + self.generate(node, 50, sync_fun=self.no_op) + # Backup created at block height 264 + node.backupwallet(node.datadir_path / 'wallet_pruned.bak') + # Generate more blocks so we can actually prune the older blocks + self.generate(node, 300, sync_fun=self.no_op) + # This gives us an actual prune height roughly in the range of 220 - 240 + node.pruneblockchain(250) + # The backup should be updated with the latest height (locator) for + # the backup to load successfully this close to the prune height + node.restorewallet('pruned', node.datadir_path / 'wallet_pruned.bak') + def run_test(self): self.log.info("Generating initial blockchain") self.generate(self.nodes[0], 1) @@ -242,6 +261,8 @@ class WalletBackupTest(BitcoinTestFramework): for sourcePath in sourcePaths: assert_raises_rpc_error(-4, "backup failed", self.nodes[0].backupwallet, sourcePath) + self.test_pruned_wallet_backup() + if __name__ == '__main__': WalletBackupTest(__file__).main() diff --git a/test/functional/wallet_backwards_compatibility.py b/test/functional/wallet_backwards_compatibility.py index e71283b928..62db32d806 100755 --- a/test/functional/wallet_backwards_compatibility.py +++ b/test/functional/wallet_backwards_compatibility.py @@ -33,7 +33,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 12 + self.num_nodes = 11 # Add new version after each release: self.extra_args = [ ["-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to mine blocks. noban for immediate tx relay @@ -47,7 +47,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.19.1 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1"], # v0.18.1 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1"], # v0.17.2 - ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1", "-wallet=wallet.dat"], # v0.16.3 ] self.wallet_names = [self.default_wallet_name] @@ -68,7 +67,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): 190100, 180100, 170200, - 160300, ]) self.start_nodes() @@ -133,18 +131,17 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): def run_test(self): node_miner = self.nodes[0] node_master = self.nodes[1] - node_v21 = self.nodes[self.num_nodes - 6] - node_v17 = self.nodes[self.num_nodes - 2] - node_v16 = self.nodes[self.num_nodes - 1] + node_v21 = self.nodes[self.num_nodes - 5] + node_v17 = self.nodes[self.num_nodes - 1] legacy_nodes = self.nodes[2:] # Nodes that support legacy wallets - legacy_only_nodes = self.nodes[-5:] # Nodes that only support legacy wallets - descriptors_nodes = self.nodes[2:-5] # Nodes that support descriptor wallets + legacy_only_nodes = self.nodes[-4:] # Nodes that only support legacy wallets + descriptors_nodes = self.nodes[2:-4] # Nodes that support descriptor wallets self.generatetoaddress(node_miner, COINBASE_MATURITY + 1, node_miner.getnewaddress()) # Sanity check the test framework: - res = node_v16.getblockchaininfo() + res = node_v17.getblockchaininfo() assert_equal(res['blocks'], COINBASE_MATURITY + 1) self.log.info("Test wallet backwards compatibility...") @@ -172,6 +169,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # Create another conflicting transaction using RBF tx3_id = node_master.sendtoaddress(return_address, 1) tx4_id = node_master.bumpfee(tx3_id)["txid"] + self.sync_mempools() # Abandon transaction, but don't confirm node_master.abandontransaction(tx3_id) @@ -215,9 +213,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # In descriptors wallet mode, run this test on the nodes that support descriptor wallets # In legacy wallets mode, run this test on the nodes that support legacy wallets for node in descriptors_nodes if self.options.descriptors else legacy_nodes: - if self.major_version_less_than(node, 17): - # loadwallet was introduced in v0.17.0 - continue self.log.info(f"- {node.version}") for wallet_name in ["w1", "w2", "w3"]: if self.major_version_less_than(node, 18) and wallet_name == "w3": @@ -290,15 +285,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): node_v17.assert_start_raises_init_error(["-wallet=w3"], "Error: Error loading w3: Wallet requires newer version of Bitcoin Core") self.start_node(node_v17.index) - # No wallet created in master can be opened in 0.16 - self.log.info("Test that wallets created in master are too new for 0.16") - self.stop_node(node_v16.index) - for wallet_name in ["w1", "w2", "w3"]: - if self.options.descriptors: - node_v16.assert_start_raises_init_error([f"-wallet={wallet_name}"], f"Error: {wallet_name} corrupt, salvage failed") - else: - node_v16.assert_start_raises_init_error([f"-wallet={wallet_name}"], f"Error: Error loading {wallet_name}: Wallet requires newer version of Bitcoin Core") - # When descriptors are enabled, w1 cannot be opened by 0.21 since it contains a taproot descriptor if self.options.descriptors: self.log.info("Test that 0.21 cannot open wallet containing tr() descriptors") diff --git a/test/functional/wallet_fast_rescan.py b/test/functional/wallet_fast_rescan.py index 4ac441516e..6cee0d3660 100755 --- a/test/functional/wallet_fast_rescan.py +++ b/test/functional/wallet_fast_rescan.py @@ -49,7 +49,7 @@ class WalletFastRescanTest(BitcoinTestFramework): assert_equal(len(descriptors), NUM_DESCRIPTORS) w.backupwallet(WALLET_BACKUP_FILENAME) - self.log.info(f"Create txs sending to end range address of each descriptor, triggering top-ups") + self.log.info("Create txs sending to end range address of each descriptor, triggering top-ups") for i in range(NUM_BLOCKS): self.log.info(f"Block {i+1}/{NUM_BLOCKS}") for desc_info in w.listdescriptors()['descriptors']: diff --git a/test/functional/wallet_inactive_hdchains.py b/test/functional/wallet_inactive_hdchains.py index 3b0c09c02b..1a2ea5b9ff 100755 --- a/test/functional/wallet_inactive_hdchains.py +++ b/test/functional/wallet_inactive_hdchains.py @@ -6,7 +6,6 @@ Test Inactive HD Chains. """ import shutil -import time from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework @@ -75,12 +74,13 @@ class InactiveHDChainsTest(BitcoinTestFramework): self.generate(self.nodes[0], 1) # Wait for the test wallet to see the transaction - while True: + def is_tx_available(txid): try: test_wallet.gettransaction(txid) - break + return True except JSONRPCException: - time.sleep(0.1) + return False + self.nodes[0].wait_until(lambda: is_tx_available(txid), timeout=10, check_interval=0.1) if encrypt: # The test wallet will not be able to generate the topped up keypool diff --git a/test/functional/wallet_migration.py b/test/functional/wallet_migration.py index 48ad8e4f2a..3a56050731 100755 --- a/test/functional/wallet_migration.py +++ b/test/functional/wallet_migration.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020-2022 The Bitcoin Core developers +# Copyright (c) 2020-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Migrating a wallet from legacy to descriptor.""" @@ -19,7 +19,8 @@ from test_framework.descriptors import descsum_create from test_framework.key import ECPubKey from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import COIN, CTransaction, CTxOut -from test_framework.script_util import key_to_p2pkh_script, script_to_p2sh_script, script_to_p2wsh_script +from test_framework.script import hash160 +from test_framework.script_util import key_to_p2pkh_script, key_to_p2pk_script, script_to_p2sh_script, script_to_p2wsh_script from test_framework.util import ( assert_equal, assert_raises_rpc_error, @@ -27,6 +28,7 @@ from test_framework.util import ( ) from test_framework.wallet_util import ( get_generate_key, + generate_keypair, ) @@ -36,42 +38,37 @@ class WalletMigrationTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 1 - self.extra_args = [[]] + self.num_nodes = 2 self.supports_cli = False + self.extra_args = [[], ["-deprecatedrpc=create_bdb"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() - self.skip_if_no_sqlite() - self.skip_if_no_bdb() + self.skip_if_no_previous_releases() + + def setup_nodes(self): + self.add_nodes(self.num_nodes, versions=[ + None, + 280000, + ]) + self.start_nodes() + self.init_wallet(node=0) def assert_is_sqlite(self, wallet_name): - wallet_file_path = self.nodes[0].wallets_path / wallet_name / self.wallet_data_filename + wallet_file_path = self.master_node.wallets_path / wallet_name / self.wallet_data_filename with open(wallet_file_path, 'rb') as f: file_magic = f.read(16) assert_equal(file_magic, b'SQLite format 3\x00') - assert_equal(self.nodes[0].get_wallet_rpc(wallet_name).getwalletinfo()["format"], "sqlite") + assert_equal(self.master_node.get_wallet_rpc(wallet_name).getwalletinfo()["format"], "sqlite") def create_legacy_wallet(self, wallet_name, **kwargs): - self.nodes[0].createwallet(wallet_name=wallet_name, descriptors=False, **kwargs) - wallet = self.nodes[0].get_wallet_rpc(wallet_name) + self.old_node.createwallet(wallet_name=wallet_name, descriptors=False, **kwargs) + wallet = self.old_node.get_wallet_rpc(wallet_name) info = wallet.getwalletinfo() assert_equal(info["descriptors"], False) assert_equal(info["format"], "bdb") return wallet - def migrate_wallet(self, wallet_rpc, *args, **kwargs): - # Helper to ensure that only migration happens - # Since we may rescan on loading of a wallet, make sure that the best block - # is written before beginning migration - # Reload to force write that record - wallet_name = wallet_rpc.getwalletinfo()["walletname"] - wallet_rpc.unloadwallet() - self.nodes[0].loadwallet(wallet_name) - # Migrate, checking that rescan does not occur - with self.nodes[0].assert_debug_log(expected_msgs=[], unexpected_msgs=["Rescanning"]): - return wallet_rpc.migratewallet(*args, **kwargs) - def assert_addr_info_equal(self, addr_info, addr_info_old): assert_equal(addr_info["address"], addr_info_old["address"]) assert_equal(addr_info["scriptPubKey"], addr_info_old["scriptPubKey"]) @@ -99,8 +96,25 @@ class WalletMigrationTest(BitcoinTestFramework): else: assert_equal(addr_info['labels'], []), + def migrate_and_get_rpc(self, wallet_name, **kwargs): + # Since we may rescan on loading of a wallet, make sure that the best block + # is written before beginning migration + # Reload to force write that record + self.old_node.unloadwallet(wallet_name) + self.old_node.loadwallet(wallet_name) + # Now unload so we can copy it to the master node for the migration test + self.old_node.unloadwallet(wallet_name) + if wallet_name == "": + shutil.copyfile(self.old_node.wallets_path / "wallet.dat", self.master_node.wallets_path / "wallet.dat") + else: + shutil.copytree(self.old_node.wallets_path / wallet_name, self.master_node.wallets_path / wallet_name) + # Migrate, checking that rescan does not occur + with self.master_node.assert_debug_log(expected_msgs=[], unexpected_msgs=["Rescanning"]): + migrate_info = self.master_node.migratewallet(wallet_name=wallet_name, **kwargs) + return migrate_info, self.master_node.get_wallet_rpc(wallet_name) + def test_basic(self): - default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + default = self.master_node.get_wallet_rpc(self.default_wallet_name) self.log.info("Test migration of a basic keys only wallet without balance") basic0 = self.create_legacy_wallet("basic0") @@ -116,7 +130,7 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(old_change_addr_info["hdkeypath"], "m/0'/1'/0'") # Note: migration could take a while. - self.migrate_wallet(basic0) + _, basic0 = self.migrate_and_get_rpc("basic0") # Verify created descriptors assert_equal(basic0.getwalletinfo()["descriptors"], True) @@ -147,35 +161,36 @@ class WalletMigrationTest(BitcoinTestFramework): for _ in range(0, 10): default.sendtoaddress(basic1.getnewaddress(), 1) - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) for _ in range(0, 5): basic1.sendtoaddress(default.getnewaddress(), 0.5) - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) bal = basic1.getbalance() txs = basic1.listtransactions() addr_gps = basic1.listaddressgroupings() - basic1_migrate = self.migrate_wallet(basic1) + basic1_migrate, basic1 = self.migrate_and_get_rpc("basic1") assert_equal(basic1.getwalletinfo()["descriptors"], True) self.assert_is_sqlite("basic1") assert_equal(basic1.getbalance(), bal) self.assert_list_txs_equal(basic1.listtransactions(), txs) self.log.info("Test backup file can be successfully restored") - self.nodes[0].restorewallet("basic1_restored", basic1_migrate['backup_path']) - basic1_restored = self.nodes[0].get_wallet_rpc("basic1_restored") + self.old_node.restorewallet("basic1_restored", basic1_migrate['backup_path']) + basic1_restored = self.old_node.get_wallet_rpc("basic1_restored") basic1_restored_wi = basic1_restored.getwalletinfo() assert_equal(basic1_restored_wi['balance'], bal) assert_equal(basic1_restored.listaddressgroupings(), addr_gps) self.assert_list_txs_equal(basic1_restored.listtransactions(), txs) - # restart node and verify that everything is still there + # restart master node and verify that everything is still there self.restart_node(0) - default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) - self.nodes[0].loadwallet("basic1") - basic1 = self.nodes[0].get_wallet_rpc("basic1") + self.connect_nodes(0, 1) + default = self.master_node.get_wallet_rpc(self.default_wallet_name) + self.master_node.loadwallet("basic1") + basic1 = self.master_node.get_wallet_rpc("basic1") assert_equal(basic1.getwalletinfo()["descriptors"], True) self.assert_is_sqlite("basic1") assert_equal(basic1.getbalance(), bal) @@ -193,12 +208,12 @@ class WalletMigrationTest(BitcoinTestFramework): send_value = random.randint(1, 4) default.sendtoaddress(addr, send_value) basic2_balance += send_value - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) assert_equal(basic2.getbalance(), basic2_balance) basic2_txs = basic2.listtransactions() - # Now migrate and test that we still see have the same balance/transactions - self.migrate_wallet(basic2) + # Now migrate and test that we still have the same balance/transactions + _, basic2 = self.migrate_and_get_rpc("basic2") assert_equal(basic2.getwalletinfo()["descriptors"], True) self.assert_is_sqlite("basic2") assert_equal(basic2.getbalance(), basic2_balance) @@ -210,10 +225,10 @@ class WalletMigrationTest(BitcoinTestFramework): self.log.info("Test \"nothing to migrate\" when the user tries to migrate an unloaded wallet with no legacy data") basic2.unloadwallet() - assert_raises_rpc_error(-4, "Error: This wallet is already a descriptor wallet", self.nodes[0].migratewallet, "basic2") + assert_raises_rpc_error(-4, "Error: This wallet is already a descriptor wallet", self.master_node.migratewallet, "basic2") def test_multisig(self): - default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + default = self.master_node.get_wallet_rpc(self.default_wallet_name) # Contrived case where all the multisig keys are in a single wallet self.log.info("Test migration of a wallet with all keys for a multisig") @@ -224,14 +239,14 @@ class WalletMigrationTest(BitcoinTestFramework): ms_info = multisig0.addmultisigaddress(2, [addr1, addr2, addr3]) - self.migrate_wallet(multisig0) + _, multisig0 = self.migrate_and_get_rpc("multisig0") assert_equal(multisig0.getwalletinfo()["descriptors"], True) self.assert_is_sqlite("multisig0") ms_addr_info = multisig0.getaddressinfo(ms_info["address"]) assert_equal(ms_addr_info["ismine"], True) assert_equal(ms_addr_info["desc"], ms_info["descriptor"]) - assert_equal("multisig0_watchonly" in self.nodes[0].listwallets(), False) - assert_equal("multisig0_solvables" in self.nodes[0].listwallets(), False) + assert_equal("multisig0_watchonly" in self.master_node.listwallets(), False) + assert_equal("multisig0_solvables" in self.master_node.listwallets(), False) pub1 = multisig0.getaddressinfo(addr1)["pubkey"] pub2 = multisig0.getaddressinfo(addr2)["pubkey"] @@ -249,7 +264,7 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(multisig1.getaddressinfo(addr1)["ismine"], False) assert_equal(multisig1.getaddressinfo(addr1)["iswatchonly"], True) assert_equal(multisig1.getaddressinfo(addr1)["solvable"], True) - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) multisig1.gettransaction(txid) assert_equal(multisig1.getbalances()["watchonly"]["trusted"], 10) assert_equal(multisig1.getaddressinfo(addr2)["ismine"], False) @@ -259,7 +274,7 @@ class WalletMigrationTest(BitcoinTestFramework): # Migrating multisig1 should see the multisig is no longer part of multisig1 # A new wallet multisig1_watchonly is created which has the multisig address # Transaction to multisig is in multisig1_watchonly and not multisig1 - self.migrate_wallet(multisig1) + _, multisig1 = self.migrate_and_get_rpc("multisig1") assert_equal(multisig1.getwalletinfo()["descriptors"], True) self.assert_is_sqlite("multisig1") assert_equal(multisig1.getaddressinfo(addr1)["ismine"], False) @@ -269,8 +284,8 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(multisig1.getbalance(), 0) assert_equal(multisig1.listtransactions(), []) - assert_equal("multisig1_watchonly" in self.nodes[0].listwallets(), True) - ms1_watchonly = self.nodes[0].get_wallet_rpc("multisig1_watchonly") + assert_equal("multisig1_watchonly" in self.master_node.listwallets(), True) + ms1_watchonly = self.master_node.get_wallet_rpc("multisig1_watchonly") ms1_wallet_info = ms1_watchonly.getwalletinfo() assert_equal(ms1_wallet_info['descriptors'], True) assert_equal(ms1_wallet_info['private_keys_enabled'], False) @@ -286,8 +301,8 @@ class WalletMigrationTest(BitcoinTestFramework): # Migrating multisig1 should see the second multisig is no longer part of multisig1 # A new wallet multisig1_solvables is created which has the second address # This should have no transactions - assert_equal("multisig1_solvables" in self.nodes[0].listwallets(), True) - ms1_solvable = self.nodes[0].get_wallet_rpc("multisig1_solvables") + assert_equal("multisig1_solvables" in self.master_node.listwallets(), True) + ms1_solvable = self.master_node.get_wallet_rpc("multisig1_solvables") ms1_wallet_info = ms1_solvable.getwalletinfo() assert_equal(ms1_wallet_info['descriptors'], True) assert_equal(ms1_wallet_info['private_keys_enabled'], False) @@ -301,7 +316,7 @@ class WalletMigrationTest(BitcoinTestFramework): def test_other_watchonly(self): - default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + default = self.master_node.get_wallet_rpc(self.default_wallet_name) # Wallet with an imported address. Should be the same thing as the multisig test self.log.info("Test migration of a wallet with watchonly imports") @@ -332,7 +347,7 @@ class WalletMigrationTest(BitcoinTestFramework): # Tx that has both a watchonly and spendable output watchonly_spendable_txid = default.send(outputs=[{received_addr: 1}, {import_addr:1}])["txid"] - self.generate(self.nodes[0], 2) + self.generate(self.master_node, 2) received_watchonly_tx_info = imports0.gettransaction(received_watchonly_txid, True) received_sent_watchonly_tx_info = imports0.gettransaction(received_sent_watchonly_utxo["txid"], True) @@ -342,10 +357,10 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(len(imports0.listtransactions(include_watchonly=True)), 6) # Mock time forward a bit so we can check that tx metadata is preserved - self.nodes[0].setmocktime(int(time.time()) + 100) + self.master_node.setmocktime(int(time.time()) + 100) # Migrate - self.migrate_wallet(imports0) + _, imports0 = self.migrate_and_get_rpc("imports0") assert_equal(imports0.getwalletinfo()["descriptors"], True) self.assert_is_sqlite("imports0") assert_raises_rpc_error(-5, "Invalid or non-wallet transaction id", imports0.gettransaction, received_watchonly_txid) @@ -356,8 +371,8 @@ class WalletMigrationTest(BitcoinTestFramework): imports0.gettransaction(watchonly_spendable_txid) assert_equal(imports0.getbalance(), spendable_bal) - assert_equal("imports0_watchonly" in self.nodes[0].listwallets(), True) - watchonly = self.nodes[0].get_wallet_rpc("imports0_watchonly") + assert_equal("imports0_watchonly" in self.master_node.listwallets(), True) + watchonly = self.master_node.get_wallet_rpc("imports0_watchonly") watchonly_info = watchonly.getwalletinfo() assert_equal(watchonly_info["descriptors"], True) self.assert_is_sqlite("imports0_watchonly") @@ -375,14 +390,14 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(len(watchonly.listtransactions(include_watchonly=True)), 4) # Check that labels were migrated and persisted to watchonly wallet - self.nodes[0].unloadwallet("imports0_watchonly") - self.nodes[0].loadwallet("imports0_watchonly") + self.master_node.unloadwallet("imports0_watchonly") + self.master_node.loadwallet("imports0_watchonly") labels = watchonly.listlabels() assert "external" in labels assert "imported" in labels def test_no_privkeys(self): - default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + default = self.master_node.get_wallet_rpc(self.default_wallet_name) # Migrating an actual watchonly wallet should not create a new watchonly wallet self.log.info("Test migration of a pure watchonly wallet") @@ -398,10 +413,10 @@ class WalletMigrationTest(BitcoinTestFramework): }]) assert_equal(res[0]['success'], True) default.sendtoaddress(addr, 10) - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) - self.migrate_wallet(watchonly0) - assert_equal("watchonly0_watchonly" in self.nodes[0].listwallets(), False) + _, watchonly0 = self.migrate_and_get_rpc("watchonly0") + assert_equal("watchonly0_watchonly" in self.master_node.listwallets(), False) info = watchonly0.getwalletinfo() assert_equal(info["descriptors"], True) assert_equal(info["private_keys_enabled"], False) @@ -432,7 +447,7 @@ class WalletMigrationTest(BitcoinTestFramework): # Before migrating, we can fetch addr1 from the keypool assert_equal(watchonly1.getnewaddress(address_type="bech32"), addr1) - self.migrate_wallet(watchonly1) + _, watchonly1 = self.migrate_and_get_rpc("watchonly1") info = watchonly1.getwalletinfo() assert_equal(info["descriptors"], True) assert_equal(info["private_keys_enabled"], False) @@ -448,36 +463,34 @@ class WalletMigrationTest(BitcoinTestFramework): addr_info = wallet.getaddressinfo(addr) desc = descsum_create("pk(" + addr_info["pubkey"] + ")") - self.nodes[0].generatetodescriptor(1, desc, invalid_call=False) + self.generatetodescriptor(self.master_node, 1, desc) bals = wallet.getbalances() - self.migrate_wallet(wallet) + _, wallet = self.migrate_and_get_rpc("pkcb") assert_equal(bals, wallet.getbalances()) def test_encrypted(self): self.log.info("Test migration of an encrypted wallet") wallet = self.create_legacy_wallet("encrypted") - default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + default = self.master_node.get_wallet_rpc(self.default_wallet_name) wallet.encryptwallet("pass") addr = wallet.getnewaddress() txid = default.sendtoaddress(addr, 1) - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) bals = wallet.getbalances() - assert_raises_rpc_error(-4, "Error: Wallet decryption failed, the wallet passphrase was not provided or was incorrect", wallet.migratewallet) - assert_raises_rpc_error(-4, "Error: Wallet decryption failed, the wallet passphrase was not provided or was incorrect", wallet.migratewallet, None, "badpass") - assert_raises_rpc_error(-4, "The passphrase contains a null character", wallet.migratewallet, None, "pass\0with\0null") - - # Check the wallet is still active post-migration failure. - # If not, it will throw an exception and abort the test. - wallet.walletpassphrase("pass", 99999) - wallet.getnewaddress() + # Use self.migrate_and_get_rpc to test this error to get everything copied over to the master node + assert_raises_rpc_error(-4, "Error: Wallet decryption failed, the wallet passphrase was not provided or was incorrect", self.migrate_and_get_rpc, "encrypted") + # Use the RPC directly on the master node for the rest of these checks + assert_raises_rpc_error(-4, "Error: Wallet decryption failed, the wallet passphrase was not provided or was incorrect", self.master_node.migratewallet, "encrypted", "badpass") + assert_raises_rpc_error(-4, "The passphrase contains a null character", self.master_node.migratewallet, "encrypted", "pass\0with\0null") # Verify we can properly migrate the encrypted wallet - self.migrate_wallet(wallet, passphrase="pass") + self.master_node.migratewallet("encrypted", passphrase="pass") + wallet = self.master_node.get_wallet_rpc("encrypted") info = wallet.getwalletinfo() assert_equal(info["descriptors"], True) @@ -487,46 +500,30 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(bals, wallet.getbalances()) - def test_unloaded(self): - self.log.info("Test migration of a wallet that isn't loaded") - wallet = self.create_legacy_wallet("notloaded") - default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) - - addr = wallet.getnewaddress() - txid = default.sendtoaddress(addr, 1) - self.generate(self.nodes[0], 1) - bals = wallet.getbalances() - - wallet.unloadwallet() - - assert_raises_rpc_error(-8, "RPC endpoint wallet and wallet_name parameter specify different wallets", wallet.migratewallet, "someotherwallet") - assert_raises_rpc_error(-8, "Either RPC endpoint wallet or wallet_name parameter must be provided", self.nodes[0].migratewallet) - self.nodes[0].migratewallet("notloaded") - - info = wallet.getwalletinfo() - assert_equal(info["descriptors"], True) - assert_equal(info["format"], "sqlite") - wallet.gettransaction(txid) - - assert_equal(bals, wallet.getbalances()) + def test_nonexistent(self): + self.log.info("Check migratewallet errors for nonexistent wallets") + default = self.master_node.get_wallet_rpc(self.default_wallet_name) + assert_raises_rpc_error(-8, "RPC endpoint wallet and wallet_name parameter specify different wallets", default.migratewallet, "someotherwallet") + assert_raises_rpc_error(-8, "Either RPC endpoint wallet or wallet_name parameter must be provided", self.master_node.migratewallet) + assert_raises_rpc_error(-4, "Error: Wallet does not exist", self.master_node.migratewallet, "notawallet") def test_unloaded_by_path(self): self.log.info("Test migration of a wallet that isn't loaded, specified by path") wallet = self.create_legacy_wallet("notloaded2") - default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + default = self.master_node.get_wallet_rpc(self.default_wallet_name) addr = wallet.getnewaddress() txid = default.sendtoaddress(addr, 1) - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) bals = wallet.getbalances() wallet.unloadwallet() - wallet_file_path = self.nodes[0].wallets_path / "notloaded2" - self.nodes[0].migratewallet(wallet_file_path) + wallet_file_path = self.old_node.wallets_path / "notloaded2" + self.master_node.migratewallet(wallet_file_path) # Because we gave the name by full path, the loaded wallet's name is that path too. - wallet = self.nodes[0].get_wallet_rpc(str(wallet_file_path)) + wallet = self.master_node.get_wallet_rpc(str(wallet_file_path)) info = wallet.getwalletinfo() assert_equal(info["descriptors"], True) @@ -541,9 +538,10 @@ class WalletMigrationTest(BitcoinTestFramework): # Set time to verify backup existence later curr_time = int(time.time()) - wallet.setmocktime(curr_time) + self.master_node.setmocktime(curr_time) - res = self.migrate_wallet(wallet) + res, wallet = self.migrate_and_get_rpc("") + self.master_node.setmocktime(0) info = wallet.getwalletinfo() assert_equal(info["descriptors"], True) assert_equal(info["format"], "sqlite") @@ -553,38 +551,35 @@ class WalletMigrationTest(BitcoinTestFramework): # Check backup existence and its non-empty wallet filename backup_filename = f"default_wallet_{curr_time}.legacy.bak" - backup_path = self.nodes[0].wallets_path / backup_filename + backup_path = self.master_node.wallets_path / backup_filename assert backup_path.exists() assert_equal(str(backup_path), res['backup_path']) assert {"name": backup_filename} not in walletdir_list["wallets"] + self.master_node.setmocktime(0) + def test_direct_file(self): self.log.info("Test migration of a wallet that is not in a wallet directory") wallet = self.create_legacy_wallet("plainfile") wallet.unloadwallet() - wallets_dir = self.nodes[0].wallets_path - wallet_path = wallets_dir / "plainfile" - wallet_dat_path = wallet_path / "wallet.dat" - shutil.copyfile(wallet_dat_path, wallets_dir / "plainfile.bak") - shutil.rmtree(wallet_path) - shutil.move(wallets_dir / "plainfile.bak", wallet_path) - - self.nodes[0].loadwallet("plainfile") - info = wallet.getwalletinfo() - assert_equal(info["descriptors"], False) - assert_equal(info["format"], "bdb") + shutil.copyfile( + self.old_node.wallets_path / "plainfile" / "wallet.dat" , + self.master_node.wallets_path / "plainfile" + ) + assert (self.master_node.wallets_path / "plainfile").is_file() - self.migrate_wallet(wallet) + self.master_node.migratewallet("plainfile") + wallet = self.master_node.get_wallet_rpc("plainfile") info = wallet.getwalletinfo() assert_equal(info["descriptors"], True) assert_equal(info["format"], "sqlite") - assert wallet_path.is_dir() - assert wallet_dat_path.is_file() + assert (self.master_node.wallets_path / "plainfile").is_dir() + assert (self.master_node.wallets_path / "plainfile" / "wallet.dat").is_file() def test_addressbook(self): - df_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + df_wallet = self.master_node.get_wallet_rpc(self.default_wallet_name) self.log.info("Test migration of address book data") wallet = self.create_legacy_wallet("legacy_addrbook") @@ -602,7 +597,7 @@ class WalletMigrationTest(BitcoinTestFramework): wallet.importpubkey(df_wallet.getaddressinfo(multi_addr3)["pubkey"]) ms_addr_info = wallet.addmultisigaddress(2, [multi_addr1, multi_addr2, multi_addr3]) - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) # Test vectors addr_external = { @@ -650,7 +645,7 @@ class WalletMigrationTest(BitcoinTestFramework): # To store the change address in the addressbook need to send coins to it wallet.send(outputs=[{wallet.getnewaddress(): 2}], options={"change_address": change_address['addr']}) - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) # Util wrapper func for 'addr_info' def check(info, node): @@ -663,9 +658,9 @@ class WalletMigrationTest(BitcoinTestFramework): check(addr_info, wallet) # Migrate wallet - info_migration = self.migrate_wallet(wallet) - wallet_wo = self.nodes[0].get_wallet_rpc(info_migration["watchonly_name"]) - wallet_solvables = self.nodes[0].get_wallet_rpc(info_migration["solvables_name"]) + info_migration, wallet = self.migrate_and_get_rpc("legacy_addrbook") + wallet_wo = self.master_node.get_wallet_rpc(info_migration["watchonly_name"]) + wallet_solvables = self.master_node.get_wallet_rpc(info_migration["solvables_name"]) ######################### # Post migration checks # @@ -690,28 +685,28 @@ class WalletMigrationTest(BitcoinTestFramework): ######################################################################################## # First the main wallet - self.nodes[0].unloadwallet("legacy_addrbook") - self.nodes[0].loadwallet("legacy_addrbook") + self.master_node.unloadwallet("legacy_addrbook") + self.master_node.loadwallet("legacy_addrbook") for addr_info in [addr_external, addr_external_with_label, addr_internal, addr_internal_with_label, change_address, ms_addr]: check(addr_info, wallet) # Watch-only wallet - self.nodes[0].unloadwallet(info_migration["watchonly_name"]) - self.nodes[0].loadwallet(info_migration["watchonly_name"]) + self.master_node.unloadwallet(info_migration["watchonly_name"]) + self.master_node.loadwallet(info_migration["watchonly_name"]) self.check_address(wallet_wo, watch_only_addr['addr'], is_mine=True, is_change=watch_only_addr['is_change'], label=watch_only_addr["label"]) for addr_info in [addr_external, addr_external_with_label, ms_addr]: check(addr_info, wallet_wo) # Solvables wallet - self.nodes[0].unloadwallet(info_migration["solvables_name"]) - self.nodes[0].loadwallet(info_migration["solvables_name"]) + self.master_node.unloadwallet(info_migration["solvables_name"]) + self.master_node.loadwallet(info_migration["solvables_name"]) self.check_address(wallet_solvables, ms_addr['addr'], is_mine=True, is_change=ms_addr['is_change'], label=ms_addr["label"]) for addr_info in [addr_external, addr_external_with_label]: check(addr_info, wallet_solvables) def test_migrate_raw_p2sh(self): self.log.info("Test migration of watch-only raw p2sh script") - df_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + df_wallet = self.master_node.get_wallet_rpc(self.default_wallet_name) wallet = self.create_legacy_wallet("raw_p2sh") def send_to_script(script, amount): @@ -721,7 +716,7 @@ class WalletMigrationTest(BitcoinTestFramework): hex_tx = df_wallet.fundrawtransaction(tx.serialize().hex())['hex'] signed_tx = df_wallet.signrawtransactionwithwallet(hex_tx) df_wallet.sendrawtransaction(signed_tx['hex']) - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) # Craft sh(pkh(key)) script and send coins to it pubkey = df_wallet.getaddressinfo(df_wallet.getnewaddress())["pubkey"] @@ -758,8 +753,8 @@ class WalletMigrationTest(BitcoinTestFramework): wallet.rpc.importaddress(address=script_sh_pkh.hex(), label=label_sh_pkh, rescan=False, p2sh=True) # Migrate wallet and re-check balance - info_migration = self.migrate_wallet(wallet) - wallet_wo = self.nodes[0].get_wallet_rpc(info_migration["watchonly_name"]) + info_migration, wallet = self.migrate_and_get_rpc("raw_p2sh") + wallet_wo = self.master_node.get_wallet_rpc(info_migration["watchonly_name"]) # Watch-only balance is under "mine". assert_equal(wallet_wo.getbalances()['mine']['trusted'], 5) @@ -781,17 +776,17 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(next((it['desc'] for it in wallet_wo.listdescriptors()['descriptors'] if it['desc'] == desc_invalid), None), None) # Just in case, also verify wallet restart - self.nodes[0].unloadwallet(info_migration["watchonly_name"]) - self.nodes[0].loadwallet(info_migration["watchonly_name"]) + self.master_node.unloadwallet(info_migration["watchonly_name"]) + self.master_node.loadwallet(info_migration["watchonly_name"]) assert_equal(wallet_wo.getbalances()['mine']['trusted'], 5) def test_conflict_txs(self): self.log.info("Test migration when wallet contains conflicting transactions") - def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + def_wallet = self.master_node.get_wallet_rpc(self.default_wallet_name) wallet = self.create_legacy_wallet("conflicts") def_wallet.sendtoaddress(wallet.getnewaddress(), 10) - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) # parent tx parent_txid = wallet.sendtoaddress(wallet.getnewaddress(), 9) @@ -799,12 +794,12 @@ class WalletMigrationTest(BitcoinTestFramework): conflict_utxo = wallet.gettransaction(txid=parent_txid, verbose=True)["decoded"]["vin"][0] # The specific assertion in MarkConflicted being tested requires that the parent tx is already loaded - # by the time the child tx is loaded. Since transactions end up being loaded in txid order due to how both - # and sqlite store things, we can just grind the child tx until it has a txid that is greater than the parent's. + # by the time the child tx is loaded. Since transactions end up being loaded in txid order due to how + # sqlite stores things, we can just grind the child tx until it has a txid that is greater than the parent's. locktime = 500000000 # Use locktime as nonce, starting at unix timestamp minimum addr = wallet.getnewaddress() while True: - child_send_res = wallet.send(outputs=[{addr: 8}], add_to_wallet=False, locktime=locktime) + child_send_res = wallet.send(outputs=[{addr: 8}], options={"add_to_wallet": False, "locktime": locktime}) child_txid = child_send_res["txid"] child_txid_bytes = bytes.fromhex(child_txid)[::-1] if (child_txid_bytes > parent_txid_bytes): @@ -813,15 +808,15 @@ class WalletMigrationTest(BitcoinTestFramework): locktime += 1 # conflict with parent - conflict_unsigned = self.nodes[0].createrawtransaction(inputs=[conflict_utxo], outputs=[{wallet.getnewaddress(): 9.9999}]) + conflict_unsigned = self.master_node.createrawtransaction(inputs=[conflict_utxo], outputs=[{wallet.getnewaddress(): 9.9999}]) conflict_signed = wallet.signrawtransactionwithwallet(conflict_unsigned)["hex"] - conflict_txid = self.nodes[0].sendrawtransaction(conflict_signed) - self.generate(self.nodes[0], 1) + conflict_txid = self.master_node.sendrawtransaction(conflict_signed) + self.generate(self.master_node, 1) assert_equal(wallet.gettransaction(txid=parent_txid)["confirmations"], -1) assert_equal(wallet.gettransaction(txid=child_txid)["confirmations"], -1) assert_equal(wallet.gettransaction(txid=conflict_txid)["confirmations"], 1) - self.migrate_wallet(wallet) + _, wallet = self.migrate_and_get_rpc("conflicts") assert_equal(wallet.gettransaction(txid=parent_txid)["confirmations"], -1) assert_equal(wallet.gettransaction(txid=child_txid)["confirmations"], -1) assert_equal(wallet.gettransaction(txid=conflict_txid)["confirmations"], 1) @@ -854,7 +849,7 @@ class WalletMigrationTest(BitcoinTestFramework): p2wpkh_addr = key_to_p2wpkh(hybrid_pubkey) wallet.importaddress(p2wpkh_addr) - migrate_info = self.migrate_wallet(wallet) + migrate_info, wallet = self.migrate_and_get_rpc("hybrid_keys") # Both addresses should only appear in the watchonly wallet p2pkh_addr_info = wallet.getaddressinfo(p2pkh_addr) @@ -864,7 +859,7 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(p2wpkh_addr_info["iswatchonly"], False) assert_equal(p2wpkh_addr_info["ismine"], False) - watchonly_wallet = self.nodes[0].get_wallet_rpc(migrate_info["watchonly_name"]) + watchonly_wallet = self.master_node.get_wallet_rpc(migrate_info["watchonly_name"]) watchonly_p2pkh_addr_info = watchonly_wallet.getaddressinfo(p2pkh_addr) assert_equal(watchonly_p2pkh_addr_info["iswatchonly"], False) assert_equal(watchonly_p2pkh_addr_info["ismine"], True) @@ -887,31 +882,32 @@ class WalletMigrationTest(BitcoinTestFramework): # Make a copy of the wallet with the solvables wallet name so that we are unable # to create the solvables wallet when migrating, thus failing to migrate wallet.unloadwallet() - solvables_path = self.nodes[0].wallets_path / "failed_solvables" - shutil.copytree(self.nodes[0].wallets_path / "failed", solvables_path) + solvables_path = self.master_node.wallets_path / "failed_solvables" + shutil.copytree(self.old_node.wallets_path / "failed", solvables_path) original_shasum = sha256sum_file(solvables_path / "wallet.dat") - self.nodes[0].loadwallet("failed") + self.old_node.loadwallet("failed") # Add a multisig so that a solvables wallet is created wallet.addmultisigaddress(2, [wallet.getnewaddress(), get_generate_key().pubkey]) wallet.importaddress(get_generate_key().p2pkh_addr) - assert_raises_rpc_error(-4, "Failed to create database", wallet.migratewallet) + self.old_node.unloadwallet("failed") + shutil.copytree(self.old_node.wallets_path / "failed", self.master_node.wallets_path / "failed") + assert_raises_rpc_error(-4, "Failed to create database", self.master_node.migratewallet, "failed") - assert "failed" in self.nodes[0].listwallets() - assert "failed_watchonly" not in self.nodes[0].listwallets() - assert "failed_solvables" not in self.nodes[0].listwallets() + assert "failed" in self.master_node.listwallets() + assert "failed_watchonly" not in self.master_node.listwallets() + assert "failed_solvables" not in self.master_node.listwallets() - assert not (self.nodes[0].wallets_path / "failed_watchonly").exists() + assert not (self.master_node.wallets_path / "failed_watchonly").exists() # Since the file in failed_solvables is one that we put there, migration shouldn't touch it assert solvables_path.exists() new_shasum = sha256sum_file(solvables_path / "wallet.dat") assert_equal(original_shasum, new_shasum) - wallet.unloadwallet() # Check the wallet we tried to migrate is still BDB - with open(self.nodes[0].wallets_path / "failed" / "wallet.dat", "rb") as f: + with open(self.master_node.wallets_path / "failed" / "wallet.dat", "rb") as f: data = f.read(16) _, _, magic = struct.unpack("QII", data) assert_equal(magic, BTREE_MAGIC) @@ -920,13 +916,13 @@ class WalletMigrationTest(BitcoinTestFramework): self.log.info("Test that a blank wallet is migrated") wallet = self.create_legacy_wallet("blank", blank=True) assert_equal(wallet.getwalletinfo()["blank"], True) - wallet.migratewallet() + _, wallet = self.migrate_and_get_rpc("blank") assert_equal(wallet.getwalletinfo()["blank"], True) assert_equal(wallet.getwalletinfo()["descriptors"], True) def test_avoidreuse(self): self.log.info("Test that avoidreuse persists after migration") - def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + def_wallet = self.master_node.get_wallet_rpc(self.default_wallet_name) wallet = self.create_legacy_wallet("avoidreuse") wallet.setwalletflag("avoid_reuse", True) @@ -941,12 +937,12 @@ class WalletMigrationTest(BitcoinTestFramework): reused_addr = wallet.getnewaddress() def_wallet.sendtoaddress(reused_addr, 2) - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) # Send funds from the test wallet with both its own and the imported wallet.sendall([def_wallet.getnewaddress()]) def_wallet.sendall(recipients=[def_wallet.getnewaddress()], inputs=imported_utxos) - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) balances = wallet.getbalances() assert_equal(balances["mine"]["trusted"], 0) assert_equal(balances["watchonly"]["trusted"], 0) @@ -954,7 +950,7 @@ class WalletMigrationTest(BitcoinTestFramework): # Reuse the addresses def_wallet.sendtoaddress(reused_addr, 1) def_wallet.sendtoaddress(reused_imported_addr, 1) - self.generate(self.nodes[0], 1) + self.generate(self.master_node, 1) balances = wallet.getbalances() assert_equal(balances["mine"]["used"], 1) # Reused watchonly will not show up in balances @@ -968,8 +964,8 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(utxo["reused"], True) # Migrate - migrate_res = wallet.migratewallet() - watchonly_wallet = self.nodes[0].get_wallet_rpc(migrate_res["watchonly_name"]) + _, wallet = self.migrate_and_get_rpc("avoidreuse") + watchonly_wallet = self.master_node.get_wallet_rpc("avoidreuse_watchonly") # One utxo in each wallet, marked used utxos = wallet.listunspent() @@ -981,13 +977,13 @@ class WalletMigrationTest(BitcoinTestFramework): def test_preserve_tx_extra_info(self): self.log.info("Test that tx extra data is preserved after migration") - def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + def_wallet = self.master_node.get_wallet_rpc(self.default_wallet_name) # Create and fund wallet wallet = self.create_legacy_wallet("persist_comments") def_wallet.sendtoaddress(wallet.getnewaddress(), 2) - self.generate(self.nodes[0], 6) + self.generate(self.master_node, 6) # Create tx and bump it to store 'replaced_by_txid' and 'replaces_txid' data within the transactions. # Additionally, store an extra comment within the original tx. @@ -1006,15 +1002,69 @@ class WalletMigrationTest(BitcoinTestFramework): # Pre-migration verification check_comments() # Migrate - wallet.migratewallet() + _, wallet = self.migrate_and_get_rpc("persist_comments") # Post-migration verification check_comments() wallet.unloadwallet() + def test_migrate_simple_watch_only(self): + self.log.info("Test migrating a watch-only p2pk script") + wallet = self.create_legacy_wallet("bare_p2pk", blank=True) + _, pubkey = generate_keypair() + p2pk_script = key_to_p2pk_script(pubkey) + wallet.importaddress(address=p2pk_script.hex()) + # Migrate wallet in the latest node + res, _ = self.migrate_and_get_rpc("bare_p2pk") + wo_wallet = self.master_node.get_wallet_rpc(res['watchonly_name']) + assert_equal(wo_wallet.listdescriptors()['descriptors'][0]['desc'], descsum_create(f'pk({pubkey.hex()})')) + wo_wallet.unloadwallet() + + def test_manual_keys_import(self): + self.log.info("Test migrating standalone private keys") + wallet = self.create_legacy_wallet("import_privkeys", blank=True) + privkey, pubkey = generate_keypair(wif=True) + wallet.importprivkey(privkey=privkey, label="hi", rescan=False) + + # Migrate and verify + res, wallet = self.migrate_and_get_rpc("import_privkeys") + + # There should be descriptors containing the imported key for: pk(), pkh(), sh(wpkh()), wpkh() + key_origin = hash160(pubkey)[:4].hex() + pubkey_hex = pubkey.hex() + combo_desc = descsum_create(f"combo([{key_origin}]{pubkey_hex})") + + # Verify all expected descriptors were migrated + migrated_desc = [item['desc'] for item in wallet.listdescriptors()['descriptors'] if pubkey.hex() in item['desc']] + assert_equal([combo_desc], migrated_desc) + wallet.unloadwallet() + + ###################################################### + self.log.info("Test migrating standalone public keys") + wallet = self.create_legacy_wallet("import_pubkeys", blank=True) + wallet.importpubkey(pubkey=pubkey_hex, rescan=False) + + res, _ = self.migrate_and_get_rpc("import_pubkeys") + + # Same as before, there should be descriptors in the watch-only wallet for the imported pubkey + wo_wallet = self.nodes[0].get_wallet_rpc(res['watchonly_name']) + # As we imported the pubkey only, there will be no key origin in the following descriptors + pk_desc = descsum_create(f'pk({pubkey_hex})') + pkh_desc = descsum_create(f'pkh({pubkey_hex})') + sh_wpkh_desc = descsum_create(f'sh(wpkh({pubkey_hex}))') + wpkh_desc = descsum_create(f'wpkh({pubkey_hex})') + expected_descs = [pk_desc, pkh_desc, sh_wpkh_desc, wpkh_desc] + + # Verify all expected descriptors were migrated + migrated_desc = [item['desc'] for item in wo_wallet.listdescriptors()['descriptors']] + assert_equal(expected_descs, migrated_desc) + wo_wallet.unloadwallet() def run_test(self): - self.generate(self.nodes[0], 101) + self.master_node = self.nodes[0] + self.old_node = self.nodes[1] + + self.generate(self.master_node, 101) # TODO: Test the actual records in the wallet for these tests too. The behavior may be correct, but the data written may not be what we actually want self.test_basic() @@ -1023,7 +1073,7 @@ class WalletMigrationTest(BitcoinTestFramework): self.test_no_privkeys() self.test_pk_coinbases() self.test_encrypted() - self.test_unloaded() + self.test_nonexistent() self.test_unloaded_by_path() self.test_default_wallet() self.test_direct_file() @@ -1035,6 +1085,9 @@ class WalletMigrationTest(BitcoinTestFramework): self.test_avoidreuse() self.test_preserve_tx_extra_info() self.test_blank() + self.test_migrate_simple_watch_only() + self.test_manual_keys_import() + if __name__ == '__main__': WalletMigrationTest(__file__).main() diff --git a/test/functional/wallet_multisig_descriptor_psbt.py b/test/functional/wallet_multisig_descriptor_psbt.py index 2e0b0d1a41..a69185b3a5 100755 --- a/test/functional/wallet_multisig_descriptor_psbt.py +++ b/test/functional/wallet_multisig_descriptor_psbt.py @@ -53,8 +53,8 @@ class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): for i, node in enumerate(self.nodes): node.createwallet(wallet_name=f"{self.name}_{i}", blank=True, descriptors=True, disable_private_keys=True) multisig = node.get_wallet_rpc(f"{self.name}_{i}") - external = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f','.join(external_xpubs)}))") - internal = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f','.join(internal_xpubs)}))") + external = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{','.join(external_xpubs)}))") + internal = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{','.join(internal_xpubs)}))") result = multisig.importdescriptors([ { # receiving addresses (internal: False) "desc": external["descriptor"], diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 156f4279b4..e241edd3fa 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -12,7 +12,6 @@ import os import platform import shutil import stat -import time from test_framework.authproxy import JSONRPCException from test_framework.blocktools import COINBASE_MATURITY @@ -21,6 +20,7 @@ from test_framework.test_node import ErrorMatch from test_framework.util import ( assert_equal, assert_raises_rpc_error, + ensure_for, get_rpc_proxy, ) @@ -204,7 +204,7 @@ class MultiWalletTest(BitcoinTestFramework): self.restart_node(0, ['-nowallet', '-walletdir=' + competing_wallet_dir]) self.nodes[0].createwallet(self.default_wallet_name) if self.options.descriptors: - exp_stderr = f"Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['PACKAGE_NAME']}?" + exp_stderr = f"Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['CLIENT_NAME']}?" else: exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\S*\"!" self.nodes[1].assert_start_raises_init_error(['-walletdir=' + competing_wallet_dir], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) @@ -229,7 +229,7 @@ class MultiWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo) # accessing wallet RPC without using wallet endpoint fails - assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo) + assert_raises_rpc_error(-19, "Multiple wallets are loaded. Please select which wallet", node.getwalletinfo) w1, w2, w3, w4, *_ = wallets self.generatetoaddress(node, nblocks=COINBASE_MATURITY + 1, address=w1.getnewaddress(), sync_fun=self.no_op) @@ -275,7 +275,7 @@ class MultiWalletTest(BitcoinTestFramework): loadwallet_name = node.loadwallet(wallet_names[1]) assert_equal(loadwallet_name['name'], wallet_names[1]) assert_equal(node.listwallets(), wallet_names[0:2]) - assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo) + assert_raises_rpc_error(-19, "Multiple wallets are loaded. Please select which wallet", node.getwalletinfo) w2 = node.get_wallet_rpc(wallet_names[1]) w2.getwalletinfo() @@ -373,8 +373,7 @@ class MultiWalletTest(BitcoinTestFramework): w2.encryptwallet('test') w2.walletpassphrase('test', 1) w2.unloadwallet() - time.sleep(1.1) - assert 'w2' not in self.nodes[0].listwallets() + ensure_for(duration=1.1, f=lambda: 'w2' not in self.nodes[0].listwallets()) # Successfully unload all wallets for wallet_name in self.nodes[0].listwallets(): diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index ef3f925ee8..c909336a25 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -185,6 +185,7 @@ class UpgradeWalletTest(BitcoinTestFramework): self.restart_node(0) copy_v16() wallet = node_master.get_wallet_rpc(self.default_wallet_name) + assert_equal(wallet.getbalance(), v16_3_balance) self.log.info("Test upgradewallet without a version argument") self.test_upgradewallet(wallet, previous_version=159900, expected_version=169900) # wallet should still contain the same balance diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index 9917eca75a..e87977edfc 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -371,10 +371,6 @@ def run_once(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, using_libfuzzer for future in as_completed(jobs): output, result, target = future.result() logging.debug(output) - if using_libfuzzer: - done_stat = [l for l in output.splitlines() if "DONE" in l] - assert len(done_stat) == 1 - stats.append((target, done_stat[0])) try: result.check_returncode() except subprocess.CalledProcessError as e: @@ -382,8 +378,12 @@ def run_once(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, using_libfuzzer logging.info(e.stdout) if e.stderr: logging.info(e.stderr) - logging.info(f"Target {result.args} failed with exit code {e.returncode}") + logging.info(f"⚠️ Failure generated from target with exit code {e.returncode}: {result.args}") sys.exit(1) + if using_libfuzzer: + done_stat = [l for l in output.splitlines() if "DONE" in l] + assert len(done_stat) == 1 + stats.append((target, done_stat[0])) if using_libfuzzer: print("Summary:") diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index 1d397e721e..1f8d92c980 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -98,6 +98,14 @@ SHA256_SUMS = { "fe6e347a66043946920c72c9c4afca301968101e6b82fb90a63d7885ebcceb32": {"tag": "v25.0", "tarball": "bitcoin-25.0-riscv64-linux-gnu.tar.gz"}, "5708fc639cdfc27347cccfd50db9b73b53647b36fb5f3a4a93537cbe8828c27f": {"tag": "v25.0", "tarball": "bitcoin-25.0-x86_64-apple-darwin.tar.gz"}, "33930d432593e49d58a9bff4c30078823e9af5d98594d2935862788ce8a20aec": {"tag": "v25.0", "tarball": "bitcoin-25.0-x86_64-linux-gnu.tar.gz"}, + + "7fa582d99a25c354d23e371a5848bd9e6a79702870f9cbbf1292b86e647d0f4e": {"tag": "v28.0", "tarball": "bitcoin-28.0-aarch64-linux-gnu.tar.gz"}, + "e004b7910bedd6dd18b6c52b4eef398d55971da666487a82cd48708d2879727e": {"tag": "v28.0", "tarball": "bitcoin-28.0-arm-linux-gnueabihf.tar.gz"}, + "c8108f30dfcc7ddffab33f5647d745414ef9d3298bfe67d243fe9b9cb4df4c12": {"tag": "v28.0", "tarball": "bitcoin-28.0-arm64-apple-darwin.tar.gz"}, + "756df50d8f0c2a3d4111389a7be5f4849e0f5014dd5bfcbc37a8c3aaaa54907b": {"tag": "v28.0", "tarball": "bitcoin-28.0-powerpc64-linux-gnu.tar.gz"}, + "6ee1a520b638132a16725020146abea045db418ce91c02493f02f541cd53062a": {"tag": "v28.0", "tarball": "bitcoin-28.0-riscv64-linux-gnu.tar.gz"}, + "77e931bbaaf47771a10c376230bf53223f5380864bad3568efc7f4d02e40a0f7": {"tag": "v28.0", "tarball": "bitcoin-28.0-x86_64-apple-darwin.tar.gz"}, + "7fe294b02b25b51acb8e8e0a0eb5af6bbafa7cd0c5b0e5fcbb61263104a82fbc": {"tag": "v28.0", "tarball": "bitcoin-28.0-x86_64-linux-gnu.tar.gz"}, } diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index f55d0f8cb7..3e9e5ba230 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -23,7 +23,7 @@ CMD_GREP_WALLET_ARGS = r"git grep --function-context 'void WalletInit::AddWallet CMD_GREP_WALLET_HIDDEN_ARGS = r"git grep --function-context 'void DummyWalletInit::AddWalletOptions' -- {}".format(CMD_ROOT_DIR) CMD_GREP_DOCS = r"git grep --perl-regexp '{}' {}".format(REGEX_DOC, CMD_ROOT_DIR) # list unsupported, deprecated and duplicate args as they need no documentation -SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb']) +SET_DOC_OPTIONAL = set(['-h', '-?', '-dbcrashratio', '-forcecompactdb']) def lint_missing_argument_documentation(): diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index fe845ed19e..52ae95fbb6 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -35,20 +35,20 @@ for commit in $(git rev-list --reverse "$1"); do git checkout --quiet "$commit"^ || exit SCRIPT="$(git rev-list --format=%b -n1 "$commit" | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d')" if test -z "$SCRIPT"; then - echo "Error: missing script for: $commit" - echo "Failed" + echo "Error: missing script for: $commit" >&2 + echo "Failed" >&2 RET=1 else - echo "Running script for: $commit" - echo "$SCRIPT" + echo "Running script for: $commit" >&2 + echo "$SCRIPT" >&2 (eval "$SCRIPT") - git --no-pager diff --exit-code "$commit" && echo "OK" || (echo "Failed"; false) || RET=1 + git --no-pager diff --exit-code "$commit" && echo "OK" >&2 || (echo "Failed" >&2; false) || RET=1 fi git reset --quiet --hard HEAD else if git rev-list "--format=%b" -n1 "$commit" | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then - echo "Error: script block marker but no scripted-diff in title of commit $commit" - echo "Failed" + echo "Error: script block marker but no scripted-diff in title of commit $commit" >&2 + echo "Failed" >&2 RET=1 fi fi diff --git a/test/lint/lint-assertions.py b/test/lint/lint-assertions.py deleted file mode 100755 index 5d01b13fd4..0000000000 --- a/test/lint/lint-assertions.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for assertions with obvious side effects. - -import sys -import subprocess - - -def git_grep(params: [], error_msg: ""): - try: - output = subprocess.check_output(["git", "grep", *params], text=True, encoding="utf8") - print(error_msg) - print(output) - return 1 - except subprocess.CalledProcessError as ex1: - if ex1.returncode > 1: - raise ex1 - return 0 - - -def main(): - # Aborting the whole process is undesirable for RPC code. So nonfatal - # checks should be used over assert. See: src/util/check.h - # src/rpc/server.cpp is excluded from this check since it's mostly meta-code. - exit_code = git_grep([ - "--line-number", - "--extended-regexp", - r"\<(A|a)ss(ume|ert)\(", - "--", - "src/rpc/", - "src/wallet/rpc*", - ":(exclude)src/rpc/server.cpp", - ], "CHECK_NONFATAL(condition) or NONFATAL_UNREACHABLE should be used instead of assert for RPC code.") - - # The `BOOST_ASSERT` macro requires to `#include boost/assert.hpp`, - # which is an unnecessary Boost dependency. - exit_code |= git_grep([ - "--line-number", - "--extended-regexp", - r"BOOST_ASSERT\(", - "--", - "*.cpp", - "*.h", - ], "BOOST_ASSERT must be replaced with Assert, BOOST_REQUIRE, or BOOST_CHECK.") - - sys.exit(exit_code) - - -if __name__ == "__main__": - main() diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index c30975fea7..86a17fb0f8 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -17,15 +17,7 @@ import sys FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS = [ 'tfm::format,1', # Assuming tfm::::format(std::ostream&, ... - 'LogError,0', - 'LogWarning,0', - 'LogInfo,0', - 'LogDebug,1', - 'LogTrace,1', - 'LogPrintf,0', - 'LogPrintLevel,2', 'strprintf,0', - 'WalletLogPrintf,0', ] RUN_LINT_FILE = 'test/lint/run-lint-format-strings.py' @@ -70,7 +62,7 @@ def main(): matching_files_filtered = [] for matching_file in matching_files: - if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|test/fuzz/strprintf.cpp)|contrib/devtools/bitcoin-tidy/example_logprintf.cpp', matching_file): + if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|test/fuzz/strprintf.cpp)', matching_file): matching_files_filtered.append(matching_file) matching_files_filtered.sort() diff --git a/test/lint/lint-git-commit-check.py b/test/lint/lint-git-commit-check.py index 5dc30cc755..f5636f22ec 100755 --- a/test/lint/lint-git-commit-check.py +++ b/test/lint/lint-git-commit-check.py @@ -36,10 +36,10 @@ def main(): assert os.getenv("COMMIT_RANGE") # E.g. COMMIT_RANGE='HEAD~n..HEAD' commit_range = os.getenv("COMMIT_RANGE") - commit_hashes = check_output(["git", "log", commit_range, "--format=%H"], text=True, encoding="utf8").splitlines() + commit_hashes = check_output(["git", "-c", "log.showSignature=false", "log", commit_range, "--format=%H"], text=True, encoding="utf8").splitlines() for hash in commit_hashes: - commit_info = check_output(["git", "log", "--format=%B", "-n", "1", hash], text=True, encoding="utf8").splitlines() + commit_info = check_output(["git", "-c", "log.showSignature=false", "log", "--format=%B", "-n", "1", hash], text=True, encoding="utf8").splitlines() if len(commit_info) >= 2: if commit_info[1]: print(f"The subject line of commit hash {hash} is followed by a non-empty line. Subject lines should always be followed by a blank line.") diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py index 90884299d5..05780c3074 100755 --- a/test/lint/lint-includes.py +++ b/test/lint/lint-includes.py @@ -20,7 +20,7 @@ from lint_ignore_dirs import SHARED_EXCLUDED_SUBTREES EXCLUDED_DIRS = ["contrib/devtools/bitcoin-tidy/", ] + SHARED_EXCLUDED_SUBTREES -EXPECTED_BOOST_INCLUDES = ["boost/date_time/posix_time/posix_time.hpp", +EXPECTED_BOOST_INCLUDES = [ "boost/multi_index/detail/hash_index_iterator.hpp", "boost/multi_index/hashed_index.hpp", "boost/multi_index/identity.hpp", diff --git a/test/lint/lint-spelling.py b/test/lint/lint-spelling.py index 3e578b218f..945288a3dd 100755 --- a/test/lint/lint-spelling.py +++ b/test/lint/lint-spelling.py @@ -14,7 +14,7 @@ from subprocess import check_output, STDOUT, CalledProcessError from lint_ignore_dirs import SHARED_EXCLUDED_SUBTREES IGNORE_WORDS_FILE = 'test/lint/spelling.ignore-words.txt' -FILES_ARGS = ['git', 'ls-files', '--', ":(exclude)build-aux/m4/", ":(exclude)contrib/seeds/*.txt", ":(exclude)depends/", ":(exclude)doc/release-notes/", ":(exclude)src/qt/locale/", ":(exclude)src/qt/*.qrc", ":(exclude)contrib/guix/patches"] +FILES_ARGS = ['git', 'ls-files', '--', ":(exclude)contrib/seeds/*.txt", ":(exclude)depends/", ":(exclude)doc/release-notes/", ":(exclude)src/qt/locale/", ":(exclude)src/qt/*.qrc", ":(exclude)contrib/guix/patches"] FILES_ARGS += [f":(exclude){dir}" for dir in SHARED_EXCLUDED_SUBTREES] diff --git a/test/lint/run-lint-format-strings.py b/test/lint/run-lint-format-strings.py index a32717653a..0e08c9f974 100755 --- a/test/lint/run-lint-format-strings.py +++ b/test/lint/run-lint-format-strings.py @@ -13,13 +13,10 @@ import re import sys FALSE_POSITIVES = [ - ("src/clientversion.cpp", "strprintf(_(COPYRIGHT_HOLDERS).translated, COPYRIGHT_HOLDERS_SUBSTITUTION)"), + ("src/clientversion.cpp", "strprintf(_(COPYRIGHT_HOLDERS), COPYRIGHT_HOLDERS_SUBSTITUTION)"), ("src/test/translation_tests.cpp", "strprintf(format, arg)"), - ("src/validationinterface.cpp", "LogDebug(BCLog::VALIDATION, fmt \"\\n\", __VA_ARGS__)"), - ("src/wallet/wallet.h", "WalletLogPrintf(const char* fmt, Params... parameters)"), - ("src/wallet/wallet.h", "LogPrintf((\"%s \" + std::string{fmt}).c_str(), GetDisplayName(), parameters...)"), - ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(const char* fmt, Params... parameters)"), - ("src/wallet/scriptpubkeyman.h", "LogPrintf((\"%s \" + std::string{fmt}).c_str(), m_storage.GetDisplayName(), parameters...)"), + ("src/test/util_string_tests.cpp", 'tfm::format(ConstevalFormatString<2>{"%*s"}, "hi", "hi")'), + ("src/test/util_string_tests.cpp", 'tfm::format(ConstevalFormatString<2>{"%.*s"}, "hi", "hi")'), ] diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index 64a1486a4c..8479fd2d64 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -29,7 +29,7 @@ fn get_linter_list() -> Vec<&'static Linter> { lint_fn: lint_doc }, &Linter { - description: "Check that no symbol from bitcoin-config.h is used without the header being included", + description: "Check that no symbol from bitcoin-build-config.h is used without the header being included", name: "includes_build_config", lint_fn: lint_includes_build_config }, @@ -49,6 +49,16 @@ fn get_linter_list() -> Vec<&'static Linter> { lint_fn: lint_std_filesystem }, &Linter { + description: "Check that fatal assertions are not used in RPC code", + name: "rpc_assert", + lint_fn: lint_rpc_assert + }, + &Linter { + description: "Check that boost assertions are not used", + name: "boost_assert", + lint_fn: lint_boost_assert + }, + &Linter { description: "Check that release note snippets are in the right folder", name: "doc_release_note_snippets", lint_fn: lint_doc_release_note_snippets @@ -226,6 +236,7 @@ fn lint_py_lint() -> LintResult { "F405", // foo_function may be undefined, or defined from star imports: bar_module "F406", // "from module import *" only allowed at module level "F407", // an undefined __future__ feature name was imported + "F541", // f-string without any placeholders "F601", // dictionary key name repeated with different values "F602", // dictionary key variable name repeated with different values "F621", // too many expressions in an assignment with star-unpacking @@ -236,6 +247,7 @@ fn lint_py_lint() -> LintResult { "F822", // undefined name name in __all__ "F823", // local variable name … referenced before assignment "F841", // local variable 'foo' is assigned to but never used + "PLE", // Pylint errors "W191", // indentation contains tabs "W291", // trailing whitespace "W292", // no newline at end of file @@ -271,6 +283,7 @@ fn lint_std_filesystem() -> LintResult { let found = git() .args([ "grep", + "--line-number", "std::filesystem", "--", "./src/", @@ -281,10 +294,66 @@ fn lint_std_filesystem() -> LintResult { .success(); if found { Err(r#" -^^^ Direct use of std::filesystem may be dangerous and buggy. Please include <util/fs.h> and use the fs:: namespace, which has unsafe filesystem functions marked as deleted. "# + .trim() + .to_string()) + } else { + Ok(()) + } +} + +fn lint_rpc_assert() -> LintResult { + let found = git() + .args([ + "grep", + "--line-number", + "--extended-regexp", + r"\<(A|a)ss(ume|ert)\(", + "--", + "src/rpc/", + "src/wallet/rpc*", + ":(exclude)src/rpc/server.cpp", + // src/rpc/server.cpp is excluded from this check since it's mostly meta-code. + ]) + .status() + .expect("command error") + .success(); + if found { + Err(r#" +CHECK_NONFATAL(condition) or NONFATAL_UNREACHABLE should be used instead of assert for RPC code. + +Aborting the whole process is undesirable for RPC code. So nonfatal +checks should be used over assert. See: src/util/check.h + "# + .trim() + .to_string()) + } else { + Ok(()) + } +} + +fn lint_boost_assert() -> LintResult { + let found = git() + .args([ + "grep", + "--line-number", + "--extended-regexp", + r"BOOST_ASSERT\(", + "--", + "*.cpp", + "*.h", + ]) + .status() + .expect("command error") + .success(); + if found { + Err(r#" +BOOST_ASSERT must be replaced with Assert, BOOST_REQUIRE, or BOOST_CHECK to avoid an unnecessary +include of the boost/assert.hpp dependency. + "# + .trim() .to_string()) } else { Ok(()) @@ -301,17 +370,15 @@ fn lint_doc_release_note_snippets() -> LintResult { if non_release_notes.is_empty() { Ok(()) } else { - Err(format!( - r#" -{} -^^^ + println!("{non_release_notes}"); + Err(r#" Release note snippets and other docs must be put into the doc/ folder directly. The doc/release-notes/ folder is for archived release notes of previous releases only. Snippets are expected to follow the naming "/doc/release-notes-<PR number>.md". - "#, - non_release_notes - )) + "# + .trim() + .to_string()) } } @@ -354,7 +421,6 @@ fn lint_trailing_whitespace() -> LintResult { .success(); if trailing_space { Err(r#" -^^^ Trailing whitespace (including Windows line endings [CR LF]) is problematic, because git may warn about it, or editors may remove it by default, forcing developers in the future to either undo the changes manually or spend time on review. @@ -364,6 +430,7 @@ Thus, it is best to remove the trailing space now. Please add any false positives, such as subtrees, Windows-related files, patch files, or externally sourced files to the exclude list. "# + .trim() .to_string()) } else { Ok(()) @@ -380,7 +447,6 @@ fn lint_tabs_whitespace() -> LintResult { .success(); if tabs { Err(r#" -^^^ Use of tabs in this codebase is problematic, because existing code uses spaces and tabs will cause display issues and conflict with editor settings. @@ -388,6 +454,7 @@ Please remove the tabs. Please add any false positives, such as subtrees, or externally sourced files to the exclude list. "# + .trim() .to_string()) } else { Ok(()) @@ -395,7 +462,7 @@ Please add any false positives, such as subtrees, or externally sourced files to } fn lint_includes_build_config() -> LintResult { - let config_path = "./cmake/bitcoin-config.h.in"; + let config_path = "./cmake/bitcoin-build-config.h.in"; let defines_regex = format!( r"^\s*(?!//).*({})", check_output(Command::new("grep").args(["define", "--", config_path])) @@ -429,7 +496,7 @@ fn lint_includes_build_config() -> LintResult { ]) .args(get_pathspecs_exclude_subtrees()) .args([ - // These are exceptions which don't use bitcoin-config.h, rather the Makefile.am adds + // These are exceptions which don't use bitcoin-build-config.h, rather CMakeLists.txt adds // these cppflags manually. ":(exclude)src/crypto/sha256_arm_shani.cpp", ":(exclude)src/crypto/sha256_avx2.cpp", @@ -447,9 +514,9 @@ fn lint_includes_build_config() -> LintResult { "--files-with-matches" }, if mode { - "^#include <config/bitcoin-config.h> // IWYU pragma: keep$" + "^#include <bitcoin-build-config.h> // IWYU pragma: keep$" } else { - "#include <config/bitcoin-config.h>" // Catch redundant includes with and without the IWYU pragma + "#include <bitcoin-build-config.h>" // Catch redundant includes with and without the IWYU pragma }, "--", ]) @@ -462,12 +529,11 @@ fn lint_includes_build_config() -> LintResult { if missing { return Err(format!( r#" -^^^ -One or more files use a symbol declared in the bitcoin-config.h header. However, they are not +One or more files use a symbol declared in the bitcoin-build-config.h header. However, they are not including the header. This is problematic, because the header may or may not be indirectly included. If the indirect include were to be intentionally or accidentally removed, the build could still succeed, but silently be buggy. For example, a slower fallback algorithm could be picked, -even though bitcoin-config.h indicates that a faster feature is available and should be used. +even though bitcoin-build-config.h indicates that a faster feature is available and should be used. If you are unsure which symbol is used, you can find it with this command: git grep --perl-regexp '{}' -- file_name @@ -475,16 +541,17 @@ git grep --perl-regexp '{}' -- file_name Make sure to include it with the IWYU pragma. Otherwise, IWYU may falsely instruct to remove the include again. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep "#, defines_regex - )); + ) + .trim() + .to_string()); } let redundant = print_affected_files(false); if redundant { return Err(r#" -^^^ -None of the files use a symbol declared in the bitcoin-config.h header. However, they are including +None of the files use a symbol declared in the bitcoin-build-config.h header. However, they are including the header. Consider removing the unused include. "# .to_string()); @@ -516,6 +583,7 @@ fn lint_markdown() -> LintResult { "--ignore-path", md_ignore_path_str.as_str(), "--gitignore", + "--gituntracked", "--root-dir", ".", ]) @@ -529,13 +597,15 @@ fn lint_markdown() -> LintResult { r#" One or more markdown links are broken. -Relative links are preferred (but not required) as jumping to file works natively within Emacs. +Note: relative links are preferred as jump-to-file works natively within Emacs, but they are not required. Markdown link errors found: {} "#, stderr - )) + ) + .trim() + .to_string()) } Err(e) if e.kind() == ErrorKind::NotFound => { println!("`mlc` was not found in $PATH, skipping markdown lint check."); @@ -587,10 +657,9 @@ fn main() -> ExitCode { env::set_current_dir(&git_root).unwrap(); if let Err(err) = (linter.lint_fn)() { println!( - "{err}\n^---- ⚠️ Failure generated from lint check '{}'!", - linter.name + "^^^\n{err}\n^---- ⚠️ Failure generated from lint check '{}' ({})!\n\n", + linter.name, linter.description, ); - println!("{}", linter.description); test_failed = true; } } diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 94bd14e6c3..6e636b56b8 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -55,7 +55,6 @@ unsigned-integer-overflow:MurmurHash3 unsigned-integer-overflow:CBlockPolicyEstimator::processBlockTx unsigned-integer-overflow:TxConfirmStats::EstimateMedianVal unsigned-integer-overflow:prevector.h -unsigned-integer-overflow:EvalScript unsigned-integer-overflow:InsecureRandomContext::rand64 unsigned-integer-overflow:InsecureRandomContext::SplitMix64 unsigned-integer-overflow:bitset_detail::PopCount diff --git a/test/util/test_runner.py b/test/util/test_runner.py index 1cd368f6f4..cac184ca30 100755 --- a/test/util/test_runner.py +++ b/test/util/test_runner.py @@ -5,7 +5,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test framework for bitcoin utils. -Runs automatically during `make check`. +Runs automatically during `ctest --test-dir build/`. Can also be run manually.""" @@ -83,13 +83,11 @@ def bctest(testDir, testObj, buildenv): execrun = [execprog] + execargs # Read the input data (if there is any) - stdinCfg = None inputData = None if "input" in testObj: filename = os.path.join(testDir, testObj["input"]) with open(filename, encoding="utf8") as f: inputData = f.read() - stdinCfg = subprocess.PIPE # Read the expected output data (if there is any) outputFn = None @@ -112,9 +110,8 @@ def bctest(testDir, testObj, buildenv): raise Exception # Run the test - proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) try: - outs = proc.communicate(input=inputData) + res = subprocess.run(execrun, capture_output=True, text=True, input=inputData) except OSError: logging.error("OSError, Failed to execute " + execprog) raise @@ -123,9 +120,9 @@ def bctest(testDir, testObj, buildenv): data_mismatch, formatting_mismatch = False, False # Parse command output and expected output try: - a_parsed = parse_output(outs[0], outputType) + a_parsed = parse_output(res.stdout, outputType) except Exception as e: - logging.error('Error parsing command output as %s: %s' % (outputType, e)) + logging.error(f"Error parsing command output as {outputType}: '{str(e)}'; res: {str(res)}") raise try: b_parsed = parse_output(outputData, outputType) @@ -134,13 +131,13 @@ def bctest(testDir, testObj, buildenv): raise # Compare data if a_parsed != b_parsed: - logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") + logging.error(f"Output data mismatch for {outputFn} (format {outputType}); res: {str(res)}") data_mismatch = True # Compare formatting - if outs[0] != outputData: - error_message = "Output formatting mismatch for " + outputFn + ":\n" + if res.stdout != outputData: + error_message = f"Output formatting mismatch for {outputFn}:\nres: {str(res)}\n" error_message += "".join(difflib.context_diff(outputData.splitlines(True), - outs[0].splitlines(True), + res.stdout.splitlines(True), fromfile=outputFn, tofile="returned")) logging.error(error_message) @@ -152,8 +149,8 @@ def bctest(testDir, testObj, buildenv): wantRC = 0 if "return_code" in testObj: wantRC = testObj['return_code'] - if proc.returncode != wantRC: - logging.error("Return code mismatch for " + outputFn) + if res.returncode != wantRC: + logging.error(f"Return code mismatch for {outputFn}; res: {str(res)}") raise Exception if "error_txt" in testObj: @@ -164,8 +161,8 @@ def bctest(testDir, testObj, buildenv): # emits DISPLAY errors when running as a windows application on # linux through wine. Just assert that the expected error text appears # somewhere in stderr. - if want_error not in outs[1]: - logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip()) + if want_error not in res.stderr: + logging.error(f"Error mismatch:\nExpected: {want_error}\nReceived: {res.stderr.rstrip()}\nres: {str(res)}") raise Exception def parse_output(a, fmt): diff --git a/vcpkg.json b/vcpkg.json index ecbccb072c..d5dfb05ebe 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,51 +1,64 @@ { + "$comment": "The builtin-baseline corresponds to 2024.09.30 Release", "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", - "builtin-baseline": "9edb1b8e590cc086563301d735cae4b6e732d2d2", - "overrides":[ - {"name": "libevent", "version": "2.1.12#7"}, - {"name": "liblzma", "version": "5.4.1#1"} - ], + "builtin-baseline": "c82f74667287d3dc386bce81e44964370c91a289", "dependencies": [ - "boost-date-time", "boost-multi-index", "boost-signals2", "libevent" ], "default-features": [ - "wallet", - "miniupnpc", - "zeromq", + "qt5", "tests", - "qt5" + "wallet", + "zeromq" ], "features": { - "wallet": { - "description": "Enable wallet", - "dependencies": [ "berkeleydb", "sqlite3" ] - }, - "sqlite": { - "description": "Enable SQLite wallet support", - "dependencies": [ "sqlite3" ] - }, "berkeleydb": { "description": "Enable Berkeley DB wallet support", - "dependencies": [ "berkeleydb" ] + "dependencies": [ + "berkeleydb" + ] }, - "miniupnpc": { - "description": "Enable UPnP", - "dependencies": [ "miniupnpc" ] + "qt5": { + "description": "Build GUI, Qt 5", + "dependencies": [ + "qt5-base", + "qt5-tools", + "libqrencode" + ] }, - "zeromq": { - "description": "Enable ZMQ notifications", - "dependencies": [ "zeromq" ] + "sqlite": { + "description": "Enable SQLite wallet support", + "dependencies": [ + "sqlite3" + ] }, "tests": { "description": "Build test_bitcoin.exe executable", - "dependencies": [ "boost-test" ] + "dependencies": [ + "boost-test" + ] }, - "qt5": { - "description": "Build GUI, Qt 5", - "dependencies": [ "qt5-base", "qt5-tools" ] + "wallet": { + "description": "Enable wallet", + "dependencies": [ + "berkeleydb", + "sqlite3" + ] + }, + "zeromq": { + "description": "Enable ZMQ notifications", + "dependencies": [ + "zeromq" + ] + } + }, + "overrides": [ + { + "$comment": "Newer unreleased libevent versions cause https://github.com/bitcoin/bitcoin/issues/30096", + "name": "libevent", + "version": "2.1.12#7" } - } + ] } |