diff options
218 files changed, 3072 insertions, 1598 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99be137a24..439d02cc8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,12 +67,12 @@ jobs: echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD $EXCLUDE_MERGE_BASE_ANCESTORS | head -1)" >> "$GITHUB_ENV" - run: | sudo apt-get update - sudo apt-get install clang ccache build-essential cmake pkg-config python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y + sudo apt-get install clang ccache build-essential cmake pkg-config python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libzmq3-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y - name: Compile and run tests run: | # Run tests on commits after the last merge commit and before the PR head commit # Use clang++, because it is a bit faster and uses less memory than g++ - git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && CC=clang CXX=clang++ cmake -B build -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWITH_BDB=ON -DWITH_NATPMP=ON -DWITH_MINIUPNPC=ON -DWITH_USDT=ON && cmake --build build -j $(nproc) && ctest --test-dir build -j $(nproc) && ./build/test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.TEST_BASE }} + git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && CC=clang CXX=clang++ cmake -B build -DWERROR=ON -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWITH_BDB=ON -DWITH_MINIUPNPC=ON -DWITH_USDT=ON && cmake --build build -j $(nproc) && ctest --test-dir build -j $(nproc) && ./build/test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.TEST_BASE }} macos-native-arm64: name: 'macOS 14 native, arm64, no depends, sqlite only, gui' @@ -105,7 +105,7 @@ jobs: run: | # A workaround for "The `brew link` step did not complete successfully" error. brew install --quiet python@3 || brew link --overwrite python@3 - brew install --quiet ninja pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq qt@5 qrencode + brew install --quiet coreutils ninja pkg-config gnu-getopt ccache boost libevent miniupnpc zeromq qt@5 qrencode - name: Set Ccache directory run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" diff --git a/.python-version b/.python-version index 43077b2460..1445aee866 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.9.18 +3.10.14 diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ae988ac90..edc4710637 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,11 +121,6 @@ option(REDUCE_EXPORTS "Attempt to reduce exported symbols in the resulting execu option(WERROR "Treat compiler warnings as errors." OFF) option(WITH_CCACHE "Attempt to use ccache for compiling." ON) -option(WITH_NATPMP "Enable NAT-PMP." OFF) -if(WITH_NATPMP) - find_package(NATPMP MODULE REQUIRED) -endif() - option(WITH_MINIUPNPC "Enable UPnP." OFF) if(WITH_MINIUPNPC) find_package(MiniUPnPc MODULE REQUIRED) @@ -182,7 +177,7 @@ if(BUILD_GUI) if(BUILD_GUI_TESTS) list(APPEND qt_components Test) endif() - find_package(Qt5 5.11.3 MODULE REQUIRED + find_package(Qt 5.11.3 MODULE REQUIRED COMPONENTS ${qt_components} ) unset(qt_components) @@ -239,7 +234,6 @@ if(BUILD_FOR_FUZZING) set(BUILD_WALLET_TOOL OFF) set(BUILD_GUI OFF) set(ENABLE_EXTERNAL_SIGNER OFF) - set(WITH_NATPMP OFF) set(WITH_MINIUPNPC OFF) set(WITH_ZMQ OFF) set(BUILD_TESTS OFF) @@ -546,7 +540,7 @@ if(WERROR) unset(werror_flag) endif() -find_package(Python3 3.9 COMPONENTS Interpreter) +find_package(Python3 3.10 COMPONENTS Interpreter) if(Python3_EXECUTABLE) set(PYTHON_COMMAND ${Python3_EXECUTABLE}) else() @@ -621,9 +615,7 @@ if(ENABLE_WALLET) message(" - legacy wallets (Berkeley DB) ..... ${WITH_BDB}") endif() message(" external signer ..................... ${ENABLE_EXTERNAL_SIGNER}") -message(" port mapping:") -message(" - using NAT-PMP .................... ${WITH_NATPMP}") -message(" - using UPnP ....................... ${WITH_MINIUPNPC}") +message(" port mapping using UPnP ............. ${WITH_MINIUPNPC}") message(" ZeroMQ .............................. ${WITH_ZMQ}") message(" USDT tracing ........................ ${WITH_USDT}") message(" QR code (GUI) ....................... ${WITH_QRENCODE}") diff --git a/CMakePresets.json b/CMakePresets.json index 041e3c1cbf..3bbb61afce 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -86,7 +86,6 @@ "WITH_BDB": "ON", "WITH_MINIUPNPC": "ON", "WITH_MULTIPROCESS": "ON", - "WITH_NATPMP": "ON", "WITH_QRENCODE": "ON", "WITH_SQLITE": "ON", "WITH_USDT": "ON", @@ -53,7 +53,8 @@ and extending unit tests can be found in [/src/test/README.md](/src/test/README. There are also [regression and integration tests](/test), written in Python. -These tests can be run (if the [test dependencies](/test) are installed) with: `test/functional/test_runner.py` +These tests can be run (if the [test dependencies](/test) are installed) with: `build/test/functional/test_runner.py` +(assuming `build` is your build directory). The CI (Continuous Integration) systems make sure that every pull request is built for Windows, Linux, and macOS, and that unit/sanity tests are run automatically. diff --git a/ci/test/00_setup_env_mac_native.sh b/ci/test/00_setup_env_mac_native.sh index 1e197d8c71..45d644d9ca 100755 --- a/ci/test/00_setup_env_mac_native.sh +++ b/ci/test/00_setup_env_mac_native.sh @@ -11,7 +11,7 @@ export LC_ALL=C.UTF-8 export PIP_PACKAGES="--break-system-packages zmq" export GOAL="install" export CMAKE_GENERATOR="Ninja" -export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DWITH_MINIUPNPC=ON -DWITH_NATPMP=ON -DREDUCE_EXPORTS=ON" +export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DWITH_MINIUPNPC=ON -DREDUCE_EXPORTS=ON" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index e27563f323..dc84ef49a4 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -19,7 +19,7 @@ else fi export CONTAINER_NAME=ci_native_asan -export PACKAGES="systemtap-sdt-dev clang-18 llvm-18 libclang-rt-18-dev python3-zmq qtbase5-dev qttools5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" +export PACKAGES="systemtap-sdt-dev clang-18 llvm-18 libclang-rt-18-dev python3-zmq qtbase5-dev qttools5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" export NO_DEPENDS=1 export GOAL="install" export BITCOIN_CONFIG="\ diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index 479628d3e8..3d5d1b7745 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel export CI_IMAGE_NAME_TAG="docker.io/debian:bookworm" -# Use minimum supported python3.9 (or best-effort 3.11) and clang-16, see doc/dependencies.md +# Use minimum supported python3.10 (or best-effort 3.11) and clang-16, see doc/dependencies.md export PACKAGES="python3-zmq clang-16 llvm-16 libc++abi-16-dev libc++-16-dev" export DEP_OPTS="NO_WALLET=1 CC=clang-16 CXX='clang++-16 -stdlib=libc++'" export GOAL="install" diff --git a/ci/test/00_setup_env_native_previous_releases.sh b/ci/test/00_setup_env_native_previous_releases.sh index 2482e545e1..717eb67a28 100755 --- a/ci/test/00_setup_env_native_previous_releases.sh +++ b/ci/test/00_setup_env_native_previous_releases.sh @@ -8,9 +8,9 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_previous_releases export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" -# Use minimum supported python3.9 (or best effort 3.10) and gcc-11, see doc/dependencies.md +# Use minimum supported python3.10 and gcc-11, see doc/dependencies.md export PACKAGES="gcc-11 g++-11 python3-zmq" -export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 CC=gcc-11 CXX=g++-11" +export DEP_OPTS="NO_UPNP=1 DEBUG=1 CC=gcc-11 CXX=g++-11" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index 4a6f5bb3e2..cc1dea09cb 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_tidy export TIDY_LLVM_V="18" -export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev qtbase5-dev qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" +export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq libevent-dev libboost-dev libminiupnpc-dev libzmq3-dev systemtap-sdt-dev qtbase5-dev qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false @@ -18,7 +18,7 @@ export RUN_CHECK_DEPS=true export RUN_TIDY=true export GOAL="install" export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DWITH_NATPMP=ON -DWITH_MINIUPNPC=ON -DWITH_USDT=ON -DWITH_BDB=ON -DWARN_INCOMPATIBLE_BDB=OFF \ + -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DWITH_MINIUPNPC=ON -DWITH_USDT=ON -DWITH_BDB=ON -DWARN_INCOMPATIBLE_BDB=OFF \ -DENABLE_HARDENING=OFF \ -DCMAKE_C_COMPILER=clang-${TIDY_LLVM_V} \ -DCMAKE_CXX_COMPILER=clang++-${TIDY_LLVM_V} \ diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 60bbe83119..3c5622cd02 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -8,14 +8,14 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang-16 llvm-16 libclang-rt-16-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" +export PACKAGES="valgrind clang-16 llvm-16 libclang-rt-16-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # feature_init excluded for now, see https://github.com/bitcoin/bitcoin/issues/30011 ; bind tests excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" # TODO enable GUI export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DWITH_BDB=ON -DWITH_NATPMP=ON -DWITH_MINIUPNPC=ON -DWARN_INCOMPATIBLE_BDB=OFF -DBUILD_GUI=OFF \ + -DWITH_ZMQ=ON -DWITH_BDB=ON -DWITH_MINIUPNPC=ON -DWARN_INCOMPATIBLE_BDB=OFF -DBUILD_GUI=OFF \ -DCMAKE_C_COMPILER=clang-16 \ -DCMAKE_CXX_COMPILER=clang++-16 \ " diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 538a58cbd5..72e3985a51 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -15,6 +15,8 @@ if [ "$(git config --global ${CFG_DONE})" == "true" ]; then exit 0 fi +MAKEJOBS="-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds. + if [ -n "$DPKG_ADD_ARCH" ]; then dpkg --add-architecture "$DPKG_ADD_ARCH" fi @@ -45,7 +47,7 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ -S /msan/llvm-project/llvm - ninja -C /msan/clang_build/ "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds + ninja -C /msan/clang_build/ "$MAKEJOBS" ninja -C /msan/clang_build/ install-runtimes update-alternatives --install /usr/bin/clang++ clang++ /msan/clang_build/bin/clang++ 100 @@ -64,7 +66,7 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then -DLIBCXX_HARDENING_MODE=debug \ -S /msan/llvm-project/runtimes - ninja -C /msan/cxx_build/ "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds + ninja -C /msan/cxx_build/ "$MAKEJOBS" # Clear no longer needed source folder du -sh /msan/llvm-project @@ -74,7 +76,7 @@ fi if [[ "${RUN_TIDY}" == "true" ]]; then ${CI_RETRY_EXE} git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_"${TIDY_LLVM_V}" /include-what-you-use cmake -B /iwyu-build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-"${TIDY_LLVM_V}" -S /include-what-you-use - make -C /iwyu-build/ install "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds + make -C /iwyu-build/ install "$MAKEJOBS" fi mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" diff --git a/ci/test/02_run_container.sh b/ci/test/02_run_container.sh index 1727f9296b..9069fba156 100755 --- a/ci/test/02_run_container.sh +++ b/ci/test/02_run_container.sh @@ -15,12 +15,22 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value and "HOME" != key and "PATH" != key and "USER" != key]' | tee "/tmp/env-$USER-$CONTAINER_NAME" # System-dependent env vars must be kept as is. So read them from the container. docker run --rm "${CI_IMAGE_NAME_TAG}" bash -c "env | grep --extended-regexp '^(HOME|PATH|USER)='" | tee --append "/tmp/env-$USER-$CONTAINER_NAME" + + # Env vars during the build can not be changed. For example, a modified + # $MAKEJOBS is ignored in the build process. Use --cpuset-cpus as an + # approximation to respect $MAKEJOBS somewhat, if cpuset is available. + MAYBE_CPUSET="" + if [ "$HAVE_CGROUP_CPUSET" ]; then + MAYBE_CPUSET="--cpuset-cpus=$( python3 -c "import random;P=$( nproc );M=min(P,int('$MAKEJOBS'.lstrip('-j')));print(','.join(map(str,sorted(random.sample(range(P),M)))))" )" + fi echo "Creating $CI_IMAGE_NAME_TAG container to run in" + # shellcheck disable=SC2086 DOCKER_BUILDKIT=1 docker build \ --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \ --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ --build-arg "FILE_ENV=${FILE_ENV}" \ + $MAYBE_CPUSET \ --label="${CI_IMAGE_LABEL}" \ --tag="${CONTAINER_NAME}" \ "${BASE_READ_ONLY_DIR}" diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index 1c1b5fa545..6eab4f7467 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -13,12 +13,11 @@ export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/l export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" +echo "Number of available processing units: $(nproc)" if [ "$CI_OS_NAME" == "macos" ]; then top -l 1 -s 0 | awk ' /PhysMem/ {print}' - echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" else free -m -h - echo "Number of CPUs (nproc): $(nproc)" echo "System info: $(uname --kernel-name --kernel-release)" lscpu fi @@ -146,7 +145,7 @@ if [ "$RUN_CHECK_DEPS" = "true" ]; then fi if [ "$RUN_UNIT_TESTS" = "true" ]; then - DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" CTEST_OUTPUT_ON_FAILURE=ON ctest "${MAKEJOBS}" + DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" CTEST_OUTPUT_ON_FAILURE=ON ctest "${MAKEJOBS}" --timeout $(( TEST_RUNNER_TIMEOUT_FACTOR * 60 )) fi if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then diff --git a/cmake/bitcoin-config.h.in b/cmake/bitcoin-build-config.h.in index 094eb8040a..094eb8040a 100644 --- a/cmake/bitcoin-config.h.in +++ b/cmake/bitcoin-build-config.h.in diff --git a/cmake/introspection.cmake b/cmake/introspection.cmake index 5435a109d4..29c93869a7 100644 --- a/cmake/introspection.cmake +++ b/cmake/introspection.cmake @@ -6,7 +6,7 @@ include(CheckCXXSourceCompiles) include(CheckCXXSymbolExists) include(CheckIncludeFileCXX) -# The following HAVE_{HEADER}_H variables go to the bitcoin-config.h header. +# The following HAVE_{HEADER}_H variables go to the bitcoin-build-config.h header. check_include_file_cxx(sys/prctl.h HAVE_SYS_PRCTL_H) check_include_file_cxx(sys/resources.h HAVE_SYS_RESOURCES_H) check_include_file_cxx(sys/vmmeter.h HAVE_SYS_VMMETER_H) diff --git a/cmake/module/FindNATPMP.cmake b/cmake/module/FindNATPMP.cmake deleted file mode 100644 index 930555232b..0000000000 --- a/cmake/module/FindNATPMP.cmake +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2023-present The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or https://opensource.org/license/mit/. - -find_path(NATPMP_INCLUDE_DIR - NAMES natpmp.h -) - -find_library(NATPMP_LIBRARY - NAMES natpmp -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(NATPMP - REQUIRED_VARS NATPMP_LIBRARY NATPMP_INCLUDE_DIR -) - -if(NATPMP_FOUND AND NOT TARGET NATPMP::NATPMP) - add_library(NATPMP::NATPMP UNKNOWN IMPORTED) - set_target_properties(NATPMP::NATPMP PROPERTIES - IMPORTED_LOCATION "${NATPMP_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${NATPMP_INCLUDE_DIR}" - ) - set_property(TARGET NATPMP::NATPMP PROPERTY - INTERFACE_COMPILE_DEFINITIONS USE_NATPMP=1 $<$<PLATFORM_ID:Windows>:NATPMP_STATICLIB> - ) -endif() - -mark_as_advanced( - NATPMP_INCLUDE_DIR - NATPMP_LIBRARY -) diff --git a/cmake/module/FindQt5.cmake b/cmake/module/FindQt.cmake index f39ee53d5b..2e43294a99 100644 --- a/cmake/module/FindQt5.cmake +++ b/cmake/module/FindQt.cmake @@ -3,10 +3,10 @@ # file COPYING or https://opensource.org/license/mit/. #[=======================================================================[ -FindQt5 -------- +FindQt +------ -Finds the Qt 5 headers and libraries. +Finds the Qt headers and libraries. This is a wrapper around find_package() command that: - facilitates searching in various build environments @@ -19,7 +19,7 @@ if(CMAKE_HOST_APPLE) find_program(HOMEBREW_EXECUTABLE brew) if(HOMEBREW_EXECUTABLE) execute_process( - COMMAND ${HOMEBREW_EXECUTABLE} --prefix qt@5 + COMMAND ${HOMEBREW_EXECUTABLE} --prefix qt@${Qt_FIND_VERSION_MAJOR} OUTPUT_VARIABLE _qt_homebrew_prefix ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE @@ -40,10 +40,10 @@ endif() # /usr/x86_64-w64-mingw32/lib/libm.a or /usr/arm-linux-gnueabihf/lib/libm.a. set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH) -find_package(Qt5 ${Qt5_FIND_VERSION} - COMPONENTS ${Qt5_FIND_COMPONENTS} +find_package(Qt${Qt_FIND_VERSION_MAJOR} ${Qt_FIND_VERSION} + COMPONENTS ${Qt_FIND_COMPONENTS} HINTS ${_qt_homebrew_prefix} - PATH_SUFFIXES Qt5 # Required on OpenBSD systems. + PATH_SUFFIXES Qt${Qt_FIND_VERSION_MAJOR} # Required on OpenBSD systems. ) unset(_qt_homebrew_prefix) @@ -56,11 +56,11 @@ else() endif() include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Qt5 - REQUIRED_VARS Qt5_DIR - VERSION_VAR Qt5_VERSION +find_package_handle_standard_args(Qt + REQUIRED_VARS Qt${Qt_FIND_VERSION_MAJOR}_DIR + VERSION_VAR Qt${Qt_FIND_VERSION_MAJOR}_VERSION ) -foreach(component IN LISTS Qt5_FIND_COMPONENTS ITEMS "") - mark_as_advanced(Qt5${component}_DIR) +foreach(component IN LISTS Qt_FIND_COMPONENTS ITEMS "") + mark_as_advanced(Qt${Qt_FIND_VERSION_MAJOR}${component}_DIR) endforeach() diff --git a/contrib/devtools/bitcoin-tidy/CMakeLists.txt b/contrib/devtools/bitcoin-tidy/CMakeLists.txt index 95345b4782..c6f683f7ab 100644 --- a/contrib/devtools/bitcoin-tidy/CMakeLists.txt +++ b/contrib/devtools/bitcoin-tidy/CMakeLists.txt @@ -25,7 +25,7 @@ find_program(CLANG_TIDY_EXE NAMES "clang-tidy-${LLVM_VERSION_MAJOR}" "clang-tidy message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Found clang-tidy: ${CLANG_TIDY_EXE}") -add_library(bitcoin-tidy MODULE bitcoin-tidy.cpp logprintf.cpp nontrivial-threadlocal.cpp) +add_library(bitcoin-tidy MODULE bitcoin-tidy.cpp nontrivial-threadlocal.cpp) target_include_directories(bitcoin-tidy SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) # Disable RTTI and exceptions as necessary @@ -58,7 +58,7 @@ else() endif() # Create a dummy library that runs clang-tidy tests as a side-effect of building -add_library(bitcoin-tidy-tests OBJECT EXCLUDE_FROM_ALL example_logprintf.cpp example_nontrivial-threadlocal.cpp) +add_library(bitcoin-tidy-tests OBJECT EXCLUDE_FROM_ALL example_nontrivial-threadlocal.cpp) add_dependencies(bitcoin-tidy-tests bitcoin-tidy) set_target_properties(bitcoin-tidy-tests PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_COMMAND}") diff --git a/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp b/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp index 1ef4494973..f2658b5a58 100644 --- a/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp +++ b/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp @@ -2,7 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "logprintf.h" #include "nontrivial-threadlocal.h" #include <clang-tidy/ClangTidyModule.h> @@ -13,7 +12,6 @@ class BitcoinModule final : public clang::tidy::ClangTidyModule public: void addCheckFactories(clang::tidy::ClangTidyCheckFactories& CheckFactories) override { - CheckFactories.registerCheck<bitcoin::LogPrintfCheck>("bitcoin-unterminated-logprintf"); CheckFactories.registerCheck<bitcoin::NonTrivialThreadLocal>("bitcoin-nontrivial-threadlocal"); } }; diff --git a/contrib/devtools/bitcoin-tidy/example_logprintf.cpp b/contrib/devtools/bitcoin-tidy/example_logprintf.cpp deleted file mode 100644 index dc77f668e3..0000000000 --- a/contrib/devtools/bitcoin-tidy/example_logprintf.cpp +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (c) 2023 Bitcoin Developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <string> - -// Test for bitcoin-unterminated-logprintf - -enum LogFlags { - NONE -}; - -enum Level { - None -}; - -template <typename... Args> -static inline void LogPrintf_(const std::string& logging_function, const std::string& source_file, const int source_line, const LogFlags flag, const Level level, const char* fmt, const Args&... args) -{ -} - -#define LogPrintLevel_(category, level, ...) LogPrintf_(__func__, __FILE__, __LINE__, category, level, __VA_ARGS__) -#define LogPrintf(...) LogPrintLevel_(LogFlags::NONE, Level::None, __VA_ARGS__) - -#define LogDebug(category, ...) \ - do { \ - LogPrintf(__VA_ARGS__); \ - } while (0) - - -class CWallet -{ - std::string GetDisplayName() const - { - return "default wallet"; - } - -public: - template <typename... Params> - void WalletLogPrintf(const char* fmt, Params... parameters) const - { - LogPrintf(("%s " + std::string{fmt}).c_str(), GetDisplayName(), parameters...); - }; -}; - -struct ScriptPubKeyMan -{ - std::string GetDisplayName() const - { - return "default wallet"; - } - - template <typename... Params> - void WalletLogPrintf(const char* fmt, Params... parameters) const - { - LogPrintf(("%s " + std::string{fmt}).c_str(), GetDisplayName(), parameters...); - }; -}; - -void good_func() -{ - LogPrintf("hello world!\n"); -} -void good_func2() -{ - CWallet wallet; - wallet.WalletLogPrintf("hi\n"); - ScriptPubKeyMan spkm; - spkm.WalletLogPrintf("hi\n"); - - const CWallet& walletref = wallet; - walletref.WalletLogPrintf("hi\n"); - - auto* walletptr = new CWallet(); - walletptr->WalletLogPrintf("hi\n"); - delete walletptr; -} -void bad_func() -{ - LogPrintf("hello world!"); -} -void bad_func2() -{ - LogPrintf(""); -} -void bad_func3() -{ - // Ending in "..." has no special meaning. - LogPrintf("hello world!..."); -} -void bad_func4_ignored() -{ - LogPrintf("hello world!"); // NOLINT(bitcoin-unterminated-logprintf) -} -void bad_func5() -{ - CWallet wallet; - wallet.WalletLogPrintf("hi"); - ScriptPubKeyMan spkm; - spkm.WalletLogPrintf("hi"); - - const CWallet& walletref = wallet; - walletref.WalletLogPrintf("hi"); - - auto* walletptr = new CWallet(); - walletptr->WalletLogPrintf("hi"); - delete walletptr; -} diff --git a/contrib/devtools/bitcoin-tidy/logprintf.cpp b/contrib/devtools/bitcoin-tidy/logprintf.cpp deleted file mode 100644 index 36beac28c8..0000000000 --- a/contrib/devtools/bitcoin-tidy/logprintf.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2023 Bitcoin Developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "logprintf.h" - -#include <clang/AST/ASTContext.h> -#include <clang/ASTMatchers/ASTMatchFinder.h> - - -namespace { -AST_MATCHER(clang::StringLiteral, unterminated) -{ - size_t len = Node.getLength(); - if (len > 0 && Node.getCodeUnit(len - 1) == '\n') { - return false; - } - return true; -} -} // namespace - -namespace bitcoin { - -void LogPrintfCheck::registerMatchers(clang::ast_matchers::MatchFinder* finder) -{ - using namespace clang::ast_matchers; - - /* - Logprintf(..., ..., ..., ..., ..., "foo", ...) - */ - - finder->addMatcher( - callExpr( - callee(functionDecl(hasName("LogPrintf_"))), - hasArgument(5, stringLiteral(unterminated()).bind("logstring"))), - this); - - /* - auto walletptr = &wallet; - wallet.WalletLogPrintf("foo"); - wallet->WalletLogPrintf("foo"); - */ - finder->addMatcher( - cxxMemberCallExpr( - callee(cxxMethodDecl(hasName("WalletLogPrintf"))), - hasArgument(0, stringLiteral(unterminated()).bind("logstring"))), - this); -} - -void LogPrintfCheck::check(const clang::ast_matchers::MatchFinder::MatchResult& Result) -{ - if (const clang::StringLiteral* lit = Result.Nodes.getNodeAs<clang::StringLiteral>("logstring")) { - const clang::ASTContext& ctx = *Result.Context; - const auto user_diag = diag(lit->getEndLoc(), "Unterminated format string used with LogPrintf"); - const auto& loc = lit->getLocationOfByte(lit->getByteLength(), *Result.SourceManager, ctx.getLangOpts(), ctx.getTargetInfo()); - user_diag << clang::FixItHint::CreateInsertion(loc, "\\n"); - } -} - -} // namespace bitcoin diff --git a/contrib/devtools/bitcoin-tidy/logprintf.h b/contrib/devtools/bitcoin-tidy/logprintf.h deleted file mode 100644 index db95dfe143..0000000000 --- a/contrib/devtools/bitcoin-tidy/logprintf.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2023 Bitcoin Developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef LOGPRINTF_CHECK_H -#define LOGPRINTF_CHECK_H - -#include <clang-tidy/ClangTidyCheck.h> - -namespace bitcoin { - -// Warn about any use of LogPrintf that does not end with a newline. -class LogPrintfCheck final : public clang::tidy::ClangTidyCheck -{ -public: - LogPrintfCheck(clang::StringRef Name, clang::tidy::ClangTidyContext* Context) - : clang::tidy::ClangTidyCheck(Name, Context) {} - - bool isLanguageVersionSupported(const clang::LangOptions& LangOpts) const override - { - return LangOpts.CPlusPlus; - } - void registerMatchers(clang::ast_matchers::MatchFinder* Finder) override; - void check(const clang::ast_matchers::MatchFinder::MatchResult& Result) override; -}; - -} // namespace bitcoin - -#endif // LOGPRINTF_CHECK_H diff --git a/contrib/devtools/check-deps.sh b/contrib/devtools/check-deps.sh index 18d5fe0eb5..cdfc4e7533 100755 --- a/contrib/devtools/check-deps.sh +++ b/contrib/devtools/check-deps.sh @@ -133,7 +133,7 @@ check_disallowed() { dst_obj=$(obj_names "$symbol" "${temp_dir}/${dst}_exports.txt") while read src_obj; do if ! check_suppress "$src_obj" "$dst_obj" "$symbol"; then - echo "Error: $src_obj depends on $dst_obj symbol '$(c++filt "$symbol")', can suppess with:" + echo "Error: $src_obj depends on $dst_obj symbol '$(c++filt "$symbol")', can suppress with:" echo " SUPPRESS[\"$src_obj $dst_obj $symbol\"]=1" result=1 fi @@ -145,7 +145,7 @@ check_disallowed() { # Declare array to track errors which were suppressed. declare -A SUPPRESSED -# Return whether error should be suppressed and record suppresssion in +# Return whether error should be suppressed and record suppression in # SUPPRESSED array. check_suppress() { local src_obj="$1" @@ -161,7 +161,7 @@ check_suppress() { return 1 } -# Warn about error which were supposed to be suppress, but were not encountered. +# Warn about error which were supposed to be suppressed, but were not encountered. check_not_suppressed() { for suppress in "${!SUPPRESS[@]}"; do if [[ ! -v SUPPRESSED[$suppress] ]]; then @@ -194,7 +194,10 @@ cd "$BUILD_DIR/src" extract_symbols "$TEMP_DIR" if check_libraries "$TEMP_DIR"; then echo "Success! No unexpected dependencies were detected." + RET=0 else echo >&2 "Error: Unexpected dependencies were detected. Check previous output." + RET=1 fi rm -r "$TEMP_DIR" +exit $RET diff --git a/contrib/devtools/gen-bitcoin-conf.sh b/contrib/devtools/gen-bitcoin-conf.sh index 2ebbd42022..d830852c9e 100755 --- a/contrib/devtools/gen-bitcoin-conf.sh +++ b/contrib/devtools/gen-bitcoin-conf.sh @@ -72,9 +72,12 @@ cat >> "${EXAMPLE_CONF_FILE}" << 'EOF' # Options for mainnet [main] -# Options for testnet +# Options for testnet3 [test] +# Options for testnet4 +[testnet4] + # Options for signet [signet] diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 1722c7d290..3f6010280a 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -235,7 +235,7 @@ def check_MACHO_libraries(binary) -> bool: return ok def check_MACHO_min_os(binary) -> bool: - if binary.build_version.minos == [11,0,0]: + if binary.build_version.minos == [13,0,0]: return True return False diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index 454dbc6bd2..c75a5e1546 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -116,7 +116,7 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cxx, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,11.0', '-Wl,11.4']), + self.assertEqual(call_symbol_check(cxx, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,13.0', '-Wl,11.4']), (1, f'{executable}: failed SDK')) def test_PE(self): diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 1ffc22a76b..3184cd4afe 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -230,8 +230,6 @@ case "$HOST" in *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac -# Make $HOST-specific native binaries from depends available in $PATH -export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" mkdir -p "$DISTSRC" ( cd "$DISTSRC" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 3da98cf651..e5ebecbf93 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -469,6 +469,7 @@ inspecting signatures in Mach-O binaries.") `(append ,flags ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html (list "--enable-stack-protector=all", + "--enable-cet", "--enable-bind-now", "--disable-werror", building-on))) diff --git a/contrib/signet/README.md b/contrib/signet/README.md index 706b296c54..5fcd8944e6 100644 --- a/contrib/signet/README.md +++ b/contrib/signet/README.md @@ -23,9 +23,8 @@ miner You will first need to pick a difficulty target. Since signet chains are primarily protected by a signature rather than proof of work, there is no need to spend as much energy as possible mining, however you may wish to choose to spend more time than the absolute minimum. The calibrate subcommand can be used to pick a target appropriate for your hardware, eg: - cd src/ - MINER="../contrib/signet/miner" - GRIND="./bitcoin-util grind" + MINER="./contrib/signet/miner" + GRIND="./build/src/bitcoin-util grind" $MINER calibrate --grind-cmd="$GRIND" nbits=1e00f403 for 25s average mining time @@ -33,7 +32,7 @@ It defaults to estimating an nbits value resulting in 25s average time to find a To mine the first block in your custom chain, you can run: - CLI="./bitcoin-cli -conf=mysignet.conf" + CLI="./build/src/bitcoin-cli -conf=mysignet.conf" ADDR=$($CLI -signet getnewaddress) NBITS=1e00f403 $MINER --cli="$CLI" generate --grind-cmd="$GRIND" --address="$ADDR" --nbits=$NBITS diff --git a/contrib/tracing/connectblock_benchmark.bt b/contrib/tracing/connectblock_benchmark.bt index 4aa4742103..de112af639 100755 --- a/contrib/tracing/connectblock_benchmark.bt +++ b/contrib/tracing/connectblock_benchmark.bt @@ -82,7 +82,7 @@ usdt:./build/src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 @inputs = @inputs + $inputs; @sigops = @sigops + $sigops; - @durations = hist($duration / 1000); + @durations = hist($duration / 1e6); if ($height == $1 && $height != 0) { @start = nsecs; @@ -92,7 +92,7 @@ usdt:./build/src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 if ($2 > 0 && $height >= $2) { @end = nsecs; $duration = @end - @start; - printf("\nTook %d ms to connect the blocks between height %d and %d.\n", $duration / 1000000, $1, $2); + printf("\nTook %d ms to connect the blocks between height %d and %d.\n", $duration / 1e9, $1, $2); exit(); } } @@ -102,7 +102,7 @@ usdt:./build/src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 blocks where the time it took to connect the block is above the <logging threshold in ms>. */ -usdt:./build/src/bitcoind:validation:block_connected / (uint64) arg5 / 1000> $3 / +usdt:./build/src/bitcoind:validation:block_connected / (uint64) arg5 / 1e6 > $3 / { $hash = arg0; $height = (int32) arg1; @@ -120,7 +120,7 @@ usdt:./build/src/bitcoind:validation:block_connected / (uint64) arg5 / 1000> $3 printf("%02x", $b); $p -= 1; } - printf(") %4d tx %5d ins %5d sigops took %4d ms\n", $transactions, $inputs, $sigops, (uint64) $duration / 1000); + printf(") %4d tx %5d ins %5d sigops took %4d ms\n", $transactions, $inputs, $sigops, (uint64) $duration / 1e6); } diff --git a/depends/Makefile b/depends/Makefile index ad1fb2b049..f1dc300b7a 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -42,7 +42,6 @@ NO_WALLET ?= NO_ZMQ ?= NO_UPNP ?= NO_USDT ?= -NO_NATPMP ?= MULTIPROCESS ?= LTO ?= NO_HARDEN ?= @@ -159,13 +158,12 @@ sqlite_packages_$(NO_SQLITE) = $(sqlite_packages) wallet_packages_$(NO_WALLET) = $(bdb_packages_) $(sqlite_packages_) upnp_packages_$(NO_UPNP) = $(upnp_packages) -natpmp_packages_$(NO_NATPMP) = $(natpmp_packages) zmq_packages_$(NO_ZMQ) = $(zmq_packages) multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) usdt_packages_$(NO_USDT) = $(usdt_$(host_os)_packages) -packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) +packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(usdt_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) ifneq ($(zmq_packages_),) @@ -191,6 +189,7 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) echo copying packages: $^ echo to: $(@D) cd $(@D); $(foreach package,$^, $(build_TAR) xf $($(package)_cached); ) + echo To build Bitcoin Core with these packages, pass \'--toolchain $(@D)/toolchain.cmake\' to the first CMake invocation. touch $@ ifeq ($(host),$(build)) @@ -233,7 +232,6 @@ $(host_prefix)/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_$(fina -e 's|@bdb_packages@|$(bdb_packages_)|' \ -e 's|@sqlite_packages@|$(sqlite_packages_)|' \ -e 's|@upnp_packages@|$(upnp_packages_)|' \ - -e 's|@natpmp_packages@|$(natpmp_packages_)|' \ -e 's|@usdt_packages@|$(usdt_packages_)|' \ -e 's|@no_harden@|$(NO_HARDEN)|' \ -e 's|@multiprocess@|$(MULTIPROCESS)|' \ diff --git a/depends/README.md b/depends/README.md index 38a0d8ccd4..4ef7247ea4 100644 --- a/depends/README.md +++ b/depends/README.md @@ -54,7 +54,7 @@ For more information, see [SDK Extraction](../contrib/macdeploy/README.md#sdk-ex #### For Win64 cross compilation -- see [build-windows.md](../doc/build-windows.md#cross-compilation-for-ubuntu-and-windows-subsystem-for-linux) + apt install g++-mingw-w64-x86-64-posix #### For linux (including i386, ARM) cross compilation @@ -113,7 +113,6 @@ The following can be set when running make: `make FOO=bar` - `NO_BDB`: Don't download/build/cache BerkeleyDB - `NO_SQLITE`: Don't download/build/cache SQLite - `NO_UPNP`: Don't download/build/cache packages needed for enabling UPnP -- `NO_NATPMP`: Don't download/build/cache packages needed for enabling NAT-PMP - `NO_USDT`: Don't download/build/cache packages needed for enabling USDT tracepoints - `MULTIPROCESS`: Build libmultiprocess (experimental) - `DEBUG`: Disable some optimizations and enable more runtime checking diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index a27d8b323b..4659d52912 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,4 +1,4 @@ -OSX_MIN_VERSION=11.0 +OSX_MIN_VERSION=13.0 OSX_SDK_VERSION=14.0 XCODE_VERSION=15.0 XCODE_BUILD_ID=15A240d diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index c09f7b1e3a..755d7aebe4 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -1,3 +1,6 @@ +ifneq ($(shell $(SHELL) $(.SHELLFLAGS) "command -v $(host)-gcc-posix"),) +mingw32_CC := $(host)-gcc-posix +endif ifneq ($(shell $(SHELL) $(.SHELLFLAGS) "command -v $(host)-g++-posix"),) mingw32_CXX := $(host)-g++-posix endif diff --git a/depends/packages/libnatpmp.mk b/depends/packages/libnatpmp.mk deleted file mode 100644 index 5a573a18e7..0000000000 --- a/depends/packages/libnatpmp.mk +++ /dev/null @@ -1,20 +0,0 @@ -package=libnatpmp -$(package)_version=f2433bec24ca3d3f22a8a7840728a3ac177f94ba -$(package)_download_path=https://github.com/miniupnp/libnatpmp/archive -$(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=ef84979950dfb3556705b63c9cd6c95501b75e887fba466234b187f3c9029669 -$(package)_build_subdir=build - -define $(package)_config_cmds - $($(package)_cmake) -S .. -B . -endef - -define $(package)_build_cmds - $(MAKE) natpmp -endef - -define $(package)_stage_cmds - mkdir -p $($(package)_staging_prefix_dir)/include $($(package)_staging_prefix_dir)/lib && \ - install ../natpmp.h ../natpmp_declspec.h $($(package)_staging_prefix_dir)/include && \ - install libnatpmp.a $($(package)_staging_prefix_dir)/lib -endef diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index 3fa5faa4ba..7c69e0f0c6 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -1,8 +1,8 @@ package=native_libmultiprocess -$(package)_version=015e95f7ebaa47619a213a19801e7fffafc56864 +$(package)_version=abe254b9734f2e2b220d1456de195532d6e6ac1e $(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=4b1266b121337f3f6f37e1863fba91c1a5ee9ad126bcffc6fe6b9ca47ad050a1 +$(package)_sha256_hash=85777073259fdc75d24ac5777a19991ec1156c5f12db50b252b861c95dcb4f46 $(package)_dependencies=native_capnp define $(package)_config_cmds diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 01ed0d7a92..08a91cbcbd 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -18,7 +18,6 @@ sqlite_packages=sqlite zmq_packages=zeromq upnp_packages=miniupnpc -natpmp_packages=libnatpmp multiprocess_packages = libmultiprocess capnp multiprocess_native_packages = native_libmultiprocess native_capnp diff --git a/depends/toolchain.cmake.in b/depends/toolchain.cmake.in index d805c1c4f9..735ebc8ea4 100644 --- a/depends/toolchain.cmake.in +++ b/depends/toolchain.cmake.in @@ -146,13 +146,6 @@ else() set(WITH_MINIUPNPC ON CACHE BOOL "") endif() -set(natpmp_packages @natpmp_packages@) -if("${natpmp_packages}" STREQUAL "") - set(WITH_NATPMP OFF CACHE BOOL "") -else() - set(WITH_NATPMP ON CACHE BOOL "") -endif() - set(usdt_packages @usdt_packages@) if("${usdt_packages}" STREQUAL "") set(WITH_USDT OFF CACHE BOOL "") diff --git a/doc/README.md b/doc/README.md index 7f5db1b5bf..79ca53ce76 100644 --- a/doc/README.md +++ b/doc/README.md @@ -3,7 +3,7 @@ Bitcoin Core Setup --------------------- -Bitcoin Core is the original Bitcoin client and it builds the backbone of the network. It downloads and, by default, stores the entire history of Bitcoin transactions, which requires a few hundred gigabytes of disk space. Depending on the speed of your computer and network connection, the synchronization process can take anywhere from a few hours to a day or more. +Bitcoin Core is the original Bitcoin client and it builds the backbone of the network. It downloads and, by default, stores the entire history of Bitcoin transactions, which requires several hundred gigabytes or more of disk space. Depending on the speed of your computer and network connection, the synchronization process can take anywhere from a few hours to several days or more. To download Bitcoin Core, visit [bitcoincore.org](https://bitcoincore.org/en/download/). diff --git a/doc/bitcoin-conf.md b/doc/bitcoin-conf.md index 76711d0e7d..9b31879790 100644 --- a/doc/bitcoin-conf.md +++ b/doc/bitcoin-conf.md @@ -31,7 +31,7 @@ Comments may appear in two ways: ### Network specific options Network specific options can be: -- placed into sections with headers `[main]` (not `[mainnet]`), `[test]` (not `[testnet]`), `[signet]` or `[regtest]`; +- placed into sections with headers `[main]` (not `[mainnet]`), `[test]` (not `[testnet]`, for testnet3), `[testnet4]`, `[signet]` or `[regtest]`; - prefixed with a chain name; e.g., `regtest.maxmempool=100`. Network specific options take precedence over non-network specific options. diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index 6a25e9a834..8b3b10ab85 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -42,7 +42,7 @@ from ports. However, you can build DB 4.8 yourself [using depends](/depends). ```bash pkg install gmake -gmake -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 +gmake -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 ``` When the build is complete, the Berkeley DB installation location will be displayed: diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index fafc91fc5f..908b750a8f 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -44,7 +44,7 @@ from ports. However you can build it yourself, [using depends](/depends). Refer to [depends/README.md](/depends/README.md) for detailed instructions. ```bash -gmake -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 +gmake -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 ... to: /path/to/bitcoin/depends/*-unknown-openbsd* ``` diff --git a/doc/build-osx.md b/doc/build-osx.md index 600eebb6ff..cb8e82dae8 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -1,15 +1,15 @@ # macOS Build Guide -**Updated for MacOS [14](https://www.apple.com/macos/sonoma/)** +**Updated for MacOS [15](https://www.apple.com/macos/macos-sequoia/)** -This guide describes how to build bitcoind, command-line utilities, and GUI on macOS +This guide describes how to build bitcoind, command-line utilities, and GUI on macOS. ## Preparation The commands in this guide should be executed in a Terminal application. macOS comes with a built-in Terminal located in: -``` +```bash /Applications/Utilities/Terminal.app ``` @@ -51,20 +51,6 @@ To install, run the following from your terminal: brew install cmake boost pkg-config libevent ``` -For macOS 11 (Big Sur) and 12 (Monterey) you need to install a more recent version of llvm. - -``` bash -brew install llvm -``` - -And append the following to the configure commands below: - -``` bash --DCMAKE_C_COMPILER="$(brew --prefix llvm)/bin/clang" -DCMAKE_CXX_COMPILER="$(brew --prefix llvm)/bin/clang++" -``` - -Try `llvm@17` if compilation fails with the default version of llvm. - ### 4. Clone Bitcoin repository `git` should already be installed by default on your system. @@ -135,17 +121,6 @@ Skip if you do not need this functionality. brew install miniupnpc ``` -###### libnatpmp - -libnatpmp may be used for NAT-PMP port mapping. -Skip if you do not need this functionality. - -``` bash -brew install libnatpmp -``` - -Check out the [further configuration](#further-configuration) section for more information. - --- #### ZMQ Dependencies diff --git a/doc/build-unix.md b/doc/build-unix.md index 4c3c659bff..086731be5a 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -60,9 +60,9 @@ executables, which are based on BerkeleyDB 4.8. Otherwise, you can build Berkele To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) -Optional port mapping libraries (see: `-DWITH_MINIUPNPC=ON` and `-DWITH_NATPMP=ON`): +Optional port mapping library (see: `-DWITH_MINIUPNPC=ON`): - sudo apt install libminiupnpc-dev libnatpmp-dev + sudo apt install libminiupnpc-dev ZMQ dependencies (provides ZMQ API): @@ -112,9 +112,9 @@ are based on Berkeley DB 4.8. Otherwise, you can build Berkeley DB [yourself](#b To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) -Optional port mapping libraries (see: `-DWITH_MINIUPNPC=ON` and `-DWITH_NATPMP=ON`): +Optional port mapping library (see: `-DWITH_MINIUPNPC=ON`): - sudo dnf install miniupnpc-devel libnatpmp-devel + sudo dnf install miniupnpc-devel ZMQ dependencies (provides ZMQ API): @@ -153,7 +153,7 @@ The legacy wallet uses Berkeley DB. To ensure backwards compatibility it is recommended to use Berkeley DB 4.8. If you have to build it yourself, and don't want to use any other libraries built in depends, you can do: ```bash -make -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 +make -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 ... to: /path/to/bitcoin/depends/x86_64-pc-linux-gnu ``` diff --git a/doc/build-windows.md b/doc/build-windows.md index 2e7b93da35..0c1418bff9 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -28,36 +28,18 @@ The steps below can be performed on Ubuntu or WSL. The depends system will also work on other Linux distributions, however the commands for installing the toolchain will be different. -First, install the general dependencies: - - sudo apt update - sudo apt upgrade - sudo apt install cmake curl g++ git make pkg-config - -A host toolchain (`g++`) is necessary because some dependency -packages need to build host utilities that are used in the build process. - -See [dependencies.md](dependencies.md) for a complete overview. +See [README.md](../depends/README.md) in the depends directory for which +dependencies to install and [dependencies.md](dependencies.md) for a complete overview. If you want to build the Windows installer using the `deploy` build target, you will need [NSIS](https://nsis.sourceforge.io/Main_Page): - sudo apt install nsis + apt install nsis Acquire the source in the usual way: git clone https://github.com/bitcoin/bitcoin.git cd bitcoin -## Building for 64-bit Windows - -The first step is to install the mingw-w64 cross-compilation toolchain: - -```sh -sudo apt install g++-mingw-w64-x86-64-posix -``` - -Once the toolchain is installed the build steps are common: - Note that for WSL the Bitcoin Core source path MUST be somewhere in the default mount file system, for example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. This means you cannot use a directory that is located directly on the host Windows file system to perform the build. diff --git a/doc/dependencies.md b/doc/dependencies.md index 4c620cf876..a3d42fc281 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -9,7 +9,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Clang](https://clang.llvm.org) | [16.0](https://github.com/bitcoin/bitcoin/pull/30263) | | [CMake](https://cmake.org/) | [3.22](https://github.com/bitcoin/bitcoin/pull/30454) | | [GCC](https://gcc.gnu.org) | [11.1](https://github.com/bitcoin/bitcoin/pull/29091) | -| [Python](https://www.python.org) (scripts, tests) | [3.9](https://github.com/bitcoin/bitcoin/pull/28211) | +| [Python](https://www.python.org) (scripts, tests) | [3.10](https://github.com/bitcoin/bitcoin/pull/30527) | | [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | ## Required @@ -34,7 +34,6 @@ You can find installation instructions in the `build-*.md` file for your platfor ### Networking | Dependency | Releases | Version used | Minimum required | Runtime | | --- | --- | --- | --- | --- | -| [libnatpmp](../depends/packages/libnatpmp.mk) | [link](https://github.com/miniupnp/libnatpmp/) | commit [f2433be...](https://github.com/bitcoin/bitcoin/pull/29708) | | No | | [MiniUPnPc](../depends/packages/miniupnpc.mk) | [link](https://miniupnp.tuxfamily.org/) | [2.2.7](https://github.com/bitcoin/bitcoin/pull/29707) | 2.1 | No | ### Notifications diff --git a/doc/fuzzing.md b/doc/fuzzing.md index f9630cca0f..927b0dc8d5 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -206,82 +206,6 @@ $ FUZZ=process_message ./honggfuzz/honggfuzz -i inputs/ -- build_fuzz/src/test/f Read the [Honggfuzz documentation](https://github.com/google/honggfuzz/blob/master/docs/USAGE.md) for more information. -## Fuzzing the Bitcoin Core P2P layer using Honggfuzz NetDriver - -Honggfuzz NetDriver allows for very easy fuzzing of TCP servers such as Bitcoin -Core without having to write any custom fuzzing harness. The `bitcoind` server -process is largely fuzzed without modification. - -This makes the fuzzing highly realistic: a bug reachable by the fuzzer is likely -also remotely triggerable by an untrusted peer. - -To quickly get started fuzzing the P2P layer using Honggfuzz NetDriver: - -```sh -$ mkdir bitcoin-honggfuzz-p2p/ -$ cd bitcoin-honggfuzz-p2p/ -$ git clone https://github.com/bitcoin/bitcoin -$ cd bitcoin/ -$ git clone https://github.com/google/honggfuzz -$ cd honggfuzz/ -$ make -$ cd .. -$ git apply << "EOF" -diff --git a/src/compat/compat.h b/src/compat/compat.h -index 8195bceaec..cce2b31ff0 100644 ---- a/src/compat/compat.h -+++ b/src/compat/compat.h -@@ -90,8 +90,12 @@ typedef char* sockopt_arg_type; - // building with a binutils < 2.36 is subject to this ld bug. - #define MAIN_FUNCTION __declspec(dllexport) int main(int argc, char* argv[]) - #else -+#ifdef HFND_FUZZING_ENTRY_FUNCTION_CXX -+#define MAIN_FUNCTION HFND_FUZZING_ENTRY_FUNCTION_CXX(int argc, char* argv[]) -+#else - #define MAIN_FUNCTION int main(int argc, char* argv[]) - #endif -+#endif - - // Note these both should work with the current usage of poll, but best to be safe - // WIN32 poll is broken https://daniel.haxx.se/blog/2012/10/10/wsapoll-is-broken/ -diff --git a/src/net.cpp b/src/net.cpp -index 7601a6ea84..702d0f56ce 100644 ---- a/src/net.cpp -+++ b/src/net.cpp -@@ -727,7 +727,7 @@ int V1TransportDeserializer::readHeader(Span<const uint8_t> msg_bytes) - } - - // Check start string, network magic -- if (memcmp(hdr.pchMessageStart, m_chain_params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { -+ if (false && memcmp(hdr.pchMessageStart, m_chain_params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { // skip network magic checking - LogDebug(BCLog::NET, "Header error: Wrong MessageStart %s received, peer=%d\n", HexStr(hdr.pchMessageStart), m_node_id); - return -1; - } -@@ -788,7 +788,7 @@ CNetMessage V1TransportDeserializer::GetMessage(const std::chrono::microseconds - RandAddEvent(ReadLE32(hash.begin())); - - // Check checksum and header message type string -- if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { -+ if (false && memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { // skip checksum checking - LogDebug(BCLog::NET, "Header error: Wrong checksum (%s, %u bytes), expected %s was %s, peer=%d\n", - SanitizeString(msg.m_type), msg.m_message_size, - HexStr(Span{hash}.first(CMessageHeader::CHECKSUM_SIZE)), -EOF -$ cmake -B build_fuzz \ - -DCMAKE_C_COMPILER="$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang" \ - -DCMAKE_CXX_COMPILER="$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang++" \ - -DENABLE_WALLET=OFF \ - -DBUILD_GUI=OFF \ - -DSANITIZERS=address,undefined -$ cmake --build build_fuzz --target bitcoind -$ mkdir -p inputs/ -$ ./honggfuzz/honggfuzz --exit_upon_crash --quiet --timeout 4 -n 1 -Q \ - -E HFND_TCP_PORT=18444 -f inputs/ -- \ - build_fuzz/src/bitcoind -regtest -discover=0 -dns=0 -dnsseed=0 -listenonion=0 \ - -nodebuglogfile -bind=127.0.0.1:18444 -logthreadnames \ - -debug -``` - # OSS-Fuzz Bitcoin Core participates in Google's [OSS-Fuzz](https://github.com/google/oss-fuzz/tree/master/projects/bitcoin-core) diff --git a/doc/release-notes-empty-template.md b/doc/release-notes-empty-template.md index 96e28c3763..1ff55b5ccc 100644 --- a/doc/release-notes-empty-template.md +++ b/doc/release-notes-empty-template.md @@ -32,11 +32,18 @@ Upgrading directly from a version of Bitcoin Core that has reached its EOL is possible, but it might take some time if the data directory needs to be migrated. Old wallet versions of Bitcoin Core are generally supported. +Running Bitcoin Core binaries on macOS requires self signing. +``` +cd /path/to/bitcoin-core/bin +xattr -d com.apple.quarantine bitcoin-cli bitcoin-qt bitcoin-tx bitcoin-util bitcoin-wallet bitcoind test_bitcoin +codesign -s - bitcoin-cli bitcoin-qt bitcoin-tx bitcoin-util bitcoin-wallet bitcoind test_bitcoin +``` + Compatibility ============== Bitcoin Core is supported and extensively tested on operating systems -using the Linux Kernel 3.17+, macOS 11.0+, and Windows 7 and newer. Bitcoin +using the Linux Kernel 3.17+, macOS 13.0+, and Windows 7 and newer. Bitcoin Core should also work on most other Unix-like systems but is not as frequently tested on them. It is not recommended to use Bitcoin Core on unsupported systems. diff --git a/doc/release-notes/release-notes-28.0.md b/doc/release-notes/release-notes-28.0.md new file mode 100644 index 0000000000..d9e6a34d0f --- /dev/null +++ b/doc/release-notes/release-notes-28.0.md @@ -0,0 +1,371 @@ +Bitcoin Core version 28.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-28.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Running Bitcoin Core binaries on macOS requires self signing. +``` +cd /path/to/bitcoin-28.0/bin +xattr -d com.apple.quarantine bitcoin-cli bitcoin-qt bitcoin-tx bitcoin-util bitcoin-wallet bitcoind test_bitcoin +codesign -s - bitcoin-cli bitcoin-qt bitcoin-tx bitcoin-util bitcoin-wallet bitcoind test_bitcoin +``` + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux Kernel 3.17+, macOS 11.0+, and Windows 7 and newer. Bitcoin +Core should also work on most other UNIX-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +Testnet4/BIP94 support +----- + +Support for Testnet4 as specified in [BIP94](https://github.com/bitcoin/bips/blob/master/bip-0094.mediawiki) +has been added. The network can be selected with the `-testnet4` option and +the section header is also named `[testnet4]`. + +While the intention is to phase out support for Testnet3 in an upcoming +version, support for it is still available via the known options in this +release. (#29775) + +Windows Data Directory +---------------------- + +The default data directory on Windows has been moved from `C:\Users\Username\AppData\Roaming\Bitcoin` +to `C:\Users\Username\AppData\Local\Bitcoin`. Bitcoin Core will check the existence +of the old directory first and continue to use that directory for backwards +compatibility if it is present. (#27064) + +JSON-RPC 2.0 Support +-------------------- + +The JSON-RPC server now recognizes JSON-RPC 2.0 requests and responds with +strict adherence to the [specification](https://www.jsonrpc.org/specification). +See [JSON-RPC-interface.md](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md#json-rpc-11-vs-20) for details. (#27101) + +JSON-RPC clients may need to be updated to be compatible with the JSON-RPC server. +Please open an issue on GitHub if any compatibility issues are found. + +libbitcoinconsensus Removal +--------------------------- + +The libbitcoin-consensus library was deprecated in 27.0 and is now completely removed. (#29648) + +P2P and Network Changes +----------------------- + +- Previously if Bitcoin Core was listening for P2P connections, either using + default settings or via `bind=addr:port` it would always also bind to + `127.0.0.1:8334` to listen for Tor connections. It was not possible to switch + this off, even if the node didn't use Tor. This has been changed and now + `bind=addr:port` results in binding on `addr:port` only. The default behavior + of binding to `0.0.0.0:8333` and `127.0.0.1:8334` has not been changed. + + If you are using a `bind=...` configuration without `bind=...=onion` and rely + on the previous implied behavior to accept incoming Tor connections at + `127.0.0.1:8334`, you need to now make this explicit by using + `bind=... bind=127.0.0.1:8334=onion`. (#22729) + +- Bitcoin Core will now fail to start up if any of its P2P binds fail, rather + than the previous behaviour where it would only abort startup if all P2P + binds had failed. (#22729) + +- UNIX domain sockets can now be used for proxy connections. Set `-onion` or `-proxy` + to the local socket path with the prefix `unix:` (e.g. `-onion=unix:/home/me/torsocket`). + (#27375) + +- UNIX socket paths are now accepted for `-zmqpubrawblock` and `-zmqpubrawtx` with + the format `-zmqpubrawtx=unix:/path/to/file` (#27679) + +- Additional "in" and "out" flags have been added to `-whitelist` to control whether + permissions apply to inbound connections and/or manual ones (default: inbound only). (#27114) + +- Transactions having a feerate that is too low will be opportunistically paired with + their child transactions and submitted as a package, thus enabling the node to download + 1-parent-1-child packages using the existing transaction relay protocol. Combined with + other mempool policies, this change allows limited "package relay" when a parent transaction + is below the mempool minimum feerate. Topologically Restricted Until Confirmation (TRUC) + parents are additionally allowed to be below the minimum relay feerate (i.e., pay 0 fees). + Use the `submitpackage` RPC to submit packages directly to the node. Warning: this P2P + feature is limited (unlike the `submitpackage` interface, a child with multiple unconfirmed + parents is not supported) and not yet reliable under adversarial conditions. (#28970) + +Mempool Policy Changes +---------------------- + +- Transactions with version number set to 3 are now treated as standard on all networks (#29496), + subject to opt-in Topologically Restricted Until Confirmation (TRUC) transaction policy as + described in [BIP 431](https://github.com/bitcoin/bips/blob/master/bip-0431.mediawiki). The + policy includes limits on spending unconfirmed outputs (#28948), eviction of a previous descendant + if a more incentive-compatible one is submitted (#29306), and a maximum transaction size of 10,000vB + (#29873). These restrictions simplify the assessment of incentive compatibility of accepting or + replacing TRUC transactions, thus ensuring any replacements are more profitable for the node and + making fee-bumping more reliable. + +- Pay To Anchor (P2A) is a new standard witness output type for spending, + a newly recognised output template. This allows for key-less anchor + outputs, with compact spending conditions for additional efficiencies on + top of an equivalent `sh(OP_TRUE)` output, in addition to the txid stability + of the spending transaction. + N.B. propagation of this output spending on the network will be limited + until a sufficient number of nodes on the network adopt this upgrade. (#30352) + +- Limited package RBF is now enabled, where the proposed conflicting package would result in + a connected component, aka cluster, of size 2 in the mempool. All clusters being conflicted + against must be of size 2 or lower. (#28984) + +- The default value of the `-mempoolfullrbf` configuration option has been changed from 0 to 1, + i.e. `mempoolfullrbf=1`. (#30493) + +Updated RPCs +------------ + +- The `dumptxoutset` RPC now returns the UTXO set dump in a new and + improved format. Correspondingly, the `loadtxoutset` RPC now expects + this new format in the dumps it tries to load. Dumps with the old + format are no longer supported and need to be recreated using the + new format to be usable. (#29612) + +- AssumeUTXO mainnet parameters have been added for height 840,000. + This means the `loadtxoutset` RPC can now be used on mainnet with + the matching UTXO set from that height. (#28553) + +- The `warnings` field in `getblockchaininfo`, `getmininginfo` and + `getnetworkinfo` now returns all the active node warnings as an array + of strings, instead of a single warning. The current behaviour + can be temporarily restored by running Bitcoin Core with the configuration + option `-deprecatedrpc=warnings`. (#29845) + +- Previously when using the `sendrawtransaction` RPC and specifying outputs + that are already in the UTXO set, an RPC error code of `-27` with the + message "Transaction already in block chain" was returned in response. + The error message has been changed to "Transaction outputs already in utxo set" + to more accurately describe the source of the issue. (#30212) + +- The default mode for the `estimatesmartfee` RPC has been updated from `conservative` to `economical`, + which is expected to reduce over-estimation for many users, particularly if Replace-by-Fee is an option. + For users that require high confidence in their fee estimates at the cost of potentially over-estimating, + the `conservative` mode remains available. (#30275) + +- RPC `scantxoutset` now returns 2 new fields in the "unspents" JSON array: `blockhash` and `confirmations`. + See the scantxoutset help for details. (#30515) + +- RPC `submitpackage` now allows 2 new arguments to be passed: `maxfeerate` and `maxburnamount`. See the + subtmitpackage help for details. (#28950) + +Changes to wallet-related RPCs can be found in the Wallet section below. + +Updated REST APIs +----------------- +- Parameter validation for `/rest/getutxos` has been improved by rejecting + truncated or overly large txids and malformed outpoint indices via raising + an HTTP_BAD_REQUEST "Parse error". These requests were previously handled + silently. (#30482, #30444) + +Build System +------------ + +- GCC 11.1 or later, or Clang 16.0 or later, +are now required to compile Bitcoin Core. (#29091, #30263) + +- The minimum required glibc to run Bitcoin Core is now +2.31. This means that RHEL 8 and Ubuntu 18.04 (Bionic) +are no-longer supported. (#29987) + +- `--enable-lcov-branch-coverage` has been removed, given +incompatibilities between lcov version 1 & 2. `LCOV_OPTS` +should be used to set any options instead. (#30192) + +Updated Settings +---------------- + +- When running with `-alertnotify`, an alert can now be raised multiple +times instead of just once. Previously, it was only raised when unknown +new consensus rules were activated. Its scope has now been increased to +include all kernel warnings. Specifically, alerts will now also be raised +when an invalid chain with a large amount of work has been detected. +Additional warnings may be added in the future. (#30058) + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +Wallet +------ + +- The wallet now detects when wallet transactions conflict with the mempool. Mempool-conflicting + transactions can be seen in the `"mempoolconflicts"` field of `gettransaction`. The inputs + of mempool-conflicted transactions can now be respent without manually abandoning the + transactions when the parent transaction is dropped from the mempool, which can cause wallet + balances to appear higher. (#27307) + +- A new `max_tx_weight` option has been added to the RPCs `fundrawtransaction`, `walletcreatefundedpsbt`, and `send`. +It specifies the maximum transaction weight. If the limit is exceeded during funding, the transaction will not be built. +The default value is 4,000,000 WU. (#29523) + +- A new `createwalletdescriptor` RPC allows users to add new automatically generated + descriptors to their wallet. This can be used to upgrade wallets created prior to the + introduction of a new standard descriptor, such as taproot. (#29130) + +- A new RPC `gethdkeys` lists all of the BIP32 HD keys in use by all of the descriptors in the wallet. + These keys can be used in conjunction with `createwalletdescriptor` to create and add single key + descriptors to the wallet for a particular key that the wallet already knows. (#29130) + +- The `sendall` RPC can now spend unconfirmed change and will include additional fees as necessary + for the resulting transaction to bump the unconfirmed transactions' feerates to the specified feerate. (#28979) + +- In RPC `bumpfee`, if a `fee_rate` is specified, the feerate is no longer restricted + to following the wallet's incremental feerate of 5 sat/vb. The feerate must still be + at least the sum of the original fee and the mempool's incremental feerate. (#27969) + +GUI Changes +----------- + +- The "Migrate Wallet" menu allows users to migrate any legacy wallet in their wallet +directory, regardless of the wallets loaded. (gui#824) + +- The "Information" window now displays the maximum mempool size along with the +mempool usage. (gui#825) + +Low-level Changes +================= + +Tests +----- + +- The BIP94 timewarp attack mitigation is now active on the `regtest` network. (#30681) + +- A new `-testdatadir` option has been added to `test_bitcoin` to allow specifying the + location of unit test data directories. (#26564) + +Blockstorage +------------ + +- Block files are now XOR'd by default with a key stored in the blocksdir. +Previous releases of Bitcoin Core or previous external software will not be able to read the blocksdir with a non-zero XOR-key. +Refer to the `-blocksxor` help for more details. (#28052) + +Chainstate +---------- + +- The chainstate database flushes that occur when blocks are pruned will no longer +empty the database cache. The cache will remain populated longer, which significantly +reduces the time for initial block download to complete. (#28280) + +Dependencies +------------ + +- The dependency on Boost.Process has been replaced with cpp-subprocess, which is contained in source. +Builders will no longer need Boost.Process to build with external signer support. (#28981) + +Credits +======= + +Thanks to everyone who directly contributed to this release: +- 0xb10c +- Alfonso Roman Zubeldia +- Andrew Toth +- AngusP +- Anthony Towns +- Antoine Poinsot +- Anton A +- Ava Chow +- Ayush Singh +- Ben Westgate +- Brandon Odiwuor +- brunoerg +- bstin +- Charlie +- Christopher Bergqvist +- Cory Fields +- crazeteam +- Daniela Brozzoni +- David Gumberg +- dergoegge +- Edil Medeiros +- Epic Curious +- Fabian Jahr +- fanquake +- furszy +- glozow +- Greg Sanders +- hanmz +- Hennadii Stepanov +- Hernan Marino +- Hodlinator +- ishaanam +- ismaelsadeeq +- Jadi +- Jon Atack +- josibake +- jrakibi +- kevkevin +- kevkevinpal +- Konstantin Akimov +- laanwj +- Larry Ruane +- Lőrinc +- Luis Schwab +- Luke Dashjr +- MarcoFalke +- marcofleon +- Marnix +- Martin Saposnic +- Martin Zumsande +- Matt Corallo +- Matthew Zipkin +- Matt Whitlock +- Max Edwards +- Michael Dietz +- Murch +- nanlour +- pablomartin4btc +- Peter Todd +- Pieter Wuille +- @RandyMcMillan +- RoboSchmied +- Roman Zeyde +- Ryan Ofsky +- Sebastian Falbesoner +- Sergi Delgado Segura +- Sjors Provoost +- spicyzboss +- StevenMia +- stickies-v +- stratospher +- Suhas Daftuar +- sunerok +- tdb3 +- TheCharlatan +- umiumi +- Vasil Dimov +- virtu +- willcl-ark + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/tracing.md b/doc/tracing.md index 3948b1ab49..c12af122db 100644 --- a/doc/tracing.md +++ b/doc/tracing.md @@ -106,7 +106,7 @@ Arguments passed: 3. Transactions in the Block as `uint64` 4. Inputs spend in the Block as `int32` 5. SigOps in the Block (excluding coinbase SigOps) `uint64` -6. Time it took to connect the Block in microseconds (µs) as `uint64` +6. Time it took to connect the Block in nanoseconds (ns) as `uint64` ### Context `utxocache` diff --git a/share/qt/Info.plist.in b/share/qt/Info.plist.in index b4e6f6a150..5ff736152f 100644 --- a/share/qt/Info.plist.in +++ b/share/qt/Info.plist.in @@ -3,7 +3,7 @@ <plist version="0.9"> <dict> <key>LSMinimumSystemVersion</key> - <string>11</string> + <string>13</string> <key>LSArchitecturePriority</key> <array> diff --git a/share/setup.nsi.in b/share/setup.nsi.in index 2ce798bd2d..f22e256967 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -95,7 +95,9 @@ Section -post SEC0001 !insertmacro MUI_STARTMENU_WRITE_BEGIN Application CreateDirectory $SMPROGRAMS\$StartMenuGroup CreateShortcut "$SMPROGRAMS\$StartMenuGroup\$(^Name).lnk" $INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@ - CreateShortcut "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (testnet, 64-bit).lnk" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" "-testnet" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" 1 + CreateShortcut "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (testnet).lnk" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" "-testnet" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" 1 + CreateShortcut "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (test signet).lnk" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" "-signet" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" 2 + CreateShortcut "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (testnet4).lnk" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" "-testnet4" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" 3 CreateShortcut "$SMPROGRAMS\$StartMenuGroup\Uninstall $(^Name).lnk" $INSTDIR\uninstall.exe !insertmacro MUI_STARTMENU_WRITE_END WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayName "$(^Name)" @@ -140,7 +142,9 @@ Section -un.post UNSEC0001 DeleteRegKey HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\Uninstall $(^Name).lnk" Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\$(^Name).lnk" - Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (testnet, 64-bit).lnk" + Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (testnet).lnk" + Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (testnet4).lnk" + Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (test signet).lnk" Delete /REBOOTOK "$SMSTARTUP\Bitcoin.lnk" Delete /REBOOTOK $INSTDIR\uninstall.exe Delete /REBOOTOK $INSTDIR\debug.log diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 36d76fbb27..4a86465bba 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,7 +5,7 @@ include(GNUInstallDirs) include(AddWindowsResources) -configure_file(${PROJECT_SOURCE_DIR}/cmake/bitcoin-config.h.in config/bitcoin-config.h USE_SOURCE_PERMISSIONS @ONLY) +configure_file(${PROJECT_SOURCE_DIR}/cmake/bitcoin-build-config.h.in bitcoin-build-config.h USE_SOURCE_PERMISSIONS @ONLY) include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) add_custom_target(generate_build_info @@ -114,6 +114,8 @@ add_library(bitcoin_common STATIC EXCLUDE_FROM_ALL common/init.cpp common/interfaces.cpp common/messages.cpp + common/netif.cpp + common/pcp.cpp common/run_command.cpp common/settings.cpp common/signmessage.cpp @@ -292,7 +294,6 @@ target_link_libraries(bitcoin_node Boost::headers $<TARGET_NAME_IF_EXISTS:libevent::libevent> $<TARGET_NAME_IF_EXISTS:libevent::pthreads> - $<TARGET_NAME_IF_EXISTS:NATPMP::NATPMP> $<TARGET_NAME_IF_EXISTS:MiniUPnPc::MiniUPnPc> $<TARGET_NAME_IF_EXISTS:bitcoin_zmq> $<TARGET_NAME_IF_EXISTS:USDT::headers> diff --git a/src/addrdb.cpp b/src/addrdb.cpp index b89141c88e..4637906441 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <addrdb.h> diff --git a/src/addrman.cpp b/src/addrman.cpp index 84d228dc82..358d4fc0a8 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <addrman.h> #include <addrman_impl.h> diff --git a/src/bench/cluster_linearize.cpp b/src/bench/cluster_linearize.cpp index cf071dda2d..7d011975dd 100644 --- a/src/bench/cluster_linearize.cpp +++ b/src/bench/cluster_linearize.cpp @@ -28,7 +28,7 @@ DepGraph<SetType> MakeLinearGraph(ClusterIndex ntx) DepGraph<SetType> depgraph; for (ClusterIndex i = 0; i < ntx; ++i) { depgraph.AddTransaction({-int32_t(i), 1}); - if (i > 0) depgraph.AddDependency(i - 1, i); + if (i > 0) depgraph.AddDependencies(SetType::Singleton(i - 1), i); } return depgraph; } @@ -43,7 +43,7 @@ DepGraph<SetType> MakeWideGraph(ClusterIndex ntx) DepGraph<SetType> depgraph; for (ClusterIndex i = 0; i < ntx; ++i) { depgraph.AddTransaction({int32_t(i) + 1, 1}); - if (i > 0) depgraph.AddDependency(0, i); + if (i > 0) depgraph.AddDependencies(SetType::Singleton(0), i); } return depgraph; } @@ -70,19 +70,19 @@ DepGraph<SetType> MakeHardGraph(ClusterIndex ntx) depgraph.AddTransaction({1, 2}); } else if (i == 1) { depgraph.AddTransaction({14, 2}); - depgraph.AddDependency(0, 1); + depgraph.AddDependencies(SetType::Singleton(0), 1); } else if (i == 2) { depgraph.AddTransaction({6, 1}); - depgraph.AddDependency(2, 1); + depgraph.AddDependencies(SetType::Singleton(2), 1); } else if (i == 3) { depgraph.AddTransaction({5, 1}); - depgraph.AddDependency(2, 3); + depgraph.AddDependencies(SetType::Singleton(2), 3); } else if ((i & 1) == 0) { depgraph.AddTransaction({7, 1}); - depgraph.AddDependency(i - 1, i); + depgraph.AddDependencies(SetType::Singleton(i - 1), i); } else { depgraph.AddTransaction({5, 1}); - depgraph.AddDependency(i, 4); + depgraph.AddDependencies(SetType::Singleton(i), 4); } } else { // Even cluster size. @@ -98,16 +98,16 @@ DepGraph<SetType> MakeHardGraph(ClusterIndex ntx) depgraph.AddTransaction({1, 1}); } else if (i == 1) { depgraph.AddTransaction({3, 1}); - depgraph.AddDependency(0, 1); + depgraph.AddDependencies(SetType::Singleton(0), 1); } else if (i == 2) { depgraph.AddTransaction({1, 1}); - depgraph.AddDependency(0, 2); + depgraph.AddDependencies(SetType::Singleton(0), 2); } else if (i & 1) { depgraph.AddTransaction({4, 1}); - depgraph.AddDependency(i - 1, i); + depgraph.AddDependencies(SetType::Singleton(i - 1), i); } else { depgraph.AddTransaction({0, 1}); - depgraph.AddDependency(i, 3); + depgraph.AddDependencies(SetType::Singleton(i), 3); } } } @@ -195,7 +195,7 @@ void BenchMergeLinearizationsWorstCase(ClusterIndex ntx, benchmark::Bench& bench DepGraph<SetType> depgraph; for (ClusterIndex i = 0; i < ntx; ++i) { depgraph.AddTransaction({i, 1}); - if (i) depgraph.AddDependency(0, i); + if (i) depgraph.AddDependencies(SetType::Singleton(0), i); } std::vector<ClusterIndex> lin1; std::vector<ClusterIndex> lin2; diff --git a/src/bench/wallet_create.cpp b/src/bench/wallet_create.cpp index 43b5b5c91e..3b916d7c39 100644 --- a/src/bench/wallet_create.cpp +++ b/src/bench/wallet_create.cpp @@ -3,7 +3,7 @@ // file COPYING or https://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <random.h> #include <support/allocators/secure.h> #include <test/util/setup_common.h> diff --git a/src/bench/wallet_ismine.cpp b/src/bench/wallet_ismine.cpp index 29e370ce29..5343814ab2 100644 --- a/src/bench/wallet_ismine.cpp +++ b/src/bench/wallet_ismine.cpp @@ -4,7 +4,7 @@ #include <addresstype.h> #include <bench/bench.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <key.h> #include <key_io.h> #include <script/descriptor.h> diff --git a/src/bench/wallet_loading.cpp b/src/bench/wallet_loading.cpp index 03459d37c1..5d92cfa0de 100644 --- a/src/bench/wallet_loading.cpp +++ b/src/bench/wallet_loading.cpp @@ -4,7 +4,7 @@ #include <addresstype.h> #include <bench/bench.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <consensus/amount.h> #include <outputtype.h> #include <primitives/transaction.h> diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 538fe592b8..38fd740b71 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chainparamsbase.h> #include <clientversion.h> @@ -678,7 +678,7 @@ public: " \".\" - we do not relay addresses to this peer (addr_relay_enabled is false)\n" " addrl Total number of addresses dropped due to rate limiting\n" " age Duration of connection to the peer, in minutes\n" - " asmap Mapped AS (Autonomous System) number in the BGP route to the peer, used for diversifying\n" + " asmap Mapped AS (Autonomous System) number at the end of the BGP route to the peer, used for diversifying\n" " peer selection (only displayed if the -asmap config option is set)\n" " id Peer index, in increasing order of peer connections since node startup\n" " address IP address and port of the peer\n" diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 89c03c1647..b3329afba4 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chainparamsbase.h> #include <clientversion.h> diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp index c8f5bc5026..46ba136d81 100644 --- a/src/bitcoin-util.cpp +++ b/src/bitcoin-util.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <arith_uint256.h> #include <chain.h> diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 7d030abe97..00f39be794 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chainparams.h> #include <chainparamsbase.h> diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 1e27924dfd..192676a10b 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chainparams.h> #include <clientversion.h> @@ -274,7 +274,7 @@ MAIN_FUNCTION if (ProcessInitCommands(args)) return EXIT_SUCCESS; // Start application - if (!AppInit(node) || !Assert(node.shutdown)->wait()) { + if (!AppInit(node) || !Assert(node.shutdown_signal)->wait()) { node.exit_status = EXIT_FAILURE; } Interrupt(node); diff --git a/src/clientversion.cpp b/src/clientversion.cpp index 7e6741a1a0..3943c4fb1d 100644 --- a/src/clientversion.cpp +++ b/src/clientversion.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <clientversion.h> #include <util/string.h> diff --git a/src/clientversion.h b/src/clientversion.h index 73aaf868e4..d1202b1259 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -7,11 +7,11 @@ #include <util/macros.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep // Check that required client information is defined #if !defined(CLIENT_VERSION_MAJOR) || !defined(CLIENT_VERSION_MINOR) || !defined(CLIENT_VERSION_BUILD) || !defined(CLIENT_VERSION_IS_RELEASE) || !defined(COPYRIGHT_YEAR) -#error Client version information missing: version is not defined by bitcoin-config.h or in any other way +#error Client version information missing: version is not defined by bitcoin-build-config.h or in any other way #endif //! Copyright string used in Windows .rc files diff --git a/src/cluster_linearize.h b/src/cluster_linearize.h index e964849f22..757c81f108 100644 --- a/src/cluster_linearize.h +++ b/src/cluster_linearize.h @@ -19,14 +19,6 @@ namespace cluster_linearize { -/** Data type to represent cluster input. - * - * cluster[i].first is tx_i's fee and size. - * cluster[i].second[j] is true iff tx_i spends one or more of tx_j's outputs. - */ -template<typename SetType> -using Cluster = std::vector<std::pair<FeeFrac, SetType>>; - /** Data type to represent transaction indices in clusters. */ using ClusterIndex = uint32_t; @@ -54,12 +46,23 @@ class DepGraph Entry(const FeeFrac& f, const SetType& a, const SetType& d) noexcept : feerate(f), ancestors(a), descendants(d) {} }; - /** Data for each transaction, in the same order as the Cluster it was constructed from. */ + /** Data for each transaction. */ std::vector<Entry> entries; + /** Which positions are used. */ + SetType m_used; + public: /** Equality operator (primarily for testing purposes). */ - friend bool operator==(const DepGraph&, const DepGraph&) noexcept = default; + friend bool operator==(const DepGraph& a, const DepGraph& b) noexcept + { + if (a.m_used != b.m_used) return false; + // Only compare the used positions within the entries vector. + for (auto idx : a.m_used) { + if (a.entries[idx] != b.entries[idx]) return false; + } + return true; + } // Default constructors. DepGraph() noexcept = default; @@ -68,80 +71,51 @@ public: DepGraph& operator=(const DepGraph&) noexcept = default; DepGraph& operator=(DepGraph&&) noexcept = default; - /** Construct a DepGraph object for ntx transactions, with no dependencies. + /** Construct a DepGraph object given another DepGraph and a mapping from old to new. * - * Complexity: O(N) where N=ntx. - **/ - explicit DepGraph(ClusterIndex ntx) noexcept - { - Assume(ntx <= SetType::Size()); - entries.resize(ntx); - for (ClusterIndex i = 0; i < ntx; ++i) { - entries[i].ancestors = SetType::Singleton(i); - entries[i].descendants = SetType::Singleton(i); - } - } - - /** Construct a DepGraph object given a cluster. + * @param depgraph The original DepGraph that is being remapped. * - * Complexity: O(N^2) where N=cluster.size(). - */ - explicit DepGraph(const Cluster<SetType>& cluster) noexcept : entries(cluster.size()) - { - for (ClusterIndex i = 0; i < cluster.size(); ++i) { - // Fill in fee and size. - entries[i].feerate = cluster[i].first; - // Fill in direct parents as ancestors. - entries[i].ancestors = cluster[i].second; - // Make sure transactions are ancestors of themselves. - entries[i].ancestors.Set(i); - } - - // Propagate ancestor information. - for (ClusterIndex i = 0; i < entries.size(); ++i) { - // At this point, entries[a].ancestors[b] is true iff b is an ancestor of a and there - // is a path from a to b through the subgraph consisting of {a, b} union - // {0, 1, ..., (i-1)}. - SetType to_merge = entries[i].ancestors; - for (ClusterIndex j = 0; j < entries.size(); ++j) { - if (entries[j].ancestors[i]) { - entries[j].ancestors |= to_merge; - } - } - } - - // Fill in descendant information by transposing the ancestor information. - for (ClusterIndex i = 0; i < entries.size(); ++i) { - for (auto j : entries[i].ancestors) { - entries[j].descendants.Set(i); - } - } - } - - /** Construct a DepGraph object given another DepGraph and a mapping from old to new. + * @param mapping A Span such that mapping[i] gives the position in the new DepGraph + * for position i in the old depgraph. Its size must be equal to + * depgraph.PositionRange(). The value of mapping[i] is ignored if + * position i is a hole in depgraph (i.e., if !depgraph.Positions()[i]). + * + * @param pos_range The PositionRange() for the new DepGraph. It must equal the largest + * value in mapping for any used position in depgraph plus 1, or 0 if + * depgraph.TxCount() == 0. * * Complexity: O(N^2) where N=depgraph.TxCount(). */ - DepGraph(const DepGraph<SetType>& depgraph, Span<const ClusterIndex> mapping) noexcept : entries(depgraph.TxCount()) + DepGraph(const DepGraph<SetType>& depgraph, Span<const ClusterIndex> mapping, ClusterIndex pos_range) noexcept : entries(pos_range) { - Assert(mapping.size() == depgraph.TxCount()); - // Fill in fee, size, ancestors. - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { - const auto& input = depgraph.entries[i]; - auto& output = entries[mapping[i]]; - output.feerate = input.feerate; - for (auto j : input.ancestors) output.ancestors.Set(mapping[j]); + Assume(mapping.size() == depgraph.PositionRange()); + Assume((pos_range == 0) == (depgraph.TxCount() == 0)); + for (ClusterIndex i : depgraph.Positions()) { + auto new_idx = mapping[i]; + Assume(new_idx < pos_range); + // Add transaction. + entries[new_idx].ancestors = SetType::Singleton(new_idx); + entries[new_idx].descendants = SetType::Singleton(new_idx); + m_used.Set(new_idx); + // Fill in fee and size. + entries[new_idx].feerate = depgraph.entries[i].feerate; } - // Fill in descendant information. - for (ClusterIndex i = 0; i < entries.size(); ++i) { - for (auto j : entries[i].ancestors) { - entries[j].descendants.Set(i); - } + for (ClusterIndex i : depgraph.Positions()) { + // Fill in dependencies by mapping direct parents. + SetType parents; + for (auto j : depgraph.GetReducedParents(i)) parents.Set(mapping[j]); + AddDependencies(parents, mapping[i]); } + // Verify that the provided pos_range was correct (no unused positions at the end). + Assume(m_used.None() ? (pos_range == 0) : (pos_range == m_used.Last() + 1)); } + /** Get the set of transactions positions in use. Complexity: O(1). */ + const SetType& Positions() const noexcept { return m_used; } + /** Get the range of positions in this DepGraph. All entries in Positions() are in [0, PositionRange() - 1]. */ + ClusterIndex PositionRange() const noexcept { return entries.size(); } /** Get the number of transactions in the graph. Complexity: O(1). */ - auto TxCount() const noexcept { return entries.size(); } + auto TxCount() const noexcept { return m_used.Count(); } /** Get the feerate of a given transaction i. Complexity: O(1). */ const FeeFrac& FeeRate(ClusterIndex i) const noexcept { return entries[i].feerate; } /** Get the mutable feerate of a given transaction i. Complexity: O(1). */ @@ -151,39 +125,120 @@ public: /** Get the descendants of a given transaction i. Complexity: O(1). */ const SetType& Descendants(ClusterIndex i) const noexcept { return entries[i].descendants; } - /** Add a new unconnected transaction to this transaction graph (at the end), and return its - * ClusterIndex. + /** Add a new unconnected transaction to this transaction graph (in the first available + * position), and return its ClusterIndex. * * Complexity: O(1) (amortized, due to resizing of backing vector). */ ClusterIndex AddTransaction(const FeeFrac& feefrac) noexcept { - Assume(TxCount() < SetType::Size()); - ClusterIndex new_idx = TxCount(); - entries.emplace_back(feefrac, SetType::Singleton(new_idx), SetType::Singleton(new_idx)); + static constexpr auto ALL_POSITIONS = SetType::Fill(SetType::Size()); + auto available = ALL_POSITIONS - m_used; + Assume(available.Any()); + ClusterIndex new_idx = available.First(); + if (new_idx == entries.size()) { + entries.emplace_back(feefrac, SetType::Singleton(new_idx), SetType::Singleton(new_idx)); + } else { + entries[new_idx] = Entry(feefrac, SetType::Singleton(new_idx), SetType::Singleton(new_idx)); + } + m_used.Set(new_idx); return new_idx; } - /** Modify this transaction graph, adding a dependency between a specified parent and child. + /** Remove the specified positions from this DepGraph. + * + * The specified positions will no longer be part of Positions(), and dependencies with them are + * removed. Note that due to DepGraph only tracking ancestors/descendants (and not direct + * dependencies), if a parent is removed while a grandparent remains, the grandparent will + * remain an ancestor. * * Complexity: O(N) where N=TxCount(). - **/ - void AddDependency(ClusterIndex parent, ClusterIndex child) noexcept + */ + void RemoveTransactions(const SetType& del) noexcept + { + m_used -= del; + // Remove now-unused trailing entries. + while (!entries.empty() && !m_used[entries.size() - 1]) { + entries.pop_back(); + } + // Remove the deleted transactions from ancestors/descendants of other transactions. Note + // that the deleted positions will retain old feerate and dependency information. This does + // not matter as they will be overwritten by AddTransaction if they get used again. + for (auto& entry : entries) { + entry.ancestors &= m_used; + entry.descendants &= m_used; + } + } + + /** Modify this transaction graph, adding multiple parents to a specified child. + * + * Complexity: O(N) where N=TxCount(). + */ + void AddDependencies(const SetType& parents, ClusterIndex child) noexcept { - // Bail out if dependency is already implied. - if (entries[child].ancestors[parent]) return; - // To each ancestor of the parent, add as descendants the descendants of the child. + Assume(m_used[child]); + Assume(parents.IsSubsetOf(m_used)); + // Compute the ancestors of parents that are not already ancestors of child. + SetType par_anc; + for (auto par : parents - Ancestors(child)) { + par_anc |= Ancestors(par); + } + par_anc -= Ancestors(child); + // Bail out if there are no such ancestors. + if (par_anc.None()) return; + // To each such ancestor, add as descendants the descendants of the child. const auto& chl_des = entries[child].descendants; - for (auto anc_of_par : Ancestors(parent)) { + for (auto anc_of_par : par_anc) { entries[anc_of_par].descendants |= chl_des; } - // To each descendant of the child, add as ancestors the ancestors of the parent. - const auto& par_anc = entries[parent].ancestors; + // To each descendant of the child, add those ancestors. for (auto dec_of_chl : Descendants(child)) { entries[dec_of_chl].ancestors |= par_anc; } } + /** Compute the (reduced) set of parents of node i in this graph. + * + * This returns the minimal subset of the parents of i whose ancestors together equal all of + * i's ancestors (unless i is part of a cycle of dependencies). Note that DepGraph does not + * store the set of parents; this information is inferred from the ancestor sets. + * + * Complexity: O(N) where N=Ancestors(i).Count() (which is bounded by TxCount()). + */ + SetType GetReducedParents(ClusterIndex i) const noexcept + { + SetType parents = Ancestors(i); + parents.Reset(i); + for (auto parent : parents) { + if (parents[parent]) { + parents -= Ancestors(parent); + parents.Set(parent); + } + } + return parents; + } + + /** Compute the (reduced) set of children of node i in this graph. + * + * This returns the minimal subset of the children of i whose descendants together equal all of + * i's descendants (unless i is part of a cycle of dependencies). Note that DepGraph does not + * store the set of children; this information is inferred from the descendant sets. + * + * Complexity: O(N) where N=Descendants(i).Count() (which is bounded by TxCount()). + */ + SetType GetReducedChildren(ClusterIndex i) const noexcept + { + SetType children = Descendants(i); + children.Reset(i); + for (auto child : children) { + if (children[child]) { + children -= Descendants(child); + children.Set(child); + } + } + return children; + } + /** Compute the aggregate feerate of a set of nodes in this graph. * * Complexity: O(N) where N=elems.Count(). @@ -237,7 +292,7 @@ public: * * Complexity: O(TxCount()). */ - bool IsConnected() const noexcept { return IsConnected(SetType::Fill(TxCount())); } + bool IsConnected() const noexcept { return IsConnected(m_used); } /** Append the entries of select to list in a topologically valid order. * @@ -487,11 +542,11 @@ public: */ AncestorCandidateFinder(const DepGraph<SetType>& depgraph LIFETIMEBOUND) noexcept : m_depgraph(depgraph), - m_todo{SetType::Fill(depgraph.TxCount())}, - m_ancestor_set_feerates(depgraph.TxCount()) + m_todo{depgraph.Positions()}, + m_ancestor_set_feerates(depgraph.PositionRange()) { // Precompute ancestor-set feerates. - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + for (ClusterIndex i : m_depgraph.Positions()) { /** The remaining ancestors for transaction i. */ SetType anc_to_add = m_depgraph.Ancestors(i); FeeFrac anc_feerate; @@ -614,22 +669,26 @@ public: SearchCandidateFinder(const DepGraph<SetType>& depgraph, uint64_t rng_seed) noexcept : m_rng(rng_seed), m_sorted_to_original(depgraph.TxCount()), - m_original_to_sorted(depgraph.TxCount()), - m_todo(SetType::Fill(depgraph.TxCount())) + m_original_to_sorted(depgraph.PositionRange()) { - // Determine reordering mapping, by sorting by decreasing feerate. - std::iota(m_sorted_to_original.begin(), m_sorted_to_original.end(), ClusterIndex{0}); + // Determine reordering mapping, by sorting by decreasing feerate. Unusued positions are + // not included, as they will never be looked up anyway. + ClusterIndex sorted_pos{0}; + for (auto i : depgraph.Positions()) { + m_sorted_to_original[sorted_pos++] = i; + } std::sort(m_sorted_to_original.begin(), m_sorted_to_original.end(), [&](auto a, auto b) { auto feerate_cmp = depgraph.FeeRate(a) <=> depgraph.FeeRate(b); if (feerate_cmp == 0) return a < b; return feerate_cmp > 0; }); // Compute reverse mapping. - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + for (ClusterIndex i = 0; i < m_sorted_to_original.size(); ++i) { m_original_to_sorted[m_sorted_to_original[i]] = i; } // Compute reordered dependency graph. - m_sorted_depgraph = DepGraph(depgraph, m_original_to_sorted); + m_sorted_depgraph = DepGraph(depgraph, m_original_to_sorted, m_sorted_to_original.size()); + m_todo = m_sorted_depgraph.Positions(); } /** Check whether any unlinearized transactions remain. */ @@ -1141,7 +1200,7 @@ void PostLinearize(const DepGraph<SetType>& depgraph, Span<ClusterIndex> lineari // During an even pass, the diagram above would correspond to linearization [2,3,0,1], with // groups [2] and [3,0,1]. - std::vector<TxEntry> entries(linearization.size() + 1); + std::vector<TxEntry> entries(depgraph.PositionRange() + 1); // Perform two passes over the linearization. for (int pass = 0; pass < 2; ++pass) { diff --git a/src/common/netif.cpp b/src/common/netif.cpp new file mode 100644 index 0000000000..08f034a412 --- /dev/null +++ b/src/common/netif.cpp @@ -0,0 +1,303 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include <bitcoin-build-config.h> // IWYU pragma: keep + +#include <common/netif.h> + +#include <logging.h> +#include <netbase.h> +#include <util/check.h> +#include <util/sock.h> +#include <util/syserror.h> + +#if defined(__linux__) +#include <linux/rtnetlink.h> +#elif defined(__FreeBSD__) +#include <osreldate.h> +#if __FreeBSD_version >= 1400000 +// Workaround https://github.com/freebsd/freebsd-src/pull/1070. +#define typeof __typeof +#include <netlink/netlink.h> +#include <netlink/netlink_route.h> +#endif +#elif defined(WIN32) +#include <iphlpapi.h> +#elif defined(__APPLE__) +#include <net/route.h> +#include <sys/sysctl.h> +#endif + +namespace { + +// Linux and FreeBSD 14.0+. For FreeBSD 13.2 the code can be compiled but +// running it requires loading a special kernel module, otherwise socket(AF_NETLINK,...) +// will fail, so we skip that. +#if defined(__linux__) || (defined(__FreeBSD__) && __FreeBSD_version >= 1400000) + +std::optional<CNetAddr> QueryDefaultGatewayImpl(sa_family_t family) +{ + // Create a netlink socket. + auto sock{CreateSock(AF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE)}; + if (!sock) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "socket(AF_NETLINK): %s\n", NetworkErrorString(errno)); + return std::nullopt; + } + + // Send request. + struct { + nlmsghdr hdr; ///< Request header. + rtmsg data; ///< Request data, a "route message". + nlattr dst_hdr; ///< One attribute, conveying the route destination address. + char dst_data[16]; ///< Route destination address. To query the default route we use 0.0.0.0/0 or [::]/0. For IPv4 the first 4 bytes are used. + } request{}; + + // Whether to use the first 4 or 16 bytes from request.dst_data. + const size_t dst_data_len = family == AF_INET ? 4 : 16; + + request.hdr.nlmsg_type = RTM_GETROUTE; + request.hdr.nlmsg_flags = NLM_F_REQUEST; +#ifdef __linux__ + // Linux IPv4 / IPv6 - this must be present, otherwise no gateway is found + // FreeBSD IPv4 - does not matter, the gateway is found with or without this + // FreeBSD IPv6 - this must be absent, otherwise no gateway is found + request.hdr.nlmsg_flags |= NLM_F_DUMP; +#endif + request.hdr.nlmsg_len = NLMSG_LENGTH(sizeof(rtmsg) + sizeof(nlattr) + dst_data_len); + request.hdr.nlmsg_seq = 0; // Sequence number, used to match which reply is to which request. Irrelevant for us because we send just one request. + request.data.rtm_family = family; + request.data.rtm_dst_len = 0; // Prefix length. +#ifdef __FreeBSD__ + // Linux IPv4 / IPv6 this must be absent, otherwise no gateway is found + // FreeBSD IPv4 - does not matter, the gateway is found with or without this + // FreeBSD IPv6 - this must be present, otherwise no gateway is found + request.data.rtm_flags = RTM_F_PREFIX; +#endif + request.dst_hdr.nla_type = RTA_DST; + request.dst_hdr.nla_len = sizeof(nlattr) + dst_data_len; + + if (sock->Send(&request, request.hdr.nlmsg_len, 0) != static_cast<ssize_t>(request.hdr.nlmsg_len)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "send() to netlink socket: %s\n", NetworkErrorString(errno)); + return std::nullopt; + } + + // Receive response. + char response[4096]; + int64_t recv_result; + do { + recv_result = sock->Recv(response, sizeof(response), 0); + } while (recv_result < 0 && (errno == EINTR || errno == EAGAIN)); + if (recv_result < 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "recv() from netlink socket: %s\n", NetworkErrorString(errno)); + return std::nullopt; + } + + for (nlmsghdr* hdr = (nlmsghdr*)response; NLMSG_OK(hdr, recv_result); hdr = NLMSG_NEXT(hdr, recv_result)) { + rtmsg* r = (rtmsg*)NLMSG_DATA(hdr); + int remaining_len = RTM_PAYLOAD(hdr); + + // Iterate over the attributes. + rtattr *rta_gateway = nullptr; + int scope_id = 0; + for (rtattr* attr = RTM_RTA(r); RTA_OK(attr, remaining_len); attr = RTA_NEXT(attr, remaining_len)) { + if (attr->rta_type == RTA_GATEWAY) { + rta_gateway = attr; + } else if (attr->rta_type == RTA_OIF && sizeof(int) == RTA_PAYLOAD(attr)) { + std::memcpy(&scope_id, RTA_DATA(attr), sizeof(scope_id)); + } + } + + // Found gateway? + if (rta_gateway != nullptr) { + if (family == AF_INET && sizeof(in_addr) == RTA_PAYLOAD(rta_gateway)) { + in_addr gw; + std::memcpy(&gw, RTA_DATA(rta_gateway), sizeof(gw)); + return CNetAddr(gw); + } else if (family == AF_INET6 && sizeof(in6_addr) == RTA_PAYLOAD(rta_gateway)) { + in6_addr gw; + std::memcpy(&gw, RTA_DATA(rta_gateway), sizeof(gw)); + return CNetAddr(gw, scope_id); + } + } + } + + return std::nullopt; +} + +#elif defined(WIN32) + +std::optional<CNetAddr> QueryDefaultGatewayImpl(sa_family_t family) +{ + NET_LUID interface_luid = {}; + SOCKADDR_INET destination_address = {}; + MIB_IPFORWARD_ROW2 best_route = {}; + SOCKADDR_INET best_source_address = {}; + DWORD best_if_idx = 0; + DWORD status = 0; + + // Pass empty destination address of the requested type (:: or 0.0.0.0) to get interface of default route. + destination_address.si_family = family; + status = GetBestInterfaceEx((sockaddr*)&destination_address, &best_if_idx); + if (status != NO_ERROR) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Could not get best interface for default route: %s\n", NetworkErrorString(status)); + return std::nullopt; + } + + // Get best route to default gateway. + // Leave interface_luid at all-zeros to use interface index instead. + status = GetBestRoute2(&interface_luid, best_if_idx, nullptr, &destination_address, 0, &best_route, &best_source_address); + if (status != NO_ERROR) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Could not get best route for default route for interface index %d: %s\n", + best_if_idx, NetworkErrorString(status)); + return std::nullopt; + } + + Assume(best_route.NextHop.si_family == family); + if (family == AF_INET) { + return CNetAddr(best_route.NextHop.Ipv4.sin_addr); + } else if(family == AF_INET6) { + return CNetAddr(best_route.NextHop.Ipv6.sin6_addr, best_route.InterfaceIndex); + } + return std::nullopt; +} + +#elif defined(__APPLE__) + +#define ROUNDUP32(a) \ + ((a) > 0 ? (1 + (((a) - 1) | (sizeof(uint32_t) - 1))) : sizeof(uint32_t)) + +std::optional<CNetAddr> FromSockAddr(const struct sockaddr* addr) +{ + // Check valid length. Note that sa_len is not part of POSIX, and exists on MacOS and some BSDs only, so we can't + // do this check in SetSockAddr. + if (!(addr->sa_family == AF_INET && addr->sa_len == sizeof(struct sockaddr_in)) && + !(addr->sa_family == AF_INET6 && addr->sa_len == sizeof(struct sockaddr_in6))) { + return std::nullopt; + } + + // Fill in a CService from the sockaddr, then drop the port part. + CService service; + if (service.SetSockAddr(addr)) { + return (CNetAddr)service; + } + return std::nullopt; +} + +//! MacOS: Get default gateway from route table. See route(4) for the format. +std::optional<CNetAddr> QueryDefaultGatewayImpl(sa_family_t family) +{ + // net.route.0.inet[6].flags.gateway + int mib[] = {CTL_NET, PF_ROUTE, 0, family, NET_RT_FLAGS, RTF_GATEWAY}; + // The size of the available data is determined by calling sysctl() with oldp=nullptr. See sysctl(3). + size_t l = 0; + if (sysctl(/*name=*/mib, /*namelen=*/sizeof(mib) / sizeof(int), /*oldp=*/nullptr, /*oldlenp=*/&l, /*newp=*/nullptr, /*newlen=*/0) < 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Could not get sysctl length of routing table: %s\n", SysErrorString(errno)); + return std::nullopt; + } + std::vector<std::byte> buf(l); + if (sysctl(/*name=*/mib, /*namelen=*/sizeof(mib) / sizeof(int), /*oldp=*/buf.data(), /*oldlenp=*/&l, /*newp=*/nullptr, /*newlen=*/0) < 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Could not get sysctl data of routing table: %s\n", SysErrorString(errno)); + return std::nullopt; + } + // Iterate over messages (each message is a routing table entry). + for (size_t msg_pos = 0; msg_pos < buf.size(); ) { + if ((msg_pos + sizeof(rt_msghdr)) > buf.size()) return std::nullopt; + const struct rt_msghdr* rt = (const struct rt_msghdr*)(buf.data() + msg_pos); + const size_t next_msg_pos = msg_pos + rt->rtm_msglen; + if (rt->rtm_msglen < sizeof(rt_msghdr) || next_msg_pos > buf.size()) return std::nullopt; + // Iterate over addresses within message, get destination and gateway (if present). + // Address data starts after header. + size_t sa_pos = msg_pos + sizeof(struct rt_msghdr); + std::optional<CNetAddr> dst, gateway; + for (int i = 0; i < RTAX_MAX; i++) { + if (rt->rtm_addrs & (1 << i)) { + // 2 is just sa_len + sa_family, the theoretical minimum size of a socket address. + if ((sa_pos + 2) > next_msg_pos) return std::nullopt; + const struct sockaddr* sa = (const struct sockaddr*)(buf.data() + sa_pos); + if ((sa_pos + sa->sa_len) > next_msg_pos) return std::nullopt; + if (i == RTAX_DST) { + dst = FromSockAddr(sa); + } else if (i == RTAX_GATEWAY) { + gateway = FromSockAddr(sa); + } + // Skip sockaddr entries for bit flags we're not interested in, + // move cursor. + sa_pos += ROUNDUP32(sa->sa_len); + } + } + // Found default gateway? + if (dst && gateway && dst->IsBindAny()) { // Route to 0.0.0.0 or :: ? + return *gateway; + } + // Skip to next message. + msg_pos = next_msg_pos; + } + return std::nullopt; +} + +#else + +// Dummy implementation. +std::optional<CNetAddr> QueryDefaultGatewayImpl(sa_family_t) +{ + return std::nullopt; +} + +#endif + +} + +std::optional<CNetAddr> QueryDefaultGateway(Network network) +{ + Assume(network == NET_IPV4 || network == NET_IPV6); + + sa_family_t family; + if (network == NET_IPV4) { + family = AF_INET; + } else if(network == NET_IPV6) { + family = AF_INET6; + } else { + return std::nullopt; + } + + std::optional<CNetAddr> ret = QueryDefaultGatewayImpl(family); + + // It's possible for the default gateway to be 0.0.0.0 or ::0 on at least Windows + // for some routing strategies. If so, return as if no default gateway was found. + if (ret && !ret->IsBindAny()) { + return ret; + } else { + return std::nullopt; + } +} + +std::vector<CNetAddr> GetLocalAddresses() +{ + std::vector<CNetAddr> addresses; +#ifdef WIN32 + char pszHostName[256] = ""; + if (gethostname(pszHostName, sizeof(pszHostName)) != SOCKET_ERROR) { + addresses = LookupHost(pszHostName, 0, true); + } +#elif (HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS) + struct ifaddrs* myaddrs; + if (getifaddrs(&myaddrs) == 0) { + for (struct ifaddrs* ifa = myaddrs; ifa != nullptr; ifa = ifa->ifa_next) + { + if (ifa->ifa_addr == nullptr) continue; + if ((ifa->ifa_flags & IFF_UP) == 0) continue; + if ((ifa->ifa_flags & IFF_LOOPBACK) != 0) continue; + if (ifa->ifa_addr->sa_family == AF_INET) { + struct sockaddr_in* s4 = (struct sockaddr_in*)(ifa->ifa_addr); + addresses.emplace_back(s4->sin_addr); + } else if (ifa->ifa_addr->sa_family == AF_INET6) { + struct sockaddr_in6* s6 = (struct sockaddr_in6*)(ifa->ifa_addr); + addresses.emplace_back(s6->sin6_addr); + } + } + freeifaddrs(myaddrs); + } +#endif + return addresses; +} diff --git a/src/common/netif.h b/src/common/netif.h new file mode 100644 index 0000000000..55bc023be6 --- /dev/null +++ b/src/common/netif.h @@ -0,0 +1,19 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMMON_NETIF_H +#define BITCOIN_COMMON_NETIF_H + +#include <netaddress.h> + +#include <optional> + +//! Query the OS for the default gateway for `network`. This only makes sense for NET_IPV4 and NET_IPV6. +//! Returns std::nullopt if it cannot be found, or there is no support for this OS. +std::optional<CNetAddr> QueryDefaultGateway(Network network); + +//! Return all local non-loopback IPv4 and IPv6 network addresses. +std::vector<CNetAddr> GetLocalAddresses(); + +#endif // BITCOIN_COMMON_NETIF_H diff --git a/src/common/pcp.cpp b/src/common/pcp.cpp new file mode 100644 index 0000000000..3cc1cba924 --- /dev/null +++ b/src/common/pcp.cpp @@ -0,0 +1,524 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include <common/pcp.h> + +#include <common/netif.h> +#include <crypto/common.h> +#include <logging.h> +#include <netaddress.h> +#include <netbase.h> +#include <random.h> +#include <span.h> +#include <util/check.h> +#include <util/readwritefile.h> +#include <util/sock.h> +#include <util/strencodings.h> + +namespace { + +// RFC6886 NAT-PMP and RFC6887 Port Control Protocol (PCP) implementation. +// NAT-PMP and PCP use network byte order (big-endian). + +// NAT-PMP (v0) protocol constants. +//! NAT-PMP uses a fixed server port number (RFC6887 section 1.1). +constexpr uint16_t NATPMP_SERVER_PORT = 5351; +//! Version byte for NATPMP (RFC6886 1.1) +constexpr uint8_t NATPMP_VERSION = 0; +//! Request opcode base (RFC6886 3). +constexpr uint8_t NATPMP_REQUEST = 0x00; +//! Response opcode base (RFC6886 3). +constexpr uint8_t NATPMP_RESPONSE = 0x80; +//! Get external address (RFC6886 3.2) +constexpr uint8_t NATPMP_OP_GETEXTERNAL = 0x00; +//! Map TCP port (RFC6886 3.3) +constexpr uint8_t NATPMP_OP_MAP_TCP = 0x02; +//! Shared request header size in bytes. +constexpr size_t NATPMP_REQUEST_HDR_SIZE = 2; +//! Shared response header (minimum) size in bytes. +constexpr size_t NATPMP_RESPONSE_HDR_SIZE = 8; +//! GETEXTERNAL request size in bytes, including header (RFC6886 3.2). +constexpr size_t NATPMP_GETEXTERNAL_REQUEST_SIZE = NATPMP_REQUEST_HDR_SIZE + 0; +//! GETEXTERNAL response size in bytes, including header (RFC6886 3.2). +constexpr size_t NATPMP_GETEXTERNAL_RESPONSE_SIZE = NATPMP_RESPONSE_HDR_SIZE + 4; +//! MAP request size in bytes, including header (RFC6886 3.3). +constexpr size_t NATPMP_MAP_REQUEST_SIZE = NATPMP_REQUEST_HDR_SIZE + 10; +//! MAP response size in bytes, including header (RFC6886 3.3). +constexpr size_t NATPMP_MAP_RESPONSE_SIZE = NATPMP_RESPONSE_HDR_SIZE + 8; + +// Shared header offsets (RFC6886 3.2, 3.3), relative to start of packet. +//! Offset of version field in packets. +constexpr size_t NATPMP_HDR_VERSION_OFS = 0; +//! Offset of opcode field in packets +constexpr size_t NATPMP_HDR_OP_OFS = 1; +//! Offset of result code in packets. Result codes are 16 bit in NAT-PMP instead of 8 bit in PCP. +constexpr size_t NATPMP_RESPONSE_HDR_RESULT_OFS = 2; + +// GETEXTERNAL response offsets (RFC6886 3.2), relative to start of packet. +//! Returned external address +constexpr size_t NATPMP_GETEXTERNAL_RESPONSE_IP_OFS = 8; + +// MAP request offsets (RFC6886 3.3), relative to start of packet. +//! Internal port to be mapped. +constexpr size_t NATPMP_MAP_REQUEST_INTERNAL_PORT_OFS = 4; +//! Suggested external port for mapping. +constexpr size_t NATPMP_MAP_REQUEST_EXTERNAL_PORT_OFS = 6; +//! Requested port mapping lifetime in seconds. +constexpr size_t NATPMP_MAP_REQUEST_LIFETIME_OFS = 8; + +// MAP response offsets (RFC6886 3.3), relative to start of packet. +//! Internal port for mapping (will match internal port of request). +constexpr size_t NATPMP_MAP_RESPONSE_INTERNAL_PORT_OFS = 8; +//! External port for mapping. +constexpr size_t NATPMP_MAP_RESPONSE_EXTERNAL_PORT_OFS = 10; +//! Created port mapping lifetime in seconds. +constexpr size_t NATPMP_MAP_RESPONSE_LIFETIME_OFS = 12; + +// Relevant NETPMP result codes (RFC6886 3.5). +//! Result code representing success status. +constexpr uint8_t NATPMP_RESULT_SUCCESS = 0; +//! Result code representing unsupported version. +constexpr uint8_t NATPMP_RESULT_UNSUPP_VERSION = 1; +//! Result code representing lack of resources. +constexpr uint8_t NATPMP_RESULT_NO_RESOURCES = 4; + +//! Mapping of NATPMP result code to string (RFC6886 3.5). Result codes <=2 match PCP. +const std::map<uint8_t, std::string> NATPMP_RESULT_STR{ + {0, "SUCCESS"}, + {1, "UNSUPP_VERSION"}, + {2, "NOT_AUTHORIZED"}, + {3, "NETWORK_FAILURE"}, + {4, "NO_RESOURCES"}, + {5, "UNSUPP_OPCODE"}, +}; + +// PCP (v2) protocol constants. +//! Maximum packet size in bytes (RFC6887 section 7). +constexpr size_t PCP_MAX_SIZE = 1100; +//! PCP uses a fixed server port number (RFC6887 section 19.1). Shared with NAT-PMP. +constexpr uint16_t PCP_SERVER_PORT = NATPMP_SERVER_PORT; +//! Version byte. 0 is NAT-PMP (RFC6886), 1 is forbidden, 2 for PCP (RFC6887). +constexpr uint8_t PCP_VERSION = 2; +//! PCP Request Header. See RFC6887 section 7.1. Shared with NAT-PMP. +constexpr uint8_t PCP_REQUEST = NATPMP_REQUEST; // R = 0 +//! PCP Response Header. See RFC6887 section 7.2. Shared with NAT-PMP. +constexpr uint8_t PCP_RESPONSE = NATPMP_RESPONSE; // R = 1 +//! Map opcode. See RFC6887 section 19.2 +constexpr uint8_t PCP_OP_MAP = 0x01; +//! TCP protocol number (IANA). +constexpr uint16_t PCP_PROTOCOL_TCP = 6; +//! Request and response header size in bytes (RFC6887 section 7.1). +constexpr size_t PCP_HDR_SIZE = 24; +//! Map request and response size in bytes (RFC6887 section 11.1). +constexpr size_t PCP_MAP_SIZE = 36; + +// Header offsets shared between request and responses (RFC6887 7.1, 7.2), relative to start of packet. +//! Version field (1 byte). +constexpr size_t PCP_HDR_VERSION_OFS = NATPMP_HDR_VERSION_OFS; +//! Opcode field (1 byte). +constexpr size_t PCP_HDR_OP_OFS = NATPMP_HDR_OP_OFS; +//! Requested lifetime (request), granted lifetime (response) (4 bytes). +constexpr size_t PCP_HDR_LIFETIME_OFS = 4; + +// Request header offsets (RFC6887 7.1), relative to start of packet. +//! PCP client's IP address (16 bytes). +constexpr size_t PCP_REQUEST_HDR_IP_OFS = 8; + +// Response header offsets (RFC6887 7.2), relative to start of packet. +//! Result code (1 byte). +constexpr size_t PCP_RESPONSE_HDR_RESULT_OFS = 3; + +// MAP request/response offsets (RFC6887 11.1), relative to start of opcode-specific data. +//! Mapping nonce (12 bytes). +constexpr size_t PCP_MAP_NONCE_OFS = 0; +//! Protocol (1 byte). +constexpr size_t PCP_MAP_PROTOCOL_OFS = 12; +//! Internal port for mapping (2 bytes). +constexpr size_t PCP_MAP_INTERNAL_PORT_OFS = 16; +//! Suggested external port (request), assigned external port (response) (2 bytes). +constexpr size_t PCP_MAP_EXTERNAL_PORT_OFS = 18; +//! Suggested external IP (request), assigned external IP (response) (16 bytes). +constexpr size_t PCP_MAP_EXTERNAL_IP_OFS = 20; + +//! Result code representing success (RFC6887 7.4), shared with NAT-PMP. +constexpr uint8_t PCP_RESULT_SUCCESS = NATPMP_RESULT_SUCCESS; +//! Result code representing lack of resources (RFC6887 7.4). +constexpr uint8_t PCP_RESULT_NO_RESOURCES = 8; + +//! Mapping of PCP result code to string (RFC6887 7.4). Result codes <=2 match NAT-PMP. +const std::map<uint8_t, std::string> PCP_RESULT_STR{ + {0, "SUCCESS"}, + {1, "UNSUPP_VERSION"}, + {2, "NOT_AUTHORIZED"}, + {3, "MALFORMED_REQUEST"}, + {4, "UNSUPP_OPCODE"}, + {5, "UNSUPP_OPTION"}, + {6, "MALFORMED_OPTION"}, + {7, "NETWORK_FAILURE"}, + {8, "NO_RESOURCES"}, + {9, "UNSUPP_PROTOCOL"}, + {10, "USER_EX_QUOTA"}, + {11, "CANNOT_PROVIDE_EXTERNAL"}, + {12, "ADDRESS_MISMATCH"}, + {13, "EXCESSIVE_REMOTE_PEER"}, +}; + +//! Return human-readable string from NATPMP result code. +std::string NATPMPResultString(uint8_t result_code) +{ + auto result_i = NATPMP_RESULT_STR.find(result_code); + return strprintf("%s (code %d)", result_i == NATPMP_RESULT_STR.end() ? "(unknown)" : result_i->second, result_code); +} + +//! Return human-readable string from PCP result code. +std::string PCPResultString(uint8_t result_code) +{ + auto result_i = PCP_RESULT_STR.find(result_code); + return strprintf("%s (code %d)", result_i == PCP_RESULT_STR.end() ? "(unknown)" : result_i->second, result_code); +} + +//! Wrap address in IPv6 according to RFC6887. wrapped_addr needs to be able to store 16 bytes. +[[nodiscard]] bool PCPWrapAddress(Span<uint8_t> wrapped_addr, const CNetAddr &addr) +{ + Assume(wrapped_addr.size() == ADDR_IPV6_SIZE); + if (addr.IsIPv4()) { + struct in_addr addr4; + if (!addr.GetInAddr(&addr4)) return false; + // Section 5: "When the address field holds an IPv4 address, an IPv4-mapped IPv6 address [RFC4291] is used (::ffff:0:0/96)." + std::memcpy(wrapped_addr.data(), IPV4_IN_IPV6_PREFIX.data(), IPV4_IN_IPV6_PREFIX.size()); + std::memcpy(wrapped_addr.data() + IPV4_IN_IPV6_PREFIX.size(), &addr4, ADDR_IPV4_SIZE); + return true; + } else if (addr.IsIPv6()) { + struct in6_addr addr6; + if (!addr.GetIn6Addr(&addr6)) return false; + std::memcpy(wrapped_addr.data(), &addr6, ADDR_IPV6_SIZE); + return true; + } else { + return false; + } +} + +//! Unwrap PCP-encoded address according to RFC6887. +CNetAddr PCPUnwrapAddress(Span<const uint8_t> wrapped_addr) +{ + Assume(wrapped_addr.size() == ADDR_IPV6_SIZE); + if (util::HasPrefix(wrapped_addr, IPV4_IN_IPV6_PREFIX)) { + struct in_addr addr4; + std::memcpy(&addr4, wrapped_addr.data() + IPV4_IN_IPV6_PREFIX.size(), ADDR_IPV4_SIZE); + return CNetAddr(addr4); + } else { + struct in6_addr addr6; + std::memcpy(&addr6, wrapped_addr.data(), ADDR_IPV6_SIZE); + return CNetAddr(addr6); + } +} + +//! PCP or NAT-PMP send-receive loop. +std::optional<std::vector<uint8_t>> PCPSendRecv(Sock &sock, const std::string &protocol, Span<const uint8_t> request, int num_tries, + std::chrono::milliseconds timeout_per_try, + std::function<bool(Span<const uint8_t>)> check_packet) +{ + using namespace std::chrono; + // UDP is a potentially lossy protocol, so we try to send again a few times. + uint8_t response[PCP_MAX_SIZE]; + bool got_response = false; + int recvsz = 0; + for (int ntry = 0; !got_response && ntry < num_tries; ++ntry) { + if (ntry > 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "%s: Retrying (%d)\n", protocol, ntry); + } + // Dispatch packet to gateway. + if (sock.Send(request.data(), request.size(), 0) != static_cast<ssize_t>(request.size())) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "%s: Could not send request: %s\n", protocol, NetworkErrorString(WSAGetLastError())); + return std::nullopt; // Network-level error, probably no use retrying. + } + + // Wait for response(s) until we get a valid response, a network error, or time out. + auto cur_time = time_point_cast<milliseconds>(steady_clock::now()); + auto deadline = cur_time + timeout_per_try; + while ((cur_time = time_point_cast<milliseconds>(steady_clock::now())) < deadline) { + Sock::Event occurred = 0; + if (!sock.Wait(deadline - cur_time, Sock::RECV, &occurred)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "%s: Could not wait on socket: %s\n", protocol, NetworkErrorString(WSAGetLastError())); + return std::nullopt; // Network-level error, probably no use retrying. + } + if (!occurred) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "%s: Timeout\n", protocol); + break; // Retry. + } + + // Receive response. + recvsz = sock.Recv(response, sizeof(response), MSG_DONTWAIT); + if (recvsz < 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "%s: Could not receive response: %s\n", protocol, NetworkErrorString(WSAGetLastError())); + return std::nullopt; // Network-level error, probably no use retrying. + } + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "%s: Received response of %d bytes: %s\n", protocol, recvsz, HexStr(Span(response, recvsz))); + + if (check_packet(Span<uint8_t>(response, recvsz))) { + got_response = true; // Got expected response, break from receive loop as well as from retry loop. + break; + } + } + } + if (!got_response) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "%s: Giving up after %d tries\n", protocol, num_tries); + return std::nullopt; + } + return std::vector<uint8_t>(response, response + recvsz); +} + +} + +std::variant<MappingResult, MappingError> NATPMPRequestPortMap(const CNetAddr &gateway, uint16_t port, uint32_t lifetime, int num_tries, std::chrono::milliseconds timeout_per_try) +{ + struct sockaddr_storage dest_addr; + socklen_t dest_addrlen = sizeof(struct sockaddr_storage); + + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "natpmp: Requesting port mapping port %d from gateway %s\n", port, gateway.ToStringAddr()); + + // Validate gateway, make sure it's IPv4. NAT-PMP does not support IPv6. + if (!CService(gateway, PCP_SERVER_PORT).GetSockAddr((struct sockaddr*)&dest_addr, &dest_addrlen)) return MappingError::NETWORK_ERROR; + if (dest_addr.ss_family != AF_INET) return MappingError::NETWORK_ERROR; + + // Create IPv4 UDP socket + auto sock{CreateSock(AF_INET, SOCK_DGRAM, IPPROTO_UDP)}; + if (!sock) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Could not create UDP socket: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + + // Associate UDP socket to gateway. + if (sock->Connect((struct sockaddr*)&dest_addr, dest_addrlen) != 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Could not connect to gateway: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + + // Use getsockname to get the address toward the default gateway (the internal address). + struct sockaddr_in internal; + socklen_t internal_addrlen = sizeof(struct sockaddr_in); + if (sock->GetSockName((struct sockaddr*)&internal, &internal_addrlen) != 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Could not get sock name: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + + // Request external IP address (RFC6886 section 3.2). + std::vector<uint8_t> request(NATPMP_GETEXTERNAL_REQUEST_SIZE); + request[NATPMP_HDR_VERSION_OFS] = NATPMP_VERSION; + request[NATPMP_HDR_OP_OFS] = NATPMP_REQUEST | NATPMP_OP_GETEXTERNAL; + + auto recv_res = PCPSendRecv(*sock, "natpmp", request, num_tries, timeout_per_try, + [&](const Span<const uint8_t> response) -> bool { + if (response.size() < NATPMP_GETEXTERNAL_RESPONSE_SIZE) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Response too small\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + if (response[NATPMP_HDR_VERSION_OFS] != NATPMP_VERSION || response[NATPMP_HDR_OP_OFS] != (NATPMP_RESPONSE | NATPMP_OP_GETEXTERNAL)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Response to wrong command\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + return true; + }); + + struct in_addr external_addr; + if (recv_res) { + const std::span<const uint8_t> response = *recv_res; + + Assume(response.size() >= NATPMP_GETEXTERNAL_RESPONSE_SIZE); + uint16_t result_code = ReadBE16(response.data() + NATPMP_RESPONSE_HDR_RESULT_OFS); + if (result_code != NATPMP_RESULT_SUCCESS) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Getting external address failed with result %s\n", NATPMPResultString(result_code)); + return MappingError::PROTOCOL_ERROR; + } + + std::memcpy(&external_addr, response.data() + NATPMP_GETEXTERNAL_RESPONSE_IP_OFS, ADDR_IPV4_SIZE); + } else { + return MappingError::NETWORK_ERROR; + } + + // Create TCP mapping request (RFC6886 section 3.3). + request = std::vector<uint8_t>(NATPMP_MAP_REQUEST_SIZE); + request[NATPMP_HDR_VERSION_OFS] = NATPMP_VERSION; + request[NATPMP_HDR_OP_OFS] = NATPMP_REQUEST | NATPMP_OP_MAP_TCP; + WriteBE16(request.data() + NATPMP_MAP_REQUEST_INTERNAL_PORT_OFS, port); + WriteBE16(request.data() + NATPMP_MAP_REQUEST_EXTERNAL_PORT_OFS, port); + WriteBE32(request.data() + NATPMP_MAP_REQUEST_LIFETIME_OFS, lifetime); + + recv_res = PCPSendRecv(*sock, "natpmp", request, num_tries, timeout_per_try, + [&](const Span<const uint8_t> response) -> bool { + if (response.size() < NATPMP_MAP_RESPONSE_SIZE) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Response too small\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + if (response[0] != NATPMP_VERSION || response[1] != (NATPMP_RESPONSE | NATPMP_OP_MAP_TCP)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Response to wrong command\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + uint16_t internal_port = ReadBE16(response.data() + NATPMP_MAP_RESPONSE_INTERNAL_PORT_OFS); + if (internal_port != port) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Response port doesn't match request\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + return true; + }); + + if (recv_res) { + const std::span<uint8_t> response = *recv_res; + + Assume(response.size() >= NATPMP_MAP_RESPONSE_SIZE); + uint16_t result_code = ReadBE16(response.data() + NATPMP_RESPONSE_HDR_RESULT_OFS); + if (result_code != NATPMP_RESULT_SUCCESS) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "natpmp: Port mapping failed with result %s\n", NATPMPResultString(result_code)); + if (result_code == NATPMP_RESULT_NO_RESOURCES) { + return MappingError::NO_RESOURCES; + } + return MappingError::PROTOCOL_ERROR; + } + + uint32_t lifetime_ret = ReadBE32(response.data() + NATPMP_MAP_RESPONSE_LIFETIME_OFS); + uint16_t external_port = ReadBE16(response.data() + NATPMP_MAP_RESPONSE_EXTERNAL_PORT_OFS); + return MappingResult(NATPMP_VERSION, CService(internal.sin_addr, port), CService(external_addr, external_port), lifetime_ret); + } else { + return MappingError::NETWORK_ERROR; + } +} + +std::variant<MappingResult, MappingError> PCPRequestPortMap(const PCPMappingNonce &nonce, const CNetAddr &gateway, const CNetAddr &bind, uint16_t port, uint32_t lifetime, int num_tries, std::chrono::milliseconds timeout_per_try) +{ + struct sockaddr_storage dest_addr, bind_addr; + socklen_t dest_addrlen = sizeof(struct sockaddr_storage), bind_addrlen = sizeof(struct sockaddr_storage); + + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "pcp: Requesting port mapping for addr %s port %d from gateway %s\n", bind.ToStringAddr(), port, gateway.ToStringAddr()); + + // Validate addresses, make sure they're the same network family. + if (!CService(gateway, PCP_SERVER_PORT).GetSockAddr((struct sockaddr*)&dest_addr, &dest_addrlen)) return MappingError::NETWORK_ERROR; + if (!CService(bind, 0).GetSockAddr((struct sockaddr*)&bind_addr, &bind_addrlen)) return MappingError::NETWORK_ERROR; + if (dest_addr.ss_family != bind_addr.ss_family) return MappingError::NETWORK_ERROR; + + // Create UDP socket (IPv4 or IPv6 based on provided gateway). + auto sock{CreateSock(dest_addr.ss_family, SOCK_DGRAM, IPPROTO_UDP)}; + if (!sock) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Could not create UDP socket: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + + // Make sure that we send from requested destination address, anything else will be + // rejected by a security-conscious router. + if (sock->Bind((struct sockaddr*)&bind_addr, bind_addrlen) != 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Could not bind to address: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + + // Associate UDP socket to gateway. + if (sock->Connect((struct sockaddr*)&dest_addr, dest_addrlen) != 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Could not connect to gateway: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + + // Use getsockname to get the address toward the default gateway (the internal address), + // in case we don't know what address to map + // (this is only needed if bind is INADDR_ANY, but it doesn't hurt as an extra check). + struct sockaddr_storage internal_addr; + socklen_t internal_addrlen = sizeof(struct sockaddr_storage); + if (sock->GetSockName((struct sockaddr*)&internal_addr, &internal_addrlen) != 0) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Could not get sock name: %s\n", NetworkErrorString(WSAGetLastError())); + return MappingError::NETWORK_ERROR; + } + CService internal; + if (!internal.SetSockAddr((struct sockaddr*)&internal_addr)) return MappingError::NETWORK_ERROR; + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "pcp: Internal address after connect: %s\n", internal.ToStringAddr()); + + // Build request packet. Make sure the packet is zeroed so that reserved fields are zero + // as required by the spec (and not potentially leak data). + // Make sure there's space for the request header and MAP specific request data. + std::vector<uint8_t> request(PCP_HDR_SIZE + PCP_MAP_SIZE); + // Fill in request header, See RFC6887 Figure 2. + size_t ofs = 0; + request[ofs + PCP_HDR_VERSION_OFS] = PCP_VERSION; + request[ofs + PCP_HDR_OP_OFS] = PCP_REQUEST | PCP_OP_MAP; + WriteBE32(request.data() + ofs + PCP_HDR_LIFETIME_OFS, lifetime); + if (!PCPWrapAddress(Span(request).subspan(ofs + PCP_REQUEST_HDR_IP_OFS, ADDR_IPV6_SIZE), internal)) return MappingError::NETWORK_ERROR; + + ofs += PCP_HDR_SIZE; + + // Fill in MAP request packet, See RFC6887 Figure 9. + // Randomize mapping nonce (this is repeated in the response, to be able to + // correlate requests and responses, and used to authenticate changes to the mapping). + std::memcpy(request.data() + ofs + PCP_MAP_NONCE_OFS, nonce.data(), PCP_MAP_NONCE_SIZE); + request[ofs + PCP_MAP_PROTOCOL_OFS] = PCP_PROTOCOL_TCP; + WriteBE16(request.data() + ofs + PCP_MAP_INTERNAL_PORT_OFS, port); + WriteBE16(request.data() + ofs + PCP_MAP_EXTERNAL_PORT_OFS, port); + if (!PCPWrapAddress(Span(request).subspan(ofs + PCP_MAP_EXTERNAL_IP_OFS, ADDR_IPV6_SIZE), bind)) return MappingError::NETWORK_ERROR; + + ofs += PCP_MAP_SIZE; + Assume(ofs == request.size()); + + // Receive loop. + bool is_natpmp = false; + auto recv_res = PCPSendRecv(*sock, "pcp", request, num_tries, timeout_per_try, + [&](const Span<const uint8_t> response) -> bool { + // Unsupported version according to RFC6887 appendix A and RFC6886 section 3.5, can fall back to NAT-PMP. + if (response.size() == NATPMP_RESPONSE_HDR_SIZE && response[PCP_HDR_VERSION_OFS] == NATPMP_VERSION && response[PCP_RESPONSE_HDR_RESULT_OFS] == NATPMP_RESULT_UNSUPP_VERSION) { + is_natpmp = true; + return true; // Let it through to caller. + } + if (response.size() < (PCP_HDR_SIZE + PCP_MAP_SIZE)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Response too small\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + if (response[PCP_HDR_VERSION_OFS] != PCP_VERSION || response[PCP_HDR_OP_OFS] != (PCP_RESPONSE | PCP_OP_MAP)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Response to wrong command\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + // Handle MAP opcode response. See RFC6887 Figure 10. + // Check that returned mapping nonce matches our request. + if (!std::ranges::equal(response.subspan(PCP_HDR_SIZE + PCP_MAP_NONCE_OFS, PCP_MAP_NONCE_SIZE), nonce)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Mapping nonce mismatch\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + uint8_t protocol = response[PCP_HDR_SIZE + 12]; + uint16_t internal_port = ReadBE16(response.data() + PCP_HDR_SIZE + 16); + if (protocol != PCP_PROTOCOL_TCP || internal_port != port) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Response protocol or port doesn't match request\n"); + return false; // Wasn't response to what we expected, try receiving next packet. + } + return true; + }); + + if (!recv_res) { + return MappingError::NETWORK_ERROR; + } + if (is_natpmp) { + return MappingError::UNSUPP_VERSION; + } + + const std::span<const uint8_t> response = *recv_res; + // If we get here, we got a valid MAP response to our request. + // Check to see if we got the result we expected. + Assume(response.size() >= (PCP_HDR_SIZE + PCP_MAP_SIZE)); + uint8_t result_code = response[PCP_RESPONSE_HDR_RESULT_OFS]; + uint32_t lifetime_ret = ReadBE32(response.data() + PCP_HDR_LIFETIME_OFS); + uint16_t external_port = ReadBE16(response.data() + PCP_HDR_SIZE + PCP_MAP_EXTERNAL_PORT_OFS); + CNetAddr external_addr{PCPUnwrapAddress(response.subspan(PCP_HDR_SIZE + PCP_MAP_EXTERNAL_IP_OFS, ADDR_IPV6_SIZE))}; + if (result_code != PCP_RESULT_SUCCESS) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "pcp: Mapping failed with result %s\n", PCPResultString(result_code)); + if (result_code == PCP_RESULT_NO_RESOURCES) { + return MappingError::NO_RESOURCES; + } + return MappingError::PROTOCOL_ERROR; + } + + return MappingResult(PCP_VERSION, CService(internal, port), CService(external_addr, external_port), lifetime_ret); +} + +std::string MappingResult::ToString() +{ + Assume(version == NATPMP_VERSION || version == PCP_VERSION); + return strprintf("%s:%s -> %s (for %ds)", + version == NATPMP_VERSION ? "natpmp" : "pcp", + external.ToStringAddrPort(), + internal.ToStringAddrPort(), + lifetime + ); +} diff --git a/src/common/pcp.h b/src/common/pcp.h new file mode 100644 index 0000000000..ce2273e140 --- /dev/null +++ b/src/common/pcp.h @@ -0,0 +1,68 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMMON_PCP_H +#define BITCOIN_COMMON_PCP_H + +#include <netaddress.h> + +#include <variant> + +// RFC6886 NAT-PMP and RFC6887 Port Control Protocol (PCP) implementation. +// NAT-PMP and PCP use network byte order (big-endian). + +//! Mapping nonce size in bytes (see RFC6887 section 11.1). +constexpr size_t PCP_MAP_NONCE_SIZE = 12; + +//! PCP mapping nonce. Arbitrary data chosen by the client to identify a mapping. +typedef std::array<uint8_t, PCP_MAP_NONCE_SIZE> PCPMappingNonce; + +//! Unsuccessful response to a port mapping. +enum class MappingError { + NETWORK_ERROR, ///< Any kind of network-level error. + PROTOCOL_ERROR, ///< Any kind of protocol-level error, except unsupported version or no resources. + UNSUPP_VERSION, ///< Unsupported protocol version. + NO_RESOURCES, ///< No resources available (port probably already mapped). +}; + +//! Successful response to a port mapping. +struct MappingResult { + MappingResult(uint8_t version, const CService &internal_in, const CService &external_in, uint32_t lifetime_in): + version(version), internal(internal_in), external(external_in), lifetime(lifetime_in) {} + //! Protocol version, one of NATPMP_VERSION or PCP_VERSION. + uint8_t version; + //! Internal host:port. + CService internal; + //! External host:port. + CService external; + //! Granted lifetime of binding (seconds). + uint32_t lifetime; + + //! Format mapping as string for logging. + std::string ToString(); +}; + +//! Try to open a port using RFC 6886 NAT-PMP. IPv4 only. +//! +//! * gateway: Destination address for PCP requests (usually the default gateway). +//! * port: Internal port, and desired external port. +//! * lifetime: Requested lifetime in seconds for mapping. The server may assign as shorter or longer lifetime. A lifetime of 0 deletes the mapping. +//! * num_tries: Number of tries in case of no response. +//! +//! Returns the external_ip:external_port of the mapping if successful, otherwise a MappingError. +std::variant<MappingResult, MappingError> NATPMPRequestPortMap(const CNetAddr &gateway, uint16_t port, uint32_t lifetime, int num_tries = 3, std::chrono::milliseconds timeout_per_try = std::chrono::milliseconds(1000)); + +//! Try to open a port using RFC 6887 Port Control Protocol (PCP). Handles IPv4 and IPv6. +//! +//! * nonce: Mapping cookie. Keep this the same over renewals. +//! * gateway: Destination address for PCP requests (usually the default gateway). +//! * bind: Specific local bind address for IPv6 pinholing. Set this as INADDR_ANY for IPv4. +//! * port: Internal port, and desired external port. +//! * lifetime: Requested lifetime in seconds for mapping. The server may assign as shorter or longer lifetime. A lifetime of 0 deletes the mapping. +//! * num_tries: Number of tries in case of no response. +//! +//! Returns the external_ip:external_port of the mapping if successful, otherwise a MappingError. +std::variant<MappingResult, MappingError> PCPRequestPortMap(const PCPMappingNonce &nonce, const CNetAddr &gateway, const CNetAddr &bind, uint16_t port, uint32_t lifetime, int num_tries = 3, std::chrono::milliseconds timeout_per_try = std::chrono::milliseconds(1000)); + +#endif // BITCOIN_COMMON_PCP_H diff --git a/src/common/run_command.cpp b/src/common/run_command.cpp index 67608b985f..1f6d51b4f4 100644 --- a/src/common/run_command.cpp +++ b/src/common/run_command.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <common/run_command.h> diff --git a/src/common/settings.cpp b/src/common/settings.cpp index c1520dacd2..0b11e246c6 100644 --- a/src/common/settings.cpp +++ b/src/common/settings.cpp @@ -4,7 +4,7 @@ #include <common/settings.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <tinyformat.h> #include <univalue.h> diff --git a/src/common/system.cpp b/src/common/system.cpp index 6d04c8a7bc..6a9463a0a5 100644 --- a/src/common/system.cpp +++ b/src/common/system.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <common/system.h> diff --git a/src/common/system.h b/src/common/system.h index d9115d3b33..a4b56be9ac 100644 --- a/src/common/system.h +++ b/src/common/system.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_COMMON_SYSTEM_H #define BITCOIN_COMMON_SYSTEM_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <cstdint> #include <string> diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp index af01902c92..dc32f0ab80 100644 --- a/src/consensus/merkle.cpp +++ b/src/consensus/merkle.cpp @@ -83,3 +83,106 @@ uint256 BlockWitnessMerkleRoot(const CBlock& block, bool* mutated) return ComputeMerkleRoot(std::move(leaves), mutated); } +/* This implements a constant-space merkle root/path calculator, limited to 2^32 leaves. */ +static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot, bool* pmutated, uint32_t branchpos, std::vector<uint256>* pbranch) { + if (pbranch) pbranch->clear(); + if (leaves.size() == 0) { + if (pmutated) *pmutated = false; + if (proot) *proot = uint256(); + return; + } + bool mutated = false; + // count is the number of leaves processed so far. + uint32_t count = 0; + // inner is an array of eagerly computed subtree hashes, indexed by tree + // level (0 being the leaves). + // For example, when count is 25 (11001 in binary), inner[4] is the hash of + // the first 16 leaves, inner[3] of the next 8 leaves, and inner[0] equal to + // the last leaf. The other inner entries are undefined. + uint256 inner[32]; + // Which position in inner is a hash that depends on the matching leaf. + int matchlevel = -1; + // First process all leaves into 'inner' values. + while (count < leaves.size()) { + uint256 h = leaves[count]; + bool matchh = count == branchpos; + count++; + int level; + // For each of the lower bits in count that are 0, do 1 step. Each + // corresponds to an inner value that existed before processing the + // current leaf, and each needs a hash to combine it. + for (level = 0; !(count & ((uint32_t{1}) << level)); level++) { + if (pbranch) { + if (matchh) { + pbranch->push_back(inner[level]); + } else if (matchlevel == level) { + pbranch->push_back(h); + matchh = true; + } + } + mutated |= (inner[level] == h); + h = Hash(inner[level], h); + } + // Store the resulting hash at inner position level. + inner[level] = h; + if (matchh) { + matchlevel = level; + } + } + // Do a final 'sweep' over the rightmost branch of the tree to process + // odd levels, and reduce everything to a single top value. + // Level is the level (counted from the bottom) up to which we've sweeped. + int level = 0; + // As long as bit number level in count is zero, skip it. It means there + // is nothing left at this level. + while (!(count & ((uint32_t{1}) << level))) { + level++; + } + uint256 h = inner[level]; + bool matchh = matchlevel == level; + while (count != ((uint32_t{1}) << level)) { + // If we reach this point, h is an inner value that is not the top. + // We combine it with itself (Bitcoin's special rule for odd levels in + // the tree) to produce a higher level one. + if (pbranch && matchh) { + pbranch->push_back(h); + } + h = Hash(h, h); + // Increment count to the value it would have if two entries at this + // level had existed. + count += ((uint32_t{1}) << level); + level++; + // And propagate the result upwards accordingly. + while (!(count & ((uint32_t{1}) << level))) { + if (pbranch) { + if (matchh) { + pbranch->push_back(inner[level]); + } else if (matchlevel == level) { + pbranch->push_back(h); + matchh = true; + } + } + h = Hash(inner[level], h); + level++; + } + } + // Return result. + if (pmutated) *pmutated = mutated; + if (proot) *proot = h; +} + +static std::vector<uint256> ComputeMerkleBranch(const std::vector<uint256>& leaves, uint32_t position) { + std::vector<uint256> ret; + MerkleComputation(leaves, nullptr, nullptr, position, &ret); + return ret; +} + +std::vector<uint256> BlockMerkleBranch(const CBlock& block, uint32_t position) +{ + std::vector<uint256> leaves; + leaves.resize(block.vtx.size()); + for (size_t s = 0; s < block.vtx.size(); s++) { + leaves[s] = block.vtx[s]->GetHash(); + } + return ComputeMerkleBranch(leaves, position); +} diff --git a/src/consensus/merkle.h b/src/consensus/merkle.h index 4ae5a5b897..363f68039c 100644 --- a/src/consensus/merkle.h +++ b/src/consensus/merkle.h @@ -24,4 +24,14 @@ uint256 BlockMerkleRoot(const CBlock& block, bool* mutated = nullptr); */ uint256 BlockWitnessMerkleRoot(const CBlock& block, bool* mutated = nullptr); +/** + * Compute merkle path to the specified transaction + * + * @param[in] block the block + * @param[in] position transaction for which to calculate the merkle path, defaults to coinbase + * + * @return merkle path ordered from the deepest + */ +std::vector<uint256> BlockMerkleBranch(const CBlock& block, uint32_t position = 0); + #endif // BITCOIN_CONSENSUS_MERKLE_H diff --git a/src/crypto/common.h b/src/crypto/common.h index 1dc4f3f55c..d45459b1f6 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -70,6 +70,12 @@ uint64_t static inline ReadBE64(const unsigned char* ptr) return be64toh_internal(x); } +void static inline WriteBE16(unsigned char* ptr, uint16_t x) +{ + uint16_t v = htobe16_internal(x); + memcpy(ptr, &v, 2); +} + void static inline WriteBE32(unsigned char* ptr, uint32_t x) { uint32_t v = htobe32_internal(x); diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index deedc0a6d1..09c5d3123e 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <crypto/sha256.h> #include <crypto/common.h> diff --git a/src/httpserver.cpp b/src/httpserver.cpp index ffc53fbd9c..b8772ed852 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <httpserver.h> diff --git a/src/index/base.cpp b/src/index/base.cpp index 810358394a..1a7eb9cd5e 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -31,7 +31,7 @@ template <typename... Args> void BaseIndex::FatalErrorf(util::ConstevalFormatString<sizeof...(Args)> fmt, const Args&... args) { auto message = tfm::format(fmt, args...); - node::AbortNode(m_chain->context()->shutdown, m_chain->context()->exit_status, Untranslated(message), m_chain->context()->warnings.get()); + node::AbortNode(m_chain->context()->shutdown_request, m_chain->context()->exit_status, Untranslated(message), m_chain->context()->warnings.get()); } CBlockLocator GetLocator(interfaces::Chain& chain, const uint256& block_hash) diff --git a/src/init.cpp b/src/init.cpp index 2a301e114e..ab53cb851d 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <init.h> @@ -123,17 +123,19 @@ using node::ApplyArgsManOptions; using node::BlockManager; using node::CacheSizes; using node::CalculateCacheSizes; +using node::ChainstateLoadResult; +using node::ChainstateLoadStatus; using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINT_MODIFIED_FEE; using node::DEFAULT_STOPATHEIGHT; using node::DumpMempool; -using node::LoadMempool; +using node::ImportBlocks; using node::KernelNotifications; using node::LoadChainstate; +using node::LoadMempool; using node::MempoolPath; using node::NodeContext; using node::ShouldPersistMempool; -using node::ImportBlocks; using node::VerifyLoadedChainstate; using util::Join; using util::ReplaceAll; @@ -205,7 +207,14 @@ void InitContext(NodeContext& node) g_shutdown.emplace(); node.args = &gArgs; - node.shutdown = &*g_shutdown; + node.shutdown_signal = &*g_shutdown; + node.shutdown_request = [&node] { + assert(node.shutdown_signal); + if (!(*node.shutdown_signal)()) return false; + // Wake any threads that may be waiting for the tip to change. + if (node.notifications) WITH_LOCK(node.notifications->m_tip_block_mutex, node.notifications->m_tip_block_cv.notify_all()); + return true; + }; } ////////////////////////////////////////////////////////////////////////////// @@ -233,7 +242,7 @@ void InitContext(NodeContext& node) bool ShutdownRequested(node::NodeContext& node) { - return bool{*Assert(node.shutdown)}; + return bool{*Assert(node.shutdown_signal)}; } #if HAVE_SYSTEM @@ -284,7 +293,7 @@ void Shutdown(NodeContext& node) StopHTTPRPC(); StopREST(); - StopRPC(&node); + StopRPC(); StopHTTPServer(); for (const auto& client : node.chain_clients) { client->flush(); @@ -557,11 +566,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) #else hidden_args.emplace_back("-upnp"); #endif -#ifdef USE_NATPMP - argsman.AddArg("-natpmp", strprintf("Use NAT-PMP to map the listening port (default: %u)", DEFAULT_NATPMP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); -#else - hidden_args.emplace_back("-natpmp"); -#endif // USE_NATPMP + argsman.AddArg("-natpmp", strprintf("Use PCP or NAT-PMP to map the listening port (default: %u)", DEFAULT_NATPMP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-whitebind=<[permissions@]addr>", "Bind to the given address and add permission flags to the peers connecting to it. " "Use [host]:port notation for IPv6. Allowed permissions: " + Join(NET_PERMISSIONS_DOC, ", ") + ". " "Specify multiple permissions separated by commas (default: download,noban,mempool,relay). Can be specified multiple times.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -680,21 +685,6 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) argsman.AddHiddenArgs(hidden_args); } -static bool fHaveGenesis = false; -static GlobalMutex g_genesis_wait_mutex; -static std::condition_variable g_genesis_wait_cv; - -static void BlockNotifyGenesisWait(const CBlockIndex* pBlockIndex) -{ - if (pBlockIndex != nullptr) { - { - LOCK(g_genesis_wait_mutex); - fHaveGenesis = true; - } - g_genesis_wait_cv.notify_all(); - } -} - #if HAVE_SYSTEM static void StartupNotify(const ArgsManager& args) { @@ -709,7 +699,7 @@ static void StartupNotify(const ArgsManager& args) static bool AppInitServers(NodeContext& node) { const ArgsManager& args = *Assert(node.args); - if (!InitHTTPServer(*Assert(node.shutdown))) { + if (!InitHTTPServer(*Assert(node.shutdown_signal))) { return false; } StartRPC(); @@ -824,7 +814,7 @@ namespace { // Variables internal to initialization process only int nMaxConnections; int available_fds; -ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK_LIMITED | NODE_WITNESS); +ServiceFlags g_local_services = ServiceFlags(NODE_NETWORK_LIMITED | NODE_WITNESS); int64_t peer_connect_timeout; std::set<BlockFilterType> g_enabled_filter_types; @@ -939,7 +929,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) // Signal NODE_P2P_V2 if BIP324 v2 transport is enabled. if (args.GetBoolArg("-v2transport", DEFAULT_V2_TRANSPORT)) { - nLocalServices = ServiceFlags(nLocalServices | NODE_P2P_V2); + g_local_services = ServiceFlags(g_local_services | NODE_P2P_V2); } // Signal NODE_COMPACT_FILTERS if peerblockfilters and basic filters index are both enabled. @@ -948,7 +938,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) return InitError(_("Cannot set -peerblockfilters without -blockfilterindex.")); } - nLocalServices = ServiceFlags(nLocalServices | NODE_COMPACT_FILTERS); + g_local_services = ServiceFlags(g_local_services | NODE_COMPACT_FILTERS); } if (args.GetIntArg("-prune", 0)) { @@ -1033,7 +1023,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) SetMockTime(args.GetIntArg("-mocktime", 0)); // SetMockTime(0) is a no-op if (args.GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) - nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); + g_local_services = ServiceFlags(g_local_services | NODE_BLOOM); if (args.IsArgSet("-test")) { if (chainparams.GetChainType() != ChainType::REGTEST) { @@ -1072,6 +1062,13 @@ bool AppInitParameterInteraction(const ArgsManager& args) if (!blockman_result) { return InitError(util::ErrorString(blockman_result)); } + CTxMemPool::Options mempool_opts{ + .check_ratio = chainparams.DefaultConsistencyChecks() ? 1 : 0, + }; + auto mempool_result{ApplyArgsManOptions(args, chainparams, mempool_opts)}; + if (!mempool_result) { + return InitError(util::ErrorString(mempool_result)); + } } return true; @@ -1176,6 +1173,104 @@ bool CheckHostPortOptions(const ArgsManager& args) { return true; } +// A GUI user may opt to retry once if there is a failure during chainstate initialization. +// The function therefore has to support re-entry. +static ChainstateLoadResult InitAndLoadChainstate( + NodeContext& node, + bool do_reindex, + const bool do_reindex_chainstate, + CacheSizes& cache_sizes, + const ArgsManager& args) +{ + const CChainParams& chainparams = Params(); + CTxMemPool::Options mempool_opts{ + .check_ratio = chainparams.DefaultConsistencyChecks() ? 1 : 0, + .signals = node.validation_signals.get(), + }; + Assert(ApplyArgsManOptions(args, chainparams, mempool_opts)); // no error can happen, already checked in AppInitParameterInteraction + bilingual_str mempool_error; + node.mempool = std::make_unique<CTxMemPool>(mempool_opts, mempool_error); + if (!mempool_error.empty()) { + return {ChainstateLoadStatus::FAILURE_FATAL, mempool_error}; + } + LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024)); + ChainstateManager::Options chainman_opts{ + .chainparams = chainparams, + .datadir = args.GetDataDirNet(), + .notifications = *node.notifications, + .signals = node.validation_signals.get(), + }; + Assert(ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction + BlockManager::Options blockman_opts{ + .chainparams = chainman_opts.chainparams, + .blocks_dir = args.GetBlocksDirPath(), + .notifications = chainman_opts.notifications, + }; + Assert(ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction + try { + node.chainman = std::make_unique<ChainstateManager>(*Assert(node.shutdown_signal), chainman_opts, blockman_opts); + } catch (std::exception& e) { + return {ChainstateLoadStatus::FAILURE_FATAL, strprintf(Untranslated("Failed to initialize ChainstateManager: %s"), e.what())}; + } + ChainstateManager& chainman = *node.chainman; + // This is defined and set here instead of inline in validation.h to avoid a hard + // dependency between validation and index/base, since the latter is not in + // libbitcoinkernel. + chainman.snapshot_download_completed = [&node]() { + if (!node.chainman->m_blockman.IsPruneMode()) { + LogPrintf("[snapshot] re-enabling NODE_NETWORK services\n"); + node.connman->AddLocalServices(NODE_NETWORK); + } + LogPrintf("[snapshot] restarting indexes\n"); + // Drain the validation interface queue to ensure that the old indexes + // don't have any pending work. + Assert(node.validation_signals)->SyncWithValidationInterfaceQueue(); + for (auto* index : node.indexes) { + index->Interrupt(); + index->Stop(); + if (!(index->Init() && index->StartBackgroundSync())) { + LogPrintf("[snapshot] WARNING failed to restart index %s on snapshot chain\n", index->GetName()); + } + } + }; + node::ChainstateLoadOptions options; + options.mempool = Assert(node.mempool.get()); + options.wipe_block_tree_db = do_reindex; + options.wipe_chainstate_db = do_reindex || do_reindex_chainstate; + options.prune = chainman.m_blockman.IsPruneMode(); + options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); + options.check_level = args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); + options.require_full_verification = args.IsArgSet("-checkblocks") || args.IsArgSet("-checklevel"); + options.coins_error_cb = [] { + uiInterface.ThreadSafeMessageBox( + _("Error reading from database, shutting down."), + "", CClientUIInterface::MSG_ERROR); + }; + uiInterface.InitMessage(_("Loading block index…").translated); + const auto load_block_index_start_time{SteadyClock::now()}; + auto catch_exceptions = [](auto&& f) { + try { + return f(); + } catch (const std::exception& e) { + LogError("%s\n", e.what()); + return std::make_tuple(node::ChainstateLoadStatus::FAILURE, _("Error opening block database")); + } + }; + auto [status, error] = catch_exceptions([&] { return LoadChainstate(chainman, cache_sizes, options); }); + if (status == node::ChainstateLoadStatus::SUCCESS) { + uiInterface.InitMessage(_("Verifying blocks…").translated); + if (chainman.m_blockman.m_have_pruned && options.check_blocks > MIN_BLOCKS_TO_KEEP) { + LogWarning("pruned datadir may not have more than %d blocks; only checking available blocks\n", + MIN_BLOCKS_TO_KEEP); + } + std::tie(status, error) = catch_exceptions([&] { return VerifyLoadedChainstate(chainman, options); }); + if (status == node::ChainstateLoadStatus::SUCCESS) { + LogPrintf(" block index %15dms\n", Ticks<std::chrono::milliseconds>(SteadyClock::now() - load_block_index_start_time)); + } + } + return {status, error}; +}; + bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) { const ArgsManager& args = *Assert(node.args); @@ -1224,7 +1319,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) constexpr uint64_t min_disk_space = 50 << 20; // 50 MB if (!CheckDiskSpace(args.GetBlocksDirPath(), min_disk_space)) { LogError("Shutting down due to lack of disk space!\n"); - if (!(*Assert(node.shutdown))()) { + if (!(Assert(node.shutdown_request))()) { LogError("Failed to send shutdown signal after disk space check\n"); } } @@ -1505,22 +1600,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 7: load block chain - node.notifications = std::make_unique<KernelNotifications>(*Assert(node.shutdown), node.exit_status, *Assert(node.warnings)); - ReadNotificationArgs(args, *node.notifications); - ChainstateManager::Options chainman_opts{ - .chainparams = chainparams, - .datadir = args.GetDataDirNet(), - .notifications = *node.notifications, - .signals = &validation_signals, - }; - Assert(ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction - - BlockManager::Options blockman_opts{ - .chainparams = chainman_opts.chainparams, - .blocks_dir = args.GetBlocksDirPath(), - .notifications = chainman_opts.notifications, - }; - Assert(ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction + node.notifications = std::make_unique<KernelNotifications>(Assert(node.shutdown_request), node.exit_status, *Assert(node.warnings)); + auto& kernel_notifications{*node.notifications}; + ReadNotificationArgs(args, kernel_notifications); // cache size calculations CacheSizes cache_sizes = CalculateCacheSizes(args, g_enabled_filter_types.size()); @@ -1539,119 +1621,39 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) assert(!node.mempool); assert(!node.chainman); - CTxMemPool::Options mempool_opts{ - .check_ratio = chainparams.DefaultConsistencyChecks() ? 1 : 0, - .signals = &validation_signals, - }; - auto result{ApplyArgsManOptions(args, chainparams, mempool_opts)}; - if (!result) { - return InitError(util::ErrorString(result)); - } - bool do_reindex{args.GetBoolArg("-reindex", false)}; const bool do_reindex_chainstate{args.GetBoolArg("-reindex-chainstate", false)}; - for (bool fLoaded = false; !fLoaded && !ShutdownRequested(node);) { - bilingual_str mempool_error; - node.mempool = std::make_unique<CTxMemPool>(mempool_opts, mempool_error); - if (!mempool_error.empty()) { - return InitError(mempool_error); - } - LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024)); - - try { - node.chainman = std::make_unique<ChainstateManager>(*Assert(node.shutdown), chainman_opts, blockman_opts); - } catch (std::exception& e) { - return InitError(strprintf(Untranslated("Failed to initialize ChainstateManager: %s"), e.what())); - } - ChainstateManager& chainman = *node.chainman; - - // This is defined and set here instead of inline in validation.h to avoid a hard - // dependency between validation and index/base, since the latter is not in - // libbitcoinkernel. - chainman.snapshot_download_completed = [&node]() { - if (!node.chainman->m_blockman.IsPruneMode()) { - LogPrintf("[snapshot] re-enabling NODE_NETWORK services\n"); - node.connman->AddLocalServices(NODE_NETWORK); - } - - LogPrintf("[snapshot] restarting indexes\n"); - - // Drain the validation interface queue to ensure that the old indexes - // don't have any pending work. - Assert(node.validation_signals)->SyncWithValidationInterfaceQueue(); - - for (auto* index : node.indexes) { - index->Interrupt(); - index->Stop(); - if (!(index->Init() && index->StartBackgroundSync())) { - LogPrintf("[snapshot] WARNING failed to restart index %s on snapshot chain\n", index->GetName()); - } - } - }; - - node::ChainstateLoadOptions options; - options.mempool = Assert(node.mempool.get()); - options.wipe_block_tree_db = do_reindex; - options.wipe_chainstate_db = do_reindex || do_reindex_chainstate; - options.prune = chainman.m_blockman.IsPruneMode(); - options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); - options.check_level = args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); - options.require_full_verification = args.IsArgSet("-checkblocks") || args.IsArgSet("-checklevel"); - options.coins_error_cb = [] { - uiInterface.ThreadSafeMessageBox( - _("Error reading from database, shutting down."), - "", CClientUIInterface::MSG_ERROR); - }; - - uiInterface.InitMessage(_("Loading block index…").translated); - const auto load_block_index_start_time{SteadyClock::now()}; - auto catch_exceptions = [](auto&& f) { - try { - return f(); - } catch (const std::exception& e) { - LogError("%s\n", e.what()); - return std::make_tuple(node::ChainstateLoadStatus::FAILURE, _("Error opening block database")); - } - }; - auto [status, error] = catch_exceptions([&]{ return LoadChainstate(chainman, cache_sizes, options); }); - if (status == node::ChainstateLoadStatus::SUCCESS) { - uiInterface.InitMessage(_("Verifying blocks…").translated); - if (chainman.m_blockman.m_have_pruned && options.check_blocks > MIN_BLOCKS_TO_KEEP) { - LogWarning("pruned datadir may not have more than %d blocks; only checking available blocks\n", - MIN_BLOCKS_TO_KEEP); - } - std::tie(status, error) = catch_exceptions([&]{ return VerifyLoadedChainstate(chainman, options);}); - if (status == node::ChainstateLoadStatus::SUCCESS) { - fLoaded = true; - LogPrintf(" block index %15dms\n", Ticks<std::chrono::milliseconds>(SteadyClock::now() - load_block_index_start_time)); - } + // Chainstate initialization and loading may be retried once with reindexing by GUI users + auto [status, error] = InitAndLoadChainstate( + node, + do_reindex, + do_reindex_chainstate, + cache_sizes, + args); + if (status == ChainstateLoadStatus::FAILURE && !do_reindex && !ShutdownRequested(node)) { + // suggest a reindex + bool do_retry = uiInterface.ThreadSafeQuestion( + error + Untranslated(".\n\n") + _("Do you want to rebuild the block database now?"), + error.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", + "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); + if (!do_retry) { + LogError("Aborted block database rebuild. Exiting.\n"); + return false; } - - if (status == node::ChainstateLoadStatus::FAILURE_FATAL || status == node::ChainstateLoadStatus::FAILURE_INCOMPATIBLE_DB || status == node::ChainstateLoadStatus::FAILURE_INSUFFICIENT_DBCACHE) { - return InitError(error); - } - - if (!fLoaded && !ShutdownRequested(node)) { - // first suggest a reindex - if (!do_reindex) { - bool fRet = uiInterface.ThreadSafeQuestion( - error + Untranslated(".\n\n") + _("Do you want to rebuild the block database now?"), - error.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", - "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); - if (fRet) { - do_reindex = true; - if (!Assert(node.shutdown)->reset()) { - LogError("Internal error: failed to reset shutdown signal.\n"); - } - } else { - LogError("Aborted block database rebuild. Exiting.\n"); - return false; - } - } else { - return InitError(error); - } + do_reindex = true; + if (!Assert(node.shutdown_signal)->reset()) { + LogError("Internal error: failed to reset shutdown signal.\n"); } + std::tie(status, error) = InitAndLoadChainstate( + node, + do_reindex, + do_reindex_chainstate, + cache_sizes, + args); + } + if (status != ChainstateLoadStatus::SUCCESS && status != ChainstateLoadStatus::INTERRUPTED) { + return InitError(error); } // As LoadBlockIndex can take several minutes, it's possible the user @@ -1714,7 +1716,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Prior to setting NODE_NETWORK, check if we can provide historical blocks. if (!WITH_LOCK(chainman.GetMutex(), return chainman.BackgroundSyncInProgress())) { LogPrintf("Setting NODE_NETWORK on non-prune mode\n"); - nLocalServices = ServiceFlags(nLocalServices | NODE_NETWORK); + g_local_services = ServiceFlags(g_local_services | NODE_NETWORK); } else { LogPrintf("Running node in NODE_NETWORK_LIMITED mode until snapshot background sync completes\n"); } @@ -1752,15 +1754,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } } - // Either install a handler to notify us when genesis activates, or set fHaveGenesis directly. - // No locking, as this happens before any background thread is started. - boost::signals2::connection block_notify_genesis_wait_connection; - if (WITH_LOCK(chainman.GetMutex(), return chainman.ActiveChain().Tip() == nullptr)) { - block_notify_genesis_wait_connection = uiInterface.NotifyBlockTip_connect(std::bind(BlockNotifyGenesisWait, std::placeholders::_2)); - } else { - fHaveGenesis = true; - } - #if HAVE_SYSTEM const std::string block_notify = args.GetArg("-blocknotify", ""); if (!block_notify.empty()) { @@ -1785,7 +1778,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) ImportBlocks(chainman, vImportFiles); if (args.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { LogPrintf("Stopping after block import\n"); - if (!(*Assert(node.shutdown))()) { + if (!(Assert(node.shutdown_request))()) { LogError("Failed to send shutdown signal after finishing block import\n"); } return; @@ -1805,15 +1798,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) }); // Wait for genesis block to be processed - { - WAIT_LOCK(g_genesis_wait_mutex, lock); - // We previously could hang here if shutdown was requested prior to - // ImportBlocks getting started, so instead we just wait on a timer to - // check ShutdownRequested() regularly. - while (!fHaveGenesis && !ShutdownRequested(node)) { - g_genesis_wait_cv.wait_for(lock, std::chrono::milliseconds(500)); - } - block_notify_genesis_wait_connection.disconnect(); + if (WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip() == nullptr)) { + WAIT_LOCK(kernel_notifications.m_tip_block_mutex, lock); + kernel_notifications.m_tip_block_cv.wait(lock, [&]() EXCLUSIVE_LOCKS_REQUIRED(kernel_notifications.m_tip_block_mutex) { + return !kernel_notifications.m_tip_block.IsNull() || ShutdownRequested(node); + }); } if (ShutdownRequested(node)) { @@ -1822,17 +1811,17 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 12: start node - //// debug print int64_t best_block_time{}; { - LOCK(cs_main); + LOCK(chainman.GetMutex()); + const auto& tip{*Assert(chainman.ActiveTip())}; LogPrintf("block tree size = %u\n", chainman.BlockIndex().size()); - chain_active_height = chainman.ActiveChain().Height(); - best_block_time = chainman.ActiveChain().Tip() ? chainman.ActiveChain().Tip()->GetBlockTime() : chainman.GetParams().GenesisBlock().GetBlockTime(); + chain_active_height = tip.nHeight; + best_block_time = tip.GetBlockTime(); if (tip_info) { tip_info->block_height = chain_active_height; tip_info->block_time = best_block_time; - tip_info->verification_progress = GuessVerificationProgress(chainman.GetParams().TxData(), chainman.ActiveChain().Tip()); + tip_info->verification_progress = GuessVerificationProgress(chainman.GetParams().TxData(), &tip); } if (tip_info && chainman.m_best_header) { tip_info->header_height = chainman.m_best_header->nHeight; @@ -1842,11 +1831,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) LogPrintf("nBestHeight = %d\n", chain_active_height); if (node.peerman) node.peerman->SetBestBlock(chain_active_height, std::chrono::seconds{best_block_time}); - // Map ports with UPnP or NAT-PMP. + // Map ports with UPnP or NAT-PMP StartMapPort(args.GetBoolArg("-upnp", DEFAULT_UPNP), args.GetBoolArg("-natpmp", DEFAULT_NATPMP)); CConnman::Options connOptions; - connOptions.nLocalServices = nLocalServices; + connOptions.m_local_services = g_local_services; connOptions.m_max_automatic_connections = nMaxConnections; connOptions.uiInterface = &uiInterface; connOptions.m_banman = node.banman.get(); diff --git a/src/init/common.cpp b/src/init/common.cpp index 36142c2b9a..dd8ca020d2 100644 --- a/src/init/common.cpp +++ b/src/init/common.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <clientversion.h> #include <common/args.h> diff --git a/src/interfaces/mining.h b/src/interfaces/mining.h index 421c5ba762..c77f3c30a2 100644 --- a/src/interfaces/mining.h +++ b/src/interfaces/mining.h @@ -42,6 +42,20 @@ public: virtual CTransactionRef getCoinbaseTx() = 0; virtual std::vector<unsigned char> getCoinbaseCommitment() = 0; virtual int getWitnessCommitmentIndex() = 0; + + /** + * Compute merkle path to the coinbase transaction + * + * @return merkle path ordered from the deepest + */ + virtual std::vector<uint256> getCoinbaseMerklePath() = 0; + + /** + * Construct and broadcast the block. + * + * @returns if the block was processed, independent of block validity + */ + virtual bool submitSolution(uint32_t version, uint32_t timestamp, uint32_t nonce, CMutableTransaction coinbase) = 0; }; //! Interface giving clients (RPC, Stratum v2 Template Provider in the future) @@ -61,11 +75,11 @@ public: virtual std::optional<BlockRef> getTip() = 0; /** - * Waits for the tip to change + * Waits for the connected tip to change. If the tip was not connected on + * startup, this will wait. * * @param[in] current_tip block hash of the current chain tip. Function waits - * for the chain tip to change if this matches, otherwise - * it returns right away. + * for the chain tip to differ from this. * @param[in] timeout how long to wait for a new tip * @returns Hash and height of the current chain tip after this call. */ diff --git a/src/interfaces/node.h b/src/interfaces/node.h index b87c78db52..91a623a65d 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -121,7 +121,7 @@ public: virtual void resetSettings() = 0; //! Map port. - virtual void mapPort(bool use_upnp, bool use_natpmp) = 0; + virtual void mapPort(bool use_upnp, bool use_pcp) = 0; //! Get proxy. virtual bool getProxy(Network net, Proxy& proxy_info) = 0; diff --git a/src/ipc/capnp/mining.capnp b/src/ipc/capnp/mining.capnp index 4ea69d16c9..5e0216acea 100644 --- a/src/ipc/capnp/mining.capnp +++ b/src/ipc/capnp/mining.capnp @@ -31,6 +31,8 @@ interface BlockTemplate $Proxy.wrap("interfaces::BlockTemplate") { getCoinbaseTx @4 (context: Proxy.Context) -> (result: Data); getCoinbaseCommitment @5 (context: Proxy.Context) -> (result: Data); getWitnessCommitmentIndex @6 (context: Proxy.Context) -> (result: Int32); + getCoinbaseMerklePath @7 (context: Proxy.Context) -> (result: List(Data)); + submitSolution@8 (context: Proxy.Context, version: UInt32, timestamp: UInt32, nonce: UInt32, coinbase :Data) -> (result: Bool); } struct BlockCreateOptions $Proxy.wrap("node::BlockCreateOptions") { diff --git a/src/kernel/CMakeLists.txt b/src/kernel/CMakeLists.txt index 65d753b3af..7bf8efc516 100644 --- a/src/kernel/CMakeLists.txt +++ b/src/kernel/CMakeLists.txt @@ -84,6 +84,7 @@ target_link_libraries(bitcoinkernel bitcoin_crypto leveldb secp256k1 + $<TARGET_NAME_IF_EXISTS:USDT::headers> PUBLIC Boost::headers ) diff --git a/src/logging.cpp b/src/logging.cpp index d04db767e6..5f055566ef 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -5,6 +5,7 @@ #include <logging.h> #include <memusage.h> +#include <util/check.h> #include <util/fs.h> #include <util/string.h> #include <util/threadnames.h> @@ -368,6 +369,8 @@ static size_t MemUsage(const BCLog::Logger::BufferedLog& buflog) void BCLog::Logger::FormatLogStrInPlace(std::string& str, BCLog::LogFlags category, BCLog::Level level, std::string_view source_file, int source_line, std::string_view logging_function, std::string_view threadname, SystemClock::time_point now, std::chrono::seconds mocktime) const { + if (!str.ends_with('\n')) str.push_back('\n'); + str.insert(0, GetLogPrefix(category, level)); if (m_log_sourcelocations) { @@ -391,21 +394,7 @@ void BCLog::Logger::LogPrintStr_(std::string_view str, std::string_view logging_ { std::string str_prefixed = LogEscapeMessage(str); - const bool starts_new_line = m_started_new_line; - m_started_new_line = !str.empty() && str[str.size()-1] == '\n'; - if (m_buffering) { - if (!starts_new_line) { - if (!m_msgs_before_open.empty()) { - m_msgs_before_open.back().str += str_prefixed; - m_cur_buffer_memusage += str_prefixed.size(); - return; - } else { - // unlikely edge case; add a marker that something was trimmed - str_prefixed.insert(0, "[...] "); - } - } - { BufferedLog buf{ .now=SystemClock::now(), @@ -435,9 +424,7 @@ void BCLog::Logger::LogPrintStr_(std::string_view str, std::string_view logging_ return; } - if (starts_new_line) { - FormatLogStrInPlace(str_prefixed, category, level, source_file, source_line, logging_function, util::ThreadGetInternalName(), SystemClock::now(), GetMockTime()); - } + FormatLogStrInPlace(str_prefixed, category, level, source_file, source_line, logging_function, util::ThreadGetInternalName(), SystemClock::now(), GetMockTime()); if (m_print_to_console) { // print to console diff --git a/src/logging.h b/src/logging.h index 8605c8cd64..fdc12c79b3 100644 --- a/src/logging.h +++ b/src/logging.h @@ -105,13 +105,6 @@ namespace BCLog { size_t m_cur_buffer_memusage GUARDED_BY(m_cs){0}; size_t m_buffer_lines_discarded GUARDED_BY(m_cs){0}; - /** - * m_started_new_line is a state variable that will suppress printing of - * the timestamp when multiple calls are made that don't end in a - * newline. - */ - std::atomic_bool m_started_new_line{true}; - //! Category-specific log level. Overrides `m_log_level`. std::unordered_map<LogFlags, Level> m_category_log_levels GUARDED_BY(m_cs); @@ -253,7 +246,6 @@ inline void LogPrintFormatInternal(std::string_view logging_function, std::strin try { log_msg = tfm::format(fmt, args...); } catch (tinyformat::format_error& fmterr) { - /* Original format string will have newline so don't add one here */ log_msg = "Error \"" + std::string{fmterr.what()} + "\" while formatting log message: " + fmt.fmt; } LogInstance().LogPrintStr(log_msg, logging_function, source_file, source_line, flag, level); diff --git a/src/mapport.cpp b/src/mapport.cpp index 1920297be6..bdeda6da34 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -2,24 +2,22 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <mapport.h> #include <clientversion.h> +#include <common/netif.h> +#include <common/pcp.h> #include <common/system.h> #include <logging.h> #include <net.h> #include <netaddress.h> #include <netbase.h> +#include <random.h> #include <util/thread.h> #include <util/threadinterrupt.h> -#ifdef USE_NATPMP -#include <compat/compat.h> -#include <natpmp.h> -#endif // USE_NATPMP - #ifdef USE_UPNP #include <miniupnpc/miniupnpc.h> #include <miniupnpc/upnpcommands.h> @@ -36,7 +34,6 @@ static_assert(MINIUPNPC_API_VERSION >= 17, "miniUPnPc API version >= 17 assumed" #include <string> #include <thread> -#if defined(USE_NATPMP) || defined(USE_UPNP) static CThreadInterrupt g_mapport_interrupt; static std::thread g_mapport_thread; static std::atomic_uint g_mapport_enabled_protos{MapPortProtoFlag::NONE}; @@ -46,104 +43,96 @@ using namespace std::chrono_literals; static constexpr auto PORT_MAPPING_REANNOUNCE_PERIOD{20min}; static constexpr auto PORT_MAPPING_RETRY_PERIOD{5min}; -#ifdef USE_NATPMP -static uint16_t g_mapport_external_port = 0; -static bool NatpmpInit(natpmp_t* natpmp) +static bool ProcessPCP() { - const int r_init = initnatpmp(natpmp, /* detect gateway automatically */ 0, /* forced gateway - NOT APPLIED*/ 0); - if (r_init == 0) return true; - LogPrintf("natpmp: initnatpmp() failed with %d error.\n", r_init); - return false; -} + // The same nonce is used for all mappings, this is allowed by the spec, and simplifies keeping track of them. + PCPMappingNonce pcp_nonce; + GetRandBytes(pcp_nonce); -static bool NatpmpDiscover(natpmp_t* natpmp, struct in_addr& external_ipv4_addr) -{ - const int r_send = sendpublicaddressrequest(natpmp); - if (r_send == 2 /* OK */) { - int r_read; - natpmpresp_t response; - do { - r_read = readnatpmpresponseorretry(natpmp, &response); - } while (r_read == NATPMP_TRYAGAIN); - - if (r_read == 0) { - external_ipv4_addr = response.pnu.publicaddress.addr; - return true; - } else if (r_read == NATPMP_ERR_NOGATEWAYSUPPORT) { - LogPrintf("natpmp: The gateway does not support NAT-PMP.\n"); + bool ret = false; + bool no_resources = false; + const uint16_t private_port = GetListenPort(); + // Multiply the reannounce period by two, as we'll try to renew approximately halfway. + const uint32_t requested_lifetime = std::chrono::seconds(PORT_MAPPING_REANNOUNCE_PERIOD * 2).count(); + uint32_t actual_lifetime = 0; + std::chrono::milliseconds sleep_time; + + // Local functor to handle result from PCP/NATPMP mapping. + auto handle_mapping = [&](std::variant<MappingResult, MappingError> &res) -> void { + if (MappingResult* mapping = std::get_if<MappingResult>(&res)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Info, "portmap: Added mapping %s\n", mapping->ToString()); + AddLocal(mapping->external, LOCAL_MAPPED); + ret = true; + actual_lifetime = std::min(actual_lifetime, mapping->lifetime); + } else if (MappingError *err = std::get_if<MappingError>(&res)) { + // Detailed error will already have been logged internally in respective Portmap function. + if (*err == MappingError::NO_RESOURCES) { + no_resources = true; + } + } + }; + + do { + actual_lifetime = requested_lifetime; + no_resources = false; // Set to true if there was any "no resources" error. + ret = false; // Set to true if any mapping succeeds. + + // IPv4 + std::optional<CNetAddr> gateway4 = QueryDefaultGateway(NET_IPV4); + if (!gateway4) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "portmap: Could not determine IPv4 default gateway\n"); } else { - LogPrintf("natpmp: readnatpmpresponseorretry() for public address failed with %d error.\n", r_read); + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "portmap: gateway [IPv4]: %s\n", gateway4->ToStringAddr()); + + // Open a port mapping on whatever local address we have toward the gateway. + struct in_addr inaddr_any; + inaddr_any.s_addr = htonl(INADDR_ANY); + auto res = PCPRequestPortMap(pcp_nonce, *gateway4, CNetAddr(inaddr_any), private_port, requested_lifetime); + MappingError* pcp_err = std::get_if<MappingError>(&res); + if (pcp_err && *pcp_err == MappingError::UNSUPP_VERSION) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "portmap: Got unsupported PCP version response, falling back to NAT-PMP\n"); + res = NATPMPRequestPortMap(*gateway4, private_port, requested_lifetime); + } + handle_mapping(res); } - } else { - LogPrintf("natpmp: sendpublicaddressrequest() failed with %d error.\n", r_send); - } - return false; -} + // IPv6 + std::optional<CNetAddr> gateway6 = QueryDefaultGateway(NET_IPV6); + if (!gateway6) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "portmap: Could not determine IPv6 default gateway\n"); + } else { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "portmap: gateway [IPv6]: %s\n", gateway6->ToStringAddr()); -static bool NatpmpMapping(natpmp_t* natpmp, const struct in_addr& external_ipv4_addr, uint16_t private_port, bool& external_ip_discovered) -{ - const uint16_t suggested_external_port = g_mapport_external_port ? g_mapport_external_port : private_port; - const int r_send = sendnewportmappingrequest(natpmp, NATPMP_PROTOCOL_TCP, private_port, suggested_external_port, 3600 /*seconds*/); - if (r_send == 12 /* OK */) { - int r_read; - natpmpresp_t response; - do { - r_read = readnatpmpresponseorretry(natpmp, &response); - } while (r_read == NATPMP_TRYAGAIN); - - if (r_read == 0) { - auto pm = response.pnu.newportmapping; - if (private_port == pm.privateport && pm.lifetime > 0) { - g_mapport_external_port = pm.mappedpublicport; - const CService external{external_ipv4_addr, pm.mappedpublicport}; - if (!external_ip_discovered && fDiscover) { - AddLocal(external, LOCAL_MAPPED); - external_ip_discovered = true; - } - LogPrintf("natpmp: Port mapping successful. External address = %s\n", external.ToStringAddrPort()); - return true; - } else { - LogPrintf("natpmp: Port mapping failed.\n"); + // Try to open pinholes for all routable local IPv6 addresses. + for (const auto &addr: GetLocalAddresses()) { + if (!addr.IsRoutable() || !addr.IsIPv6()) continue; + auto res = PCPRequestPortMap(pcp_nonce, *gateway6, addr, private_port, requested_lifetime); + handle_mapping(res); } - } else if (r_read == NATPMP_ERR_NOGATEWAYSUPPORT) { - LogPrintf("natpmp: The gateway does not support NAT-PMP.\n"); - } else { - LogPrintf("natpmp: readnatpmpresponseorretry() for port mapping failed with %d error.\n", r_read); } - } else { - LogPrintf("natpmp: sendnewportmappingrequest() failed with %d error.\n", r_send); - } - - return false; -} -static bool ProcessNatpmp() -{ - bool ret = false; - natpmp_t natpmp; - struct in_addr external_ipv4_addr; - if (NatpmpInit(&natpmp) && NatpmpDiscover(&natpmp, external_ipv4_addr)) { - bool external_ip_discovered = false; - const uint16_t private_port = GetListenPort(); - do { - ret = NatpmpMapping(&natpmp, external_ipv4_addr, private_port, external_ip_discovered); - } while (ret && g_mapport_interrupt.sleep_for(PORT_MAPPING_REANNOUNCE_PERIOD)); - g_mapport_interrupt.reset(); + // Log message if we got NO_RESOURCES. + if (no_resources) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "portmap: At least one mapping failed because of a NO_RESOURCES error. This usually indicates that the port is already used on the router. If this is the only instance of bitcoin running on the network, this will resolve itself automatically. Otherwise, you might want to choose a different P2P port to prevent this conflict.\n"); + } - const int r_send = sendnewportmappingrequest(&natpmp, NATPMP_PROTOCOL_TCP, private_port, g_mapport_external_port, /* remove a port mapping */ 0); - g_mapport_external_port = 0; - if (r_send == 12 /* OK */) { - LogPrintf("natpmp: Port mapping removed successfully.\n"); - } else { - LogPrintf("natpmp: sendnewportmappingrequest(0) failed with %d error.\n", r_send); + // Sanity-check returned lifetime. + if (actual_lifetime < 30) { + LogPrintLevel(BCLog::NET, BCLog::Level::Warning, "portmap: Got impossibly short mapping lifetime of %d seconds\n", actual_lifetime); + return false; } - } + // RFC6887 11.2.1 recommends that clients send their first renewal packet at a time chosen with uniform random + // distribution in the range 1/2 to 5/8 of expiration time. + std::chrono::seconds sleep_time_min(actual_lifetime / 2); + std::chrono::seconds sleep_time_max(actual_lifetime * 5 / 8); + sleep_time = sleep_time_min + FastRandomContext().randrange<std::chrono::milliseconds>(sleep_time_max - sleep_time_min); + } while (ret && g_mapport_interrupt.sleep_for(sleep_time)); + + // We don't delete the mappings when the thread is interrupted because this would add additional complexity, so + // we rather just choose a fairly short expiry time. - closenatpmp(&natpmp); return ret; } -#endif // USE_NATPMP #ifdef USE_UPNP static bool ProcessUpnp() @@ -223,23 +212,21 @@ static void ThreadMapPort() do { ok = false; -#ifdef USE_UPNP // High priority protocol. - if (g_mapport_enabled_protos & MapPortProtoFlag::UPNP) { - g_mapport_current_proto = MapPortProtoFlag::UPNP; - ok = ProcessUpnp(); + if (g_mapport_enabled_protos & MapPortProtoFlag::PCP) { + g_mapport_current_proto = MapPortProtoFlag::PCP; + ok = ProcessPCP(); if (ok) continue; } -#endif // USE_UPNP -#ifdef USE_NATPMP +#ifdef USE_UPNP // Low priority protocol. - if (g_mapport_enabled_protos & MapPortProtoFlag::NAT_PMP) { - g_mapport_current_proto = MapPortProtoFlag::NAT_PMP; - ok = ProcessNatpmp(); + if (g_mapport_enabled_protos & MapPortProtoFlag::UPNP) { + g_mapport_current_proto = MapPortProtoFlag::UPNP; + ok = ProcessUpnp(); if (ok) continue; } -#endif // USE_NATPMP +#endif // USE_UPNP g_mapport_current_proto = MapPortProtoFlag::NONE; if (g_mapport_enabled_protos == MapPortProtoFlag::NONE) { @@ -281,7 +268,7 @@ static void DispatchMapPort() assert(g_mapport_thread.joinable()); assert(!g_mapport_interrupt); - // Interrupt a protocol-specific loop in the ThreadUpnp() or in the ThreadNatpmp() + // Interrupt a protocol-specific loop in the ThreadUpnp() or in the ThreadPCP() // to force trying the next protocol in the ThreadMapPort() loop. g_mapport_interrupt(); } @@ -295,10 +282,10 @@ static void MapPortProtoSetEnabled(MapPortProtoFlag proto, bool enabled) } } -void StartMapPort(bool use_upnp, bool use_natpmp) +void StartMapPort(bool use_upnp, bool use_pcp) { MapPortProtoSetEnabled(MapPortProtoFlag::UPNP, use_upnp); - MapPortProtoSetEnabled(MapPortProtoFlag::NAT_PMP, use_natpmp); + MapPortProtoSetEnabled(MapPortProtoFlag::PCP, use_pcp); DispatchMapPort(); } @@ -317,18 +304,3 @@ void StopMapPort() g_mapport_interrupt.reset(); } } - -#else // #if defined(USE_NATPMP) || defined(USE_UPNP) -void StartMapPort(bool use_upnp, bool use_natpmp) -{ - // Intentionally left blank. -} -void InterruptMapPort() -{ - // Intentionally left blank. -} -void StopMapPort() -{ - // Intentionally left blank. -} -#endif // #if defined(USE_NATPMP) || defined(USE_UPNP) diff --git a/src/mapport.h b/src/mapport.h index 6f55c46f6c..51202687f2 100644 --- a/src/mapport.h +++ b/src/mapport.h @@ -12,10 +12,10 @@ static constexpr bool DEFAULT_NATPMP = false; enum MapPortProtoFlag : unsigned int { NONE = 0x00, UPNP = 0x01, - NAT_PMP = 0x02, + PCP = 0x02, // PCP with NAT-PMP fallback. }; -void StartMapPort(bool use_upnp, bool use_natpmp); +void StartMapPort(bool use_upnp, bool use_pcp); void InterruptMapPort(); void StopMapPort(); diff --git a/src/net.cpp b/src/net.cpp index d87adcc031..477240cdf2 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <net.h> @@ -12,6 +12,7 @@ #include <banman.h> #include <clientversion.h> #include <common/args.h> +#include <common/netif.h> #include <compat/compat.h> #include <consensus/consensus.h> #include <crypto/sha256.h> @@ -2949,7 +2950,7 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai return; pnode->grantOutbound = std::move(grant_outbound); - m_msgproc->InitializeNode(*pnode, nLocalServices); + m_msgproc->InitializeNode(*pnode, m_local_services); { LOCK(m_nodes_mutex); m_nodes.push_back(pnode); @@ -3118,46 +3119,10 @@ void Discover() if (!fDiscover) return; -#ifdef WIN32 - // Get local host IP - char pszHostName[256] = ""; - if (gethostname(pszHostName, sizeof(pszHostName)) != SOCKET_ERROR) - { - const std::vector<CNetAddr> addresses{LookupHost(pszHostName, 0, true)}; - for (const CNetAddr& addr : addresses) - { - if (AddLocal(addr, LOCAL_IF)) - LogPrintf("%s: %s - %s\n", __func__, pszHostName, addr.ToStringAddr()); - } + for (const CNetAddr &addr: GetLocalAddresses()) { + if (AddLocal(addr, LOCAL_IF)) + LogPrintf("%s: %s\n", __func__, addr.ToStringAddr()); } -#elif (HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS) - // Get local host ip - struct ifaddrs* myaddrs; - if (getifaddrs(&myaddrs) == 0) - { - for (struct ifaddrs* ifa = myaddrs; ifa != nullptr; ifa = ifa->ifa_next) - { - if (ifa->ifa_addr == nullptr) continue; - if ((ifa->ifa_flags & IFF_UP) == 0) continue; - if ((ifa->ifa_flags & IFF_LOOPBACK) != 0) continue; - if (ifa->ifa_addr->sa_family == AF_INET) - { - struct sockaddr_in* s4 = (struct sockaddr_in*)(ifa->ifa_addr); - CNetAddr addr(s4->sin_addr); - if (AddLocal(addr, LOCAL_IF)) - LogPrintf("%s: IPv4 %s: %s\n", __func__, ifa->ifa_name, addr.ToStringAddr()); - } - else if (ifa->ifa_addr->sa_family == AF_INET6) - { - struct sockaddr_in6* s6 = (struct sockaddr_in6*)(ifa->ifa_addr); - CNetAddr addr(s6->sin6_addr); - if (AddLocal(addr, LOCAL_IF)) - LogPrintf("%s: IPv6 %s: %s\n", __func__, ifa->ifa_name, addr.ToStringAddr()); - } - } - freeifaddrs(myaddrs); - } -#endif } void CConnman::SetNetworkActive(bool active) @@ -3746,7 +3711,7 @@ uint64_t CConnman::GetTotalBytesSent() const ServiceFlags CConnman::GetLocalServices() const { - return nLocalServices; + return m_local_services; } static std::unique_ptr<Transport> MakeTransport(NodeId id, bool use_v2transport, bool inbound) noexcept @@ -148,7 +148,7 @@ enum LOCAL_NONE, // unknown LOCAL_IF, // address a local interface listens on LOCAL_BIND, // address explicit bound to - LOCAL_MAPPED, // address reported by UPnP or NAT-PMP + LOCAL_MAPPED, // address reported by UPnP or PCP LOCAL_MANUAL, // address explicitly specified (-externalip=) LOCAL_MAX @@ -1035,7 +1035,7 @@ public: struct Options { - ServiceFlags nLocalServices = NODE_NONE; + ServiceFlags m_local_services = NODE_NONE; int m_max_automatic_connections = 0; CClientUIInterface* uiInterface = nullptr; NetEventsInterface* m_msgproc = nullptr; @@ -1065,7 +1065,7 @@ public: { AssertLockNotHeld(m_total_bytes_sent_mutex); - nLocalServices = connOptions.nLocalServices; + m_local_services = connOptions.m_local_services; m_max_automatic_connections = connOptions.m_max_automatic_connections; m_max_outbound_full_relay = std::min(MAX_OUTBOUND_FULL_RELAY_CONNECTIONS, m_max_automatic_connections); m_max_outbound_block_relay = std::min(MAX_BLOCK_RELAY_ONLY_CONNECTIONS, m_max_automatic_connections - m_max_outbound_full_relay); @@ -1223,8 +1223,8 @@ public: //! Updates the local services that this node advertises to other peers //! during connection handshake. - void AddLocalServices(ServiceFlags services) { nLocalServices = ServiceFlags(nLocalServices | services); }; - void RemoveLocalServices(ServiceFlags services) { nLocalServices = ServiceFlags(nLocalServices & ~services); } + void AddLocalServices(ServiceFlags services) { m_local_services = ServiceFlags(m_local_services | services); }; + void RemoveLocalServices(ServiceFlags services) { m_local_services = ServiceFlags(m_local_services & ~services); } uint64_t GetMaxOutboundTarget() const EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); std::chrono::seconds GetMaxOutboundTimeframe() const; @@ -1470,7 +1470,7 @@ private: * * \sa Peer::our_services */ - std::atomic<ServiceFlags> nLocalServices; + std::atomic<ServiceFlags> m_local_services; std::unique_ptr<CSemaphore> semOutbound; std::unique_ptr<CSemaphore> semAddnode; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index b7d0f5360d..be16884011 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -515,6 +515,7 @@ public: std::optional<std::string> FetchBlock(NodeId peer_id, const CBlockIndex& block_index) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); + std::vector<TxOrphanage::OrphanTxBase> GetOrphanTransactions() override EXCLUSIVE_LOCKS_REQUIRED(!m_tx_download_mutex); PeerManagerInfo GetInfo() const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void SendPings() override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void RelayTransaction(const uint256& txid, const uint256& wtxid) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); @@ -1917,6 +1918,12 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c return true; } +std::vector<TxOrphanage::OrphanTxBase> PeerManagerImpl::GetOrphanTransactions() +{ + LOCK(m_tx_download_mutex); + return m_orphanage.GetOrphanTransactions(); +} + PeerManagerInfo PeerManagerImpl::GetInfo() const { return PeerManagerInfo{ diff --git a/src/net_processing.h b/src/net_processing.h index ccacd15e42..0d2dc59c5a 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -7,6 +7,7 @@ #define BITCOIN_NET_PROCESSING_H #include <net.h> +#include <txorphanage.h> #include <validationinterface.h> #include <chrono> @@ -99,6 +100,8 @@ public: /** Get statistics from node state */ virtual bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const = 0; + virtual std::vector<TxOrphanage::OrphanTxBase> GetOrphanTransactions() = 0; + /** Get peer manager info. */ virtual PeerManagerInfo GetInfo() const = 0; diff --git a/src/netbase.cpp b/src/netbase.cpp index 76a5f9834e..eaca5a16c1 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <netbase.h> diff --git a/src/node/abort.cpp b/src/node/abort.cpp index 8a17c41fd2..c15bf047c8 100644 --- a/src/node/abort.cpp +++ b/src/node/abort.cpp @@ -15,12 +15,12 @@ namespace node { -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message, node::Warnings* warnings) +void AbortNode(const std::function<bool()>& shutdown_request, std::atomic<int>& exit_status, const bilingual_str& message, node::Warnings* warnings) { if (warnings) warnings->Set(Warning::FATAL_INTERNAL_ERROR, message); InitError(_("A fatal internal error occurred, see debug.log for details: ") + message); exit_status.store(EXIT_FAILURE); - if (shutdown && !(*shutdown)()) { + if (shutdown_request && !shutdown_request()) { LogError("Failed to send shutdown signal\n"); }; } diff --git a/src/node/abort.h b/src/node/abort.h index c881af4634..c8514628bc 100644 --- a/src/node/abort.h +++ b/src/node/abort.h @@ -6,16 +6,13 @@ #define BITCOIN_NODE_ABORT_H #include <atomic> +#include <functional> struct bilingual_str; -namespace util { -class SignalInterrupt; -} // namespace util - namespace node { class Warnings; -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message, node::Warnings* warnings); +void AbortNode(const std::function<bool()>& shutdown_request, std::atomic<int>& exit_status, const bilingual_str& message, node::Warnings* warnings); } // namespace node #endif // BITCOIN_NODE_ABORT_H diff --git a/src/node/context.h b/src/node/context.h index 398089ff3c..debc122120 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -59,8 +59,10 @@ struct NodeContext { std::unique_ptr<ECC_Context> ecc_context; //! Init interface for initializing current process and connecting to other processes. interfaces::Init* init{nullptr}; + //! Function to request a shutdown. + std::function<bool()> shutdown_request; //! Interrupt object used to track whether node shutdown was requested. - util::SignalInterrupt* shutdown{nullptr}; + util::SignalInterrupt* shutdown_signal{nullptr}; std::unique_ptr<AddrMan> addrman; std::unique_ptr<CConnman> connman; std::unique_ptr<CTxMemPool> mempool; diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 1fedcd6cfa..0010c104a8 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -8,6 +8,7 @@ #include <chain.h> #include <chainparams.h> #include <common/args.h> +#include <consensus/merkle.h> #include <consensus/validation.h> #include <deploymentstatus.h> #include <external_signer.h> @@ -60,7 +61,7 @@ #include <validation.h> #include <validationinterface.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <any> #include <memory> @@ -134,13 +135,15 @@ public: } void startShutdown() override { - if (!(*Assert(Assert(m_context)->shutdown))()) { + NodeContext& ctx{*Assert(m_context)}; + if (!(Assert(ctx.shutdown_request))()) { LogError("Failed to send shutdown signal\n"); } + // Stop RPC for clean shutdown if any of waitfor* commands is executed. if (args().GetBoolArg("-server", false)) { InterruptRPC(); - StopRPC(m_context); + StopRPC(); } } bool shutdownRequested() override { return ShutdownRequested(*Assert(m_context)); }; @@ -184,7 +187,7 @@ public: }); args().WriteSettingsFile(); } - void mapPort(bool use_upnp, bool use_natpmp) override { StartMapPort(use_upnp, use_natpmp); } + void mapPort(bool use_upnp, bool use_pcp) override { StartMapPort(use_upnp, use_pcp); } bool getProxy(Network net, Proxy& proxy_info) override { return GetProxy(net, proxy_info); } size_t getNodeCount(ConnectionDirection flags) override { @@ -870,7 +873,7 @@ public: class BlockTemplateImpl : public BlockTemplate { public: - explicit BlockTemplateImpl(std::unique_ptr<CBlockTemplate> block_template) : m_block_template(std::move(block_template)) + explicit BlockTemplateImpl(std::unique_ptr<CBlockTemplate> block_template, NodeContext& node) : m_block_template(std::move(block_template)), m_node(node) { assert(m_block_template); } @@ -910,7 +913,37 @@ public: return GetWitnessCommitmentIndex(m_block_template->block); } + std::vector<uint256> getCoinbaseMerklePath() override + { + return BlockMerkleBranch(m_block_template->block); + } + + bool submitSolution(uint32_t version, uint32_t timestamp, uint32_t nonce, CMutableTransaction coinbase) override + { + CBlock block{m_block_template->block}; + + auto cb = MakeTransactionRef(std::move(coinbase)); + + if (block.vtx.size() == 0) { + block.vtx.push_back(cb); + } else { + block.vtx[0] = cb; + } + + block.nVersion = version; + block.nTime = timestamp; + block.nNonce = nonce; + + block.hashMerkleRoot = BlockMerkleRoot(block); + + auto block_ptr = std::make_shared<const CBlock>(block); + return chainman().ProcessNewBlock(block_ptr, /*force_processing=*/true, /*min_pow_checked=*/true, /*new_block=*/nullptr); + } + const std::unique_ptr<CBlockTemplate> m_block_template; + + ChainstateManager& chainman() { return *Assert(m_node.chainman); } + NodeContext& m_node; }; class MinerImpl : public Mining @@ -938,19 +971,12 @@ public: BlockRef waitTipChanged(uint256 current_tip, MillisecondsDouble timeout) override { - // Interrupt check interval - const MillisecondsDouble tick{1000}; - auto now{std::chrono::steady_clock::now()}; - auto deadline = now + timeout; - // std::chrono does not check against overflow - if (deadline < now) deadline = std::chrono::steady_clock::time_point::max(); + if (timeout > std::chrono::years{100}) timeout = std::chrono::years{100}; // Upper bound to avoid UB in std::chrono { WAIT_LOCK(notifications().m_tip_block_mutex, lock); - while ((notifications().m_tip_block == uint256() || notifications().m_tip_block == current_tip) && !chainman().m_interrupt) { - now = std::chrono::steady_clock::now(); - if (now >= deadline) break; - notifications().m_tip_block_cv.wait_until(lock, std::min(deadline, now + tick)); - } + notifications().m_tip_block_cv.wait_for(lock, timeout, [&]() EXCLUSIVE_LOCKS_REQUIRED(notifications().m_tip_block_mutex) { + return (notifications().m_tip_block != current_tip && notifications().m_tip_block != uint256::ZERO) || chainman().m_interrupt; + }); } // Must release m_tip_block_mutex before locking cs_main, to avoid deadlocks. LOCK(::cs_main); @@ -984,7 +1010,7 @@ public: { BlockAssembler::Options assemble_options{options}; ApplyArgsManOptions(*Assert(m_node.args), assemble_options); - return std::make_unique<BlockTemplateImpl>(BlockAssembler{chainman().ActiveChainstate(), context()->mempool.get(), assemble_options}.CreateNewBlock(script_pub_key)); + return std::make_unique<BlockTemplateImpl>(BlockAssembler{chainman().ActiveChainstate(), context()->mempool.get(), assemble_options}.CreateNewBlock(script_pub_key), m_node); } NodeContext* context() override { return &m_node; } diff --git a/src/node/kernel_notifications.cpp b/src/node/kernel_notifications.cpp index 40d45c8c2b..a09803165c 100644 --- a/src/node/kernel_notifications.cpp +++ b/src/node/kernel_notifications.cpp @@ -4,7 +4,7 @@ #include <node/kernel_notifications.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chain.h> #include <common/args.h> @@ -58,7 +58,7 @@ kernel::InterruptResult KernelNotifications::blockTip(SynchronizationState state uiInterface.NotifyBlockTip(state, &index); if (m_stop_at_height && index.nHeight >= m_stop_at_height) { - if (!m_shutdown()) { + if (!m_shutdown_request()) { LogError("Failed to send shutdown signal after reaching stop height\n"); } return kernel::Interrupted{}; @@ -90,12 +90,12 @@ void KernelNotifications::warningUnset(kernel::Warning id) void KernelNotifications::flushError(const bilingual_str& message) { - AbortNode(&m_shutdown, m_exit_status, message, &m_warnings); + AbortNode(m_shutdown_request, m_exit_status, message, &m_warnings); } void KernelNotifications::fatalError(const bilingual_str& message) { - node::AbortNode(m_shutdown_on_fatal_error ? &m_shutdown : nullptr, + node::AbortNode(m_shutdown_on_fatal_error ? m_shutdown_request : nullptr, m_exit_status, message, &m_warnings); } diff --git a/src/node/kernel_notifications.h b/src/node/kernel_notifications.h index 9ff980ea56..296b9c426d 100644 --- a/src/node/kernel_notifications.h +++ b/src/node/kernel_notifications.h @@ -13,6 +13,7 @@ #include <atomic> #include <cstdint> +#include <functional> class ArgsManager; class CBlockIndex; @@ -23,10 +24,6 @@ namespace kernel { enum class Warning; } // namespace kernel -namespace util { -class SignalInterrupt; -} // namespace util - namespace node { class Warnings; @@ -35,8 +32,8 @@ static constexpr int DEFAULT_STOPATHEIGHT{0}; class KernelNotifications : public kernel::Notifications { public: - KernelNotifications(util::SignalInterrupt& shutdown, std::atomic<int>& exit_status, node::Warnings& warnings) - : m_shutdown(shutdown), m_exit_status{exit_status}, m_warnings{warnings} {} + KernelNotifications(const std::function<bool()>& shutdown_request, std::atomic<int>& exit_status, node::Warnings& warnings) + : m_shutdown_request(shutdown_request), m_exit_status{exit_status}, m_warnings{warnings} {} [[nodiscard]] kernel::InterruptResult blockTip(SynchronizationState state, CBlockIndex& index) override EXCLUSIVE_LOCKS_REQUIRED(!m_tip_block_mutex); @@ -58,12 +55,14 @@ public: bool m_shutdown_on_fatal_error{true}; Mutex m_tip_block_mutex; - std::condition_variable m_tip_block_cv; + std::condition_variable m_tip_block_cv GUARDED_BY(m_tip_block_mutex); //! The block for which the last blockTip notification was received for. - uint256 m_tip_block; + //! The initial ZERO means that no block has been connected yet, which may + //! be true even long after startup, until shutdown. + uint256 m_tip_block GUARDED_BY(m_tip_block_mutex){uint256::ZERO}; private: - util::SignalInterrupt& m_shutdown; + const std::function<bool()>& m_shutdown_request; std::atomic<int>& m_exit_status; node::Warnings& m_warnings; }; diff --git a/src/node/warnings.cpp b/src/node/warnings.cpp index 87389e472b..255d8dba6e 100644 --- a/src/node/warnings.cpp +++ b/src/node/warnings.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <node/warnings.h> diff --git a/src/qt/CMakeLists.txt b/src/qt/CMakeLists.txt index 297408a926..7ec2b74cc8 100644 --- a/src/qt/CMakeLists.txt +++ b/src/qt/CMakeLists.txt @@ -133,7 +133,6 @@ target_link_libraries(bitcoinqt bitcoin_cli leveldb Boost::headers - $<TARGET_NAME_IF_EXISTS:NATPMP::NATPMP> $<TARGET_NAME_IF_EXISTS:MiniUPnPc::MiniUPnPc> $<TARGET_NAME_IF_EXISTS:PkgConfig::libqrencode> $<$<PLATFORM_ID:Darwin>:-framework\ AppKit> diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index ec415f0bac..c43cb77866 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/bitcoin.h> diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 1423a8bbc6..52b117eed7 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_BITCOIN_H #define BITCOIN_QT_BITCOIN_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <interfaces/node.h> #include <qt/initexecutor.h> diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 6d66c7473b..6d7e183e98 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/bitcoingui.h> diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 73adbda5a5..32fb7488fb 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_BITCOINGUI_H #define BITCOIN_QT_BITCOINGUI_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/bitcoinunits.h> #include <qt/clientmodel.h> diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 5c70c2695c..fb81dee8da 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/clientmodel.h> diff --git a/src/qt/createwalletdialog.cpp b/src/qt/createwalletdialog.cpp index 3e8d1461e5..2908043d28 100644 --- a/src/qt/createwalletdialog.cpp +++ b/src/qt/createwalletdialog.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <interfaces/node.h> #include <qt/createwalletdialog.h> diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 1ce23c72bb..2056b2cccd 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -328,10 +328,10 @@ <item> <widget class="QCheckBox" name="mapPortNatpmp"> <property name="toolTip"> - <string>Automatically open the Bitcoin client port on the router. This only works when your router supports NAT-PMP and it is enabled. The external port could be random.</string> + <string>Automatically open the Bitcoin client port on the router. This only works when your router supports PCP or NAT-PMP and it is enabled. The external port could be random.</string> </property> <property name="text"> - <string>Map port using NA&T-PMP</string> + <string>Map port using PCP or NA&T-PMP</string> </property> </widget> </item> diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 26b42deb64..bc770b71aa 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chainparams.h> #include <qt/intro.h> diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index 7580f6b47a..522ecfd801 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/modaloverlay.h> #include <qt/forms/ui_modaloverlay.h> diff --git a/src/qt/notificator.cpp b/src/qt/notificator.cpp index 85bdeee49a..af97bb2143 100644 --- a/src/qt/notificator.cpp +++ b/src/qt/notificator.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/notificator.h> diff --git a/src/qt/notificator.h b/src/qt/notificator.h index 8808716aa4..932cfc4651 100644 --- a/src/qt/notificator.h +++ b/src/qt/notificator.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_NOTIFICATOR_H #define BITCOIN_QT_NOTIFICATOR_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <QIcon> #include <QObject> diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 4d590c3eed..b70769ed24 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/optionsdialog.h> #include <qt/forms/ui_optionsdialog.h> @@ -108,9 +108,6 @@ OptionsDialog::OptionsDialog(QWidget* parent, bool enableWallet) #ifndef USE_UPNP ui->mapPortUpnp->setEnabled(false); #endif -#ifndef USE_NATPMP - ui->mapPortNatpmp->setEnabled(false); -#endif ui->proxyIp->setEnabled(false); ui->proxyPort->setEnabled(false); @@ -169,8 +166,15 @@ OptionsDialog::OptionsDialog(QWidget* parent, bool enableWallet) /** check if the locale name consists of 2 parts (language_country) */ if(langStr.contains("_")) { - /** display language strings as "native language - native country (locale name)", e.g. "Deutsch - Deutschland (de)" */ - ui->lang->addItem(locale.nativeLanguageName() + QString(" - ") + locale.nativeCountryName() + QString(" (") + langStr + QString(")"), QVariant(langStr)); + /** display language strings as "native language - native country/territory (locale name)", e.g. "Deutsch - Deutschland (de)" */ + ui->lang->addItem(locale.nativeLanguageName() + QString(" - ") + +#if (QT_VERSION >= QT_VERSION_CHECK(6, 2, 0)) + locale.nativeTerritoryName() + +#else + locale.nativeCountryName() + +#endif + QString(" (") + langStr + QString(")"), QVariant(langStr)); + } else { diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 0c21c6748d..5bca5c5320 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/optionsmodel.h> @@ -320,10 +320,15 @@ static ProxySetting ParseProxyString(const QString& proxy) if (proxy.isEmpty()) { return default_val; } - // contains IP at index 0 and port at index 1 - QStringList ip_port = GUIUtil::SplitSkipEmptyParts(proxy, ":"); - if (ip_port.size() == 2) { - return {true, ip_port.at(0), ip_port.at(1)}; + uint16_t port{0}; + std::string hostname; + if (SplitHostPort(proxy.toStdString(), port, hostname) && port != 0) { + // Valid and port within the valid range + // Check if the hostname contains a colon, indicating an IPv6 address + if (hostname.find(':') != std::string::npos) { + hostname = "[" + hostname + "]"; // Wrap IPv6 address in brackets + } + return {true, QString::fromStdString(hostname), QString::number(port)}; } else { // Invalid: return default return default_val; } @@ -414,11 +419,7 @@ QVariant OptionsModel::getOption(OptionID option, const std::string& suffix) con return false; #endif // USE_UPNP case MapPortNatpmp: -#ifdef USE_NATPMP return SettingToBool(setting(), DEFAULT_NATPMP); -#else - return false; -#endif // USE_NATPMP case MinimizeOnClose: return fMinimizeOnClose; diff --git a/src/qt/qrimagewidget.cpp b/src/qt/qrimagewidget.cpp index f6e712a047..e912dafa60 100644 --- a/src/qt/qrimagewidget.cpp +++ b/src/qt/qrimagewidget.cpp @@ -15,7 +15,7 @@ #include <QMouseEvent> #include <QPainter> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #ifdef USE_QRCODE #include <qrencode.h> diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index b4322ddc0f..a5ee6583e0 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -14,7 +14,7 @@ #include <QDialog> #include <QString> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep ReceiveRequestDialog::ReceiveRequestDialog(QWidget* parent) : QDialog(parent, GUIUtil::dialog_flags), diff --git a/src/qt/res/bitcoin-qt-res.rc b/src/qt/res/bitcoin-qt-res.rc index e590b407b0..d84744f91e 100644 --- a/src/qt/res/bitcoin-qt-res.rc +++ b/src/qt/res/bitcoin-qt-res.rc @@ -1,5 +1,7 @@ IDI_ICON1 ICON DISCARDABLE "icons/bitcoin.ico" IDI_ICON2 ICON DISCARDABLE "icons/bitcoin_testnet.ico" +IDI_ICON3 ICON DISCARDABLE "icons/bitcoin_signet.ico" +IDI_ICON4 ICON DISCARDABLE "icons/bitcoin_testnet.ico" // testnet4 #include <windows.h> // needed for VERSIONINFO #include "../../clientversion.h" // holds the needed client version information diff --git a/src/qt/res/icons/bitcoin_signet.ico b/src/qt/res/icons/bitcoin_signet.ico Binary files differnew file mode 100644 index 0000000000..fb9be5153b --- /dev/null +++ b/src/qt/res/icons/bitcoin_signet.ico diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index ae3f9aa686..018c22a4a8 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/rpcconsole.h> #include <qt/forms/ui_debugwindow.h> @@ -16,7 +16,9 @@ #include <qt/guiutil.h> #include <qt/peertablesortproxy.h> #include <qt/platformstyle.h> +#ifdef ENABLE_WALLET #include <qt/walletmodel.h> +#endif // ENABLE_WALLET #include <rpc/client.h> #include <rpc/server.h> #include <util/strencodings.h> diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index 4747e611d0..894ecb1fdf 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_RPCCONSOLE_H #define BITCOIN_QT_RPCCONSOLE_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/clientmodel.h> #include <qt/guiutil.h> diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 03173ec80e..4019ca5e9d 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/sendcoinsdialog.h> #include <qt/forms/ui_sendcoinsdialog.h> diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 012186ee4d..0b1d3c6c3a 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -11,7 +11,7 @@ #include <qt/walletmodel.h> #include <common/signmessage.h> // For MessageSign(), MessageVerify() -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <key_io.h> #include <wallet/wallet.h> diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index ffd6689910..e194b5fd32 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/splashscreen.h> diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index f7d66f316e..3d5cb4a863 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -222,8 +222,8 @@ void AddressBookTests::addressBookTests() // framework when it tries to look up unimplemented cocoa functions, // and fails to handle returned nulls // (https://bugreports.qt.io/browse/QTBUG-49686). - QWARN("Skipping AddressBookTests on mac build with 'minimal' platform set due to Qt bugs. To run AppTests, invoke " - "with 'QT_QPA_PLATFORM=cocoa test_bitcoin-qt' on mac, or else use a linux or windows build."); + qWarning() << "Skipping AddressBookTests on mac build with 'minimal' platform set due to Qt bugs. To run AppTests, invoke " + "with 'QT_QPA_PLATFORM=cocoa test_bitcoin-qt' on mac, or else use a linux or windows build."; return; } #endif diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 10abcb00eb..73e04b55c8 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -60,8 +60,8 @@ void AppTests::appTests() // framework when it tries to look up unimplemented cocoa functions, // and fails to handle returned nulls // (https://bugreports.qt.io/browse/QTBUG-49686). - QWARN("Skipping AppTests on mac build with 'minimal' platform set due to Qt bugs. To run AppTests, invoke " - "with 'QT_QPA_PLATFORM=cocoa test_bitcoin-qt' on mac, or else use a linux or windows build."); + qWarning() << "Skipping AppTests on mac build with 'minimal' platform set due to Qt bugs. To run AppTests, invoke " + "with 'QT_QPA_PLATFORM=cocoa test_bitcoin-qt' on mac, or else use a linux or windows build."; return; } #endif diff --git a/src/qt/test/optiontests.cpp b/src/qt/test/optiontests.cpp index 0f82f65f3e..4f3eb778c5 100644 --- a/src/qt/test/optiontests.cpp +++ b/src/qt/test/optiontests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <common/args.h> #include <init.h> diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 72e8055425..0797d31a71 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -127,6 +127,11 @@ void RPCNestedTests::rpcNestedTests() RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest( abc , cba )"); QVERIFY(result == "[\"abc\",\"cba\"]"); +// Handle deprecated macro, can be removed once minimum Qt is at least 6.3.0. +#if (QT_VERSION >= QT_VERSION_CHECK(6, 3, 0)) +#undef QVERIFY_EXCEPTION_THROWN +#define QVERIFY_EXCEPTION_THROWN(expression, exceptiontype) QVERIFY_THROWS_EXCEPTION(exceptiontype, expression) +#endif QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo() .\n"), std::runtime_error); //invalid syntax QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo() getblockchaininfo()"), std::runtime_error); //invalid syntax RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo("); //tolerate non closing brackets if we have no arguments diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 958cc7ae88..172c06f4ea 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <interfaces/init.h> #include <interfaces/node.h> @@ -58,7 +58,7 @@ int main(int argc, char* argv[]) gArgs.ForceSetArg("-natpmp", "0"); std::string error; - if (!gArgs.ReadConfigFiles(error, true)) QWARN(error.c_str()); + if (!gArgs.ReadConfigFiles(error, true)) qWarning() << error.c_str(); // Prefer the "minimal" platform for the test instead of the normal default // platform ("xcb", "windows", or "cocoa") so tests can't unintentionally diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 6a573d284c..98dfe12f08 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -475,8 +475,8 @@ void WalletTests::walletTests() // framework when it tries to look up unimplemented cocoa functions, // and fails to handle returned nulls // (https://bugreports.qt.io/browse/QTBUG-49686). - QWARN("Skipping WalletTests on mac build with 'minimal' platform set due to Qt bugs. To run AppTests, invoke " - "with 'QT_QPA_PLATFORM=cocoa test_bitcoin-qt' on mac, or else use a linux or windows build."); + qWarning() << "Skipping WalletTests on mac build with 'minimal' platform set due to Qt bugs. To run AppTests, invoke " + "with 'QT_QPA_PLATFORM=cocoa test_bitcoin-qt' on mac, or else use a linux or windows build."; return; } #endif diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index f261c6409d..29b7f5c401 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <qt/utilitydialog.h> diff --git a/src/qt/winshutdownmonitor.cpp b/src/qt/winshutdownmonitor.cpp index 0b5278c192..9ccd7028da 100644 --- a/src/qt/winshutdownmonitor.cpp +++ b/src/qt/winshutdownmonitor.cpp @@ -12,7 +12,11 @@ // If we don't want a message to be processed by Qt, return true and set result to // the value that the window procedure should return. Otherwise return false. +#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)) +bool WinShutdownMonitor::nativeEventFilter(const QByteArray &eventType, void *pMessage, qintptr *pnResult) +#else bool WinShutdownMonitor::nativeEventFilter(const QByteArray &eventType, void *pMessage, long *pnResult) +#endif { Q_UNUSED(eventType); diff --git a/src/qt/winshutdownmonitor.h b/src/qt/winshutdownmonitor.h index 060d8546e3..a8b626065d 100644 --- a/src/qt/winshutdownmonitor.h +++ b/src/qt/winshutdownmonitor.h @@ -20,7 +20,11 @@ public: WinShutdownMonitor(std::function<void()> shutdown_fn) : m_shutdown_fn{std::move(shutdown_fn)} {} /** Implements QAbstractNativeEventFilter interface for processing Windows messages */ +#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)) + bool nativeEventFilter(const QByteArray &eventType, void *pMessage, qintptr *pnResult) override; +#else bool nativeEventFilter(const QByteArray &eventType, void *pMessage, long *pnResult) override; +#endif /** Register the reason for blocking shutdown on Windows to allow clean client exit */ static void registerShutdownBlockReason(const QString& strReason, const HWND& mainWinId); diff --git a/src/random.cpp b/src/random.cpp index c89ee9f38b..163112585a 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <random.h> diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 4754b597c5..dee48481c5 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <randomenv.h> diff --git a/src/rest.cpp b/src/rest.cpp index 4732922a15..ca26c699b5 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <rest.h> diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index c4fc06b34e..360f24ec55 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -766,14 +766,7 @@ static RPCHelpMan getblock() { uint256 hash(ParseHashV(request.params[0], "blockhash")); - int verbosity = 1; - if (!request.params[1].isNull()) { - if (request.params[1].isBool()) { - verbosity = request.params[1].get_bool() ? 1 : 0; - } else { - verbosity = request.params[1].getInt<int>(); - } - } + int verbosity{ParseVerbosity(request.params[1], /*default_verbosity=*/1)}; const CBlockIndex* pblockindex; const CBlockIndex* tip; diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 0112a261ce..601e4fa7bf 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -254,6 +254,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "keypoolrefill", 0, "newsize" }, { "getrawmempool", 0, "verbose" }, { "getrawmempool", 1, "mempool_sequence" }, + { "getorphantxs", 0, "verbosity" }, + { "getorphantxs", 0, "verbose" }, { "estimatesmartfee", 0, "conf_target" }, { "estimaterawfee", 0, "conf_target" }, { "estimaterawfee", 1, "threshold" }, diff --git a/src/rpc/external_signer.cpp b/src/rpc/external_signer.cpp index 3ad7a940e0..44de5443fa 100644 --- a/src/rpc/external_signer.cpp +++ b/src/rpc/external_signer.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <common/args.h> #include <common/system.h> diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index d61898260b..27a00c5d91 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -8,8 +8,10 @@ #include <node/mempool_persist.h> #include <chainparams.h> +#include <consensus/validation.h> #include <core_io.h> #include <kernel/mempool_entry.h> +#include <net_processing.h> #include <node/mempool_persist_args.h> #include <node/types.h> #include <policy/rbf.h> @@ -24,6 +26,7 @@ #include <util/moneystr.h> #include <util/strencodings.h> #include <util/time.h> +#include <util/vector.h> #include <utility> @@ -812,6 +815,104 @@ static RPCHelpMan savemempool() }; } +static std::vector<RPCResult> OrphanDescription() +{ + return { + RPCResult{RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, + RPCResult{RPCResult::Type::STR_HEX, "wtxid", "The transaction witness hash in hex"}, + RPCResult{RPCResult::Type::NUM, "bytes", "The serialized transaction size in bytes"}, + RPCResult{RPCResult::Type::NUM, "vsize", "The virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted."}, + RPCResult{RPCResult::Type::NUM, "weight", "The transaction weight as defined in BIP 141."}, + RPCResult{RPCResult::Type::NUM_TIME, "expiration", "The orphan expiration time expressed in " + UNIX_EPOCH_TIME}, + RPCResult{RPCResult::Type::ARR, "from", "", + { + RPCResult{RPCResult::Type::NUM, "peer_id", "Peer ID"}, + }}, + }; +} + +static UniValue OrphanToJSON(const TxOrphanage::OrphanTxBase& orphan) +{ + UniValue o(UniValue::VOBJ); + o.pushKV("txid", orphan.tx->GetHash().ToString()); + o.pushKV("wtxid", orphan.tx->GetWitnessHash().ToString()); + o.pushKV("bytes", orphan.tx->GetTotalSize()); + o.pushKV("vsize", GetVirtualTransactionSize(*orphan.tx)); + o.pushKV("weight", GetTransactionWeight(*orphan.tx)); + o.pushKV("expiration", int64_t{TicksSinceEpoch<std::chrono::seconds>(orphan.nTimeExpire)}); + UniValue from(UniValue::VARR); + from.push_back(orphan.fromPeer); // only one fromPeer for now + o.pushKV("from", from); + return o; +} + +static RPCHelpMan getorphantxs() +{ + return RPCHelpMan{"getorphantxs", + "\nShows transactions in the tx orphanage.\n" + "\nEXPERIMENTAL warning: this call may be changed in future releases.\n", + { + {"verbosity|verbose", RPCArg::Type::NUM, RPCArg::Default{0}, "0 for an array of txids (may contain duplicates), 1 for an array of objects with tx details, and 2 for details from (1) and tx hex", + RPCArgOptions{.skip_type_check = true}}, + }, + { + RPCResult{"for verbose = 0", + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, + }}, + RPCResult{"for verbose = 1", + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::OBJ, "", "", OrphanDescription()}, + }}, + RPCResult{"for verbose = 2", + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::OBJ, "", "", + Cat<std::vector<RPCResult>>( + OrphanDescription(), + {{RPCResult::Type::STR_HEX, "hex", "The serialized, hex-encoded transaction data"}} + ) + }, + }}, + }, + RPCExamples{ + HelpExampleCli("getorphantxs", "2") + + HelpExampleRpc("getorphantxs", "2") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + const NodeContext& node = EnsureAnyNodeContext(request.context); + PeerManager& peerman = EnsurePeerman(node); + std::vector<TxOrphanage::OrphanTxBase> orphanage = peerman.GetOrphanTransactions(); + + int verbosity{ParseVerbosity(request.params[0], /*default_verbosity=*/0)}; + + UniValue ret(UniValue::VARR); + + if (verbosity <= 0) { + for (auto const& orphan : orphanage) { + ret.push_back(orphan.tx->GetHash().ToString()); + } + } else if (verbosity == 1) { + for (auto const& orphan : orphanage) { + ret.push_back(OrphanToJSON(orphan)); + } + } else { + // >= 2 + for (auto const& orphan : orphanage) { + UniValue o{OrphanToJSON(orphan)}; + o.pushKV("hex", EncodeHexTx(*orphan.tx)); + ret.push_back(o); + } + } + + return ret; + }, + }; +} + static RPCHelpMan submitpackage() { return RPCHelpMan{"submitpackage", @@ -1027,6 +1128,7 @@ void RegisterMempoolRPCCommands(CRPCTable& t) {"blockchain", &getrawmempool}, {"blockchain", &importmempool}, {"blockchain", &savemempool}, + {"hidden", &getorphantxs}, {"rawtransactions", &submitpackage}, }; for (const auto& c : commands) { diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 6270496b1b..44605cbc89 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chain.h> #include <chainparams.h> diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 1119a3e668..5398685074 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -129,8 +129,8 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::STR, "addrbind", /*optional=*/true, "(ip:port) Bind address of the connection to the peer"}, {RPCResult::Type::STR, "addrlocal", /*optional=*/true, "(ip:port) Local address as reported by the peer"}, {RPCResult::Type::STR, "network", "Network (" + Join(GetNetworkNames(/*append_unroutable=*/true), ", ") + ")"}, - {RPCResult::Type::NUM, "mapped_as", /*optional=*/true, "The AS in the BGP route to the peer used for diversifying\n" - "peer selection (only available if the asmap config flag is set)"}, + {RPCResult::Type::NUM, "mapped_as", /*optional=*/true, "Mapped AS (Autonomous System) number at the end of the BGP route to the peer, used for diversifying\n" + "peer selection (only displayed if the -asmap config option is set)"}, {RPCResult::Type::STR_HEX, "services", "The services offered"}, {RPCResult::Type::ARR, "servicesnames", "the services offered, in human-readable form", { @@ -1102,12 +1102,12 @@ static RPCHelpMan getaddrmaninfo() }; } -UniValue AddrmanEntryToJSON(const AddrInfo& info, CConnman& connman) +UniValue AddrmanEntryToJSON(const AddrInfo& info, const CConnman& connman) { UniValue ret(UniValue::VOBJ); ret.pushKV("address", info.ToStringAddr()); - const auto mapped_as{connman.GetMappedAS(info)}; - if (mapped_as != 0) { + const uint32_t mapped_as{connman.GetMappedAS(info)}; + if (mapped_as) { ret.pushKV("mapped_as", mapped_as); } ret.pushKV("port", info.GetPort()); @@ -1116,14 +1116,14 @@ UniValue AddrmanEntryToJSON(const AddrInfo& info, CConnman& connman) ret.pushKV("network", GetNetworkName(info.GetNetClass())); ret.pushKV("source", info.source.ToStringAddr()); ret.pushKV("source_network", GetNetworkName(info.source.GetNetClass())); - const auto source_mapped_as{connman.GetMappedAS(info.source)}; - if (source_mapped_as != 0) { + const uint32_t source_mapped_as{connman.GetMappedAS(info.source)}; + if (source_mapped_as) { ret.pushKV("source_mapped_as", source_mapped_as); } return ret; } -UniValue AddrmanTableToJSON(const std::vector<std::pair<AddrInfo, AddressPosition>>& tableInfos, CConnman& connman) +UniValue AddrmanTableToJSON(const std::vector<std::pair<AddrInfo, AddressPosition>>& tableInfos, const CConnman& connman) { UniValue table(UniValue::VOBJ); for (const auto& e : tableInfos) { @@ -1150,14 +1150,14 @@ static RPCHelpMan getrawaddrman() {RPCResult::Type::OBJ_DYN, "table", "buckets with addresses in the address manager table ( new, tried )", { {RPCResult::Type::OBJ, "bucket/position", "the location in the address manager table (<bucket>/<position>)", { {RPCResult::Type::STR, "address", "The address of the node"}, - {RPCResult::Type::NUM, "mapped_as", /*optional=*/true, "The ASN mapped to the IP of this peer per our current ASMap"}, + {RPCResult::Type::NUM, "mapped_as", /*optional=*/true, "Mapped AS (Autonomous System) number at the end of the BGP route to the peer, used for diversifying peer selection (only displayed if the -asmap config option is set)"}, {RPCResult::Type::NUM, "port", "The port number of the node"}, {RPCResult::Type::STR, "network", "The network (" + Join(GetNetworkNames(), ", ") + ") of the address"}, {RPCResult::Type::NUM, "services", "The services offered by the node"}, {RPCResult::Type::NUM_TIME, "time", "The " + UNIX_EPOCH_TIME + " when the node was last seen"}, {RPCResult::Type::STR, "source", "The address that relayed the address to us"}, {RPCResult::Type::STR, "source_network", "The network (" + Join(GetNetworkNames(), ", ") + ") of the source address"}, - {RPCResult::Type::NUM, "source_mapped_as", /*optional=*/true, "The ASN mapped to the IP of this peer's source per our current ASMap"} + {RPCResult::Type::NUM, "source_mapped_as", /*optional=*/true, "Mapped AS (Autonomous System) number at the end of the BGP route to the source, used for diversifying peer selection (only displayed if the -asmap config option is set)"} }} }} } diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index af2e1d0eaa..5e36273cf4 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chainparams.h> #include <httpserver.h> diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index bb072370ce..65e6e40b0d 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -338,15 +338,7 @@ static RPCHelpMan getrawtransaction() throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "The genesis block coinbase is not considered an ordinary transaction and cannot be retrieved"); } - // Accept either a bool (true) or a num (>=0) to indicate verbosity. - int verbosity{0}; - if (!request.params[1].isNull()) { - if (request.params[1].isBool()) { - verbosity = request.params[1].get_bool(); - } else { - verbosity = request.params[1].getInt<int>(); - } - } + int verbosity{ParseVerbosity(request.params[1], /*default_verbosity=*/0)}; if (!request.params[2].isNull()) { LOCK(cs_main); diff --git a/src/rpc/register.h b/src/rpc/register.h index 65fd29ff08..17ed6c142c 100644 --- a/src/rpc/register.h +++ b/src/rpc/register.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_RPC_REGISTER_H #define BITCOIN_RPC_REGISTER_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep /** These are in one header file to avoid creating tons of single-function * headers for everything under src/rpc/ */ diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 086f609ac3..01f2dc0c0e 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <rpc/server.h> @@ -169,7 +169,7 @@ static RPCHelpMan stop() { // Event loop will exit after current HTTP requests have been handled, so // this reply will get back to the client. - CHECK_NONFATAL((*CHECK_NONFATAL(EnsureAnyNodeContext(jsonRequest.context).shutdown))()); + CHECK_NONFATAL((CHECK_NONFATAL(EnsureAnyNodeContext(jsonRequest.context).shutdown_request))()); if (jsonRequest.params[0].isNum()) { UninterruptibleSleep(std::chrono::milliseconds{jsonRequest.params[0].getInt<int>()}); } @@ -294,7 +294,7 @@ void InterruptRPC() }); } -void StopRPC(const std::any& context) +void StopRPC() { static std::once_flag g_rpc_stop_flag; // This function could be called twice if the GUI has been started with -server=1. @@ -303,9 +303,6 @@ void StopRPC(const std::any& context) LogDebug(BCLog::RPC, "Stopping RPC\n"); WITH_LOCK(g_deadline_timers_mutex, deadlineTimers.clear()); DeleteAuthCookie(); - node::NodeContext& node = EnsureAnyNodeContext(context); - // The notifications interface doesn't exist between initialization step 4a and 7. - if (node.notifications) node.notifications->m_tip_block_cv.notify_all(); LogDebug(BCLog::RPC, "RPC stopped.\n"); }); } diff --git a/src/rpc/server.h b/src/rpc/server.h index a8896d4247..5a22279a58 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -9,7 +9,6 @@ #include <rpc/request.h> #include <rpc/util.h> -#include <any> #include <functional> #include <map> #include <stdint.h> @@ -173,7 +172,7 @@ extern CRPCTable tableRPC; void StartRPC(); void InterruptRPC(); -void StopRPC(const std::any& context); +void StopRPC(); UniValue JSONRPCExec(const JSONRPCRequest& jreq, bool catch_errors); #endif // BITCOIN_RPC_SERVER_H diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 678bac7a18..d71d7d737b 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <clientversion.h> #include <common/args.h> @@ -81,6 +81,18 @@ void RPCTypeCheckObj(const UniValue& o, } } +int ParseVerbosity(const UniValue& arg, int default_verbosity) +{ + if (!arg.isNull()) { + if (arg.isBool()) { + return arg.get_bool(); // true = 1 + } else { + return arg.getInt<int>(); + } + } + return default_verbosity; +} + CAmount AmountFromValue(const UniValue& value, int decimals) { if (!value.isNum() && !value.isStr()) diff --git a/src/rpc/util.h b/src/rpc/util.h index 23024376e0..b8e6759768 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -101,6 +101,15 @@ std::vector<unsigned char> ParseHexV(const UniValue& v, std::string_view name); std::vector<unsigned char> ParseHexO(const UniValue& o, std::string_view strKey); /** + * Parses verbosity from provided UniValue. + * + * @param[in] arg The verbosity argument as a bool (true) or int (0, 1, 2,...) + * @param[in] default_verbosity The value to return if verbosity argument is null + * @returns An integer describing the verbosity level (e.g. 0, 1, 2, etc.) + */ +int ParseVerbosity(const UniValue& arg, int default_verbosity); + +/** * Validate and return a CAmount from a UniValue number or string. * * @param[in] value UniValue number or string to parse. diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index f6024f1ef3..48ae874fcd 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -144,7 +144,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) BOOST_REQUIRE(filter_index.StartBackgroundSync()); // Allow filter index to catch up with the block index. - IndexWaitSynced(filter_index, *Assert(m_node.shutdown)); + IndexWaitSynced(filter_index, *Assert(m_node.shutdown_signal)); // Check that filter index has all blocks that were in the chain before it started. { diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index 121f00bd25..c2b95dd861 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -28,13 +28,13 @@ BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) { const auto params {CreateChainParams(ArgsManager{}, ChainType::MAIN)}; - KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)}; + KernelNotifications notifications{Assert(m_node.shutdown_request), m_node.exit_status, *Assert(m_node.warnings)}; const BlockManager::Options blockman_opts{ .chainparams = *params, .blocks_dir = m_args.GetBlocksDirPath(), .notifications = notifications, }; - BlockManager blockman{*Assert(m_node.shutdown), blockman_opts}; + BlockManager blockman{*Assert(m_node.shutdown_signal), blockman_opts}; // simulate adding a genesis block normally BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); // simulate what happens during reindex @@ -135,13 +135,13 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_block_data_availability, TestChain100Setup) BOOST_AUTO_TEST_CASE(blockmanager_flush_block_file) { - KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)}; + KernelNotifications notifications{Assert(m_node.shutdown_request), m_node.exit_status, *Assert(m_node.warnings)}; node::BlockManager::Options blockman_opts{ .chainparams = Params(), .blocks_dir = m_args.GetBlocksDirPath(), .notifications = notifications, }; - BlockManager blockman{*Assert(m_node.shutdown), blockman_opts}; + BlockManager blockman{*Assert(m_node.shutdown_signal), blockman_opts}; // Test blocks with no transactions, not even a coinbase CBlock block1; diff --git a/src/test/cluster_linearize_tests.cpp b/src/test/cluster_linearize_tests.cpp index d15e783ea1..265ccdc805 100644 --- a/src/test/cluster_linearize_tests.cpp +++ b/src/test/cluster_linearize_tests.cpp @@ -18,13 +18,24 @@ using namespace cluster_linearize; namespace { +/** Special magic value that indicates to TestDepGraphSerialization that a cluster entry represents + * a hole. */ +constexpr std::pair<FeeFrac, TestBitSet> HOLE{FeeFrac{0, 0x3FFFFF}, {}}; + template<typename SetType> -void TestDepGraphSerialization(const Cluster<SetType>& cluster, const std::string& hexenc) +void TestDepGraphSerialization(const std::vector<std::pair<FeeFrac, SetType>>& cluster, const std::string& hexenc) { - DepGraph depgraph(cluster); - - // Run normal sanity and correspondence checks, which includes a round-trip test. - VerifyDepGraphFromCluster(cluster, depgraph); + // Construct DepGraph from cluster argument. + DepGraph<SetType> depgraph; + SetType holes; + for (ClusterIndex i = 0; i < cluster.size(); ++i) { + depgraph.AddTransaction(cluster[i].first); + if (cluster[i] == HOLE) holes.Set(i); + } + for (ClusterIndex i = 0; i < cluster.size(); ++i) { + depgraph.AddDependencies(cluster[i].second, i); + } + depgraph.RemoveTransactions(holes); // There may be multiple serializations of the same graph, but DepGraphFormatter's serializer // only produces one of those. Verify that hexenc matches that canonical serialization. @@ -133,6 +144,34 @@ BOOST_AUTO_TEST_CASE(depgraph_ser_tests) skip insertion C): D,A,B,E,C */ "00" /* end of graph */ ); + + // Transactions: A(1,2), B(3,1), C(2,1), D(1,3), E(1,1). Deps: C->A, D->A, D->B, E->D. + // In order: [_, D, _, _, A, _, B, _, _, _, E, _, _, C] (_ being holes). Internally serialized + // in order A,B,C,D,E. + TestDepGraphSerialization<TestBitSet>( + {HOLE, {{1, 3}, {4, 6}}, HOLE, HOLE, {{1, 2}, {}}, HOLE, {{3, 1}, {}}, HOLE, HOLE, HOLE, {{1, 1}, {1}}, HOLE, HOLE, {{2, 1}, {4}}}, + "02" /* A size */ + "02" /* A fee */ + "03" /* A insertion position (3 holes): _, _, _, A */ + "01" /* B size */ + "06" /* B fee */ + "06" /* B insertion position (skip B->A dependency, skip 4 inserts, add 1 hole): _, _, _, A, _, B */ + "01" /* C size */ + "04" /* C fee */ + "01" /* C->A dependency (skip C->B dependency) */ + "0b" /* C insertion position (skip 6 inserts, add 5 holes): _, _, _, A, _, B, _, _, _, _, _, C */ + "03" /* D size */ + "02" /* D fee */ + "01" /* D->B dependency (skip D->C dependency) */ + "00" /* D->A dependency (no skips) */ + "0b" /* D insertion position (skip 11 inserts): _, D, _, _, A, _, B, _, _, _, _, _, C */ + "01" /* E size */ + "02" /* E fee */ + "00" /* E->D dependency (no skips) */ + "04" /* E insertion position (skip E->C dependency, E->B and E->A are implied, skip 3 + inserts): _, D, _, _, A, _, B, _, _, _, E, _, _, C */ + "00" /* end of graph */ + ); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 08814c1499..e09aad05e9 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -35,7 +35,7 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) BOOST_REQUIRE(coin_stats_index.StartBackgroundSync()); - IndexWaitSynced(coin_stats_index, *Assert(m_node.shutdown)); + IndexWaitSynced(coin_stats_index, *Assert(m_node.shutdown_signal)); // Check that CoinStatsIndex works for genesis block. const CBlockIndex* genesis_block_index; @@ -86,7 +86,7 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup) CoinStatsIndex index{interfaces::MakeChain(m_node), 1 << 20}; BOOST_REQUIRE(index.Init()); BOOST_REQUIRE(index.StartBackgroundSync()); - IndexWaitSynced(index, *Assert(m_node.shutdown)); + IndexWaitSynced(index, *Assert(m_node.shutdown_signal)); std::shared_ptr<const CBlock> new_block; CBlockIndex* new_block_index = nullptr; { diff --git a/src/test/feefrac_tests.cpp b/src/test/feefrac_tests.cpp index 5af3c3d7ed..41c9c0a633 100644 --- a/src/test/feefrac_tests.cpp +++ b/src/test/feefrac_tests.cpp @@ -15,7 +15,7 @@ BOOST_AUTO_TEST_CASE(feefrac_operators) FeeFrac sum{1500, 400}; FeeFrac diff{500, -200}; FeeFrac empty{0, 0}; - FeeFrac zero_fee{0, 1}; // zero-fee allowed + [[maybe_unused]] FeeFrac zero_fee{0, 1}; // zero-fee allowed BOOST_CHECK(empty == FeeFrac{}); // same as no-args diff --git a/src/test/fuzz/cluster_linearize.cpp b/src/test/fuzz/cluster_linearize.cpp index d91f85d867..5b3770636a 100644 --- a/src/test/fuzz/cluster_linearize.cpp +++ b/src/test/fuzz/cluster_linearize.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <cluster_linearize.h> +#include <random.h> #include <serialize.h> #include <streams.h> #include <test/fuzz/fuzz.h> @@ -36,7 +37,7 @@ class SimpleCandidateFinder public: /** Construct an SimpleCandidateFinder for a given graph. */ SimpleCandidateFinder(const DepGraph<SetType>& depgraph LIFETIMEBOUND) noexcept : - m_depgraph(depgraph), m_todo{SetType::Fill(depgraph.TxCount())} {} + m_depgraph(depgraph), m_todo{depgraph.Positions()} {} /** Remove a set of transactions from the set of to-be-linearized ones. */ void MarkDone(SetType select) noexcept { m_todo -= select; } @@ -106,7 +107,7 @@ class ExhaustiveCandidateFinder public: /** Construct an ExhaustiveCandidateFinder for a given graph. */ ExhaustiveCandidateFinder(const DepGraph<SetType>& depgraph LIFETIMEBOUND) noexcept : - m_depgraph(depgraph), m_todo{SetType::Fill(depgraph.TxCount())} {} + m_depgraph(depgraph), m_todo{depgraph.Positions()} {} /** Remove a set of transactions from the set of to-be-linearized ones. */ void MarkDone(SetType select) noexcept { m_todo -= select; } @@ -152,7 +153,7 @@ std::pair<std::vector<ClusterIndex>, bool> SimpleLinearize(const DepGraph<SetTyp { std::vector<ClusterIndex> linearization; SimpleCandidateFinder finder(depgraph); - SetType todo = SetType::Fill(depgraph.TxCount()); + SetType todo = depgraph.Positions(); bool optimal = true; while (todo.Any()) { auto [candidate, iterations_done] = finder.FindCandidateSet(max_iterations); @@ -169,14 +170,14 @@ std::pair<std::vector<ClusterIndex>, bool> SimpleLinearize(const DepGraph<SetTyp template<typename BS> void MakeConnected(DepGraph<BS>& depgraph) { - auto todo = BS::Fill(depgraph.TxCount()); + auto todo = depgraph.Positions(); auto comp = depgraph.FindConnectedComponent(todo); Assume(depgraph.IsConnected(comp)); todo -= comp; while (todo.Any()) { auto nextcomp = depgraph.FindConnectedComponent(todo); Assume(depgraph.IsConnected(nextcomp)); - depgraph.AddDependency(comp.Last(), nextcomp.First()); + depgraph.AddDependencies(BS::Singleton(comp.Last()), nextcomp.First()); todo -= nextcomp; comp = nextcomp; } @@ -205,7 +206,7 @@ template<typename BS> std::vector<ClusterIndex> ReadLinearization(const DepGraph<BS>& depgraph, SpanReader& reader) { std::vector<ClusterIndex> linearization; - TestBitSet todo = TestBitSet::Fill(depgraph.TxCount()); + TestBitSet todo = depgraph.Positions(); // In every iteration one topologically-valid transaction is appended to linearization. while (todo.Any()) { // Compute the set of transactions with no not-yet-included ancestors. @@ -240,59 +241,157 @@ std::vector<ClusterIndex> ReadLinearization(const DepGraph<BS>& depgraph, SpanRe } // namespace -FUZZ_TARGET(clusterlin_add_dependency) +FUZZ_TARGET(clusterlin_depgraph_sim) { - // Verify that computing a DepGraph from a cluster, or building it step by step using AddDependency - // have the same effect. + // Simulation test to verify the full behavior of DepGraph. - // Construct a cluster of a certain length, with no dependencies. FuzzedDataProvider provider(buffer.data(), buffer.size()); - auto num_tx = provider.ConsumeIntegralInRange<ClusterIndex>(2, 32); - Cluster<TestBitSet> cluster(num_tx, std::pair{FeeFrac{0, 1}, TestBitSet{}}); - // Construct the corresponding DepGraph object (also no dependencies). - DepGraph depgraph(cluster); - SanityCheck(depgraph); - // Read (parent, child) pairs, and add them to the cluster and depgraph. - LIMITED_WHILE(provider.remaining_bytes() > 0, TestBitSet::Size() * TestBitSet::Size()) { - auto parent = provider.ConsumeIntegralInRange<ClusterIndex>(0, num_tx - 1); - auto child = provider.ConsumeIntegralInRange<ClusterIndex>(0, num_tx - 2); - child += (child >= parent); - cluster[child].second.Set(parent); - depgraph.AddDependency(parent, child); - assert(depgraph.Ancestors(child)[parent]); - assert(depgraph.Descendants(parent)[child]); - } - // Sanity check the result. - SanityCheck(depgraph); - // Verify that the resulting DepGraph matches one recomputed from the cluster. - assert(DepGraph(cluster) == depgraph); -} -FUZZ_TARGET(clusterlin_cluster_serialization) -{ - // Verify that any graph of transactions has its ancestry correctly computed by DepGraph, and - // if it is a DAG, that it can be serialized as a DepGraph in a way that roundtrips. This - // guarantees that any acyclic cluster has a corresponding DepGraph serialization. + /** Real DepGraph being tested. */ + DepGraph<TestBitSet> real; + /** Simulated DepGraph (sim[i] is std::nullopt if position i does not exist; otherwise, + * sim[i]->first is its individual feerate, and sim[i]->second is its set of ancestors. */ + std::array<std::optional<std::pair<FeeFrac, TestBitSet>>, TestBitSet::Size()> sim; + /** The number of non-nullopt position in sim. */ + ClusterIndex num_tx_sim{0}; + + /** Read a valid index of a transaction from the provider. */ + auto idx_fn = [&]() { + auto offset = provider.ConsumeIntegralInRange<ClusterIndex>(0, num_tx_sim - 1); + for (ClusterIndex i = 0; i < sim.size(); ++i) { + if (!sim[i].has_value()) continue; + if (offset == 0) return i; + --offset; + } + assert(false); + return ClusterIndex(-1); + }; - FuzzedDataProvider provider(buffer.data(), buffer.size()); + /** Read a valid subset of the transactions from the provider. */ + auto subset_fn = [&]() { + auto range = (uint64_t{1} << num_tx_sim) - 1; + const auto mask = provider.ConsumeIntegralInRange<uint64_t>(0, range); + auto mask_shifted = mask; + TestBitSet subset; + for (ClusterIndex i = 0; i < sim.size(); ++i) { + if (!sim[i].has_value()) continue; + if (mask_shifted & 1) { + subset.Set(i); + } + mask_shifted >>= 1; + } + assert(mask_shifted == 0); + return subset; + }; - // Construct a cluster in a naive way (using a FuzzedDataProvider-based serialization). - Cluster<TestBitSet> cluster; - auto num_tx = provider.ConsumeIntegralInRange<ClusterIndex>(1, 32); - cluster.resize(num_tx); - for (ClusterIndex i = 0; i < num_tx; ++i) { - cluster[i].first.size = provider.ConsumeIntegralInRange<int32_t>(1, 0x3fffff); - cluster[i].first.fee = provider.ConsumeIntegralInRange<int64_t>(-0x8000000000000, 0x7ffffffffffff); - for (ClusterIndex j = 0; j < num_tx; ++j) { - if (i == j) continue; - if (provider.ConsumeBool()) cluster[i].second.Set(j); + /** Read any set of transactions from the provider (including unused positions). */ + auto set_fn = [&]() { + auto range = (uint64_t{1} << sim.size()) - 1; + const auto mask = provider.ConsumeIntegralInRange<uint64_t>(0, range); + TestBitSet set; + for (ClusterIndex i = 0; i < sim.size(); ++i) { + if ((mask >> i) & 1) { + set.Set(i); + } } + return set; + }; + + /** Propagate ancestor information in sim. */ + auto anc_update_fn = [&]() { + while (true) { + bool updates{false}; + for (ClusterIndex chl = 0; chl < sim.size(); ++chl) { + if (!sim[chl].has_value()) continue; + for (auto par : sim[chl]->second) { + if (!sim[chl]->second.IsSupersetOf(sim[par]->second)) { + sim[chl]->second |= sim[par]->second; + updates = true; + } + } + } + if (!updates) break; + } + }; + + /** Compare the state of transaction i in the simulation with the real one. */ + auto check_fn = [&](ClusterIndex i) { + // Compare used positions. + assert(real.Positions()[i] == sim[i].has_value()); + if (sim[i].has_value()) { + // Compare feerate. + assert(real.FeeRate(i) == sim[i]->first); + // Compare ancestors (note that SanityCheck verifies correspondence between ancestors + // and descendants, so we can restrict ourselves to ancestors here). + assert(real.Ancestors(i) == sim[i]->second); + } + }; + + LIMITED_WHILE(provider.remaining_bytes() > 0, 1000) { + uint8_t command = provider.ConsumeIntegral<uint8_t>(); + if (num_tx_sim == 0 || ((command % 3) <= 0 && num_tx_sim < TestBitSet::Size())) { + // AddTransaction. + auto fee = provider.ConsumeIntegralInRange<int64_t>(-0x8000000000000, 0x7ffffffffffff); + auto size = provider.ConsumeIntegralInRange<int32_t>(1, 0x3fffff); + FeeFrac feerate{fee, size}; + // Apply to DepGraph. + auto idx = real.AddTransaction(feerate); + // Verify that the returned index is correct. + assert(!sim[idx].has_value()); + for (ClusterIndex i = 0; i < TestBitSet::Size(); ++i) { + if (!sim[i].has_value()) { + assert(idx == i); + break; + } + } + // Update sim. + sim[idx] = {feerate, TestBitSet::Singleton(idx)}; + ++num_tx_sim; + continue; + } + if ((command % 3) <= 1 && num_tx_sim > 0) { + // AddDependencies. + ClusterIndex child = idx_fn(); + auto parents = subset_fn(); + // Apply to DepGraph. + real.AddDependencies(parents, child); + // Apply to sim. + sim[child]->second |= parents; + continue; + } + if (num_tx_sim > 0) { + // Remove transactions. + auto del = set_fn(); + // Propagate all ancestry information before deleting anything in the simulation (as + // intermediary transactions may be deleted which impact connectivity). + anc_update_fn(); + // Compare the state of the transactions being deleted. + for (auto i : del) check_fn(i); + // Apply to DepGraph. + real.RemoveTransactions(del); + // Apply to sim. + for (ClusterIndex i = 0; i < sim.size(); ++i) { + if (sim[i].has_value()) { + if (del[i]) { + --num_tx_sim; + sim[i] = std::nullopt; + } else { + sim[i]->second -= del; + } + } + } + continue; + } + // This should be unreachable (one of the 3 above actions should always be possible). + assert(false); } - // Construct dependency graph, and verify it matches the cluster (which includes a round-trip - // check for the serialization). - DepGraph depgraph(cluster); - VerifyDepGraphFromCluster(cluster, depgraph); + // Compare the real obtained depgraph against the simulation. + anc_update_fn(); + for (ClusterIndex i = 0; i < sim.size(); ++i) check_fn(i); + assert(real.TxCount() == num_tx_sim); + // Sanity check the result (which includes round-tripping serialization, if applicable). + SanityCheck(real); } FUZZ_TARGET(clusterlin_depgraph_serialization) @@ -322,7 +421,7 @@ FUZZ_TARGET(clusterlin_components) reader >> Using<DepGraphFormatter>(depgraph); } catch (const std::ios_base::failure&) {} - TestBitSet todo = TestBitSet::Fill(depgraph.TxCount()); + TestBitSet todo = depgraph.Positions(); while (todo.Any()) { // Find a connected component inside todo. auto component = depgraph.FindConnectedComponent(todo); @@ -333,7 +432,7 @@ FUZZ_TARGET(clusterlin_components) // If todo is the entire graph, and the entire graph is connected, then the component must // be the entire graph. - if (todo == TestBitSet::Fill(depgraph.TxCount())) { + if (todo == depgraph.Positions()) { assert((component == todo) == depgraph.IsConnected()); } @@ -370,7 +469,7 @@ FUZZ_TARGET(clusterlin_components) reader >> VARINT(subset_bits); } catch (const std::ios_base::failure&) {} TestBitSet subset; - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + for (ClusterIndex i : depgraph.Positions()) { if (todo[i]) { if (subset_bits & 1) subset.Set(i); subset_bits >>= 1; @@ -423,7 +522,7 @@ FUZZ_TARGET(clusterlin_chunking) } // Naively recompute the chunks (each is the highest-feerate prefix of what remains). - auto todo = TestBitSet::Fill(depgraph.TxCount()); + auto todo = depgraph.Positions(); for (const auto& chunk_feerate : chunking) { assert(todo.Any()); SetInfo<TestBitSet> accumulator, best; @@ -454,7 +553,7 @@ FUZZ_TARGET(clusterlin_ancestor_finder) } catch (const std::ios_base::failure&) {} AncestorCandidateFinder anc_finder(depgraph); - auto todo = TestBitSet::Fill(depgraph.TxCount()); + auto todo = depgraph.Positions(); while (todo.Any()) { // Call the ancestor finder's FindCandidateSet for what remains of the graph. assert(!anc_finder.AllDone()); @@ -519,7 +618,7 @@ FUZZ_TARGET(clusterlin_search_finder) ExhaustiveCandidateFinder exh_finder(depgraph); AncestorCandidateFinder anc_finder(depgraph); - auto todo = TestBitSet::Fill(depgraph.TxCount()); + auto todo = depgraph.Positions(); while (todo.Any()) { assert(!src_finder.AllDone()); assert(!smp_finder.AllDone()); @@ -623,7 +722,7 @@ FUZZ_TARGET(clusterlin_linearization_chunking) } catch (const std::ios_base::failure&) {} // Retrieve a topologically-valid subset of depgraph. - auto todo = TestBitSet::Fill(depgraph.TxCount()); + auto todo = depgraph.Positions(); auto subset = SetInfo(depgraph, ReadTopologicalSet(depgraph, todo, reader)); // Retrieve a valid linearization for depgraph. @@ -806,8 +905,8 @@ FUZZ_TARGET(clusterlin_linearize) // Only for very small clusters, test every topologically-valid permutation. if (depgraph.TxCount() <= 7) { - std::vector<ClusterIndex> perm_linearization(depgraph.TxCount()); - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) perm_linearization[i] = i; + std::vector<ClusterIndex> perm_linearization; + for (ClusterIndex i : depgraph.Positions()) perm_linearization.push_back(i); // Iterate over all valid permutations. do { // Determine whether perm_linearization is topological. @@ -891,30 +990,30 @@ FUZZ_TARGET(clusterlin_postlinearize_tree) // Now construct a new graph, copying the nodes, but leaving only the first parent (even // direction) or the first child (odd direction). DepGraph<TestBitSet> depgraph_tree; - for (ClusterIndex i = 0; i < depgraph_gen.TxCount(); ++i) { - depgraph_tree.AddTransaction(depgraph_gen.FeeRate(i)); + for (ClusterIndex i = 0; i < depgraph_gen.PositionRange(); ++i) { + if (depgraph_gen.Positions()[i]) { + depgraph_tree.AddTransaction(depgraph_gen.FeeRate(i)); + } else { + // For holes, add a dummy transaction which is deleted below, so that non-hole + // transactions retain their position. + depgraph_tree.AddTransaction(FeeFrac{}); + } } + depgraph_tree.RemoveTransactions(TestBitSet::Fill(depgraph_gen.PositionRange()) - depgraph_gen.Positions()); + if (direction & 1) { for (ClusterIndex i = 0; i < depgraph_gen.TxCount(); ++i) { - auto children = depgraph_gen.Descendants(i) - TestBitSet::Singleton(i); - // Remove descendants that are children of other descendants. - for (auto j : children) { - if (!children[j]) continue; - children -= depgraph_gen.Descendants(j); - children.Set(j); + auto children = depgraph_gen.GetReducedChildren(i); + if (children.Any()) { + depgraph_tree.AddDependencies(TestBitSet::Singleton(i), children.First()); } - if (children.Any()) depgraph_tree.AddDependency(i, children.First()); } } else { for (ClusterIndex i = 0; i < depgraph_gen.TxCount(); ++i) { - auto parents = depgraph_gen.Ancestors(i) - TestBitSet::Singleton(i); - // Remove ancestors that are parents of other ancestors. - for (auto j : parents) { - if (!parents[j]) continue; - parents -= depgraph_gen.Ancestors(j); - parents.Set(j); + auto parents = depgraph_gen.GetReducedParents(i); + if (parents.Any()) { + depgraph_tree.AddDependencies(TestBitSet::Singleton(parents.First()), i); } - if (parents.Any()) depgraph_tree.AddDependency(parents.First(), i); } } diff --git a/src/test/fuzz/crypto_common.cpp b/src/test/fuzz/crypto_common.cpp index 8e07dfedb9..5a76d4e1a9 100644 --- a/src/test/fuzz/crypto_common.cpp +++ b/src/test/fuzz/crypto_common.cpp @@ -35,6 +35,10 @@ FUZZ_TARGET(crypto_common) WriteLE64(writele64_arr.data(), random_u64); assert(ReadLE64(writele64_arr.data()) == random_u64); + std::array<uint8_t, 2> writebe16_arr; + WriteBE16(writebe16_arr.data(), random_u16); + assert(ReadBE16(writebe16_arr.data()) == random_u16); + std::array<uint8_t, 4> writebe32_arr; WriteBE32(writebe32_arr.data(), random_u32); assert(ReadBE32(writebe32_arr.data()) == random_u32); diff --git a/src/test/fuzz/p2p_headers_presync.cpp b/src/test/fuzz/p2p_headers_presync.cpp index e22087303a..2670aa8ee4 100644 --- a/src/test/fuzz/p2p_headers_presync.cpp +++ b/src/test/fuzz/p2p_headers_presync.cpp @@ -197,15 +197,15 @@ FUZZ_TARGET(p2p_headers_presync, .init = initialize) auto headers_msg = NetMsg::Make(NetMsgType::BLOCK, TX_WITH_WITNESS(block)); g_testing_setup->SendMessage(fuzzed_data_provider, std::move(headers_msg)); }); + } - // This is a conservative overestimate, as base is only moved forward when sending headers. In theory, - // the longest chain generated by this test is 1600 (FUZZ_MAX_HEADERS_RESULTS * 100) headers. In that case, - // this variable will accurately reflect the chain's total work. - total_work += CalculateClaimedHeadersWork(all_headers); + // This is a conservative overestimate, as base is only moved forward when sending headers. In theory, + // the longest chain generated by this test is 1600 (FUZZ_MAX_HEADERS_RESULTS * 100) headers. In that case, + // this variable will accurately reflect the chain's total work. + total_work += CalculateClaimedHeadersWork(all_headers); - // This test should never create a chain with more work than MinimumChainWork. - assert(total_work < chainman.MinimumChainWork()); - } + // This test should never create a chain with more work than MinimumChainWork. + assert(total_work < chainman.MinimumChainWork()); // The headers/blocks sent in this test should never be stored, as the chains don't have the work required // to meet the anti-DoS work threshold. So, if at any point the block index grew in size, then there's a bug diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 9122617e46..4db37ab7b7 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -143,6 +143,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "getnetworkhashps", "getnetworkinfo", "getnodeaddresses", + "getorphantxs", "getpeerinfo", "getprioritisedtransactions", "getrawaddrman", diff --git a/src/test/ipc_test.cpp b/src/test/ipc_test.cpp index 91eba9214f..af37434980 100644 --- a/src/test/ipc_test.cpp +++ b/src/test/ipc_test.cpp @@ -62,7 +62,7 @@ void IpcPipeTest() auto connection_client = std::make_unique<mp::Connection>(loop, kj::mv(pipe.ends[0])); auto foo_client = std::make_unique<mp::ProxyClient<gen::FooInterface>>( - connection_client->m_rpc_system.bootstrap(mp::ServerVatId().vat_id).castAs<gen::FooInterface>(), + connection_client->m_rpc_system->bootstrap(mp::ServerVatId().vat_id).castAs<gen::FooInterface>(), connection_client.get(), /* destroy_connection= */ false); foo_promise.set_value(std::move(foo_client)); disconnect_client = [&] { loop.sync([&] { connection_client.reset(); }); }; diff --git a/src/test/logging_tests.cpp b/src/test/logging_tests.cpp index 8217a2385c..77ec81e597 100644 --- a/src/test/logging_tests.cpp +++ b/src/test/logging_tests.cpp @@ -86,12 +86,12 @@ BOOST_AUTO_TEST_CASE(logging_timer) BOOST_FIXTURE_TEST_CASE(logging_LogPrintStr, LogSetup) { LogInstance().m_log_sourcelocations = true; - LogInstance().LogPrintStr("foo1: bar1\n", "fn1", "src1", 1, BCLog::LogFlags::NET, BCLog::Level::Debug); - LogInstance().LogPrintStr("foo2: bar2\n", "fn2", "src2", 2, BCLog::LogFlags::NET, BCLog::Level::Info); - LogInstance().LogPrintStr("foo3: bar3\n", "fn3", "src3", 3, BCLog::LogFlags::ALL, BCLog::Level::Debug); - LogInstance().LogPrintStr("foo4: bar4\n", "fn4", "src4", 4, BCLog::LogFlags::ALL, BCLog::Level::Info); - LogInstance().LogPrintStr("foo5: bar5\n", "fn5", "src5", 5, BCLog::LogFlags::NONE, BCLog::Level::Debug); - LogInstance().LogPrintStr("foo6: bar6\n", "fn6", "src6", 6, BCLog::LogFlags::NONE, BCLog::Level::Info); + LogInstance().LogPrintStr("foo1: bar1", "fn1", "src1", 1, BCLog::LogFlags::NET, BCLog::Level::Debug); + LogInstance().LogPrintStr("foo2: bar2", "fn2", "src2", 2, BCLog::LogFlags::NET, BCLog::Level::Info); + LogInstance().LogPrintStr("foo3: bar3", "fn3", "src3", 3, BCLog::LogFlags::ALL, BCLog::Level::Debug); + LogInstance().LogPrintStr("foo4: bar4", "fn4", "src4", 4, BCLog::LogFlags::ALL, BCLog::Level::Info); + LogInstance().LogPrintStr("foo5: bar5", "fn5", "src5", 5, BCLog::LogFlags::NONE, BCLog::Level::Debug); + LogInstance().LogPrintStr("foo6: bar6", "fn6", "src6", 6, BCLog::LogFlags::NONE, BCLog::Level::Info); std::ifstream file{tmp_log_path}; std::vector<std::string> log_lines; for (std::string log; std::getline(file, log);) { @@ -133,11 +133,11 @@ BOOST_FIXTURE_TEST_CASE(logging_LogPrintMacrosDeprecated, LogSetup) BOOST_FIXTURE_TEST_CASE(logging_LogPrintMacros, LogSetup) { - LogTrace(BCLog::NET, "foo6: %s\n", "bar6"); // not logged - LogDebug(BCLog::NET, "foo7: %s\n", "bar7"); - LogInfo("foo8: %s\n", "bar8"); - LogWarning("foo9: %s\n", "bar9"); - LogError("foo10: %s\n", "bar10"); + LogTrace(BCLog::NET, "foo6: %s", "bar6"); // not logged + LogDebug(BCLog::NET, "foo7: %s", "bar7"); + LogInfo("foo8: %s", "bar8"); + LogWarning("foo9: %s", "bar9"); + LogError("foo10: %s", "bar10"); std::ifstream file{tmp_log_path}; std::vector<std::string> log_lines; for (std::string log; std::getline(file, log);) { diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp index 70308cb29a..2b1cf8595d 100644 --- a/src/test/merkle_tests.cpp +++ b/src/test/merkle_tests.cpp @@ -23,110 +23,6 @@ static uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vecto return hash; } -/* This implements a constant-space merkle root/path calculator, limited to 2^32 leaves. */ -static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot, bool* pmutated, uint32_t branchpos, std::vector<uint256>* pbranch) { - if (pbranch) pbranch->clear(); - if (leaves.size() == 0) { - if (pmutated) *pmutated = false; - if (proot) *proot = uint256(); - return; - } - bool mutated = false; - // count is the number of leaves processed so far. - uint32_t count = 0; - // inner is an array of eagerly computed subtree hashes, indexed by tree - // level (0 being the leaves). - // For example, when count is 25 (11001 in binary), inner[4] is the hash of - // the first 16 leaves, inner[3] of the next 8 leaves, and inner[0] equal to - // the last leaf. The other inner entries are undefined. - uint256 inner[32]; - // Which position in inner is a hash that depends on the matching leaf. - int matchlevel = -1; - // First process all leaves into 'inner' values. - while (count < leaves.size()) { - uint256 h = leaves[count]; - bool matchh = count == branchpos; - count++; - int level; - // For each of the lower bits in count that are 0, do 1 step. Each - // corresponds to an inner value that existed before processing the - // current leaf, and each needs a hash to combine it. - for (level = 0; !(count & ((uint32_t{1}) << level)); level++) { - if (pbranch) { - if (matchh) { - pbranch->push_back(inner[level]); - } else if (matchlevel == level) { - pbranch->push_back(h); - matchh = true; - } - } - mutated |= (inner[level] == h); - h = Hash(inner[level], h); - } - // Store the resulting hash at inner position level. - inner[level] = h; - if (matchh) { - matchlevel = level; - } - } - // Do a final 'sweep' over the rightmost branch of the tree to process - // odd levels, and reduce everything to a single top value. - // Level is the level (counted from the bottom) up to which we've sweeped. - int level = 0; - // As long as bit number level in count is zero, skip it. It means there - // is nothing left at this level. - while (!(count & ((uint32_t{1}) << level))) { - level++; - } - uint256 h = inner[level]; - bool matchh = matchlevel == level; - while (count != ((uint32_t{1}) << level)) { - // If we reach this point, h is an inner value that is not the top. - // We combine it with itself (Bitcoin's special rule for odd levels in - // the tree) to produce a higher level one. - if (pbranch && matchh) { - pbranch->push_back(h); - } - h = Hash(h, h); - // Increment count to the value it would have if two entries at this - // level had existed. - count += ((uint32_t{1}) << level); - level++; - // And propagate the result upwards accordingly. - while (!(count & ((uint32_t{1}) << level))) { - if (pbranch) { - if (matchh) { - pbranch->push_back(inner[level]); - } else if (matchlevel == level) { - pbranch->push_back(h); - matchh = true; - } - } - h = Hash(inner[level], h); - level++; - } - } - // Return result. - if (pmutated) *pmutated = mutated; - if (proot) *proot = h; -} - -static std::vector<uint256> ComputeMerkleBranch(const std::vector<uint256>& leaves, uint32_t position) { - std::vector<uint256> ret; - MerkleComputation(leaves, nullptr, nullptr, position, &ret); - return ret; -} - -static std::vector<uint256> BlockMerkleBranch(const CBlock& block, uint32_t position) -{ - std::vector<uint256> leaves; - leaves.resize(block.vtx.size()); - for (size_t s = 0; s < block.vtx.size(); s++) { - leaves[s] = block.vtx[s]->GetHash(); - } - return ComputeMerkleBranch(leaves, position); -} - // Older version of the merkle root computation code, for comparison. static uint256 BlockBuildMerkleTree(const CBlock& block, bool* fMutated, std::vector<uint256>& vMerkleTree) { diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 8c3806d22f..59eb90bd27 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -123,7 +123,6 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript ScriptError err; const CTransaction txCredit{BuildCreditingTransaction(scriptPubKey, nValue)}; CMutableTransaction tx = BuildSpendingTransaction(scriptSig, scriptWitness, txCredit); - CMutableTransaction tx2 = tx; BOOST_CHECK_MESSAGE(VerifyScript(scriptSig, scriptPubKey, &scriptWitness, flags, MutableTransactionSignatureChecker(&tx, 0, txCredit.vout[0].nValue, MissingDataBehavior::ASSERT_FAIL), &err) == expect, message); BOOST_CHECK_MESSAGE(err == scriptError, FormatScriptError(err) + " where " + FormatScriptError((ScriptError_t)scriptError) + " expected: " + message); diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index 7a9a14d5ac..a5d9be07d5 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. // -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <test/util/setup_common.h> #include <common/run_command.h> #include <univalue.h> diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 5a32b02ad9..9ee5387830 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -33,7 +33,7 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) BOOST_REQUIRE(txindex.StartBackgroundSync()); // Allow tx index to catch up with the block index. - IndexWaitSynced(txindex, *Assert(m_node.shutdown)); + IndexWaitSynced(txindex, *Assert(m_node.shutdown_signal)); // Check that txindex excludes genesis block transactions. const CBlock& genesis_block = Params().GenesisBlock(); diff --git a/src/test/util/cluster_linearize.h b/src/test/util/cluster_linearize.h index b86ebcd78b..871aa9d74e 100644 --- a/src/test/util/cluster_linearize.h +++ b/src/test/util/cluster_linearize.h @@ -27,7 +27,7 @@ using TestBitSet = BitSet<32>; template<typename SetType> bool IsAcyclic(const DepGraph<SetType>& depgraph) noexcept { - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + for (ClusterIndex i : depgraph.Positions()) { if ((depgraph.Ancestors(i) & depgraph.Descendants(i)) != SetType::Singleton(i)) { return false; } @@ -57,11 +57,14 @@ bool IsAcyclic(const DepGraph<SetType>& depgraph) noexcept * by parent relations that were serialized before it). * - The various insertion positions in the cluster, from the very end of the cluster, to the * front. + * - The appending of 1, 2, 3, ... holes at the end of the cluster, followed by appending the new + * transaction. * - * Let's say you have a 7-transaction cluster, consisting of transactions F,A,C,B,G,E,D, but - * serialized in order A,B,C,D,E,F,G, because that happens to be a topological ordering. By the - * time G gets serialized, what has been serialized already represents the cluster F,A,C,B,E,D (in - * that order). G has B and E as direct parents, and E depends on C. + * Let's say you have a 7-transaction cluster, consisting of transactions F,A,C,B,_,G,E,_,D + * (where _ represent holes; unused positions within the DepGraph) but serialized in order + * A,B,C,D,E,F,G, because that happens to be a topological ordering. By the time G gets serialized, + * what has been serialized already represents the cluster F,A,C,B,_,E,_,D (in that order). G has B + * and E as direct parents, and E depends on C. * * In this case, the possibilities are, in order: * - [ ] the dependency G->F @@ -71,17 +74,23 @@ bool IsAcyclic(const DepGraph<SetType>& depgraph) noexcept * - [ ] the dependency G->A * - [ ] put G at the end of the cluster * - [ ] put G before D + * - [ ] put G before the hole before D * - [X] put G before E + * - [ ] put G before the hole before E * - [ ] put G before B * - [ ] put G before C * - [ ] put G before A * - [ ] put G before F + * - [ ] add 1 hole at the end of the cluster, followed by G + * - [ ] add 2 holes at the end of the cluster, followed by G + * - [ ] add ... * - * The skip values in this case are 1 (G->F), 1 (G->D), 3 (G->A, G at end, G before D). No skip - * after 3 is needed (or permitted), because there can only be one position for G. Also note that - * G->C is not included in the list of possibilities, as it is implied by the included G->E and - * E->C that came before it. On deserialization, if the last skip value was 8 or larger (putting - * G before the beginning of the cluster), it is interpreted as wrapping around back to the end. + * The skip values in this case are 1 (G->F), 1 (G->D), 4 (G->A, G at end, G before D, G before + * hole). No skip after 4 is needed (or permitted), because there can only be one position for G. + * Also note that G->C is not included in the list of possibilities, as it is implied by the + * included G->E and E->C that came before it. On deserialization, if the last skip value was 8 or + * larger (putting G before the beginning of the cluster), it is interpreted as wrapping around + * back to the end. * * * Rationale: @@ -125,18 +134,18 @@ struct DepGraphFormatter static void Ser(Stream& s, const DepGraph<SetType>& depgraph) { /** Construct a topological order to serialize the transactions in. */ - std::vector<ClusterIndex> topo_order(depgraph.TxCount()); - std::iota(topo_order.begin(), topo_order.end(), ClusterIndex{0}); + std::vector<ClusterIndex> topo_order; + topo_order.reserve(depgraph.TxCount()); + for (auto i : depgraph.Positions()) topo_order.push_back(i); std::sort(topo_order.begin(), topo_order.end(), [&](ClusterIndex a, ClusterIndex b) { auto anc_a = depgraph.Ancestors(a).Count(), anc_b = depgraph.Ancestors(b).Count(); if (anc_a != anc_b) return anc_a < anc_b; return a < b; }); - /** Which transactions the deserializer already knows when it has deserialized what has - * been serialized here so far, and in what order. */ - std::vector<ClusterIndex> rebuilt_order; - rebuilt_order.reserve(depgraph.TxCount()); + /** Which positions (incl. holes) the deserializer already knows when it has deserialized + * what has been serialized here so far. */ + SetType done; // Loop over the transactions in topological order. for (ClusterIndex topo_idx = 0; topo_idx < topo_order.size(); ++topo_idx) { @@ -166,14 +175,20 @@ struct DepGraphFormatter } } // Write position information. - ClusterIndex insert_distance = 0; - while (insert_distance < rebuilt_order.size()) { - // Loop to find how far from the end in rebuilt_order to insert. - if (idx > *(rebuilt_order.end() - 1 - insert_distance)) break; - ++insert_distance; + auto add_holes = SetType::Fill(idx) - done - depgraph.Positions(); + if (add_holes.None()) { + // The new transaction is to be inserted N positions back from the end of the + // cluster. Emit N to indicate that that many insertion choices are skipped. + auto skips = (done - SetType::Fill(idx)).Count(); + s << VARINT(diff + skips); + } else { + // The new transaction is to be appended at the end of the cluster, after N holes. + // Emit current_cluster_size + N, to indicate all insertion choices are skipped, + // plus N possibilities for the number of holes. + s << VARINT(diff + done.Count() + add_holes.Count()); + done |= add_holes; } - rebuilt_order.insert(rebuilt_order.end() - insert_distance, idx); - s << VARINT(diff + insert_distance); + done.Set(idx); } // Output a final 0 to denote the end of the graph. @@ -189,10 +204,16 @@ struct DepGraphFormatter /** Mapping from serialization order to cluster order, used later to reconstruct the * cluster order. */ std::vector<ClusterIndex> reordering; + /** How big the entries vector in the reconstructed depgraph will be (including holes). */ + ClusterIndex total_size{0}; // Read transactions in topological order. - try { - while (true) { + while (true) { + FeeFrac new_feerate; //!< The new transaction's fee and size. + SetType new_ancestors; //!< The new transaction's ancestors (excluding itself). + uint64_t diff{0}; //!< How many potential parents/insertions we have to skip. + bool read_error{false}; + try { // Read size. Size 0 signifies the end of the DepGraph. int32_t size; s >> VARINT_MODE(size, VarIntMode::NONNEGATIVE_SIGNED); @@ -204,21 +225,18 @@ struct DepGraphFormatter s >> VARINT(coded_fee); coded_fee &= 0xFFFFFFFFFFFFF; // Enough for fee between -21M...21M BTC. static_assert(0xFFFFFFFFFFFFF > uint64_t{2} * 21000000 * 100000000); - auto fee = UnsignedToSigned(coded_fee); - // Extend topo_depgraph with the new transaction (preliminarily at the end). - auto topo_idx = topo_depgraph.AddTransaction({fee, size}); - reordering.push_back(reordering.size()); + new_feerate = {UnsignedToSigned(coded_fee), size}; // Read dependency information. - uint64_t diff = 0; //!< How many potential parents we have to skip. + auto topo_idx = reordering.size(); s >> VARINT(diff); for (ClusterIndex dep_dist = 0; dep_dist < topo_idx; ++dep_dist) { /** Which topo_depgraph index we are currently considering as parent of topo_idx. */ ClusterIndex dep_topo_idx = topo_idx - 1 - dep_dist; // Ignore transactions which are already known ancestors of topo_idx. - if (topo_depgraph.Descendants(dep_topo_idx)[topo_idx]) continue; + if (new_ancestors[dep_topo_idx]) continue; if (diff == 0) { // When the skip counter has reached 0, add an actual dependency. - topo_depgraph.AddDependency(dep_topo_idx, topo_idx); + new_ancestors |= topo_depgraph.Ancestors(dep_topo_idx); // And read the number of skips after it. s >> VARINT(diff); } else { @@ -226,23 +244,52 @@ struct DepGraphFormatter --diff; } } - // If we reach this point, we can interpret the remaining skip value as how far - // from the end of reordering the new transaction should be placed (wrapping - // around), so remove the preliminary position it was put in above (which was to - // make sure that if a deserialization exception occurs, the new transaction still - // has some entry in reordering). - reordering.pop_back(); - ClusterIndex insert_distance = diff % (reordering.size() + 1); - // And then update reordering to reflect this new transaction's insertion. - for (auto& pos : reordering) { - pos += (pos >= reordering.size() - insert_distance); + } catch (const std::ios_base::failure&) { + // Continue even if a read error was encountered. + read_error = true; + } + // Construct a new transaction whenever we made it past the new_feerate construction. + if (new_feerate.IsEmpty()) break; + assert(reordering.size() < SetType::Size()); + auto topo_idx = topo_depgraph.AddTransaction(new_feerate); + topo_depgraph.AddDependencies(new_ancestors, topo_idx); + if (total_size < SetType::Size()) { + // Normal case. + diff %= SetType::Size(); + if (diff <= total_size) { + // Insert the new transaction at distance diff back from the end. + for (auto& pos : reordering) { + pos += (pos >= total_size - diff); + } + reordering.push_back(total_size++ - diff); + } else { + // Append diff - total_size holes at the end, plus the new transaction. + total_size = diff; + reordering.push_back(total_size++); + } + } else { + // In case total_size == SetType::Size, it is not possible to insert the new + // transaction without exceeding SetType's size. Instead, interpret diff as an + // index into the holes, and overwrite a position there. This branch is never used + // when deserializing the output of the serializer, but gives meaning to otherwise + // invalid input. + diff %= (SetType::Size() - reordering.size()); + SetType holes = SetType::Fill(SetType::Size()); + for (auto pos : reordering) holes.Reset(pos); + for (auto pos : holes) { + if (diff == 0) { + reordering.push_back(pos); + break; + } + --diff; } - reordering.push_back(reordering.size() - insert_distance); } - } catch (const std::ios_base::failure&) {} + // Stop if a read error was encountered during deserialization. + if (read_error) break; + } // Construct the original cluster order depgraph. - depgraph = DepGraph(topo_depgraph, reordering); + depgraph = DepGraph(topo_depgraph, reordering, total_size); } }; @@ -250,8 +297,19 @@ struct DepGraphFormatter template<typename SetType> void SanityCheck(const DepGraph<SetType>& depgraph) { + // Verify Positions and PositionRange consistency. + ClusterIndex num_positions{0}; + ClusterIndex position_range{0}; + for (ClusterIndex i : depgraph.Positions()) { + ++num_positions; + position_range = i + 1; + } + assert(num_positions == depgraph.TxCount()); + assert(position_range == depgraph.PositionRange()); + assert(position_range >= num_positions); + assert(position_range <= SetType::Size()); // Consistency check between ancestors internally. - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + for (ClusterIndex i : depgraph.Positions()) { // Transactions include themselves as ancestors. assert(depgraph.Ancestors(i)[i]); // If a is an ancestor of b, then b's ancestors must include all of a's ancestors. @@ -260,13 +318,27 @@ void SanityCheck(const DepGraph<SetType>& depgraph) } } // Consistency check between ancestors and descendants. - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { - for (ClusterIndex j = 0; j < depgraph.TxCount(); ++j) { + for (ClusterIndex i : depgraph.Positions()) { + for (ClusterIndex j : depgraph.Positions()) { assert(depgraph.Ancestors(i)[j] == depgraph.Descendants(j)[i]); } + // No transaction is a parent or child of itself. + auto parents = depgraph.GetReducedParents(i); + auto children = depgraph.GetReducedChildren(i); + assert(!parents[i]); + assert(!children[i]); + // Parents of a transaction do not have ancestors inside those parents (except itself). + // Note that even the transaction itself may be missing (if it is part of a cycle). + for (auto parent : parents) { + assert((depgraph.Ancestors(parent) & parents).IsSubsetOf(SetType::Singleton(parent))); + } + // Similar for children and descendants. + for (auto child : children) { + assert((depgraph.Descendants(child) & children).IsSubsetOf(SetType::Singleton(child))); + } } - // If DepGraph is acyclic, serialize + deserialize must roundtrip. if (IsAcyclic(depgraph)) { + // If DepGraph is acyclic, serialize + deserialize must roundtrip. std::vector<unsigned char> ser; VectorWriter writer(ser, 0); writer << Using<DepGraphFormatter>(depgraph); @@ -284,42 +356,36 @@ void SanityCheck(const DepGraph<SetType>& depgraph) reader >> Using<DepGraphFormatter>(decoded_depgraph); assert(depgraph == decoded_depgraph); assert(reader.empty()); - } -} -/** Verify that a DepGraph corresponds to the information in a cluster. */ -template<typename SetType> -void VerifyDepGraphFromCluster(const Cluster<SetType>& cluster, const DepGraph<SetType>& depgraph) -{ - // Sanity check the depgraph, which includes a check for correspondence between ancestors and - // descendants, so it suffices to check just ancestors below. - SanityCheck(depgraph); - // Verify transaction count. - assert(cluster.size() == depgraph.TxCount()); - // Verify feerates. - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { - assert(depgraph.FeeRate(i) == cluster[i].first); - } - // Verify ancestors. - for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { - // Start with the transaction having itself as ancestor. - auto ancestors = SetType::Singleton(i); - // Add parents of ancestors to the set of ancestors until it stops changing. - while (true) { - const auto old_ancestors = ancestors; - for (auto ancestor : ancestors) { - ancestors |= cluster[ancestor].second; - } - if (old_ancestors == ancestors) break; + // In acyclic graphs, the union of parents with parents of parents etc. yields the + // full ancestor set (and similar for children and descendants). + std::vector<SetType> parents(depgraph.PositionRange()), children(depgraph.PositionRange()); + for (ClusterIndex i : depgraph.Positions()) { + parents[i] = depgraph.GetReducedParents(i); + children[i] = depgraph.GetReducedChildren(i); } - // Compare against depgraph. - assert(depgraph.Ancestors(i) == ancestors); - // Some additional sanity tests: - // - Every transaction has itself as ancestor. - assert(ancestors[i]); - // - Every transaction has its direct parents as ancestors. - for (auto parent : cluster[i].second) { - assert(ancestors[parent]); + for (auto i : depgraph.Positions()) { + // Initialize the set of ancestors with just the current transaction itself. + SetType ancestors = SetType::Singleton(i); + // Iteratively add parents of all transactions in the ancestor set to itself. + while (true) { + const auto old_ancestors = ancestors; + for (auto j : ancestors) ancestors |= parents[j]; + // Stop when no more changes are being made. + if (old_ancestors == ancestors) break; + } + assert(ancestors == depgraph.Ancestors(i)); + + // Initialize the set of descendants with just the current transaction itself. + SetType descendants = SetType::Singleton(i); + // Iteratively add children of all transactions in the descendant set to itself. + while (true) { + const auto old_descendants = descendants; + for (auto j : descendants) descendants |= children[j]; + // Stop when no more changes are being made. + if (old_descendants == descendants) break; + } + assert(descendants == depgraph.Descendants(i)); } } } @@ -333,7 +399,7 @@ void SanityCheck(const DepGraph<SetType>& depgraph, Span<const ClusterIndex> lin TestBitSet done; for (auto i : linearization) { // Check transaction position is in range. - assert(i < depgraph.TxCount()); + assert(depgraph.Positions()[i]); // Check topology and lack of duplicates. assert((depgraph.Ancestors(i) - done) == TestBitSet::Singleton(i)); done.Set(i); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index fa89ceb332..7465846356 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -104,7 +104,8 @@ static void ExitFailure(std::string_view str_err) BasicTestingSetup::BasicTestingSetup(const ChainType chainType, TestOpts opts) : m_args{} { - m_node.shutdown = &m_interrupt; + m_node.shutdown_signal = &m_interrupt; + m_node.shutdown_request = [this]{ return m_interrupt(); }; m_node.args = &gArgs; std::vector<const char*> arguments = Cat( { @@ -224,7 +225,7 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts) m_cache_sizes = CalculateCacheSizes(m_args); - m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)); + m_node.notifications = std::make_unique<KernelNotifications>(Assert(m_node.shutdown_request), m_node.exit_status, *Assert(m_node.warnings)); m_make_chainman = [this, &chainparams, opts] { Assert(!m_node.chainman); @@ -245,7 +246,7 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts) .blocks_dir = m_args.GetBlocksDirPath(), .notifications = chainman_opts.notifications, }; - m_node.chainman = std::make_unique<ChainstateManager>(*Assert(m_node.shutdown), chainman_opts, blockman_opts); + m_node.chainman = std::make_unique<ChainstateManager>(*Assert(m_node.shutdown_signal), chainman_opts, blockman_opts); LOCK(m_node.chainman->GetMutex()); m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<BlockTreeDB>(DBParams{ .path = m_args.GetDataDirNet() / "blocks" / "index", diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index f5a3c0dcee..f9cf5d9157 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -75,6 +75,23 @@ struct BasicTestingSetup { fs::path m_path_root; fs::path m_path_lock; bool m_has_custom_datadir{false}; + /** @brief Test-specific arguments and settings. + * + * This member is intended to be the primary source of settings for code + * being tested by unit tests. It exists to make tests more self-contained + * and reduce reliance on global state. + * + * Usage guidelines: + * 1. Prefer using m_args where possible in test code. + * 2. If m_args is not accessible, use m_node.args as a fallback. + * 3. Avoid direct references to gArgs in test code. + * + * Note: Currently, m_node.args points to gArgs for backwards + * compatibility. In the future, it will point to m_args to further isolate + * test environments. + * + * @see https://github.com/bitcoin/bitcoin/issues/25055 for additional context. + */ ArgsManager m_args; }; diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 702af6578d..c9cca8af04 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -70,14 +70,18 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) { ChainstateManager& chainman = *Assert(m_node.chainman); - uint256 curr_tip = m_node.notifications->m_tip_block; + const auto get_notify_tip{[&]() { + LOCK(m_node.notifications->m_tip_block_mutex); + return m_node.notifications->m_tip_block; + }}; + uint256 curr_tip = get_notify_tip(); // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can // be found. mineBlocks(10); // After adding some blocks to the tip, best block should have changed. - BOOST_CHECK(m_node.notifications->m_tip_block != curr_tip); + BOOST_CHECK(get_notify_tip() != curr_tip); // Grab block 1 from disk; we'll add it to the background chain later. std::shared_ptr<CBlock> pblockone = std::make_shared<CBlock>(); @@ -92,15 +96,15 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) // Ensure our active chain is the snapshot chainstate. BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.IsSnapshotActive())); - curr_tip = m_node.notifications->m_tip_block; + curr_tip = get_notify_tip(); // Mine a new block on top of the activated snapshot chainstate. mineBlocks(1); // Defined in TestChain100Setup. // After adding some blocks to the snapshot tip, best block should have changed. - BOOST_CHECK(m_node.notifications->m_tip_block != curr_tip); + BOOST_CHECK(get_notify_tip() != curr_tip); - curr_tip = m_node.notifications->m_tip_block; + curr_tip = get_notify_tip(); BOOST_CHECK_EQUAL(chainman.GetAll().size(), 2); @@ -136,10 +140,10 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) // Ensure tip is as expected BOOST_CHECK_EQUAL(background_cs.m_chain.Tip()->GetBlockHash(), pblockone->GetHash()); - // g_best_block should be unchanged after adding a block to the background + // get_notify_tip() should be unchanged after adding a block to the background // validation chain. BOOST_CHECK(block_added); - BOOST_CHECK_EQUAL(curr_tip, m_node.notifications->m_tip_block); + BOOST_CHECK_EQUAL(curr_tip, get_notify_tip()); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index b4fcfbd853..6c2a825e64 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -382,7 +382,7 @@ struct SnapshotTestSetup : TestChain100Setup { LOCK(::cs_main); chainman.ResetChainstates(); BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0); - m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)); + m_node.notifications = std::make_unique<KernelNotifications>(Assert(m_node.shutdown_request), m_node.exit_status, *Assert(m_node.warnings)); const ChainstateManager::Options chainman_opts{ .chainparams = ::Params(), .datadir = chainman.m_options.datadir, @@ -397,7 +397,7 @@ struct SnapshotTestSetup : TestChain100Setup { // For robustness, ensure the old manager is destroyed before creating a // new one. m_node.chainman.reset(); - m_node.chainman = std::make_unique<ChainstateManager>(*Assert(m_node.shutdown), chainman_opts, blockman_opts); + m_node.chainman = std::make_unique<ChainstateManager>(*Assert(m_node.shutdown_signal), chainman_opts, blockman_opts); } return *Assert(m_node.chainman); } diff --git a/src/txorphanage.cpp b/src/txorphanage.cpp index 35a215c88a..ba4ba6c3b6 100644 --- a/src/txorphanage.cpp +++ b/src/txorphanage.cpp @@ -33,7 +33,7 @@ bool TxOrphanage::AddTx(const CTransactionRef& tx, NodeId peer) return false; } - auto ret = m_orphans.emplace(wtxid, OrphanTx{tx, peer, Now<NodeSeconds>() + ORPHAN_TX_EXPIRE_TIME, m_orphan_list.size()}); + auto ret = m_orphans.emplace(wtxid, OrphanTx{{tx, peer, Now<NodeSeconds>() + ORPHAN_TX_EXPIRE_TIME}, m_orphan_list.size()}); assert(ret.second); m_orphan_list.push_back(ret.first); for (const CTxIn& txin : tx->vin) { @@ -277,3 +277,13 @@ std::vector<std::pair<CTransactionRef, NodeId>> TxOrphanage::GetChildrenFromDiff } return children_found; } + +std::vector<TxOrphanage::OrphanTxBase> TxOrphanage::GetOrphanTransactions() const +{ + std::vector<OrphanTxBase> ret; + ret.reserve(m_orphans.size()); + for (auto const& o : m_orphans) { + ret.push_back({o.second.tx, o.second.fromPeer, o.second.nTimeExpire}); + } + return ret; +} diff --git a/src/txorphanage.h b/src/txorphanage.h index 2c53d1d40f..5501d10922 100644 --- a/src/txorphanage.h +++ b/src/txorphanage.h @@ -72,11 +72,17 @@ public: return m_orphans.size(); } -protected: - struct OrphanTx { + /** Allows providing orphan information externally */ + struct OrphanTxBase { CTransactionRef tx; NodeId fromPeer; NodeSeconds nTimeExpire; + }; + + std::vector<OrphanTxBase> GetOrphanTransactions() const; + +protected: + struct OrphanTx : public OrphanTxBase { size_t list_pos; }; diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 26c6271f9b..4999dbf13f 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -42,4 +42,5 @@ target_link_libraries(bitcoin_util bitcoin_clientversion bitcoin_crypto $<$<PLATFORM_ID:Windows>:ws2_32> + $<$<PLATFORM_ID:Windows>:iphlpapi> ) diff --git a/src/util/check.cpp b/src/util/check.cpp index eb3885832b..e1956042c3 100644 --- a/src/util/check.cpp +++ b/src/util/check.cpp @@ -4,7 +4,7 @@ #include <util/check.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <clientversion.h> #include <tinyformat.h> diff --git a/src/util/check.h b/src/util/check.h index a02a1de8dc..8f28f5dc94 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -40,7 +40,7 @@ void assertion_fail(std::string_view file, int line, std::string_view func, std: /** Helper for Assert()/Assume() */ template <bool IS_ASSERT, typename T> -T&& inline_assertion_check(LIFETIMEBOUND T&& val, [[maybe_unused]] const char* file, [[maybe_unused]] int line, [[maybe_unused]] const char* func, [[maybe_unused]] const char* assertion) +constexpr T&& inline_assertion_check(LIFETIMEBOUND T&& val, [[maybe_unused]] const char* file, [[maybe_unused]] int line, [[maybe_unused]] const char* func, [[maybe_unused]] const char* assertion) { if constexpr (IS_ASSERT #ifdef ABORT_ON_FAILED_ASSUME diff --git a/src/util/feefrac.h b/src/util/feefrac.h index 9772162010..161322b50a 100644 --- a/src/util/feefrac.h +++ b/src/util/feefrac.h @@ -64,13 +64,13 @@ struct FeeFrac int32_t size; /** Construct an IsEmpty() FeeFrac. */ - inline FeeFrac() noexcept : fee{0}, size{0} {} + constexpr inline FeeFrac() noexcept : fee{0}, size{0} {} /** Construct a FeeFrac with specified fee and size. */ - inline FeeFrac(int64_t f, int32_t s) noexcept : fee{f}, size{s} {} + constexpr inline FeeFrac(int64_t f, int32_t s) noexcept : fee{f}, size{s} {} - inline FeeFrac(const FeeFrac&) noexcept = default; - inline FeeFrac& operator=(const FeeFrac&) noexcept = default; + constexpr inline FeeFrac(const FeeFrac&) noexcept = default; + constexpr inline FeeFrac& operator=(const FeeFrac&) noexcept = default; /** Check if this is empty (size and fee are 0). */ bool inline IsEmpty() const noexcept { diff --git a/src/util/fs_helpers.cpp b/src/util/fs_helpers.cpp index e01f9a61e9..7ac7b829d8 100644 --- a/src/util/fs_helpers.cpp +++ b/src/util/fs_helpers.cpp @@ -5,7 +5,7 @@ #include <util/fs_helpers.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <logging.h> #include <sync.h> diff --git a/src/util/syserror.cpp b/src/util/syserror.cpp index 6f3a724483..a902826f8e 100644 --- a/src/util/syserror.cpp +++ b/src/util/syserror.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <tinyformat.h> #include <util/syserror.h> diff --git a/src/util/threadnames.cpp b/src/util/threadnames.cpp index 0249de37e3..37c5b8f617 100644 --- a/src/util/threadnames.cpp +++ b/src/util/threadnames.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <cstring> #include <string> diff --git a/src/util/tokenpipe.cpp b/src/util/tokenpipe.cpp index 16fbb664ea..9425c62ebf 100644 --- a/src/util/tokenpipe.cpp +++ b/src/util/tokenpipe.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/tokenpipe.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #ifndef WIN32 diff --git a/src/util/trace.h b/src/util/trace.h index d9ed65e3aa..72a486d562 100644 --- a/src/util/trace.h +++ b/src/util/trace.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_UTIL_TRACE_H #define BITCOIN_UTIL_TRACE_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #ifdef ENABLE_TRACING diff --git a/src/validation.cpp b/src/validation.cpp index 5da3387ad2..ab94ac02eb 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <validation.h> @@ -2179,6 +2179,8 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, if (pvChecks) { pvChecks->emplace_back(std::move(check)); } else if (!check()) { + ScriptError error{check.GetScriptError()}; + if (flags & STANDARD_NOT_MANDATORY_VERIFY_FLAGS) { // Check whether the failure was caused by a // non-mandatory script verification check, such as @@ -2192,6 +2194,14 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheSigStore, &txdata); if (check2()) return state.Invalid(TxValidationResult::TX_NOT_STANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); + + // If the second check failed, it failed due to a mandatory script verification + // flag, but the first check might have failed on a non-mandatory script + // verification flag. + // + // Avoid reporting a mandatory script check failure with a non-mandatory error + // string by reporting the error from the second check. + error = check2.GetScriptError(); } // MANDATORY flag failures correspond to // TxValidationResult::TX_CONSENSUS. Because CONSENSUS @@ -2202,7 +2212,7 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, // support, to avoid splitting the network (but this // depends on the details of how net_processing handles // such errors). - return state.Invalid(TxValidationResult::TX_CONSENSUS, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); + return state.Invalid(TxValidationResult::TX_CONSENSUS, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(error))); } } @@ -2736,7 +2746,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, block.vtx.size(), nInputs, nSigOpsCost, - time_5 - time_start // in microseconds (µs) + Ticks<std::chrono::nanoseconds>(time_5 - time_start) ); return true; @@ -3533,7 +3543,6 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< m_chainman.m_options.signals->UpdatedBlockTip(pindexNewTip, pindexFork, still_in_ibd); } - // Always notify the UI if a new block tip was connected if (kernel::IsInterrupted(m_chainman.GetNotifications().blockTip(GetSynchronizationState(still_in_ibd, m_chainman.m_blockman.m_blockfiles_indexed), *pindexNewTip))) { // Just breaking and returning success for now. This could // be changed to bubble up the kernel::Interrupted value to diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index e8ff1d78e3..da2685d771 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -9,6 +9,7 @@ #include <consensus/validation.h> #include <kernel/chain.h> #include <kernel/mempool_entry.h> +#include <kernel/mempool_removal_reason.h> #include <logging.h> #include <primitives/block.h> #include <primitives/transaction.h> @@ -19,8 +20,6 @@ #include <unordered_map> #include <utility> -std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept; - /** * ValidationSignalsImpl manages a list of shared_ptr<CValidationInterface> callbacks. * diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 14d22bb54e..cfd09a2e10 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <common/args.h> #include <init.h> diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index 838d062108..1c2951deee 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <core_io.h> #include <key_io.h> diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 20d09b1d9a..4ffc6f1e0d 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <chain.h> #include <clientversion.h> diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index 39582b3f6a..5140ac8c05 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <core_io.h> #include <key_io.h> diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 7abf7f59c0..aceed24a86 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -1167,6 +1167,7 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( result.GetSelectedValue()); // vouts to the payees + txNew.vout.reserve(vecSend.size() + 1); // + 1 because of possible later insert for (const auto& recipient : vecSend) { txNew.vout.emplace_back(recipient.nAmount, GetScriptForDestination(recipient.dest)); @@ -1217,6 +1218,7 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( // behavior." bool use_anti_fee_sniping = true; const uint32_t default_sequence{coin_control.m_signal_bip125_rbf.value_or(wallet.m_signal_rbf) ? MAX_BIP125_RBF_SEQUENCE : CTxIn::MAX_SEQUENCE_NONFINAL}; + txNew.vin.reserve(selected_coins.size()); for (const auto& coin : selected_coins) { std::optional<uint32_t> sequence = coin_control.GetSequence(coin->outpoint); if (sequence) { diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index f2110ea3f7..ab082327de 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <wallet/sqlite.h> diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 2fac356263..ea32199497 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <boost/test/unit_test.hpp> @@ -28,7 +28,7 @@ inline std::ostream& operator<<(std::ostream& os, const std::pair<const Serializ { Span key{kv.first}, value{kv.second}; os << "(\"" << std::string_view{reinterpret_cast<const char*>(key.data()), key.size()} << "\", \"" - << std::string_view{reinterpret_cast<const char*>(key.data()), key.size()} << "\")"; + << std::string_view{reinterpret_cast<const char*>(value.data()), value.size()} << "\")"; return os; } diff --git a/src/wallet/test/fuzz/wallet_bdb_parser.cpp b/src/wallet/test/fuzz/wallet_bdb_parser.cpp index 5ec24faede..6482b65d06 100644 --- a/src/wallet/test/fuzz/wallet_bdb_parser.cpp +++ b/src/wallet/test/fuzz/wallet_bdb_parser.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> diff --git a/src/wallet/test/util.h b/src/wallet/test/util.h index fc7674e961..ba12f5f6bf 100644 --- a/src/wallet/test/util.h +++ b/src/wallet/test/util.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_WALLET_TEST_UTIL_H #define BITCOIN_WALLET_TEST_UTIL_H -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <addresstype.h> #include <wallet/db.h> diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index b921bc5e49..de565102cc 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -5,7 +5,7 @@ #include <wallet/wallet.h> -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <addresstype.h> #include <blockfilter.h> #include <chain.h> diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 47b84f7e6a..597a4ef9a4 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <wallet/walletdb.h> diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 10785ad354..b78985264a 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep #include <wallet/wallettool.h> diff --git a/test/functional/README.md b/test/functional/README.md index a4994f2e7c..a34bf1827c 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -10,7 +10,8 @@ that file and modify to fit your needs. #### Coverage -Running `test/functional/test_runner.py` with the `--coverage` argument tracks which RPCs are +Assuming the build directory is `build`, +running `build/test/functional/test_runner.py` with the `--coverage` argument tracks which RPCs are called by the tests and prints a report of uncovered RPCs in the summary. This can be used (along with the `--extended` argument) to find out which RPCs we don't have test cases for. diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index 2e4ca83bf0..d2d7202d86 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -263,6 +263,17 @@ def getDisabledOpcodeTemplate(opcode): 'valid_in_block' : True }) +class NonStandardAndInvalid(BadTxTemplate): + """A non-standard transaction which is also consensus-invalid should return the consensus error.""" + reject_reason = "mandatory-script-verify-flag-failed (OP_RETURN was encountered)" + expect_disconnect = True + valid_in_block = False + + def get_tx(self): + return create_tx_with_script( + self.spend_tx, 0, script_sig=b'\x00' * 3 + b'\xab\x6a', + amount=(self.spend_avail // 2)) + # Disabled opcode tx templates (CVE-2010-5137) DisabledOpcodeTemplates = [getDisabledOpcodeTemplate(opcode) for opcode in [ OP_CAT, diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 384ca311c7..43bf61c174 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -88,6 +88,7 @@ class FullBlockTest(BitcoinTestFramework): self.extra_args = [[ '-acceptnonstdtxn=1', # This is a consensus block test, we don't care about tx policy '-testactivationheight=bip34@2', + '-par=1', # Until https://github.com/bitcoin/bitcoin/issues/30960 is fixed ]] def run_test(self): diff --git a/test/functional/feature_blocksxor.py b/test/functional/feature_blocksxor.py index 7698a66ec4..9824bf9715 100755 --- a/test/functional/feature_blocksxor.py +++ b/test/functional/feature_blocksxor.py @@ -31,7 +31,7 @@ class BlocksXORTest(BitcoinTestFramework): node = self.nodes[0] wallet = MiniWallet(node) for _ in range(5): - wallet.send_self_transfer(from_node=node, target_weight=80000) + wallet.send_self_transfer(from_node=node, target_vsize=20000) self.generate(wallet, 1) block_files = list(node.blocks_path.glob('blk[0-9][0-9][0-9][0-9][0-9].dat')) diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 83627ff5c2..974d8268a2 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -398,6 +398,7 @@ class EstimateFeeTest(BitcoinTestFramework): self.start_node(0) self.connect_nodes(0, 1) self.connect_nodes(0, 2) + self.sync_blocks() assert_equal(self.nodes[0].estimatesmartfee(1)["errors"], ["Insufficient data or no feerate found"]) def broadcast_and_mine(self, broadcaster, miner, feerate, count): diff --git a/test/functional/feature_framework_miniwallet.py b/test/functional/feature_framework_miniwallet.py index d1aa24e7cd..f723f7f31e 100755 --- a/test/functional/feature_framework_miniwallet.py +++ b/test/functional/feature_framework_miniwallet.py @@ -9,7 +9,7 @@ import string from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( - assert_greater_than_or_equal, + assert_equal, ) from test_framework.wallet import ( MiniWallet, @@ -22,17 +22,15 @@ class FeatureFrameworkMiniWalletTest(BitcoinTestFramework): self.num_nodes = 1 def test_tx_padding(self): - """Verify that MiniWallet's transaction padding (`target_weight` parameter) - works accurately enough (i.e. at most 3 WUs higher) with all modes.""" + """Verify that MiniWallet's transaction padding (`target_vsize` parameter) + works accurately with all modes.""" for mode_name, wallet in self.wallets: self.log.info(f"Test tx padding with MiniWallet mode {mode_name}...") utxo = wallet.get_utxo(mark_as_spent=False) - for target_weight in [1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 4000000, - 989, 2001, 4337, 13371, 23219, 49153, 102035, 223419, 3999989]: - tx = wallet.create_self_transfer(utxo_to_spend=utxo, target_weight=target_weight)["tx"] - self.log.debug(f"-> target weight: {target_weight}, actual weight: {tx.get_weight()}") - assert_greater_than_or_equal(tx.get_weight(), target_weight) - assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) + for target_vsize in [250, 500, 1250, 2500, 5000, 12500, 25000, 50000, 1000000, + 248, 501, 1085, 3343, 5805, 12289, 25509, 55855, 999998]: + tx = wallet.create_self_transfer(utxo_to_spend=utxo, target_vsize=target_vsize)["tx"] + assert_equal(tx.get_vsize(), target_vsize) def test_wallet_tagging(self): """Verify that tagged wallet instances are able to send funds.""" diff --git a/test/functional/interface_usdt_validation.py b/test/functional/interface_usdt_validation.py index 9a37b96ada..8a98a452de 100755 --- a/test/functional/interface_usdt_validation.py +++ b/test/functional/interface_usdt_validation.py @@ -8,6 +8,7 @@ """ import ctypes +import time # Test will be skipped if we don't have bcc installed try: @@ -105,10 +106,12 @@ class ValidationTracepointTest(BitcoinTestFramework): handle_blockconnected) self.log.info(f"mine {BLOCKS_EXPECTED} blocks") - block_hashes = self.generatetoaddress( - self.nodes[0], BLOCKS_EXPECTED, ADDRESS_BCRT1_UNSPENDABLE) - for block_hash in block_hashes: - expected_blocks[block_hash] = self.nodes[0].getblock(block_hash, 2) + generatetoaddress_duration = dict() + for _ in range(BLOCKS_EXPECTED): + start = time.time() + hash = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] + generatetoaddress_duration[hash] = (time.time() - start) * 1e9 # in nanoseconds + expected_blocks[hash] = self.nodes[0].getblock(hash, 2) bpf.perf_buffer_poll(timeout=200) @@ -123,6 +126,10 @@ class ValidationTracepointTest(BitcoinTestFramework): assert_equal(0, event.sigops) # no sigops in coinbase tx # only plausibility checks assert event.duration > 0 + # generatetoaddress (mining and connecting) takes longer than + # connecting the block. In case the duration unit is off, we'll + # detect it with this assert. + assert event.duration < generatetoaddress_duration[block_hash] del expected_blocks[block_hash] assert_equal(BLOCKS_EXPECTED, len(events)) assert_equal(0, len(expected_blocks)) diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index 626928a49a..a29c103c3f 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -55,12 +55,12 @@ class MempoolLimitTest(BitcoinTestFramework): self.generate(node, 1) # tx_A needs to be RBF'd, set minfee at set size - A_weight = 1000 + A_vsize = 250 mempoolmin_feerate = node.getmempoolinfo()["mempoolminfee"] tx_A = self.wallet.send_self_transfer( from_node=node, fee_rate=mempoolmin_feerate, - target_weight=A_weight, + target_vsize=A_vsize, utxo_to_spend=rbf_utxo, confirmed_only=True ) @@ -68,15 +68,15 @@ class MempoolLimitTest(BitcoinTestFramework): # RBF's tx_A, is not yet submitted tx_B = self.wallet.create_self_transfer( fee=tx_A["fee"] * 4, - target_weight=A_weight, + target_vsize=A_vsize, utxo_to_spend=rbf_utxo, confirmed_only=True ) # Spends tx_B's output, too big for cpfp carveout (because that would also increase the descendant limit by 1) - non_cpfp_carveout_weight = 40001 # EXTRA_DESCENDANT_TX_SIZE_LIMIT + 1 + non_cpfp_carveout_vsize = 10001 # EXTRA_DESCENDANT_TX_SIZE_LIMIT + 1 tx_C = self.wallet.create_self_transfer( - target_weight=non_cpfp_carveout_weight, + target_vsize=non_cpfp_carveout_vsize, fee_rate=mempoolmin_feerate, utxo_to_spend=tx_B["new_utxo"], confirmed_only=True @@ -103,14 +103,14 @@ class MempoolLimitTest(BitcoinTestFramework): # UTXOs to be spent by the ultimate child transaction parent_utxos = [] - evicted_weight = 8000 + evicted_vsize = 2000 # Mempool transaction which is evicted due to being at the "bottom" of the mempool when the # mempool overflows and evicts by descendant score. It's important that the eviction doesn't # happen in the middle of package evaluation, as it can invalidate the coins cache. mempool_evicted_tx = self.wallet.send_self_transfer( from_node=node, fee_rate=mempoolmin_feerate, - target_weight=evicted_weight, + target_vsize=evicted_vsize, confirmed_only=True ) # Already in mempool when package is submitted. @@ -132,14 +132,16 @@ class MempoolLimitTest(BitcoinTestFramework): # Series of parents that don't need CPFP and are submitted individually. Each one is large and # high feerate, which means they should trigger eviction but not be evicted. - parent_weight = 100000 + parent_vsize = 25000 num_big_parents = 3 - assert_greater_than(parent_weight * num_big_parents, current_info["maxmempool"] - current_info["bytes"]) + # Need to be large enough to trigger eviction + # (note that the mempool usage of a tx is about three times its vsize) + assert_greater_than(parent_vsize * num_big_parents * 3, current_info["maxmempool"] - current_info["bytes"]) parent_feerate = 100 * mempoolmin_feerate big_parent_txids = [] for i in range(num_big_parents): - parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_weight=parent_weight, confirmed_only=True) + parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_vsize=parent_vsize, confirmed_only=True) parent_utxos.append(parent["new_utxo"]) package_hex.append(parent["hex"]) big_parent_txids.append(parent["txid"]) @@ -311,8 +313,9 @@ class MempoolLimitTest(BitcoinTestFramework): entry = node.getmempoolentry(txid) worst_feerate_btcvb = min(worst_feerate_btcvb, entry["fees"]["descendant"] / entry["descendantsize"]) # Needs to be large enough to trigger eviction - target_weight_each = 200000 - assert_greater_than(target_weight_each * 2, node.getmempoolinfo()["maxmempool"] - node.getmempoolinfo()["bytes"]) + # (note that the mempool usage of a tx is about three times its vsize) + target_vsize_each = 50000 + assert_greater_than(target_vsize_each * 2 * 3, node.getmempoolinfo()["maxmempool"] - node.getmempoolinfo()["bytes"]) # Should be a true CPFP: parent's feerate is just below mempool min feerate parent_feerate = mempoolmin_feerate - Decimal("0.000001") # 0.1 sats/vbyte below min feerate # Parent + child is above mempool minimum feerate @@ -320,8 +323,8 @@ class MempoolLimitTest(BitcoinTestFramework): # However, when eviction is triggered, these transactions should be at the bottom. # This assertion assumes parent and child are the same size. miniwallet.rescan_utxos() - tx_parent_just_below = miniwallet.create_self_transfer(fee_rate=parent_feerate, target_weight=target_weight_each) - tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee_rate=child_feerate, target_weight=target_weight_each) + tx_parent_just_below = miniwallet.create_self_transfer(fee_rate=parent_feerate, target_vsize=target_vsize_each) + tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee_rate=child_feerate, target_vsize=target_vsize_each) # This package ranks below the lowest descendant package in the mempool package_fee = tx_parent_just_below["fee"] + tx_child_just_above["fee"] package_vsize = tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize() diff --git a/test/functional/mempool_package_limits.py b/test/functional/mempool_package_limits.py index 6e26a684e2..3290ff43c4 100755 --- a/test/functional/mempool_package_limits.py +++ b/test/functional/mempool_package_limits.py @@ -4,9 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for limiting mempool and package ancestors/descendants.""" from test_framework.blocktools import COINBASE_MATURITY -from test_framework.messages import ( - WITNESS_SCALE_FACTOR, -) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -290,19 +287,18 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): parent_utxos = [] target_vsize = 30_000 high_fee = 10 * target_vsize # 10 sats/vB - target_weight = target_vsize * WITNESS_SCALE_FACTOR self.log.info("Check that in-mempool and in-package ancestor size limits are calculated properly in packages") # Mempool transactions A and B for _ in range(2): - bulked_tx = self.wallet.create_self_transfer(target_weight=target_weight) + bulked_tx = self.wallet.create_self_transfer(target_vsize=target_vsize) self.wallet.sendrawtransaction(from_node=node, tx_hex=bulked_tx["hex"]) parent_utxos.append(bulked_tx["new_utxo"]) # Package transaction C - pc_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_utxos, fee_per_output=high_fee, target_weight=target_weight) + pc_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_utxos, fee_per_output=high_fee, target_vsize=target_vsize) # Package transaction D - pd_tx = self.wallet.create_self_transfer(utxo_to_spend=pc_tx["new_utxos"][0], target_weight=target_weight) + pd_tx = self.wallet.create_self_transfer(utxo_to_spend=pc_tx["new_utxos"][0], target_vsize=target_vsize) assert_equal(2, node.getmempoolinfo()["size"]) return [pc_tx["hex"], pd_tx["hex"]] @@ -321,20 +317,19 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): node = self.nodes[0] target_vsize = 21_000 high_fee = 10 * target_vsize # 10 sats/vB - target_weight = target_vsize * WITNESS_SCALE_FACTOR self.log.info("Check that in-mempool and in-package descendant sizes are calculated properly in packages") # Top parent in mempool, Ma - ma_tx = self.wallet.create_self_transfer_multi(num_outputs=2, fee_per_output=high_fee // 2, target_weight=target_weight) + ma_tx = self.wallet.create_self_transfer_multi(num_outputs=2, fee_per_output=high_fee // 2, target_vsize=target_vsize) self.wallet.sendrawtransaction(from_node=node, tx_hex=ma_tx["hex"]) package_hex = [] for j in range(2): # Two legs (left and right) # Mempool transaction (Mb and Mc) - mempool_tx = self.wallet.create_self_transfer(utxo_to_spend=ma_tx["new_utxos"][j], target_weight=target_weight) + mempool_tx = self.wallet.create_self_transfer(utxo_to_spend=ma_tx["new_utxos"][j], target_vsize=target_vsize) self.wallet.sendrawtransaction(from_node=node, tx_hex=mempool_tx["hex"]) # Package transaction (Pd and Pe) - package_tx = self.wallet.create_self_transfer(utxo_to_spend=mempool_tx["new_utxo"], target_weight=target_weight) + package_tx = self.wallet.create_self_transfer(utxo_to_spend=mempool_tx["new_utxo"], target_vsize=target_vsize) package_hex.append(package_tx["hex"]) assert_equal(3, node.getmempoolinfo()["size"]) diff --git a/test/functional/mempool_truc.py b/test/functional/mempool_truc.py index 28f3256ef1..54a258215d 100755 --- a/test/functional/mempool_truc.py +++ b/test/functional/mempool_truc.py @@ -6,7 +6,6 @@ from decimal import Decimal from test_framework.messages import ( MAX_BIP125_RBF_SEQUENCE, - WITNESS_SCALE_FACTOR, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -23,6 +22,7 @@ from test_framework.wallet import ( MAX_REPLACEMENT_CANDIDATES = 100 TRUC_MAX_VSIZE = 10000 +TRUC_CHILD_MAX_VSIZE = 1000 def cleanup(extra_args=None): def decorator(func): @@ -55,14 +55,14 @@ class MempoolTRUC(BitcoinTestFramework): def test_truc_max_vsize(self): node = self.nodes[0] self.log.info("Test TRUC-specific maximum transaction vsize") - tx_v3_heavy = self.wallet.create_self_transfer(target_weight=(TRUC_MAX_VSIZE + 1) * WITNESS_SCALE_FACTOR, version=3) + tx_v3_heavy = self.wallet.create_self_transfer(target_vsize=TRUC_MAX_VSIZE + 1, version=3) assert_greater_than_or_equal(tx_v3_heavy["tx"].get_vsize(), TRUC_MAX_VSIZE) expected_error_heavy = f"TRUC-violation, version=3 tx {tx_v3_heavy['txid']} (wtxid={tx_v3_heavy['wtxid']}) is too big" assert_raises_rpc_error(-26, expected_error_heavy, node.sendrawtransaction, tx_v3_heavy["hex"]) self.check_mempool([]) # Ensure we are hitting the TRUC-specific limit and not something else - tx_v2_heavy = self.wallet.send_self_transfer(from_node=node, target_weight=(TRUC_MAX_VSIZE + 1) * WITNESS_SCALE_FACTOR, version=2) + tx_v2_heavy = self.wallet.send_self_transfer(from_node=node, target_vsize=TRUC_MAX_VSIZE + 1, version=2) self.check_mempool([tx_v2_heavy["txid"]]) @cleanup(extra_args=["-datacarriersize=1000"]) @@ -73,10 +73,10 @@ class MempoolTRUC(BitcoinTestFramework): self.check_mempool([tx_v3_parent_normal["txid"]]) tx_v3_child_heavy = self.wallet.create_self_transfer( utxo_to_spend=tx_v3_parent_normal["new_utxo"], - target_weight=4004, + target_vsize=TRUC_CHILD_MAX_VSIZE + 1, version=3 ) - assert_greater_than_or_equal(tx_v3_child_heavy["tx"].get_vsize(), 1000) + assert_greater_than_or_equal(tx_v3_child_heavy["tx"].get_vsize(), TRUC_CHILD_MAX_VSIZE) expected_error_child_heavy = f"TRUC-violation, version=3 child tx {tx_v3_child_heavy['txid']} (wtxid={tx_v3_child_heavy['wtxid']}) is too big" assert_raises_rpc_error(-26, expected_error_child_heavy, node.sendrawtransaction, tx_v3_child_heavy["hex"]) self.check_mempool([tx_v3_parent_normal["txid"]]) @@ -88,20 +88,21 @@ class MempoolTRUC(BitcoinTestFramework): from_node=node, fee_rate=DEFAULT_FEE, utxo_to_spend=tx_v3_parent_normal["new_utxo"], - target_weight=3987, + target_vsize=TRUC_CHILD_MAX_VSIZE - 3, version=3 ) - assert_greater_than_or_equal(1000, tx_v3_child_almost_heavy["tx"].get_vsize()) + assert_greater_than_or_equal(TRUC_CHILD_MAX_VSIZE, tx_v3_child_almost_heavy["tx"].get_vsize()) self.check_mempool([tx_v3_parent_normal["txid"], tx_v3_child_almost_heavy["txid"]]) assert_equal(node.getmempoolentry(tx_v3_parent_normal["txid"])["descendantcount"], 2) tx_v3_child_almost_heavy_rbf = self.wallet.send_self_transfer( from_node=node, fee_rate=DEFAULT_FEE * 2, utxo_to_spend=tx_v3_parent_normal["new_utxo"], - target_weight=3500, + target_vsize=875, version=3 ) - assert_greater_than_or_equal(tx_v3_child_almost_heavy["tx"].get_vsize() + tx_v3_child_almost_heavy_rbf["tx"].get_vsize(), 1000) + assert_greater_than_or_equal(tx_v3_child_almost_heavy["tx"].get_vsize() + tx_v3_child_almost_heavy_rbf["tx"].get_vsize(), + TRUC_CHILD_MAX_VSIZE) self.check_mempool([tx_v3_parent_normal["txid"], tx_v3_child_almost_heavy_rbf["txid"]]) assert_equal(node.getmempoolentry(tx_v3_parent_normal["txid"])["descendantcount"], 2) @@ -199,8 +200,8 @@ class MempoolTRUC(BitcoinTestFramework): self.check_mempool([]) tx_v2_from_v3 = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block["new_utxo"], version=2) tx_v3_from_v2 = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v2_block["new_utxo"], version=3) - tx_v3_child_large = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block2["new_utxo"], target_weight=5000, version=3) - assert_greater_than(node.getmempoolentry(tx_v3_child_large["txid"])["vsize"], 1000) + tx_v3_child_large = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block2["new_utxo"], target_vsize=1250, version=3) + assert_greater_than(node.getmempoolentry(tx_v3_child_large["txid"])["vsize"], TRUC_CHILD_MAX_VSIZE) self.check_mempool([tx_v2_from_v3["txid"], tx_v3_from_v2["txid"], tx_v3_child_large["txid"]]) node.invalidateblock(block[0]) self.check_mempool([tx_v3_block["txid"], tx_v2_block["txid"], tx_v3_block2["txid"], tx_v2_from_v3["txid"], tx_v3_from_v2["txid"], tx_v3_child_large["txid"]]) @@ -217,22 +218,22 @@ class MempoolTRUC(BitcoinTestFramework): """ node = self.nodes[0] self.log.info("Test that a decreased limitdescendantsize also applies to TRUC child") - parent_target_weight = 9990 * WITNESS_SCALE_FACTOR - child_target_weight = 500 * WITNESS_SCALE_FACTOR + parent_target_vsize = 9990 + child_target_vsize = 500 tx_v3_parent_large1 = self.wallet.send_self_transfer( from_node=node, - target_weight=parent_target_weight, + target_vsize=parent_target_vsize, version=3 ) tx_v3_child_large1 = self.wallet.create_self_transfer( utxo_to_spend=tx_v3_parent_large1["new_utxo"], - target_weight=child_target_weight, + target_vsize=child_target_vsize, version=3 ) # Parent and child are within v3 limits, but parent's 10kvB descendant limit is exceeded assert_greater_than_or_equal(TRUC_MAX_VSIZE, tx_v3_parent_large1["tx"].get_vsize()) - assert_greater_than_or_equal(1000, tx_v3_child_large1["tx"].get_vsize()) + assert_greater_than_or_equal(TRUC_CHILD_MAX_VSIZE, tx_v3_child_large1["tx"].get_vsize()) assert_greater_than(tx_v3_parent_large1["tx"].get_vsize() + tx_v3_child_large1["tx"].get_vsize(), 10000) assert_raises_rpc_error(-26, f"too-long-mempool-chain, exceeds descendant size limit for tx {tx_v3_parent_large1['txid']}", node.sendrawtransaction, tx_v3_child_large1["hex"]) @@ -244,18 +245,18 @@ class MempoolTRUC(BitcoinTestFramework): self.restart_node(0, extra_args=["-limitancestorsize=10", "-datacarriersize=40000"]) tx_v3_parent_large2 = self.wallet.send_self_transfer( from_node=node, - target_weight=parent_target_weight, + target_vsize=parent_target_vsize, version=3 ) tx_v3_child_large2 = self.wallet.create_self_transfer( utxo_to_spend=tx_v3_parent_large2["new_utxo"], - target_weight=child_target_weight, + target_vsize=child_target_vsize, version=3 ) # Parent and child are within TRUC limits assert_greater_than_or_equal(TRUC_MAX_VSIZE, tx_v3_parent_large2["tx"].get_vsize()) - assert_greater_than_or_equal(1000, tx_v3_child_large2["tx"].get_vsize()) + assert_greater_than_or_equal(TRUC_CHILD_MAX_VSIZE, tx_v3_child_large2["tx"].get_vsize()) assert_greater_than(tx_v3_parent_large2["tx"].get_vsize() + tx_v3_child_large2["tx"].get_vsize(), 10000) assert_raises_rpc_error(-26, f"too-long-mempool-chain, exceeds ancestor size limit", node.sendrawtransaction, tx_v3_child_large2["hex"]) @@ -267,12 +268,12 @@ class MempoolTRUC(BitcoinTestFramework): node = self.nodes[0] tx_v3_parent_normal = self.wallet.create_self_transfer( fee_rate=0, - target_weight=4004, + target_vsize=1001, version=3 ) tx_v3_parent_2_normal = self.wallet.create_self_transfer( fee_rate=0, - target_weight=4004, + target_vsize=1001, version=3 ) tx_v3_child_multiparent = self.wallet.create_self_transfer_multi( @@ -282,7 +283,7 @@ class MempoolTRUC(BitcoinTestFramework): ) tx_v3_child_heavy = self.wallet.create_self_transfer_multi( utxos_to_spend=[tx_v3_parent_normal["new_utxo"]], - target_weight=4004, + target_vsize=TRUC_CHILD_MAX_VSIZE + 1, fee_per_output=10000, version=3 ) @@ -294,7 +295,7 @@ class MempoolTRUC(BitcoinTestFramework): self.check_mempool([]) result = node.submitpackage([tx_v3_parent_normal["hex"], tx_v3_child_heavy["hex"]]) - # tx_v3_child_heavy is heavy based on weight, not sigops. + # tx_v3_child_heavy is heavy based on vsize, not sigops. assert_equal(result['package_msg'], f"TRUC-violation, version=3 child tx {tx_v3_child_heavy['txid']} (wtxid={tx_v3_child_heavy['wtxid']}) is too big: {tx_v3_child_heavy['tx'].get_vsize()} > 1000 virtual bytes") self.check_mempool([]) @@ -416,7 +417,7 @@ class MempoolTRUC(BitcoinTestFramework): node = self.nodes[0] tx_v3_parent = self.wallet.create_self_transfer( fee_rate=0, - target_weight=4004, + target_vsize=1001, version=3 ) tx_v2_child = self.wallet.create_self_transfer_multi( diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 241aefab24..ee8c6c16ca 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -165,7 +165,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): node.p2ps[0].send_txs_and_test([rejected_parent], node, success=False) self.log.info('Test that a peer disconnection causes erase its transactions from the orphan pool') - with node.assert_debug_log(['Erased 100 orphan transaction(s) from peer=25']): + with node.assert_debug_log(['Erased 100 orphan transaction(s) from peer=26']): self.reconnect_p2p(num_connections=1) self.log.info('Test that a transaction in the orphan pool is included in a new tip block causes erase this transaction from the orphan pool') diff --git a/test/functional/rpc_getorphantxs.py b/test/functional/rpc_getorphantxs.py new file mode 100755 index 0000000000..8d32ce1638 --- /dev/null +++ b/test/functional/rpc_getorphantxs.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the getorphantxs RPC.""" + +from test_framework.mempool_util import tx_in_orphanage +from test_framework.messages import msg_tx +from test_framework.p2p import P2PInterface +from test_framework.util import assert_equal +from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet import MiniWallet + + +class GetOrphanTxsTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.test_orphan_activity() + self.test_orphan_details() + + def test_orphan_activity(self): + self.log.info("Check that orphaned transactions are returned with getorphantxs") + node = self.nodes[0] + + self.log.info("Create two 1P1C packages, but only broadcast the children") + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + tx_parent_2 = self.wallet.create_self_transfer() + tx_child_2 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_2["new_utxo"]) + peer = node.add_p2p_connection(P2PInterface()) + peer.send_and_ping(msg_tx(tx_child_1["tx"])) + peer.send_and_ping(msg_tx(tx_child_2["tx"])) + + self.log.info("Check that neither parent is in the mempool") + assert_equal(node.getmempoolinfo()["size"], 0) + + self.log.info("Check that both children are in the orphanage") + + orphanage = node.getorphantxs(verbosity=0) + self.log.info("Check the size of the orphanage") + assert_equal(len(orphanage), 2) + self.log.info("Check that negative verbosity is treated as 0") + assert_equal(orphanage, node.getorphantxs(verbosity=-1)) + assert tx_in_orphanage(node, tx_child_1["tx"]) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Broadcast parent 1") + peer.send_and_ping(msg_tx(tx_parent_1["tx"])) + self.log.info("Check that parent 1 and child 1 are in the mempool") + raw_mempool = node.getrawmempool() + assert_equal(len(raw_mempool), 2) + assert tx_parent_1["txid"] in raw_mempool + assert tx_child_1["txid"] in raw_mempool + + self.log.info("Check that orphanage only contains child 2") + orphanage = node.getorphantxs() + assert_equal(len(orphanage), 1) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + peer.send_and_ping(msg_tx(tx_parent_2["tx"])) + self.log.info("Check that all parents and children are now in the mempool") + raw_mempool = node.getrawmempool() + assert_equal(len(raw_mempool), 4) + assert tx_parent_1["txid"] in raw_mempool + assert tx_child_1["txid"] in raw_mempool + assert tx_parent_2["txid"] in raw_mempool + assert tx_child_2["txid"] in raw_mempool + self.log.info("Check that the orphanage is empty") + assert_equal(len(node.getorphantxs()), 0) + + self.log.info("Confirm the transactions (clears mempool)") + self.generate(node, 1) + assert_equal(node.getmempoolinfo()["size"], 0) + + def test_orphan_details(self): + self.log.info("Check the transaction details returned from getorphantxs") + node = self.nodes[0] + + self.log.info("Create two orphans, from different peers") + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + tx_parent_2 = self.wallet.create_self_transfer() + tx_child_2 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_2["new_utxo"]) + peer_1 = node.add_p2p_connection(P2PInterface()) + peer_2 = node.add_p2p_connection(P2PInterface()) + peer_1.send_and_ping(msg_tx(tx_child_1["tx"])) + peer_2.send_and_ping(msg_tx(tx_child_2["tx"])) + + orphanage = node.getorphantxs(verbosity=2) + assert tx_in_orphanage(node, tx_child_1["tx"]) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Check that orphan 1 and 2 were from different peers") + assert orphanage[0]["from"][0] != orphanage[1]["from"][0] + + self.log.info("Unorphan child 2") + peer_2.send_and_ping(msg_tx(tx_parent_2["tx"])) + assert not tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Checking orphan details") + orphanage = node.getorphantxs(verbosity=1) + assert_equal(len(node.getorphantxs()), 1) + orphan_1 = orphanage[0] + self.orphan_details_match(orphan_1, tx_child_1, verbosity=1) + + self.log.info("Checking orphan details (verbosity 2)") + orphanage = node.getorphantxs(verbosity=2) + orphan_1 = orphanage[0] + self.orphan_details_match(orphan_1, tx_child_1, verbosity=2) + + def orphan_details_match(self, orphan, tx, verbosity): + self.log.info("Check txid/wtxid of orphan") + assert_equal(orphan["txid"], tx["txid"]) + assert_equal(orphan["wtxid"], tx["wtxid"]) + + self.log.info("Check the sizes of orphan") + assert_equal(orphan["bytes"], len(tx["tx"].serialize())) + assert_equal(orphan["vsize"], tx["tx"].get_vsize()) + assert_equal(orphan["weight"], tx["tx"].get_weight()) + + if verbosity == 2: + self.log.info("Check the transaction hex of orphan") + assert_equal(orphan["hex"], tx["hex"]) + + +if __name__ == '__main__': + GetOrphanTxsTest(__file__).main() diff --git a/test/functional/test_framework/mempool_util.py b/test/functional/test_framework/mempool_util.py index fe47123e13..a6a7940c60 100644 --- a/test/functional/test_framework/mempool_util.py +++ b/test/functional/test_framework/mempool_util.py @@ -8,6 +8,7 @@ from decimal import Decimal from .blocktools import ( COINBASE_MATURITY, ) +from .messages import CTransaction from .util import ( assert_equal, assert_greater_than, @@ -83,3 +84,8 @@ def fill_mempool(test_framework, node, *, tx_sync_fun=None): test_framework.log.debug("Check that mempoolminfee is larger than minrelaytxfee") assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) + +def tx_in_orphanage(node, tx: CTransaction) -> bool: + """Returns true if the transaction is in the orphanage.""" + found = [o for o in node.getorphantxs(verbosity=1) if o["txid"] == tx.rehash() and o["wtxid"] == tx.getwtxid()] + return len(found) == 1 diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index f3713f297e..1cef714705 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -7,7 +7,6 @@ from copy import deepcopy from decimal import Decimal from enum import Enum -import math from typing import ( Any, Optional, @@ -35,7 +34,6 @@ from test_framework.messages import ( CTxOut, hash256, ser_compact_size, - WITNESS_SCALE_FACTOR, ) from test_framework.script import ( CScript, @@ -119,20 +117,18 @@ class MiniWallet: def _create_utxo(self, *, txid, vout, value, height, coinbase, confirmations): return {"txid": txid, "vout": vout, "value": value, "height": height, "coinbase": coinbase, "confirmations": confirmations} - def _bulk_tx(self, tx, target_weight): - """Pad a transaction with extra outputs until it reaches a target weight (or higher). + def _bulk_tx(self, tx, target_vsize): + """Pad a transaction with extra outputs until it reaches a target vsize. returns the tx """ tx.vout.append(CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN]))) - # determine number of needed padding bytes by converting weight difference to vbytes - dummy_vbytes = (target_weight - tx.get_weight() + 3) // 4 + # determine number of needed padding bytes + dummy_vbytes = target_vsize - tx.get_vsize() # compensate for the increase of the compact-size encoded script length # (note that the length encoding of the unpadded output script needs one byte) dummy_vbytes -= len(ser_compact_size(dummy_vbytes)) - 1 tx.vout[-1].scriptPubKey = CScript([OP_RETURN] + [OP_1] * dummy_vbytes) - # Actual weight should be at most 3 higher than target weight - assert_greater_than_or_equal(tx.get_weight(), target_weight) - assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) + assert_equal(tx.get_vsize(), target_vsize) def get_balance(self): return sum(u['value'] for u in self._utxos) @@ -309,7 +305,7 @@ class MiniWallet: locktime=0, sequence=0, fee_per_output=1000, - target_weight=0, + target_vsize=0, confirmed_only=False, ): """ @@ -338,8 +334,8 @@ class MiniWallet: self.sign_tx(tx) - if target_weight: - self._bulk_tx(tx, target_weight) + if target_vsize: + self._bulk_tx(tx, target_vsize) txid = tx.rehash() return { @@ -364,7 +360,7 @@ class MiniWallet: fee_rate=Decimal("0.003"), fee=Decimal("0"), utxo_to_spend=None, - target_weight=0, + target_vsize=0, confirmed_only=False, **kwargs, ): @@ -379,20 +375,18 @@ class MiniWallet: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) else: assert False - if target_weight and not fee: # respect fee_rate if target weight is passed - # the actual weight might be off by 3 WUs, so calculate based on that (see self._bulk_tx) - max_actual_weight = target_weight + 3 - fee = get_fee(math.ceil(max_actual_weight / WITNESS_SCALE_FACTOR), fee_rate) + if target_vsize and not fee: # respect fee_rate if target vsize is passed + fee = get_fee(target_vsize, fee_rate) send_value = utxo_to_spend["value"] - (fee or (fee_rate * vsize / 1000)) # create tx tx = self.create_self_transfer_multi( utxos_to_spend=[utxo_to_spend], amount_per_output=int(COIN * send_value), - target_weight=target_weight, + target_vsize=target_vsize, **kwargs, ) - if not target_weight: + if not target_vsize: assert_equal(tx["tx"].get_vsize(), vsize) tx["new_utxo"] = tx.pop("new_utxos")[0] diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 301f62d7b0..3d8c230066 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -160,6 +160,7 @@ BASE_SCRIPTS = [ 'wallet_importmulti.py --legacy-wallet', 'mempool_limit.py', 'rpc_txoutproof.py', + 'rpc_getorphantxs.py', 'wallet_listreceivedby.py --legacy-wallet', 'wallet_listreceivedby.py --descriptors', 'wallet_abandonconflict.py --legacy-wallet', @@ -446,8 +447,8 @@ def main(): help="Leave bitcoinds and test.* datadir on exit or error") parser.add_argument('--resultsfile', '-r', help='store test results (as CSV) to the provided file') - args, unknown_args = parser.parse_known_args() + fail_on_warn = args.ci if not args.ansi: global DEFAULT, BOLD, GREEN, RED DEFAULT = ("", "") @@ -524,8 +525,12 @@ def main(): # Remove the test cases that the user has explicitly asked to exclude. # The user can specify a test case with or without the .py extension. if args.exclude: + def print_warning_missing_test(test_name): - print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], test_name)) + print("{}WARNING!{} Test '{}' not found in current test list. Check the --exclude list.".format(BOLD[1], BOLD[0], test_name)) + if fail_on_warn: + sys.exit(1) + def remove_tests(exclude_list): if not exclude_list: print_warning_missing_test(exclude_test) @@ -562,7 +567,7 @@ def main(): f"A minimum of {MIN_NO_CLEANUP_SPACE // (1024 * 1024 * 1024)} GB of free space is required.") passon_args.append("--nocleanup") - check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=args.ci) + check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=fail_on_warn) check_script_prefixes() if not args.keepcache: @@ -871,7 +876,6 @@ def check_script_list(*, src_dir, fail_on_warn): if len(missed_tests) != 0: print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests))) if fail_on_warn: - # On CI this warning is an error to prevent merging incomplete commits into master sys.exit(1) diff --git a/test/functional/wallet_backwards_compatibility.py b/test/functional/wallet_backwards_compatibility.py index e71283b928..775786fbb1 100755 --- a/test/functional/wallet_backwards_compatibility.py +++ b/test/functional/wallet_backwards_compatibility.py @@ -33,7 +33,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 12 + self.num_nodes = 11 # Add new version after each release: self.extra_args = [ ["-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to mine blocks. noban for immediate tx relay @@ -47,7 +47,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.19.1 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1"], # v0.18.1 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1"], # v0.17.2 - ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1", "-wallet=wallet.dat"], # v0.16.3 ] self.wallet_names = [self.default_wallet_name] @@ -68,7 +67,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): 190100, 180100, 170200, - 160300, ]) self.start_nodes() @@ -133,18 +131,17 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): def run_test(self): node_miner = self.nodes[0] node_master = self.nodes[1] - node_v21 = self.nodes[self.num_nodes - 6] - node_v17 = self.nodes[self.num_nodes - 2] - node_v16 = self.nodes[self.num_nodes - 1] + node_v21 = self.nodes[self.num_nodes - 5] + node_v17 = self.nodes[self.num_nodes - 1] legacy_nodes = self.nodes[2:] # Nodes that support legacy wallets - legacy_only_nodes = self.nodes[-5:] # Nodes that only support legacy wallets - descriptors_nodes = self.nodes[2:-5] # Nodes that support descriptor wallets + legacy_only_nodes = self.nodes[-4:] # Nodes that only support legacy wallets + descriptors_nodes = self.nodes[2:-4] # Nodes that support descriptor wallets self.generatetoaddress(node_miner, COINBASE_MATURITY + 1, node_miner.getnewaddress()) # Sanity check the test framework: - res = node_v16.getblockchaininfo() + res = node_v17.getblockchaininfo() assert_equal(res['blocks'], COINBASE_MATURITY + 1) self.log.info("Test wallet backwards compatibility...") @@ -215,9 +212,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # In descriptors wallet mode, run this test on the nodes that support descriptor wallets # In legacy wallets mode, run this test on the nodes that support legacy wallets for node in descriptors_nodes if self.options.descriptors else legacy_nodes: - if self.major_version_less_than(node, 17): - # loadwallet was introduced in v0.17.0 - continue self.log.info(f"- {node.version}") for wallet_name in ["w1", "w2", "w3"]: if self.major_version_less_than(node, 18) and wallet_name == "w3": @@ -290,15 +284,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): node_v17.assert_start_raises_init_error(["-wallet=w3"], "Error: Error loading w3: Wallet requires newer version of Bitcoin Core") self.start_node(node_v17.index) - # No wallet created in master can be opened in 0.16 - self.log.info("Test that wallets created in master are too new for 0.16") - self.stop_node(node_v16.index) - for wallet_name in ["w1", "w2", "w3"]: - if self.options.descriptors: - node_v16.assert_start_raises_init_error([f"-wallet={wallet_name}"], f"Error: {wallet_name} corrupt, salvage failed") - else: - node_v16.assert_start_raises_init_error([f"-wallet={wallet_name}"], f"Error: Error loading {wallet_name}: Wallet requires newer version of Bitcoin Core") - # When descriptors are enabled, w1 cannot be opened by 0.21 since it contains a taproot descriptor if self.options.descriptors: self.log.info("Test that 0.21 cannot open wallet containing tr() descriptors") diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index ef3f925ee8..c909336a25 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -185,6 +185,7 @@ class UpgradeWalletTest(BitcoinTestFramework): self.restart_node(0) copy_v16() wallet = node_master.get_wallet_rpc(self.default_wallet_name) + assert_equal(wallet.getbalance(), v16_3_balance) self.log.info("Test upgradewallet without a version argument") self.test_upgradewallet(wallet, previous_version=159900, expected_version=169900) # wallet should still contain the same balance diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index fe845ed19e..52ae95fbb6 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -35,20 +35,20 @@ for commit in $(git rev-list --reverse "$1"); do git checkout --quiet "$commit"^ || exit SCRIPT="$(git rev-list --format=%b -n1 "$commit" | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d')" if test -z "$SCRIPT"; then - echo "Error: missing script for: $commit" - echo "Failed" + echo "Error: missing script for: $commit" >&2 + echo "Failed" >&2 RET=1 else - echo "Running script for: $commit" - echo "$SCRIPT" + echo "Running script for: $commit" >&2 + echo "$SCRIPT" >&2 (eval "$SCRIPT") - git --no-pager diff --exit-code "$commit" && echo "OK" || (echo "Failed"; false) || RET=1 + git --no-pager diff --exit-code "$commit" && echo "OK" >&2 || (echo "Failed" >&2; false) || RET=1 fi git reset --quiet --hard HEAD else if git rev-list "--format=%b" -n1 "$commit" | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then - echo "Error: script block marker but no scripted-diff in title of commit $commit" - echo "Failed" + echo "Error: script block marker but no scripted-diff in title of commit $commit" >&2 + echo "Failed" >&2 RET=1 fi fi diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index a809851ec6..86a17fb0f8 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -62,7 +62,7 @@ def main(): matching_files_filtered = [] for matching_file in matching_files: - if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|test/fuzz/strprintf.cpp)|contrib/devtools/bitcoin-tidy/example_logprintf.cpp', matching_file): + if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|test/fuzz/strprintf.cpp)', matching_file): matching_files_filtered.append(matching_file) matching_files_filtered.sort() diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index 64a1486a4c..42c880052e 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -29,7 +29,7 @@ fn get_linter_list() -> Vec<&'static Linter> { lint_fn: lint_doc }, &Linter { - description: "Check that no symbol from bitcoin-config.h is used without the header being included", + description: "Check that no symbol from bitcoin-build-config.h is used without the header being included", name: "includes_build_config", lint_fn: lint_includes_build_config }, @@ -395,7 +395,7 @@ Please add any false positives, such as subtrees, or externally sourced files to } fn lint_includes_build_config() -> LintResult { - let config_path = "./cmake/bitcoin-config.h.in"; + let config_path = "./cmake/bitcoin-build-config.h.in"; let defines_regex = format!( r"^\s*(?!//).*({})", check_output(Command::new("grep").args(["define", "--", config_path])) @@ -429,7 +429,7 @@ fn lint_includes_build_config() -> LintResult { ]) .args(get_pathspecs_exclude_subtrees()) .args([ - // These are exceptions which don't use bitcoin-config.h, rather the Makefile.am adds + // These are exceptions which don't use bitcoin-build-config.h, rather CMakeLists.txt adds // these cppflags manually. ":(exclude)src/crypto/sha256_arm_shani.cpp", ":(exclude)src/crypto/sha256_avx2.cpp", @@ -447,9 +447,9 @@ fn lint_includes_build_config() -> LintResult { "--files-with-matches" }, if mode { - "^#include <config/bitcoin-config.h> // IWYU pragma: keep$" + "^#include <bitcoin-build-config.h> // IWYU pragma: keep$" } else { - "#include <config/bitcoin-config.h>" // Catch redundant includes with and without the IWYU pragma + "#include <bitcoin-build-config.h>" // Catch redundant includes with and without the IWYU pragma }, "--", ]) @@ -463,11 +463,11 @@ fn lint_includes_build_config() -> LintResult { return Err(format!( r#" ^^^ -One or more files use a symbol declared in the bitcoin-config.h header. However, they are not +One or more files use a symbol declared in the bitcoin-build-config.h header. However, they are not including the header. This is problematic, because the header may or may not be indirectly included. If the indirect include were to be intentionally or accidentally removed, the build could still succeed, but silently be buggy. For example, a slower fallback algorithm could be picked, -even though bitcoin-config.h indicates that a faster feature is available and should be used. +even though bitcoin-build-config.h indicates that a faster feature is available and should be used. If you are unsure which symbol is used, you can find it with this command: git grep --perl-regexp '{}' -- file_name @@ -475,7 +475,7 @@ git grep --perl-regexp '{}' -- file_name Make sure to include it with the IWYU pragma. Otherwise, IWYU may falsely instruct to remove the include again. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep "#, defines_regex )); @@ -484,7 +484,7 @@ include again. if redundant { return Err(r#" ^^^ -None of the files use a symbol declared in the bitcoin-config.h header. However, they are including +None of the files use a symbol declared in the bitcoin-build-config.h header. However, they are including the header. Consider removing the unused include. "# .to_string()); diff --git a/test/util/test_runner.py b/test/util/test_runner.py index e4a77d53d6..cac184ca30 100755 --- a/test/util/test_runner.py +++ b/test/util/test_runner.py @@ -83,13 +83,11 @@ def bctest(testDir, testObj, buildenv): execrun = [execprog] + execargs # Read the input data (if there is any) - stdinCfg = None inputData = None if "input" in testObj: filename = os.path.join(testDir, testObj["input"]) with open(filename, encoding="utf8") as f: inputData = f.read() - stdinCfg = subprocess.PIPE # Read the expected output data (if there is any) outputFn = None @@ -112,9 +110,8 @@ def bctest(testDir, testObj, buildenv): raise Exception # Run the test - proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) try: - outs = proc.communicate(input=inputData) + res = subprocess.run(execrun, capture_output=True, text=True, input=inputData) except OSError: logging.error("OSError, Failed to execute " + execprog) raise @@ -123,9 +120,9 @@ def bctest(testDir, testObj, buildenv): data_mismatch, formatting_mismatch = False, False # Parse command output and expected output try: - a_parsed = parse_output(outs[0], outputType) + a_parsed = parse_output(res.stdout, outputType) except Exception as e: - logging.error('Error parsing command output as %s: %s' % (outputType, e)) + logging.error(f"Error parsing command output as {outputType}: '{str(e)}'; res: {str(res)}") raise try: b_parsed = parse_output(outputData, outputType) @@ -134,13 +131,13 @@ def bctest(testDir, testObj, buildenv): raise # Compare data if a_parsed != b_parsed: - logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") + logging.error(f"Output data mismatch for {outputFn} (format {outputType}); res: {str(res)}") data_mismatch = True # Compare formatting - if outs[0] != outputData: - error_message = "Output formatting mismatch for " + outputFn + ":\n" + if res.stdout != outputData: + error_message = f"Output formatting mismatch for {outputFn}:\nres: {str(res)}\n" error_message += "".join(difflib.context_diff(outputData.splitlines(True), - outs[0].splitlines(True), + res.stdout.splitlines(True), fromfile=outputFn, tofile="returned")) logging.error(error_message) @@ -152,8 +149,8 @@ def bctest(testDir, testObj, buildenv): wantRC = 0 if "return_code" in testObj: wantRC = testObj['return_code'] - if proc.returncode != wantRC: - logging.error("Return code mismatch for " + outputFn) + if res.returncode != wantRC: + logging.error(f"Return code mismatch for {outputFn}; res: {str(res)}") raise Exception if "error_txt" in testObj: @@ -164,8 +161,8 @@ def bctest(testDir, testObj, buildenv): # emits DISPLAY errors when running as a windows application on # linux through wine. Just assert that the expected error text appears # somewhere in stderr. - if want_error not in outs[1]: - logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip()) + if want_error not in res.stderr: + logging.error(f"Error mismatch:\nExpected: {want_error}\nReceived: {res.stderr.rstrip()}\nres: {str(res)}") raise Exception def parse_output(a, fmt): |