diff options
382 files changed, 8833 insertions, 3745 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a86b00fdc6..b92e3c66d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD ^$(git rev-list -n1 --merges HEAD)^@ | head -1)" >> "$GITHUB_ENV" - run: | sudo apt-get update - sudo apt-get install clang-15 ccache build-essential libtool autotools-dev automake pkg-config bsdmainutils python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libnatpmp-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y + sudo apt-get install clang-15 ccache build-essential libtool autotools-dev automake pkg-config bsdmainutils python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libnatpmp-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y - name: Compile and run tests run: | # Run tests on commits after the last merge commit and before the PR head commit @@ -96,7 +96,10 @@ jobs: - name: Install Homebrew packages env: HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 - run: brew install automake libtool pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq qt@5 qrencode + run: | + # A workaround for "The `brew link` step did not complete successfully" error. + brew install python@3 || brew link --overwrite python@3 + brew install automake libtool pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq qt@5 qrencode - name: Set Ccache directory run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" @@ -157,6 +160,7 @@ jobs: $env:CI_QT_URL | Out-File -FilePath "$env:GITHUB_WORKSPACE\qt_url" $env:CI_QT_CONF | Out-File -FilePath "$env:GITHUB_WORKSPACE\qt_conf" py -3 --version + Write-Host "PowerShell version $($PSVersionTable.PSVersion.ToString())" - name: Restore static Qt cache id: static-qt-cache diff --git a/.gitignore b/.gitignore index c77303f50e..f7bcbd1459 100644 --- a/.gitignore +++ b/.gitignore @@ -130,12 +130,12 @@ win32-build test/config.ini test/cache/* test/.mypy_cache/ +test/lint/test_runner/target/ !src/leveldb*/Makefile /doc/doxygen/ -libbitcoinconsensus.pc contrib/devtools/split-debug.sh # Output from running db4 installation @@ -144,7 +144,6 @@ db4/ # clang-check *.plist -osx_volname dist/ /guix-build-* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ae4ff1a92..0ac6db76ed 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,9 +66,10 @@ Discussion about codebase improvements happens in GitHub issues and pull requests. The developer -[mailing list](https://lists.linuxfoundation.org/mailman/listinfo/bitcoin-dev) +[mailing list](https://groups.google.com/g/bitcoindev) should be used to discuss complicated or controversial consensus or P2P protocol changes before working on a patch set. +Archives can be found on [https://gnusha.org/pi/bitcoindev/](https://gnusha.org/pi/bitcoindev/). Contributor Workflow diff --git a/Makefile.am b/Makefile.am index eec498dc0e..e79dc4e21b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -14,11 +14,6 @@ endif .PHONY: deploy FORCE .INTERMEDIATE: $(COVERAGE_INFO) -if BUILD_BITCOIN_LIBS -pkgconfigdir = $(libdir)/pkgconfig -pkgconfig_DATA = libbitcoinconsensus.pc -endif - BITCOIND_BIN=$(top_builddir)/src/$(BITCOIN_DAEMON_NAME)$(EXEEXT) BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT) BITCOIN_TEST_BIN=$(top_builddir)/src/test/$(BITCOIN_TEST_NAME)$(EXEEXT) @@ -118,9 +113,6 @@ OSX_APP_BUILT=$(OSX_APP)/Contents/PkgInfo $(OSX_APP)/Contents/Resources/empty.lp $(OSX_APP)/Contents/Resources/bitcoin.icns $(OSX_APP)/Contents/Info.plist \ $(OSX_APP)/Contents/MacOS/Bitcoin-Qt $(OSX_APP)/Contents/Resources/Base.lproj/InfoPlist.strings -osx_volname: - echo $(OSX_VOLNAME) >$@ - if BUILD_DARWIN $(OSX_ZIP): $(OSX_APP_BUILT) $(OSX_PACKAGING) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -zip @@ -134,7 +126,7 @@ $(OSX_ZIP): deploydir cd $(APP_DIST_DIR) && find . | sort | $(ZIP) -X@ $@ $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING) - INSTALL_NAME_TOOL=$(INSTALL_NAME_TOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) + OTOOL=$(OTOOL) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) deploydir: $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt endif !BUILD_DARWIN @@ -338,7 +330,7 @@ clean-docs: clean-local: clean-docs rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ fuzz.coverage/ test/tmp/ cache/ $(OSX_APP) rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__ - rm -rf osx_volname dist/ + rm -rf dist/ test/lint/test_runner/target/ test/lint/__pycache__ test-security-check: if TARGET_DARWIN diff --git a/build-aux/m4/l_atomic.m4 b/build-aux/m4/l_atomic.m4 index aa00168fce..859ddaabbb 100644 --- a/build-aux/m4/l_atomic.m4 +++ b/build-aux/m4/l_atomic.m4 @@ -7,7 +7,7 @@ dnl warranty. # Clang, when building for 32-bit, # and linking against libstdc++, requires linking with # -latomic if using the C++ atomic library. -# Can be tested with: clang++ test.cpp -m32 +# Can be tested with: clang++ -std=c++20 test.cpp -m32 # # Sourced from http://bugs.debian.org/797228 @@ -27,8 +27,11 @@ m4_define([_CHECK_ATOMIC_testbody], [[ auto t1 = t.load(); t.compare_exchange_strong(t1, 3s); - std::atomic<int64_t> a{}; + std::atomic<double> d{}; + d.store(3.14); + auto d1 = d.load(); + std::atomic<int64_t> a{}; int64_t v = 5; int64_t r = a.fetch_add(v); return static_cast<int>(r); diff --git a/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj b/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj index 95fdcdb79b..a34ef41d16 100644 --- a/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj +++ b/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj @@ -15,7 +15,6 @@ <ClCompile Include="..\..\src\primitives\block.cpp" /> <ClCompile Include="..\..\src\primitives\transaction.cpp" /> <ClCompile Include="..\..\src\pubkey.cpp" /> - <ClCompile Include="..\..\src\script\bitcoinconsensus.cpp" /> <ClCompile Include="..\..\src\script\interpreter.cpp" /> <ClCompile Include="..\..\src\script\script.cpp" /> <ClCompile Include="..\..\src\script\script_error.cpp" /> diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 47cb8864c2..6b12c53f2a 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -46,7 +46,7 @@ if [ ! -d "${LINT_RUNNER_PATH}" ]; then fi ${CI_RETRY_EXE} pip3 install \ - codespell==2.2.5 \ + codespell==2.2.6 \ flake8==6.1.0 \ lief==0.13.2 \ mypy==1.4.1 \ diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index 318b2bb819..cdf0f60147 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -8,21 +8,24 @@ export LC_ALL=C set -ex -if [ -n "$LOCAL_BRANCH" ]; then - # To faithfully recreate CI linting locally, specify all commits on the current - # branch. - COMMIT_RANGE="$(git merge-base HEAD master)..HEAD" -elif [ -n "$CIRRUS_PR" ]; then +if [ -n "$CIRRUS_PR" ]; then COMMIT_RANGE="HEAD~..HEAD" - echo - git log --no-merges --oneline "$COMMIT_RANGE" - echo - test/lint/commit-script-check.sh "$COMMIT_RANGE" + if [ "$(git rev-list -1 HEAD)" != "$(git rev-list -1 --merges HEAD)" ]; then + echo "Error: The top commit must be a merge commit, usually the remote 'pull/${PR_NUMBER}/merge' branch." + false + fi else - COMMIT_RANGE="SKIP_EMPTY_NOT_A_PR" + # Otherwise, assume that a merge commit exists. This merge commit is assumed + # to be the base, after which linting will be done. If the merge commit is + # HEAD, the range will be empty. + COMMIT_RANGE="$( git rev-list --max-count=1 --merges HEAD )..HEAD" fi export COMMIT_RANGE +echo +git log --no-merges --oneline "$COMMIT_RANGE" +echo +test/lint/commit-script-check.sh "$COMMIT_RANGE" RUST_BACKTRACE=1 "${LINT_RUNNER_PATH}/test_runner" if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; then diff --git a/ci/lint/container-entrypoint.sh b/ci/lint/container-entrypoint.sh index a403f923a2..c8519a3912 100755 --- a/ci/lint/container-entrypoint.sh +++ b/ci/lint/container-entrypoint.sh @@ -14,7 +14,7 @@ export PATH="/python_build/bin:${PATH}" export LINT_RUNNER_PATH="/lint_test_runner" if [ -z "$1" ]; then - LOCAL_BRANCH=1 bash -ic "./ci/lint/06_script.sh" + bash -ic "./ci/lint/06_script.sh" else exec "$@" fi diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 65d37f01d9..c80036164f 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,7 +10,7 @@ export HOST=arm-linux-gnueabihf export DPKG_ADD_ARCH="armhf" export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" export CONTAINER_NAME=ci_arm_linux -export CI_IMAGE_NAME_TAG="docker.io/arm64v8/debian:bookworm" +export CI_IMAGE_NAME_TAG="docker.io/arm64v8/debian:bookworm" # Check that https://packages.debian.org/bookworm/g++-arm-linux-gnueabihf (version 12.2, similar to guix) can cross-compile export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 0d18e2f029..00a4d781c2 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -8,10 +8,11 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_multiprocess -export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:22.04" +export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:24.04" export PACKAGES="llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" +export TEST_RUNNER_EXTRA="--v2transport" export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' \ CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE'" export BITCOIND=bitcoin-node # Used in functional tests diff --git a/ci/test/00_setup_env_mac_native.sh b/ci/test/00_setup_env_mac_native.sh index 439fba16ef..c47f13f96e 100755 --- a/ci/test/00_setup_env_mac_native.sh +++ b/ci/test/00_setup_env_mac_native.sh @@ -7,7 +7,9 @@ export LC_ALL=C.UTF-8 export HOST=x86_64-apple-darwin -export PIP_PACKAGES="zmq" +# Homebrew's python@3.12 is marked as externally managed (PEP 668). +# Therefore, `--break-system-packages` is needed. +export PIP_PACKAGES="--break-system-packages zmq" export GOAL="install" export BITCOIN_CONFIG="--with-gui --with-miniupnpc --with-natpmp --enable-reduce-exports" export CI_OS_NAME="macos" diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 60486f8f61..668e9ecc8a 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -17,10 +17,10 @@ else fi export CONTAINER_NAME=ci_native_asan -export PACKAGES="systemtap-sdt-dev clang-17 llvm-17 libclang-rt-17-dev python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" +export PACKAGES="systemtap-sdt-dev clang-18 llvm-18 libclang-rt-18-dev python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \ +export BITCOIN_CONFIG="--enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \ CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \ --with-sanitizers=address,float-divide-by-zero,integer,undefined \ -CC='clang-17 -ftrivial-auto-var-init=pattern' CXX='clang++-17 -ftrivial-auto-var-init=pattern'" +CC='clang-18 -ftrivial-auto-var-init=pattern' CXX='clang++-18 -ftrivial-auto-var-init=pattern'" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index abee3c1541..f50561f875 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_fuzz -export PACKAGES="clang-17 llvm-17 libclang-rt-17-dev libevent-dev libboost-dev libsqlite3-dev" +export PACKAGES="clang-18 llvm-18 libclang-rt-18-dev libevent-dev libboost-dev libsqlite3-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false @@ -16,6 +16,6 @@ export RUN_FUZZ_TESTS=true export GOAL="install" export CI_CONTAINER_CAP="--cap-add SYS_PTRACE" # If run with (ASan + LSan), the container needs access to ptrace (https://github.com/google/sanitizers/issues/764) export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,float-divide-by-zero,integer \ -CC='clang-17 -ftrivial-auto-var-init=pattern' CXX='clang++-17 -ftrivial-auto-var-init=pattern'" +CC='clang-18 -ftrivial-auto-var-init=pattern' CXX='clang++-18 -ftrivial-auto-var-init=pattern'" export CCACHE_MAXSIZE=200M -export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-17" +export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-18" diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 0b32049013..f1c358082d 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" @@ -17,7 +17,8 @@ export PACKAGES="ninja-build" # BDB generates false-positives and will be removed in future export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +# _FORTIFY_SOURCE is not compatible with MSAN. +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE -U_FORTIFY_SOURCE'" export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 4f80d7ed42..bf4d1573e3 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -6,15 +6,14 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="docker.io/debian:bookworm" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="clang llvm libclang-rt-dev libevent-dev libboost-dev libsqlite3-dev valgrind" +export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libevent-dev libboost-dev libsqlite3-dev valgrind" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" -# Temporarily pin dwarf 4, until using Valgrind 3.20 or later -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CFLAGS=-gdwarf-4 CXXFLAGS=-gdwarf-4" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang-16 CXX=clang++-16" export CCACHE_MAXSIZE=200M diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 60987f5011..dd465cac2e 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" @@ -17,7 +17,8 @@ export PACKAGES="ninja-build" # BDB generates false-positives and will be removed in future export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +# _FORTIFY_SOURCE is not compatible with MSAN. +export BITCOIN_CONFIG="--with-sanitizers=memory CPPFLAGS='-U_FORTIFY_SOURCE'" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" export CCACHE_MAXSIZE=250M diff --git a/ci/test/00_setup_env_native_previous_releases.sh b/ci/test/00_setup_env_native_previous_releases.sh index 94e88f872f..9da3b18999 100755 --- a/ci/test/00_setup_env_native_previous_releases.sh +++ b/ci/test/00_setup_env_native_previous_releases.sh @@ -7,14 +7,14 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_previous_releases -export CI_IMAGE_NAME_TAG="docker.io/debian:bullseye" -# Use minimum supported python3.9 and gcc-10, see doc/dependencies.md -export PACKAGES="gcc-10 g++-10 python3-zmq" -export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 CC=gcc-10 CXX=g++-10" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +# Use minimum supported python3.9 (or best effort 3.10) and gcc-11, see doc/dependencies.md +export PACKAGES="gcc-11 g++-11 python3-zmq" +export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 CC=gcc-11 CXX=g++-11" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" export DOWNLOAD_PREVIOUS_RELEASES="true" -export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports --enable-debug \ +export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports --enable-debug \ CFLAGS=\"-g0 -O2 -funsigned-char\" CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS=\"-g0 -O2 -funsigned-char\"" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index c12044f461..5f422bbdb6 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -8,13 +8,13 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_tidy -export TIDY_LLVM_V="17" -export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq bear libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" +export TIDY_LLVM_V="18" +export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq bear libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev qtbase5-dev qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=false export RUN_TIDY=true export GOAL="install" -export BITCOIN_CONFIG="CC=clang-${TIDY_LLVM_V} CXX=clang++-${TIDY_LLVM_V} --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0 -I/usr/lib/llvm-${TIDY_LLVM_V}/lib/clang/${TIDY_LLVM_V}/include'" +export BITCOIN_CONFIG="CC=clang-${TIDY_LLVM_V} CXX=clang++-${TIDY_LLVM_V} --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0'" export CCACHE_MAXSIZE=200M diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index aa23bad809..3fcaa8c6c6 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" -export PACKAGES="clang-17 llvm-17 libclang-rt-17-dev libc++abi-17-dev libc++-17-dev python3-zmq" -export DEP_OPTS="CC=clang-17 CXX='clang++-17 -stdlib=libc++'" +export PACKAGES="clang-18 llvm-18 libclang-rt-18-dev libc++abi-18-dev libc++-18-dev python3-zmq" +export DEP_OPTS="CC=clang-18 CXX='clang++-18 -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' CXXFLAGS='-g' --with-sanitizers=thread" +export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' --with-sanitizers=thread" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 9bdb2b7860..720e522a6a 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -6,12 +6,11 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="docker.io/debian:bookworm" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang llvm libclang-rt-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" +export PACKAGES="valgrind clang-16 llvm-16 libclang-rt-16-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" -# Temporarily pin dwarf 4, until using Valgrind 3.20 or later -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++ CFLAGS=-gdwarf-4 CXXFLAGS=-gdwarf-4" # TODO enable GUI +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang-16 CXX=clang++-16" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index ca84ecce51..2fd94e253c 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -9,8 +9,8 @@ export LC_ALL=C.UTF-8 export HOST=s390x-linux-gnu export PACKAGES="python3-zmq" export CONTAINER_NAME=ci_s390x -export CI_IMAGE_NAME_TAG="docker.io/s390x/debian:bookworm" -export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export CI_IMAGE_NAME_TAG="docker.io/s390x/ubuntu:24.04" +export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true export GOAL="install" export BITCOIN_CONFIG="--enable-reduce-exports" diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index ba9af480ac..bce5ee74b6 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:22.04" # Check that Jammy can cross-compile to win64 +export CI_IMAGE_NAME_TAG="docker.io/amd64/debian:bookworm" # Check that https://packages.debian.org/bookworm/g++-mingw-w64-x86-64-posix (version 12.2, similar to guix) can cross-compile export HOST=x86_64-w64-mingw32 export DPKG_ADD_ARCH="i386" export PACKAGES="nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 99813da106..25962a53e5 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2022 The Bitcoin Core developers +# Copyright (c) 2018-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -36,7 +36,7 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-17.0.6 /msan/llvm-project + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-18.1.3" /msan/llvm-project cmake -G Ninja -B /msan/clang_build/ \ -DLLVM_ENABLE_PROJECTS="clang" \ @@ -53,13 +53,14 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /msan/clang_build/bin/llvm-symbolizer 100 cmake -G Ninja -B /msan/cxx_build/ \ - -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_USE_SANITIZER=MemoryWithOrigins \ -DCMAKE_C_COMPILER=clang \ -DCMAKE_CXX_COMPILER=clang++ \ -DLLVM_TARGETS_TO_BUILD=Native \ -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ + -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \ -DLIBCXX_HARDENING_MODE=debug \ -S /msan/llvm-project/runtimes diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index 786cb08bf6..f5da7bc55d 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -10,7 +10,7 @@ set -ex export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" -export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" +export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" if [ "$CI_OS_NAME" == "macos" ]; then @@ -71,8 +71,6 @@ elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ] fi fi -mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" - if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) @@ -182,7 +180,11 @@ if [ "${RUN_TIDY}" = "true" ]; then set -eo pipefail cd "${BASE_BUILD_DIR}/bitcoin-$HOST/src/" - ( run-clang-tidy-"${TIDY_LLVM_V}" -quiet -load="/tidy-build/libbitcoin-tidy.so" "${MAKEJOBS}" ) | grep -C5 "error" + if ! ( run-clang-tidy-"${TIDY_LLVM_V}" -quiet -load="/tidy-build/libbitcoin-tidy.so" "${MAKEJOBS}" | tee tmp.tidy-out.txt ); then + grep -C5 "error: " tmp.tidy-out.txt + echo "^^^ ⚠️ Failure generated from clang-tidy" + false + fi # Filter out files by regex here, because regex may not be # accepted in src/.bear-tidy-config # Filter out: diff --git a/configure.ac b/configure.ac index aeb8b71437..c7d124a1f1 100644 --- a/configure.ac +++ b/configure.ac @@ -307,9 +307,9 @@ AC_ARG_ENABLE([werror], [enable_werror=no]) AC_ARG_ENABLE([external-signer], - [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is auto, requires Boost::Process)])], + [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is yes)])], [use_external_signer=$enableval], - [use_external_signer=auto]) + [use_external_signer=yes]) AC_LANG_PUSH([C++]) @@ -626,15 +626,9 @@ AC_ARG_ENABLE([experimental-util-chainstate], [build_bitcoin_chainstate=$enableval], [build_bitcoin_chainstate=no]) -AC_ARG_WITH([libs], - [AS_HELP_STRING([--with-libs], - [build libraries (default=yes)])], - [build_bitcoin_libs=$withval], - [build_bitcoin_libs=yes]) - AC_ARG_WITH([experimental-kernel-lib], [AS_HELP_STRING([--with-experimental-kernel-lib], - [build experimental bitcoinkernel library (default is to build if we're building libraries and the experimental build-chainstate executable)])], + [build experimental bitcoinkernel library (default is to build if we're building the experimental build-chainstate executable)])], [build_experimental_kernel_lib=$withval], [build_experimental_kernel_lib=auto]) @@ -702,6 +696,9 @@ case $host in TARGET_OS=darwin if test $cross_compiling != "yes"; then BUILD_OS=darwin + + AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR]) + AC_CHECK_PROG([BREW], [brew], [brew]) if test "$BREW" = "brew"; then dnl These Homebrew packages may be keg-only, meaning that they won't be found @@ -765,7 +762,6 @@ case $host in ;; *) AC_PATH_TOOL([DSYMUTIL], [dsymutil], [dsymutil]) - AC_PATH_TOOL([INSTALL_NAME_TOOL], [install_name_tool], [install_name_tool]) AC_PATH_TOOL([OTOOL], [otool], [otool]) AC_PATH_PROG([ZIP], [zip], [zip]) @@ -778,7 +774,6 @@ case $host in esac fi - AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR]) CORE_CPPFLAGS="$CORE_CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" dnl ignore deprecated-declarations warnings coming from objcxx code @@ -974,24 +969,6 @@ AC_CHECK_DECLS([setsid]) AC_CHECK_DECLS([pipe2]) -AC_CHECK_FUNCS([timingsafe_bcmp]) - -AC_MSG_CHECKING([for __builtin_clzl]) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ - (void) __builtin_clzl(0); - ]])], - [ AC_MSG_RESULT([yes]); have_clzl=yes; AC_DEFINE([HAVE_BUILTIN_CLZL], [1], [Define this symbol if you have __builtin_clzl])], - [ AC_MSG_RESULT([no]); have_clzl=no;] -) - -AC_MSG_CHECKING([for __builtin_clzll]) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ - (void) __builtin_clzll(0); - ]])], - [ AC_MSG_RESULT([yes]); have_clzll=yes; AC_DEFINE([HAVE_BUILTIN_CLZLL], [1], [Define this symbol if you have __builtin_clzll])], - [ AC_MSG_RESULT([no]); have_clzll=no;] -) - dnl Check for malloc_info (for memory statistics information in getmemoryinfo) AC_MSG_CHECKING([for getmemoryinfo]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <malloc.h>]], @@ -1092,22 +1069,6 @@ if test "$use_thread_local" = "yes" || test "$use_thread_local" = "auto"; then LDFLAGS="$TEMP_LDFLAGS" fi -dnl check for gmtime_r(), fallback to gmtime_s() if that is unavailable -dnl fail if neither are available. -AC_MSG_CHECKING([for gmtime_r]) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ctime>]], - [[ gmtime_r((const time_t *) nullptr, (struct tm *) nullptr); ]])], - [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GMTIME_R], [1], [Define this symbol if gmtime_r is available]) ], - [ AC_MSG_RESULT([no]); - AC_MSG_CHECKING([for gmtime_s]); - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ctime>]], - [[ gmtime_s((struct tm *) nullptr, (const time_t *) nullptr); ]])], - [ AC_MSG_RESULT([yes])], - [ AC_MSG_RESULT([no]); AC_MSG_ERROR([Both gmtime_r and gmtime_s are unavailable]) ] - ) - ] -) - dnl Check for different ways of gathering OS randomness AC_MSG_CHECKING([for Linux getrandom function]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ @@ -1198,10 +1159,23 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ getauxval(AT_HWCAP); ]])], - [ AC_MSG_RESULT([yes]); HAVE_STRONG_GETAUXVAL=1; AC_DEFINE([HAVE_STRONG_GETAUXVAL], [1], [Define this symbol to build code that uses getauxval)]) ], + [ AC_MSG_RESULT([yes]); HAVE_STRONG_GETAUXVAL=1; AC_DEFINE([HAVE_STRONG_GETAUXVAL], [1], [Define this symbol to build code that uses getauxval]) ], [ AC_MSG_RESULT([no]); HAVE_STRONG_GETAUXVAL=0 ] ) +# Check for UNIX sockets +AC_MSG_CHECKING(for sockaddr_un) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include <sys/socket.h> + #include <sys/un.h> + ]], [[ + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + ]])], + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_SOCKADDR_UN], [1], [Define this symbol if platform supports unix domain sockets]) ], + [ AC_MSG_RESULT([no]); ] +) + have_any_system=no AC_MSG_CHECKING([for std::system]) AC_LINK_IFELSE( @@ -1249,7 +1223,6 @@ if test "$enable_fuzz" = "yes"; then build_bitcoin_chainstate=no build_bitcoin_wallet=no build_bitcoind=no - build_bitcoin_libs=no bitcoin_enable_qt=no bitcoin_enable_qt_test=no bitcoin_enable_qt_dbus=no @@ -1408,7 +1381,7 @@ if test "$use_boost" = "yes"; then dnl Check for Boost headers AX_BOOST_BASE([1.73.0],[],[AC_MSG_ERROR([Boost is not available!])]) if test "$want_boost" = "no"; then - AC_MSG_ERROR([only libbitcoinconsensus can be built without Boost]) + AC_MSG_ERROR([Boost is required]) fi dnl we don't use multi_index serialization @@ -1426,56 +1399,14 @@ if test "$use_boost" = "yes"; then fi fi -if test "$use_external_signer" != "no"; then - AC_MSG_CHECKING([whether Boost.Process can be used]) - TEMP_CXXFLAGS="$CXXFLAGS" - dnl Boost 1.78 requires the following workaround. - dnl See: https://github.com/boostorg/process/issues/235 - CXXFLAGS="$CXXFLAGS -Wno-error=narrowing" - TEMP_CPPFLAGS="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - TEMP_LDFLAGS="$LDFLAGS" - dnl Boost 1.73 and older require the following workaround. - LDFLAGS="$LDFLAGS $PTHREAD_CFLAGS" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[ - #define BOOST_PROCESS_USE_STD_FS - #include <boost/process.hpp> - ]],[[ - namespace bp = boost::process; - bp::opstream stdin_stream; - bp::ipstream stdout_stream; - bp::child c("dummy", bp::std_out > stdout_stream, bp::std_err > stdout_stream, bp::std_in < stdin_stream); - stdin_stream << std::string{"test"} << std::endl; - if (c.running()) c.terminate(); - c.wait(); - c.exit_code(); - ]])], - [have_boost_process="yes"], - [have_boost_process="no"]) - LDFLAGS="$TEMP_LDFLAGS" - CPPFLAGS="$TEMP_CPPFLAGS" - CXXFLAGS="$TEMP_CXXFLAGS" - AC_MSG_RESULT([$have_boost_process]) - if test "$have_boost_process" = "yes"; then - case $host in - dnl Boost Process for Windows uses Boost ASIO. Boost ASIO performs - dnl pre-main init of Windows networking libraries, which we do not - dnl want. - *mingw*) - use_external_signer="no" - ;; - *) - use_external_signer="yes" - AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled]) - AC_DEFINE([BOOST_PROCESS_USE_STD_FS], [1], [Defined to avoid Boost::Process trying to use Boost Filesystem]) - ;; - esac - else - if test "$use_external_signer" = "yes"; then - AC_MSG_ERROR([External signing is not supported for this Boost version]) - fi - use_external_signer="no"; - fi +case $host in + dnl Re-enable it after enabling Windows support in cpp-subprocess. + *mingw*) + use_external_signer="no" + ;; +esac +if test "$use_external_signer" = "yes"; then + AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled]) fi AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "$use_external_signer" = "yes"]) @@ -1622,18 +1553,8 @@ fi AM_CONDITIONAL([BUILD_BITCOIN_CHAINSTATE], [test $build_bitcoin_chainstate = "yes"]) AC_MSG_RESULT($build_bitcoin_chainstate) -AC_MSG_CHECKING([whether to build libraries]) -AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test $build_bitcoin_libs = "yes"]) - -if test "$build_bitcoin_libs" = "yes"; then - AC_DEFINE([HAVE_CONSENSUS_LIB], [1], [Define this symbol if the consensus lib has been built]) - AC_CONFIG_FILES([libbitcoinconsensus.pc:libbitcoinconsensus.pc.in]) -fi - AM_CONDITIONAL([BUILD_BITCOIN_KERNEL_LIB], [test "$build_experimental_kernel_lib" != "no" && ( test "$build_experimental_kernel_lib" = "yes" || test "$build_bitcoin_chainstate" = "yes" )]) -AC_MSG_RESULT($build_bitcoin_libs) - AC_LANG_POP if test "$use_ccache" != "no"; then @@ -1767,8 +1688,8 @@ else AC_MSG_RESULT([no]) fi -if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_bench$use_tests" = "nononononononononono"; then - AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-fuzz(-binary) --enable-bench or --enable-tests]) +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_bench$use_tests" = "nonononononononono"; then + AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-daemon --with-gui --enable-fuzz(-binary) --enable-bench or --enable-tests]) fi AM_CONDITIONAL([TARGET_DARWIN], [test "$TARGET_OS" = "darwin"]) @@ -1786,7 +1707,6 @@ AM_CONDITIONAL([ENABLE_QT_TESTS], [test "$BUILD_TEST_QT" = "yes"]) AM_CONDITIONAL([ENABLE_BENCH], [test "$use_bench" = "yes"]) AM_CONDITIONAL([USE_QRCODE], [test "$use_qr" = "yes"]) AM_CONDITIONAL([USE_LCOV], [test "$use_lcov" = "yes"]) -AM_CONDITIONAL([USE_LIBEVENT], [test "$use_libevent" = "yes"]) AM_CONDITIONAL([HARDEN], [test "$use_hardening" = "yes"]) AM_CONDITIONAL([ENABLE_SSE42], [test "$enable_sse42" = "yes"]) AM_CONDITIONAL([ENABLE_SSE41], [test "$enable_sse41" = "yes"]) @@ -1800,7 +1720,6 @@ AM_CONDITIONAL([USE_UPNP], [test "$use_upnp" = "yes"]) dnl for minisketch AM_CONDITIONAL([ENABLE_CLMUL], [test "$enable_clmul" = "yes"]) -AM_CONDITIONAL([HAVE_CLZ], [test "$have_clzl$have_clzll" = "yesyes"]) AC_DEFINE([CLIENT_VERSION_MAJOR], [_CLIENT_VERSION_MAJOR], [Major version]) AC_DEFINE([CLIENT_VERSION_MINOR], [_CLIENT_VERSION_MINOR], [Minor version]) @@ -1866,7 +1785,6 @@ AC_SUBST(MINIUPNPC_CPPFLAGS) AC_SUBST(MINIUPNPC_LIBS) AC_SUBST(NATPMP_CPPFLAGS) AC_SUBST(NATPMP_LIBS) -AC_SUBST(HAVE_GMTIME_R) AC_SUBST(HAVE_FDATASYNC) AC_SUBST(HAVE_FULLFSYNC) AC_SUBST(HAVE_O_CLOEXEC) @@ -1928,7 +1846,6 @@ echo echo "Options used to compile and link:" echo " external signer = $use_external_signer" echo " multiprocess = $build_multiprocess" -echo " with libs = $build_bitcoin_libs" echo " with wallet = $enable_wallet" if test "$enable_wallet" != "no"; then echo " with sqlite = $use_sqlite" diff --git a/contrib/devtools/bitcoin-tidy/CMakeLists.txt b/contrib/devtools/bitcoin-tidy/CMakeLists.txt index 35e60d1d87..1260c71423 100644 --- a/contrib/devtools/bitcoin-tidy/CMakeLists.txt +++ b/contrib/devtools/bitcoin-tidy/CMakeLists.txt @@ -1,14 +1,25 @@ -cmake_minimum_required(VERSION 3.9) +cmake_minimum_required(VERSION 3.22) -project(bitcoin-tidy VERSION 1.0.0 DESCRIPTION "clang-tidy checks for Bitcoin Core") +project(bitcoin-tidy + VERSION + 1.0.0 + DESCRIPTION "clang-tidy checks for Bitcoin Core" + LANGUAGES CXX) include(GNUInstallDirs) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_CXX_EXTENSIONS False) -# TODO: Figure out how to avoid the terminfo check +set(CMAKE_DISABLE_FIND_PACKAGE_CURL ON) +set(CMAKE_DISABLE_FIND_PACKAGE_FFI ON) +set(CMAKE_DISABLE_FIND_PACKAGE_LibEdit ON) +set(CMAKE_DISABLE_FIND_PACKAGE_LibXml2 ON) +set(CMAKE_DISABLE_FIND_PACKAGE_Terminfo ON) +set(CMAKE_DISABLE_FIND_PACKAGE_ZLIB ON) +set(CMAKE_DISABLE_FIND_PACKAGE_zstd ON) + find_package(LLVM REQUIRED CONFIG) find_program(CLANG_TIDY_EXE NAMES "clang-tidy-${LLVM_VERSION_MAJOR}" "clang-tidy" HINTS ${LLVM_TOOLS_BINARY_DIR}) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") diff --git a/contrib/devtools/bitcoin-tidy/README b/contrib/devtools/bitcoin-tidy/README.md index c15e07c4ed..c15e07c4ed 100644 --- a/contrib/devtools/bitcoin-tidy/README +++ b/contrib/devtools/bitcoin-tidy/README.md diff --git a/contrib/devtools/gen-manpages.py b/contrib/devtools/gen-manpages.py index 2860e7db99..92acd9a403 100755 --- a/contrib/devtools/gen-manpages.py +++ b/contrib/devtools/gen-manpages.py @@ -62,6 +62,10 @@ with tempfile.NamedTemporaryFile('w', suffix='.h2m') as footer: # Copyright is the same for all binaries, so just use the first. footer.write('[COPYRIGHT]\n') footer.write('\n'.join(versions[0][2]).strip()) + # Create SEE ALSO section + footer.write('\n[SEE ALSO]\n') + footer.write(', '.join(s.rpartition('/')[2] + '(1)' for s in BINARIES)) + footer.write('\n') footer.flush() # Call the binaries through help2man to produce a manual page for each of them. diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 298e7bfbd6..870938cb52 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -74,7 +74,8 @@ mkdir -p "$VERSION_BASE" ################ # Default to building for all supported HOSTs (overridable by environment) -export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu +# powerpc64le-linux-gnu currently disabled due non-determinism issues across build arches. +export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu x86_64-w64-mingw32 x86_64-apple-darwin arm64-apple-darwin}" diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index b301369ad9..1e9b682f3f 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -61,7 +61,6 @@ store_path() { # Set environment variables to point the NATIVE toolchain to the right # includes/libs NATIVE_GCC="$(store_path gcc-toolchain)" -NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" unset LIBRARY_PATH unset CPATH @@ -70,12 +69,20 @@ unset CPLUS_INCLUDE_PATH unset OBJC_INCLUDE_PATH unset OBJCPLUS_INCLUDE_PATH -export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC_STATIC}/lib" export C_INCLUDE_PATH="${NATIVE_GCC}/include" export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include" export OBJCPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" +case "$HOST" in + *darwin*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; + *mingw*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; + *) + NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" + export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC_STATIC}/lib" + ;; +esac + # Set environment variables to point the CROSS toolchain to the right # includes/libs for $HOST case "$HOST" in @@ -300,11 +307,9 @@ mkdir -p "$DISTSRC" case "$HOST" in *darwin*) - make osx_volname ${V:+V=1} make deploydir ${V:+V=1} mkdir -p "unsigned-app-${HOST}" cp --target-directory="unsigned-app-${HOST}" \ - osx_volname \ contrib/macdeploy/detached-sig-create.sh mv --target-directory="unsigned-app-${HOST}" dist ( @@ -321,26 +326,16 @@ mkdir -p "$DISTSRC" ( cd installed - case "$HOST" in - *mingw*) - mv --target-directory="$DISTNAME"/lib/ "$DISTNAME"/bin/*.dll - ;; - esac - # Prune libtool and object archives find . -name "lib*.la" -delete find . -name "lib*.a" -delete - # Prune pkg-config files - rm -rf "${DISTNAME}/lib/pkgconfig" - case "$HOST" in *darwin*) ;; *) - # Split binaries and libraries from their debug symbols + # Split binaries from their debug symbols { find "${DISTNAME}/bin" -type f -executable -print0 - find "${DISTNAME}/lib" -type f -print0 } | xargs -0 -P"$JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg ;; esac diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 6c912ca748..ce6a9562b4 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -51,7 +51,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=d5ca4d4fd713a9f7e17e074a1e37dda99bbb09fc \ + --commit=dc4842797bfdc5f9f3f5f725bf189c2b68bd6b5a \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 7335596107..8f13c642d3 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -91,7 +91,7 @@ chain for " target " development.")) (home-page (package-home-page xgcc)) (license (package-license xgcc))))) -(define base-gcc gcc-10) +(define base-gcc gcc-12) (define base-linux-kernel-headers linux-libre-headers-6.1) (define* (make-bitcoin-cross-toolchain target @@ -110,12 +110,15 @@ desirable for building Bitcoin Core release binaries." (define (gcc-mingw-patches gcc) (package-with-extra-patches gcc - (search-our-patches "gcc-remap-guix-store.patch" - "vmov-alignment.patch"))) + (search-our-patches "gcc-remap-guix-store.patch"))) + +(define (binutils-mingw-patches binutils) + (package-with-extra-patches binutils + (search-our-patches "binutils-unaligned-default.patch"))) (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (cross-binutils target)) + (let* ((xbinutils (binutils-mingw-patches (cross-binutils target))) (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (cross-gcc target #:xgcc (gcc-mingw-patches mingw-w64-base-gcc) @@ -423,6 +426,7 @@ inspecting signatures in Mach-O binaries.") (list "--enable-initfini-array=yes", "--enable-default-ssp=yes", "--enable-default-pie=yes", + "--enable-standard-branch-protection=yes", building-on))) ((#:phases phases) `(modify-phases ,phases @@ -499,15 +503,13 @@ inspecting signatures in Mach-O binaries.") gzip xz ;; Build tools + cmake-minimal gnu-make libtool autoconf-2.71 automake pkg-config bison - ;; Native GCC 10 toolchain - gcc-toolchain-10 - (list gcc-toolchain-10 "static") ;; Scripting python-minimal ;; (3.10) ;; Git @@ -516,14 +518,23 @@ inspecting signatures in Mach-O binaries.") python-lief) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) - ;; Windows - (list zip + (list ;; Native GCC 12 toolchain + gcc-toolchain-12 + zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") nsis-x86_64 nss-certs osslsigncode)) ((string-contains target "-linux-") - (list (make-bitcoin-cross-toolchain target))) + (list ;; Native GCC 12 toolchain + gcc-toolchain-12 + (list gcc-toolchain-12 "static") + (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list clang-toolchain-17 binutils cmake-minimal python-signapple zip)) + (list ;; Native GCC 11 toolchain + gcc-toolchain-11 + binutils + clang-toolchain-17 + python-signapple + zip)) (else '()))))) diff --git a/contrib/guix/patches/binutils-unaligned-default.patch b/contrib/guix/patches/binutils-unaligned-default.patch new file mode 100644 index 0000000000..d1bc71aee1 --- /dev/null +++ b/contrib/guix/patches/binutils-unaligned-default.patch @@ -0,0 +1,22 @@ +commit 6537181f59ed186a341db621812a6bc35e22eaf6 +Author: fanquake <fanquake@gmail.com> +Date: Wed Apr 10 12:15:52 2024 +0200 + + build: turn on -muse-unaligned-vector-move by default + + This allows us to avoid (more invasively) patching GCC, to avoid + unaligned instruction use. + +diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c +index e0632681477..14a9653abdf 100644 +--- a/gas/config/tc-i386.c ++++ b/gas/config/tc-i386.c +@@ -801,7 +801,7 @@ static unsigned int no_cond_jump_promotion = 0; + static unsigned int sse2avx; + + /* Encode aligned vector move as unaligned vector move. */ +-static unsigned int use_unaligned_vector_move; ++static unsigned int use_unaligned_vector_move = 1; + + /* Encode scalar AVX instructions with specific vector length. */ + static enum diff --git a/contrib/guix/patches/glibc-2.27-fcommon.patch b/contrib/guix/patches/glibc-2.27-fcommon.patch index 817aa85bb9..f8d14837fc 100644 --- a/contrib/guix/patches/glibc-2.27-fcommon.patch +++ b/contrib/guix/patches/glibc-2.27-fcommon.patch @@ -5,7 +5,7 @@ Date: Fri May 6 11:03:04 2022 +0100 build: use -fcommon to retain legacy behaviour with GCC 10 GCC 10 started using -fno-common by default, which causes issues with - the powerpc builds using gibc 2.27. A patch was commited to glibc to fix + the powerpc builds using gibc 2.27. A patch was committed to glibc to fix the issue, 18363b4f010da9ba459b13310b113ac0647c2fcc but is non-trvial to backport, and was broken in at least one way, see the followup in commit 7650321ce037302bfc2f026aa19e0213b8d02fe6. diff --git a/contrib/guix/patches/vmov-alignment.patch b/contrib/guix/patches/vmov-alignment.patch deleted file mode 100644 index 7976b864af..0000000000 --- a/contrib/guix/patches/vmov-alignment.patch +++ /dev/null @@ -1,268 +0,0 @@ -Description: Use unaligned VMOV instructions -Author: Stephen Kitt <skitt@debian.org> -Bug-Debian: https://bugs.debian.org/939559 -See also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412 - -Based on a patch originally by Claude Heiland-Allen <claude@mathr.co.uk> - ---- a/gcc/config/i386/sse.md -+++ b/gcc/config/i386/sse.md -@@ -1058,17 +1058,11 @@ - { - if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode))) - { -- if (misaligned_operand (operands[1], <MODE>mode)) -- return "vmovu<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; -- else -- return "vmova<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; -+ return "vmovu<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; - } - else - { -- if (misaligned_operand (operands[1], <MODE>mode)) -- return "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; -- else -- return "vmovdqa<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; -+ return "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; - } - } - [(set_attr "type" "ssemov") -@@ -1184,17 +1178,11 @@ - { - if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode))) - { -- if (misaligned_operand (operands[0], <MODE>mode)) -- return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; -- else -- return "vmova<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; -+ return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; - } - else - { -- if (misaligned_operand (operands[0], <MODE>mode)) -- return "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; -- else -- return "vmovdqa<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; -+ return "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; - } - } - [(set_attr "type" "ssemov") -@@ -7806,7 +7794,7 @@ - "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))" - "@ - %vmovlps\t{%1, %0|%q0, %1} -- %vmovaps\t{%1, %0|%0, %1} -+ %vmovups\t{%1, %0|%0, %1} - %vmovlps\t{%1, %d0|%d0, %q1}" - [(set_attr "type" "ssemov") - (set_attr "prefix" "maybe_vex") -@@ -13997,29 +13985,15 @@ - switch (<MODE>mode) - { - case E_V8DFmode: -- if (misaligned_operand (operands[2], <ssequartermode>mode)) -- return "vmovupd\t{%2, %x0|%x0, %2}"; -- else -- return "vmovapd\t{%2, %x0|%x0, %2}"; -+ return "vmovupd\t{%2, %x0|%x0, %2}"; - case E_V16SFmode: -- if (misaligned_operand (operands[2], <ssequartermode>mode)) -- return "vmovups\t{%2, %x0|%x0, %2}"; -- else -- return "vmovaps\t{%2, %x0|%x0, %2}"; -+ return "vmovups\t{%2, %x0|%x0, %2}"; - case E_V8DImode: -- if (misaligned_operand (operands[2], <ssequartermode>mode)) -- return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" -+ return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" - : "vmovdqu\t{%2, %x0|%x0, %2}"; -- else -- return which_alternative == 2 ? "vmovdqa64\t{%2, %x0|%x0, %2}" -- : "vmovdqa\t{%2, %x0|%x0, %2}"; - case E_V16SImode: -- if (misaligned_operand (operands[2], <ssequartermode>mode)) -- return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" -+ return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" - : "vmovdqu\t{%2, %x0|%x0, %2}"; -- else -- return which_alternative == 2 ? "vmovdqa32\t{%2, %x0|%x0, %2}" -- : "vmovdqa\t{%2, %x0|%x0, %2}"; - default: - gcc_unreachable (); - } -@@ -21225,63 +21199,27 @@ - switch (get_attr_mode (insn)) - { - case MODE_V16SF: -- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) -- return "vmovups\t{%1, %t0|%t0, %1}"; -- else -- return "vmovaps\t{%1, %t0|%t0, %1}"; -+ return "vmovups\t{%1, %t0|%t0, %1}"; - case MODE_V8DF: -- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) -- return "vmovupd\t{%1, %t0|%t0, %1}"; -- else -- return "vmovapd\t{%1, %t0|%t0, %1}"; -+ return "vmovupd\t{%1, %t0|%t0, %1}"; - case MODE_V8SF: -- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) -- return "vmovups\t{%1, %x0|%x0, %1}"; -- else -- return "vmovaps\t{%1, %x0|%x0, %1}"; -+ return "vmovups\t{%1, %x0|%x0, %1}"; - case MODE_V4DF: -- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) -- return "vmovupd\t{%1, %x0|%x0, %1}"; -- else -- return "vmovapd\t{%1, %x0|%x0, %1}"; -+ return "vmovupd\t{%1, %x0|%x0, %1}"; - case MODE_XI: -- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) -- { -- if (which_alternative == 2) -- return "vmovdqu\t{%1, %t0|%t0, %1}"; -- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) -- return "vmovdqu64\t{%1, %t0|%t0, %1}"; -- else -- return "vmovdqu32\t{%1, %t0|%t0, %1}"; -- } -+ if (which_alternative == 2) -+ return "vmovdqu\t{%1, %t0|%t0, %1}"; -+ else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) -+ return "vmovdqu64\t{%1, %t0|%t0, %1}"; - else -- { -- if (which_alternative == 2) -- return "vmovdqa\t{%1, %t0|%t0, %1}"; -- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) -- return "vmovdqa64\t{%1, %t0|%t0, %1}"; -- else -- return "vmovdqa32\t{%1, %t0|%t0, %1}"; -- } -+ return "vmovdqu32\t{%1, %t0|%t0, %1}"; - case MODE_OI: -- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) -- { -- if (which_alternative == 2) -- return "vmovdqu\t{%1, %x0|%x0, %1}"; -- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) -- return "vmovdqu64\t{%1, %x0|%x0, %1}"; -- else -- return "vmovdqu32\t{%1, %x0|%x0, %1}"; -- } -+ if (which_alternative == 2) -+ return "vmovdqu\t{%1, %x0|%x0, %1}"; -+ else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) -+ return "vmovdqu64\t{%1, %x0|%x0, %1}"; - else -- { -- if (which_alternative == 2) -- return "vmovdqa\t{%1, %x0|%x0, %1}"; -- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) -- return "vmovdqa64\t{%1, %x0|%x0, %1}"; -- else -- return "vmovdqa32\t{%1, %x0|%x0, %1}"; -- } -+ return "vmovdqu32\t{%1, %x0|%x0, %1}"; - default: - gcc_unreachable (); - } ---- a/gcc/config/i386/i386.c -+++ b/gcc/config/i386/i386.c -@@ -4981,13 +4981,13 @@ - switch (type) - { - case opcode_int: -- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; -+ opcode = "vmovdqu32"; - break; - case opcode_float: -- opcode = misaligned_p ? "vmovups" : "vmovaps"; -+ opcode = "vmovups"; - break; - case opcode_double: -- opcode = misaligned_p ? "vmovupd" : "vmovapd"; -+ opcode = "vmovupd"; - break; - } - } -@@ -4996,16 +4996,16 @@ - switch (scalar_mode) - { - case E_SFmode: -- opcode = misaligned_p ? "%vmovups" : "%vmovaps"; -+ opcode = "%vmovups"; - break; - case E_DFmode: -- opcode = misaligned_p ? "%vmovupd" : "%vmovapd"; -+ opcode = "%vmovupd"; - break; - case E_TFmode: - if (evex_reg_p) -- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; -+ opcode = "vmovdqu64"; - else -- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; -+ opcode = "%vmovdqu"; - break; - default: - gcc_unreachable (); -@@ -5017,48 +5017,32 @@ - { - case E_QImode: - if (evex_reg_p) -- opcode = (misaligned_p -- ? (TARGET_AVX512BW -- ? "vmovdqu8" -- : "vmovdqu64") -- : "vmovdqa64"); -+ opcode = TARGET_AVX512BW ? "vmovdqu8" : "vmovdqu64"; - else -- opcode = (misaligned_p -- ? (TARGET_AVX512BW -- ? "vmovdqu8" -- : "%vmovdqu") -- : "%vmovdqa"); -+ opcode = TARGET_AVX512BW ? "vmovdqu8" : "%vmovdqu"; - break; - case E_HImode: - if (evex_reg_p) -- opcode = (misaligned_p -- ? (TARGET_AVX512BW -- ? "vmovdqu16" -- : "vmovdqu64") -- : "vmovdqa64"); -+ opcode = TARGET_AVX512BW ? "vmovdqu16" : "vmovdqu64"; - else -- opcode = (misaligned_p -- ? (TARGET_AVX512BW -- ? "vmovdqu16" -- : "%vmovdqu") -- : "%vmovdqa"); -+ opcode = TARGET_AVX512BW ? "vmovdqu16" : "%vmovdqu"; - break; - case E_SImode: - if (evex_reg_p) -- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; -+ opcode = "vmovdqu32"; - else -- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; -+ opcode = "%vmovdqu"; - break; - case E_DImode: - case E_TImode: - case E_OImode: - if (evex_reg_p) -- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; -+ opcode = "vmovdqu64"; - else -- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; -+ opcode = "%vmovdqu"; - break; - case E_XImode: -- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; -+ opcode = "vmovdqu64"; - break; - default: - gcc_unreachable (); diff --git a/contrib/init/bitcoind.service b/contrib/init/bitcoind.service index 87da17f955..ade8a05926 100644 --- a/contrib/init/bitcoind.service +++ b/contrib/init/bitcoind.service @@ -81,5 +81,8 @@ PrivateDevices=true # Deny the creation of writable and executable memory mappings. MemoryDenyWriteExecute=true +# Restrict ABIs to help ensure MemoryDenyWriteExecute is enforced +SystemCallArchitectures=native + [Install] WantedBy=multi-user.target diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index ea599df3d8..d1df3062f8 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -66,9 +66,8 @@ building for macOS. Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several -other tools are needed as well such as `install_name_tool`, `lipo`, and `nmedit`. These -do not build under Linux, so they have been patched to do so. The work here was used as -a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). +other tools are needed as well. These do not build under Linux, so they have been patched to +do so. The work here was used as a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). In order to build a working toolchain, the following source packages are needed from Apple: `cctools`, `dyld`, and `ld64`. diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index ee91acd5ef..c537f9e7ec 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -13,8 +13,8 @@ # # Note that suppressions may depend on OS and/or library versions. # Tested on: -# * aarch64 (Debian Bookworm system libs, clang, without gui) -# * x86_64 (Debian Bookworm system libs, clang, without gui) +# * aarch64 (Ubuntu Noble system libs, clang, without gui) +# * x86_64 (Ubuntu Noble system libs, clang, without gui) { Suppress libdb warning - https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=662917 Memcheck:Cond diff --git a/depends/Makefile b/depends/Makefile index 88aae7ad81..005d9696fb 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -234,7 +234,6 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@NM@|$(host_NM)|' \ -e 's|@STRIP@|$(host_STRIP)|' \ -e 's|@OTOOL@|$(host_OTOOL)|' \ - -e 's|@INSTALL_NAME_TOOL@|$(host_INSTALL_NAME_TOOL)|' \ -e 's|@DSYMUTIL@|$(host_DSYMUTIL)|' \ -e 's|@build_os@|$(build_os)|' \ -e 's|@host_os@|$(host_os)|' \ diff --git a/depends/README.md b/depends/README.md index a8dfc83e3b..10e0985cf4 100644 --- a/depends/README.md +++ b/depends/README.md @@ -85,6 +85,10 @@ For linux S390X cross compilation: sudo apt-get install g++-s390x-linux-gnu binutils-s390x-linux-gnu +### Install the required dependencies: FreeBSD + + pkg install bash + ### Install the required dependencies: OpenBSD pkg_add bash gtar diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index a5f07643de..554bfd2c3e 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -5,7 +5,6 @@ build_darwin_RANLIB:=$(shell xcrun -f ranlib) build_darwin_STRIP:=$(shell xcrun -f strip) build_darwin_OTOOL:=$(shell xcrun -f otool) build_darwin_NM:=$(shell xcrun -f nm) -build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) build_darwin_DSYMUTIL:=$(shell xcrun -f dsymutil) build_darwin_SHA256SUM=shasum -a 256 build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o @@ -18,7 +17,6 @@ darwin_RANLIB:=$(shell xcrun -f ranlib) darwin_STRIP:=$(shell xcrun -f strip) darwin_OTOOL:=$(shell xcrun -f otool) darwin_NM:=$(shell xcrun -f nm) -darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) darwin_DSYMUTIL:=$(shell xcrun -f dsymutil) darwin_native_binutils= darwin_native_toolchain= diff --git a/depends/builders/default.mk b/depends/builders/default.mk index cc6dec66c2..50869cd8a2 100644 --- a/depends/builders/default.mk +++ b/depends/builders/default.mk @@ -5,13 +5,14 @@ default_build_TAR = tar default_build_RANLIB = ranlib default_build_STRIP = strip default_build_NM = nm +default_build_TOUCH = touch -h -m -t 200001011200 define add_build_tool_func build_$(build_os)_$1 ?= $$(default_build_$1) build_$(build_arch)_$(build_os)_$1 ?= $$(build_$(build_os)_$1) build_$1=$$(build_$(build_arch)_$(build_os)_$1) endef -$(foreach var,CC CXX AR TAR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL INSTALL_NAME_TOOL DSYMUTIL,$(eval $(call add_build_tool_func,$(var)))) +$(foreach var,CC CXX AR TAR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL DSYMUTIL TOUCH,$(eval $(call add_build_tool_func,$(var)))) define add_build_flags_func build_$(build_arch)_$(build_os)_$1 += $(build_$(build_os)_$1) build_$1=$$(build_$(build_arch)_$(build_os)_$1) diff --git a/depends/builders/openbsd.mk b/depends/builders/openbsd.mk index 44825d106a..9c94c4baae 100644 --- a/depends/builders/openbsd.mk +++ b/depends/builders/openbsd.mk @@ -5,3 +5,5 @@ build_openbsd_SHA256SUM = sha256 build_openbsd_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o build_openbsd_TAR = gtar +# openBSD touch doesn't understand -h +build_openbsd_TOUCH = touch -m -t 200001011200 diff --git a/depends/config.site.in b/depends/config.site.in index 29b2a67ed0..81975f02b9 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -123,11 +123,6 @@ if test "@host_os@" = darwin; then ac_cv_path_OTOOL="${OTOOL}" fi - if test -n "@INSTALL_NAME_TOOL@"; then - INSTALL_NAME_TOOL="@INSTALL_NAME_TOOL@" - ac_cv_path_INSTALL_NAME_TOOL="${INSTALL_NAME_TOOL}" - fi - if test -n "@DSYMUTIL@"; then DSYMUTIL="@DSYMUTIL@" ac_cv_path_DSYMUTIL="${DSYMUTIL}" diff --git a/depends/description.md b/depends/description.md index 69ee5bd36c..fa345bfe85 100644 --- a/depends/description.md +++ b/depends/description.md @@ -11,7 +11,7 @@ on new hosts. ### No reliance on timestamps File presence is used to determine what needs to be built. This makes the -results distributable and easily digestable by automated builders. +results distributable and easily digestible by automated builders. ### Each build only has its specified dependencies available at build-time. diff --git a/depends/funcs.mk b/depends/funcs.mk index 24c911b0f7..494ed5d324 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -147,7 +147,7 @@ $(1)_stage_env+=PATH="$(build_prefix)/bin:$(PATH)" # config.guess, which is what we set it too here. This also quells autoconf # warnings, "If you wanted to set the --build type, don't use --host.", # when using versions older than 2.70. -$(1)_autoconf=./configure --build=$(BUILD) --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" +$(1)_autoconf=./configure --build=$(BUILD) --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) --with-pic $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" ifneq ($($(1)_nm),) $(1)_autoconf += NM="$$($(1)_nm)" endif @@ -170,12 +170,19 @@ ifneq ($($(1)_ldflags),) $(1)_autoconf += LDFLAGS="$$($(1)_ldflags)" endif +# We hardcode the library install path to "lib" to match the PKG_CONFIG_PATH +# setting in depends/config.site.in, which also hardcodes "lib". +# Without this setting, CMake by default would use the OS library +# directory, which might be "lib64" or something else, not "lib", on multiarch systems. $(1)_cmake=env CC="$$($(1)_cc)" \ CFLAGS="$$($(1)_cppflags) $$($(1)_cflags)" \ CXX="$$($(1)_cxx)" \ CXXFLAGS="$$($(1)_cppflags) $$($(1)_cxxflags)" \ LDFLAGS="$$($(1)_ldflags)" \ - cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" $$($(1)_config_opts) + cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" \ + -DCMAKE_INSTALL_LIBDIR=lib/ \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + $$($(1)_config_opts) ifeq ($($(1)_type),build) $(1)_cmake += -DCMAKE_INSTALL_RPATH:PATH="$$($($(1)_type)_prefix)/lib" else @@ -234,7 +241,7 @@ $($(1)_postprocessed): | $($(1)_staged) $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) echo Caching $(1)... cd $$($(1)_staging_dir)/$(host_prefix); \ - find . ! -name '.stamp_postprocessed' -print0 | TZ=UTC xargs -0r touch -h -m -t 200001011200; \ + find . ! -name '.stamp_postprocessed' -print0 | TZ=UTC xargs -0r $(build_TOUCH); \ find . ! -name '.stamp_postprocessed' | LC_ALL=C sort | $(build_TAR) --numeric-owner --no-recursion -czf $$($(1)_staging_dir)/$$(@F) -T - mkdir -p $$(@D) rm -rf $$(@D) && mkdir -p $$(@D) diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 29ad7ef252..639259ace3 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -39,7 +39,7 @@ llvm_config_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-config") llvm_lib_dir=$(shell $(llvm_config_prog) --libdir) endif -cctools_TOOLS=AR RANLIB STRIP NM OTOOL INSTALL_NAME_TOOL DSYMUTIL +cctools_TOOLS=AR RANLIB STRIP NM OTOOL DSYMUTIL # Make-only lowercase function lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1)))))))))))))))))))))))))) @@ -105,7 +105,7 @@ endif darwin_release_CFLAGS=-O2 darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) -darwin_debug_CFLAGS=-O1 +darwin_debug_CFLAGS=-O1 -g darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) darwin_cmake_system=Darwin diff --git a/depends/hosts/default.mk b/depends/hosts/default.mk index cf3c90441d..6a6cab6cc6 100644 --- a/depends/hosts/default.mk +++ b/depends/hosts/default.mk @@ -38,5 +38,5 @@ host_$1 = $$($(host_arch)_$(host_os)_$1) host_$(release_type)_$1 = $$($(host_arch)_$(host_os)_$(release_type)_$1) endef -$(foreach tool,CC CXX AR RANLIB STRIP NM OBJCOPY OTOOL INSTALL_NAME_TOOL DSYMUTIL,$(eval $(call add_host_tool_func,$(tool)))) +$(foreach tool,CC CXX AR RANLIB STRIP NM OBJCOPY OTOOL DSYMUTIL,$(eval $(call add_host_tool_func,$(tool)))) $(foreach flags,CFLAGS CXXFLAGS CPPFLAGS LDFLAGS, $(eval $(call add_host_flags_func,$(flags)))) diff --git a/depends/hosts/linux.mk b/depends/hosts/linux.mk index 8be23be57d..f5ce2bb0b8 100644 --- a/depends/hosts/linux.mk +++ b/depends/hosts/linux.mk @@ -10,10 +10,14 @@ endif linux_release_CFLAGS=-O2 linux_release_CXXFLAGS=$(linux_release_CFLAGS) -linux_debug_CFLAGS=-O1 +linux_debug_CFLAGS=-O1 -g linux_debug_CXXFLAGS=$(linux_debug_CFLAGS) -linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -D_LIBCPP_ENABLE_DEBUG_MODE=1 +# https://gcc.gnu.org/onlinedocs/libstdc++/manual/debug_mode.html +linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC + +# https://libcxx.llvm.org/Hardening.html +linux_debug_CPPFLAGS+=-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG ifeq (86,$(findstring 86,$(build_arch))) i686_linux_CC=gcc -m32 diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index 15aa7cd25a..4c657358f6 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -14,7 +14,7 @@ endif mingw32_release_CFLAGS=-O2 mingw32_release_CXXFLAGS=$(mingw32_release_CFLAGS) -mingw32_debug_CFLAGS=-O1 +mingw32_debug_CFLAGS=-O1 -g mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS) mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC diff --git a/depends/packages.md b/depends/packages.md index ad91eaffee..7a7a42afa1 100644 --- a/depends/packages.md +++ b/depends/packages.md @@ -162,6 +162,9 @@ From the [Gentoo Wiki entry](https://wiki.gentoo.org/wiki/Project:Quality_Assura > creates. This leads to massive overlinking, which is toxic to the Gentoo > ecosystem, as it leads to a massive number of unnecessary rebuilds. +Where possible, packages are built with Position Independent Code. Either using +the Autotools `--with-pic` flag, or `CMAKE_POSITION_INDEPENDENT_CODE` with CMake. + ## Secondary dependencies: Secondary dependency packages relative to the bitcoin binaries/libraries (i.e. diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index 1a21238152..be82b0d309 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -9,11 +9,6 @@ $(package)_patches=clang_cxx_11.patch define $(package)_set_vars $(package)_config_opts=--disable-shared --enable-cxx --disable-replication --enable-option-checking $(package)_config_opts_mingw32=--enable-mingw -$(package)_config_opts_linux=--with-pic -$(package)_config_opts_freebsd=--with-pic -$(package)_config_opts_netbsd=--with-pic -$(package)_config_opts_openbsd=--with-pic -$(package)_config_opts_android=--with-pic $(package)_cflags+=-Wno-error=implicit-function-declaration -Wno-error=format-security -Wno-error=implicit-int $(package)_cppflags_freebsd=-D_XOPEN_SOURCE=600 -D__BSD_VISIBLE=1 $(package)_cppflags_netbsd=-D_XOPEN_SOURCE=600 diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index ab43764b38..ebc097d686 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -3,11 +3,6 @@ $(package)_version=1.81.0 $(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$($(package)_version)/source/ $(package)_file_name=boost_$(subst .,_,$($(package)_version)).tar.bz2 $(package)_sha256_hash=71feeed900fbccca04a3b4f2f84a7c217186f28a940ed8b7ed4725986baf99fa -$(package)_patches=process_macos_sdk.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/process_macos_sdk.patch -endef define $(package)_stage_cmds mkdir -p $($(package)_staging_prefix_dir)/include && \ diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index 2465c8091b..6d792db711 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -5,15 +5,10 @@ $(package)_download_file=$(native_$(package)_download_file) $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) -# Hardcode library install path to "lib" to match the PKG_CONFIG_PATH -# setting in depends/config.site.in, which also hardcodes "lib". -# Without this setting, cmake by default would use the OS library -# directory, which might be "lib64" or something else, not "lib", on multiarch systems. define $(package)_set_vars := $(package)_config_opts := -DBUILD_TESTING=OFF $(package)_config_opts += -DWITH_OPENSSL=OFF $(package)_config_opts += -DWITH_ZLIB=OFF - $(package)_config_opts += -DCMAKE_INSTALL_LIBDIR=lib/ endef define $(package)_config_cmds diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index bb203d06f8..2ec660109c 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -6,12 +6,11 @@ $(package)_sha256_hash=f79b8f904b749e3e0d20afeadecf8249c55b2e32d4ebb089ae378df47 # -D_DEFAULT_SOURCE defines __USE_MISC, which exposes additional # definitions in endian.h, which are required for a working -# endianess check in configure when building with -flto. +# endianness check in configure when building with -flto. define $(package)_set_vars $(package)_config_opts=--disable-shared --without-docbook --without-tests --without-examples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts += --without-xmlwf - $(package)_config_opts_linux=--with-pic $(package)_cppflags += -D_DEFAULT_SOURCE endef diff --git a/depends/packages/freetype.mk b/depends/packages/freetype.mk index 6f5dbe0f01..c28259ed67 100644 --- a/depends/packages/freetype.mk +++ b/depends/packages/freetype.mk @@ -7,7 +7,6 @@ $(package)_sha256_hash=8bee39bd3968c4804b70614a0a3ad597299ad0e824bc8aad5ce8aaf48 define $(package)_set_vars $(package)_config_opts=--without-zlib --without-png --without-harfbuzz --without-bzip2 --disable-static $(package)_config_opts += --enable-option-checking --without-brotli - $(package)_config_opts_linux=--with-pic endef define $(package)_config_cmds diff --git a/depends/packages/libXau.mk b/depends/packages/libXau.mk index b7e032c0b2..aeb14dcd6e 100644 --- a/depends/packages/libXau.mk +++ b/depends/packages/libXau.mk @@ -10,7 +10,6 @@ $(package)_dependencies=xproto define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-lint-library --without-lint $(package)_config_opts += --disable-dependency-tracking --enable-option-checking - $(package)_config_opts_linux=--with-pic endef define $(package)_preprocess_cmds diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 9650f77db9..d764be5d0a 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -11,11 +11,6 @@ define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress --disable-samples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_release=--disable-debug-mode - $(package)_config_opts_linux=--with-pic - $(package)_config_opts_freebsd=--with-pic - $(package)_config_opts_netbsd=--with-pic - $(package)_config_opts_openbsd=--with-pic - $(package)_config_opts_android=--with-pic $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601 ifeq ($(NO_HARDEN),) diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk index d237f52dbb..c292c49bfb 100644 --- a/depends/packages/libmultiprocess.mk +++ b/depends/packages/libmultiprocess.mk @@ -8,13 +8,7 @@ ifneq ($(host),$(build)) $(package)_dependencies += native_capnp endif -# Hardcode library install path to "lib" to match the PKG_CONFIG_PATH -# setting in depends/config.site.in, which also hardcodes "lib". -# Without this setting, cmake by default would use the OS library -# directory, which might be "lib64" or something else, not "lib", on multiarch systems. define $(package)_set_vars := -$(package)_config_opts += -DCMAKE_INSTALL_LIBDIR=lib/ -$(package)_config_opts += -DCMAKE_POSITION_INDEPENDENT_CODE=ON ifneq ($(host),$(build)) $(package)_config_opts := -DCAPNP_EXECUTABLE="$$(native_capnp_prefixbin)/capnp" $(package)_config_opts += -DCAPNPC_CXX_EXECUTABLE="$$(native_capnp_prefixbin)/capnpc-c++" @@ -26,7 +20,7 @@ define $(package)_config_cmds endef define $(package)_build_cmds - $(MAKE) + $(MAKE) multiprocess endef define $(package)_stage_cmds diff --git a/depends/packages/libnatpmp.mk b/depends/packages/libnatpmp.mk index caa809ec0b..5a573a18e7 100644 --- a/depends/packages/libnatpmp.mk +++ b/depends/packages/libnatpmp.mk @@ -1,26 +1,20 @@ package=libnatpmp -$(package)_version=07004b97cf691774efebe70404cf22201e4d330d +$(package)_version=f2433bec24ca3d3f22a8a7840728a3ac177f94ba $(package)_download_path=https://github.com/miniupnp/libnatpmp/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=9321953ceb39d07c25463e266e50d0ae7b64676bb3a986d932b18881ed94f1fb -$(package)_patches=no_libtool.patch +$(package)_sha256_hash=ef84979950dfb3556705b63c9cd6c95501b75e887fba466234b187f3c9029669 +$(package)_build_subdir=build -define $(package)_set_vars - $(package)_build_opts=CC="$($(package)_cc)" - $(package)_build_opts_mingw32=CPPFLAGS=-DNATPMP_STATICLIB - $(package)_build_env+=CFLAGS="$($(package)_cflags) $($(package)_cppflags)" AR="$($(package)_ar)" -endef - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/no_libtool.patch +define $(package)_config_cmds + $($(package)_cmake) -S .. -B . endef define $(package)_build_cmds - $(MAKE) libnatpmp.a $($(package)_build_opts) + $(MAKE) natpmp endef define $(package)_stage_cmds - mkdir -p $($(package)_staging_prefix_dir)/include $($(package)_staging_prefix_dir)/lib &&\ - install *.h $($(package)_staging_prefix_dir)/include &&\ + mkdir -p $($(package)_staging_prefix_dir)/include $($(package)_staging_prefix_dir)/lib && \ + install ../natpmp.h ../natpmp_declspec.h $($(package)_staging_prefix_dir)/include && \ install libnatpmp.a $($(package)_staging_prefix_dir)/lib endef diff --git a/depends/packages/libxcb_util.mk b/depends/packages/libxcb_util.mk index 6f1b9cd7c6..6e4c7359b2 100644 --- a/depends/packages/libxcb_util.mk +++ b/depends/packages/libxcb_util.mk @@ -8,7 +8,6 @@ $(package)_dependencies=libxcb define $(package)_set_vars $(package)_config_opts = --disable-shared --disable-devel-docs --without-doxygen $(package)_config_opts += --disable-dependency-tracking --enable-option-checking -$(package)_config_opts += --with-pic endef define $(package)_preprocess_cmds diff --git a/depends/packages/qrencode.mk b/depends/packages/qrencode.mk index 2afd95d7c4..4d852d833d 100644 --- a/depends/packages/qrencode.mk +++ b/depends/packages/qrencode.mk @@ -3,22 +3,22 @@ $(package)_version=4.1.1 $(package)_download_path=https://fukuchi.org/works/qrencode/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=e455d9732f8041cf5b9c388e345a641fd15707860f928e94507b1961256a6923 +$(package)_patches=cmake_fixups.patch define $(package)_set_vars -$(package)_config_opts=--disable-shared --without-tools --without-tests --without-png -$(package)_config_opts += --disable-gprof --disable-gcov --disable-mudflap -$(package)_config_opts += --disable-dependency-tracking --enable-option-checking -$(package)_config_opts_linux=--with-pic -$(package)_config_opts_android=--with-pic +$(package)_config_opts := -DWITH_TOOLS=NO -DWITH_TESTS=NO -DGPROF=OFF -DCOVERAGE=OFF +$(package)_config_opts += -DCMAKE_DISABLE_FIND_PACKAGE_PNG=TRUE -DWITHOUT_PNG=ON +$(package)_config_opts += -DCMAKE_DISABLE_FIND_PACKAGE_ICONV=TRUE $(package)_cflags += -Wno-int-conversion -Wno-implicit-function-declaration endef define $(package)_preprocess_cmds - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub use + patch -p1 < $($(package)_patch_dir)/cmake_fixups.patch endef + define $(package)_config_cmds - $($(package)_autoconf) + $($(package)_cmake) -S . -B . endef define $(package)_build_cmds @@ -28,7 +28,3 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef - -define $(package)_postprocess_cmds - rm lib/*.la -endef diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 5608e5f073..0acf4cf565 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,9 +1,9 @@ package=qt -$(package)_version=5.15.11 +$(package)_version=5.15.13 $(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules $(package)_suffix=everywhere-opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=425ad301acd91ca66c10c0dabee0704e2d0cd2801a6b670115800cbb95f84846 +$(package)_sha256_hash=4cca51dcc1f22ceeee6b3e33cd1c3a60b14e85e24644dca3af89a2c2989ab809 $(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_linguist_tools = lrelease lupdate lconvert @@ -15,10 +15,8 @@ $(package)_patches += no-xlib.patch $(package)_patches += fix_android_jni_static.patch $(package)_patches += dont_hardcode_pwd.patch $(package)_patches += qtbase-moc-ignore-gcc-macro.patch -$(package)_patches += use_android_ndk23.patch $(package)_patches += rcc_hardcode_timestamp.patch $(package)_patches += duplicate_lcqpafonts.patch -$(package)_patches += fast_fixed_dtoa_no_optimize.patch $(package)_patches += guix_cross_lib_path.patch $(package)_patches += fix-macos-linker.patch $(package)_patches += memory_resource.patch @@ -26,10 +24,10 @@ $(package)_patches += utc_from_string_no_optimize.patch $(package)_patches += windows_lto.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=a31785948c640b7c66d9fe2db4993728ca07f64e41c560b3625ad191b276ff20 +$(package)_qttranslations_sha256_hash=24d4c58bc2a40c0f44f59ee64af4192c7d0038c1e45af61646cfc5b65058f271 $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=7cd847ae6ff09416df617136eadcaf0eb98e3bc9b89979219a3ea8111fb8d339 +$(package)_qttools_sha256_hash=57c9794c572c4e02871f2e7581525752b0cf85ea16cfab23a4ac9ba7b39a5d34 $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) @@ -178,6 +176,7 @@ $(package)_config_opts_mingw32 += -xplatform win32-g++ $(package)_config_opts_mingw32 += "QMAKE_CFLAGS = '$($(package)_cflags) $($(package)_cppflags)'" $(package)_config_opts_mingw32 += "QMAKE_CXX = '$($(package)_cxx)'" $(package)_config_opts_mingw32 += "QMAKE_CXXFLAGS = '$($(package)_cxxflags) $($(package)_cppflags)'" +$(package)_config_opts_mingw32 += "QMAKE_LINK = '$($(package)_cxx)'" $(package)_config_opts_mingw32 += "QMAKE_LFLAGS = '$($(package)_ldflags)'" $(package)_config_opts_mingw32 += "QMAKE_LIB = '$($(package)_ar) rc'" $(package)_config_opts_mingw32 += -device-option CROSS_COMPILE="$(host)-" @@ -246,12 +245,10 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ - patch -p1 -i $($(package)_patch_dir)/use_android_ndk23.patch && \ patch -p1 -i $($(package)_patch_dir)/memory_resource.patch && \ patch -p1 -i $($(package)_patch_dir)/rcc_hardcode_timestamp.patch && \ patch -p1 -i $($(package)_patch_dir)/duplicate_lcqpafonts.patch && \ patch -p1 -i $($(package)_patch_dir)/utc_from_string_no_optimize.patch && \ - patch -p1 -i $($(package)_patch_dir)/fast_fixed_dtoa_no_optimize.patch && \ patch -p1 -i $($(package)_patch_dir)/guix_cross_lib_path.patch && \ patch -p1 -i $($(package)_patch_dir)/windows_lto.patch && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ diff --git a/depends/packages/sqlite.mk b/depends/packages/sqlite.mk index 6809b39113..15bc0f4d7a 100644 --- a/depends/packages/sqlite.mk +++ b/depends/packages/sqlite.mk @@ -7,12 +7,7 @@ $(package)_sha256_hash=5af07de982ba658fd91a03170c945f99c971f6955bc79df3266544373 define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-readline --disable-dynamic-extensions --enable-option-checking $(package)_config_opts+= --disable-rtree --disable-fts4 --disable-fts5 -$(package)_config_opts_linux=--with-pic -$(package)_config_opts_freebsd=--with-pic -$(package)_config_opts_netbsd=--with-pic -$(package)_config_opts_openbsd=--with-pic # We avoid using `--enable-debug` because it overrides CFLAGS, a behavior we want to prevent. -$(package)_cflags_debug += -g $(package)_cppflags_debug += -DSQLITE_DEBUG $(package)_cppflags+=-DSQLITE_DQS=0 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED $(package)_cppflags+=-DSQLITE_OMIT_SHARED_CACHE -DSQLITE_OMIT_JSON -DSQLITE_LIKE_DOESNT_MATCH_BLOBS diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index cc78999dbb..bfa5e97c60 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -11,11 +11,6 @@ define $(package)_set_vars $(package)_config_opts += --without-libsodium --without-libgssapi_krb5 --without-pgm --without-norm --without-vmci $(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov --disable-dependency-tracking $(package)_config_opts += --disable-Werror --disable-drafts --enable-option-checking - $(package)_config_opts_linux=--with-pic - $(package)_config_opts_freebsd=--with-pic - $(package)_config_opts_netbsd=--with-pic - $(package)_config_opts_openbsd=--with-pic - $(package)_config_opts_android=--with-pic endef define $(package)_preprocess_cmds diff --git a/depends/patches/boost/process_macos_sdk.patch b/depends/patches/boost/process_macos_sdk.patch deleted file mode 100644 index ebc556d972..0000000000 --- a/depends/patches/boost/process_macos_sdk.patch +++ /dev/null @@ -1,27 +0,0 @@ -Fix Boost Process compilation with macOS 14 SDK. -Can be dropped with Boost 1.84.0. -https://github.com/boostorg/process/pull/343. -https://github.com/boostorg/process/issues/342. - -diff --git a/boost/process/detail/posix/handles.hpp b/boost/process/detail/posix/handles.hpp -index cd9e1ce5a..304e77b1c 100644 ---- a/boost/process/detail/posix/handles.hpp -+++ b/boost/process/detail/posix/handles.hpp -@@ -33,7 +33,7 @@ inline std::vector<native_handle_type> get_handles(std::error_code & ec) - else - ec.clear(); - -- auto my_fd = ::dirfd(dir.get()); -+ auto my_fd = dirfd(dir.get()); - - struct ::dirent * ent_p; - -@@ -117,7 +117,7 @@ struct limit_handles_ : handler_base_ext - return; - } - -- auto my_fd = ::dirfd(dir); -+ auto my_fd = dirfd(dir); - struct ::dirent * ent_p; - - while ((ent_p = readdir(dir)) != nullptr) diff --git a/depends/patches/libnatpmp/no_libtool.patch b/depends/patches/libnatpmp/no_libtool.patch deleted file mode 100644 index 2b9f01f6eb..0000000000 --- a/depends/patches/libnatpmp/no_libtool.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff -ruN libnatpmp-07004b97cf691774efebe70404cf22201e4d330d/Makefile libnatpmp-07004b97cf691774efebe70404cf22201e4d330d.new/Makefile ---- libnatpmp-07004b97cf691774efebe70404cf22201e4d330d/Makefile 2022-07-05 07:49:50.000000000 +0000 -+++ libnatpmp-07004b97cf691774efebe70404cf22201e4d330d.new/Makefile 2024-01-23 20:59:35.674821779 +0000 -@@ -197,11 +197,7 @@ - $(CC) $(LDFLAGS) -o $@ $^ $(LDLIBS) - - $(STATICLIB): $(LIBOBJS) --ifneq (, $(findstring darwin, $(OS))) -- $(LIBTOOL) -static -o $@ $? --else - $(AR) crs $@ $? --endif - - $(SHAREDLIB): $(LIBOBJS) - ifneq (, $(findstring darwin, $(OS))) diff --git a/depends/patches/qrencode/cmake_fixups.patch b/depends/patches/qrencode/cmake_fixups.patch new file mode 100644 index 0000000000..7518d756cb --- /dev/null +++ b/depends/patches/qrencode/cmake_fixups.patch @@ -0,0 +1,23 @@ +cmake: set minimum version to 3.5 + +Correct some dev warning output. + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 773e037..a558145 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -1,4 +1,4 @@ +-cmake_minimum_required(VERSION 3.1.0) ++cmake_minimum_required(VERSION 3.5) + + project(QRencode VERSION 4.1.1 LANGUAGES C) + +@@ -20,7 +20,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + set(CMAKE_THREAD_PREFER_PTHREAD ON) + find_package(Threads) + find_package(PNG) +-find_package(Iconv) ++find_package(ICONV) + + if(CMAKE_USE_PTHREADS_INIT) + add_definitions(-DHAVE_LIBPTHREAD=1) diff --git a/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch b/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch deleted file mode 100644 index d4d6539f56..0000000000 --- a/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch +++ /dev/null @@ -1,20 +0,0 @@ -Modify the optimisation flags for FastFixedDtoa. -This fixes a non-determinism issue in the asm produced for -this function when cross-compiling on x86_64 and aarch64 for -the arm-linux-gnueabihf HOST. - ---- a/qtbase/src/3rdparty/double-conversion/fixed-dtoa.h -+++ b/qtbase/src/3rdparty/double-conversion/fixed-dtoa.h -@@ -48,9 +48,12 @@ namespace double_conversion { - // - // This method only works for some parameters. If it can't handle the input it - // returns false. The output is null-terminated when the function succeeds. -+#pragma GCC push_options -+#pragma GCC optimize ("-O1") - bool FastFixedDtoa(double v, int fractional_count, - Vector<char> buffer, int* length, int* decimal_point); - -+#pragma GCC pop_options - } // namespace double_conversion - - #endif // DOUBLE_CONVERSION_FIXED_DTOA_H_ diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch index 89c96026fb..79824f244a 100644 --- a/depends/patches/qt/fix_android_jni_static.patch +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -15,4 +15,3 @@ QWindowSystemInterfacePrivate::TabletEvent::setPlatformSynthesizesMouse(false); m_javaVM = vm; - diff --git a/depends/patches/qt/memory_resource.patch b/depends/patches/qt/memory_resource.patch index 650c328528..312f0669f6 100644 --- a/depends/patches/qt/memory_resource.patch +++ b/depends/patches/qt/memory_resource.patch @@ -17,7 +17,7 @@ and https://bugreports.qt.io/browse/QTBUG-114316 --- a/qtbase/src/corelib/global/qcompilerdetection.h +++ b/qtbase/src/corelib/global/qcompilerdetection.h -@@ -1050,16 +1050,22 @@ +@@ -1055,16 +1055,22 @@ # endif // !_HAS_CONSTEXPR # endif // !__GLIBCXX__ && !_LIBCPP_VERSION # endif // Q_OS_QNX diff --git a/depends/patches/qt/use_android_ndk23.patch b/depends/patches/qt/use_android_ndk23.patch deleted file mode 100644 index f22367d527..0000000000 --- a/depends/patches/qt/use_android_ndk23.patch +++ /dev/null @@ -1,13 +0,0 @@ -Use Android NDK r23 LTS - ---- old/qtbase/mkspecs/features/android/default_pre.prf -+++ new/qtbase/mkspecs/features/android/default_pre.prf -@@ -76,7 +76,7 @@ else: equals(QT_ARCH, x86_64): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/x86_64-linux- - else: equals(QT_ARCH, arm64-v8a): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/aarch64-linux-android- - else: CROSS_COMPILE = $$NDK_LLVM_PATH/bin/arm-linux-androideabi- - --QMAKE_RANLIB = $${CROSS_COMPILE}ranlib -+QMAKE_RANLIB = $$NDK_LLVM_PATH/bin/llvm-ranlib - QMAKE_LINK_SHLIB = $$QMAKE_LINK - QMAKE_LFLAGS = - diff --git a/doc/README.md b/doc/README.md index 446684b482..7b6dacaf4f 100644 --- a/doc/README.md +++ b/doc/README.md @@ -59,7 +59,6 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Translation Strings Policy](translation_strings_policy.md) - [JSON-RPC Interface](JSON-RPC-interface.md) - [Unauthenticated REST Interface](REST-interface.md) -- [Shared Libraries](shared-libraries.md) - [BIPS](bips.md) - [Dnsseed Policy](dnsseed-policy.md) - [Benchmarking](benchmarking.md) diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index aa10e4a891..bf86a0ee4b 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -1,6 +1,6 @@ # FreeBSD Build Guide -**Updated for FreeBSD [12.3](https://www.freebsd.org/releases/12.3R/announce/)** +**Updated for FreeBSD [14.0](https://www.freebsd.org/releases/14.0R/announce/)** This guide describes how to build bitcoind, command-line utilities, and GUI on FreeBSD. @@ -64,10 +64,11 @@ sh/bash: export BDB_PREFIX=[path displayed above] #### GUI Dependencies ###### Qt5 -Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, we need to install `qt5`. Skip if you don't intend to use the GUI. +Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, we need to install the necessary parts of Qt. Skip if you don't intend to use the GUI. ```bash -pkg install qt5 +pkg install qt5-buildtools qt5-core qt5-gui qt5-linguisttools qt5-testlib qt5-widgets ``` + ###### libqrencode The GUI can encode addresses in a QR Code. To build in QR support for the GUI, install `libqrencode`. Skip if not using the GUI or don't want QR code functionality. diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 7ed83853a8..264d5f2d08 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -1,6 +1,6 @@ # OpenBSD Build Guide -**Updated for OpenBSD [7.4](https://www.openbsd.org/74.html)** +**Updated for OpenBSD [7.5](https://www.openbsd.org/75.html)** This guide describes how to build bitcoind, command-line utilities, and GUI on OpenBSD. @@ -63,7 +63,7 @@ export BDB_PREFIX="/path/to/bitcoin/depends/x86_64-unknown-openbsd" Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, Qt 5 is required. ```bash -pkg_add qt5 +pkg_add qtbase qttools ``` ## Building Bitcoin Core diff --git a/doc/build-unix.md b/doc/build-unix.md index bf367fc421..de54fb4eeb 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -30,7 +30,7 @@ tuned to conserve memory with additional CXXFLAGS: Alternatively, or in addition, debugging information can be skipped for compilation. The default compile flags are `-g -O2`, and can be changed with: - ./configure CXXFLAGS="-O2" + ./configure CXXFLAGS="-g0" Finally, clang (often less resource hungry) can be used instead of gcc, which is used by default: @@ -81,7 +81,7 @@ To build without GUI pass `--without-gui`. To build with Qt 5 you need the following: - sudo apt-get install libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools + sudo apt-get install qtbase5-dev qttools5-dev qttools5-dev-tools Additionally, to support Wayland protocol for modern desktop environments: diff --git a/doc/dependencies.md b/doc/dependencies.md index e992b50b06..4003f22b91 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -9,7 +9,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Autoconf](https://www.gnu.org/software/autoconf/) | [2.69](https://github.com/bitcoin/bitcoin/pull/17769) | | [Automake](https://www.gnu.org/software/automake/) | [1.13](https://github.com/bitcoin/bitcoin/pull/18290) | | [Clang](https://clang.llvm.org) | [14.0](https://github.com/bitcoin/bitcoin/pull/29208) | -| [GCC](https://gcc.gnu.org) | [10.1](https://github.com/bitcoin/bitcoin/pull/28348) | +| [GCC](https://gcc.gnu.org) | [11.1](https://github.com/bitcoin/bitcoin/pull/29091) | | [Python](https://www.python.org) (scripts, tests) | [3.9](https://github.com/bitcoin/bitcoin/pull/28211) | | [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | @@ -30,12 +30,12 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Fontconfig](../depends/packages/fontconfig.mk) | [link](https://www.freedesktop.org/wiki/Software/fontconfig/) | [2.12.6](https://github.com/bitcoin/bitcoin/pull/23495) | 2.6 | Yes | | [FreeType](../depends/packages/freetype.mk) | [link](https://freetype.org) | [2.11.0](https://github.com/bitcoin/bitcoin/commit/01544dd78ccc0b0474571da854e27adef97137fb) | 2.3.0 | Yes | | [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [4.1.1](https://github.com/bitcoin/bitcoin/pull/27312) | | No | -| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.11](https://github.com/bitcoin/bitcoin/pull/28769) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | +| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.13](https://github.com/bitcoin/bitcoin/pull/29732) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | ### Networking | Dependency | Releases | Version used | Minimum required | Runtime | | --- | --- | --- | --- | --- | -| [libnatpmp](../depends/packages/libnatpmp.mk) | [link](https://github.com/miniupnp/libnatpmp/) | commit [07004b9...](https://github.com/bitcoin/bitcoin/pull/25917) | | No | +| [libnatpmp](../depends/packages/libnatpmp.mk) | [link](https://github.com/miniupnp/libnatpmp/) | commit [f2433be...](https://github.com/bitcoin/bitcoin/pull/29708) | | No | | [MiniUPnPc](../depends/packages/miniupnpc.mk) | [link](https://miniupnp.tuxfamily.org/) | [2.2.2](https://github.com/bitcoin/bitcoin/pull/20421) | 2.1 | No | ### Notifications diff --git a/doc/descriptors.md b/doc/descriptors.md index 1baf652f30..3b94ec03e4 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -243,7 +243,18 @@ Often it is useful to communicate a description of scripts along with the necessary private keys. For this reason, anywhere a public key or xpub is supported, a private key in WIF format or xprv may be provided instead. This is useful when private keys are necessary for hardened derivation -steps, or for dumping wallet descriptors including private key material. +steps, for signing transactions, or for dumping wallet descriptors +including private key material. + +For example, after importing the following 2-of-3 multisig descriptor +into a wallet, one could use `signrawtransactionwithwallet` +to sign a transaction with the first key: +``` +sh(multi(2,xprv.../84'/0'/0'/0/0,xpub1...,xpub2...)) +``` +Note how the first key is an xprv private key with a specific derivation path, +while the other two are public keys. + ### Compatibility with old wallets diff --git a/doc/design/assumeutxo.md b/doc/design/assumeutxo.md index abb623fc69..a4980729d0 100644 --- a/doc/design/assumeutxo.md +++ b/doc/design/assumeutxo.md @@ -16,7 +16,7 @@ load it. A pruned node can load a snapshot. To save space, it's possible to delete the snapshot file as soon as `loadtxoutset` finishes. -The minimum `-dbcache` setting is 550 MiB, but this functionality ignores that +The minimum `-prune` setting is 550 MiB, but this functionality ignores that minimum and uses at least 1100 MiB. As the background sync continues there will be temporarily two chainstate @@ -51,18 +51,12 @@ The utility script ## Design notes -- A new block index `nStatus` flag is introduced, `BLOCK_ASSUMED_VALID`, to mark block - index entries that are required to be assumed-valid by a chainstate created - from a UTXO snapshot. This flag is used as a way to modify certain - CheckBlockIndex() logic to account for index entries that are pending validation by a - chainstate running asynchronously in the background. - - The concept of UTXO snapshots is treated as an implementation detail that lives behind the ChainstateManager interface. The external presentation of the changes required to facilitate the use of UTXO snapshots is the understanding that there are - now certain regions of the chain that can be temporarily assumed to be valid (using - the nStatus flag mentioned above). In certain cases, e.g. wallet rescanning, this is - very similar to dealing with a pruned chain. + now certain regions of the chain that can be temporarily assumed to be valid. + In certain cases, e.g. wallet rescanning, this is very similar to dealing with + a pruned chain. Logic outside ChainstateManager should try not to know about snapshots, instead preferring to work in terms of more general states like assumed-valid. diff --git a/doc/design/libraries.md b/doc/design/libraries.md index 7cda64e713..3346c8e81b 100644 --- a/doc/design/libraries.md +++ b/doc/design/libraries.md @@ -4,10 +4,9 @@ |--------------------------|-------------| | *libbitcoin_cli* | RPC client functionality used by *bitcoin-cli* executable | | *libbitcoin_common* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_util*, but higher-level (see [Dependencies](#dependencies)). | -| *libbitcoin_consensus* | Stable, backwards-compatible consensus functionality used by *libbitcoin_node* and *libbitcoin_wallet* and also exposed as a [shared library](../shared-libraries.md). | -| *libbitcoinconsensus* | Shared library build of static *libbitcoin_consensus* library | -| *libbitcoin_kernel* | Consensus engine and support library used for validation by *libbitcoin_node* and also exposed as a [shared library](../shared-libraries.md). | -| *libbitcoinqt* | GUI functionality used by *bitcoin-qt* and *bitcoin-gui* executables | +| *libbitcoin_consensus* | Stable, backwards-compatible consensus functionality used by *libbitcoin_node* and *libbitcoin_wallet*. | +| *libbitcoin_kernel* | Consensus engine and support library used for validation by *libbitcoin_node*. | +| *libbitcoinqt* | GUI functionality used by *bitcoin-qt* and *bitcoin-gui* executables. | | *libbitcoin_ipc* | IPC functionality used by *bitcoin-node*, *bitcoin-wallet*, *bitcoin-gui* executables to communicate when [`--enable-multiprocess`](multiprocess.md) is used. | | *libbitcoin_node* | P2P and RPC server functionality used by *bitcoind* and *bitcoin-qt* executables. | | *libbitcoin_util* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_common*, but lower-level (see [Dependencies](#dependencies)). | @@ -17,7 +16,7 @@ ## Conventions -- Most libraries are internal libraries and have APIs which are completely unstable! There are few or no restrictions on backwards compatibility or rules about external dependencies. Exceptions are *libbitcoin_consensus* and *libbitcoin_kernel* which have external interfaces documented at [../shared-libraries.md](../shared-libraries.md). +- Most libraries are internal libraries and have APIs which are completely unstable! There are few or no restrictions on backwards compatibility or rules about external dependencies. An exception is *libbitcoin_kernel*, which, at some future point, will have a documented external interface. - Generally each library should have a corresponding source directory and namespace. Source code organization is a work in progress, so it is true that some namespaces are applied inconsistently, and if you look at [`libbitcoin_*_SOURCES`](../../src/Makefile.am) lists you can see that many libraries pull in files from outside their source directory. But when working with libraries, it is good to follow a consistent pattern like: diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 89c13600eb..cc3f0518e5 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -115,6 +115,8 @@ code. Use `reinterpret_cast` and `const_cast` as appropriate. - Prefer [`list initialization ({})`](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Res-list) where possible. For example `int x{0};` instead of `int x = 0;` or `int x(0);` + - Recursion is checked by clang-tidy and thus must be made explicit. Use + `NOLINTNEXTLINE(misc-no-recursion)` to suppress the check. For function calls a namespace should be specified explicitly, unless such functions have been declared within it. Otherwise, [argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl), also known as ADL, could be diff --git a/doc/dnsseed-policy.md b/doc/dnsseed-policy.md index 55a5c28258..99f48f2670 100644 --- a/doc/dnsseed-policy.md +++ b/doc/dnsseed-policy.md @@ -44,7 +44,7 @@ related to the DNS seed operation. If these expectations cannot be satisfied the operator should discontinue providing services and contact the active Bitcoin Core development team as well as posting on -[bitcoin-dev](https://lists.linuxfoundation.org/mailman/listinfo/bitcoin-dev). +[bitcoin-dev](https://groups.google.com/g/bitcoindev). Behavior outside of these expectations may be reasonable in some situations but should be discussed in public in advance. diff --git a/doc/external-signer.md b/doc/external-signer.md index de44cdd880..1468e852ef 100644 --- a/doc/external-signer.md +++ b/doc/external-signer.md @@ -150,6 +150,9 @@ Example, display the first native SegWit receive address on Testnet: The command MUST be able to figure out the address type from the descriptor. +The command MUST return an object containing `{"address": "[the address]"}`. +As a sanity check, for devices that support this, it SHOULD ask the device to derive the address. + If <descriptor> contains a master key fingerprint, the command MUST fail if it does not match the fingerprint known by the device. If <descriptor> contains an xpub, the command MUST fail if it does not match the xpub known by the device. diff --git a/doc/managing-wallets.md b/doc/managing-wallets.md index 22e006c963..b99d88877b 100644 --- a/doc/managing-wallets.md +++ b/doc/managing-wallets.md @@ -122,6 +122,22 @@ $ bitcoin-cli -rpcwallet="restored-wallet" getwalletinfo The restored wallet can also be loaded in the GUI via `File` ->`Open wallet`. +## Wallet Passphrase + +Understanding wallet security is crucial for safely storing your Bitcoin. A key aspect is the wallet passphrase, used for encryption. Let's explore its nuances, role, encryption process, and limitations. + +- **Not the Seed:** +The wallet passphrase and the seed are two separate components in wallet security. The seed, or HD seed, functions as a master key for deriving private and public keys in a hierarchical deterministic (HD) wallet. In contrast, the passphrase serves as an additional layer of security specifically designed to secure the private keys within the wallet. The passphrase serves as a safeguard, demanding an additional layer of authentication to access funds in the wallet. + +- **Protection Against Unauthorized Access:** +The passphrase serves as a protective measure, securing your funds in situations where an unauthorized user gains access to your unlocked computer or device while your wallet application is active. Without the passphrase, they would be unable to access your wallet's funds or execute transactions. However, it's essential to be aware that someone with access can potentially compromise the security of your passphrase by installing a keylogger. + +- **Doesn't Encrypt Metadata or Public Keys:** +It's important to note that the passphrase primarily secures the private keys and access to funds within the wallet. It does not encrypt metadata associated with transactions or public keys. Information about your transaction history and the public keys involved may still be visible. + +- **Risk of Fund Loss if Forgotten or Lost:** +If the wallet passphrase is too complex and is subsequently forgotten or lost, there is a risk of losing access to the funds permanently. A forgotten passphrase will result in the inability to unlock the wallet and access the funds. + ## Migrating Legacy Wallets to Descriptor Wallets Legacy wallets (traditional non-descriptor wallets) can be migrated to become Descriptor wallets diff --git a/doc/release-notes-27114.md b/doc/release-notes-27114.md new file mode 100644 index 0000000000..980ffd78a4 --- /dev/null +++ b/doc/release-notes-27114.md @@ -0,0 +1,2 @@ +- Additional flags "in" and "out" have been added to `-whitelist` to control whether + permissions apply to incoming connections and/or manual (default: incoming only).
\ No newline at end of file diff --git a/doc/release-notes-27375.md b/doc/release-notes-27375.md new file mode 100644 index 0000000000..e3f4ebdf77 --- /dev/null +++ b/doc/release-notes-27375.md @@ -0,0 +1,6 @@ +P2P +--- + +UNIX domain sockets can now be used for proxy connections. Set `-onion` or `-proxy` +to the local socket path with the prefix `unix:` (e.g. `-onion=unix:/home/me/torsocket`). +(#27375)
\ No newline at end of file diff --git a/doc/release-notes-27679.md b/doc/release-notes-27679.md new file mode 100644 index 0000000000..dbbb30428c --- /dev/null +++ b/doc/release-notes-27679.md @@ -0,0 +1,2 @@ +- unix socket paths are now accepted for `-zmqpubrawblock` and `-zmqpubrawtx` with +the format `-zmqpubrawtx=unix:/path/to/file`
\ No newline at end of file diff --git a/doc/release-notes/release-notes-25.2.md b/doc/release-notes/release-notes-25.2.md new file mode 100644 index 0000000000..3f050ebaef --- /dev/null +++ b/doc/release-notes/release-notes-25.2.md @@ -0,0 +1,74 @@ +25.2 Release Notes +================== + +Bitcoin Core version 25.2 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-25.2> + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +### Gui + +- gui#774 Fix crash on selecting "Mask values" in transaction view + +### RPC + +- #29003 rpc: fix getrawtransaction segfault + +### Wallet + +- #29176 wallet: Fix use-after-free in WalletBatch::EraseRecords +- #29510 wallet: `getrawchangeaddress` and `getnewaddress` failures should not affect keypools for descriptor wallets + +### P2P and network changes + +- #29412 p2p: Don't process mutated blocks +- #29524 p2p: Don't consider blocks mutated if they don't connect to known prev block + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Martin Zumsande +- Sebastian Falbesoner +- MarcoFalke +- UdjinM6 +- dergoegge +- Greg Sanders + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-26.1.md b/doc/release-notes/release-notes-26.1.md new file mode 100644 index 0000000000..b5c6e63007 --- /dev/null +++ b/doc/release-notes/release-notes-26.1.md @@ -0,0 +1,106 @@ +26.1 Release Notes +================== + +Bitcoin Core version 26.1 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-26.1/> + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 11.0+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +### Wallet + +- #28994 wallet: skip BnB when SFFO is enabled +- #28920 wallet: birth time update during tx scanning +- #29176 wallet: Fix use-after-free in WalletBatch::EraseRecords +- #29510 wallet: getrawchangeaddress and getnewaddress failures should not affect keypools for descriptor wallets + +### RPC + +- #29003 rpc: fix getrawtransaction segfault +- #28784 rpc: keep .cookie file if it was not generated + +### Logs + +- #29227 log mempool loading progress + +### P2P and network changes + +- #29200 net: create I2P sessions using both ECIES-X25519 and ElGamal encryption +- #29412 p2p: Don't process mutated blocks +- #29524 p2p: Don't consider blocks mutated if they don't connect to known prev block + +### Build + +- #29127 Use hardened runtime on macOS release builds. +- #29195 build: Fix -Xclang -internal-isystem option + +### CI + +- #28992 ci: Use Ubuntu 24.04 Noble for asan,tsan,tidy,fuzz +- #29080 ci: Set HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK to avoid unrelated failures +- #29610 ci: Fix "macOS native" job + +### Miscellaneous + +- #28391 refactor: Simplify CTxMempool/BlockAssembler fields, remove some external mapTx access +- #29179 test: wallet rescan with reorged parent + IsFromMe child in mempool +- #28791 snapshots: don't core dump when running -checkblockindex after loadtxoutset +- #29357 test: Drop x modifier in fsbridge::fopen call for MinGW builds +- #29529 fuzz: restrict fopencookie usage to Linux & FreeBSD + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- dergoegge +- fanquake +- furszy +- glozow +- Greg Sanders +- Hennadii Stepanov +- Jon Atack +- MarcoFalke +- Mark Friedenbach +- Martin Zumsande +- Murch +- Roman Zeyde +- stickies-v +- UdjinM6 + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). + diff --git a/doc/release-notes/release-notes-27.0.md b/doc/release-notes/release-notes-27.0.md new file mode 100644 index 0000000000..5060068328 --- /dev/null +++ b/doc/release-notes/release-notes-27.0.md @@ -0,0 +1,217 @@ +Bitcoin Core version 27.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-27.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux Kernel 3.17+, macOS 11.0+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +libbitcoinconsensus +------------------- + +- libbitcoinconsensus is deprecated and will be removed for v28. This library has + existed for nearly 10 years with very little known uptake or impact. It has + become a maintenance burden. + + The underlying functionality does not change between versions, so any users of + the library can continue to use the final release indefinitely, with the + understanding that Taproot is its final consensus update. + + In the future, libbitcoinkernel will provide a much more useful API that is + aware of the UTXO set, and therefore be able to fully validate transactions and + blocks. (#29189) + +mempool.dat compatibility +------------------------- + +- The `mempool.dat` file created by -persistmempool or the savemempool RPC will + be written in a new format. This new format includes the XOR'ing of transaction + contents to mitigate issues where external programs (such as anti-virus) attempt + to interpret and potentially modify the file. + + This new format can not be read by previous software releases. To allow for a + downgrade, a temporary setting `-persistmempoolv1` has been added to fall back + to the legacy format. (#28207) + +P2P and network changes +----------------------- + +- BIP324 v2 transport is now enabled by default. It remains possible to disable v2 + by running with `-v2transport=0`. (#29347) +- Manual connection options (`-connect`, `-addnode` and `-seednode`) will + now follow `-v2transport` to connect with v2 by default. They will retry with + v1 on failure. (#29058) + +- Network-adjusted time has been removed from consensus code. It is replaced + with (unadjusted) system time. The warning for a large median time offset + (70 minutes or more) is kept. This removes the implicit security assumption of + requiring an honest majority of outbound peers, and increases the importance + of the node operator ensuring their system time is (and stays) correct to not + fall out of consensus with the network. (#28956) + +Mempool Policy Changes +---------------------- + +- Opt-in Topologically Restricted Until Confirmation (TRUC) Transactions policy + (aka v3 transaction policy) is available for use on test networks when + `-acceptnonstdtxn=1` is set. By setting the transaction version number to 3, TRUC transactions + request the application of limits on spending of their unconfirmed outputs. These + restrictions simplify the assessment of incentive compatibility of accepting or + replacing TRUC transactions, thus ensuring any replacements are more profitable for + the node and making fee-bumping more reliable. TRUC transactions are currently + nonstandard and can only be used on test networks where the standardness rules are + relaxed or disabled (e.g. with `-acceptnonstdtxn=1`). (#28948) + +External Signing +---------------- + +- Support for external signing on Windows has been disabled. It will be re-enabled + once the underlying dependency (Boost Process), has been replaced with a different + library. (#28967) + +Updated RPCs +------------ + +- The addnode RPC now follows the `-v2transport` option (now on by default, see above) for making connections. + It remains possible to specify the transport type manually with the v2transport argument of addnode. (#29239) + +Build System +------------ + +- A C++20 capable compiler is now required to build Bitcoin Core. (#28349) +- MacOS releases are configured to use the hardened runtime libraries (#29127) + +Wallet +------ + +- The CoinGrinder coin selection algorithm has been introduced to mitigate unnecessary + large input sets and lower transaction costs at high feerates. CoinGrinder + searches for the input set with minimal weight. Solutions found by + CoinGrinder will produce a change output. CoinGrinder is only active at + elevated feerates (default: 30+ sat/vB, based on `-consolidatefeerate`×3). (#27877) +- The Branch And Bound coin selection algorithm will be disabled when the subtract fee + from outputs feature is used. (#28994) +- If the birth time of a descriptor is detected to be later than the first transaction + involving that descriptor, the birth time will be reset to the earlier time. (#28920) + +Low-level changes +================= + +Pruning +------- + +- When pruning during initial block download, more blocks will be pruned at each + flush in order to speed up the syncing of such nodes. (#20827) + +Init +---- + +- Various fixes to prevent issues where subsequent instances of Bitcoin Core would + result in deletion of files in use by an existing instance. (#28784, #28946) +- Improved handling of empty `settings.json` files. (#29144) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 22388o⚡️ +- Aaron Clauson +- Amiti Uttarwar +- Andrew Toth +- Anthony Towns +- Antoine Poinsot +- Ava Chow +- Brandon Odiwuor +- brunoerg +- Chris Stewart +- Cory Fields +- dergoegge +- djschnei21 +- Fabian Jahr +- fanquake +- furszy +- Gloria Zhao +- Greg Sanders +- Hennadii Stepanov +- Hernan Marino +- iamcarlos94 +- ismaelsadeeq +- Jameson Lopp +- Jesse Barton +- John Moffett +- Jon Atack +- josibake +- jrakibi +- Justin Dhillon +- Kashif Smith +- kevkevin +- Kristaps Kaupe +- L0la L33tz +- Luke Dashjr +- Lőrinc +- marco +- MarcoFalke +- Mark Friedenbach +- Marnix +- Martin Leitner-Ankerl +- Martin Zumsande +- Max Edwards +- Murch +- muxator +- naiyoma +- Nikodemas Tuckus +- ns-xvrn +- pablomartin4btc +- Peter Todd +- Pieter Wuille +- Richard Myers +- Roman Zeyde +- Russell Yanofsky +- Ryan Ofsky +- Sebastian Falbesoner +- Sergi Delgado Segura +- Sjors Provoost +- stickies-v +- stratospher +- Supachai Kheawjuy +- TheCharlatan +- UdjinM6 +- Vasil Dimov +- w0xlt +- willcl-ark + + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/shared-libraries.md b/doc/shared-libraries.md deleted file mode 100644 index 3a448c6556..0000000000 --- a/doc/shared-libraries.md +++ /dev/null @@ -1,78 +0,0 @@ -Shared Libraries -================ - -## bitcoinconsensus -***This library is deprecated and will be removed in v28*** - -The purpose of this library is to make the verification functionality that is critical to Bitcoin's consensus available to other applications, e.g. to language bindings. - -### API - -The interface is defined in the C header `bitcoinconsensus.h` located in `src/script/bitcoinconsensus.h`. - -#### Version - -`bitcoinconsensus_version` returns an `unsigned int` with the API version *(currently `2`)*. - -#### Script Validation - -`bitcoinconsensus_verify_script`, `bitcoinconsensus_verify_script_with_amount` and `bitcoinconsensus_verify_script_with_spent_outputs` return an `int` with the status of the verification. It will be `1` if the input script correctly spends the previous output `scriptPubKey`. - -##### Parameters -###### bitcoinconsensus_verify_script -- `const unsigned char *scriptPubKey` - The previous output script that encumbers spending. -- `unsigned int scriptPubKeyLen` - The number of bytes for the `scriptPubKey`. -- `const unsigned char *txTo` - The transaction with the input that is spending the previous output. -- `unsigned int txToLen` - The number of bytes for the `txTo`. -- `unsigned int nIn` - The index of the input in `txTo` that spends the `scriptPubKey`. -- `unsigned int flags` - The script validation flags *(see below)*. -- `bitcoinconsensus_error* err` - Will have the error/success code for the operation *(see below)*. - -###### bitcoinconsensus_verify_script_with_amount -- `const unsigned char *scriptPubKey` - The previous output script that encumbers spending. -- `unsigned int scriptPubKeyLen` - The number of bytes for the `scriptPubKey`. -- `int64_t amount` - The amount spent in the input -- `const unsigned char *txTo` - The transaction with the input that is spending the previous output. -- `unsigned int txToLen` - The number of bytes for the `txTo`. -- `unsigned int nIn` - The index of the input in `txTo` that spends the `scriptPubKey`. -- `unsigned int flags` - The script validation flags *(see below)*. -- `bitcoinconsensus_error* err` - Will have the error/success code for the operation *(see below)*. - -###### bitcoinconsensus_verify_script_with_spent_outputs -- `const unsigned char *scriptPubKey` - The previous output script that encumbers spending. -- `unsigned int scriptPubKeyLen` - The number of bytes for the `scriptPubKey`. -- `int64_t amount` - The amount spent in the input -- `const unsigned char *txTo` - The transaction with the input that is spending the previous output. -- `unsigned int txToLen` - The number of bytes for the `txTo`. -- `UTXO *spentOutputs` - Previous outputs spent in the transaction. `UTXO` is a struct composed by `const unsigned char *scriptPubKey`, `unsigned int scriptPubKeySize` (the number of bytes for the `scriptPubKey`) and `unsigned int value`. -- `unsigned int spentOutputsLen` - The number of bytes for the `spentOutputs`. -- `unsigned int nIn` - The index of the input in `txTo` that spends the `scriptPubKey`. -- `unsigned int flags` - The script validation flags *(see below)*. -- `bitcoinconsensus_error* err` - Will have the error/success code for the operation *(see below)*. - -##### Script Flags -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NONE` -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_P2SH` - Evaluate P2SH ([BIP16](https://github.com/bitcoin/bips/blob/master/bip-0016.mediawiki)) subscripts -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_DERSIG` - Enforce strict DER ([BIP66](https://github.com/bitcoin/bips/blob/master/bip-0066.mediawiki)) compliance -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NULLDUMMY` - Enforce NULLDUMMY ([BIP147](https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki)) -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY` - Enable CHECKLOCKTIMEVERIFY ([BIP65](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki)) -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKSEQUENCEVERIFY` - Enable CHECKSEQUENCEVERIFY ([BIP112](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki)) -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS` - Enable WITNESS ([BIP141](https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki)) -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT` - Enable TAPROOT ([BIP340](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki), [BIP341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki), [BIP342](https://github.com/bitcoin/bips/blob/master/bip-0342.mediawiki)) - -##### Errors -- `bitcoinconsensus_ERR_OK` - No errors with input parameters *(see the return value of `bitcoinconsensus_verify_script` for the verification status)* -- `bitcoinconsensus_ERR_TX_INDEX` - An invalid index for `txTo` -- `bitcoinconsensus_ERR_TX_SIZE_MISMATCH` - `txToLen` did not match with the size of `txTo` -- `bitcoinconsensus_ERR_DESERIALIZE` - An error deserializing `txTo` -- `bitcoinconsensus_ERR_AMOUNT_REQUIRED` - Input amount is required if WITNESS is used -- `bitcoinconsensus_ERR_INVALID_FLAGS` - Script verification `flags` are invalid (i.e. not part of the libconsensus interface) -- `bitcoinconsensus_ERR_SPENT_OUTPUTS_REQUIRED` - Spent outputs are required if TAPROOT is used -- `bitcoinconsensus_ERR_SPENT_OUTPUTS_MISMATCH` - Spent outputs size doesn't match tx inputs size - -### Example Implementations -- [NBitcoin](https://github.com/MetacoSA/NBitcoin/blob/5e1055cd7c4186dee4227c344af8892aea54faec/NBitcoin/Script.cs#L979-#L1031) (.NET Bindings) -- [node-libbitcoinconsensus](https://github.com/bitpay/node-libbitcoinconsensus) (Node.js Bindings) -- [java-libbitcoinconsensus](https://github.com/dexX7/java-libbitcoinconsensus) (Java Bindings) -- [bitcoinconsensus-php](https://github.com/Bit-Wasp/bitcoinconsensus-php) (PHP Bindings) -- [rust-bitcoinconsensus](https://github.com/rust-bitcoin/rust-bitcoinconsensus) (Rust Bindings)
\ No newline at end of file diff --git a/libbitcoinconsensus.pc.in b/libbitcoinconsensus.pc.in deleted file mode 100644 index 1ceab280bb..0000000000 --- a/libbitcoinconsensus.pc.in +++ /dev/null @@ -1,10 +0,0 @@ -prefix=@prefix@ -exec_prefix=@exec_prefix@ -libdir=@libdir@ -includedir=@includedir@ - -Name: @PACKAGE_NAME@ consensus library -Description: Library for the Bitcoin consensus protocol. -Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lbitcoinconsensus -Cflags: -I${includedir} diff --git a/src/.clang-tidy b/src/.clang-tidy index bfaa5ab8e7..a00400f083 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -6,12 +6,14 @@ bugprone-string-constructor, bugprone-use-after-move, bugprone-lambda-function-name, misc-unused-using-decls, +misc-no-recursion, modernize-use-default-member-init, modernize-use-emplace, modernize-use-noexcept, modernize-use-nullptr, performance-*, -performance-avoid-endl, +-performance-enum-size, -performance-inefficient-string-concatenation, -performance-no-int-to-ptr, -performance-noexcept-move-constructor, diff --git a/src/Makefile.am b/src/Makefile.am index 1f55bfbafd..c2e0c7b5b8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -39,9 +39,6 @@ LIBSECP256K1=secp256k1/libsecp256k1.la if ENABLE_ZMQ LIBBITCOIN_ZMQ=libbitcoin_zmq.a endif -if BUILD_BITCOIN_LIBS -LIBBITCOINCONSENSUS=libbitcoinconsensus.la -endif if BUILD_BITCOIN_KERNEL_LIB LIBBITCOINKERNEL=libbitcoinkernel.la endif @@ -302,6 +299,7 @@ BITCOIN_CORE_H = \ util/error.h \ util/exception.h \ util/fastrange.h \ + util/feefrac.h \ util/fees.h \ util/fs.h \ util/fs_helpers.h \ @@ -322,6 +320,7 @@ BITCOIN_CORE_H = \ util/sock.h \ util/spanparsing.h \ util/string.h \ + util/subprocess.h \ util/syserror.h \ util/task_runner.h \ util/thread.h \ @@ -648,7 +647,6 @@ libbitcoin_consensus_a_SOURCES = \ primitives/transaction.h \ pubkey.cpp \ pubkey.h \ - script/bitcoinconsensus.cpp \ script/interpreter.cpp \ script/interpreter.h \ script/script.cpp \ @@ -681,6 +679,7 @@ libbitcoin_common_a_SOURCES = \ common/run_command.cpp \ common/settings.cpp \ common/system.cpp \ + common/url.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ @@ -713,11 +712,6 @@ libbitcoin_common_a_SOURCES = \ script/solver.cpp \ warnings.cpp \ $(BITCOIN_CORE_H) - -if USE_LIBEVENT -libbitcoin_common_a_CPPFLAGS += $(EVENT_CFLAGS) -libbitcoin_common_a_SOURCES += common/url.cpp -endif # # util # @@ -741,6 +735,7 @@ libbitcoin_util_a_SOURCES = \ util/check.cpp \ util/error.cpp \ util/exception.cpp \ + util/feefrac.cpp \ util/fees.cpp \ util/fs.cpp \ util/fs_helpers.cpp \ @@ -983,6 +978,7 @@ libbitcoinkernel_la_SOURCES = \ util/batchpriority.cpp \ util/chaintype.cpp \ util/check.cpp \ + util/feefrac.cpp \ util/fs.cpp \ util/fs_helpers.cpp \ util/hasher.cpp \ @@ -1007,21 +1003,6 @@ libbitcoinkernel_la-clientversion.l$(OBJEXT): obj/build.h endif # BUILD_BITCOIN_KERNEL_LIB # -# bitcoinconsensus library # -if BUILD_BITCOIN_LIBS -lib_LTLIBRARIES += $(LIBBITCOINCONSENSUS) - -include_HEADERS = script/bitcoinconsensus.h -libbitcoinconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_la_SOURCES) $(libbitcoin_consensus_a_SOURCES) - -libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) -libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) -libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL -DDISABLE_OPTIMIZED_SHA256 -libbitcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) - -endif -# - CTAES_DIST = crypto/ctaes/bench.c CTAES_DIST += crypto/ctaes/ctaes.c CTAES_DIST += crypto/ctaes/ctaes.h diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index b24405ce19..7ba0111fa6 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -34,6 +34,7 @@ bench_bench_bitcoin_SOURCES = \ bench/examples.cpp \ bench/gcs_filter.cpp \ bench/hashpadding.cpp \ + bench/index_blockfilter.cpp \ bench/load_external.cpp \ bench/lockedpool.cpp \ bench/logging.cpp \ @@ -42,6 +43,7 @@ bench_bench_bitcoin_SOURCES = \ bench/merkle_root.cpp \ bench/nanobench.cpp \ bench/nanobench.h \ + bench/parse_hex.cpp \ bench/peer_eviction.cpp \ bench/poly1305.cpp \ bench/pool.cpp \ diff --git a/src/Makefile.minisketch.include b/src/Makefile.minisketch.include index 1363bec34e..c6f894f0ca 100644 --- a/src/Makefile.minisketch.include +++ b/src/Makefile.minisketch.include @@ -12,10 +12,6 @@ LIBMINISKETCH_CPPFLAGS += -DHAVE_CLMUL MINISKETCH_LIBS += $(LIBMINISKETCH_CLMUL) endif -if HAVE_CLZ -LIBMINISKETCH_CPPFLAGS += -DHAVE_CLZ -endif - EXTRA_LIBRARIES += $(MINISKETCH_LIBS) minisketch_libminisketch_clmul_a_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 9f9bdbbd0c..942e0bf69b 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -85,6 +85,7 @@ BITCOIN_TESTS =\ test/checkqueue_tests.cpp \ test/coins_tests.cpp \ test/coinstatsindex_tests.cpp \ + test/common_url_tests.cpp \ test/compilerbug_tests.cpp \ test/compress_tests.cpp \ test/crypto_tests.cpp \ @@ -93,6 +94,7 @@ BITCOIN_TESTS =\ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ test/disconnected_transactions.cpp \ + test/feefrac_tests.cpp \ test/flatfile_tests.cpp \ test/fs_tests.cpp \ test/getarg_tests.cpp \ @@ -313,7 +315,9 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/descriptor_parse.cpp \ test/fuzz/deserialize.cpp \ test/fuzz/eval_script.cpp \ + test/fuzz/feefrac.cpp \ test/fuzz/fee_rate.cpp \ + test/fuzz/feeratediagram.cpp \ test/fuzz/fees.cpp \ test/fuzz/flatfile.cpp \ test/fuzz/float.cpp \ @@ -362,7 +366,6 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/rpc.cpp \ test/fuzz/script.cpp \ test/fuzz/script_assets_test_minimizer.cpp \ - test/fuzz/script_bitcoin_consensus.cpp \ test/fuzz/script_descriptor_cache.cpp \ test/fuzz/script_flags.cpp \ test/fuzz/script_format.cpp \ diff --git a/src/addrdb.cpp b/src/addrdb.cpp index fd2a363b8a..14dc314c36 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -44,7 +44,8 @@ bool SerializeDB(Stream& stream, const Data& data) hashwriter << Params().MessageStart() << data; stream << hashwriter.GetHash(); } catch (const std::exception& e) { - return error("%s: Serialize or I/O error - %s", __func__, e.what()); + LogError("%s: Serialize or I/O error - %s\n", __func__, e.what()); + return false; } return true; @@ -64,7 +65,8 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data if (fileout.IsNull()) { fileout.fclose(); remove(pathTmp); - return error("%s: Failed to open file %s", __func__, fs::PathToString(pathTmp)); + LogError("%s: Failed to open file %s\n", __func__, fs::PathToString(pathTmp)); + return false; } // Serialize @@ -76,14 +78,16 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data if (!FileCommit(fileout.Get())) { fileout.fclose(); remove(pathTmp); - return error("%s: Failed to flush file %s", __func__, fs::PathToString(pathTmp)); + LogError("%s: Failed to flush file %s\n", __func__, fs::PathToString(pathTmp)); + return false; } fileout.fclose(); // replace existing file, if any, with new file if (!RenameOver(pathTmp, path)) { remove(pathTmp); - return error("%s: Rename-into-place failed", __func__); + LogError("%s: Rename-into-place failed\n", __func__); + return false; } return true; @@ -140,7 +144,7 @@ bool CBanDB::Write(const banmap_t& banSet) } for (const auto& err : errors) { - error("%s", err); + LogError("%s\n", err); } return false; } @@ -189,7 +193,9 @@ void ReadFromStream(AddrMan& addr, DataStream& ssPeers) util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args) { auto check_addrman = std::clamp<int32_t>(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); - auto addrman{std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman)}; + bool deterministic = HasTestOption(args, "addrman"); // use a deterministic addrman only for tests + + auto addrman{std::make_unique<AddrMan>(netgroupman, deterministic, /*consistency_check_ratio=*/check_addrman)}; const auto start{SteadyClock::now()}; const auto path_addr{args.GetDataDirNet() / "peers.dat"}; @@ -198,7 +204,7 @@ util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgro LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->Size(), Ticks<std::chrono::milliseconds>(SteadyClock::now() - start)); } catch (const DbNotFoundError&) { // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); + addrman = std::make_unique<AddrMan>(netgroupman, deterministic, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating peers.dat because the file was not found (%s)\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const InvalidAddrManVersionError&) { @@ -206,7 +212,7 @@ util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgro return util::Error{strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again."))}; } // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); + addrman = std::make_unique<AddrMan>(netgroupman, deterministic, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 84b66bc4b2..a13a693ad7 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -23,6 +23,8 @@ const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS{}; +const std::function<std::string()> G_TEST_GET_FULL_NAME{}; + namespace { void GenerateTemplateResults(const std::vector<ankerl::nanobench::Result>& benchmarkResults, const fs::path& file, const char* tpl) diff --git a/src/bench/bip324_ecdh.cpp b/src/bench/bip324_ecdh.cpp index 659da0f08e..fb10c2957e 100644 --- a/src/bench/bip324_ecdh.cpp +++ b/src/bench/bip324_ecdh.cpp @@ -27,7 +27,7 @@ static void BIP324_ECDH(benchmark::Bench& bench) bench.batch(1).unit("ecdh").run([&] { CKey key; - key.Set(UCharCast(key_data.data()), UCharCast(key_data.data()) + 32, true); + key.Set(key_data.data(), key_data.data() + 32, true); EllSwiftPubKey our_ellswift(our_ellswift_data); EllSwiftPubKey their_ellswift(their_ellswift_data); diff --git a/src/bench/ellswift.cpp b/src/bench/ellswift.cpp index 82e8dec982..9441b4863e 100644 --- a/src/bench/ellswift.cpp +++ b/src/bench/ellswift.cpp @@ -17,7 +17,7 @@ static void EllSwiftCreate(benchmark::Bench& bench) bench.batch(1).unit("pubkey").run([&] { auto ret = key.EllSwiftCreate(MakeByteSpan(entropy)); /* Use the first 32 bytes of the ellswift encoded public key as next private key. */ - key.Set(UCharCast(ret.data()), UCharCast(ret.data()) + 32, true); + key.Set(ret.data(), ret.data() + 32, true); assert(key.IsValid()); /* Use the last 32 bytes of the ellswift encoded public key as next entropy. */ std::copy(ret.begin() + 32, ret.begin() + 64, MakeWritableByteSpan(entropy).begin()); diff --git a/src/bench/index_blockfilter.cpp b/src/bench/index_blockfilter.cpp new file mode 100644 index 0000000000..5e0bfbfea6 --- /dev/null +++ b/src/bench/index_blockfilter.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> + +#include <addresstype.h> +#include <index/blockfilterindex.h> +#include <node/chainstate.h> +#include <node/context.h> +#include <test/util/setup_common.h> +#include <util/strencodings.h> + +// Very simple block filter index sync benchmark, only using coinbase outputs. +static void BlockFilterIndexSync(benchmark::Bench& bench) +{ + const auto test_setup = MakeNoLogFileContext<TestChain100Setup>(); + + // Create more blocks + int CHAIN_SIZE = 600; + CPubKey pubkey{ParseHex("02ed26169896db86ced4cbb7b3ecef9859b5952825adbeab998fb5b307e54949c9")}; + CScript script = GetScriptForDestination(WitnessV0KeyHash(pubkey)); + std::vector<CMutableTransaction> noTxns; + for (int i = 0; i < CHAIN_SIZE - 100; i++) { + test_setup->CreateAndProcessBlock(noTxns, script); + SetMockTime(GetTime() + 1); + } + assert(WITH_LOCK(::cs_main, return test_setup->m_node.chainman->ActiveHeight() == CHAIN_SIZE)); + + bench.minEpochIterations(5).run([&] { + BlockFilterIndex filter_index(interfaces::MakeChain(test_setup->m_node), BlockFilterType::BASIC, + /*n_cache_size=*/0, /*f_memory=*/false, /*f_wipe=*/true); + assert(filter_index.Init()); + assert(!filter_index.BlockUntilSyncedToCurrentChain()); + filter_index.Sync(); + + IndexSummary summary = filter_index.GetSummary(); + assert(summary.synced); + assert(summary.best_block_hash == WITH_LOCK(::cs_main, return test_setup->m_node.chainman->ActiveTip()->GetBlockHash())); + }); +} + +BENCHMARK(BlockFilterIndexSync, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/parse_hex.cpp b/src/bench/parse_hex.cpp new file mode 100644 index 0000000000..db3ead043c --- /dev/null +++ b/src/bench/parse_hex.cpp @@ -0,0 +1,36 @@ +// Copyright (c) 2024- The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <random.h> +#include <stddef.h> +#include <util/strencodings.h> +#include <cassert> +#include <optional> +#include <vector> + +std::string generateHexString(size_t length) { + const auto hex_digits = "0123456789ABCDEF"; + FastRandomContext rng(/*fDeterministic=*/true); + + std::string data; + while (data.size() < length) { + auto digit = hex_digits[rng.randbits(4)]; + data.push_back(digit); + } + return data; +} + +static void HexParse(benchmark::Bench& bench) +{ + auto data = generateHexString(130); // Generates 678B0EDA0A1FD30904D5A65E3568DB82DB2D918B0AD8DEA18A63FECCB877D07CAD1495C7157584D877420EF38B8DA473A6348B4F51811AC13C786B962BEE5668F9 by default + + bench.batch(data.size()).unit("base16").run([&] { + auto result = TryParseHex(data); + assert(result != std::nullopt); // make sure we're measuring the successful case + ankerl::nanobench::doNotOptimizeAway(result); + }); +} + +BENCHMARK(HexParse, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index e7166a91cf..ee750bc1f8 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -2,15 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - #include <bench/bench.h> #include <key.h> -#if defined(HAVE_CONSENSUS_LIB) -#include <script/bitcoinconsensus.h> -#endif #include <script/script.h> #include <script/interpreter.h> #include <streams.h> @@ -63,17 +56,6 @@ static void VerifyScriptBench(benchmark::Bench& bench) &err); assert(err == SCRIPT_ERR_OK); assert(success); - -#if defined(HAVE_CONSENSUS_LIB) - DataStream stream; - stream << TX_WITH_WITNESS(txSpend); - int csuccess = bitcoinconsensus_verify_script_with_amount( - txCredit.vout[0].scriptPubKey.data(), - txCredit.vout[0].scriptPubKey.size(), - txCredit.vout[0].nValue, - (const unsigned char*)stream.data(), stream.size(), 0, flags, nullptr); - assert(csuccess == 1); -#endif }); ECC_Stop(); } diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 3eb64aa344..642af06e82 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -89,14 +89,13 @@ int main(int argc, char* argv[]) { std::cout << "Warning: " << warning.original << std::endl; } - void flushError(const std::string& debug_message) override + void flushError(const bilingual_str& message) override { - std::cerr << "Error flushing block data to disk: " << debug_message << std::endl; + std::cerr << "Error flushing block data to disk: " << message.original << std::endl; } - void fatalError(const std::string& debug_message, const bilingual_str& user_message) override + void fatalError(const bilingual_str& message) override { - std::cerr << "Error: " << debug_message << std::endl; - std::cerr << (user_message.empty() ? "A fatal internal error occurred." : user_message.original) << std::endl; + std::cerr << "Error: " << message.original << std::endl; } }; auto notifications = std::make_unique<KernelNotifications>(); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index f0e27cb675..8901d10ef6 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -11,7 +11,6 @@ #include <clientversion.h> #include <common/args.h> #include <common/system.h> -#include <common/url.h> #include <compat/compat.h> #include <compat/stdin.h> #include <policy/feerate.h> @@ -51,7 +50,6 @@ using CliClock = std::chrono::system_clock; const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; -UrlDecodeFn* const URL_DECODE = urlDecode; static const char DEFAULT_RPCCONNECT[] = "127.0.0.1"; static const int DEFAULT_HTTP_CLIENT_TIMEOUT=900; @@ -551,7 +549,7 @@ public: peer.is_outbound ? "out" : "in", ConnectionTypeForNetinfo(peer.conn_type), peer.network, - peer.transport_protocol_type.starts_with('v') ? peer.transport_protocol_type[1] : ' ', + (peer.transport_protocol_type.size() == 2 && peer.transport_protocol_type[0] == 'v') ? peer.transport_protocol_type[1] : ' ', PingTimeToString(peer.min_ping), PingTimeToString(peer.ping), peer.last_send ? ToString(time_now - peer.last_send) : "", @@ -827,7 +825,10 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co if (response.error != -1) { responseErrorMessage = strprintf(" (error code %d - \"%s\")", response.error, http_errorstring(response.error)); } - throw CConnectionFailed(strprintf("Could not connect to the server %s:%d%s\n\nMake sure the bitcoind server is running and that you are connecting to the correct RPC port.", host, port, responseErrorMessage)); + throw CConnectionFailed(strprintf("Could not connect to the server %s:%d%s\n\n" + "Make sure the bitcoind server is running and that you are connecting to the correct RPC port.\n" + "Use \"bitcoin-cli -help\" for more info.", + host, port, responseErrorMessage)); } else if (response.status == HTTP_UNAUTHORIZED) { if (failedToGetAuthCookie) { throw std::runtime_error(strprintf( diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index d5dfbbec27..fe90958a5f 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -11,7 +11,6 @@ #include <clientversion.h> #include <common/args.h> #include <common/system.h> -#include <common/url.h> #include <compat/compat.h> #include <interfaces/init.h> #include <key.h> @@ -28,7 +27,6 @@ #include <tuple> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; -UrlDecodeFn* const URL_DECODE = nullptr; static void SetupWalletToolArgs(ArgsManager& argsman) { diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 4f0a816388..54796c5abb 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -12,7 +12,6 @@ #include <common/args.h> #include <common/init.h> #include <common/system.h> -#include <common/url.h> #include <compat/compat.h> #include <init.h> #include <interfaces/chain.h> @@ -35,7 +34,6 @@ using node::NodeContext; const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; -UrlDecodeFn* const URL_DECODE = urlDecode; #if HAVE_DECL_FORK diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 1e940e8f03..5a4513d281 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -40,14 +40,14 @@ void CBlockHeaderAndShortTxIDs::FillShortTxIDSelector() const { shorttxidk1 = shorttxidhash.GetUint64(1); } -uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const uint256& txhash) const { +uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const Wtxid& wtxid) const { static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids calculation assumes 6-byte shorttxids"); - return SipHashUint256(shorttxidk0, shorttxidk1, txhash) & 0xffffffffffffL; + return SipHashUint256(shorttxidk0, shorttxidk1, wtxid) & 0xffffffffffffL; } -ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector<std::pair<uint256, CTransactionRef>>& extra_txn) { +ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector<CTransactionRef>& extra_txn) { if (cmpctblock.header.IsNull() || (cmpctblock.shorttxids.empty() && cmpctblock.prefilledtxn.empty())) return READ_STATUS_INVALID; if (cmpctblock.shorttxids.size() + cmpctblock.prefilledtxn.size() > MAX_BLOCK_WEIGHT / MIN_SERIALIZABLE_TRANSACTION_WEIGHT) @@ -134,11 +134,14 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c } for (size_t i = 0; i < extra_txn.size(); i++) { - uint64_t shortid = cmpctblock.GetShortID(extra_txn[i].first); + if (extra_txn[i] == nullptr) { + continue; + } + uint64_t shortid = cmpctblock.GetShortID(extra_txn[i]->GetWitnessHash()); std::unordered_map<uint64_t, uint16_t>::iterator idit = shorttxids.find(shortid); if (idit != shorttxids.end()) { if (!have_txn[idit->second]) { - txn_available[idit->second] = extra_txn[i].second; + txn_available[idit->second] = extra_txn[i]; have_txn[idit->second] = true; mempool_count++; extra_count++; @@ -150,7 +153,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c // Note that we don't want duplication between extra_txn and mempool to // trigger this case, so we compare witness hashes first if (txn_available[idit->second] && - txn_available[idit->second]->GetWitnessHash() != extra_txn[i].second->GetWitnessHash()) { + txn_available[idit->second]->GetWitnessHash() != extra_txn[i]->GetWitnessHash()) { txn_available[idit->second].reset(); mempool_count--; extra_count--; diff --git a/src/blockencodings.h b/src/blockencodings.h index fb0f734ff8..2b1fabadd6 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -111,7 +111,7 @@ public: CBlockHeaderAndShortTxIDs(const CBlock& block); - uint64_t GetShortID(const uint256& txhash) const; + uint64_t GetShortID(const Wtxid& wtxid) const; size_t BlockTxCount() const { return shorttxids.size() + prefilledtxn.size(); } @@ -142,7 +142,7 @@ public: explicit PartiallyDownloadedBlock(CTxMemPool* poolIn) : pool(poolIn) {} // extra_txn is a list of extra transactions to look at, in <witness hash, reference> form - ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector<std::pair<uint256, CTransactionRef>>& extra_txn); + ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector<CTransactionRef>& extra_txn); bool IsTxAvailable(size_t index) const; ReadStatus FillBlock(CBlock& block, const std::vector<CTransactionRef>& vtx_missing); }; diff --git a/src/chain.h b/src/chain.h index fa165a4aa7..bb70dbd8bc 100644 --- a/src/chain.h +++ b/src/chain.h @@ -98,16 +98,20 @@ enum BlockStatus : uint32_t { /** * Only first tx is coinbase, 2 <= coinbase input script length <= 100, transactions valid, no duplicate txids, - * sigops, size, merkle root. Implies all parents are at least TREE but not necessarily TRANSACTIONS. When all - * parent blocks also have TRANSACTIONS, CBlockIndex::nChainTx will be set. + * sigops, size, merkle root. Implies all parents are at least TREE but not necessarily TRANSACTIONS. + * + * If a block's validity is at least VALID_TRANSACTIONS, CBlockIndex::nTx will be set. If a block and all previous + * blocks back to the genesis block or an assumeutxo snapshot block are at least VALID_TRANSACTIONS, + * CBlockIndex::nChainTx will be set. */ BLOCK_VALID_TRANSACTIONS = 3, //! Outputs do not overspend inputs, no double spends, coinbase output ok, no immature coinbase spends, BIP30. - //! Implies all parents are either at least VALID_CHAIN, or are ASSUMED_VALID + //! Implies all previous blocks back to the genesis block or an assumeutxo snapshot block are at least VALID_CHAIN. BLOCK_VALID_CHAIN = 4, - //! Scripts & signatures ok. Implies all parents are either at least VALID_SCRIPTS, or are ASSUMED_VALID. + //! Scripts & signatures ok. Implies all previous blocks back to the genesis block or an assumeutxo snapshot block + //! are at least VALID_SCRIPTS. BLOCK_VALID_SCRIPTS = 5, //! All validity bits. @@ -124,21 +128,8 @@ enum BlockStatus : uint32_t { BLOCK_OPT_WITNESS = 128, //!< block data in blk*.dat was received with a witness-enforcing client - /** - * If ASSUMED_VALID is set, it means that this block has not been validated - * and has validity status less than VALID_SCRIPTS. Also that it may have - * descendant blocks with VALID_SCRIPTS set, because they can be validated - * based on an assumeutxo snapshot. - * - * When an assumeutxo snapshot is loaded, the ASSUMED_VALID flag is added to - * unvalidated blocks at the snapshot height and below. Then, as the background - * validation progresses, and these blocks are validated, the ASSUMED_VALID - * flags are removed. See `doc/design/assumeutxo.md` for details. - * - * This flag is only used to implement checks in CheckBlockIndex() and - * should not be used elsewhere. - */ - BLOCK_ASSUMED_VALID = 256, + BLOCK_STATUS_RESERVED = 256, //!< Unused flag that was previously set on assumeutxo snapshot blocks and their + //!< ancestors before they were validated, and unset when they were validated. }; /** The block chain is a tree shaped structure starting with the @@ -173,21 +164,16 @@ public: //! (memory only) Total amount of work (expected number of hashes) in the chain up to and including this block arith_uint256 nChainWork{}; - //! Number of transactions in this block. + //! Number of transactions in this block. This will be nonzero if the block + //! reached the VALID_TRANSACTIONS level, and zero otherwise. //! Note: in a potential headers-first mode, this number cannot be relied upon - //! Note: this value is faked during UTXO snapshot load to ensure that - //! LoadBlockIndex() will load index entries for blocks that we lack data for. - //! @sa ActivateSnapshot unsigned int nTx{0}; //! (memory only) Number of transactions in the chain up to and including this block. - //! This value will be non-zero only if and only if transactions for this block and all its parents are available. + //! This value will be non-zero if this block and all previous blocks back + //! to the genesis block or an assumeutxo snapshot block have reached the + //! VALID_TRANSACTIONS level. //! Change to 64-bit type before 2024 (assuming worst case of 60 byte transactions). - //! - //! Note: this value is faked during use of a UTXO snapshot because we don't - //! have the underlying block data available during snapshot load. - //! @sa AssumeutxoData - //! @sa ActivateSnapshot unsigned int nChainTx{0}; //! Verification status of this block. See enum BlockStatus @@ -262,15 +248,14 @@ public: } /** - * Check whether this block's and all previous blocks' transactions have been - * downloaded (and stored to disk) at some point. + * Check whether this block and all previous blocks back to the genesis block or an assumeutxo snapshot block have + * reached VALID_TRANSACTIONS and had transactions downloaded (and stored to disk) at some point. * * Does not imply the transactions are consensus-valid (ConnectTip might fail) * Does not imply the transactions are still stored on disk. (IsBlockPruned might return true) * - * Note that this will be true for the snapshot base block, if one is loaded (and - * all subsequent assumed-valid blocks) since its nChainTx value will have been set - * manually based on the related AssumeutxoData entry. + * Note that this will be true for the snapshot base block, if one is loaded, since its nChainTx value will have + * been set manually based on the related AssumeutxoData entry. */ bool HaveNumChainTxs() const { return nChainTx != 0; } @@ -318,14 +303,6 @@ public: return ((nStatus & BLOCK_VALID_MASK) >= nUpTo); } - //! @returns true if the block is assumed-valid; this means it is queued to be - //! validated by a background chainstate. - bool IsAssumedValid() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main) - { - AssertLockHeld(::cs_main); - return nStatus & BLOCK_ASSUMED_VALID; - } - //! Raise the validity level of this block index entry. //! Returns true if the validity was changed. bool RaiseValidity(enum BlockStatus nUpTo) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) @@ -335,12 +312,6 @@ public: if (nStatus & BLOCK_FAILED_MASK) return false; if ((nStatus & BLOCK_VALID_MASK) < nUpTo) { - // If this block had been marked assumed-valid and we're raising - // its validity to a certain point, there is no longer an assumption. - if (nStatus & BLOCK_ASSUMED_VALID && nUpTo >= BLOCK_VALID_SCRIPTS) { - nStatus &= ~BLOCK_ASSUMED_VALID; - } - nStatus = (nStatus & ~BLOCK_VALID_MASK) | nUpTo; return true; } diff --git a/src/common/args.cpp b/src/common/args.cpp index a9108e5916..c90eb0c685 100644 --- a/src/common/args.cpp +++ b/src/common/args.cpp @@ -682,6 +682,18 @@ std::string HelpMessageOpt(const std::string &option, const std::string &message std::string("\n\n"); } +const std::vector<std::string> TEST_OPTIONS_DOC{ + "addrman (use deterministic addrman)", +}; + +bool HasTestOption(const ArgsManager& args, const std::string& test_option) +{ + const auto options = args.GetArgs("-test"); + return std::any_of(options.begin(), options.end(), [test_option](const auto& option) { + return option == test_option; + }); +} + fs::path GetDefaultDataDir() { // Windows: C:\Users\Username\AppData\Roaming\Bitcoin diff --git a/src/common/args.h b/src/common/args.h index 6451b194d1..78a61313b9 100644 --- a/src/common/args.h +++ b/src/common/args.h @@ -447,6 +447,11 @@ bool HelpRequested(const ArgsManager& args); /** Add help options to the args manager */ void SetupHelpOptions(ArgsManager& args); +extern const std::vector<std::string> TEST_OPTIONS_DOC; + +/** Checks if a particular test option is present in -test command-line arg options */ +bool HasTestOption(const ArgsManager& args, const std::string& test_option); + /** * Format a string to be used as group of options in help messages * diff --git a/src/common/run_command.cpp b/src/common/run_command.cpp index 8bd5febd53..347b486095 100644 --- a/src/common/run_command.cpp +++ b/src/common/run_command.cpp @@ -12,39 +12,34 @@ #include <univalue.h> #ifdef ENABLE_EXTERNAL_SIGNER -#include <boost/process.hpp> +#include <util/subprocess.h> #endif // ENABLE_EXTERNAL_SIGNER UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in) { #ifdef ENABLE_EXTERNAL_SIGNER - namespace bp = boost::process; + namespace sp = subprocess; UniValue result_json; - bp::opstream stdin_stream; - bp::ipstream stdout_stream; - bp::ipstream stderr_stream; + std::istringstream stdout_stream; + std::istringstream stderr_stream; if (str_command.empty()) return UniValue::VNULL; - bp::child c( - str_command, - bp::std_out > stdout_stream, - bp::std_err > stderr_stream, - bp::std_in < stdin_stream - ); + auto c = sp::Popen(str_command, sp::input{sp::PIPE}, sp::output{sp::PIPE}, sp::error{sp::PIPE}); if (!str_std_in.empty()) { - stdin_stream << str_std_in << std::endl; + c.send(str_std_in); } - stdin_stream.pipe().close(); + auto [out_res, err_res] = c.communicate(); + stdout_stream.str(std::string{out_res.buf.begin(), out_res.buf.end()}); + stderr_stream.str(std::string{err_res.buf.begin(), err_res.buf.end()}); std::string result; std::string error; std::getline(stdout_stream, result); std::getline(stderr_stream, error); - c.wait(); - const int n_error = c.exit_code(); + const int n_error = c.retcode(); if (n_error) throw std::runtime_error(strprintf("RunCommandParseJSON error: process(%s) returned %d: %s\n", str_command, n_error, error)); if (!result_json.read(result)) throw std::runtime_error("Unable to parse JSON: " + result); diff --git a/src/common/url.cpp b/src/common/url.cpp index 053e1a825c..ecf88d07ea 100644 --- a/src/common/url.cpp +++ b/src/common/url.cpp @@ -4,19 +4,36 @@ #include <common/url.h> -#include <event2/http.h> - -#include <cstdlib> +#include <charconv> #include <string> +#include <string_view> +#include <system_error> -std::string urlDecode(const std::string &urlEncoded) { +std::string UrlDecode(std::string_view url_encoded) +{ std::string res; - if (!urlEncoded.empty()) { - char *decoded = evhttp_uridecode(urlEncoded.c_str(), false, nullptr); - if (decoded) { - res = std::string(decoded); - free(decoded); + res.reserve(url_encoded.size()); + + for (size_t i = 0; i < url_encoded.size(); ++i) { + char c = url_encoded[i]; + // Special handling for percent which should be followed by two hex digits + // representing an octet values, see RFC 3986, Section 2.1 Percent-Encoding + if (c == '%' && i + 2 < url_encoded.size()) { + unsigned int decoded_value{0}; + auto [p, ec] = std::from_chars(url_encoded.data() + i + 1, url_encoded.data() + i + 3, decoded_value, 16); + + // Only if there is no error and the pointer is set to the end of + // the string, we can be sure both characters were valid hex + if (ec == std::errc{} && p == url_encoded.data() + i + 3) { + res += static_cast<char>(decoded_value); + // Next two characters are part of the percent encoding + i += 2; + continue; + } + // In case of invalid percent encoding, add the '%' and continue } + res += c; } + return res; } diff --git a/src/common/url.h b/src/common/url.h index b16b8241af..203f41c70f 100644 --- a/src/common/url.h +++ b/src/common/url.h @@ -6,9 +6,12 @@ #define BITCOIN_COMMON_URL_H #include <string> +#include <string_view> -using UrlDecodeFn = std::string(const std::string& url_encoded); -UrlDecodeFn urlDecode; -extern UrlDecodeFn* const URL_DECODE; +/* Decode a URL. + * + * Notably this implementation does not decode a '+' to a ' '. + */ +std::string UrlDecode(std::string_view url_encoded); #endif // BITCOIN_COMMON_URL_H diff --git a/src/compat/compat.h b/src/compat/compat.h index 9ff9a335f8..366c648ae7 100644 --- a/src/compat/compat.h +++ b/src/compat/compat.h @@ -32,6 +32,13 @@ #include <unistd.h> // IWYU pragma: export #endif +// Windows does not have `sa_family_t` - it defines `sockaddr::sa_family` as `u_short`. +// Thus define `sa_family_t` on Windows too so that the rest of the code can use `sa_family_t`. +// See https://learn.microsoft.com/en-us/windows/win32/api/winsock/ns-winsock-sockaddr#syntax +#ifdef WIN32 +typedef u_short sa_family_t; +#endif + // We map Linux / BSD error functions and codes, to the equivalent // Windows definitions, and use the WSA* names throughout our code. // Note that glibc defines EWOULDBLOCK as EAGAIN (see errno.h). diff --git a/src/core_read.cpp b/src/core_read.cpp index e32e46d1b9..5956d9df5f 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -256,6 +256,6 @@ util::Result<int> SighashFromStr(const std::string& sighash) if (it != map_sighash_values.end()) { return it->second; } else { - return util::Error{Untranslated(sighash + " is not a valid sighash parameter.")}; + return util::Error{Untranslated("'" + sighash + "' is not a valid sighash parameter.")}; } } diff --git a/src/crypto/chacha20poly1305.cpp b/src/crypto/chacha20poly1305.cpp index 3e8051c2dc..b969bb1a29 100644 --- a/src/crypto/chacha20poly1305.cpp +++ b/src/crypto/chacha20poly1305.cpp @@ -2,10 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - #include <crypto/chacha20poly1305.h> #include <crypto/common.h> @@ -30,10 +26,7 @@ void AEADChaCha20Poly1305::SetKey(Span<const std::byte> key) noexcept namespace { -#ifndef HAVE_TIMINGSAFE_BCMP -#define HAVE_TIMINGSAFE_BCMP - -int timingsafe_bcmp(const unsigned char* b1, const unsigned char* b2, size_t n) noexcept +int timingsafe_bcmp_internal(const unsigned char* b1, const unsigned char* b2, size_t n) noexcept { const unsigned char *p1 = b1, *p2 = b2; int ret = 0; @@ -42,8 +35,6 @@ int timingsafe_bcmp(const unsigned char* b1, const unsigned char* b2, size_t n) return (ret != 0); } -#endif - /** Compute poly1305 tag. chacha20 must be set to the right nonce, block 0. Will be at block 1 after. */ void ComputeTag(ChaCha20& chacha20, Span<const std::byte> aad, Span<const std::byte> cipher, Span<std::byte> tag) noexcept { @@ -97,7 +88,7 @@ bool AEADChaCha20Poly1305::Decrypt(Span<const std::byte> cipher, Span<const std: m_chacha20.Seek(nonce, 0); std::byte expected_tag[EXPANSION]; ComputeTag(m_chacha20, aad, cipher.first(cipher.size() - EXPANSION), expected_tag); - if (timingsafe_bcmp(UCharCast(expected_tag), UCharCast(cipher.last(EXPANSION).data()), EXPANSION)) return false; + if (timingsafe_bcmp_internal(UCharCast(expected_tag), UCharCast(cipher.last(EXPANSION).data()), EXPANSION)) return false; // Decrypt (starting at block 1). m_chacha20.Crypt(cipher.first(plain1.size()), plain1); diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 4c7bb6f20f..301f22a248 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -20,7 +20,7 @@ #include <asm/hwcap.h> #endif -#if defined(MAC_OSX) && defined(ENABLE_ARM_SHANI) +#if defined(__APPLE__) && defined(ENABLE_ARM_SHANI) #include <sys/types.h> #include <sys/sysctl.h> #endif @@ -670,7 +670,7 @@ std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implem #endif #endif -#if defined(MAC_OSX) +#if defined(__APPLE__) int val = 0; size_t len = sizeof(val); if (sysctlbyname("hw.optional.arm.FEAT_SHA256", &val, &len, nullptr, 0) == 0) { diff --git a/src/cuckoocache.h b/src/cuckoocache.h index cb0b362143..df320ed465 100644 --- a/src/cuckoocache.h +++ b/src/cuckoocache.h @@ -359,7 +359,7 @@ public: * @param bytes the approximate number of bytes to use for this data * structure * @returns A pair of the maximum number of elements storable (see setup() - * documentation for more detail) and the approxmiate total size of these + * documentation for more detail) and the approximate total size of these * elements in bytes or std::nullopt if the size requested is too large. */ std::optional<std::pair<uint32_t, size_t>> setup_bytes(size_t bytes) diff --git a/src/external_signer.cpp b/src/external_signer.cpp index 749bb5f74f..ff159a2aa5 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -62,12 +62,12 @@ bool ExternalSigner::Enumerate(const std::string& command, std::vector<ExternalS UniValue ExternalSigner::DisplayAddress(const std::string& descriptor) const { - return RunCommandParseJSON(m_command + " --fingerprint \"" + m_fingerprint + "\"" + NetworkArg() + " displayaddress --desc \"" + descriptor + "\""); + return RunCommandParseJSON(m_command + " --fingerprint " + m_fingerprint + NetworkArg() + " displayaddress --desc " + descriptor); } UniValue ExternalSigner::GetDescriptors(const int account) { - return RunCommandParseJSON(m_command + " --fingerprint \"" + m_fingerprint + "\"" + NetworkArg() + " getdescriptors --account " + strprintf("%d", account)); + return RunCommandParseJSON(m_command + " --fingerprint " + m_fingerprint + NetworkArg() + " getdescriptors --account " + strprintf("%d", account)); } bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::string& error) @@ -93,8 +93,8 @@ bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::str return false; } - const std::string command = m_command + " --stdin --fingerprint \"" + m_fingerprint + "\"" + NetworkArg(); - const std::string stdinStr = "signtx \"" + EncodeBase64(ssTx.str()) + "\""; + const std::string command = m_command + " --stdin --fingerprint " + m_fingerprint + NetworkArg(); + const std::string stdinStr = "signtx " + EncodeBase64(ssTx.str()); const UniValue signer_result = RunCommandParseJSON(command, stdinStr); diff --git a/src/flatfile.cpp b/src/flatfile.cpp index 59861a08ad..2bff663d8b 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -82,15 +82,18 @@ bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize) { FILE* file = Open(FlatFilePos(pos.nFile, 0)); // Avoid fseek to nPos if (!file) { - return error("%s: failed to open file %d", __func__, pos.nFile); + LogError("%s: failed to open file %d\n", __func__, pos.nFile); + return false; } if (finalize && !TruncateFile(file, pos.nPos)) { fclose(file); - return error("%s: failed to truncate file %d", __func__, pos.nFile); + LogError("%s: failed to truncate file %d\n", __func__, pos.nFile); + return false; } if (!FileCommit(file)) { fclose(file); - return error("%s: failed to commit file %d", __func__, pos.nFile); + LogError("%s: failed to commit file %d\n", __func__, pos.nFile); + return false; } DirectoryCommit(m_dir); diff --git a/src/i2p.cpp b/src/i2p.cpp index 02f2c1cea2..962adb124d 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -115,7 +115,7 @@ static CNetAddr DestB64ToAddr(const std::string& dest) namespace sam { Session::Session(const fs::path& private_key_file, - const CService& control_host, + const Proxy& control_host, CThreadInterrupt* interrupt) : m_private_key_file{private_key_file}, m_control_host{control_host}, @@ -124,7 +124,7 @@ Session::Session(const fs::path& private_key_file, { } -Session::Session(const CService& control_host, CThreadInterrupt* interrupt) +Session::Session(const Proxy& control_host, CThreadInterrupt* interrupt) : m_control_host{control_host}, m_interrupt{interrupt}, m_transient{true} @@ -327,14 +327,10 @@ Session::Reply Session::SendRequestAndGetReply(const Sock& sock, std::unique_ptr<Sock> Session::Hello() const { - auto sock = CreateSock(m_control_host); + auto sock = m_control_host.Connect(); if (!sock) { - throw std::runtime_error("Cannot create socket"); - } - - if (!ConnectSocketDirectly(m_control_host, *sock, nConnectTimeout, true)) { - throw std::runtime_error(strprintf("Cannot connect to %s", m_control_host.ToStringAddrPort())); + throw std::runtime_error(strprintf("Cannot connect to %s", m_control_host.ToString())); } SendRequestAndGetReply(*sock, "HELLO VERSION MIN=3.1 MAX=3.1"); @@ -418,7 +414,7 @@ void Session::CreateIfNotCreatedAlready() const auto session_type = m_transient ? "transient" : "persistent"; const auto session_id = GetRandHash().GetHex().substr(0, 10); // full is overkill, too verbose in the logs - Log("Creating %s SAM session %s with %s", session_type, session_id, m_control_host.ToStringAddrPort()); + Log("Creating %s SAM session %s with %s", session_type, session_id, m_control_host.ToString()); auto sock = Hello(); @@ -7,6 +7,7 @@ #include <compat/compat.h> #include <netaddress.h> +#include <netbase.h> #include <sync.h> #include <util/fs.h> #include <util/sock.h> @@ -67,7 +68,7 @@ public: * `Session` object. */ Session(const fs::path& private_key_file, - const CService& control_host, + const Proxy& control_host, CThreadInterrupt* interrupt); /** @@ -81,7 +82,7 @@ public: * `CThreadInterrupt` object is saved, so it must not be destroyed earlier than this * `Session` object. */ - Session(const CService& control_host, CThreadInterrupt* interrupt); + Session(const Proxy& control_host, CThreadInterrupt* interrupt); /** * Destroy the session, closing the internally used sockets. The sockets that have been @@ -235,9 +236,9 @@ private: const fs::path m_private_key_file; /** - * The host and port of the SAM control service. + * The SAM control service proxy. */ - const CService m_control_host; + const Proxy m_control_host; /** * Cease network activity when this is signaled. diff --git a/src/index/base.cpp b/src/index/base.cpp index 2287437f8f..a203ce4a9f 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -31,7 +31,7 @@ template <typename... Args> void BaseIndex::FatalErrorf(const char* fmt, const Args&... args) { auto message = tfm::format(fmt, args...); - node::AbortNode(m_chain->context()->shutdown, m_chain->context()->exit_status, message); + node::AbortNode(m_chain->context()->shutdown, m_chain->context()->exit_status, Untranslated(message)); } CBlockLocator GetLocator(interfaces::Chain& chain, const uint256& block_hash) @@ -141,7 +141,7 @@ static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev, CChain& return chain.Next(chain.FindFork(pindex_prev)); } -void BaseIndex::ThreadSync() +void BaseIndex::Sync() { const CBlockIndex* pindex = m_best_block_index.load(); if (!m_synced) { @@ -159,37 +159,20 @@ void BaseIndex::ThreadSync() return; } - { - LOCK(cs_main); - const CBlockIndex* pindex_next = NextSyncBlock(pindex, m_chainstate->m_chain); - if (!pindex_next) { - SetBestBlockIndex(pindex); - m_synced = true; - // No need to handle errors in Commit. See rationale above. - Commit(); - break; - } - if (pindex_next->pprev != pindex && !Rewind(pindex, pindex_next->pprev)) { - FatalErrorf("%s: Failed to rewind index %s to a previous chain tip", - __func__, GetName()); - return; - } - pindex = pindex_next; - } - - auto current_time{std::chrono::steady_clock::now()}; - if (last_log_time + SYNC_LOG_INTERVAL < current_time) { - LogPrintf("Syncing %s with block chain from height %d\n", - GetName(), pindex->nHeight); - last_log_time = current_time; - } - - if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { - SetBestBlockIndex(pindex->pprev); - last_locator_write_time = current_time; + const CBlockIndex* pindex_next = WITH_LOCK(cs_main, return NextSyncBlock(pindex, m_chainstate->m_chain)); + if (!pindex_next) { + SetBestBlockIndex(pindex); // No need to handle errors in Commit. See rationale above. Commit(); + m_synced = true; + break; } + if (pindex_next->pprev != pindex && !Rewind(pindex, pindex_next->pprev)) { + FatalErrorf("%s: Failed to rewind index %s to a previous chain tip", __func__, GetName()); + return; + } + pindex = pindex_next; + CBlock block; interfaces::BlockInfo block_info = kernel::MakeBlockInfo(pindex); @@ -205,6 +188,20 @@ void BaseIndex::ThreadSync() __func__, pindex->GetBlockHash().ToString()); return; } + + auto current_time{std::chrono::steady_clock::now()}; + if (last_log_time + SYNC_LOG_INTERVAL < current_time) { + LogPrintf("Syncing %s with block chain from height %d\n", + GetName(), pindex->nHeight); + last_log_time = current_time; + } + + if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { + SetBestBlockIndex(pindex); + last_locator_write_time = current_time; + // No need to handle errors in Commit. See rationale above. + Commit(); + } } } @@ -229,7 +226,8 @@ bool BaseIndex::Commit() } } if (!ok) { - return error("%s: Failed to commit latest %s state", __func__, GetName()); + LogError("%s: Failed to commit latest %s state\n", __func__, GetName()); + return false; } return true; } @@ -393,7 +391,7 @@ bool BaseIndex::StartBackgroundSync() { if (!m_init) throw std::logic_error("Error: Cannot start a non-initialized index"); - m_thread_sync = std::thread(&util::TraceThread, GetName(), [this] { ThreadSync(); }); + m_thread_sync = std::thread(&util::TraceThread, GetName(), [this] { Sync(); }); return true; } diff --git a/src/index/base.h b/src/index/base.h index 154061fb19..0eb1d9ca3b 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -78,13 +78,6 @@ private: std::thread m_thread_sync; CThreadInterrupt m_interrupt; - /// Sync the index with the block index starting from the current best block. - /// Intended to be run in its own thread, m_thread_sync, and can be - /// interrupted with m_interrupt. Once the index gets in sync, the m_synced - /// flag is set and the BlockConnected ValidationInterface callback takes - /// over and the sync thread exits. - void ThreadSync(); - /// Write the current index state (eg. chain block locator and subclass-specific items) to disk. /// /// Recommendations for error handling: @@ -152,9 +145,16 @@ public: /// validation interface so that it stays in sync with blockchain updates. [[nodiscard]] bool Init(); - /// Starts the initial sync process. + /// Starts the initial sync process on a background thread. [[nodiscard]] bool StartBackgroundSync(); + /// Sync the index with the block index starting from the current best block. + /// Intended to be run in its own thread, m_thread_sync, and can be + /// interrupted with m_interrupt. Once the index gets in sync, the m_synced + /// flag is set and the BlockConnected ValidationInterface callback takes + /// over and the sync thread exits. + void Sync(); + /// Stops the instance from staying in sync with blockchain updates. void Stop(); diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index 58f777b326..41bdca9df5 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -119,14 +119,25 @@ bool BlockFilterIndex::CustomInit(const std::optional<interfaces::BlockKey>& blo // indicate database corruption or a disk failure, and starting the index would cause // further corruption. if (m_db->Exists(DB_FILTER_POS)) { - return error("%s: Cannot read current %s state; index may be corrupted", + LogError("%s: Cannot read current %s state; index may be corrupted\n", __func__, GetName()); + return false; } // If the DB_FILTER_POS is not set, then initialize to the first location. m_next_filter_pos.nFile = 0; m_next_filter_pos.nPos = 0; } + + if (block) { + auto op_last_header = ReadFilterHeader(block->height, block->hash); + if (!op_last_header) { + LogError("Cannot read last block filter header; index may be corrupted\n"); + return false; + } + m_last_header = *op_last_header; + } + return true; } @@ -137,10 +148,12 @@ bool BlockFilterIndex::CustomCommit(CDBBatch& batch) // Flush current filter file to disk. AutoFile file{m_filter_fileseq->Open(pos)}; if (file.IsNull()) { - return error("%s: Failed to open filter file %d", __func__, pos.nFile); + LogError("%s: Failed to open filter file %d\n", __func__, pos.nFile); + return false; } if (!FileCommit(file.Get())) { - return error("%s: Failed to commit filter file %d", __func__, pos.nFile); + LogError("%s: Failed to commit filter file %d\n", __func__, pos.nFile); + return false; } batch.Write(DB_FILTER_POS, pos); @@ -159,11 +172,15 @@ bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, const uint256& std::vector<uint8_t> encoded_filter; try { filein >> block_hash >> encoded_filter; - if (Hash(encoded_filter) != hash) return error("Checksum mismatch in filter decode."); + if (Hash(encoded_filter) != hash) { + LogError("Checksum mismatch in filter decode.\n"); + return false; + } filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter), /*skip_decode_check=*/true); } catch (const std::exception& e) { - return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what()); + LogError("%s: Failed to deserialize block filter from disk: %s\n", __func__, e.what()); + return false; } return true; @@ -215,10 +232,25 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& return data_size; } +std::optional<uint256> BlockFilterIndex::ReadFilterHeader(int height, const uint256& expected_block_hash) +{ + std::pair<uint256, DBVal> read_out; + if (!m_db->Read(DBHeightKey(height), read_out)) { + return std::nullopt; + } + + if (read_out.first != expected_block_hash) { + LogError("%s: previous block header belongs to unexpected block %s; expected %s\n", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + return std::nullopt; + } + + return read_out.second.header; +} + bool BlockFilterIndex::CustomAppend(const interfaces::BlockInfo& block) { CBlockUndo block_undo; - uint256 prev_header; if (block.height > 0) { // pindex variable gives indexing code access to node internals. It @@ -227,33 +259,28 @@ bool BlockFilterIndex::CustomAppend(const interfaces::BlockInfo& block) if (!m_chainstate->m_blockman.UndoReadFromDisk(block_undo, *pindex)) { return false; } - - std::pair<uint256, DBVal> read_out; - if (!m_db->Read(DBHeightKey(block.height - 1), read_out)) { - return false; - } - - uint256 expected_block_hash = *Assert(block.prev_hash); - if (read_out.first != expected_block_hash) { - return error("%s: previous block header belongs to unexpected block %s; expected %s", - __func__, read_out.first.ToString(), expected_block_hash.ToString()); - } - - prev_header = read_out.second.header; } BlockFilter filter(m_filter_type, *Assert(block.data), block_undo); + const uint256& header = filter.ComputeHeader(m_last_header); + bool res = Write(filter, block.height, header); + if (res) m_last_header = header; // update last header + return res; +} + +bool BlockFilterIndex::Write(const BlockFilter& filter, uint32_t block_height, const uint256& filter_header) +{ size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter); if (bytes_written == 0) return false; std::pair<uint256, DBVal> value; - value.first = block.hash; + value.first = filter.GetBlockHash(); value.second.hash = filter.GetHash(); - value.second.header = filter.ComputeHeader(prev_header); + value.second.header = filter_header; value.second.pos = m_next_filter_pos; - if (!m_db->Write(DBHeightKey(block.height), value)) { + if (!m_db->Write(DBHeightKey(block_height), value)) { return false; } @@ -270,14 +297,16 @@ bool BlockFilterIndex::CustomAppend(const interfaces::BlockInfo& block) for (int height = start_height; height <= stop_height; ++height) { if (!db_it.GetKey(key) || key.height != height) { - return error("%s: unexpected key in %s: expected (%c, %d)", + LogError("%s: unexpected key in %s: expected (%c, %d)\n", __func__, index_name, DB_BLOCK_HEIGHT, height); + return false; } std::pair<uint256, DBVal> value; if (!db_it.GetValue(value)) { - return error("%s: unable to read value in %s at key (%c, %d)", + LogError("%s: unable to read value in %s at key (%c, %d)\n", __func__, index_name, DB_BLOCK_HEIGHT, height); + return false; } batch.Write(DBHashKey(value.first), std::move(value.second)); @@ -305,6 +334,8 @@ bool BlockFilterIndex::CustomRewind(const interfaces::BlockKey& current_tip, con batch.Write(DB_FILTER_POS, m_next_filter_pos); if (!m_db->WriteBatch(batch)) return false; + // Update cached header + m_last_header = *Assert(ReadFilterHeader(new_tip.height, new_tip.hash)); return true; } @@ -330,11 +361,13 @@ static bool LookupRange(CDBWrapper& db, const std::string& index_name, int start const CBlockIndex* stop_index, std::vector<DBVal>& results) { if (start_height < 0) { - return error("%s: start height (%d) is negative", __func__, start_height); + LogError("%s: start height (%d) is negative\n", __func__, start_height); + return false; } if (start_height > stop_index->nHeight) { - return error("%s: start height (%d) is greater than stop height (%d)", + LogError("%s: start height (%d) is greater than stop height (%d)\n", __func__, start_height, stop_index->nHeight); + return false; } size_t results_size = static_cast<size_t>(stop_index->nHeight - start_height + 1); @@ -350,8 +383,9 @@ static bool LookupRange(CDBWrapper& db, const std::string& index_name, int start size_t i = static_cast<size_t>(height - start_height); if (!db_it->GetValue(values[i])) { - return error("%s: unable to read value in %s at key (%c, %d)", + LogError("%s: unable to read value in %s at key (%c, %d)\n", __func__, index_name, DB_BLOCK_HEIGHT, height); + return false; } db_it->Next(); @@ -373,8 +407,9 @@ static bool LookupRange(CDBWrapper& db, const std::string& index_name, int start } if (!db.Read(DBHashKey(block_hash), results[i])) { - return error("%s: unable to read value in %s at key (%c, %s)", + LogError("%s: unable to read value in %s at key (%c, %s)\n", __func__, index_name, DB_BLOCK_HASH, block_hash.ToString()); + return false; } } diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index 10a1cfd2ee..cdb9563fb8 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -42,8 +42,15 @@ private: /** cache of block hash to filter header, to avoid disk access when responding to getcfcheckpt. */ std::unordered_map<uint256, uint256, FilterHeaderHasher> m_headers_cache GUARDED_BY(m_cs_headers_cache); + // Last computed header to avoid disk reads on every new block. + uint256 m_last_header{}; + bool AllowPrune() const override { return true; } + bool Write(const BlockFilter& filter, uint32_t block_height, const uint256& filter_header); + + std::optional<uint256> ReadFilterHeader(int height, const uint256& expected_block_hash); + protected: bool CustomInit(const std::optional<interfaces::BlockKey>& block) override; diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index ecd3fd21b5..dff8e50a4e 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -138,8 +138,9 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) read_out.first.ToString(), expected_block_hash.ToString()); if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { - return error("%s: previous block header not found; expected %s", + LogError("%s: previous block header not found; expected %s\n", __func__, expected_block_hash.ToString()); + return false; } } @@ -245,14 +246,16 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) for (int height = start_height; height <= stop_height; ++height) { if (!db_it.GetKey(key) || key.height != height) { - return error("%s: unexpected key in %s: expected (%c, %d)", + LogError("%s: unexpected key in %s: expected (%c, %d)\n", __func__, index_name, DB_BLOCK_HEIGHT, height); + return false; } std::pair<uint256, DBVal> value; if (!db_it.GetValue(value)) { - return error("%s: unable to read value in %s at key (%c, %d)", + LogError("%s: unable to read value in %s at key (%c, %d)\n", __func__, index_name, DB_BLOCK_HEIGHT, height); + return false; } batch.Write(DBHashKey(value.first), std::move(value.second)); @@ -285,8 +288,9 @@ bool CoinStatsIndex::CustomRewind(const interfaces::BlockKey& current_tip, const CBlock block; if (!m_chainstate->m_blockman.ReadBlockFromDisk(block, *iter_tip)) { - return error("%s: Failed to read block %s from disk", + LogError("%s: Failed to read block %s from disk\n", __func__, iter_tip->GetBlockHash().ToString()); + return false; } if (!ReverseBlock(block, iter_tip)) { @@ -353,23 +357,26 @@ bool CoinStatsIndex::CustomInit(const std::optional<interfaces::BlockKey>& block // exist. Any other errors indicate database corruption or a disk // failure, and starting the index would cause further corruption. if (m_db->Exists(DB_MUHASH)) { - return error("%s: Cannot read current %s state; index may be corrupted", + LogError("%s: Cannot read current %s state; index may be corrupted\n", __func__, GetName()); + return false; } } if (block) { DBVal entry; if (!LookUpOne(*m_db, *block, entry)) { - return error("%s: Cannot read current %s state; index may be corrupted", + LogError("%s: Cannot read current %s state; index may be corrupted\n", __func__, GetName()); + return false; } uint256 out; m_muhash.Finalize(out); if (entry.muhash != out) { - return error("%s: Cannot read current %s state; index may be corrupted", + LogError("%s: Cannot read current %s state; index may be corrupted\n", __func__, GetName()); + return false; } m_transaction_output_count = entry.transaction_output_count; @@ -422,8 +429,9 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex read_out.first.ToString(), expected_block_hash.ToString()); if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { - return error("%s: previous block header not found; expected %s", + LogError("%s: previous block header not found; expected %s\n", __func__, expected_block_hash.ToString()); + return false; } } } diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 4983926e68..80f615ed0e 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -81,20 +81,24 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe AutoFile file{m_chainstate->m_blockman.OpenBlockFile(postx, true)}; if (file.IsNull()) { - return error("%s: OpenBlockFile failed", __func__); + LogError("%s: OpenBlockFile failed\n", __func__); + return false; } CBlockHeader header; try { file >> header; if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) { - return error("%s: fseek(...) failed", __func__); + LogError("%s: fseek(...) failed\n", __func__); + return false; } file >> TX_WITH_WITNESS(tx); } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + LogError("%s: Deserialize or I/O error - %s\n", __func__, e.what()); + return false; } if (tx->GetHash() != tx_hash) { - return error("%s: txid mismatch", __func__); + LogError("%s: txid mismatch\n", __func__); + return false; } block_hash = header.GetHash(); return true; diff --git a/src/init.cpp b/src/init.cpp index 9ea7b881cb..c19d596c7f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -256,12 +256,8 @@ void Interrupt(NodeContext& node) InterruptMapPort(); if (node.connman) node.connman->Interrupt(); - if (g_txindex) { - g_txindex->Interrupt(); - } - ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); }); - if (g_coin_stats_index) { - g_coin_stats_index->Interrupt(); + for (auto* index : node.indexes) { + index->Interrupt(); } } @@ -337,16 +333,11 @@ void Shutdown(NodeContext& node) if (node.validation_signals) node.validation_signals->FlushBackgroundCallbacks(); // Stop and delete all indexes only after flushing background callbacks. - if (g_txindex) { - g_txindex->Stop(); - g_txindex.reset(); - } - if (g_coin_stats_index) { - g_coin_stats_index->Stop(); - g_coin_stats_index.reset(); - } - ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); }); + for (auto* index : node.indexes) index->Stop(); + if (g_txindex) g_txindex.reset(); + if (g_coin_stats_index) g_coin_stats_index.reset(); DestroyAllBlockFilterIndexes(); + node.indexes.clear(); // all instances are nullptr now // Any future callbacks will be dropped. This should absolutely be safe - if // missing a callback results in an unrecoverable situation, unclean shutdown @@ -477,7 +468,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-blocknotify=<cmd>", "Execute command when the best block changes (%s in cmd is replaced by block hash)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #endif argsman.AddArg("-blockreconstructionextratxn=<n>", strprintf("Extra transactions to keep in memory for compact block reconstructions (default: %u)", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Automatic broadcast and rebroadcast of any transactions from inbound peers is disabled, unless the peer has the 'forcerelay' permission. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Disables automatic broadcast and rebroadcast of transactions, unless the source peer has the 'forcerelay' permission. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-coinstatsindex", strprintf("Maintain coinstats index used by the gettxoutsetinfo RPC (default: %u)", DEFAULT_COINSTATSINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-conf=<file>", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location (only useable from command line, not configuration file) (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -534,7 +525,11 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection memory usage for the send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by outbound peers forward or backward by this amount (default: %u seconds).", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target per 24h. Limit does not apply to peers with 'download' permission or blocks created within past week. 0 = no limit (default: %s). Optional suffix units [k|K|m|M|g|G|t|T] (default: M). Lowercase is 1000 base while uppercase is 1024 base", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); +#if HAVE_SOCKADDR_UN + argsman.AddArg("-onion=<ip:port|path>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy). May be a local file path prefixed with 'unix:'.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); +#else argsman.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); +#endif argsman.AddArg("-i2psam=<ip:port>", "I2P SAM proxy to reach I2P peers and accept I2P connections (default: none)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-i2pacceptincoming", strprintf("Whether to accept inbound I2P connections (default: %i). Ignored if -i2psam is not set. Listening for inbound I2P connections is done through the SAM proxy, not by binding to a local address and port.", DEFAULT_I2P_ACCEPT_INCOMING), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onlynet=<net>", "Make automatic outbound connections only to network <net> (" + Join(GetNetworkNames(), ", ") + "). Inbound and manual connections are not affected by this option. It can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -545,7 +540,11 @@ void SetupServerArgs(ArgsManager& argsman) // TODO: remove the sentence "Nodes not using ... incoming connections." once the changes from // https://github.com/bitcoin/bitcoin/pull/23542 have become widespread. argsman.AddArg("-port=<port>", strprintf("Listen for connections on <port>. Nodes not using the default ports (default: %u, testnet: %u, signet: %u, regtest: %u) are unlikely to get incoming connections. Not relevant for I2P (see doc/i2p.md).", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), signetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); +#if HAVE_SOCKADDR_UN + argsman.AddArg("-proxy=<ip:port|path>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled). May be a local file path prefixed with 'unix:' if the proxy supports it.", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_ELISION, OptionsCategory::CONNECTION); +#else argsman.AddArg("-proxy=<ip:port>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled)", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_ELISION, OptionsCategory::CONNECTION); +#endif argsman.AddArg("-proxyrandomize", strprintf("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)", DEFAULT_PROXYRANDOMIZE), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-networkactive", "Enable all P2P network activity (default: 1). Can be changed by the setnetworkactive RPC command", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -554,16 +553,12 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control host and port to use if onion listening enabled (default: %s). If no port is specified, the default port of %i will be used.", DEFAULT_TOR_CONTROL, DEFAULT_TOR_CONTROL_PORT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::CONNECTION); #ifdef USE_UPNP -#if USE_UPNP - argsman.AddArg("-upnp", "Use UPnP to map the listening port (default: 1 when listening and no -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); -#else - argsman.AddArg("-upnp", strprintf("Use UPnP to map the listening port (default: %u)", 0), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); -#endif + argsman.AddArg("-upnp", strprintf("Use UPnP to map the listening port (default: %u)", DEFAULT_UPNP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); #else hidden_args.emplace_back("-upnp"); #endif #ifdef USE_NATPMP - argsman.AddArg("-natpmp", strprintf("Use NAT-PMP to map the listening port (default: %s)", DEFAULT_NATPMP ? "1 when listening and no -proxy" : "0"), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-natpmp", strprintf("Use NAT-PMP to map the listening port (default: %u)", DEFAULT_NATPMP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); #else hidden_args.emplace_back("-natpmp"); #endif // USE_NATPMP @@ -571,9 +566,11 @@ void SetupServerArgs(ArgsManager& argsman) "Use [host]:port notation for IPv6. Allowed permissions: " + Join(NET_PERMISSIONS_DOC, ", ") + ". " "Specify multiple permissions separated by commas (default: download,noban,mempool,relay). Can be specified multiple times.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-whitelist=<[permissions@]IP address or network>", "Add permission flags to the peers connecting from the given IP address (e.g. 1.2.3.4) or " + argsman.AddArg("-whitelist=<[permissions@]IP address or network>", "Add permission flags to the peers using the given IP address (e.g. 1.2.3.4) or " "CIDR-notated network (e.g. 1.2.3.0/24). Uses the same permissions as " - "-whitebind. Can be specified multiple times." , ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + "-whitebind. " + "Additional flags \"in\" and \"out\" control whether permissions apply to incoming connections and/or manual (default: incoming only). " + "Can be specified multiple times.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); g_wallet_init_interface.AddWalletOptions(argsman); @@ -614,7 +611,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-limitancestorsize=<n>", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds <n> kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT_KVB), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT_KVB), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-addrmantest", "Allows to test address relay on localhost", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-test=<option>", "Pass a test-only option. Options include : " + Join(TEST_OPTIONS_DOC, ", ") + ".", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_BYTES >> 20), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -643,8 +640,8 @@ void SetupServerArgs(ArgsManager& argsman) OptionsCategory::NODE_RELAY); argsman.AddArg("-minrelaytxfee=<amt>", strprintf("Fees (in %s/kvB) smaller than this are considered zero fee for relaying, mining and transaction creation (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_MIN_RELAY_TX_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); - argsman.AddArg("-whitelistforcerelay", strprintf("Add 'forcerelay' permission to whitelisted inbound peers with default permissions. This will relay transactions even if the transactions were already in the mempool. (default: %d)", DEFAULT_WHITELISTFORCERELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); - argsman.AddArg("-whitelistrelay", strprintf("Add 'relay' permission to whitelisted inbound peers with default permissions. This will accept relayed transactions even when not relaying transactions (default: %d)", DEFAULT_WHITELISTRELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); + argsman.AddArg("-whitelistforcerelay", strprintf("Add 'forcerelay' permission to whitelisted peers with default permissions. This will relay transactions even if the transactions were already in the mempool. (default: %d)", DEFAULT_WHITELISTFORCERELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); + argsman.AddArg("-whitelistrelay", strprintf("Add 'relay' permission to whitelisted peers with default permissions. This will accept relayed transactions even when not relaying transactions (default: %d)", DEFAULT_WHITELISTRELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); argsman.AddArg("-blockmaxweight=<n>", strprintf("Set maximum BIP141 block weight (default: %d)", DEFAULT_BLOCK_MAX_WEIGHT), ArgsManager::ALLOW_ANY, OptionsCategory::BLOCK_CREATION); @@ -1028,6 +1025,22 @@ bool AppInitParameterInteraction(const ArgsManager& args) if (args.GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); + if (args.IsArgSet("-test")) { + if (chainparams.GetChainType() != ChainType::REGTEST) { + return InitError(Untranslated("-test=<option> can only be used with regtest")); + } + const std::vector<std::string> options = args.GetArgs("-test"); + for (const std::string& option : options) { + auto it = std::find_if(TEST_OPTIONS_DOC.begin(), TEST_OPTIONS_DOC.end(), [&option](const std::string& doc_option) { + size_t pos = doc_option.find(" ("); + return (pos != std::string::npos) && (doc_option.substr(0, pos) == option); + }); + if (it == TEST_OPTIONS_DOC.end()) { + InitWarning(strprintf(_("Unrecognised option \"%s\" provided in -test=<option>."), option)); + } + } + } + // Also report errors from parsing before daemonization { kernel::Notifications notifications{}; @@ -1288,24 +1301,34 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } } - for (const std::string port_option : { - "-i2psam", - "-onion", - "-proxy", - "-rpcbind", - "-torcontrol", - "-whitebind", - "-zmqpubhashblock", - "-zmqpubhashtx", - "-zmqpubrawblock", - "-zmqpubrawtx", - "-zmqpubsequence", + for (const auto &port_option : std::vector<std::pair<std::string, bool>>{ + // arg name UNIX socket support + {"-i2psam", false}, + {"-onion", true}, + {"-proxy", true}, + {"-rpcbind", false}, + {"-torcontrol", false}, + {"-whitebind", false}, + {"-zmqpubhashblock", true}, + {"-zmqpubhashtx", true}, + {"-zmqpubrawblock", true}, + {"-zmqpubrawtx", true}, + {"-zmqpubsequence", true} }) { - for (const std::string& socket_addr : args.GetArgs(port_option)) { + const std::string arg{port_option.first}; + const bool unix{port_option.second}; + for (const std::string& socket_addr : args.GetArgs(arg)) { std::string host_out; uint16_t port_out{0}; if (!SplitHostPort(socket_addr, port_out, host_out)) { - return InitError(InvalidPortErrMsg(port_option, socket_addr)); +#if HAVE_SOCKADDR_UN + // Allow unix domain sockets for some options e.g. unix:/some/file/path + if (!unix || socket_addr.find(ADDR_PREFIX_UNIX) != 0) { + return InitError(InvalidPortErrMsg(arg, socket_addr)); + } +#else + return InitError(InvalidPortErrMsg(arg, socket_addr)); +#endif } } } @@ -1372,12 +1395,18 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // -noproxy (or -proxy=0) as well as the empty string can be used to not set a proxy, this is the default std::string proxyArg = args.GetArg("-proxy", ""); if (proxyArg != "" && proxyArg != "0") { - const std::optional<CService> proxyAddr{Lookup(proxyArg, 9050, fNameLookup)}; - if (!proxyAddr.has_value()) { - return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); + Proxy addrProxy; + if (IsUnixSocketPath(proxyArg)) { + addrProxy = Proxy(proxyArg, proxyRandomize); + } else { + const std::optional<CService> proxyAddr{Lookup(proxyArg, 9050, fNameLookup)}; + if (!proxyAddr.has_value()) { + return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); + } + + addrProxy = Proxy(proxyAddr.value(), proxyRandomize); } - Proxy addrProxy = Proxy(proxyAddr.value(), proxyRandomize); if (!addrProxy.IsValid()) return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); @@ -1403,11 +1432,16 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) "reaching the Tor network is explicitly forbidden: -onion=0")); } } else { - const std::optional<CService> addr{Lookup(onionArg, 9050, fNameLookup)}; - if (!addr.has_value() || !addr->IsValid()) { - return InitError(strprintf(_("Invalid -onion address or hostname: '%s'"), onionArg)); + if (IsUnixSocketPath(onionArg)) { + onion_proxy = Proxy(onionArg, proxyRandomize); + } else { + const std::optional<CService> addr{Lookup(onionArg, 9050, fNameLookup)}; + if (!addr.has_value() || !addr->IsValid()) { + return InitError(strprintf(_("Invalid -onion address or hostname: '%s'"), onionArg)); + } + + onion_proxy = Proxy(addr.value(), proxyRandomize); } - onion_proxy = Proxy{addr.value(), proxyRandomize}; } } @@ -1436,9 +1470,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) #if ENABLE_ZMQ g_zmq_notification_interface = CZMQNotificationInterface::Create( - [&chainman = node.chainman](CBlock& block, const CBlockIndex& index) { + [&chainman = node.chainman](std::vector<uint8_t>& block, const CBlockIndex& index) { assert(chainman); - return chainman->m_blockman.ReadBlockFromDisk(block, index); + return chainman->m_blockman.ReadRawBlockFromDisk(block, WITH_LOCK(cs_main, return index.GetBlockPos())); }); if (g_zmq_notification_interface) { @@ -1722,7 +1756,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Start indexes initial sync if (!StartIndexBackgroundSync(node)) { bilingual_str err_str = _("Failed to start indexes, shutting down.."); - chainman.GetNotifications().fatalError(err_str.original, err_str); + chainman.GetNotifications().fatalError(err_str); return; } // Load mempool from disk @@ -1784,6 +1818,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) connOptions.m_added_nodes = args.GetArgs("-addnode"); connOptions.nMaxOutboundLimit = *opt_max_upload; connOptions.m_peer_connect_timeout = peer_connect_timeout; + connOptions.whitelist_forcerelay = args.GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY); + connOptions.whitelist_relay = args.GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY); // Port to bind to if `-bind=addr` is provided without a `:port` suffix. const uint16_t default_bind_port = @@ -1868,9 +1904,15 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) for (const auto& net : args.GetArgs("-whitelist")) { NetWhitelistPermissions subnet; + ConnectionDirection connection_direction; bilingual_str error; - if (!NetWhitelistPermissions::TryParse(net, subnet, error)) return InitError(error); - connOptions.vWhitelistedRange.push_back(subnet); + if (!NetWhitelistPermissions::TryParse(net, subnet, connection_direction, error)) return InitError(error); + if (connection_direction & ConnectionDirection::In) { + connOptions.vWhitelistedRangeIncoming.push_back(subnet); + } + if (connection_direction & ConnectionDirection::Out) { + connOptions.vWhitelistedRangeOutgoing.push_back(subnet); + } } connOptions.vSeedNodes = args.GetArgs("-seednode"); diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 6114236623..c41f35829d 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -127,7 +127,7 @@ public: virtual bool setAddressReceiveRequest(const CTxDestination& dest, const std::string& id, const std::string& value) = 0; //! Display address on external signer - virtual bool displayAddress(const CTxDestination& dest) = 0; + virtual util::Result<void> displayAddress(const CTxDestination& dest) = 0; //! Lock coin. virtual bool lockCoin(const COutPoint& output, const bool write_to_db) = 0; diff --git a/src/kernel/chainparams.cpp b/src/kernel/chainparams.cpp index 264a2fd681..26c261eba2 100644 --- a/src/kernel/chainparams.cpp +++ b/src/kernel/chainparams.cpp @@ -133,7 +133,7 @@ public: // release ASAP to avoid it where possible. vSeeds.emplace_back("seed.bitcoin.sipa.be."); // Pieter Wuille, only supports x1, x5, x9, and xd vSeeds.emplace_back("dnsseed.bluematt.me."); // Matt Corallo, only supports x9 - vSeeds.emplace_back("dnsseed.bitcoin.dashjr.org."); // Luke Dashjr + vSeeds.emplace_back("dnsseed.bitcoin.dashjr-list-of-p2p-nodes.us."); // Luke Dashjr vSeeds.emplace_back("seed.bitcoinstats.com."); // Christian Decker, supports x1 - xf vSeeds.emplace_back("seed.bitcoin.jonasschnelli.ch."); // Jonas Schnelli, only supports x1, x5, x9, and xd vSeeds.emplace_back("seed.btc.petertodd.net."); // Peter Todd, only supports x1, x5, x9, and xd diff --git a/src/kernel/checks.cpp b/src/kernel/checks.cpp index bf8a2ec74c..45a5e25093 100644 --- a/src/kernel/checks.cpp +++ b/src/kernel/checks.cpp @@ -6,7 +6,6 @@ #include <key.h> #include <random.h> -#include <util/time.h> #include <util/translation.h> #include <memory> @@ -23,10 +22,6 @@ util::Result<void> SanityChecks(const Context&) return util::Error{Untranslated("OS cryptographic RNG sanity check failure. Aborting.")}; } - if (!ChronoSanityCheck()) { - return util::Error{Untranslated("Clock epoch mismatch. Aborting.")}; - } - return {}; } diff --git a/src/kernel/coinstats.cpp b/src/kernel/coinstats.cpp index ff8a33e804..81c496ab34 100644 --- a/src/kernel/coinstats.cpp +++ b/src/kernel/coinstats.cpp @@ -134,7 +134,8 @@ static bool ComputeUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj, c outputs[key.n] = std::move(coin); stats.coins_count++; } else { - return error("%s: unable to read value", __func__); + LogError("%s: unable to read value\n", __func__); + return false; } pcursor->Next(); } diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp index 57c5168e9f..f06f609379 100644 --- a/src/kernel/mempool_persist.cpp +++ b/src/kernel/mempool_persist.cpp @@ -44,7 +44,7 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active AutoFile file{opts.mockable_fopen_function(load_path, "rb")}; if (file.IsNull()) { - LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); + LogInfo("Failed to open mempool file. Continuing anyway.\n"); return false; } @@ -70,12 +70,12 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active uint64_t total_txns_to_load; file >> total_txns_to_load; uint64_t txns_tried = 0; - LogInfo("Loading %u mempool transactions from disk...\n", total_txns_to_load); + LogInfo("Loading %u mempool transactions from file...\n", total_txns_to_load); int next_tenth_to_report = 0; while (txns_tried < total_txns_to_load) { const int percentage_done(100.0 * txns_tried / total_txns_to_load); if (next_tenth_to_report < percentage_done / 10) { - LogInfo("Progress loading mempool transactions from disk: %d%% (tried %u, %u remaining)\n", + LogInfo("Progress loading mempool transactions from file: %d%% (tried %u, %u remaining)\n", percentage_done, txns_tried, total_txns_to_load - txns_tried); next_tenth_to_report = percentage_done / 10; } @@ -138,11 +138,11 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active } } } catch (const std::exception& e) { - LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); + LogInfo("Failed to deserialize mempool data on file: %s. Continuing anyway.\n", e.what()); return false; } - LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); + LogInfo("Imported mempool transactions from file: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); return true; } @@ -184,7 +184,9 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock } file.SetXor(xor_key); - file << (uint64_t)vinfo.size(); + uint64_t mempool_transactions_to_write(vinfo.size()); + file << mempool_transactions_to_write; + LogInfo("Writing %u mempool transactions to file...\n", mempool_transactions_to_write); for (const auto& i : vinfo) { file << TX_WITH_WITNESS(*(i.tx)); file << int64_t{count_seconds(i.m_time)}; @@ -194,7 +196,7 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock file << mapDeltas; - LogPrintf("Writing %d unbroadcast transactions to disk.\n", unbroadcast_txids.size()); + LogInfo("Writing %d unbroadcast transactions to file.\n", unbroadcast_txids.size()); file << unbroadcast_txids; if (!skip_file_commit && !FileCommit(file.Get())) @@ -205,11 +207,12 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock } auto last = SteadyClock::now(); - LogPrintf("Dumped mempool: %.3fs to copy, %.3fs to dump\n", + LogInfo("Dumped mempool: %.3fs to copy, %.3fs to dump, %d bytes dumped to file\n", Ticks<SecondsDouble>(mid - start), - Ticks<SecondsDouble>(last - mid)); + Ticks<SecondsDouble>(last - mid), + fs::file_size(dump_path)); } catch (const std::exception& e) { - LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); + LogInfo("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); return false; } return true; diff --git a/src/kernel/notifications_interface.h b/src/kernel/notifications_interface.h index c5e77b0df9..7283a88e86 100644 --- a/src/kernel/notifications_interface.h +++ b/src/kernel/notifications_interface.h @@ -5,14 +5,12 @@ #ifndef BITCOIN_KERNEL_NOTIFICATIONS_INTERFACE_H #define BITCOIN_KERNEL_NOTIFICATIONS_INTERFACE_H -#include <util/translation.h> - #include <cstdint> -#include <string> #include <variant> class CBlockIndex; enum class SynchronizationState; +struct bilingual_str; namespace kernel { @@ -48,7 +46,7 @@ public: //! perform. Applications can choose to handle the flush error notification //! by logging the error, or notifying the user, or triggering an early //! shutdown as a precaution against causing more errors. - virtual void flushError(const std::string& debug_message) {} + virtual void flushError(const bilingual_str& message) {} //! The fatal error notification is sent to notify the user when an error //! occurs in kernel code that can't be recovered from. After this @@ -57,7 +55,7 @@ public: //! handle the fatal error notification by logging the error, or notifying //! the user, or triggering an early shutdown as a precaution against //! causing more errors. - virtual void fatalError(const std::string& debug_message, const bilingual_str& user_message = {}) {} + virtual void fatalError(const bilingual_str& message) {} }; } // namespace kernel @@ -223,6 +223,12 @@ struct CExtKey { a.key == b.key; } + CExtKey() = default; + CExtKey(const CExtPubKey& xpub, const CKey& key_in) : nDepth(xpub.nDepth), nChild(xpub.nChild), chaincode(xpub.chaincode), key(key_in) + { + std::copy(xpub.vchFingerprint, xpub.vchFingerprint + sizeof(xpub.vchFingerprint), vchFingerprint); + } + void Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const; void Decode(const unsigned char code[BIP32_EXTKEY_SIZE]); [[nodiscard]] bool Derive(CExtKey& out, unsigned int nChild) const; diff --git a/src/logging.cpp b/src/logging.cpp index 42f100ded6..578650f856 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -9,9 +9,8 @@ #include <util/threadnames.h> #include <util/time.h> -#include <algorithm> #include <array> -#include <mutex> +#include <map> #include <optional> const char * const DEFAULT_DEBUGLOGFILE = "debug.log"; @@ -142,49 +141,57 @@ bool BCLog::Logger::DefaultShrinkDebugFile() const return m_categories == BCLog::NONE; } -struct CLogCategoryDesc { - BCLog::LogFlags flag; - std::string category; -}; - -const CLogCategoryDesc LogCategories[] = -{ - {BCLog::NONE, "0"}, - {BCLog::NONE, ""}, - {BCLog::NET, "net"}, - {BCLog::TOR, "tor"}, - {BCLog::MEMPOOL, "mempool"}, - {BCLog::HTTP, "http"}, - {BCLog::BENCH, "bench"}, - {BCLog::ZMQ, "zmq"}, - {BCLog::WALLETDB, "walletdb"}, - {BCLog::RPC, "rpc"}, - {BCLog::ESTIMATEFEE, "estimatefee"}, - {BCLog::ADDRMAN, "addrman"}, - {BCLog::SELECTCOINS, "selectcoins"}, - {BCLog::REINDEX, "reindex"}, - {BCLog::CMPCTBLOCK, "cmpctblock"}, - {BCLog::RAND, "rand"}, - {BCLog::PRUNE, "prune"}, - {BCLog::PROXY, "proxy"}, - {BCLog::MEMPOOLREJ, "mempoolrej"}, - {BCLog::LIBEVENT, "libevent"}, - {BCLog::COINDB, "coindb"}, - {BCLog::QT, "qt"}, - {BCLog::LEVELDB, "leveldb"}, - {BCLog::VALIDATION, "validation"}, - {BCLog::I2P, "i2p"}, - {BCLog::IPC, "ipc"}, +static const std::map<std::string, BCLog::LogFlags> LOG_CATEGORIES_BY_STR{ + {"0", BCLog::NONE}, + {"", BCLog::NONE}, + {"net", BCLog::NET}, + {"tor", BCLog::TOR}, + {"mempool", BCLog::MEMPOOL}, + {"http", BCLog::HTTP}, + {"bench", BCLog::BENCH}, + {"zmq", BCLog::ZMQ}, + {"walletdb", BCLog::WALLETDB}, + {"rpc", BCLog::RPC}, + {"estimatefee", BCLog::ESTIMATEFEE}, + {"addrman", BCLog::ADDRMAN}, + {"selectcoins", BCLog::SELECTCOINS}, + {"reindex", BCLog::REINDEX}, + {"cmpctblock", BCLog::CMPCTBLOCK}, + {"rand", BCLog::RAND}, + {"prune", BCLog::PRUNE}, + {"proxy", BCLog::PROXY}, + {"mempoolrej", BCLog::MEMPOOLREJ}, + {"libevent", BCLog::LIBEVENT}, + {"coindb", BCLog::COINDB}, + {"qt", BCLog::QT}, + {"leveldb", BCLog::LEVELDB}, + {"validation", BCLog::VALIDATION}, + {"i2p", BCLog::I2P}, + {"ipc", BCLog::IPC}, #ifdef DEBUG_LOCKCONTENTION - {BCLog::LOCK, "lock"}, + {"lock", BCLog::LOCK}, #endif - {BCLog::UTIL, "util"}, - {BCLog::BLOCKSTORAGE, "blockstorage"}, - {BCLog::TXRECONCILIATION, "txreconciliation"}, - {BCLog::SCAN, "scan"}, - {BCLog::TXPACKAGES, "txpackages"}, - {BCLog::ALL, "1"}, - {BCLog::ALL, "all"}, + {"blockstorage", BCLog::BLOCKSTORAGE}, + {"txreconciliation", BCLog::TXRECONCILIATION}, + {"scan", BCLog::SCAN}, + {"txpackages", BCLog::TXPACKAGES}, + {"1", BCLog::ALL}, + {"all", BCLog::ALL}, +}; + +static const std::unordered_map<BCLog::LogFlags, std::string> LOG_CATEGORIES_BY_FLAG{ + // Swap keys and values from LOG_CATEGORIES_BY_STR. + [](const std::map<std::string, BCLog::LogFlags>& in) { + std::unordered_map<BCLog::LogFlags, std::string> out; + for (const auto& [k, v] : in) { + switch (v) { + case BCLog::NONE: out.emplace(BCLog::NONE, ""); break; + case BCLog::ALL: out.emplace(BCLog::ALL, "all"); break; + default: out.emplace(v, k); + } + } + return out; + }(LOG_CATEGORIES_BY_STR) }; bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str) @@ -193,11 +200,10 @@ bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str) flag = BCLog::ALL; return true; } - for (const CLogCategoryDesc& category_desc : LogCategories) { - if (category_desc.category == str) { - flag = category_desc.flag; - return true; - } + auto it = LOG_CATEGORIES_BY_STR.find(str); + if (it != LOG_CATEGORIES_BY_STR.end()) { + flag = it->second; + return true; } return false; } @@ -221,76 +227,9 @@ std::string BCLog::Logger::LogLevelToStr(BCLog::Level level) std::string LogCategoryToStr(BCLog::LogFlags category) { - // Each log category string representation should sync with LogCategories - switch (category) { - case BCLog::LogFlags::NONE: - return ""; - case BCLog::LogFlags::NET: - return "net"; - case BCLog::LogFlags::TOR: - return "tor"; - case BCLog::LogFlags::MEMPOOL: - return "mempool"; - case BCLog::LogFlags::HTTP: - return "http"; - case BCLog::LogFlags::BENCH: - return "bench"; - case BCLog::LogFlags::ZMQ: - return "zmq"; - case BCLog::LogFlags::WALLETDB: - return "walletdb"; - case BCLog::LogFlags::RPC: - return "rpc"; - case BCLog::LogFlags::ESTIMATEFEE: - return "estimatefee"; - case BCLog::LogFlags::ADDRMAN: - return "addrman"; - case BCLog::LogFlags::SELECTCOINS: - return "selectcoins"; - case BCLog::LogFlags::REINDEX: - return "reindex"; - case BCLog::LogFlags::CMPCTBLOCK: - return "cmpctblock"; - case BCLog::LogFlags::RAND: - return "rand"; - case BCLog::LogFlags::PRUNE: - return "prune"; - case BCLog::LogFlags::PROXY: - return "proxy"; - case BCLog::LogFlags::MEMPOOLREJ: - return "mempoolrej"; - case BCLog::LogFlags::LIBEVENT: - return "libevent"; - case BCLog::LogFlags::COINDB: - return "coindb"; - case BCLog::LogFlags::QT: - return "qt"; - case BCLog::LogFlags::LEVELDB: - return "leveldb"; - case BCLog::LogFlags::VALIDATION: - return "validation"; - case BCLog::LogFlags::I2P: - return "i2p"; - case BCLog::LogFlags::IPC: - return "ipc"; -#ifdef DEBUG_LOCKCONTENTION - case BCLog::LogFlags::LOCK: - return "lock"; -#endif - case BCLog::LogFlags::UTIL: - return "util"; - case BCLog::LogFlags::BLOCKSTORAGE: - return "blockstorage"; - case BCLog::LogFlags::TXRECONCILIATION: - return "txreconciliation"; - case BCLog::LogFlags::SCAN: - return "scan"; - case BCLog::LogFlags::TXPACKAGES: - return "txpackages"; - case BCLog::LogFlags::ALL: - return "all"; - } - assert(false); + auto it = LOG_CATEGORIES_BY_FLAG.find(category); + assert(it != LOG_CATEGORIES_BY_FLAG.end()); + return it->second; } static std::optional<BCLog::Level> GetLogLevel(const std::string& level_str) @@ -312,18 +251,11 @@ static std::optional<BCLog::Level> GetLogLevel(const std::string& level_str) std::vector<LogCategory> BCLog::Logger::LogCategoriesList() const { - // Sort log categories by alphabetical order. - std::array<CLogCategoryDesc, std::size(LogCategories)> categories; - std::copy(std::begin(LogCategories), std::end(LogCategories), categories.begin()); - std::sort(categories.begin(), categories.end(), [](auto a, auto b) { return a.category < b.category; }); - std::vector<LogCategory> ret; - for (const CLogCategoryDesc& category_desc : categories) { - if (category_desc.flag == BCLog::NONE || category_desc.flag == BCLog::ALL) continue; - LogCategory catActive; - catActive.category = category_desc.category; - catActive.active = WillLogCategory(category_desc.flag); - ret.push_back(catActive); + for (const auto& [category, flag] : LOG_CATEGORIES_BY_STR) { + if (flag != BCLog::NONE && flag != BCLog::ALL) { + ret.push_back(LogCategory{.category = category, .active = WillLogCategory(flag)}); + } } return ret; } diff --git a/src/logging.h b/src/logging.h index 525e0aec6d..cfef65221f 100644 --- a/src/logging.h +++ b/src/logging.h @@ -65,11 +65,10 @@ namespace BCLog { #ifdef DEBUG_LOCKCONTENTION LOCK = (1 << 24), #endif - UTIL = (1 << 25), - BLOCKSTORAGE = (1 << 26), - TXRECONCILIATION = (1 << 27), - SCAN = (1 << 28), - TXPACKAGES = (1 << 29), + BLOCKSTORAGE = (1 << 25), + TXRECONCILIATION = (1 << 26), + SCAN = (1 << 27), + TXPACKAGES = (1 << 28), ALL = ~(uint32_t)0, }; enum class Level { @@ -215,7 +214,7 @@ static inline bool LogAcceptCategory(BCLog::LogFlags category, BCLog::Level leve /** Return true if str parses as a log category and set the flag */ bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str); -// Be conservative when using LogPrintf/error or other things which +// Be conservative when using functions that // unconditionally log to debug.log! It should not be the case that an inbound // peer can fill up a user's disk with debug.log entries. @@ -263,11 +262,4 @@ static inline void LogPrintf_(const std::string& logging_function, const std::st // Deprecated conditional logging #define LogPrint(category, ...) LogDebug(category, __VA_ARGS__) -template <typename... Args> -bool error(const char* fmt, const Args&... args) -{ - LogPrintf("ERROR: %s\n", tfm::format(fmt, args...)); - return false; -} - #endif // BITCOIN_LOGGING_H diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index c75f5c5e60..669c6e3b70 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -54,6 +54,7 @@ CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter* filter, const std: txn = CPartialMerkleTree(vHashes, vMatch); } +// NOLINTNEXTLINE(misc-no-recursion) uint256 CPartialMerkleTree::CalcHash(int height, unsigned int pos, const std::vector<uint256> &vTxid) { //we can never have zero txs in a merkle block, we always need the coinbase tx //if we do not have this assert, we can hit a memory access violation when indexing into vTxid @@ -74,6 +75,7 @@ uint256 CPartialMerkleTree::CalcHash(int height, unsigned int pos, const std::ve } } +// NOLINTNEXTLINE(misc-no-recursion) void CPartialMerkleTree::TraverseAndBuild(int height, unsigned int pos, const std::vector<uint256> &vTxid, const std::vector<bool> &vMatch) { // determine whether this node is the parent of at least one matched txid bool fParentOfMatch = false; @@ -92,6 +94,7 @@ void CPartialMerkleTree::TraverseAndBuild(int height, unsigned int pos, const st } } +// NOLINTNEXTLINE(misc-no-recursion) uint256 CPartialMerkleTree::TraverseAndExtract(int height, unsigned int pos, unsigned int &nBitsUsed, unsigned int &nHashUsed, std::vector<uint256> &vMatch, std::vector<unsigned int> &vnIndex) { if (nBitsUsed >= vBits.size()) { // overflowed the bits array - failure diff --git a/src/minisketch/.cirrus.yml b/src/minisketch/.cirrus.yml index 4a5353f137..5ceefee2cf 100644 --- a/src/minisketch/.cirrus.yml +++ b/src/minisketch/.cirrus.yml @@ -36,17 +36,6 @@ env_matrix_snippet: &ENV_MATRIX_VALGRIND TESTRUNS: 1 BUILD: -env_matrix_snippet: &ENV_MATRIX_SAN - - env: - ENABLE_FIELDS: 28 - - env: - BUILD: distcheck - - env: - CXXFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" - LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" - BENCH: no - env_matrix_snippet: &ENV_MATRIX_SAN_VALGRIND - env: ENABLE_FIELDS: "11,64,37" @@ -72,9 +61,9 @@ task: << : *ENV_MATRIX_SAN_VALGRIND matrix: - env: - CC: gcc + CXX: g++ - env: - CC: clang + CXX: clang++ -gdwarf-4 << : *MERGE_BASE test_script: - ./ci/cirrus.sh @@ -92,30 +81,45 @@ task: << : *ENV_MATRIX_VALGRIND matrix: - env: - CC: i686-linux-gnu-gcc + CXX: i686-linux-gnu-g++ - env: - CC: clang --target=i686-pc-linux-gnu -isystem /usr/i686-linux-gnu/include + CXX: clang++ --target=i686-linux-gnu -gdwarf-4 + CXXFLAGS: -g -O2 -isystem /usr/i686-linux-gnu/include -isystem /usr/i686-linux-gnu/include/c++/10/i686-linux-gnu test_script: - ./ci/cirrus.sh << : *CAT_LOGS task: - name: "x86_64: macOS Catalina" + name: "arm64: macOS Monterey" macos_instance: - image: catalina-base + image: ghcr.io/cirruslabs/macos-monterey-base:latest env: - # Cirrus gives us a fixed number of 12 virtual CPUs. - MAKEFLAGS: -j13 - matrix: - << : *ENV_MATRIX_SAN + # Cirrus gives us a fixed number of 4 virtual CPUs. + MAKEFLAGS: -j5 matrix: - env: - CC: gcc-9 + CXX: g++-11 + # Homebrew's gcc for arm64 has no libubsan. + matrix: + - env: + ENABLE_FIELDS: 28 + - env: + BUILD: distcheck - env: - CC: clang + CXX: clang++ + matrix: + - env: + ENABLE_FIELDS: 28 + - env: + BUILD: distcheck + - env: + CXXFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" + LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" + BENCH: no brew_script: - brew update - - brew install automake libtool gcc@9 + - brew install automake libtool gcc@11 << : *MERGE_BASE test_script: - ./ci/cirrus.sh @@ -128,13 +132,11 @@ task: cpu: 4 memory: 2G env: - EXEC_CMD: qemu-s390x -L /usr/s390x-linux-gnu + EXEC_CMD: qemu-s390x HOST: s390x-linux-gnu BUILD: << : *MERGE_BASE test_script: - # https://sourceware.org/bugzilla/show_bug.cgi?id=27008 - - rm /etc/ld.so.cache - ./ci/cirrus.sh << : *CAT_LOGS @@ -146,6 +148,7 @@ task: memory: 2G env: EXEC_CMD: wine + EXEC_EXT: .exe HOST: x86_64-w64-mingw32 BUILD: << : *MERGE_BASE diff --git a/src/minisketch/ci/cirrus.sh b/src/minisketch/ci/cirrus.sh index 02f737ca7f..36250d1651 100755 --- a/src/minisketch/ci/cirrus.sh +++ b/src/minisketch/ci/cirrus.sh @@ -7,7 +7,7 @@ export LC_ALL=C env >> test_env.log -$CC -v || true +$CXX -v || true valgrind --version || true ./autogen.sh @@ -32,10 +32,10 @@ then fi if [ -n "$EXEC_CMD" ]; then - $EXEC_CMD ./test $TESTRUNS - $EXEC_CMD ./test-verify $TESTRUNS + $EXEC_CMD "./test$EXEC_EXT" $TESTRUNS + $EXEC_CMD "./test-verify$EXEC_EXT" $TESTRUNS fi if [ "$BENCH" = "yes" ]; then - $EXEC_CMD ./bench + $EXEC_CMD "./bench$EXEC_EXT" fi diff --git a/src/minisketch/ci/linux-debian.Dockerfile b/src/minisketch/ci/linux-debian.Dockerfile index 63e5412ee7..122af36e1f 100644 --- a/src/minisketch/ci/linux-debian.Dockerfile +++ b/src/minisketch/ci/linux-debian.Dockerfile @@ -8,10 +8,10 @@ RUN apt-get update RUN apt-get install --no-install-recommends --no-upgrade -y \ git ca-certificates \ make automake libtool pkg-config dpkg-dev valgrind qemu-user \ - gcc g++ clang libc6-dbg \ + gcc g++ clang libclang-rt-dev libc6-dbg \ gcc-i686-linux-gnu g++-i686-linux-gnu libc6-dev-i386-cross libc6-dbg:i386 \ - g++-s390x-linux-gnu gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x \ - wine g++-mingw-w64-x86-64 + g++-s390x-linux-gnu libstdc++6:s390x gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x \ + wine wine64 g++-mingw-w64-x86-64 # Run a dummy command in wine to make it set up configuration RUN wine true || true diff --git a/src/minisketch/configure.ac b/src/minisketch/configure.ac index 83910448a2..cd52d7f412 100644 --- a/src/minisketch/configure.ac +++ b/src/minisketch/configure.ac @@ -104,11 +104,6 @@ esac AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"],[],[$CXXFLAG_WERROR]) -## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all -## unknown options if any other warning is produced. Test the -Wfoo case, and -## set the -Wno-foo case if it works. -AX_CHECK_COMPILE_FLAG([-Wshift-count-overflow],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-shift-count-overflow"],,[[$CXXFLAG_WERROR]]) - if test "x$use_ccache" != "xno"; then AC_MSG_CHECKING(if ccache should be used) if test x$CCACHE = x; then @@ -119,7 +114,6 @@ if test "x$use_ccache" != "xno"; then fi else use_ccache=yes - CC="$ac_cv_path_CCACHE $CC" CXX="$ac_cv_path_CCACHE $CXX" fi AC_MSG_RESULT($use_ccache) diff --git a/src/minisketch/src/int_utils.h b/src/minisketch/src/int_utils.h index d21ba56f33..2b3d8cb402 100644 --- a/src/minisketch/src/int_utils.h +++ b/src/minisketch/src/int_utils.h @@ -7,13 +7,16 @@ #ifndef _MINISKETCH_INT_UTILS_H_ #define _MINISKETCH_INT_UTILS_H_ +#include <stdint.h> #include <stdlib.h> #include <limits> #include <algorithm> #include <type_traits> -#ifdef _MSC_VER +#if defined(__cpp_lib_int_pow2) && __cpp_lib_int_pow2 >= 202002L +# include <bit> +#elif defined(_MSC_VER) # include <intrin.h> #endif @@ -54,11 +57,10 @@ class BitWriter { int offset = 0; unsigned char* out; -public: - BitWriter(unsigned char* output) : out(output) {} - template<int BITS, typename I> - inline void Write(I val) { + inline void WriteInner(I val) { + // We right shift by up to 8 bits below. Verify that's well defined for the type I. + static_assert(std::numeric_limits<I>::digits > 8, "BitWriter::WriteInner needs I > 8 bits"); int bits = BITS; if (bits + offset >= 8) { state |= ((val & ((I(1) << (8 - offset)) - 1)) << offset); @@ -77,6 +79,19 @@ public: offset += bits; } + +public: + BitWriter(unsigned char* output) : out(output) {} + + template<int BITS, typename I> + inline void Write(I val) { + // If I is smaller than an unsigned int, invoke WriteInner with argument converted to unsigned. + using compute_type = typename std::conditional< + (std::numeric_limits<I>::digits < std::numeric_limits<unsigned>::digits), + unsigned, I>::type; + return WriteInner<BITS, compute_type>(val); + } + inline void Flush() { if (offset) { *(out++) = state; @@ -129,7 +144,11 @@ constexpr inline I Mask() { return ((I((I(-1)) << (std::numeric_limits<I>::digit /** Compute the smallest power of two that is larger than val. */ template<typename I> static inline int CountBits(I val, int max) { -#ifdef _MSC_VER +#if defined(__cpp_lib_int_pow2) && __cpp_lib_int_pow2 >= 202002L + // c++20 impl + (void)max; + return std::bit_width(val); +#elif defined(_MSC_VER) (void)max; unsigned long index; unsigned char ret; @@ -175,6 +194,7 @@ public: } static constexpr inline bool IsZero(I a) { return a == 0; } + static constexpr inline bool IsOne(I a) { return a == 1; } static constexpr inline I Mask(I val) { return val & MASK; } static constexpr inline I Shift(I val, int bits) { return ((val << bits) & MASK); } static constexpr inline I UnsafeShift(I val, int bits) { return (val << bits); } @@ -233,7 +253,7 @@ template<typename I, int N, typename L, typename F> inline constexpr I GFMul(con template<typename I, typename F, int BITS, uint32_t MOD> inline I InvExtGCD(I x) { - if (F::IsZero(x)) return x; + if (F::IsZero(x) || F::IsOne(x)) return x; I t(0), newt(1); I r(MOD), newr = x; int rlen = BITS + 1, newrlen = F::Bits(newr, BITS); diff --git a/src/net.cpp b/src/net.cpp index 7c82f01d75..3e959c187c 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -238,10 +238,6 @@ static int GetnScore(const CService& addr) std::optional<CService> GetLocalAddrForPeer(CNode& node) { CService addrLocal{GetLocalAddress(node)}; - if (gArgs.GetBoolArg("-addrmantest", false)) { - // use IPv4 loopback during addrmantest - addrLocal = CService(LookupNumeric("127.0.0.1", GetListenPort())); - } // If discovery is enabled, sometimes give our peer the address it // tells us that it sees us as in case it has a better idea of our // address than we do. @@ -261,8 +257,7 @@ std::optional<CService> GetLocalAddrForPeer(CNode& node) addrLocal.SetIP(node.GetAddrLocal()); } } - if (addrLocal.IsRoutable() || gArgs.GetBoolArg("-addrmantest", false)) - { + if (addrLocal.IsRoutable()) { LogPrint(BCLog::NET, "Advertising address %s to peer=%d\n", addrLocal.ToStringAddrPort(), node.GetId()); return addrLocal; } @@ -442,7 +437,6 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo } // Connect - bool connected = false; std::unique_ptr<Sock> sock; Proxy proxy; CAddress addr_bind; @@ -455,6 +449,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo if (addrConnect.IsI2P() && use_proxy) { i2p::Connection conn; + bool connected{false}; if (m_i2p_sam_session) { connected = m_i2p_sam_session->Connect(addrConnect, conn, proxyConnectionFailed); @@ -463,7 +458,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo LOCK(m_unused_i2p_sessions_mutex); if (m_unused_i2p_sessions.empty()) { i2p_transient_session = - std::make_unique<i2p::sam::Session>(proxy.proxy, &interruptNet); + std::make_unique<i2p::sam::Session>(proxy, &interruptNet); } else { i2p_transient_session.swap(m_unused_i2p_sessions.front()); m_unused_i2p_sessions.pop(); @@ -483,20 +478,11 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo addr_bind = CAddress{conn.me, NODE_NONE}; } } else if (use_proxy) { - sock = CreateSock(proxy.proxy); - if (!sock) { - return nullptr; - } - connected = ConnectThroughProxy(proxy, addrConnect.ToStringAddr(), addrConnect.GetPort(), - *sock, nConnectTimeout, proxyConnectionFailed); + LogPrintLevel(BCLog::PROXY, BCLog::Level::Debug, "Using proxy: %s to connect to %s:%s\n", proxy.ToString(), addrConnect.ToStringAddr(), addrConnect.GetPort()); + sock = ConnectThroughProxy(proxy, addrConnect.ToStringAddr(), addrConnect.GetPort(), proxyConnectionFailed); } else { // no proxy needed (none set for target network) - sock = CreateSock(addrConnect); - if (!sock) { - return nullptr; - } - connected = ConnectSocketDirectly(addrConnect, *sock, nConnectTimeout, - conn_type == ConnectionType::MANUAL); + sock = ConnectDirectly(addrConnect, conn_type == ConnectionType::MANUAL); } if (!proxyConnectionFailed) { // If a connection to the node was attempted, and failure (if any) is not caused by a problem connecting to @@ -504,21 +490,20 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo addrman.Attempt(addrConnect, fCountFailure); } } else if (pszDest && GetNameProxy(proxy)) { - sock = CreateSock(proxy.proxy); - if (!sock) { - return nullptr; - } std::string host; uint16_t port{default_port}; SplitHostPort(std::string(pszDest), port, host); bool proxyConnectionFailed; - connected = ConnectThroughProxy(proxy, host, port, *sock, nConnectTimeout, - proxyConnectionFailed); + sock = ConnectThroughProxy(proxy, host, port, proxyConnectionFailed); } - if (!connected) { + if (!sock) { return nullptr; } + NetPermissionFlags permission_flags = NetPermissionFlags::None; + std::vector<NetWhitelistPermissions> whitelist_permissions = conn_type == ConnectionType::MANUAL ? vWhitelistedRangeOutgoing : std::vector<NetWhitelistPermissions>{}; + AddWhitelistPermissionFlags(permission_flags, addrConnect, whitelist_permissions); + // Add node NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); @@ -535,6 +520,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo conn_type, /*inbound_onion=*/false, CNodeOptions{ + .permission_flags = permission_flags, .i2p_sam_session = std::move(i2p_transient_session), .recv_flood_size = nReceiveFloodSize, .use_v2transport = use_v2transport, @@ -558,9 +544,18 @@ void CNode::CloseSocketDisconnect() m_i2p_sam_session.reset(); } -void CConnman::AddWhitelistPermissionFlags(NetPermissionFlags& flags, const CNetAddr &addr) const { - for (const auto& subnet : vWhitelistedRange) { - if (subnet.m_subnet.Match(addr)) NetPermissions::AddFlag(flags, subnet.m_flags); +void CConnman::AddWhitelistPermissionFlags(NetPermissionFlags& flags, const CNetAddr &addr, const std::vector<NetWhitelistPermissions>& ranges) const { + for (const auto& subnet : ranges) { + if (subnet.m_subnet.Match(addr)) { + NetPermissions::AddFlag(flags, subnet.m_flags); + } + } + if (NetPermissions::HasFlag(flags, NetPermissionFlags::Implicit)) { + NetPermissions::ClearFlag(flags, NetPermissionFlags::Implicit); + if (whitelist_forcerelay) NetPermissions::AddFlag(flags, NetPermissionFlags::ForceRelay); + if (whitelist_relay) NetPermissions::AddFlag(flags, NetPermissionFlags::Relay); + NetPermissions::AddFlag(flags, NetPermissionFlags::Mempool); + NetPermissions::AddFlag(flags, NetPermissionFlags::NoBan); } } @@ -575,7 +570,7 @@ void CNode::SetAddrLocal(const CService& addrLocalIn) { AssertLockNotHeld(m_addr_local_mutex); LOCK(m_addr_local_mutex); if (addrLocal.IsValid()) { - error("Addr local already set for node: %i. Refusing to change from %s to %s", id, addrLocal.ToStringAddrPort(), addrLocalIn.ToStringAddrPort()); + LogError("Addr local already set for node: %i. Refusing to change from %s to %s\n", id, addrLocal.ToStringAddrPort(), addrLocalIn.ToStringAddrPort()); } else { addrLocal = addrLocalIn; } @@ -1726,14 +1721,7 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, { int nInbound = 0; - AddWhitelistPermissionFlags(permission_flags, addr); - if (NetPermissions::HasFlag(permission_flags, NetPermissionFlags::Implicit)) { - NetPermissions::ClearFlag(permission_flags, NetPermissionFlags::Implicit); - if (gArgs.GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY)) NetPermissions::AddFlag(permission_flags, NetPermissionFlags::ForceRelay); - if (gArgs.GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY)) NetPermissions::AddFlag(permission_flags, NetPermissionFlags::Relay); - NetPermissions::AddFlag(permission_flags, NetPermissionFlags::Mempool); - NetPermissions::AddFlag(permission_flags, NetPermissionFlags::NoBan); - } + AddWhitelistPermissionFlags(permission_flags, addr, vWhitelistedRangeIncoming); { LOCK(m_nodes_mutex); @@ -1788,15 +1776,10 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); - ServiceFlags nodeServices = nLocalServices; - if (NetPermissions::HasFlag(permission_flags, NetPermissionFlags::BloomFilter)) { - nodeServices = static_cast<ServiceFlags>(nodeServices | NODE_BLOOM); - } - const bool inbound_onion = std::find(m_onion_binds.begin(), m_onion_binds.end(), addr_bind) != m_onion_binds.end(); // The V2Transport transparently falls back to V1 behavior when an incoming V1 connection is // detected, so use it whenever we signal NODE_P2P_V2. - const bool use_v2transport(nodeServices & NODE_P2P_V2); + const bool use_v2transport(nLocalServices & NODE_P2P_V2); CNode* pnode = new CNode(id, std::move(sock), @@ -1814,7 +1797,7 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, .use_v2transport = use_v2transport, }); pnode->AddRef(); - m_msgproc->InitializeNode(*pnode, nodeServices); + m_msgproc->InitializeNode(*pnode, nLocalServices); LogPrint(BCLog::NET, "connection from %s accepted\n", addr.ToStringAddrPort()); @@ -2273,7 +2256,11 @@ void CConnman::ThreadDNSAddressSeed() if (!resolveSource.SetInternal(host)) { continue; } - unsigned int nMaxIPs = 256; // Limits number of IPs learned from a DNS seed + // Limit number of IPs learned from a single DNS seed. This limit exists to prevent the results from + // one DNS seed from dominating AddrMan. Note that the number of results from a UDP DNS query is + // bounded to 33 already, but it is possible for it to use TCP where a larger number of results can be + // returned. + unsigned int nMaxIPs = 32; const auto addresses{LookupHost(host, nMaxIPs, true)}; if (!addresses.empty()) { for (const CNetAddr& ip : addresses) { @@ -2993,7 +2980,7 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, return false; } - std::unique_ptr<Sock> sock = CreateSock(addrBind); + std::unique_ptr<Sock> sock = CreateSock(addrBind.GetSAFamily()); if (!sock) { strError = strprintf(Untranslated("Couldn't open socket for incoming connections (socket returned error %s)"), NetworkErrorString(WSAGetLastError())); LogPrintLevel(BCLog::NET, BCLog::Level::Error, "%s\n", strError.original); @@ -3200,7 +3187,7 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) Proxy i2p_sam; if (GetProxy(NET_I2P, i2p_sam) && connOptions.m_i2p_accept_incoming) { m_i2p_sam_session = std::make_unique<i2p::sam::Session>(gArgs.GetDataDirNet() / "i2p_private_key", - i2p_sam.proxy, &interruptNet); + i2p_sam, &interruptNet); } for (const auto& strDest : connOptions.vSeedNodes) { @@ -53,11 +53,6 @@ class CNode; class CScheduler; struct bilingual_str; -/** Default for -whitelistrelay. */ -static const bool DEFAULT_WHITELISTRELAY = true; -/** Default for -whitelistforcerelay. */ -static const bool DEFAULT_WHITELISTFORCERELAY = false; - /** Time after which to disconnect, after waiting for a ping response (or inactivity). */ static constexpr std::chrono::minutes TIMEOUT_INTERVAL{20}; /** Run the feeler connection loop once every 2 minutes. **/ @@ -1053,7 +1048,8 @@ public: uint64_t nMaxOutboundLimit = 0; int64_t m_peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT; std::vector<std::string> vSeedNodes; - std::vector<NetWhitelistPermissions> vWhitelistedRange; + std::vector<NetWhitelistPermissions> vWhitelistedRangeIncoming; + std::vector<NetWhitelistPermissions> vWhitelistedRangeOutgoing; std::vector<NetWhitebindPermissions> vWhiteBinds; std::vector<CService> vBinds; std::vector<CService> onion_binds; @@ -1064,6 +1060,8 @@ public: std::vector<std::string> m_specified_outgoing; std::vector<std::string> m_added_nodes; bool m_i2p_accept_incoming; + bool whitelist_forcerelay = DEFAULT_WHITELISTFORCERELAY; + bool whitelist_relay = DEFAULT_WHITELISTRELAY; }; void Init(const Options& connOptions) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex, !m_total_bytes_sent_mutex) @@ -1087,7 +1085,8 @@ public: LOCK(m_total_bytes_sent_mutex); nMaxOutboundLimit = connOptions.nMaxOutboundLimit; } - vWhitelistedRange = connOptions.vWhitelistedRange; + vWhitelistedRangeIncoming = connOptions.vWhitelistedRangeIncoming; + vWhitelistedRangeOutgoing = connOptions.vWhitelistedRangeOutgoing; { LOCK(m_added_nodes_mutex); // Attempt v2 connection if we support v2 - we'll reconnect with v1 if our @@ -1098,6 +1097,8 @@ public: } } m_onion_binds = connOptions.onion_binds; + whitelist_forcerelay = connOptions.whitelist_forcerelay; + whitelist_relay = connOptions.whitelist_relay; } CConnman(uint64_t seed0, uint64_t seed1, AddrMan& addrman, const NetGroupManager& netgroupman, @@ -1339,7 +1340,7 @@ private: bool AttemptToEvictConnection(); CNode* ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure, ConnectionType conn_type, bool use_v2transport) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); - void AddWhitelistPermissionFlags(NetPermissionFlags& flags, const CNetAddr &addr) const; + void AddWhitelistPermissionFlags(NetPermissionFlags& flags, const CNetAddr &addr, const std::vector<NetWhitelistPermissions>& ranges) const; void DeleteNode(CNode* pnode); @@ -1398,7 +1399,9 @@ private: // Whitelisted ranges. Any node connecting from these is automatically // whitelisted (as well as those connecting to whitelisted binds). - std::vector<NetWhitelistPermissions> vWhitelistedRange; + std::vector<NetWhitelistPermissions> vWhitelistedRangeIncoming; + // Whitelisted ranges for outgoing connections. + std::vector<NetWhitelistPermissions> vWhitelistedRangeOutgoing; unsigned int nSendBufferMaxSize{0}; unsigned int nReceiveFloodSize{0}; @@ -1552,6 +1555,18 @@ private: std::vector<CService> m_onion_binds; /** + * flag for adding 'forcerelay' permission to whitelisted inbound + * and manual peers with default permissions. + */ + bool whitelist_forcerelay; + + /** + * flag for adding 'relay' permission to whitelisted inbound + * and manual peers with default permissions. + */ + bool whitelist_relay; + + /** * Mutex protecting m_i2p_sam_sessions. */ Mutex m_unused_i2p_sessions_mutex; diff --git a/src/net_permissions.cpp b/src/net_permissions.cpp index a134a55264..b01b2f643d 100644 --- a/src/net_permissions.cpp +++ b/src/net_permissions.cpp @@ -21,9 +21,10 @@ const std::vector<std::string> NET_PERMISSIONS_DOC{ namespace { // Parse the following format: "perm1,perm2@xxxxxx" -bool TryParsePermissionFlags(const std::string& str, NetPermissionFlags& output, size_t& readen, bilingual_str& error) +static bool TryParsePermissionFlags(const std::string& str, NetPermissionFlags& output, ConnectionDirection* output_connection_direction, size_t& readen, bilingual_str& error) { NetPermissionFlags flags = NetPermissionFlags::None; + ConnectionDirection connection_direction = ConnectionDirection::None; const auto atSeparator = str.find('@'); // if '@' is not found (ie, "xxxxx"), the caller should apply implicit permissions @@ -52,6 +53,15 @@ bool TryParsePermissionFlags(const std::string& str, NetPermissionFlags& output, else if (permission == "all") NetPermissions::AddFlag(flags, NetPermissionFlags::All); else if (permission == "relay") NetPermissions::AddFlag(flags, NetPermissionFlags::Relay); else if (permission == "addr") NetPermissions::AddFlag(flags, NetPermissionFlags::Addr); + else if (permission == "in") connection_direction |= ConnectionDirection::In; + else if (permission == "out") { + if (output_connection_direction == nullptr) { + // Only NetWhitebindPermissions() should pass a nullptr. + error = _("whitebind may only be used for incoming connections (\"out\" was passed)"); + return false; + } + connection_direction |= ConnectionDirection::Out; + } else if (permission.length() == 0); // Allow empty entries else { error = strprintf(_("Invalid P2P permission: '%s'"), permission); @@ -61,7 +71,16 @@ bool TryParsePermissionFlags(const std::string& str, NetPermissionFlags& output, readen++; } + // By default, whitelist only applies to incoming connections + if (connection_direction == ConnectionDirection::None) { + connection_direction = ConnectionDirection::In; + } else if (flags == NetPermissionFlags::None) { + error = strprintf(_("Only direction was set, no permissions: '%s'"), str); + return false; + } + output = flags; + if (output_connection_direction) *output_connection_direction = connection_direction; error = Untranslated(""); return true; } @@ -85,7 +104,7 @@ bool NetWhitebindPermissions::TryParse(const std::string& str, NetWhitebindPermi { NetPermissionFlags flags; size_t offset; - if (!TryParsePermissionFlags(str, flags, offset, error)) return false; + if (!TryParsePermissionFlags(str, flags, /*output_connection_direction=*/nullptr, offset, error)) return false; const std::string strBind = str.substr(offset); const std::optional<CService> addrBind{Lookup(strBind, 0, false)}; @@ -104,11 +123,12 @@ bool NetWhitebindPermissions::TryParse(const std::string& str, NetWhitebindPermi return true; } -bool NetWhitelistPermissions::TryParse(const std::string& str, NetWhitelistPermissions& output, bilingual_str& error) +bool NetWhitelistPermissions::TryParse(const std::string& str, NetWhitelistPermissions& output, ConnectionDirection& output_connection_direction, bilingual_str& error) { NetPermissionFlags flags; size_t offset; - if (!TryParsePermissionFlags(str, flags, offset, error)) return false; + // Only NetWhitebindPermissions should pass a nullptr for output_connection_direction. + if (!TryParsePermissionFlags(str, flags, &output_connection_direction, offset, error)) return false; const std::string net = str.substr(offset); const CSubNet subnet{LookupSubNet(net)}; diff --git a/src/net_permissions.h b/src/net_permissions.h index b7f3bffe1c..33babd6204 100644 --- a/src/net_permissions.h +++ b/src/net_permissions.h @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <netaddress.h> +#include <netbase.h> #include <string> #include <type_traits> @@ -15,6 +16,11 @@ struct bilingual_str; extern const std::vector<std::string> NET_PERMISSIONS_DOC; +/** Default for -whitelistrelay. */ +constexpr bool DEFAULT_WHITELISTRELAY = true; +/** Default for -whitelistforcerelay. */ +constexpr bool DEFAULT_WHITELISTFORCERELAY = false; + enum class NetPermissionFlags : uint32_t { None = 0, // Can query bloomfilter even if -peerbloomfilters is false @@ -83,7 +89,7 @@ public: class NetWhitelistPermissions : public NetPermissions { public: - static bool TryParse(const std::string& str, NetWhitelistPermissions& output, bilingual_str& error); + static bool TryParse(const std::string& str, NetWhitelistPermissions& output, ConnectionDirection& output_connection_direction, bilingual_str& error); CSubNet m_subnet; }; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 30d2d43e58..39ffff97d2 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -582,6 +582,20 @@ private: */ bool MaybeDiscourageAndDisconnect(CNode& pnode, Peer& peer); + /** Handle a transaction whose result was not MempoolAcceptResult::ResultType::VALID. + * @param[in] maybe_add_extra_compact_tx Whether this tx should be added to vExtraTxnForCompact. + * Set to false if the tx has already been rejected before, + * e.g. is an orphan, to avoid adding duplicate entries. + * Updates m_txrequest, m_recent_rejects, m_orphanage, and vExtraTxnForCompact. */ + void ProcessInvalidTx(NodeId nodeid, const CTransactionRef& tx, const TxValidationState& result, + bool maybe_add_extra_compact_tx) + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, cs_main); + + /** Handle a transaction whose result was MempoolAcceptResult::ResultType::VALID. + * Updates m_txrequest, m_orphanage, and vExtraTxnForCompact. Also queues the tx for relay. */ + void ProcessValidTx(NodeId nodeid, const CTransactionRef& tx, const std::list<CTransactionRef>& replaced_transactions) + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, cs_main); + /** * Reconsider orphan transactions after a parent has been accepted to the mempool. * @@ -992,7 +1006,7 @@ private: /** Orphan/conflicted/etc transactions that are kept for compact block reconstruction. * The last -blockreconstructionextratxn/DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN of * these are kept in a ring buffer */ - std::vector<std::pair<uint256, CTransactionRef>> vExtraTxnForCompact GUARDED_BY(g_msgproc_mutex); + std::vector<CTransactionRef> vExtraTxnForCompact GUARDED_BY(g_msgproc_mutex); /** Offset into vExtraTxnForCompact to insert the next tx */ size_t vExtraTxnForCompactIt GUARDED_BY(g_msgproc_mutex) = 0; @@ -1451,6 +1465,7 @@ void PeerManagerImpl::FindNextBlocks(std::vector<const CBlockIndex*>& vBlocks, c { std::vector<const CBlockIndex*> vToFetch; int nMaxHeight = std::min<int>(state->pindexBestKnownBlock->nHeight, nWindowEnd + 1); + bool is_limited_peer = IsLimitedPeer(peer); NodeId waitingfor = -1; while (pindexWalk->nHeight < nMaxHeight) { // Read up to 128 (or more, if more blocks than that are needed) successors of pindexWalk (towards @@ -1473,30 +1488,46 @@ void PeerManagerImpl::FindNextBlocks(std::vector<const CBlockIndex*>& vBlocks, c // We consider the chain that this peer is on invalid. return; } + if (!CanServeWitnesses(peer) && DeploymentActiveAt(*pindex, m_chainman, Consensus::DEPLOYMENT_SEGWIT)) { // We wouldn't download this block or its descendants from this peer. return; } + if (pindex->nStatus & BLOCK_HAVE_DATA || (activeChain && activeChain->Contains(pindex))) { - if (activeChain && pindex->HaveNumChainTxs()) + if (activeChain && pindex->HaveNumChainTxs()) { state->pindexLastCommonBlock = pindex; - } else if (!IsBlockRequested(pindex->GetBlockHash())) { - // The block is not already downloaded, and not yet in flight. - if (pindex->nHeight > nWindowEnd) { - // We reached the end of the window. - if (vBlocks.size() == 0 && waitingfor != peer.m_id) { - // We aren't able to fetch anything, but we would be if the download window was one larger. - if (nodeStaller) *nodeStaller = waitingfor; - } - return; } - vBlocks.push_back(pindex); - if (vBlocks.size() == count) { - return; + continue; + } + + // Is block in-flight? + if (IsBlockRequested(pindex->GetBlockHash())) { + if (waitingfor == -1) { + // This is the first already-in-flight block. + waitingfor = mapBlocksInFlight.lower_bound(pindex->GetBlockHash())->second.first; } - } else if (waitingfor == -1) { - // This is the first already-in-flight block. - waitingfor = mapBlocksInFlight.lower_bound(pindex->GetBlockHash())->second.first; + continue; + } + + // The block is not already downloaded, and not yet in flight. + if (pindex->nHeight > nWindowEnd) { + // We reached the end of the window. + if (vBlocks.size() == 0 && waitingfor != peer.m_id) { + // We aren't able to fetch anything, but we would be if the download window was one larger. + if (nodeStaller) *nodeStaller = waitingfor; + } + return; + } + + // Don't request blocks that go further than what limited peers can provide + if (is_limited_peer && (state->pindexBestKnownBlock->nHeight - pindex->nHeight >= static_cast<int>(NODE_NETWORK_LIMITED_MIN_BLOCKS) - 2 /* two blocks buffer for possible races */)) { + continue; + } + + vBlocks.push_back(pindex); + if (vBlocks.size() == count) { + return; } } } @@ -1571,6 +1602,11 @@ void PeerManagerImpl::InitializeNode(CNode& node, ServiceFlags our_services) m_node_states.emplace_hint(m_node_states.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(node.IsInboundConn())); assert(m_txrequest.Count(nodeid) == 0); } + + if (NetPermissions::HasFlag(node.m_permission_flags, NetPermissionFlags::BloomFilter)) { + our_services = static_cast<ServiceFlags>(our_services | NODE_BLOOM); + } + PeerRef peer = std::make_shared<Peer>(nodeid, our_services); { LOCK(m_peer_mutex); @@ -1766,7 +1802,7 @@ void PeerManagerImpl::AddToCompactExtraTransactions(const CTransactionRef& tx) return; if (!vExtraTxnForCompact.size()) vExtraTxnForCompact.resize(m_opts.max_extra_txs); - vExtraTxnForCompact[vExtraTxnForCompactIt] = std::make_pair(tx->GetWitnessHash(), tx); + vExtraTxnForCompact[vExtraTxnForCompactIt] = tx; vExtraTxnForCompactIt = (vExtraTxnForCompactIt + 1) % m_opts.max_extra_txs; } @@ -3032,6 +3068,91 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, return; } +void PeerManagerImpl::ProcessInvalidTx(NodeId nodeid, const CTransactionRef& ptx, const TxValidationState& state, + bool maybe_add_extra_compact_tx) +{ + AssertLockNotHeld(m_peer_mutex); + AssertLockHeld(g_msgproc_mutex); + AssertLockHeld(cs_main); + + LogDebug(BCLog::MEMPOOLREJ, "%s (wtxid=%s) from peer=%d was not accepted: %s\n", + ptx->GetHash().ToString(), + ptx->GetWitnessHash().ToString(), + nodeid, + state.ToString()); + + if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { + return; + } else if (state.GetResult() != TxValidationResult::TX_WITNESS_STRIPPED) { + // We can add the wtxid of this transaction to our reject filter. + // Do not add txids of witness transactions or witness-stripped + // transactions to the filter, as they can have been malleated; + // adding such txids to the reject filter would potentially + // interfere with relay of valid transactions from peers that + // do not support wtxid-based relay. See + // https://github.com/bitcoin/bitcoin/issues/8279 for details. + // We can remove this restriction (and always add wtxids to + // the filter even for witness stripped transactions) once + // wtxid-based relay is broadly deployed. + // See also comments in https://github.com/bitcoin/bitcoin/pull/18044#discussion_r443419034 + // for concerns around weakening security of unupgraded nodes + // if we start doing this too early. + m_recent_rejects.insert(ptx->GetWitnessHash().ToUint256()); + m_txrequest.ForgetTxHash(ptx->GetWitnessHash()); + // If the transaction failed for TX_INPUTS_NOT_STANDARD, + // then we know that the witness was irrelevant to the policy + // failure, since this check depends only on the txid + // (the scriptPubKey being spent is covered by the txid). + // Add the txid to the reject filter to prevent repeated + // processing of this transaction in the event that child + // transactions are later received (resulting in + // parent-fetching by txid via the orphan-handling logic). + if (state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && ptx->HasWitness()) { + m_recent_rejects.insert(ptx->GetHash().ToUint256()); + m_txrequest.ForgetTxHash(ptx->GetHash()); + } + if (maybe_add_extra_compact_tx && RecursiveDynamicUsage(*ptx) < 100000) { + AddToCompactExtraTransactions(ptx); + } + } + + MaybePunishNodeForTx(nodeid, state); + + // If the tx failed in ProcessOrphanTx, it should be removed from the orphanage unless the + // tx was still missing inputs. If the tx was not in the orphanage, EraseTx does nothing and returns 0. + if (Assume(state.GetResult() != TxValidationResult::TX_MISSING_INPUTS) && m_orphanage.EraseTx(ptx->GetHash()) > 0) { + LogDebug(BCLog::TXPACKAGES, " removed orphan tx %s (wtxid=%s)\n", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString()); + } +} + +void PeerManagerImpl::ProcessValidTx(NodeId nodeid, const CTransactionRef& tx, const std::list<CTransactionRef>& replaced_transactions) +{ + AssertLockNotHeld(m_peer_mutex); + AssertLockHeld(g_msgproc_mutex); + AssertLockHeld(cs_main); + + // As this version of the transaction was acceptable, we can forget about any requests for it. + // No-op if the tx is not in txrequest. + m_txrequest.ForgetTxHash(tx->GetHash()); + m_txrequest.ForgetTxHash(tx->GetWitnessHash()); + + m_orphanage.AddChildrenToWorkSet(*tx); + // If it came from the orphanage, remove it. No-op if the tx is not in txorphanage. + m_orphanage.EraseTx(tx->GetHash()); + + LogDebug(BCLog::MEMPOOL, "AcceptToMemoryPool: peer=%d: accepted %s (wtxid=%s) (poolsz %u txn, %u kB)\n", + nodeid, + tx->GetHash().ToString(), + tx->GetWitnessHash().ToString(), + m_mempool.size(), m_mempool.DynamicMemoryUsage() / 1000); + + RelayTransaction(tx->GetHash(), tx->GetWitnessHash()); + + for (const CTransactionRef& removedTx : replaced_transactions) { + AddToCompactExtraTransactions(removedTx); + } +} + bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) { AssertLockHeld(g_msgproc_mutex); @@ -3047,66 +3168,23 @@ bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { LogPrint(BCLog::TXPACKAGES, " accepted orphan tx %s (wtxid=%s)\n", orphanHash.ToString(), orphan_wtxid.ToString()); - LogPrint(BCLog::MEMPOOL, "AcceptToMemoryPool: peer=%d: accepted %s (wtxid=%s) (poolsz %u txn, %u kB)\n", - peer.m_id, - orphanHash.ToString(), - orphan_wtxid.ToString(), - m_mempool.size(), m_mempool.DynamicMemoryUsage() / 1000); - RelayTransaction(orphanHash, porphanTx->GetWitnessHash()); - m_orphanage.AddChildrenToWorkSet(*porphanTx); - m_orphanage.EraseTx(orphanHash); - for (const CTransactionRef& removedTx : result.m_replaced_transactions.value()) { - AddToCompactExtraTransactions(removedTx); - } + Assume(result.m_replaced_transactions.has_value()); + std::list<CTransactionRef> empty_replacement_list; + ProcessValidTx(peer.m_id, porphanTx, result.m_replaced_transactions.value_or(empty_replacement_list)); return true; } else if (state.GetResult() != TxValidationResult::TX_MISSING_INPUTS) { - if (state.IsInvalid()) { - LogPrint(BCLog::TXPACKAGES, " invalid orphan tx %s (wtxid=%s) from peer=%d. %s\n", - orphanHash.ToString(), - orphan_wtxid.ToString(), - peer.m_id, - state.ToString()); - LogPrint(BCLog::MEMPOOLREJ, "%s (wtxid=%s) from peer=%d was not accepted: %s\n", - orphanHash.ToString(), - orphan_wtxid.ToString(), - peer.m_id, - state.ToString()); - // Maybe punish peer that gave us an invalid orphan tx - MaybePunishNodeForTx(peer.m_id, state); - } - // Has inputs but not accepted to mempool - // Probably non-standard or insufficient fee - LogPrint(BCLog::TXPACKAGES, " removed orphan tx %s (wtxid=%s)\n", orphanHash.ToString(), orphan_wtxid.ToString()); - if (state.GetResult() != TxValidationResult::TX_WITNESS_STRIPPED) { - // We can add the wtxid of this transaction to our reject filter. - // Do not add txids of witness transactions or witness-stripped - // transactions to the filter, as they can have been malleated; - // adding such txids to the reject filter would potentially - // interfere with relay of valid transactions from peers that - // do not support wtxid-based relay. See - // https://github.com/bitcoin/bitcoin/issues/8279 for details. - // We can remove this restriction (and always add wtxids to - // the filter even for witness stripped transactions) once - // wtxid-based relay is broadly deployed. - // See also comments in https://github.com/bitcoin/bitcoin/pull/18044#discussion_r443419034 - // for concerns around weakening security of unupgraded nodes - // if we start doing this too early. - m_recent_rejects.insert(porphanTx->GetWitnessHash().ToUint256()); - // If the transaction failed for TX_INPUTS_NOT_STANDARD, - // then we know that the witness was irrelevant to the policy - // failure, since this check depends only on the txid - // (the scriptPubKey being spent is covered by the txid). - // Add the txid to the reject filter to prevent repeated - // processing of this transaction in the event that child - // transactions are later received (resulting in - // parent-fetching by txid via the orphan-handling logic). - if (state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && porphanTx->HasWitness()) { - // We only add the txid if it differs from the wtxid, to - // avoid wasting entries in the rolling bloom filter. - m_recent_rejects.insert(porphanTx->GetHash().ToUint256()); - } + LogPrint(BCLog::TXPACKAGES, " invalid orphan tx %s (wtxid=%s) from peer=%d. %s\n", + orphanHash.ToString(), + orphan_wtxid.ToString(), + peer.m_id, + state.ToString()); + + if (Assume(state.IsInvalid() && + state.GetResult() != TxValidationResult::TX_UNKNOWN && + state.GetResult() != TxValidationResult::TX_NO_MEMPOOL && + state.GetResult() != TxValidationResult::TX_RESULT_UNSET)) { + ProcessInvalidTx(peer.m_id, porphanTx, state, /*maybe_add_extra_compact_tx=*/false); } - m_orphanage.EraseTx(orphanHash); return true; } } @@ -4276,24 +4354,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const TxValidationState& state = result.m_state; if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { - // As this version of the transaction was acceptable, we can forget about any - // requests for it. - m_txrequest.ForgetTxHash(tx.GetHash()); - m_txrequest.ForgetTxHash(tx.GetWitnessHash()); - RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); - m_orphanage.AddChildrenToWorkSet(tx); - + ProcessValidTx(pfrom.GetId(), ptx, result.m_replaced_transactions.value()); pfrom.m_last_tx_time = GetTime<std::chrono::seconds>(); - - LogPrint(BCLog::MEMPOOL, "AcceptToMemoryPool: peer=%d: accepted %s (wtxid=%s) (poolsz %u txn, %u kB)\n", - pfrom.GetId(), - tx.GetHash().ToString(), - tx.GetWitnessHash().ToString(), - m_mempool.size(), m_mempool.DynamicMemoryUsage() / 1000); - - for (const CTransactionRef& removedTx : result.m_replaced_transactions.value()) { - AddToCompactExtraTransactions(removedTx); - } } else if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { @@ -4354,48 +4416,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_txrequest.ForgetTxHash(tx.GetHash()); m_txrequest.ForgetTxHash(tx.GetWitnessHash()); } - } else { - if (state.GetResult() != TxValidationResult::TX_WITNESS_STRIPPED) { - // We can add the wtxid of this transaction to our reject filter. - // Do not add txids of witness transactions or witness-stripped - // transactions to the filter, as they can have been malleated; - // adding such txids to the reject filter would potentially - // interfere with relay of valid transactions from peers that - // do not support wtxid-based relay. See - // https://github.com/bitcoin/bitcoin/issues/8279 for details. - // We can remove this restriction (and always add wtxids to - // the filter even for witness stripped transactions) once - // wtxid-based relay is broadly deployed. - // See also comments in https://github.com/bitcoin/bitcoin/pull/18044#discussion_r443419034 - // for concerns around weakening security of unupgraded nodes - // if we start doing this too early. - m_recent_rejects.insert(tx.GetWitnessHash().ToUint256()); - m_txrequest.ForgetTxHash(tx.GetWitnessHash()); - // If the transaction failed for TX_INPUTS_NOT_STANDARD, - // then we know that the witness was irrelevant to the policy - // failure, since this check depends only on the txid - // (the scriptPubKey being spent is covered by the txid). - // Add the txid to the reject filter to prevent repeated - // processing of this transaction in the event that child - // transactions are later received (resulting in - // parent-fetching by txid via the orphan-handling logic). - if (state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && tx.HasWitness()) { - m_recent_rejects.insert(tx.GetHash().ToUint256()); - m_txrequest.ForgetTxHash(tx.GetHash()); - } - if (RecursiveDynamicUsage(*ptx) < 100000) { - AddToCompactExtraTransactions(ptx); - } - } } - if (state.IsInvalid()) { - LogPrint(BCLog::MEMPOOLREJ, "%s (wtxid=%s) from peer=%d was not accepted: %s\n", - tx.GetHash().ToString(), - tx.GetWitnessHash().ToString(), - pfrom.GetId(), - state.ToString()); - MaybePunishNodeForTx(pfrom.GetId(), state); + ProcessInvalidTx(pfrom.GetId(), ptx, state, /*maybe_add_extra_compact_tx=*/true); } return; } diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 7530334db1..74ab6dd8d8 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -818,6 +818,19 @@ bool CService::SetSockAddr(const struct sockaddr *paddr) } } +sa_family_t CService::GetSAFamily() const +{ + switch (m_net) { + case NET_IPV4: + return AF_INET; + case NET_IPV6: + case NET_CJDNS: + return AF_INET6; + default: + return AF_UNSPEC; + } +} + uint16_t CService::GetPort() const { return port; diff --git a/src/netaddress.h b/src/netaddress.h index c697b7e0a3..c63bd4b4e5 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -540,6 +540,11 @@ public: uint16_t GetPort() const; bool GetSockAddr(struct sockaddr* paddr, socklen_t* addrlen) const; bool SetSockAddr(const struct sockaddr* paddr); + /** + * Get the address family + * @returns AF_UNSPEC if unspecified + */ + [[nodiscard]] sa_family_t GetSAFamily() const; friend bool operator==(const CService& a, const CService& b); friend bool operator!=(const CService& a, const CService& b) { return !(a == b); } friend bool operator<(const CService& a, const CService& b); diff --git a/src/netbase.cpp b/src/netbase.cpp index 9fbd9f7dea..f0fa298378 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -3,6 +3,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + #include <netbase.h> #include <compat/compat.h> @@ -21,6 +25,10 @@ #include <limits> #include <memory> +#if HAVE_SOCKADDR_UN +#include <sys/un.h> +#endif + // Settings static GlobalMutex g_proxyinfo_mutex; static Proxy proxyInfo[NET_MAX] GUARDED_BY(g_proxyinfo_mutex); @@ -208,6 +216,24 @@ CService LookupNumeric(const std::string& name, uint16_t portDefault, DNSLookupF return Lookup(name, portDefault, /*fAllowLookup=*/false, dns_lookup_function).value_or(CService{}); } +bool IsUnixSocketPath(const std::string& name) +{ +#if HAVE_SOCKADDR_UN + if (name.find(ADDR_PREFIX_UNIX) != 0) return false; + + // Split off "unix:" prefix + std::string str{name.substr(ADDR_PREFIX_UNIX.length())}; + + // Path size limit is platform-dependent + // see https://manpages.ubuntu.com/manpages/xenial/en/man7/unix.7.html + if (str.size() + 1 > sizeof(((sockaddr_un*)nullptr)->sun_path)) return false; + + return true; +#else + return false; +#endif +} + /** SOCKS version */ enum SOCKSVersion: uint8_t { SOCKS4 = 0x04, @@ -338,7 +364,8 @@ bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* a IntrRecvError recvr; LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); if (strDest.size() > 255) { - return error("Hostname too long"); + LogError("Hostname too long\n"); + return false; } // Construct the version identifier/method selection message std::vector<uint8_t> vSocks5Init; @@ -358,14 +385,17 @@ bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* a return false; } if (pchRet1[0] != SOCKSVersion::SOCKS5) { - return error("Proxy failed to initialize"); + LogError("Proxy failed to initialize\n"); + return false; } if (pchRet1[1] == SOCKS5Method::USER_PASS && auth) { // Perform username/password authentication (as described in RFC1929) std::vector<uint8_t> vAuth; vAuth.push_back(0x01); // Current (and only) version of user/pass subnegotiation - if (auth->username.size() > 255 || auth->password.size() > 255) - return error("Proxy username or password too long"); + if (auth->username.size() > 255 || auth->password.size() > 255) { + LogError("Proxy username or password too long\n"); + return false; + } vAuth.push_back(auth->username.size()); vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); vAuth.push_back(auth->password.size()); @@ -374,15 +404,18 @@ bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* a LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); uint8_t pchRetA[2]; if (InterruptibleRecv(pchRetA, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - return error("Error reading proxy authentication response"); + LogError("Error reading proxy authentication response\n"); + return false; } if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { - return error("Proxy authentication unsuccessful"); + LogError("Proxy authentication unsuccessful\n"); + return false; } } else if (pchRet1[1] == SOCKS5Method::NOAUTH) { // Perform no authentication } else { - return error("Proxy requested wrong authentication method %02x", pchRet1[1]); + LogError("Proxy requested wrong authentication method %02x\n", pchRet1[1]); + return false; } std::vector<uint8_t> vSocks5; vSocks5.push_back(SOCKSVersion::SOCKS5); // VER protocol version @@ -402,11 +435,13 @@ bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* a * error message. */ return false; } else { - return error("Error while reading proxy response"); + LogError("Error while reading proxy response\n"); + return false; } } if (pchRet2[0] != SOCKSVersion::SOCKS5) { - return error("Proxy failed to accept request"); + LogError("Proxy failed to accept request\n"); + return false; } if (pchRet2[1] != SOCKS5Reply::SUCCEEDED) { // Failures to connect to a peer that are not proxy errors @@ -414,7 +449,8 @@ bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* a return false; } if (pchRet2[2] != 0x00) { // Reserved field must be 0 - return error("Error: malformed proxy response"); + LogError("Error: malformed proxy response\n"); + return false; } uint8_t pchRet3[256]; switch (pchRet2[3]) { @@ -423,39 +459,46 @@ bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* a case SOCKS5Atyp::DOMAINNAME: { recvr = InterruptibleRecv(pchRet3, 1, g_socks5_recv_timeout, sock); if (recvr != IntrRecvError::OK) { - return error("Error reading from proxy"); + LogError("Error reading from proxy\n"); + return false; } int nRecv = pchRet3[0]; recvr = InterruptibleRecv(pchRet3, nRecv, g_socks5_recv_timeout, sock); break; } - default: return error("Error: malformed proxy response"); + default: { + LogError("Error: malformed proxy response\n"); + return false; + } } if (recvr != IntrRecvError::OK) { - return error("Error reading from proxy"); + LogError("Error reading from proxy\n"); + return false; } if (InterruptibleRecv(pchRet3, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - return error("Error reading from proxy"); + LogError("Error reading from proxy\n"); + return false; } LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); return true; } catch (const std::runtime_error& e) { - return error("Error during SOCKS5 proxy handshake: %s", e.what()); + LogError("Error during SOCKS5 proxy handshake: %s\n", e.what()); + return false; } } -std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) +std::unique_ptr<Sock> CreateSockOS(sa_family_t address_family) { - // Create a sockaddr from the specified service. - struct sockaddr_storage sockaddr; - socklen_t len = sizeof(sockaddr); - if (!address_family.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { - LogPrintf("Cannot create socket for %s: unsupported network\n", address_family.ToStringAddrPort()); - return nullptr; - } + // Not IPv4, IPv6 or UNIX + if (address_family == AF_UNSPEC) return nullptr; + + int protocol{IPPROTO_TCP}; +#if HAVE_SOCKADDR_UN + if (address_family == AF_UNIX) protocol = 0; +#endif - // Create a TCP socket in the address family of the specified service. - SOCKET hSocket = socket(((struct sockaddr*)&sockaddr)->sa_family, SOCK_STREAM, IPPROTO_TCP); + // Create a socket in the specified address family. + SOCKET hSocket = socket(address_family, SOCK_STREAM, protocol); if (hSocket == INVALID_SOCKET) { return nullptr; } @@ -479,21 +522,25 @@ std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) } #endif - // Set the no-delay option (disable Nagle's algorithm) on the TCP socket. - const int on{1}; - if (sock->SetSockOpt(IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on)) == SOCKET_ERROR) { - LogPrint(BCLog::NET, "Unable to set TCP_NODELAY on a newly created socket, continuing anyway\n"); - } - // Set the non-blocking option on the socket. if (!sock->SetNonBlocking()) { LogPrintf("Error setting socket to non-blocking: %s\n", NetworkErrorString(WSAGetLastError())); return nullptr; } + +#if HAVE_SOCKADDR_UN + if (address_family == AF_UNIX) return sock; +#endif + + // Set the no-delay option (disable Nagle's algorithm) on the TCP socket. + const int on{1}; + if (sock->SetSockOpt(IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on)) == SOCKET_ERROR) { + LogPrint(BCLog::NET, "Unable to set TCP_NODELAY on a newly created socket, continuing anyway\n"); + } return sock; } -std::function<std::unique_ptr<Sock>(const CService&)> CreateSock = CreateSockTCP; +std::function<std::unique_ptr<Sock>(const sa_family_t&)> CreateSock = CreateSockOS; template<typename... Args> static void LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args) { @@ -505,18 +552,10 @@ static void LogConnectFailure(bool manual_connection, const char* fmt, const Arg } } -bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nTimeout, bool manual_connection) +static bool ConnectToSocket(const Sock& sock, struct sockaddr* sockaddr, socklen_t len, const std::string& dest_str, bool manual_connection) { - // Create a sockaddr from the specified service. - struct sockaddr_storage sockaddr; - socklen_t len = sizeof(sockaddr); - if (!addrConnect.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { - LogPrintf("Cannot connect to %s: unsupported network\n", addrConnect.ToStringAddrPort()); - return false; - } - - // Connect to the addrConnect service on the hSocket socket. - if (sock.Connect(reinterpret_cast<struct sockaddr*>(&sockaddr), len) == SOCKET_ERROR) { + // Connect to `sockaddr` using `sock`. + if (sock.Connect(sockaddr, len) == SOCKET_ERROR) { int nErr = WSAGetLastError(); // WSAEINVAL is here because some legacy version of winsock uses it if (nErr == WSAEINPROGRESS || nErr == WSAEWOULDBLOCK || nErr == WSAEINVAL) @@ -526,13 +565,13 @@ bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nT // synchronously to check for successful connection with a timeout. const Sock::Event requested = Sock::RECV | Sock::SEND; Sock::Event occurred; - if (!sock.Wait(std::chrono::milliseconds{nTimeout}, requested, &occurred)) { + if (!sock.Wait(std::chrono::milliseconds{nConnectTimeout}, requested, &occurred)) { LogPrintf("wait for connect to %s failed: %s\n", - addrConnect.ToStringAddrPort(), + dest_str, NetworkErrorString(WSAGetLastError())); return false; } else if (occurred == 0) { - LogPrint(BCLog::NET, "connection attempt to %s timed out\n", addrConnect.ToStringAddrPort()); + LogPrint(BCLog::NET, "connection attempt to %s timed out\n", dest_str); return false; } @@ -544,13 +583,13 @@ bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nT socklen_t sockerr_len = sizeof(sockerr); if (sock.GetSockOpt(SOL_SOCKET, SO_ERROR, (sockopt_arg_type)&sockerr, &sockerr_len) == SOCKET_ERROR) { - LogPrintf("getsockopt() for %s failed: %s\n", addrConnect.ToStringAddrPort(), NetworkErrorString(WSAGetLastError())); + LogPrintf("getsockopt() for %s failed: %s\n", dest_str, NetworkErrorString(WSAGetLastError())); return false; } if (sockerr != 0) { LogConnectFailure(manual_connection, "connect() to %s failed after wait: %s", - addrConnect.ToStringAddrPort(), + dest_str, NetworkErrorString(sockerr)); return false; } @@ -561,13 +600,68 @@ bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nT else #endif { - LogConnectFailure(manual_connection, "connect() to %s failed: %s", addrConnect.ToStringAddrPort(), NetworkErrorString(WSAGetLastError())); + LogConnectFailure(manual_connection, "connect() to %s failed: %s", dest_str, NetworkErrorString(WSAGetLastError())); return false; } } return true; } +std::unique_ptr<Sock> ConnectDirectly(const CService& dest, bool manual_connection) +{ + auto sock = CreateSock(dest.GetSAFamily()); + if (!sock) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Cannot create a socket for connecting to %s\n", dest.ToStringAddrPort()); + return {}; + } + + // Create a sockaddr from the specified service. + struct sockaddr_storage sockaddr; + socklen_t len = sizeof(sockaddr); + if (!dest.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { + LogPrintf("Cannot get sockaddr for %s: unsupported network\n", dest.ToStringAddrPort()); + return {}; + } + + if (!ConnectToSocket(*sock, (struct sockaddr*)&sockaddr, len, dest.ToStringAddrPort(), manual_connection)) { + return {}; + } + + return sock; +} + +std::unique_ptr<Sock> Proxy::Connect() const +{ + if (!IsValid()) return {}; + + if (!m_is_unix_socket) return ConnectDirectly(proxy, /*manual_connection=*/true); + +#if HAVE_SOCKADDR_UN + auto sock = CreateSock(AF_UNIX); + if (!sock) { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Cannot create a socket for connecting to %s\n", m_unix_socket_path); + return {}; + } + + const std::string path{m_unix_socket_path.substr(ADDR_PREFIX_UNIX.length())}; + + struct sockaddr_un addrun; + memset(&addrun, 0, sizeof(addrun)); + addrun.sun_family = AF_UNIX; + // leave the last char in addrun.sun_path[] to be always '\0' + memcpy(addrun.sun_path, path.c_str(), std::min(sizeof(addrun.sun_path) - 1, path.length())); + socklen_t len = sizeof(addrun); + + if(!ConnectToSocket(*sock, (struct sockaddr*)&addrun, len, path, /*manual_connection=*/true)) { + return {}; + } + + return sock; +#else + return {}; +#endif +} + bool SetProxy(enum Network net, const Proxy &addrProxy) { assert(net >= 0 && net < NET_MAX); if (!addrProxy.IsValid()) @@ -616,27 +710,32 @@ bool IsProxy(const CNetAddr &addr) { return false; } -bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_t port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed) +std::unique_ptr<Sock> ConnectThroughProxy(const Proxy& proxy, + const std::string& dest, + uint16_t port, + bool& proxy_connection_failed) { // first connect to proxy server - if (!ConnectSocketDirectly(proxy.proxy, sock, nTimeout, true)) { - outProxyConnectionFailed = true; - return false; + auto sock = proxy.Connect(); + if (!sock) { + proxy_connection_failed = true; + return {}; } + // do socks negotiation - if (proxy.randomize_credentials) { + if (proxy.m_randomize_credentials) { ProxyCredentials random_auth; static std::atomic_int counter(0); random_auth.username = random_auth.password = strprintf("%i", counter++); - if (!Socks5(strDest, port, &random_auth, sock)) { - return false; + if (!Socks5(dest, port, &random_auth, *sock)) { + return {}; } } else { - if (!Socks5(strDest, port, nullptr, sock)) { - return false; + if (!Socks5(dest, port, nullptr, *sock)) { + return {}; } } - return true; + return sock; } CSubNet LookupSubNet(const std::string& subnet_str) diff --git a/src/netbase.h b/src/netbase.h index 1bd95ba0d9..321c288f67 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -27,6 +27,9 @@ static const int DEFAULT_CONNECT_TIMEOUT = 5000; //! -dns default static const int DEFAULT_NAME_LOOKUP = true; +/** Prefix for unix domain socket addresses (which are local filesystem paths) */ +const std::string ADDR_PREFIX_UNIX = "unix:"; + enum class ConnectionDirection { None = 0, In = (1U << 0), @@ -43,16 +46,46 @@ static inline bool operator&(ConnectionDirection a, ConnectionDirection b) { return (underlying(a) & underlying(b)); } +/** + * Check if a string is a valid UNIX domain socket path + * + * @param name The string provided by the user representing a local path + * + * @returns Whether the string has proper format, length, and points to an existing file path + */ +bool IsUnixSocketPath(const std::string& name); + class Proxy { public: - Proxy(): randomize_credentials(false) {} - explicit Proxy(const CService &_proxy, bool _randomize_credentials=false): proxy(_proxy), randomize_credentials(_randomize_credentials) {} - - bool IsValid() const { return proxy.IsValid(); } + Proxy() : m_is_unix_socket(false), m_randomize_credentials(false) {} + explicit Proxy(const CService& _proxy, bool _randomize_credentials = false) : proxy(_proxy), m_is_unix_socket(false), m_randomize_credentials(_randomize_credentials) {} + explicit Proxy(const std::string path, bool _randomize_credentials = false) : m_unix_socket_path(path), m_is_unix_socket(true), m_randomize_credentials(_randomize_credentials) {} CService proxy; - bool randomize_credentials; + std::string m_unix_socket_path; + bool m_is_unix_socket; + bool m_randomize_credentials; + + bool IsValid() const + { + if (m_is_unix_socket) return IsUnixSocketPath(m_unix_socket_path); + return proxy.IsValid(); + } + + sa_family_t GetFamily() const + { + if (m_is_unix_socket) return AF_UNIX; + return proxy.GetSAFamily(); + } + + std::string ToString() const + { + if (m_is_unix_socket) return m_unix_socket_path; + return proxy.ToStringAddrPort(); + } + + std::unique_ptr<Sock> Connect() const; }; /** Credentials for proxy authentication */ @@ -229,47 +262,42 @@ CService LookupNumeric(const std::string& name, uint16_t portDefault = 0, DNSLoo CSubNet LookupSubNet(const std::string& subnet_str); /** - * Create a TCP socket in the given address family. - * @param[in] address_family The socket is created in the same address family as this address. + * Create a TCP or UNIX socket in the given address family. + * @param[in] address_family to use for the socket. * @return pointer to the created Sock object or unique_ptr that owns nothing in case of failure */ -std::unique_ptr<Sock> CreateSockTCP(const CService& address_family); +std::unique_ptr<Sock> CreateSockOS(sa_family_t address_family); /** - * Socket factory. Defaults to `CreateSockTCP()`, but can be overridden by unit tests. + * Socket factory. Defaults to `CreateSockOS()`, but can be overridden by unit tests. */ -extern std::function<std::unique_ptr<Sock>(const CService&)> CreateSock; +extern std::function<std::unique_ptr<Sock>(const sa_family_t&)> CreateSock; /** - * Try to connect to the specified service on the specified socket. + * Create a socket and try to connect to the specified service. * - * @param addrConnect The service to which to connect. - * @param sock The socket on which to connect. - * @param nTimeout Wait this many milliseconds for the connection to be - * established. - * @param manual_connection Whether or not the connection was manually requested - * (e.g. through the addnode RPC) + * @param[in] dest The service to which to connect. + * @param[in] manual_connection Whether or not the connection was manually requested (e.g. through the addnode RPC) * - * @returns Whether or not a connection was successfully made. + * @returns the connected socket if the operation succeeded, empty unique_ptr otherwise */ -bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nTimeout, bool manual_connection); +std::unique_ptr<Sock> ConnectDirectly(const CService& dest, bool manual_connection); /** * Connect to a specified destination service through a SOCKS5 proxy by first * connecting to the SOCKS5 proxy. * - * @param proxy The SOCKS5 proxy. - * @param strDest The destination service to which to connect. - * @param port The destination port. - * @param sock The socket on which to connect to the SOCKS5 proxy. - * @param nTimeout Wait this many milliseconds for the connection to the SOCKS5 - * proxy to be established. - * @param[out] outProxyConnectionFailed Whether or not the connection to the - * SOCKS5 proxy failed. + * @param[in] proxy The SOCKS5 proxy. + * @param[in] dest The destination service to which to connect. + * @param[in] port The destination port. + * @param[out] proxy_connection_failed Whether or not the connection to the SOCKS5 proxy failed. * - * @returns Whether or not the operation succeeded. + * @returns the connected socket if the operation succeeded. Otherwise an empty unique_ptr. */ -bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_t port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed); +std::unique_ptr<Sock> ConnectThroughProxy(const Proxy& proxy, + const std::string& dest, + uint16_t port, + bool& proxy_connection_failed); /** * Interrupt SOCKS5 reads or writes. diff --git a/src/node/abort.cpp b/src/node/abort.cpp index 1bdc91670d..b727608384 100644 --- a/src/node/abort.cpp +++ b/src/node/abort.cpp @@ -16,14 +16,13 @@ namespace node { -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const std::string& debug_message, const bilingual_str& user_message) +void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message) { - SetMiscWarning(Untranslated(debug_message)); - LogPrintf("*** %s\n", debug_message); - InitError(user_message.empty() ? _("A fatal internal error occurred, see debug.log for details") : user_message); + SetMiscWarning(message); + InitError(_("A fatal internal error occurred, see debug.log for details: ") + message); exit_status.store(EXIT_FAILURE); if (shutdown && !(*shutdown)()) { - LogPrintf("Error: failed to send shutdown signal\n"); + LogError("Failed to send shutdown signal\n"); }; } } // namespace node diff --git a/src/node/abort.h b/src/node/abort.h index 28d021cc78..1092279142 100644 --- a/src/node/abort.h +++ b/src/node/abort.h @@ -5,17 +5,16 @@ #ifndef BITCOIN_NODE_ABORT_H #define BITCOIN_NODE_ABORT_H -#include <util/translation.h> - #include <atomic> -#include <string> + +struct bilingual_str; namespace util { class SignalInterrupt; } // namespace util namespace node { -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const std::string& debug_message, const bilingual_str& user_message = {}); +void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message); } // namespace node #endif // BITCOIN_NODE_ABORT_H diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index c499bbfa6a..576c07a833 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -131,12 +131,14 @@ bool BlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, s pindexNew->nTx = diskindex.nTx; if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams)) { - return error("%s: CheckProofOfWork failed: %s", __func__, pindexNew->ToString()); + LogError("%s: CheckProofOfWork failed: %s\n", __func__, pindexNew->ToString()); + return false; } pcursor->Next(); } else { - return error("%s: failed to read value", __func__); + LogError("%s: failed to read value\n", __func__); + return false; } } else { break; @@ -402,7 +404,7 @@ bool BlockManager::LoadBlockIndex(const std::optional<uint256>& snapshot_blockha if (snapshot_blockhash) { const std::optional<AssumeutxoData> maybe_au_data = GetParams().AssumeutxoForBlockhash(*snapshot_blockhash); if (!maybe_au_data) { - m_opts.notifications.fatalError(strprintf("Assumeutxo data not found for the given blockhash '%s'.", snapshot_blockhash->ToString())); + m_opts.notifications.fatalError(strprintf(_("Assumeutxo data not found for the given blockhash '%s'."), snapshot_blockhash->ToString())); return false; } const AssumeutxoData& au_data = *Assert(maybe_au_data); @@ -432,7 +434,8 @@ bool BlockManager::LoadBlockIndex(const std::optional<uint256>& snapshot_blockha for (CBlockIndex* pindex : vSortedByHeight) { if (m_interrupt) return false; if (previous_index && pindex->nHeight > previous_index->nHeight + 1) { - return error("%s: block index is non-contiguous, index of height %d missing", __func__, previous_index->nHeight + 1); + LogError("%s: block index is non-contiguous, index of height %d missing\n", __func__, previous_index->nHeight + 1); + return false; } previous_index = pindex; pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); @@ -671,7 +674,8 @@ bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos // Open history file to append AutoFile fileout{OpenUndoFile(pos)}; if (fileout.IsNull()) { - return error("%s: OpenUndoFile failed", __func__); + LogError("%s: OpenUndoFile failed\n", __func__); + return false; } // Write index header @@ -681,7 +685,8 @@ bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos // Write undo data long fileOutPos = ftell(fileout.Get()); if (fileOutPos < 0) { - return error("%s: ftell failed", __func__); + LogError("%s: ftell failed\n", __func__); + return false; } pos.nPos = (unsigned int)fileOutPos; fileout << blockundo; @@ -700,13 +705,15 @@ bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& in const FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())}; if (pos.IsNull()) { - return error("%s: no undo data available", __func__); + LogError("%s: no undo data available\n", __func__); + return false; } // Open history file to read AutoFile filein{OpenUndoFile(pos, true)}; if (filein.IsNull()) { - return error("%s: OpenUndoFile failed", __func__); + LogError("%s: OpenUndoFile failed\n", __func__); + return false; } // Read block @@ -717,12 +724,14 @@ bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& in verifier >> blockundo; filein >> hashChecksum; } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + LogError("%s: Deserialize or I/O error - %s\n", __func__, e.what()); + return false; } // Verify checksum if (hashChecksum != verifier.GetHash()) { - return error("%s: Checksum mismatch", __func__); + LogError("%s: Checksum mismatch\n", __func__); + return false; } return true; @@ -732,7 +741,7 @@ bool BlockManager::FlushUndoFile(int block_file, bool finalize) { FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize); if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { - m_opts.notifications.flushError("Flushing undo file to disk failed. This is likely the result of an I/O error."); + m_opts.notifications.flushError(_("Flushing undo file to disk failed. This is likely the result of an I/O error.")); return false; } return true; @@ -754,7 +763,7 @@ bool BlockManager::FlushBlockFile(int blockfile_num, bool fFinalize, bool finali FlatFilePos block_pos_old(blockfile_num, m_blockfile_info[blockfile_num].nSize); if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { - m_opts.notifications.flushError("Flushing block file to disk failed. This is likely the result of an I/O error."); + m_opts.notifications.flushError(_("Flushing block file to disk failed. This is likely the result of an I/O error.")); success = false; } // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, @@ -897,19 +906,19 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne if (!fKnown) { LogPrint(BCLog::BLOCKSTORAGE, "Leaving block file %i: %s (onto %i) (height %i)\n", last_blockfile, m_blockfile_info[last_blockfile].ToString(), nFile, nHeight); - } - // Do not propagate the return code. The flush concerns a previous block - // and undo file that has already been written to. If a flush fails - // here, and we crash, there is no expected additional block data - // inconsistency arising from the flush failure here. However, the undo - // data may be inconsistent after a crash if the flush is called during - // a reindex. A flush error might also leave some of the data files - // untrimmed. - if (!FlushBlockFile(last_blockfile, !fKnown, finalize_undo)) { - LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, - "Failed to flush previous block file %05i (finalize=%i, finalize_undo=%i) before opening new block file %05i\n", - last_blockfile, !fKnown, finalize_undo, nFile); + // Do not propagate the return code. The flush concerns a previous block + // and undo file that has already been written to. If a flush fails + // here, and we crash, there is no expected additional block data + // inconsistency arising from the flush failure here. However, the undo + // data may be inconsistent after a crash if the flush is called during + // a reindex. A flush error might also leave some of the data files + // untrimmed. + if (!FlushBlockFile(last_blockfile, !fKnown, finalize_undo)) { + LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, + "Failed to flush previous block file %05i (finalize=%i, finalize_undo=%i) before opening new block file %05i\n", + last_blockfile, !fKnown, finalize_undo, nFile); + } } // No undo data yet in the new file, so reset our undo-height tracking. m_blockfile_cursors[chain_type] = BlockfileCursor{nFile}; @@ -926,7 +935,7 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne bool out_of_space; size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); if (out_of_space) { - m_opts.notifications.fatalError("Disk space is too low!", _("Disk space is too low!")); + m_opts.notifications.fatalError(_("Disk space is too low!")); return false; } if (bytes_allocated != 0 && IsPruneMode()) { @@ -951,7 +960,7 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP bool out_of_space; size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); if (out_of_space) { - return FatalError(m_opts.notifications, state, "Disk space is too low!", _("Disk space is too low!")); + return FatalError(m_opts.notifications, state, _("Disk space is too low!")); } if (bytes_allocated != 0 && IsPruneMode()) { m_check_for_pruning = true; @@ -965,7 +974,8 @@ bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const // Open history file to append AutoFile fileout{OpenBlockFile(pos)}; if (fileout.IsNull()) { - return error("WriteBlockToDisk: OpenBlockFile failed"); + LogError("WriteBlockToDisk: OpenBlockFile failed\n"); + return false; } // Write index header @@ -975,7 +985,8 @@ bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const // Write block long fileOutPos = ftell(fileout.Get()); if (fileOutPos < 0) { - return error("WriteBlockToDisk: ftell failed"); + LogError("WriteBlockToDisk: ftell failed\n"); + return false; } pos.nPos = (unsigned int)fileOutPos; fileout << TX_WITH_WITNESS(block); @@ -993,10 +1004,11 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid if (block.GetUndoPos().IsNull()) { FlatFilePos _pos; if (!FindUndoPos(state, block.nFile, _pos, ::GetSerializeSize(blockundo) + 40)) { - return error("ConnectBlock(): FindUndoPos failed"); + LogError("ConnectBlock(): FindUndoPos failed\n"); + return false; } if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash())) { - return FatalError(m_opts.notifications, state, "Failed to write undo data"); + return FatalError(m_opts.notifications, state, _("Failed to write undo data.")); } // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height @@ -1031,24 +1043,28 @@ bool BlockManager::ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) cons // Open history file to read AutoFile filein{OpenBlockFile(pos, true)}; if (filein.IsNull()) { - return error("ReadBlockFromDisk: OpenBlockFile failed for %s", pos.ToString()); + LogError("ReadBlockFromDisk: OpenBlockFile failed for %s\n", pos.ToString()); + return false; } // Read block try { filein >> TX_WITH_WITNESS(block); } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString()); + LogError("%s: Deserialize or I/O error - %s at %s\n", __func__, e.what(), pos.ToString()); + return false; } // Check the header if (!CheckProofOfWork(block.GetHash(), block.nBits, GetConsensus())) { - return error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString()); + LogError("ReadBlockFromDisk: Errors in block header at %s\n", pos.ToString()); + return false; } // Signet only: check block solution if (GetConsensus().signet_blocks && !CheckSignetBlockSolution(block, GetConsensus())) { - return error("ReadBlockFromDisk: Errors in block solution at %s", pos.ToString()); + LogError("ReadBlockFromDisk: Errors in block solution at %s\n", pos.ToString()); + return false; } return true; @@ -1062,8 +1078,9 @@ bool BlockManager::ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) co return false; } if (block.GetHash() != index.GetBlockHash()) { - return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", + LogError("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s\n", index.ToString(), block_pos.ToString()); + return false; } return true; } @@ -1071,10 +1088,17 @@ bool BlockManager::ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) co bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos) const { FlatFilePos hpos = pos; + // If nPos is less than 8 the pos is null and we don't have the block data + // Return early to prevent undefined behavior of unsigned int underflow + if (hpos.nPos < 8) { + LogError("%s: OpenBlockFile failed for %s\n", __func__, pos.ToString()); + return false; + } hpos.nPos -= 8; // Seek back 8 bytes for meta header AutoFile filein{OpenBlockFile(hpos, true)}; if (filein.IsNull()) { - return error("%s: OpenBlockFile failed for %s", __func__, pos.ToString()); + LogError("%s: OpenBlockFile failed for %s\n", __func__, pos.ToString()); + return false; } try { @@ -1084,20 +1108,23 @@ bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatF filein >> blk_start >> blk_size; if (blk_start != GetParams().MessageStart()) { - return error("%s: Block magic mismatch for %s: %s versus expected %s", __func__, pos.ToString(), + LogError("%s: Block magic mismatch for %s: %s versus expected %s\n", __func__, pos.ToString(), HexStr(blk_start), HexStr(GetParams().MessageStart())); + return false; } if (blk_size > MAX_SIZE) { - return error("%s: Block data is larger than maximum deserialization size for %s: %s versus %s", __func__, pos.ToString(), + LogError("%s: Block data is larger than maximum deserialization size for %s: %s versus %s\n", __func__, pos.ToString(), blk_size, MAX_SIZE); + return false; } block.resize(blk_size); // Zeroing of memory is intentional here filein.read(MakeWritableByteSpan(block)); } catch (const std::exception& e) { - return error("%s: Read from block file failed: %s for %s", __func__, e.what(), pos.ToString()); + LogError("%s: Read from block file failed: %s for %s\n", __func__, e.what(), pos.ToString()); + return false; } return true; @@ -1117,12 +1144,12 @@ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, cons nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE); } if (!FindBlockPos(blockPos, nBlockSize, nHeight, block.GetBlockTime(), position_known)) { - error("%s: FindBlockPos failed", __func__); + LogError("%s: FindBlockPos failed\n", __func__); return FlatFilePos(); } if (!position_known) { if (!WriteBlockToDisk(block, blockPos)) { - m_opts.notifications.fatalError("Failed to write block"); + m_opts.notifications.fatalError(_("Failed to write block.")); return FlatFilePos(); } } @@ -1206,7 +1233,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile for (Chainstate* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { BlockValidationState state; if (!chainstate->ActivateBestChain(state, nullptr)) { - chainman.GetNotifications().fatalError(strprintf("Failed to connect best block (%s)", state.ToString())); + chainman.GetNotifications().fatalError(strprintf(_("Failed to connect best block (%s)."), state.ToString())); return; } } diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index f9a372e3de..4d2d83812e 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -406,6 +406,7 @@ public: NodeContext* m_context{nullptr}; }; +// NOLINTNEXTLINE(misc-no-recursion) bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<RecursiveMutex>& lock, const CChain& active, const BlockManager& blockman) { if (!index) return false; diff --git a/src/node/kernel_notifications.cpp b/src/node/kernel_notifications.cpp index 1fd3bad296..99f909ff75 100644 --- a/src/node/kernel_notifications.cpp +++ b/src/node/kernel_notifications.cpp @@ -84,15 +84,15 @@ void KernelNotifications::warning(const bilingual_str& warning) DoWarning(warning); } -void KernelNotifications::flushError(const std::string& debug_message) +void KernelNotifications::flushError(const bilingual_str& message) { - AbortNode(&m_shutdown, m_exit_status, debug_message); + AbortNode(&m_shutdown, m_exit_status, message); } -void KernelNotifications::fatalError(const std::string& debug_message, const bilingual_str& user_message) +void KernelNotifications::fatalError(const bilingual_str& message) { node::AbortNode(m_shutdown_on_fatal_error ? &m_shutdown : nullptr, - m_exit_status, debug_message, user_message); + m_exit_status, message); } void ReadNotificationArgs(const ArgsManager& args, KernelNotifications& notifications) diff --git a/src/node/kernel_notifications.h b/src/node/kernel_notifications.h index 38d8600ac6..f4d97a0fff 100644 --- a/src/node/kernel_notifications.h +++ b/src/node/kernel_notifications.h @@ -9,7 +9,6 @@ #include <atomic> #include <cstdint> -#include <string> class ArgsManager; class CBlockIndex; @@ -37,9 +36,9 @@ public: void warning(const bilingual_str& warning) override; - void flushError(const std::string& debug_message) override; + void flushError(const bilingual_str& message) override; - void fatalError(const std::string& debug_message, const bilingual_str& user_message = {}) override; + void fatalError(const bilingual_str& message) override; //! Block height after which blockTip notification will return Interrupted{}, if >0. int m_stop_at_height{DEFAULT_STOPATHEIGHT}; diff --git a/src/node/transaction.h b/src/node/transaction.h index 168273594c..6782536ace 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -26,6 +26,12 @@ struct NodeContext; */ static const CFeeRate DEFAULT_MAX_RAW_TX_FEE_RATE{COIN / 10}; +/** Maximum burn value for sendrawtransaction, submitpackage, and testmempoolaccept RPC calls. + * By default, a transaction with a burn value higher than this will be rejected + * by these RPCs and the GUI. This can be overridden with the maxburnamount argument. + */ +static const CAmount DEFAULT_MAX_BURN_AMOUNT{0}; + /** * Submit a transaction to the mempool and (optionally) relay it to all P2P peers. * diff --git a/src/noui.cpp b/src/noui.cpp index af5a180ce3..23637dfa1f 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -28,20 +28,21 @@ bool noui_ThreadSafeMessageBox(const bilingual_str& message, const std::string& switch (style) { case CClientUIInterface::MSG_ERROR: strCaption = "Error: "; + if (!fSecure) LogError("%s\n", message.original); break; case CClientUIInterface::MSG_WARNING: strCaption = "Warning: "; + if (!fSecure) LogWarning("%s\n", message.original); break; case CClientUIInterface::MSG_INFORMATION: strCaption = "Information: "; + if (!fSecure) LogInfo("%s\n", message.original); break; default: strCaption = caption + ": "; // Use supplied caption (can be empty) + if (!fSecure) LogInfo("%s%s\n", strCaption, message.original); } - if (!fSecure) { - LogPrintf("%s%s\n", strCaption, message.original); - } tfm::format(std::cerr, "%s%s\n", strCaption, message.original); return false; } diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index f0830d8f22..2ad79b6f99 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -19,6 +19,8 @@ #include <limits> #include <vector> +#include <compare> + RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) { AssertLockHeld(pool.cs); @@ -181,3 +183,22 @@ std::optional<std::string> PaysForRBF(CAmount original_fees, } return std::nullopt; } + +std::optional<std::pair<DiagramCheckError, std::string>> ImprovesFeerateDiagram(CTxMemPool& pool, + const CTxMemPool::setEntries& direct_conflicts, + const CTxMemPool::setEntries& all_conflicts, + CAmount replacement_fees, + int64_t replacement_vsize) +{ + // Require that the replacement strictly improves the mempool's feerate diagram. + const auto chunk_results{pool.CalculateChunksForRBF(replacement_fees, replacement_vsize, direct_conflicts, all_conflicts)}; + + if (!chunk_results.has_value()) { + return std::make_pair(DiagramCheckError::UNCALCULABLE, util::ErrorString(chunk_results).original); + } + + if (!std::is_gt(CompareChunks(chunk_results.value().second, chunk_results.value().first))) { + return std::make_pair(DiagramCheckError::FAILURE, "insufficient feerate: does not improve feerate diagram"); + } + return std::nullopt; +} diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 5a33ed64a3..252fbec8e3 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -9,7 +9,9 @@ #include <primitives/transaction.h> #include <threadsafety.h> #include <txmempool.h> +#include <util/feefrac.h> +#include <compare> #include <cstddef> #include <cstdint> #include <optional> @@ -33,6 +35,13 @@ enum class RBFTransactionState { FINAL, }; +enum class DiagramCheckError { + /** Unable to calculate due to topology or other reason */ + UNCALCULABLE, + /** New diagram wasn't strictly superior */ + FAILURE, +}; + /** * Determine whether an unconfirmed transaction is signaling opt-in to RBF * according to BIP 125 @@ -106,4 +115,21 @@ std::optional<std::string> PaysForRBF(CAmount original_fees, CFeeRate relay_fee, const uint256& txid); +/** + * The replacement transaction must improve the feerate diagram of the mempool. + * @param[in] pool The mempool. + * @param[in] direct_conflicts Set of in-mempool txids corresponding to the direct conflicts i.e. + * input double-spends with the proposed transaction + * @param[in] all_conflicts Set of mempool entries corresponding to all transactions to be evicted + * @param[in] replacement_fees Fees of proposed replacement package + * @param[in] replacement_vsize Size of proposed replacement package + * @returns error type and string if mempool diagram doesn't improve, otherwise std::nullopt. + */ +std::optional<std::pair<DiagramCheckError, std::string>> ImprovesFeerateDiagram(CTxMemPool& pool, + const CTxMemPool::setEntries& direct_conflicts, + const CTxMemPool::setEntries& all_conflicts, + CAmount replacement_fees, + int64_t replacement_vsize) + EXCLUSIVE_LOCKS_REQUIRED(pool.cs); + #endif // BITCOIN_POLICY_RBF_H diff --git a/src/policy/v3_policy.cpp b/src/policy/v3_policy.cpp index f838dc6c0f..3c3942d707 100644 --- a/src/policy/v3_policy.cpp +++ b/src/policy/v3_policy.cpp @@ -130,10 +130,11 @@ std::optional<std::string> PackageV3Checks(const CTransactionRef& ptx, int64_t v } // It shouldn't be possible to have any mempool siblings at this point. SingleV3Checks - // catches mempool siblings. Also, if the package consists of connected transactions, + // catches mempool siblings and sibling eviction is not extended to packages. Also, if the package consists of connected transactions, // any tx having a mempool ancestor would mean the package exceeds ancestor limits. if (!Assume(!parent_info.m_has_mempool_descendant)) { - return strprintf("tx %u would exceed descendant count limit", parent_info.m_wtxid.ToString()); + return strprintf("tx %s (wtxid=%s) would exceed descendant count limit", + parent_info.m_txid.ToString(), parent_info.m_wtxid.ToString()); } } } else { @@ -158,7 +159,7 @@ std::optional<std::string> PackageV3Checks(const CTransactionRef& ptx, int64_t v return std::nullopt; } -std::optional<std::string> SingleV3Checks(const CTransactionRef& ptx, +std::optional<std::pair<std::string, CTransactionRef>> SingleV3Checks(const CTransactionRef& ptx, const CTxMemPool::setEntries& mempool_ancestors, const std::set<Txid>& direct_conflicts, int64_t vsize) @@ -166,13 +167,15 @@ std::optional<std::string> SingleV3Checks(const CTransactionRef& ptx, // Check v3 and non-v3 inheritance. for (const auto& entry : mempool_ancestors) { if (ptx->nVersion != 3 && entry->GetTx().nVersion == 3) { - return strprintf("non-v3 tx %s (wtxid=%s) cannot spend from v3 tx %s (wtxid=%s)", + return std::make_pair(strprintf("non-v3 tx %s (wtxid=%s) cannot spend from v3 tx %s (wtxid=%s)", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), - entry->GetSharedTx()->GetHash().ToString(), entry->GetSharedTx()->GetWitnessHash().ToString()); + entry->GetSharedTx()->GetHash().ToString(), entry->GetSharedTx()->GetWitnessHash().ToString()), + nullptr); } else if (ptx->nVersion == 3 && entry->GetTx().nVersion != 3) { - return strprintf("v3 tx %s (wtxid=%s) cannot spend from non-v3 tx %s (wtxid=%s)", + return std::make_pair(strprintf("v3 tx %s (wtxid=%s) cannot spend from non-v3 tx %s (wtxid=%s)", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), - entry->GetSharedTx()->GetHash().ToString(), entry->GetSharedTx()->GetWitnessHash().ToString()); + entry->GetSharedTx()->GetHash().ToString(), entry->GetSharedTx()->GetWitnessHash().ToString()), + nullptr); } } @@ -185,16 +188,18 @@ std::optional<std::string> SingleV3Checks(const CTransactionRef& ptx, // Check that V3_ANCESTOR_LIMIT would not be violated. if (mempool_ancestors.size() + 1 > V3_ANCESTOR_LIMIT) { - return strprintf("tx %s (wtxid=%s) would have too many ancestors", - ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString()); + return std::make_pair(strprintf("tx %s (wtxid=%s) would have too many ancestors", + ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString()), + nullptr); } // Remaining checks only pertain to transactions with unconfirmed ancestors. if (mempool_ancestors.size() > 0) { // If this transaction spends V3 parents, it cannot be too large. if (vsize > V3_CHILD_MAX_VSIZE) { - return strprintf("v3 child tx %s (wtxid=%s) is too big: %u > %u virtual bytes", - ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), vsize, V3_CHILD_MAX_VSIZE); + return std::make_pair(strprintf("v3 child tx %s (wtxid=%s) is too big: %u > %u virtual bytes", + ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), vsize, V3_CHILD_MAX_VSIZE), + nullptr); } // Check the descendant counts of in-mempool ancestors. @@ -210,9 +215,20 @@ std::optional<std::string> SingleV3Checks(const CTransactionRef& ptx, std::any_of(children.cbegin(), children.cend(), [&direct_conflicts](const CTxMemPoolEntry& child){return direct_conflicts.count(child.GetTx().GetHash()) > 0;}); if (parent_entry->GetCountWithDescendants() + 1 > V3_DESCENDANT_LIMIT && !child_will_be_replaced) { - return strprintf("tx %u (wtxid=%s) would exceed descendant count limit", - parent_entry->GetSharedTx()->GetHash().ToString(), - parent_entry->GetSharedTx()->GetWitnessHash().ToString()); + // Allow sibling eviction for v3 transaction: if another child already exists, even if + // we don't conflict inputs with it, consider evicting it under RBF rules. We rely on v3 rules + // only permitting 1 descendant, as otherwise we would need to have logic for deciding + // which descendant to evict. Skip if this isn't true, e.g. if the transaction has + // multiple children or the sibling also has descendants due to a reorg. + const bool consider_sibling_eviction{parent_entry->GetCountWithDescendants() == 2 && + children.begin()->get().GetCountWithAncestors() == 2}; + + // Return the sibling if its eviction can be considered. Provide the "descendant count + // limit" string either way, as the caller may decide not to do sibling eviction. + return std::make_pair(strprintf("tx %u (wtxid=%s) would exceed descendant count limit", + parent_entry->GetSharedTx()->GetHash().ToString(), + parent_entry->GetSharedTx()->GetWitnessHash().ToString()), + consider_sibling_eviction ? children.begin()->get().GetSharedTx() : nullptr); } } return std::nullopt; diff --git a/src/policy/v3_policy.h b/src/policy/v3_policy.h index 9e871915e5..2e56f8822b 100644 --- a/src/policy/v3_policy.h +++ b/src/policy/v3_policy.h @@ -48,9 +48,15 @@ static_assert(V3_CHILD_MAX_VSIZE + MAX_STANDARD_TX_WEIGHT / WITNESS_SCALE_FACTOR * count of in-mempool ancestors. * @param[in] vsize The sigop-adjusted virtual size of ptx. * - * @returns debug string if an error occurs, std::nullopt otherwise. + * @returns 3 possibilities: + * - std::nullopt if all v3 checks were applied successfully + * - debug string + pointer to a mempool sibling if this transaction would be the second child in a + * 1-parent-1-child cluster; the caller may consider evicting the specified sibling or return an + * error with the debug string. + * - debug string + nullptr if this transaction violates some v3 rule and sibling eviction is not + * applicable. */ -std::optional<std::string> SingleV3Checks(const CTransactionRef& ptx, +std::optional<std::pair<std::string, CTransactionRef>> SingleV3Checks(const CTransactionRef& ptx, const CTxMemPool::setEntries& mempool_ancestors, const std::set<Txid>& direct_conflicts, int64_t vsize); diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 084b9a6615..1d1a84e375 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -109,22 +109,26 @@ QFont fixedPitchFont(bool use_embedded_font) return QFontDatabase::systemFont(QFontDatabase::FixedFont); } -// Just some dummy data to generate a convincing random-looking (but consistent) address -static const uint8_t dummydata[] = {0xeb,0x15,0x23,0x1d,0xfc,0xeb,0x60,0x92,0x58,0x86,0xb6,0x7d,0x06,0x52,0x99,0x92,0x59,0x15,0xae,0xb1,0x72,0xc0,0x66,0x47}; - -// Generate a dummy address with invalid CRC, starting with the network prefix. +// Return a pre-generated dummy bech32m address (P2TR) with invalid checksum. static std::string DummyAddress(const CChainParams ¶ms) { - std::vector<unsigned char> sourcedata = params.Base58Prefix(CChainParams::PUBKEY_ADDRESS); - sourcedata.insert(sourcedata.end(), dummydata, dummydata + sizeof(dummydata)); - for(int i=0; i<256; ++i) { // Try every trailing byte - std::string s = EncodeBase58(sourcedata); - if (!IsValidDestinationString(s)) { - return s; - } - sourcedata[sourcedata.size()-1] += 1; - } - return ""; + std::string addr; + switch (params.GetChainType()) { + case ChainType::MAIN: + addr = "bc1p35yvjel7srp783ztf8v6jdra7dhfzk5jaun8xz2qp6ws7z80n4tq2jku9f"; + break; + case ChainType::SIGNET: + case ChainType::TESTNET: + addr = "tb1p35yvjel7srp783ztf8v6jdra7dhfzk5jaun8xz2qp6ws7z80n4tqa6qnlg"; + break; + case ChainType::REGTEST: + addr = "bcrt1p35yvjel7srp783ztf8v6jdra7dhfzk5jaun8xz2qp6ws7z80n4tqsr2427"; + break; + } // no default case, so the compiler can warn about missing cases + assert(!addr.empty()); + + if (Assume(!IsValidDestinationString(addr))) return addr; + return {}; } void setupAddressWidget(QValidatedLineEdit *widget, QWidget *parent) diff --git a/src/qt/main.cpp b/src/qt/main.cpp index c84dd78b44..16befd99e8 100644 --- a/src/qt/main.cpp +++ b/src/qt/main.cpp @@ -4,7 +4,6 @@ #include <qt/bitcoin.h> -#include <common/url.h> #include <compat/compat.h> #include <util/translation.h> @@ -17,7 +16,8 @@ extern const std::function<std::string(const char*)> G_TRANSLATION_FUN = [](const char* psz) { return QCoreApplication::translate("bitcoin-core", psz).toStdString(); }; -UrlDecodeFn* const URL_DECODE = urlDecode; + +const std::function<std::string()> G_TEST_GET_FULL_NAME{}; MAIN_FUNCTION { diff --git a/src/qt/notificator.cpp b/src/qt/notificator.cpp index 2021e5f9dc..551c0ffd13 100644 --- a/src/qt/notificator.cpp +++ b/src/qt/notificator.cpp @@ -112,10 +112,10 @@ FreedesktopImage::FreedesktopImage(const QImage &img): for(unsigned int ptr = 0; ptr < num_pixels; ++ptr) { - image[ptr*BYTES_PER_PIXEL+0] = data[ptr] >> 16; // R - image[ptr*BYTES_PER_PIXEL+1] = data[ptr] >> 8; // G - image[ptr*BYTES_PER_PIXEL+2] = data[ptr]; // B - image[ptr*BYTES_PER_PIXEL+3] = data[ptr] >> 24; // A + image[ptr * BYTES_PER_PIXEL + 0] = char(data[ptr] >> 16); // R + image[ptr * BYTES_PER_PIXEL + 1] = char(data[ptr] >> 8); // G + image[ptr * BYTES_PER_PIXEL + 2] = char(data[ptr]); // B + image[ptr * BYTES_PER_PIXEL + 3] = char(data[ptr] >> 24); // A } } diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index a87bef796c..dd654a7abe 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -454,20 +454,24 @@ void OptionsDialog::updateProxyValidationState() void OptionsDialog::updateDefaultProxyNets() { - const std::optional<CNetAddr> ui_proxy_netaddr{LookupHost(ui->proxyIp->text().toStdString(), /*fAllowLookup=*/false)}; - const CService ui_proxy{ui_proxy_netaddr.value_or(CNetAddr{}), ui->proxyPort->text().toUShort()}; + std::string proxyIpText{ui->proxyIp->text().toStdString()}; + if (!IsUnixSocketPath(proxyIpText)) { + const std::optional<CNetAddr> ui_proxy_netaddr{LookupHost(proxyIpText, /*fAllowLookup=*/false)}; + const CService ui_proxy{ui_proxy_netaddr.value_or(CNetAddr{}), ui->proxyPort->text().toUShort()}; + proxyIpText = ui_proxy.ToStringAddrPort(); + } Proxy proxy; bool has_proxy; has_proxy = model->node().getProxy(NET_IPV4, proxy); - ui->proxyReachIPv4->setChecked(has_proxy && proxy.proxy == ui_proxy); + ui->proxyReachIPv4->setChecked(has_proxy && proxy.ToString() == proxyIpText); has_proxy = model->node().getProxy(NET_IPV6, proxy); - ui->proxyReachIPv6->setChecked(has_proxy && proxy.proxy == ui_proxy); + ui->proxyReachIPv6->setChecked(has_proxy && proxy.ToString() == proxyIpText); has_proxy = model->node().getProxy(NET_ONION, proxy); - ui->proxyReachTor->setChecked(has_proxy && proxy.proxy == ui_proxy); + ui->proxyReachTor->setChecked(has_proxy && proxy.ToString() == proxyIpText); } ProxyAddressValidator::ProxyAddressValidator(QObject *parent) : diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index d816a72ca3..d0f7c64357 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -396,6 +396,7 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in return successful; } +// NOLINTNEXTLINE(misc-no-recursion) QVariant OptionsModel::getOption(OptionID option, const std::string& suffix) const { auto setting = [&]{ return node().getPersistentSetting(SettingName(option) + suffix); }; @@ -508,6 +509,7 @@ QFont OptionsModel::getFontForMoney() const return getFontForChoice(m_font_money); } +// NOLINTNEXTLINE(misc-no-recursion) bool OptionsModel::setOption(OptionID option, const QVariant& value, const std::string& suffix) { auto changed = [&] { return value.isValid() && value != getOption(option, suffix); }; diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 8decc27bd7..c5405cca98 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -50,6 +50,8 @@ const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS{}; +const std::function<std::string()> G_TEST_GET_FULL_NAME{}; + // This is all you need to run all the tests int main(int argc, char* argv[]) { diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 1bdf94d3b5..fe000bcbb8 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -569,16 +569,17 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) return true; } -bool WalletModel::displayAddress(std::string sAddress) const +void WalletModel::displayAddress(std::string sAddress) const { CTxDestination dest = DecodeDestination(sAddress); - bool res = false; try { - res = m_wallet->displayAddress(dest); + util::Result<void> result = m_wallet->displayAddress(dest); + if (!result) { + QMessageBox::warning(nullptr, tr("Signer error"), QString::fromStdString(util::ErrorString(result).translated)); + } } catch (const std::runtime_error& e) { QMessageBox::critical(nullptr, tr("Can't display address"), e.what()); } - return res; } bool WalletModel::isWalletEnabled() diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 503ee16823..ab2096c1fe 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -130,7 +130,7 @@ public: UnlockContext requestUnlock(); bool bumpFee(uint256 hash, uint256& new_hash); - bool displayAddress(std::string sAddress) const; + void displayAddress(std::string sAddress) const; static bool isWalletEnabled(); diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index 09e0771534..4926d1e80b 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -134,7 +134,7 @@ void WalletView::processNewTransaction(const QModelIndex& parent, int start, int return; QString date = ttm->index(start, TransactionTableModel::Date, parent).data().toString(); - qint64 amount = ttm->index(start, TransactionTableModel::Amount, parent).data(Qt::EditRole).toULongLong(); + qint64 amount = ttm->index(start, TransactionTableModel::Amount, parent).data(Qt::EditRole).toLongLong(); QString type = ttm->index(start, TransactionTableModel::Type, parent).data().toString(); QModelIndex index = ttm->index(start, 0, parent); QString address = ttm->data(index, TransactionTableModel::AddressRole).toString(); diff --git a/src/randomenv.cpp b/src/randomenv.cpp index da81a61651..123b5cc06c 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -13,6 +13,7 @@ #include <compat/compat.h> #include <compat/cpuid.h> #include <crypto/sha512.h> +#include <span.h> #include <support/cleanse.h> #include <util/time.h> @@ -357,10 +358,19 @@ void RandAddStaticEnv(CSHA512& hasher) hasher << &hasher << &RandAddStaticEnv << &malloc << &errno << &environ; // Hostname +#ifdef WIN32 + constexpr DWORD max_size = MAX_COMPUTERNAME_LENGTH + 1; + char hname[max_size]; + DWORD size = max_size; + if (GetComputerNameA(hname, &size) != 0) { + hasher.Write(UCharCast(hname), size); + } +#else char hname[256]; if (gethostname(hname, 256) == 0) { hasher.Write((const unsigned char*)hname, strnlen(hname, 256)); } +#endif #if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS // Network interfaces diff --git a/src/rest.cpp b/src/rest.cpp index 91184745c8..89c033b8a3 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -13,6 +13,7 @@ #include <chain.h> #include <chainparams.h> #include <core_io.h> +#include <flatfile.h> #include <httpserver.h> #include <index/blockfilterindex.h> #include <index/txindex.h> @@ -34,7 +35,7 @@ #include <validation.h> #include <any> -#include <string> +#include <vector> #include <univalue.h> @@ -295,7 +296,7 @@ static bool rest_block(const std::any& context, if (!ParseHashStr(hashStr, hash)) return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); - CBlock block; + FlatFilePos pos{}; const CBlockIndex* pblockindex = nullptr; const CBlockIndex* tip = nullptr; ChainstateManager* maybe_chainman = GetChainman(context, req); @@ -311,32 +312,33 @@ static bool rest_block(const std::any& context, if (chainman.m_blockman.IsBlockPruned(*pblockindex)) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); } + pos = pblockindex->GetBlockPos(); } - if (!chainman.m_blockman.ReadBlockFromDisk(block, *pblockindex)) { + std::vector<uint8_t> block_data{}; + if (!chainman.m_blockman.ReadRawBlockFromDisk(block_data, pos)) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } switch (rf) { case RESTResponseFormat::BINARY: { - DataStream ssBlock; - ssBlock << TX_WITH_WITNESS(block); - std::string binaryBlock = ssBlock.str(); + const std::string binaryBlock{block_data.begin(), block_data.end()}; req->WriteHeader("Content-Type", "application/octet-stream"); req->WriteReply(HTTP_OK, binaryBlock); return true; } case RESTResponseFormat::HEX: { - DataStream ssBlock; - ssBlock << TX_WITH_WITNESS(block); - std::string strHex = HexStr(ssBlock) + "\n"; + const std::string strHex{HexStr(block_data) + "\n"}; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; } case RESTResponseFormat::JSON: { + CBlock block{}; + DataStream block_stream{block_data}; + block_stream >> TX_WITH_WITNESS(block); UniValue objBlock = blockToJSON(chainman.m_blockman, block, *tip, *pblockindex, tx_verbosity); std::string strJSON = objBlock.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index dfdddeacea..a1135c27d4 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -17,6 +17,7 @@ #include <core_io.h> #include <deploymentinfo.h> #include <deploymentstatus.h> +#include <flatfile.h> #include <hash.h> #include <index/blockfilterindex.h> #include <index/coinstatsindex.h> @@ -595,6 +596,28 @@ static CBlock GetBlockChecked(BlockManager& blockman, const CBlockIndex& blockin return block; } +static std::vector<uint8_t> GetRawBlockChecked(BlockManager& blockman, const CBlockIndex& blockindex) +{ + std::vector<uint8_t> data{}; + FlatFilePos pos{}; + { + LOCK(cs_main); + if (blockman.IsBlockPruned(blockindex)) { + throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); + } + pos = blockindex.GetBlockPos(); + } + + if (!blockman.ReadRawBlockFromDisk(data, pos)) { + // Block not found on disk. This could be because we have the block + // header in our index but not yet have the block or did not accept the + // block. Or if the block was pruned right after we released the lock above. + throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); + } + + return data; +} + static CBlockUndo GetUndoChecked(BlockManager& blockman, const CBlockIndex& blockindex) { CBlockUndo blockUndo; @@ -735,15 +758,16 @@ static RPCHelpMan getblock() } } - const CBlock block{GetBlockChecked(chainman.m_blockman, *pblockindex)}; + const std::vector<uint8_t> block_data{GetRawBlockChecked(chainman.m_blockman, *pblockindex)}; if (verbosity <= 0) { - DataStream ssBlock; - ssBlock << TX_WITH_WITNESS(block); - std::string strHex = HexStr(ssBlock); - return strHex; + return HexStr(block_data); } + DataStream block_stream{block_data}; + CBlock block{}; + block_stream >> TX_WITH_WITNESS(block); + TxVerbosity tx_verbosity; if (verbosity == 1) { tx_verbosity = TxVerbosity::SHOW_TXID; diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 5825efdf82..b8dc148eae 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -128,6 +128,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "testmempoolaccept", 0, "rawtxs" }, { "testmempoolaccept", 1, "maxfeerate" }, { "submitpackage", 0, "package" }, + { "submitpackage", 1, "maxfeerate" }, + { "submitpackage", 2, "maxburnamount" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, { "fundrawtransaction", 1, "add_inputs"}, @@ -275,6 +277,11 @@ static const CRPCConvertParam vRPCConvertParams[] = { "logging", 1, "exclude" }, { "disconnectnode", 1, "nodeid" }, { "upgradewallet", 0, "version" }, + { "gethdkeys", 0, "active_only" }, + { "gethdkeys", 0, "options" }, + { "gethdkeys", 0, "private" }, + { "createwalletdescriptor", 1, "options" }, + { "createwalletdescriptor", 1, "internal" }, // Echo with conversion (For testing only) { "echojson", 0, "arg0" }, { "echojson", 1, "arg1" }, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 25bfec2d45..920bb9ea7f 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -28,6 +28,7 @@ using kernel::DumpMempool; +using node::DEFAULT_MAX_BURN_AMOUNT; using node::DEFAULT_MAX_RAW_TX_FEE_RATE; using node::MempoolPath; using node::NodeContext; @@ -46,7 +47,7 @@ static RPCHelpMan sendrawtransaction() {"maxfeerate", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK())}, "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + "/kvB.\nFee rates larger than 1BTC/kvB are rejected.\nSet to 0 to accept any fee rate."}, - {"maxburnamount", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(0)}, + {"maxburnamount", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(DEFAULT_MAX_BURN_AMOUNT)}, "Reject transactions with provably unspendable outputs (e.g. 'datacarrier' outputs that use the OP_RETURN opcode) greater than the specified value, expressed in " + CURRENCY_UNIT + ".\n" "If burning funds through unspendable outputs is desired, increase this value.\n" "This check is based on heuristics and does not guarantee spendability of outputs.\n"}, @@ -180,7 +181,7 @@ static RPCHelpMan testmempoolaccept() Chainstate& chainstate = chainman.ActiveChainstate(); const PackageMempoolAcceptResult package_result = [&] { LOCK(::cs_main); - if (txns.size() > 1) return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/true); + if (txns.size() > 1) return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/true, /*client_maxfeerate=*/{}); return PackageMempoolAcceptResult(txns[0]->GetWitnessHash(), chainman.ProcessTransaction(txns[0], /*test_accept=*/true)); }(); @@ -823,6 +824,14 @@ static RPCHelpMan submitpackage() {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, }, }, + {"maxfeerate", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK())}, + "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + + "/kvB.\nFee rates larger than 1BTC/kvB are rejected.\nSet to 0 to accept any fee rate."}, + {"maxburnamount", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(DEFAULT_MAX_BURN_AMOUNT)}, + "Reject transactions with provably unspendable outputs (e.g. 'datacarrier' outputs that use the OP_RETURN opcode) greater than the specified value, expressed in " + CURRENCY_UNIT + ".\n" + "If burning funds through unspendable outputs is desired, increase this value.\n" + "This check is based on heuristics and does not guarantee spendability of outputs.\n" + }, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -862,6 +871,17 @@ static RPCHelpMan submitpackage() "Array must contain between 1 and " + ToString(MAX_PACKAGE_COUNT) + " transactions."); } + // Fee check needs to be run with chainstate and package context + const CFeeRate max_raw_tx_fee_rate = ParseFeeRate(self.Arg<UniValue>(1)); + std::optional<CFeeRate> client_maxfeerate{max_raw_tx_fee_rate}; + // 0-value is special; it's mapped to no sanity check + if (max_raw_tx_fee_rate == CFeeRate(0)) { + client_maxfeerate = std::nullopt; + } + + // Burn sanity check is run with no context + const CAmount max_burn_amount = request.params[2].isNull() ? 0 : AmountFromValue(request.params[2]); + std::vector<CTransactionRef> txns; txns.reserve(raw_transactions.size()); for (const auto& rawtx : raw_transactions.getValues()) { @@ -870,6 +890,13 @@ static RPCHelpMan submitpackage() throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed: " + rawtx.get_str() + " Make sure the tx has at least one input."); } + + for (const auto& out : mtx.vout) { + if((out.scriptPubKey.IsUnspendable() || !out.scriptPubKey.HasValidOps()) && out.nValue > max_burn_amount) { + throw JSONRPCTransactionError(TransactionError::MAX_BURN_EXCEEDED); + } + } + txns.emplace_back(MakeTransactionRef(std::move(mtx))); } if (!IsChildWithParentsTree(txns)) { @@ -879,7 +906,7 @@ static RPCHelpMan submitpackage() NodeContext& node = EnsureAnyNodeContext(request.context); CTxMemPool& mempool = EnsureMemPool(node); Chainstate& chainstate = EnsureChainman(node).ActiveChainstate(); - const auto package_result = WITH_LOCK(::cs_main, return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/ false)); + const auto package_result = WITH_LOCK(::cs_main, return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/ false, client_maxfeerate)); std::string package_msg = "success"; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 5e6f42b596..f935a3b08f 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -607,8 +607,8 @@ static UniValue GetNetworksInfo() obj.pushKV("name", GetNetworkName(network)); obj.pushKV("limited", !g_reachable_nets.Contains(network)); obj.pushKV("reachable", g_reachable_nets.Contains(network)); - obj.pushKV("proxy", proxy.IsValid() ? proxy.proxy.ToStringAddrPort() : std::string()); - obj.pushKV("proxy_randomize_credentials", proxy.randomize_credentials); + obj.pushKV("proxy", proxy.IsValid() ? proxy.ToString() : std::string()); + obj.pushKV("proxy_randomize_credentials", proxy.m_randomize_credentials); networks.push_back(obj); } return networks; @@ -951,7 +951,7 @@ static RPCHelpMan getnodeaddresses() static RPCHelpMan addpeeraddress() { return RPCHelpMan{"addpeeraddress", - "\nAdd the address of a potential peer to the address manager. This RPC is for testing only.\n", + "Add the address of a potential peer to an address manager table. This RPC is for testing only.", { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The IP address of the peer"}, {"port", RPCArg::Type::NUM, RPCArg::Optional::NO, "The port of the peer"}, @@ -960,7 +960,8 @@ static RPCHelpMan addpeeraddress() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "success", "whether the peer address was successfully added to the address manager"}, + {RPCResult::Type::BOOL, "success", "whether the peer address was successfully added to the address manager table"}, + {RPCResult::Type::STR, "error", /*optional=*/true, "error description, if the address could not be added"}, }, }, RPCExamples{ @@ -989,8 +990,13 @@ static RPCHelpMan addpeeraddress() success = true; if (tried) { // Attempt to move the address to the tried addresses table. - addrman.Good(address); + if (!addrman.Good(address)) { + success = false; + obj.pushKV("error", "failed-adding-to-tried"); + } } + } else { + obj.pushKV("error", "failed-adding-to-new"); } } diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index b8c0080aef..ffc2ee5ab0 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -26,6 +26,7 @@ #include <univalue.h> #include <util/any.h> #include <util/check.h> +#include <util/time.h> #include <stdint.h> #ifdef HAVE_MALLOC_INFO @@ -58,9 +59,11 @@ static RPCHelpMan setmocktime() LOCK(cs_main); const int64_t time{request.params[0].getInt<int64_t>()}; - if (time < 0) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Mocktime cannot be negative: %s.", time)); + constexpr int64_t max_time{Ticks<std::chrono::seconds>(std::chrono::nanoseconds::max())}; + if (time < 0 || time > max_time) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Mocktime must be in the range [0, %s], not %s.", max_time, time)); } + SetMockTime(time); const NodeContext& node_context{EnsureAnyNodeContext(request.context)}; for (const auto& chain_client : node_context.chain_clients) { diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 51c88cc1ba..6e332e3855 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -414,6 +414,7 @@ struct Sections { /** * Recursive helper to translate an RPCArg into sections */ + // NOLINTNEXTLINE(misc-no-recursion) void Push(const RPCArg& arg, const size_t current_indent = 5, const OuterType outer_type = OuterType::NONE) { const auto indent = std::string(current_indent, ' '); @@ -953,6 +954,7 @@ std::string RPCArg::ToDescriptionString(bool is_named_arg) const return ret; } +// NOLINTNEXTLINE(misc-no-recursion) void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const int current_indent) const { // Indentation @@ -1086,6 +1088,7 @@ static std::optional<UniValue::VType> ExpectedType(RPCResult::Type type) NONFATAL_UNREACHABLE(); } +// NOLINTNEXTLINE(misc-no-recursion) UniValue RPCResult::MatchesType(const UniValue& result) const { if (m_skip_type_check) { @@ -1164,6 +1167,7 @@ void RPCResult::CheckInnerDoc() const CHECK_NONFATAL(inner_needed != m_inner.empty()); } +// NOLINTNEXTLINE(misc-no-recursion) std::string RPCArg::ToStringObj(const bool oneline) const { std::string res; @@ -1202,6 +1206,7 @@ std::string RPCArg::ToStringObj(const bool oneline) const NONFATAL_UNREACHABLE(); } +// NOLINTNEXTLINE(misc-no-recursion) std::string RPCArg::ToString(const bool oneline) const { if (oneline && !m_opts.oneline_description.empty()) { @@ -1228,6 +1233,7 @@ std::string RPCArg::ToString(const bool oneline) const case Type::OBJ: case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: { + // NOLINTNEXTLINE(misc-no-recursion) const std::string res = Join(m_inner, ",", [&](const RPCArg& i) { return i.ToStringObj(oneline); }); if (m_type == Type::OBJ) { return "{" + res + "}"; diff --git a/src/rpc/util.h b/src/rpc/util.h index ad3ed97b2e..f6ee6a317a 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -162,6 +162,7 @@ struct RPCArgOptions { //!< methods set the also_positional flag and read values from both positions. }; +// NOLINTNEXTLINE(misc-no-recursion) struct RPCArg { enum class Type { OBJ, @@ -271,6 +272,7 @@ struct RPCArg { std::string ToDescriptionString(bool is_named_arg) const; }; +// NOLINTNEXTLINE(misc-no-recursion) struct RPCResult { enum class Type { OBJ, diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp deleted file mode 100644 index c4eccacf41..0000000000 --- a/src/script/bitcoinconsensus.cpp +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <script/bitcoinconsensus.h> - -#include <primitives/transaction.h> -#include <pubkey.h> -#include <script/interpreter.h> - -namespace { - -/** A class that deserializes a single CTransaction one time. */ -class TxInputStream -{ -public: - TxInputStream(const unsigned char *txTo, size_t txToLen) : - m_data(txTo), - m_remaining(txToLen) - {} - - void read(Span<std::byte> dst) - { - if (dst.size() > m_remaining) { - throw std::ios_base::failure(std::string(__func__) + ": end of data"); - } - - if (dst.data() == nullptr) { - throw std::ios_base::failure(std::string(__func__) + ": bad destination buffer"); - } - - if (m_data == nullptr) { - throw std::ios_base::failure(std::string(__func__) + ": bad source buffer"); - } - - memcpy(dst.data(), m_data, dst.size()); - m_remaining -= dst.size(); - m_data += dst.size(); - } - - template<typename T> - TxInputStream& operator>>(T&& obj) - { - ::Unserialize(*this, obj); - return *this; - } - -private: - const unsigned char* m_data; - size_t m_remaining; -}; - -inline int set_error(bitcoinconsensus_error* ret, bitcoinconsensus_error serror) -{ - if (ret) - *ret = serror; - return 0; -} - -} // namespace - -/** Check that all specified flags are part of the libconsensus interface. */ -static bool verify_flags(unsigned int flags) -{ - return (flags & ~(bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL)) == 0; -} - -static int verify_script(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, CAmount amount, - const unsigned char *txTo , unsigned int txToLen, - const UTXO *spentOutputs, unsigned int spentOutputsLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err) -{ - if (!verify_flags(flags)) { - return set_error(err, bitcoinconsensus_ERR_INVALID_FLAGS); - } - - if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT && spentOutputs == nullptr) { - return set_error(err, bitcoinconsensus_ERR_SPENT_OUTPUTS_REQUIRED); - } - - try { - TxInputStream stream(txTo, txToLen); - CTransaction tx(deserialize, TX_WITH_WITNESS, stream); - - std::vector<CTxOut> spent_outputs; - if (spentOutputs != nullptr) { - if (spentOutputsLen != tx.vin.size()) { - return set_error(err, bitcoinconsensus_ERR_SPENT_OUTPUTS_MISMATCH); - } - for (size_t i = 0; i < spentOutputsLen; i++) { - CScript spk = CScript(spentOutputs[i].scriptPubKey, spentOutputs[i].scriptPubKey + spentOutputs[i].scriptPubKeySize); - const CAmount& value = spentOutputs[i].value; - CTxOut tx_out = CTxOut(value, spk); - spent_outputs.push_back(tx_out); - } - } - - if (nIn >= tx.vin.size()) - return set_error(err, bitcoinconsensus_ERR_TX_INDEX); - if (GetSerializeSize(TX_WITH_WITNESS(tx)) != txToLen) - return set_error(err, bitcoinconsensus_ERR_TX_SIZE_MISMATCH); - - // Regardless of the verification result, the tx did not error. - set_error(err, bitcoinconsensus_ERR_OK); - - PrecomputedTransactionData txdata(tx); - - if (spentOutputs != nullptr && flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT) { - txdata.Init(tx, std::move(spent_outputs)); - } - - return VerifyScript(tx.vin[nIn].scriptSig, CScript(scriptPubKey, scriptPubKey + scriptPubKeyLen), &tx.vin[nIn].scriptWitness, flags, TransactionSignatureChecker(&tx, nIn, amount, txdata, MissingDataBehavior::FAIL), nullptr); - } catch (const std::exception&) { - return set_error(err, bitcoinconsensus_ERR_TX_DESERIALIZE); // Error deserializing - } -} - -int bitcoinconsensus_verify_script_with_spent_outputs(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, int64_t amount, - const unsigned char *txTo , unsigned int txToLen, - const UTXO *spentOutputs, unsigned int spentOutputsLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err) -{ - CAmount am(amount); - return ::verify_script(scriptPubKey, scriptPubKeyLen, am, txTo, txToLen, spentOutputs, spentOutputsLen, nIn, flags, err); -} - -int bitcoinconsensus_verify_script_with_amount(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, int64_t amount, - const unsigned char *txTo , unsigned int txToLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err) -{ - CAmount am(amount); - UTXO *spentOutputs = nullptr; - unsigned int spentOutputsLen = 0; - return ::verify_script(scriptPubKey, scriptPubKeyLen, am, txTo, txToLen, spentOutputs, spentOutputsLen, nIn, flags, err); -} - - -int bitcoinconsensus_verify_script(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, - const unsigned char *txTo , unsigned int txToLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err) -{ - if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS) { - return set_error(err, bitcoinconsensus_ERR_AMOUNT_REQUIRED); - } - - CAmount am(0); - UTXO *spentOutputs = nullptr; - unsigned int spentOutputsLen = 0; - return ::verify_script(scriptPubKey, scriptPubKeyLen, am, txTo, txToLen, spentOutputs, spentOutputsLen, nIn, flags, err); -} - -unsigned int bitcoinconsensus_version() -{ - // Just use the API version for now - return BITCOINCONSENSUS_API_VER; -} diff --git a/src/script/bitcoinconsensus.h b/src/script/bitcoinconsensus.h deleted file mode 100644 index a202b5ba06..0000000000 --- a/src/script/bitcoinconsensus.h +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_SCRIPT_BITCOINCONSENSUS_H -#define BITCOIN_SCRIPT_BITCOINCONSENSUS_H - -#include <stdint.h> - -#if defined(BUILD_BITCOIN_INTERNAL) && defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> - #if defined(_WIN32) - #if defined(HAVE_DLLEXPORT_ATTRIBUTE) - #define EXPORT_SYMBOL __declspec(dllexport) - #else - #define EXPORT_SYMBOL - #endif - #elif defined(HAVE_DEFAULT_VISIBILITY_ATTRIBUTE) - #define EXPORT_SYMBOL __attribute__ ((visibility ("default"))) - #endif -#elif defined(MSC_VER) && !defined(STATIC_LIBBITCOINCONSENSUS) - #define EXPORT_SYMBOL __declspec(dllimport) -#endif - -#ifndef EXPORT_SYMBOL - #define EXPORT_SYMBOL -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#define BITCOINCONSENSUS_API_VER 2 - -typedef enum bitcoinconsensus_error_t -{ - bitcoinconsensus_ERR_OK = 0, - bitcoinconsensus_ERR_TX_INDEX, - bitcoinconsensus_ERR_TX_SIZE_MISMATCH, - bitcoinconsensus_ERR_TX_DESERIALIZE, - bitcoinconsensus_ERR_AMOUNT_REQUIRED, - bitcoinconsensus_ERR_INVALID_FLAGS, - bitcoinconsensus_ERR_SPENT_OUTPUTS_REQUIRED, - bitcoinconsensus_ERR_SPENT_OUTPUTS_MISMATCH -} bitcoinconsensus_error; - -/** Script verification flags */ -enum -{ - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NONE = 0, - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_P2SH = (1U << 0), // evaluate P2SH (BIP16) subscripts - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_DERSIG = (1U << 2), // enforce strict DER (BIP66) compliance - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NULLDUMMY = (1U << 4), // enforce NULLDUMMY (BIP147) - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY = (1U << 9), // enable CHECKLOCKTIMEVERIFY (BIP65) - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKSEQUENCEVERIFY = (1U << 10), // enable CHECKSEQUENCEVERIFY (BIP112) - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS = (1U << 11), // enable WITNESS (BIP141) - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT = (1U << 17), // enable TAPROOT (BIPs 341 & 342) - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL = bitcoinconsensus_SCRIPT_FLAGS_VERIFY_P2SH | bitcoinconsensus_SCRIPT_FLAGS_VERIFY_DERSIG | - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NULLDUMMY | bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY | - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKSEQUENCEVERIFY | bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS | - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT -}; - -typedef struct { - const unsigned char *scriptPubKey; - unsigned int scriptPubKeySize; - int64_t value; -} UTXO; - -/// Returns 1 if the input nIn of the serialized transaction pointed to by -/// txTo correctly spends the scriptPubKey pointed to by scriptPubKey under -/// the additional constraints specified by flags. -/// If not nullptr, err will contain an error/success code for the operation -EXPORT_SYMBOL int bitcoinconsensus_verify_script(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, - const unsigned char *txTo , unsigned int txToLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err); - -EXPORT_SYMBOL int bitcoinconsensus_verify_script_with_amount(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, int64_t amount, - const unsigned char *txTo , unsigned int txToLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err); - -EXPORT_SYMBOL int bitcoinconsensus_verify_script_with_spent_outputs(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, int64_t amount, - const unsigned char *txTo , unsigned int txToLen, - const UTXO *spentOutputs, unsigned int spentOutputsLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err); - -EXPORT_SYMBOL unsigned int bitcoinconsensus_version(); - -#ifdef __cplusplus -} // extern "C" -#endif - -#undef EXPORT_SYMBOL - -#endif // BITCOIN_SCRIPT_BITCOINCONSENSUS_H diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index c6bc5f8f1d..a11d4dcbd5 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -212,6 +212,11 @@ public: /** Derive a private key, if private data is available in arg. */ virtual bool GetPrivKey(int pos, const SigningProvider& arg, CKey& key) const = 0; + + /** Return the non-extended public key for this PubkeyProvider, if it has one. */ + virtual std::optional<CPubKey> GetRootPubKey() const = 0; + /** Return the extended public key for this PubkeyProvider, if it has one. */ + virtual std::optional<CExtPubKey> GetRootExtPubKey() const = 0; }; class OriginPubkeyProvider final : public PubkeyProvider @@ -265,6 +270,14 @@ public: { return m_provider->GetPrivKey(pos, arg, key); } + std::optional<CPubKey> GetRootPubKey() const override + { + return m_provider->GetRootPubKey(); + } + std::optional<CExtPubKey> GetRootExtPubKey() const override + { + return m_provider->GetRootExtPubKey(); + } }; /** An object representing a parsed constant public key in a descriptor. */ @@ -310,6 +323,14 @@ public: { return arg.GetKey(m_pubkey.GetID(), key); } + std::optional<CPubKey> GetRootPubKey() const override + { + return m_pubkey; + } + std::optional<CExtPubKey> GetRootExtPubKey() const override + { + return std::nullopt; + } }; enum class DeriveType { @@ -525,6 +546,14 @@ public: key = extkey.key; return true; } + std::optional<CPubKey> GetRootPubKey() const override + { + return std::nullopt; + } + std::optional<CExtPubKey> GetRootExtPubKey() const override + { + return m_root_extkey; + } }; /** Base class for all Descriptor implementations. */ @@ -570,6 +599,7 @@ public: COMPAT, // string calculation that mustn't change over time to stay compatible with previous software versions }; + // NOLINTNEXTLINE(misc-no-recursion) bool IsSolvable() const override { for (const auto& arg : m_subdescriptor_args) { @@ -578,6 +608,7 @@ public: return true; } + // NOLINTNEXTLINE(misc-no-recursion) bool IsRange() const final { for (const auto& pubkey : m_pubkey_args) { @@ -589,6 +620,7 @@ public: return false; } + // NOLINTNEXTLINE(misc-no-recursion) virtual bool ToStringSubScriptHelper(const SigningProvider* arg, std::string& ret, const StringType type, const DescriptorCache* cache = nullptr) const { size_t pos = 0; @@ -601,6 +633,7 @@ public: return true; } + // NOLINTNEXTLINE(misc-no-recursion) virtual bool ToStringHelper(const SigningProvider* arg, std::string& out, const StringType type, const DescriptorCache* cache = nullptr) const { std::string extra = ToStringExtra(); @@ -653,6 +686,7 @@ public: return ret; } + // NOLINTNEXTLINE(misc-no-recursion) bool ExpandHelper(int pos, const SigningProvider& arg, const DescriptorCache* read_cache, std::vector<CScript>& output_scripts, FlatSigningProvider& out, DescriptorCache* write_cache) const { std::vector<std::pair<CPubKey, KeyOriginInfo>> entries; @@ -694,6 +728,7 @@ public: return ExpandHelper(pos, DUMMY_SIGNING_PROVIDER, &read_cache, output_scripts, out, nullptr); } + // NOLINTNEXTLINE(misc-no-recursion) void ExpandPrivate(int pos, const SigningProvider& provider, FlatSigningProvider& out) const final { for (const auto& p : m_pubkey_args) { @@ -720,6 +755,20 @@ public: std::optional<int64_t> MaxSatisfactionWeight(bool) const override { return {}; } std::optional<int64_t> MaxSatisfactionElems() const override { return {}; } + + // NOLINTNEXTLINE(misc-no-recursion) + void GetPubKeys(std::set<CPubKey>& pubkeys, std::set<CExtPubKey>& ext_pubs) const override + { + for (const auto& p : m_pubkey_args) { + std::optional<CPubKey> pub = p->GetRootPubKey(); + if (pub) pubkeys.insert(*pub); + std::optional<CExtPubKey> ext_pub = p->GetRootExtPubKey(); + if (ext_pub) ext_pubs.insert(*ext_pub); + } + for (const auto& arg : m_subdescriptor_args) { + arg->GetPubKeys(pubkeys, ext_pubs); + } + } }; /** A parsed addr(A) descriptor. */ @@ -1537,6 +1586,7 @@ struct KeyParser { }; /** Parse a script in a particular context. */ +// NOLINTNEXTLINE(misc-no-recursion) std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, std::string& error) { using namespace spanparsing; @@ -1844,6 +1894,7 @@ std::unique_ptr<DescriptorImpl> InferMultiA(const CScript& script, ParseScriptCo return std::make_unique<MultiADescriptor>(match->first, std::move(keys)); } +// NOLINTNEXTLINE(misc-no-recursion) std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptContext ctx, const SigningProvider& provider) { if (ctx == ParseScriptContext::P2TR && script.size() == 34 && script[0] == 32 && script[33] == OP_CHECKSIG) { diff --git a/src/script/descriptor.h b/src/script/descriptor.h index caa5d1608d..e78a775330 100644 --- a/src/script/descriptor.h +++ b/src/script/descriptor.h @@ -158,6 +158,13 @@ struct Descriptor { /** Get the maximum size number of stack elements for satisfying this descriptor. */ virtual std::optional<int64_t> MaxSatisfactionElems() const = 0; + + /** Return all (extended) public keys for this descriptor, including any from subdescriptors. + * + * @param[out] pubkeys Any public keys + * @param[out] ext_pubs Any extended public keys + */ + virtual void GetPubKeys(std::set<CPubKey>& pubkeys, std::set<CExtPubKey>& ext_pubs) const = 0; }; /** Parse a `descriptor` string. Included private keys are put in `out`. diff --git a/src/script/miniscript.h b/src/script/miniscript.h index 76b952350b..f635fa7340 100644 --- a/src/script/miniscript.h +++ b/src/script/miniscript.h @@ -1617,7 +1617,7 @@ public: //! Produce a witness for this script, if possible and given the information available in the context. //! The non-malleable satisfaction is guaranteed to be valid if it exists, and ValidSatisfaction() //! is true. If IsSane() holds, this satisfaction is guaranteed to succeed in case the node's - //! conditions are satisfied (private keys and hash preimages available, locktimes satsified). + //! conditions are satisfied (private keys and hash preimages available, locktimes satisfied). template<typename Ctx> Availability Satisfy(const Ctx& ctx, std::vector<std::vector<unsigned char>>& stack, bool nonmalleable = true) const { auto ret = ProduceInput(ctx); diff --git a/src/script/sign.cpp b/src/script/sign.cpp index be4b357568..22ac062a63 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -295,7 +295,7 @@ struct TapSatisfier: Satisfier<XOnlyPubKey> { //! Conversion from a raw xonly public key. template <typename I> std::optional<XOnlyPubKey> FromPKBytes(I first, I last) const { - CHECK_NONFATAL(last - first == 32); + if (last - first != 32) return {}; XOnlyPubKey pubkey; std::copy(first, last, pubkey.begin()); return pubkey; diff --git a/src/script/signingprovider.cpp b/src/script/signingprovider.cpp index ff02ab5a12..baabd4d5b5 100644 --- a/src/script/signingprovider.cpp +++ b/src/script/signingprovider.cpp @@ -157,8 +157,10 @@ bool FillableSigningProvider::GetKey(const CKeyID &address, CKey &keyOut) const bool FillableSigningProvider::AddCScript(const CScript& redeemScript) { - if (redeemScript.size() > MAX_SCRIPT_ELEMENT_SIZE) - return error("FillableSigningProvider::AddCScript(): redeemScripts > %i bytes are invalid", MAX_SCRIPT_ELEMENT_SIZE); + if (redeemScript.size() > MAX_SCRIPT_ELEMENT_SIZE) { + LogError("FillableSigningProvider::AddCScript(): redeemScripts > %i bytes are invalid\n", MAX_SCRIPT_ELEMENT_SIZE); + return false; + } LOCK(cs_KeyStore); mapScripts[CScriptID(redeemScript)] = redeemScript; @@ -368,8 +370,6 @@ TaprootBuilder& TaprootBuilder::Add(int depth, Span<const unsigned char> script, /* Construct NodeInfo object with leaf hash and (if track is true) also leaf information. */ NodeInfo node; node.hash = ComputeTapleafHash(leaf_version, script); - // due to bug in clang-tidy-17: - // NOLINTNEXTLINE(modernize-use-emplace) if (track) node.leaves.emplace_back(LeafInfo{std::vector<unsigned char>(script.begin(), script.end()), leaf_version, {}}); /* Insert into the branch. */ Insert(std::move(node), depth); diff --git a/src/secp256k1/.github/actions/install-homebrew-valgrind/action.yml b/src/secp256k1/.github/actions/install-homebrew-valgrind/action.yml index 094ff891f7..ce10eb2686 100644 --- a/src/secp256k1/.github/actions/install-homebrew-valgrind/action.yml +++ b/src/secp256k1/.github/actions/install-homebrew-valgrind/action.yml @@ -16,7 +16,7 @@ runs: cat valgrind_fingerprint shell: bash - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: cache with: path: ${{ env.CI_HOMEBREW_CELLAR_VALGRIND }} diff --git a/src/secp256k1/.github/actions/run-in-docker-action/action.yml b/src/secp256k1/.github/actions/run-in-docker-action/action.yml index dbfaa4fece..74933686a0 100644 --- a/src/secp256k1/.github/actions/run-in-docker-action/action.yml +++ b/src/secp256k1/.github/actions/run-in-docker-action/action.yml @@ -36,6 +36,11 @@ runs: load: true cache-from: type=gha + - # Workaround for https://github.com/google/sanitizers/issues/1614 . + # The underlying issue has been fixed in clang 18.1.3. + run: sudo sysctl -w vm.mmap_rnd_bits=28 + shell: bash + - # Tell Docker to pass environment variables in `env` into the container. run: > docker run \ diff --git a/src/secp256k1/CMakeLists.txt b/src/secp256k1/CMakeLists.txt index cf0dc3ba93..9ef7defe51 100644 --- a/src/secp256k1/CMakeLists.txt +++ b/src/secp256k1/CMakeLists.txt @@ -51,29 +51,40 @@ endif() option(SECP256K1_INSTALL "Enable installation." ${PROJECT_IS_TOP_LEVEL}) -option(SECP256K1_ENABLE_MODULE_ECDH "Enable ECDH module." ON) -if(SECP256K1_ENABLE_MODULE_ECDH) - add_compile_definitions(ENABLE_MODULE_ECDH=1) -endif() +## Modules +# We declare all options before processing them, to make sure we can express +# dependendencies while processing. +option(SECP256K1_ENABLE_MODULE_ECDH "Enable ECDH module." ON) option(SECP256K1_ENABLE_MODULE_RECOVERY "Enable ECDSA pubkey recovery module." OFF) -if(SECP256K1_ENABLE_MODULE_RECOVERY) - add_compile_definitions(ENABLE_MODULE_RECOVERY=1) -endif() - option(SECP256K1_ENABLE_MODULE_EXTRAKEYS "Enable extrakeys module." ON) option(SECP256K1_ENABLE_MODULE_SCHNORRSIG "Enable schnorrsig module." ON) +option(SECP256K1_ENABLE_MODULE_ELLSWIFT "Enable ElligatorSwift module." ON) + +# Processing must be done in a topological sorting of the dependency graph +# (dependent module first). +if(SECP256K1_ENABLE_MODULE_ELLSWIFT) + add_compile_definitions(ENABLE_MODULE_ELLSWIFT=1) +endif() + if(SECP256K1_ENABLE_MODULE_SCHNORRSIG) + if(DEFINED SECP256K1_ENABLE_MODULE_EXTRAKEYS AND NOT SECP256K1_ENABLE_MODULE_EXTRAKEYS) + message(FATAL_ERROR "Module dependency error: You have disabled the extrakeys module explicitly, but it is required by the schnorrsig module.") + endif() set(SECP256K1_ENABLE_MODULE_EXTRAKEYS ON) add_compile_definitions(ENABLE_MODULE_SCHNORRSIG=1) endif() + if(SECP256K1_ENABLE_MODULE_EXTRAKEYS) add_compile_definitions(ENABLE_MODULE_EXTRAKEYS=1) endif() -option(SECP256K1_ENABLE_MODULE_ELLSWIFT "Enable ElligatorSwift module." ON) -if(SECP256K1_ENABLE_MODULE_ELLSWIFT) - add_compile_definitions(ENABLE_MODULE_ELLSWIFT=1) +if(SECP256K1_ENABLE_MODULE_RECOVERY) + add_compile_definitions(ENABLE_MODULE_RECOVERY=1) +endif() + +if(SECP256K1_ENABLE_MODULE_ECDH) + add_compile_definitions(ENABLE_MODULE_ECDH=1) endif() option(SECP256K1_USE_EXTERNAL_DEFAULT_CALLBACKS "Enable external default callback functions." OFF) @@ -254,9 +265,14 @@ if(SECP256K1_BUILD_BENCHMARK OR SECP256K1_BUILD_TESTS OR SECP256K1_BUILD_EXHAUST enable_testing() endif() +set(SECP256K1_LATE_CFLAGS "" CACHE STRING "Compiler flags that are added to the command line after all other flags added by the build system.") +include(AllTargetsCompileOptions) + add_subdirectory(src) +all_targets_compile_options(src "${SECP256K1_LATE_CFLAGS}") if(SECP256K1_BUILD_EXAMPLES) add_subdirectory(examples) + all_targets_compile_options(examples "${SECP256K1_LATE_CFLAGS}") endif() message("\n") @@ -330,6 +346,9 @@ else() message(" - LDFLAGS for executables ............ ${CMAKE_EXE_LINKER_FLAGS_DEBUG}") message(" - LDFLAGS for shared libraries ....... ${CMAKE_SHARED_LINKER_FLAGS_DEBUG}") endif() +if(SECP256K1_LATE_CFLAGS) + message("SECP256K1_LATE_CFLAGS ................. ${SECP256K1_LATE_CFLAGS}") +endif() message("\n") if(SECP256K1_EXPERIMENTAL) message( diff --git a/src/secp256k1/CONTRIBUTING.md b/src/secp256k1/CONTRIBUTING.md index a5e457913a..5fbf7332c9 100644 --- a/src/secp256k1/CONTRIBUTING.md +++ b/src/secp256k1/CONTRIBUTING.md @@ -44,7 +44,7 @@ The Contributor Workflow & Peer Review in libsecp256k1 are similar to Bitcoin Co In addition, libsecp256k1 tries to maintain the following coding conventions: -* No runtime heap allocation (e.g., no `malloc`) unless explicitly requested by the caller (via `secp256k1_context_create` or `secp256k1_scratch_space_create`, for example). Morever, it should be possible to use the library without any heap allocations. +* No runtime heap allocation (e.g., no `malloc`) unless explicitly requested by the caller (via `secp256k1_context_create` or `secp256k1_scratch_space_create`, for example). Moreover, it should be possible to use the library without any heap allocations. * The tests should cover all lines and branches of the library (see [Test coverage](#coverage)). * Operations involving secret data should be tested for being constant time with respect to the secrets (see [src/ctime_tests.c](src/ctime_tests.c)). * Local variables containing secret data should be cleared explicitly to try to delete secrets from memory. diff --git a/src/secp256k1/README.md b/src/secp256k1/README.md index 4013e6a93b..6e88eb4ecb 100644 --- a/src/secp256k1/README.md +++ b/src/secp256k1/README.md @@ -79,9 +79,9 @@ To maintain a pristine source tree, CMake encourages to perform an out-of-source $ mkdir build && cd build $ cmake .. - $ make - $ make check # run the test suite - $ sudo make install # optional + $ cmake --build . + $ ctest # run the test suite + $ sudo cmake --build . --target install # optional To compile optional modules (such as Schnorr signatures), you need to run `cmake` with additional flags (such as `-DSECP256K1_ENABLE_MODULE_SCHNORRSIG=ON`). Run `cmake .. -LH` to see the full list of available flags. diff --git a/src/secp256k1/ci/ci.sh b/src/secp256k1/ci/ci.sh index 9cc715955e..3999af4f1c 100755 --- a/src/secp256k1/ci/ci.sh +++ b/src/secp256k1/ci/ci.sh @@ -17,7 +17,8 @@ print_environment() { SECP256K1_TEST_ITERS BENCH SECP256K1_BENCH_ITERS CTIMETESTS\ EXAMPLES \ HOST WRAPPER_CMD \ - CC CFLAGS CPPFLAGS AR NM + CC CFLAGS CPPFLAGS AR NM \ + UBSAN_OPTIONS ASAN_OPTIONS LSAN_OPTIONS do eval "isset=\${$var+x}" if [ -n "$isset" ]; then diff --git a/src/secp256k1/cmake/AllTargetsCompileOptions.cmake b/src/secp256k1/cmake/AllTargetsCompileOptions.cmake new file mode 100644 index 0000000000..6e420e0fde --- /dev/null +++ b/src/secp256k1/cmake/AllTargetsCompileOptions.cmake @@ -0,0 +1,12 @@ +# Add compile options to all targets added in the subdirectory. +function(all_targets_compile_options dir options) + get_directory_property(targets DIRECTORY ${dir} BUILDSYSTEM_TARGETS) + separate_arguments(options) + set(compiled_target_types STATIC_LIBRARY SHARED_LIBRARY OBJECT_LIBRARY EXECUTABLE) + foreach(target ${targets}) + get_target_property(type ${target} TYPE) + if(type IN_LIST compiled_target_types) + target_compile_options(${target} PRIVATE ${options}) + endif() + endforeach() +endfunction() diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index 2c1596775e..158ed5d769 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -387,29 +387,32 @@ SECP_CFLAGS="$SECP_CFLAGS $WERROR_CFLAGS" ### Handle module options ### -if test x"$enable_module_ecdh" = x"yes"; then - SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ECDH=1" -fi - -if test x"$enable_module_recovery" = x"yes"; then - SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_RECOVERY=1" +# Processing must be done in a reverse topological sorting of the dependency graph +# (dependent module first). +if test x"$enable_module_ellswift" = x"yes"; then + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ELLSWIFT=1" fi if test x"$enable_module_schnorrsig" = x"yes"; then - SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_SCHNORRSIG=1" + if test x"$enable_module_extrakeys" = x"no"; then + AC_MSG_ERROR([Module dependency error: You have disabled the extrakeys module explicitly, but it is required by the schnorrsig module.]) + fi enable_module_extrakeys=yes + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_SCHNORRSIG=1" fi -if test x"$enable_module_ellswift" = x"yes"; then - SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ELLSWIFT=1" -fi - -# Test if extrakeys is set after the schnorrsig module to allow the schnorrsig -# module to set enable_module_extrakeys=yes if test x"$enable_module_extrakeys" = x"yes"; then SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_EXTRAKEYS=1" fi +if test x"$enable_module_recovery" = x"yes"; then + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_RECOVERY=1" +fi + +if test x"$enable_module_ecdh" = x"yes"; then + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ECDH=1" +fi + if test x"$enable_external_default_callbacks" = x"yes"; then SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DUSE_EXTERNAL_DEFAULT_CALLBACKS=1" fi diff --git a/src/secp256k1/contrib/lax_der_parsing.h b/src/secp256k1/contrib/lax_der_parsing.h index 034a38e6a0..37c8c691f2 100644 --- a/src/secp256k1/contrib/lax_der_parsing.h +++ b/src/secp256k1/contrib/lax_der_parsing.h @@ -67,8 +67,8 @@ extern "C" { * * Returns: 1 when the signature could be parsed, 0 otherwise. * Args: ctx: a secp256k1 context object - * Out: sig: a pointer to a signature object - * In: input: a pointer to the signature to be parsed + * Out: sig: pointer to a signature object + * In: input: pointer to the signature to be parsed * inputlen: the length of the array pointed to be input * * This function will accept any valid DER encoded signature, even if the diff --git a/src/secp256k1/doc/release-process.md b/src/secp256k1/doc/release-process.md index 51e337a5ab..cdf62430df 100644 --- a/src/secp256k1/doc/release-process.md +++ b/src/secp256k1/doc/release-process.md @@ -1,4 +1,4 @@ -# Release Process +# Release process This document outlines the process for releasing versions of the form `$MAJOR.$MINOR.$PATCH`. @@ -14,31 +14,30 @@ This process also assumes that there will be no minor releases for old major rel We aim to cut a regular release every 3-4 months, approximately twice as frequent as major Bitcoin Core releases. Every second release should be published one month before the feature freeze of the next major Bitcoin Core release, allowing sufficient time to update the library in Core. -## Sanity Checks -Perform these checks before creating a release: +## Sanity checks +Perform these checks when reviewing the release PR (see below): 1. Ensure `make distcheck` doesn't fail. -```shell -./autogen.sh && ./configure --enable-dev-mode && make distcheck -``` + ```shell + ./autogen.sh && ./configure --enable-dev-mode && make distcheck + ``` 2. Check installation with autotools: -```shell -dir=$(mktemp -d) -./autogen.sh && ./configure --prefix=$dir && make clean && make install && ls -RlAh $dir -gcc -o ecdsa examples/ecdsa.c $(PKG_CONFIG_PATH=$dir/lib/pkgconfig pkg-config --cflags --libs libsecp256k1) -Wl,-rpath,"$dir/lib" && ./ecdsa -``` + ```shell + dir=$(mktemp -d) + ./autogen.sh && ./configure --prefix=$dir && make clean && make install && ls -RlAh $dir + gcc -o ecdsa examples/ecdsa.c $(PKG_CONFIG_PATH=$dir/lib/pkgconfig pkg-config --cflags --libs libsecp256k1) -Wl,-rpath,"$dir/lib" && ./ecdsa + ``` 3. Check installation with CMake: -```shell -dir=$(mktemp -d) -build=$(mktemp -d) -cmake -B $build -DCMAKE_INSTALL_PREFIX=$dir && cmake --build $build --target install && ls -RlAh $dir -gcc -o ecdsa examples/ecdsa.c -I $dir/include -L $dir/lib*/ -l secp256k1 -Wl,-rpath,"$dir/lib",-rpath,"$dir/lib64" && ./ecdsa -``` -4. Use the [`check-abi.sh`](/tools/check-abi.sh) tool to ensure there are no unexpected ABI incompatibilities and that the version number and release notes accurately reflect all potential ABI changes. To run this tool, the `abi-dumper` and `abi-compliance-checker` packages are required. - -```shell -tools/check-abi.sh -``` + ```shell + dir=$(mktemp -d) + build=$(mktemp -d) + cmake -B $build -DCMAKE_INSTALL_PREFIX=$dir && cmake --build $build --target install && ls -RlAh $dir + gcc -o ecdsa examples/ecdsa.c -I $dir/include -L $dir/lib*/ -l secp256k1 -Wl,-rpath,"$dir/lib",-rpath,"$dir/lib64" && ./ecdsa + ``` +4. Use the [`check-abi.sh`](/tools/check-abi.sh) tool to verify that there are no unexpected ABI incompatibilities and that the version number and the release notes accurately reflect all potential ABI changes. To run this tool, the `abi-dumper` and `abi-compliance-checker` packages are required. + ```shell + tools/check-abi.sh + ``` ## Regular release @@ -47,27 +46,29 @@ tools/check-abi.sh * adding a section for the release (make sure that the version number is a link to a diff between the previous and new version), * removing the `[Unreleased]` section header, and * including an entry for `### ABI Compatibility` if it doesn't exist, - * sets `_PKG_VERSION_IS_RELEASE` to `true` in `configure.ac`, and - * if this is not a patch release - * updates `_PKG_VERSION_*` and `_LIB_VERSION_*` in `configure.ac` and + * sets `_PKG_VERSION_IS_RELEASE` to `true` in `configure.ac`, and, + * if this is not a patch release, + * updates `_PKG_VERSION_*` and `_LIB_VERSION_*` in `configure.ac`, and * updates `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_*` in `CMakeLists.txt`. -2. After the PR is merged, tag the commit and push it: +2. Perform the [sanity checks](#sanity-checks) on the PR branch. +3. After the PR is merged, tag the commit, and push the tag: ``` RELEASE_COMMIT=<merge commit of step 1> git tag -s v$MAJOR.$MINOR.$PATCH -m "libsecp256k1 $MAJOR.$MINOR.$PATCH" $RELEASE_COMMIT git push git@github.com:bitcoin-core/secp256k1.git v$MAJOR.$MINOR.$PATCH ``` -3. Open a PR to the master branch with a commit (using message `"release cleanup: bump version after $MAJOR.$MINOR.$PATCH"`, for example) that +4. Open a PR to the master branch with a commit (using message `"release cleanup: bump version after $MAJOR.$MINOR.$PATCH"`, for example) that * sets `_PKG_VERSION_IS_RELEASE` to `false` and increments `_PKG_VERSION_PATCH` and `_LIB_VERSION_REVISION` in `configure.ac`, * increments the `$PATCH` component of `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_REVISION` in `CMakeLists.txt`, and * adds an `[Unreleased]` section header to the [CHANGELOG.md](../CHANGELOG.md). If other maintainers are not present to approve the PR, it can be merged without ACKs. -4. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md). +5. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md). +6. Send an announcement email to the bitcoin-dev mailing list. ## Maintenance release -Note that bugfixes only need to be backported to releases for which no compatible release without the bug exists. +Note that bug fixes need to be backported only to releases for which no compatible release without the bug exists. 1. If there's no maintenance branch `$MAJOR.$MINOR`, create one: ``` @@ -75,19 +76,18 @@ Note that bugfixes only need to be backported to releases for which no compatibl git push git@github.com:bitcoin-core/secp256k1.git $MAJOR.$MINOR ``` 2. Open a pull request to the `$MAJOR.$MINOR` branch that - * includes the bugfixes, + * includes the bug fixes, * finalizes the release notes similar to a regular release, * increments `_PKG_VERSION_PATCH` and `_LIB_VERSION_REVISION` in `configure.ac` and the `$PATCH` component of `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_REVISION` in `CMakeLists.txt` (with commit message `"release: bump versions for $MAJOR.$MINOR.$PATCH"`, for example). -3. After the PRs are merged, update the release branch and tag the commit: +3. Perform the [sanity checks](#sanity-checks) on the PR branch. +4. After the PRs are merged, update the release branch, tag the commit, and push the tag: ``` git checkout $MAJOR.$MINOR && git pull git tag -s v$MAJOR.$MINOR.$PATCH -m "libsecp256k1 $MAJOR.$MINOR.$PATCH" - ``` -4. Push tag: - ``` git push git@github.com:bitcoin-core/secp256k1.git v$MAJOR.$MINOR.$PATCH ``` -5. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md). -6. Open PR to the master branch that includes a commit (with commit message `"release notes: add $MAJOR.$MINOR.$PATCH"`, for example) that adds release notes to [CHANGELOG.md](../CHANGELOG.md). +6. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md). +7. Send an announcement email to the bitcoin-dev mailing list. +8. Open PR to the master branch that includes a commit (with commit message `"release notes: add $MAJOR.$MINOR.$PATCH"`, for example) that adds release notes to [CHANGELOG.md](../CHANGELOG.md). diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index 936f0b42b7..f4053f2a93 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -265,7 +265,7 @@ SECP256K1_API void secp256k1_selftest(void); * memory allocation entirely, see secp256k1_context_static and the functions in * secp256k1_preallocated.h. * - * Returns: a newly created context object. + * Returns: pointer to a newly created context object. * In: flags: Always set to SECP256K1_CONTEXT_NONE (see below). * * The only valid non-deprecated flag in recent library versions is @@ -296,8 +296,8 @@ SECP256K1_API secp256k1_context *secp256k1_context_create( * Cloning secp256k1_context_static is not possible, and should not be emulated by * the caller (e.g., using memcpy). Create a new context instead. * - * Returns: a newly created context object. - * Args: ctx: an existing context to copy (not secp256k1_context_static) + * Returns: pointer to a newly created context object. + * Args: ctx: pointer to a context to copy (not secp256k1_context_static). */ SECP256K1_API secp256k1_context *secp256k1_context_clone( const secp256k1_context *ctx @@ -313,7 +313,7 @@ SECP256K1_API secp256k1_context *secp256k1_context_clone( * behaviour is undefined. In that case, secp256k1_context_preallocated_destroy must * be used instead. * - * Args: ctx: an existing context to destroy, constructed using + * Args: ctx: pointer to a context to destroy, constructed using * secp256k1_context_create or secp256k1_context_clone * (i.e., not secp256k1_context_static). */ @@ -350,8 +350,8 @@ SECP256K1_API void secp256k1_context_destroy( * fails. In this case, the corresponding default handler will be called with * the data pointer argument set to NULL. * - * Args: ctx: an existing context object. - * In: fun: a pointer to a function to call when an illegal argument is + * Args: ctx: pointer to a context object. + * In: fun: pointer to a function to call when an illegal argument is * passed to the API, taking a message and an opaque pointer. * (NULL restores the default handler.) * data: the opaque pointer to pass to fun above, must be NULL for the default handler. @@ -377,8 +377,8 @@ SECP256K1_API void secp256k1_context_set_illegal_callback( * for that). After this callback returns, anything may happen, including * crashing. * - * Args: ctx: an existing context object. - * In: fun: a pointer to a function to call when an internal error occurs, + * Args: ctx: pointer to a context object. + * In: fun: pointer to a function to call when an internal error occurs, * taking a message and an opaque pointer (NULL restores the * default handler, see secp256k1_context_set_illegal_callback * for details). @@ -395,7 +395,7 @@ SECP256K1_API void secp256k1_context_set_error_callback( /** Create a secp256k1 scratch space object. * * Returns: a newly created scratch space. - * Args: ctx: an existing context object. + * Args: ctx: pointer to a context object. * In: size: amount of memory to be available as scratch space. Some extra * (<100 bytes) will be allocated for extra accounting. */ @@ -407,7 +407,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT secp256k1_scratch_space *secp256k1_sc /** Destroy a secp256k1 scratch space. * * The pointer may not be used afterwards. - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object. * scratch: space to destroy */ SECP256K1_API void secp256k1_scratch_space_destroy( @@ -419,7 +419,7 @@ SECP256K1_API void secp256k1_scratch_space_destroy( * * Returns: 1 if the public key was fully valid. * 0 if the public key could not be parsed or is invalid. - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object. * Out: pubkey: pointer to a pubkey object. If 1 is returned, it is set to a * parsed version of input. If not, its value is undefined. * In: input: pointer to a serialized public key @@ -439,14 +439,14 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_parse( /** Serialize a pubkey object into a serialized byte sequence. * * Returns: 1 always. - * Args: ctx: a secp256k1 context object. - * Out: output: a pointer to a 65-byte (if compressed==0) or 33-byte (if + * Args: ctx: pointer to a context object. + * Out: output: pointer to a 65-byte (if compressed==0) or 33-byte (if * compressed==1) byte array to place the serialized key * in. - * In/Out: outputlen: a pointer to an integer which is initially set to the + * In/Out: outputlen: pointer to an integer which is initially set to the * size of output, and is overwritten with the written * size. - * In: pubkey: a pointer to a secp256k1_pubkey containing an + * In: pubkey: pointer to a secp256k1_pubkey containing an * initialized public key. * flags: SECP256K1_EC_COMPRESSED if serialization should be in * compressed format, otherwise SECP256K1_EC_UNCOMPRESSED. @@ -464,7 +464,7 @@ SECP256K1_API int secp256k1_ec_pubkey_serialize( * Returns: <0 if the first public key is less than the second * >0 if the first public key is greater than the second * 0 if the two public keys are equal - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object * In: pubkey1: first public key to compare * pubkey2: second public key to compare */ @@ -477,9 +477,9 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_cmp( /** Parse an ECDSA signature in compact (64 bytes) format. * * Returns: 1 when the signature could be parsed, 0 otherwise. - * Args: ctx: a secp256k1 context object - * Out: sig: a pointer to a signature object - * In: input64: a pointer to the 64-byte array to parse + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: input64: pointer to the 64-byte array to parse * * The signature must consist of a 32-byte big endian R value, followed by a * 32-byte big endian S value. If R or S fall outside of [0..order-1], the @@ -498,9 +498,9 @@ SECP256K1_API int secp256k1_ecdsa_signature_parse_compact( /** Parse a DER ECDSA signature. * * Returns: 1 when the signature could be parsed, 0 otherwise. - * Args: ctx: a secp256k1 context object - * Out: sig: a pointer to a signature object - * In: input: a pointer to the signature to be parsed + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: input: pointer to the signature to be parsed * inputlen: the length of the array pointed to be input * * This function will accept any valid DER encoded signature, even if the @@ -520,13 +520,13 @@ SECP256K1_API int secp256k1_ecdsa_signature_parse_der( /** Serialize an ECDSA signature in DER format. * * Returns: 1 if enough space was available to serialize, 0 otherwise - * Args: ctx: a secp256k1 context object - * Out: output: a pointer to an array to store the DER serialization - * In/Out: outputlen: a pointer to a length integer. Initially, this integer + * Args: ctx: pointer to a context object + * Out: output: pointer to an array to store the DER serialization + * In/Out: outputlen: pointer to a length integer. Initially, this integer * should be set to the length of output. After the call * it will be set to the length of the serialization (even * if 0 was returned). - * In: sig: a pointer to an initialized signature object + * In: sig: pointer to an initialized signature object */ SECP256K1_API int secp256k1_ecdsa_signature_serialize_der( const secp256k1_context *ctx, @@ -538,9 +538,9 @@ SECP256K1_API int secp256k1_ecdsa_signature_serialize_der( /** Serialize an ECDSA signature in compact (64 byte) format. * * Returns: 1 - * Args: ctx: a secp256k1 context object - * Out: output64: a pointer to a 64-byte array to store the compact serialization - * In: sig: a pointer to an initialized signature object + * Args: ctx: pointer to a context object + * Out: output64: pointer to a 64-byte array to store the compact serialization + * In: sig: pointer to an initialized signature object * * See secp256k1_ecdsa_signature_parse_compact for details about the encoding. */ @@ -554,7 +554,7 @@ SECP256K1_API int secp256k1_ecdsa_signature_serialize_compact( * * Returns: 1: correct signature * 0: incorrect or unparseable signature - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object * In: sig: the signature being verified. * msghash32: the 32-byte message hash being verified. * The verifier must make sure to apply a cryptographic @@ -585,12 +585,12 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ecdsa_verify( /** Convert a signature to a normalized lower-S form. * * Returns: 1 if sigin was not normalized, 0 if it already was. - * Args: ctx: a secp256k1 context object - * Out: sigout: a pointer to a signature to fill with the normalized form, + * Args: ctx: pointer to a context object + * Out: sigout: pointer to a signature to fill with the normalized form, * or copy if the input was already normalized. (can be NULL if * you're only interested in whether the input was already * normalized). - * In: sigin: a pointer to a signature to check/normalize (can be identical to sigout) + * In: sigin: pointer to a signature to check/normalize (can be identical to sigout) * * With ECDSA a third-party can forge a second distinct signature of the same * message, given a single initial signature, but without knowing the key. This diff --git a/src/secp256k1/include/secp256k1_ecdh.h b/src/secp256k1/include/secp256k1_ecdh.h index 515e174299..4d9da3461d 100644 --- a/src/secp256k1/include/secp256k1_ecdh.h +++ b/src/secp256k1/include/secp256k1_ecdh.h @@ -39,7 +39,7 @@ SECP256K1_API const secp256k1_ecdh_hash_function secp256k1_ecdh_hash_function_de * 0: scalar was invalid (zero or overflow) or hashfp returned 0 * Args: ctx: pointer to a context object. * Out: output: pointer to an array to be filled by hashfp. - * In: pubkey: a pointer to a secp256k1_pubkey containing an initialized public key. + * In: pubkey: pointer to a secp256k1_pubkey containing an initialized public key. * seckey: a 32-byte scalar with which to multiply the point. * hashfp: pointer to a hash function. If NULL, * secp256k1_ecdh_hash_function_sha256 is used diff --git a/src/secp256k1/include/secp256k1_ellswift.h b/src/secp256k1/include/secp256k1_ellswift.h index f79bd88396..ae37287f82 100644 --- a/src/secp256k1/include/secp256k1_ellswift.h +++ b/src/secp256k1/include/secp256k1_ellswift.h @@ -87,7 +87,7 @@ SECP256K1_API const secp256k1_ellswift_xdh_hash_function secp256k1_ellswift_xdh_ * Returns: 1 always. * Args: ctx: pointer to a context object * Out: ell64: pointer to a 64-byte array to be filled - * In: pubkey: a pointer to a secp256k1_pubkey containing an + * In: pubkey: pointer to a secp256k1_pubkey containing an * initialized public key * rnd32: pointer to 32 bytes of randomness * @@ -169,7 +169,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ellswift_create( * (will not be NULL) * ell_b64: pointer to the 64-byte encoded public key of party B * (will not be NULL) - * seckey32: a pointer to our 32-byte secret key + * seckey32: pointer to our 32-byte secret key * party: boolean indicating which party we are: zero if we are * party A, non-zero if we are party B. seckey32 must be * the private key corresponding to that party's ell_?64. diff --git a/src/secp256k1/include/secp256k1_extrakeys.h b/src/secp256k1/include/secp256k1_extrakeys.h index 7fcce68e68..ad70b92f95 100644 --- a/src/secp256k1/include/secp256k1_extrakeys.h +++ b/src/secp256k1/include/secp256k1_extrakeys.h @@ -39,7 +39,7 @@ typedef struct { * Returns: 1 if the public key was fully valid. * 0 if the public key could not be parsed or is invalid. * - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object. * Out: pubkey: pointer to a pubkey object. If 1 is returned, it is set to a * parsed version of input. If not, it's set to an invalid value. * In: input32: pointer to a serialized xonly_pubkey. @@ -54,9 +54,9 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_parse( * * Returns: 1 always. * - * Args: ctx: a secp256k1 context object. - * Out: output32: a pointer to a 32-byte array to place the serialized key in. - * In: pubkey: a pointer to a secp256k1_xonly_pubkey containing an initialized public key. + * Args: ctx: pointer to a context object. + * Out: output32: pointer to a 32-byte array to place the serialized key in. + * In: pubkey: pointer to a secp256k1_xonly_pubkey containing an initialized public key. */ SECP256K1_API int secp256k1_xonly_pubkey_serialize( const secp256k1_context *ctx, @@ -69,7 +69,7 @@ SECP256K1_API int secp256k1_xonly_pubkey_serialize( * Returns: <0 if the first public key is less than the second * >0 if the first public key is greater than the second * 0 if the two public keys are equal - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object. * In: pubkey1: first public key to compare * pubkey2: second public key to compare */ diff --git a/src/secp256k1/include/secp256k1_preallocated.h b/src/secp256k1/include/secp256k1_preallocated.h index f37744777b..f2d95c245e 100644 --- a/src/secp256k1/include/secp256k1_preallocated.h +++ b/src/secp256k1/include/secp256k1_preallocated.h @@ -52,8 +52,8 @@ SECP256K1_API size_t secp256k1_context_preallocated_size( * in the memory. In simpler words, the prealloc pointer (or any pointer derived * from it) should not be used during the lifetime of the context object. * - * Returns: a newly created context object. - * In: prealloc: a pointer to a rewritable contiguous block of memory of + * Returns: pointer to newly created context object. + * In: prealloc: pointer to a rewritable contiguous block of memory of * size at least secp256k1_context_preallocated_size(flags) * bytes, as detailed above. * flags: which parts of the context to initialize. @@ -72,7 +72,7 @@ SECP256K1_API secp256k1_context *secp256k1_context_preallocated_create( * caller-provided memory. * * Returns: the required size of the caller-provided memory block. - * In: ctx: an existing context to copy. + * In: ctx: pointer to a context to copy. */ SECP256K1_API size_t secp256k1_context_preallocated_clone_size( const secp256k1_context *ctx @@ -91,9 +91,9 @@ SECP256K1_API size_t secp256k1_context_preallocated_clone_size( * Cloning secp256k1_context_static is not possible, and should not be emulated by * the caller (e.g., using memcpy). Create a new context instead. * - * Returns: a newly created context object. - * Args: ctx: an existing context to copy (not secp256k1_context_static). - * In: prealloc: a pointer to a rewritable contiguous block of memory of + * Returns: pointer to a newly created context object. + * Args: ctx: pointer to a context to copy (not secp256k1_context_static). + * In: prealloc: pointer to a rewritable contiguous block of memory of * size at least secp256k1_context_preallocated_size(flags) * bytes, as detailed above. */ @@ -118,7 +118,7 @@ SECP256K1_API secp256k1_context *secp256k1_context_preallocated_clone( * preallocated pointer given to secp256k1_context_preallocated_create or * secp256k1_context_preallocated_clone. * - * Args: ctx: an existing context to destroy, constructed using + * Args: ctx: pointer to a context to destroy, constructed using * secp256k1_context_preallocated_create or * secp256k1_context_preallocated_clone * (i.e., not secp256k1_context_static). diff --git a/src/secp256k1/include/secp256k1_recovery.h b/src/secp256k1/include/secp256k1_recovery.h index b12ca4d972..341b8bac63 100644 --- a/src/secp256k1/include/secp256k1_recovery.h +++ b/src/secp256k1/include/secp256k1_recovery.h @@ -28,9 +28,9 @@ typedef struct { /** Parse a compact ECDSA signature (64 bytes + recovery id). * * Returns: 1 when the signature could be parsed, 0 otherwise - * Args: ctx: a secp256k1 context object - * Out: sig: a pointer to a signature object - * In: input64: a pointer to a 64-byte compact signature + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: input64: pointer to a 64-byte compact signature * recid: the recovery id (0, 1, 2 or 3) */ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_parse_compact( @@ -43,9 +43,9 @@ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_parse_compact( /** Convert a recoverable signature into a normal signature. * * Returns: 1 - * Args: ctx: a secp256k1 context object. - * Out: sig: a pointer to a normal signature. - * In: sigin: a pointer to a recoverable signature. + * Args: ctx: pointer to a context object. + * Out: sig: pointer to a normal signature. + * In: sigin: pointer to a recoverable signature. */ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_convert( const secp256k1_context *ctx, @@ -56,10 +56,10 @@ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_convert( /** Serialize an ECDSA signature in compact format (64 bytes + recovery id). * * Returns: 1 - * Args: ctx: a secp256k1 context object. - * Out: output64: a pointer to a 64-byte array of the compact signature. - * recid: a pointer to an integer to hold the recovery id. - * In: sig: a pointer to an initialized signature object. + * Args: ctx: pointer to a context object. + * Out: output64: pointer to a 64-byte array of the compact signature. + * recid: pointer to an integer to hold the recovery id. + * In: sig: pointer to an initialized signature object. */ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_serialize_compact( const secp256k1_context *ctx, diff --git a/src/secp256k1/include/secp256k1_schnorrsig.h b/src/secp256k1/include/secp256k1_schnorrsig.h index 26358533f6..23163de2fb 100644 --- a/src/secp256k1/include/secp256k1_schnorrsig.h +++ b/src/secp256k1/include/secp256k1_schnorrsig.h @@ -169,11 +169,11 @@ SECP256K1_API int secp256k1_schnorrsig_sign_custom( * * Returns: 1: correct signature * 0: incorrect signature - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object. * In: sig64: pointer to the 64-byte signature to verify. * msg: the message being verified. Can only be NULL if msglen is 0. * msglen: length of the message - * pubkey: pointer to an x-only public key to verify with (cannot be NULL) + * pubkey: pointer to an x-only public key to verify with */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorrsig_verify( const secp256k1_context *ctx, diff --git a/src/secp256k1/src/assumptions.h b/src/secp256k1/src/assumptions.h index 8ed04209e9..7961005350 100644 --- a/src/secp256k1/src/assumptions.h +++ b/src/secp256k1/src/assumptions.h @@ -19,65 +19,69 @@ reduce the odds of experiencing an unwelcome surprise. */ -struct secp256k1_assumption_checker { - /* This uses a trick to implement a static assertion in C89: a type with an array of negative size is not - allowed. */ - int dummy_array[( - /* Bytes are 8 bits. */ - (CHAR_BIT == 8) && +#if defined(__has_attribute) +# if __has_attribute(__unavailable__) +__attribute__((__unavailable__("Don't call this function. It only exists because STATIC_ASSERT cannot be used outside a function."))) +# endif +#endif +static void secp256k1_assumption_checker(void) { + /* Bytes are 8 bits. */ + STATIC_ASSERT(CHAR_BIT == 8); - /* No integer promotion for uint32_t. This ensures that we can multiply uintXX_t values where XX >= 32 - without signed overflow, which would be undefined behaviour. */ - (UINT_MAX <= UINT32_MAX) && + /* No integer promotion for uint32_t. This ensures that we can multiply uintXX_t values where XX >= 32 + without signed overflow, which would be undefined behaviour. */ + STATIC_ASSERT(UINT_MAX <= UINT32_MAX); - /* Conversions from unsigned to signed outside of the bounds of the signed type are - implementation-defined. Verify that they function as reinterpreting the lower - bits of the input in two's complement notation. Do this for conversions: - - from uint(N)_t to int(N)_t with negative result - - from uint(2N)_t to int(N)_t with negative result - - from int(2N)_t to int(N)_t with negative result - - from int(2N)_t to int(N)_t with positive result */ + /* Conversions from unsigned to signed outside of the bounds of the signed type are + implementation-defined. Verify that they function as reinterpreting the lower + bits of the input in two's complement notation. Do this for conversions: + - from uint(N)_t to int(N)_t with negative result + - from uint(2N)_t to int(N)_t with negative result + - from int(2N)_t to int(N)_t with negative result + - from int(2N)_t to int(N)_t with positive result */ - /* To int8_t. */ - ((int8_t)(uint8_t)0xAB == (int8_t)-(int8_t)0x55) && - ((int8_t)(uint16_t)0xABCD == (int8_t)-(int8_t)0x33) && - ((int8_t)(int16_t)(uint16_t)0xCDEF == (int8_t)(uint8_t)0xEF) && - ((int8_t)(int16_t)(uint16_t)0x9234 == (int8_t)(uint8_t)0x34) && + /* To int8_t. */ + STATIC_ASSERT(((int8_t)(uint8_t)0xAB == (int8_t)-(int8_t)0x55)); + STATIC_ASSERT((int8_t)(uint16_t)0xABCD == (int8_t)-(int8_t)0x33); + STATIC_ASSERT((int8_t)(int16_t)(uint16_t)0xCDEF == (int8_t)(uint8_t)0xEF); + STATIC_ASSERT((int8_t)(int16_t)(uint16_t)0x9234 == (int8_t)(uint8_t)0x34); - /* To int16_t. */ - ((int16_t)(uint16_t)0xBCDE == (int16_t)-(int16_t)0x4322) && - ((int16_t)(uint32_t)0xA1B2C3D4 == (int16_t)-(int16_t)0x3C2C) && - ((int16_t)(int32_t)(uint32_t)0xC1D2E3F4 == (int16_t)(uint16_t)0xE3F4) && - ((int16_t)(int32_t)(uint32_t)0x92345678 == (int16_t)(uint16_t)0x5678) && + /* To int16_t. */ + STATIC_ASSERT((int16_t)(uint16_t)0xBCDE == (int16_t)-(int16_t)0x4322); + STATIC_ASSERT((int16_t)(uint32_t)0xA1B2C3D4 == (int16_t)-(int16_t)0x3C2C); + STATIC_ASSERT((int16_t)(int32_t)(uint32_t)0xC1D2E3F4 == (int16_t)(uint16_t)0xE3F4); + STATIC_ASSERT((int16_t)(int32_t)(uint32_t)0x92345678 == (int16_t)(uint16_t)0x5678); - /* To int32_t. */ - ((int32_t)(uint32_t)0xB2C3D4E5 == (int32_t)-(int32_t)0x4D3C2B1B) && - ((int32_t)(uint64_t)0xA123B456C789D012ULL == (int32_t)-(int32_t)0x38762FEE) && - ((int32_t)(int64_t)(uint64_t)0xC1D2E3F4A5B6C7D8ULL == (int32_t)(uint32_t)0xA5B6C7D8) && - ((int32_t)(int64_t)(uint64_t)0xABCDEF0123456789ULL == (int32_t)(uint32_t)0x23456789) && + /* To int32_t. */ + STATIC_ASSERT((int32_t)(uint32_t)0xB2C3D4E5 == (int32_t)-(int32_t)0x4D3C2B1B); + STATIC_ASSERT((int32_t)(uint64_t)0xA123B456C789D012ULL == (int32_t)-(int32_t)0x38762FEE); + STATIC_ASSERT((int32_t)(int64_t)(uint64_t)0xC1D2E3F4A5B6C7D8ULL == (int32_t)(uint32_t)0xA5B6C7D8); + STATIC_ASSERT((int32_t)(int64_t)(uint64_t)0xABCDEF0123456789ULL == (int32_t)(uint32_t)0x23456789); - /* To int64_t. */ - ((int64_t)(uint64_t)0xB123C456D789E012ULL == (int64_t)-(int64_t)0x4EDC3BA928761FEEULL) && + /* To int64_t. */ + STATIC_ASSERT((int64_t)(uint64_t)0xB123C456D789E012ULL == (int64_t)-(int64_t)0x4EDC3BA928761FEEULL); #if defined(SECP256K1_INT128_NATIVE) - ((int64_t)(((uint128_t)0xA1234567B8901234ULL << 64) + 0xC5678901D2345678ULL) == (int64_t)-(int64_t)0x3A9876FE2DCBA988ULL) && - (((int64_t)(int128_t)(((uint128_t)0xB1C2D3E4F5A6B7C8ULL << 64) + 0xD9E0F1A2B3C4D5E6ULL)) == (int64_t)(uint64_t)0xD9E0F1A2B3C4D5E6ULL) && - (((int64_t)(int128_t)(((uint128_t)0xABCDEF0123456789ULL << 64) + 0x0123456789ABCDEFULL)) == (int64_t)(uint64_t)0x0123456789ABCDEFULL) && + STATIC_ASSERT((int64_t)(((uint128_t)0xA1234567B8901234ULL << 64) + 0xC5678901D2345678ULL) == (int64_t)-(int64_t)0x3A9876FE2DCBA988ULL); + STATIC_ASSERT(((int64_t)(int128_t)(((uint128_t)0xB1C2D3E4F5A6B7C8ULL << 64) + 0xD9E0F1A2B3C4D5E6ULL)) == (int64_t)(uint64_t)0xD9E0F1A2B3C4D5E6ULL); + STATIC_ASSERT(((int64_t)(int128_t)(((uint128_t)0xABCDEF0123456789ULL << 64) + 0x0123456789ABCDEFULL)) == (int64_t)(uint64_t)0x0123456789ABCDEFULL); - /* To int128_t. */ - ((int128_t)(((uint128_t)0xB1234567C8901234ULL << 64) + 0xD5678901E2345678ULL) == (int128_t)(-(int128_t)0x8E1648B3F50E80DCULL * 0x8E1648B3F50E80DDULL + 0x5EA688D5482F9464ULL)) && + /* To int128_t. */ + STATIC_ASSERT((int128_t)(((uint128_t)0xB1234567C8901234ULL << 64) + 0xD5678901E2345678ULL) == (int128_t)(-(int128_t)0x8E1648B3F50E80DCULL * 0x8E1648B3F50E80DDULL + 0x5EA688D5482F9464ULL)); #endif - /* Right shift on negative signed values is implementation defined. Verify that it - acts as a right shift in two's complement with sign extension (i.e duplicating - the top bit into newly added bits). */ - ((((int8_t)0xE8) >> 2) == (int8_t)(uint8_t)0xFA) && - ((((int16_t)0xE9AC) >> 4) == (int16_t)(uint16_t)0xFE9A) && - ((((int32_t)0x937C918A) >> 9) == (int32_t)(uint32_t)0xFFC9BE48) && - ((((int64_t)0xA8B72231DF9CF4B9ULL) >> 19) == (int64_t)(uint64_t)0xFFFFF516E4463BF3ULL) && + /* Right shift on negative signed values is implementation defined. Verify that it + acts as a right shift in two's complement with sign extension (i.e duplicating + the top bit into newly added bits). */ + STATIC_ASSERT((((int8_t)0xE8) >> 2) == (int8_t)(uint8_t)0xFA); + STATIC_ASSERT((((int16_t)0xE9AC) >> 4) == (int16_t)(uint16_t)0xFE9A); + STATIC_ASSERT((((int32_t)0x937C918A) >> 9) == (int32_t)(uint32_t)0xFFC9BE48); + STATIC_ASSERT((((int64_t)0xA8B72231DF9CF4B9ULL) >> 19) == (int64_t)(uint64_t)0xFFFFF516E4463BF3ULL); #if defined(SECP256K1_INT128_NATIVE) - ((((int128_t)(((uint128_t)0xCD833A65684A0DBCULL << 64) + 0xB349312F71EA7637ULL)) >> 39) == (int128_t)(((uint128_t)0xFFFFFFFFFF9B0674ULL << 64) + 0xCAD0941B79669262ULL)) && + STATIC_ASSERT((((int128_t)(((uint128_t)0xCD833A65684A0DBCULL << 64) + 0xB349312F71EA7637ULL)) >> 39) == (int128_t)(((uint128_t)0xFFFFFFFFFF9B0674ULL << 64) + 0xCAD0941B79669262ULL)); #endif - 1) * 2 - 1]; -}; + + /* This function is not supposed to be called. */ + VERIFY_CHECK(0); +} #endif /* SECP256K1_ASSUMPTIONS_H */ diff --git a/src/secp256k1/src/checkmem.h b/src/secp256k1/src/checkmem.h index f2169decfc..7e333ce5f3 100644 --- a/src/secp256k1/src/checkmem.h +++ b/src/secp256k1/src/checkmem.h @@ -30,6 +30,8 @@ * - SECP256K1_CHECKMEM_DEFINE(p, len): * - marks the len-byte memory pointed to by p as defined data (public data, in the * context of constant-time checking). + * - SECP256K1_CHECKMEM_MSAN_DEFINE(p, len): + * - Like SECP256K1_CHECKMEM_DEFINE, but applies only to memory_sanitizer. * */ @@ -48,11 +50,16 @@ # define SECP256K1_CHECKMEM_ENABLED 1 # define SECP256K1_CHECKMEM_UNDEFINE(p, len) __msan_allocated_memory((p), (len)) # define SECP256K1_CHECKMEM_DEFINE(p, len) __msan_unpoison((p), (len)) +# define SECP256K1_CHECKMEM_MSAN_DEFINE(p, len) __msan_unpoison((p), (len)) # define SECP256K1_CHECKMEM_CHECK(p, len) __msan_check_mem_is_initialized((p), (len)) # define SECP256K1_CHECKMEM_RUNNING() (1) # endif #endif +#if !defined SECP256K1_CHECKMEM_MSAN_DEFINE +# define SECP256K1_CHECKMEM_MSAN_DEFINE(p, len) SECP256K1_CHECKMEM_NOOP((p), (len)) +#endif + /* If valgrind integration is desired (through the VALGRIND define), implement the * SECP256K1_CHECKMEM_* macros using valgrind. */ #if !defined SECP256K1_CHECKMEM_ENABLED diff --git a/src/secp256k1/src/field.h b/src/secp256k1/src/field.h index bd589bf8a8..8c65a3aff6 100644 --- a/src/secp256k1/src/field.h +++ b/src/secp256k1/src/field.h @@ -255,8 +255,8 @@ static void secp256k1_fe_add(secp256k1_fe *r, const secp256k1_fe *a); /** Multiply two field elements. * * On input, a and b must be valid field elements; r does not need to be initialized. - * r and a may point to the same object, but neither can be equal to b. The magnitudes - * of a and b must not exceed 8. + * r and a may point to the same object, but neither may point to the object pointed + * to by b. The magnitudes of a and b must not exceed 8. * Performs {r = a * b} * On output, r will have magnitude 1, but won't be normalized. */ diff --git a/src/secp256k1/src/modules/ellswift/tests_impl.h b/src/secp256k1/src/modules/ellswift/tests_impl.h index 7d1efbc492..f96e3a1268 100644 --- a/src/secp256k1/src/modules/ellswift/tests_impl.h +++ b/src/secp256k1/src/modules/ellswift/tests_impl.h @@ -188,9 +188,9 @@ void run_ellswift_tests(void) { CHECK(ret == ((testcase->enc_bitmap >> c) & 1)); if (ret) { secp256k1_fe x2; - CHECK(check_fe_equal(&t, &testcase->encs[c])); + CHECK(fe_equal(&t, &testcase->encs[c])); secp256k1_ellswift_xswiftec_var(&x2, &testcase->u, &testcase->encs[c]); - CHECK(check_fe_equal(&testcase->x, &x2)); + CHECK(fe_equal(&testcase->x, &x2)); } } } @@ -203,7 +203,7 @@ void run_ellswift_tests(void) { CHECK(ret); ret = secp256k1_pubkey_load(CTX, &ge, &pubkey); CHECK(ret); - CHECK(check_fe_equal(&testcase->x, &ge.x)); + CHECK(fe_equal(&testcase->x, &ge.x)); CHECK(secp256k1_fe_is_odd(&ge.y) == testcase->odd_y); } for (i = 0; (unsigned)i < sizeof(ellswift_xdh_tests_bip324) / sizeof(ellswift_xdh_tests_bip324[0]); ++i) { @@ -290,7 +290,7 @@ void run_ellswift_tests(void) { secp256k1_ecmult(&resj, &decj, &sec, NULL); secp256k1_ge_set_gej(&res, &resj); /* Compare. */ - CHECK(check_fe_equal(&res.x, &share_x)); + CHECK(fe_equal(&res.x, &share_x)); } /* Verify the joint behavior of secp256k1_ellswift_xdh */ for (i = 0; i < 200 * COUNT; i++) { diff --git a/src/secp256k1/src/scalar_4x64_impl.h b/src/secp256k1/src/scalar_4x64_impl.h index 7b9c542f07..82cd957f16 100644 --- a/src/secp256k1/src/scalar_4x64_impl.h +++ b/src/secp256k1/src/scalar_4x64_impl.h @@ -462,6 +462,14 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) : "S"(l), "i"(SECP256K1_N_C_0), "i"(SECP256K1_N_C_1) : "rax", "rdx", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "cc"); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m0, sizeof(m0)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m1, sizeof(m1)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m2, sizeof(m2)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m3, sizeof(m3)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m4, sizeof(m4)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m5, sizeof(m5)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m6, sizeof(m6)); + /* Reduce 385 bits into 258. */ __asm__ __volatile__( /* Preload */ @@ -541,6 +549,12 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) : "g"(m0), "g"(m1), "g"(m2), "g"(m3), "g"(m4), "g"(m5), "g"(m6), "i"(SECP256K1_N_C_0), "i"(SECP256K1_N_C_1) : "rax", "rdx", "r8", "r9", "r10", "r11", "r12", "r13", "cc"); + SECP256K1_CHECKMEM_MSAN_DEFINE(&p0, sizeof(p0)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&p1, sizeof(p1)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&p2, sizeof(p2)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&p3, sizeof(p3)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&p4, sizeof(p4)); + /* Reduce 258 bits into 256. */ __asm__ __volatile__( /* Preload */ @@ -586,6 +600,10 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) : "=g"(c) : "g"(p0), "g"(p1), "g"(p2), "g"(p3), "g"(p4), "D"(r), "i"(SECP256K1_N_C_0), "i"(SECP256K1_N_C_1) : "rax", "rdx", "r8", "r9", "r10", "cc", "memory"); + + SECP256K1_CHECKMEM_MSAN_DEFINE(r, sizeof(*r)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&c, sizeof(c)); + #else secp256k1_uint128 c128; uint64_t c, c0, c1, c2; @@ -663,7 +681,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) secp256k1_scalar_reduce(r, c + secp256k1_scalar_check_overflow(r)); } -static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, const secp256k1_scalar *b) { +static void secp256k1_scalar_mul_512(uint64_t *l8, const secp256k1_scalar *a, const secp256k1_scalar *b) { #ifdef USE_ASM_X86_64 const uint64_t *pb = b->d; __asm__ __volatile__( @@ -678,7 +696,7 @@ static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, c /* (rax,rdx) = a0 * b0 */ "movq %%r15, %%rax\n" "mulq %%r11\n" - /* Extract l0 */ + /* Extract l8[0] */ "movq %%rax, 0(%%rsi)\n" /* (r8,r9,r10) = (rdx) */ "movq %%rdx, %%r8\n" @@ -696,7 +714,7 @@ static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, c "addq %%rax, %%r8\n" "adcq %%rdx, %%r9\n" "adcq $0, %%r10\n" - /* Extract l1 */ + /* Extract l8[1] */ "movq %%r8, 8(%%rsi)\n" "xorq %%r8, %%r8\n" /* (r9,r10,r8) += a0 * b2 */ @@ -717,7 +735,7 @@ static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, c "addq %%rax, %%r9\n" "adcq %%rdx, %%r10\n" "adcq $0, %%r8\n" - /* Extract l2 */ + /* Extract l8[2] */ "movq %%r9, 16(%%rsi)\n" "xorq %%r9, %%r9\n" /* (r10,r8,r9) += a0 * b3 */ @@ -746,7 +764,7 @@ static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, c "addq %%rax, %%r10\n" "adcq %%rdx, %%r8\n" "adcq $0, %%r9\n" - /* Extract l3 */ + /* Extract l8[3] */ "movq %%r10, 24(%%rsi)\n" "xorq %%r10, %%r10\n" /* (r8,r9,r10) += a1 * b3 */ @@ -767,7 +785,7 @@ static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, c "addq %%rax, %%r8\n" "adcq %%rdx, %%r9\n" "adcq $0, %%r10\n" - /* Extract l4 */ + /* Extract l8[4] */ "movq %%r8, 32(%%rsi)\n" "xorq %%r8, %%r8\n" /* (r9,r10,r8) += a2 * b3 */ @@ -782,51 +800,54 @@ static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, c "addq %%rax, %%r9\n" "adcq %%rdx, %%r10\n" "adcq $0, %%r8\n" - /* Extract l5 */ + /* Extract l8[5] */ "movq %%r9, 40(%%rsi)\n" /* (r10,r8) += a3 * b3 */ "movq %%r15, %%rax\n" "mulq %%r14\n" "addq %%rax, %%r10\n" "adcq %%rdx, %%r8\n" - /* Extract l6 */ + /* Extract l8[6] */ "movq %%r10, 48(%%rsi)\n" - /* Extract l7 */ + /* Extract l8[7] */ "movq %%r8, 56(%%rsi)\n" : "+d"(pb) - : "S"(l), "D"(a->d) + : "S"(l8), "D"(a->d) : "rax", "rbx", "rcx", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", "cc", "memory"); + + SECP256K1_CHECKMEM_MSAN_DEFINE(l8, sizeof(*l8) * 8); + #else /* 160 bit accumulator. */ uint64_t c0 = 0, c1 = 0; uint32_t c2 = 0; - /* l[0..7] = a[0..3] * b[0..3]. */ + /* l8[0..7] = a[0..3] * b[0..3]. */ muladd_fast(a->d[0], b->d[0]); - extract_fast(l[0]); + extract_fast(l8[0]); muladd(a->d[0], b->d[1]); muladd(a->d[1], b->d[0]); - extract(l[1]); + extract(l8[1]); muladd(a->d[0], b->d[2]); muladd(a->d[1], b->d[1]); muladd(a->d[2], b->d[0]); - extract(l[2]); + extract(l8[2]); muladd(a->d[0], b->d[3]); muladd(a->d[1], b->d[2]); muladd(a->d[2], b->d[1]); muladd(a->d[3], b->d[0]); - extract(l[3]); + extract(l8[3]); muladd(a->d[1], b->d[3]); muladd(a->d[2], b->d[2]); muladd(a->d[3], b->d[1]); - extract(l[4]); + extract(l8[4]); muladd(a->d[2], b->d[3]); muladd(a->d[3], b->d[2]); - extract(l[5]); + extract(l8[5]); muladd_fast(a->d[3], b->d[3]); - extract_fast(l[6]); + extract_fast(l8[6]); VERIFY_CHECK(c1 == 0); - l[7] = c0; + l8[7] = c0; #endif } diff --git a/src/secp256k1/src/scalar_impl.h b/src/secp256k1/src/scalar_impl.h index bbba83e937..972d8041b0 100644 --- a/src/secp256k1/src/scalar_impl.h +++ b/src/secp256k1/src/scalar_impl.h @@ -229,7 +229,7 @@ static void secp256k1_scalar_split_lambda(secp256k1_scalar * SECP256K1_RESTRICT * <= {triangle inequality} * a1*|k*b2/n - c1| + a2*|k*(-b1)/n - c2| * < {Lemma 1 and Lemma 2} - * a1*(2^-1 + epslion1) + a2*(2^-1 + epsilon2) + * a1*(2^-1 + epsilon1) + a2*(2^-1 + epsilon2) * < {rounding up to an integer} * (a1 + a2 + 1)/2 * < {rounding up to a power of 2} @@ -247,7 +247,7 @@ static void secp256k1_scalar_split_lambda(secp256k1_scalar * SECP256K1_RESTRICT * <= {triangle inequality} * (-b1)*|k*b2/n - c1| + b2*|k*(-b1)/n - c2| * < {Lemma 1 and Lemma 2} - * (-b1)*(2^-1 + epslion1) + b2*(2^-1 + epsilon2) + * (-b1)*(2^-1 + epsilon1) + b2*(2^-1 + epsilon2) * < {rounding up to an integer} * (-b1 + b2)/2 + 1 * < {rounding up to a power of 2} diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index 4c11e7f0b8..15a5eede67 100644 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -237,36 +237,25 @@ static SECP256K1_INLINE void secp256k1_declassify(const secp256k1_context* ctx, } static int secp256k1_pubkey_load(const secp256k1_context* ctx, secp256k1_ge* ge, const secp256k1_pubkey* pubkey) { - if (sizeof(secp256k1_ge_storage) == 64) { - /* When the secp256k1_ge_storage type is exactly 64 byte, use its - * representation inside secp256k1_pubkey, as conversion is very fast. - * Note that secp256k1_pubkey_save must use the same representation. */ - secp256k1_ge_storage s; - memcpy(&s, &pubkey->data[0], sizeof(s)); - secp256k1_ge_from_storage(ge, &s); - } else { - /* Otherwise, fall back to 32-byte big endian for X and Y. */ - secp256k1_fe x, y; - ARG_CHECK(secp256k1_fe_set_b32_limit(&x, pubkey->data)); - ARG_CHECK(secp256k1_fe_set_b32_limit(&y, pubkey->data + 32)); - secp256k1_ge_set_xy(ge, &x, &y); - } + secp256k1_ge_storage s; + + /* We require that the secp256k1_ge_storage type is exactly 64 bytes. + * This is formally not guaranteed by the C standard, but should hold on any + * sane compiler in the real world. */ + STATIC_ASSERT(sizeof(secp256k1_ge_storage) == 64); + memcpy(&s, &pubkey->data[0], 64); + secp256k1_ge_from_storage(ge, &s); ARG_CHECK(!secp256k1_fe_is_zero(&ge->x)); return 1; } static void secp256k1_pubkey_save(secp256k1_pubkey* pubkey, secp256k1_ge* ge) { - if (sizeof(secp256k1_ge_storage) == 64) { - secp256k1_ge_storage s; - secp256k1_ge_to_storage(&s, ge); - memcpy(&pubkey->data[0], &s, sizeof(s)); - } else { - VERIFY_CHECK(!secp256k1_ge_is_infinity(ge)); - secp256k1_fe_normalize_var(&ge->x); - secp256k1_fe_normalize_var(&ge->y); - secp256k1_fe_get_b32(pubkey->data, &ge->x); - secp256k1_fe_get_b32(pubkey->data + 32, &ge->y); - } + secp256k1_ge_storage s; + + STATIC_ASSERT(sizeof(secp256k1_ge_storage) == 64); + VERIFY_CHECK(!secp256k1_ge_is_infinity(ge)); + secp256k1_ge_to_storage(&s, ge); + memcpy(&pubkey->data[0], &s, 64); } int secp256k1_ec_pubkey_parse(const secp256k1_context* ctx, secp256k1_pubkey* pubkey, const unsigned char *input, size_t inputlen) { diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index bec1c45585..85b4881295 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -2927,20 +2927,18 @@ static void run_scalar_tests(void) { secp256k1_scalar_set_b32(&r2, res[i][1], &overflow); CHECK(!overflow); secp256k1_scalar_mul(&z, &x, &y); - CHECK(!secp256k1_scalar_check_overflow(&z)); CHECK(secp256k1_scalar_eq(&r1, &z)); if (!secp256k1_scalar_is_zero(&y)) { secp256k1_scalar_inverse(&zz, &y); - CHECK(!secp256k1_scalar_check_overflow(&zz)); secp256k1_scalar_inverse_var(&zzv, &y); CHECK(secp256k1_scalar_eq(&zzv, &zz)); secp256k1_scalar_mul(&z, &z, &zz); - CHECK(!secp256k1_scalar_check_overflow(&z)); CHECK(secp256k1_scalar_eq(&x, &z)); secp256k1_scalar_mul(&zz, &zz, &y); - CHECK(!secp256k1_scalar_check_overflow(&zz)); CHECK(secp256k1_scalar_eq(&secp256k1_scalar_one, &zz)); } + secp256k1_scalar_mul(&z, &x, &x); + CHECK(secp256k1_scalar_eq(&r2, &z)); } } } @@ -2955,7 +2953,7 @@ static void random_fe_non_square(secp256k1_fe *ns) { } } -static int check_fe_equal(const secp256k1_fe *a, const secp256k1_fe *b) { +static int fe_equal(const secp256k1_fe *a, const secp256k1_fe *b) { secp256k1_fe an = *a; secp256k1_fe bn = *b; secp256k1_fe_normalize_weak(&an); @@ -3092,7 +3090,7 @@ static void run_field_half(void) { #endif secp256k1_fe_normalize_weak(&u); secp256k1_fe_add(&u, &u); - CHECK(check_fe_equal(&t, &u)); + CHECK(fe_equal(&t, &u)); /* Check worst-case input: ensure the LSB is 1 so that P will be added, * which will also cause all carries to be 1, since all limbs that can @@ -3111,7 +3109,7 @@ static void run_field_half(void) { #endif secp256k1_fe_normalize_weak(&u); secp256k1_fe_add(&u, &u); - CHECK(check_fe_equal(&t, &u)); + CHECK(fe_equal(&t, &u)); } } @@ -3138,7 +3136,7 @@ static void run_field_misc(void) { secp256k1_fe_add(&z, &q); /* z = x+v */ q = x; /* q = x */ secp256k1_fe_add_int(&q, v); /* q = x+v */ - CHECK(check_fe_equal(&q, &z)); + CHECK(fe_equal(&q, &z)); /* Test the fe equality and comparison operations. */ CHECK(secp256k1_fe_cmp_var(&x, &x) == 0); CHECK(secp256k1_fe_equal(&x, &x)); @@ -3198,27 +3196,27 @@ static void run_field_misc(void) { secp256k1_fe_add(&y, &x); z = x; secp256k1_fe_mul_int(&z, 3); - CHECK(check_fe_equal(&y, &z)); + CHECK(fe_equal(&y, &z)); secp256k1_fe_add(&y, &x); secp256k1_fe_add(&z, &x); - CHECK(check_fe_equal(&z, &y)); + CHECK(fe_equal(&z, &y)); z = x; secp256k1_fe_mul_int(&z, 5); secp256k1_fe_mul(&q, &x, &fe5); - CHECK(check_fe_equal(&z, &q)); + CHECK(fe_equal(&z, &q)); secp256k1_fe_negate(&x, &x, 1); secp256k1_fe_add(&z, &x); secp256k1_fe_add(&q, &x); - CHECK(check_fe_equal(&y, &z)); - CHECK(check_fe_equal(&q, &y)); + CHECK(fe_equal(&y, &z)); + CHECK(fe_equal(&q, &y)); /* Check secp256k1_fe_half. */ z = x; secp256k1_fe_half(&z); secp256k1_fe_add(&z, &z); - CHECK(check_fe_equal(&x, &z)); + CHECK(fe_equal(&x, &z)); secp256k1_fe_add(&z, &z); secp256k1_fe_half(&z); - CHECK(check_fe_equal(&x, &z)); + CHECK(fe_equal(&x, &z)); } } @@ -3287,18 +3285,31 @@ static void run_fe_mul(void) { } static void run_sqr(void) { - secp256k1_fe x, s; + int i; + secp256k1_fe x, y, lhs, rhs, tmp; - { - int i; - secp256k1_fe_set_int(&x, 1); - secp256k1_fe_negate(&x, &x, 1); + secp256k1_fe_set_int(&x, 1); + secp256k1_fe_negate(&x, &x, 1); - for (i = 1; i <= 512; ++i) { - secp256k1_fe_mul_int(&x, 2); - secp256k1_fe_normalize(&x); - secp256k1_fe_sqr(&s, &x); - } + for (i = 1; i <= 512; ++i) { + secp256k1_fe_mul_int(&x, 2); + secp256k1_fe_normalize(&x); + + /* Check that (x+y)*(x-y) = x^2 - y*2 for some random values y */ + random_fe_test(&y); + + lhs = x; + secp256k1_fe_add(&lhs, &y); /* lhs = x+y */ + secp256k1_fe_negate(&tmp, &y, 1); /* tmp = -y */ + secp256k1_fe_add(&tmp, &x); /* tmp = x-y */ + secp256k1_fe_mul(&lhs, &lhs, &tmp); /* lhs = (x+y)*(x-y) */ + + secp256k1_fe_sqr(&rhs, &x); /* rhs = x^2 */ + secp256k1_fe_sqr(&tmp, &y); /* tmp = y^2 */ + secp256k1_fe_negate(&tmp, &tmp, 1); /* tmp = -y^2 */ + secp256k1_fe_add(&rhs, &tmp); /* rhs = x^2 - y^2 */ + + CHECK(fe_equal(&lhs, &rhs)); } } @@ -3620,9 +3631,9 @@ static void run_inverse_tests(void) for (i = 0; (size_t)i < sizeof(fe_cases)/sizeof(fe_cases[0]); ++i) { for (var = 0; var <= 1; ++var) { test_inverse_field(&x_fe, &fe_cases[i][0], var); - check_fe_equal(&x_fe, &fe_cases[i][1]); + CHECK(fe_equal(&x_fe, &fe_cases[i][1])); test_inverse_field(&x_fe, &fe_cases[i][1], var); - check_fe_equal(&x_fe, &fe_cases[i][0]); + CHECK(fe_equal(&x_fe, &fe_cases[i][0])); } } for (i = 0; (size_t)i < sizeof(scalar_cases)/sizeof(scalar_cases[0]); ++i) { @@ -4558,7 +4569,7 @@ static void ecmult_const_mult_xonly(void) { /* Check that resj's X coordinate corresponds with resx. */ secp256k1_fe_sqr(&v, &resj.z); secp256k1_fe_mul(&v, &v, &resx); - CHECK(check_fe_equal(&v, &resj.x)); + CHECK(fe_equal(&v, &resj.x)); } /* Test that secp256k1_ecmult_const_xonly correctly rejects X coordinates not on curve. */ diff --git a/src/secp256k1/src/util.h b/src/secp256k1/src/util.h index 187bf1c5e0..154d9ebcf1 100644 --- a/src/secp256k1/src/util.h +++ b/src/secp256k1/src/util.h @@ -51,13 +51,27 @@ static void print_buf_plain(const unsigned char *buf, size_t len) { # define SECP256K1_INLINE inline # endif +/** Assert statically that expr is true. + * + * This is a statement-like macro and can only be used inside functions. + */ +#define STATIC_ASSERT(expr) do { \ + switch(0) { \ + case 0: \ + /* If expr evaluates to 0, we have two case labels "0", which is illegal. */ \ + case /* ERROR: static assertion failed */ (expr): \ + ; \ + } \ +} while(0) + /** Assert statically that expr is an integer constant expression, and run stmt. * * Useful for example to enforce that magnitude arguments are constant. */ #define ASSERT_INT_CONST_AND_DO(expr, stmt) do { \ switch(42) { \ - case /* ERROR: integer argument is not constant */ expr: \ + /* C allows only integer constant expressions as case labels. */ \ + case /* ERROR: integer argument is not constant */ (expr): \ break; \ default: ; \ } \ diff --git a/src/secp256k1/tools/check-abi.sh b/src/secp256k1/tools/check-abi.sh index 8f6119cd8e..55c945ac16 100755 --- a/src/secp256k1/tools/check-abi.sh +++ b/src/secp256k1/tools/check-abi.sh @@ -3,17 +3,19 @@ set -eu default_base_version="$(git describe --match "v*.*.*" --abbrev=0)" -default_new_version="master" +default_new_version="HEAD" display_help_and_exit() { - echo "Usage: $0 <base_ver> <new_ver>" + echo "Usage: $0 [<base_ver> [<new_ver>]]" echo "" echo "Description: This script uses the ABI Compliance Checker tool to determine if the ABI" echo " of a new version of libsecp256k1 has changed in a backward-incompatible way." echo "" echo "Options:" - echo " base_ver Specify the base version (default: $default_base_version)" - echo " new_ver Specify the new version (default: $default_new_version)" + echo " base_ver Specify the base version as a git commit-ish" + echo " (default: most recent reachable tag matching \"v.*.*\", currently \"$default_base_version\")" + echo " new_ver Specify the new version as a git commit-ish" + echo " (default: $default_new_version)" echo " -h, --help Display this help message" exit 0 } @@ -23,9 +25,11 @@ if [ "$#" -eq 0 ]; then new_version="$default_new_version" elif [ "$#" -eq 1 ] && { [ "$1" = "-h" ] || [ "$1" = "--help" ]; }; then display_help_and_exit -elif [ "$#" -eq 2 ]; then +elif [ "$#" -eq 1 ] || [ "$#" -eq 2 ]; then base_version="$1" - new_version="$2" + if [ "$#" -eq 2 ]; then + new_version="$2" + fi else echo "Invalid usage. See help:" echo "" @@ -33,7 +37,8 @@ else fi checkout_and_build() { - git worktree add -d "$1" "$2" + _orig_dir="$(pwd)" + git worktree add --detach "$1" "$2" cd "$1" mkdir build && cd build cmake -S .. --preset dev-mode \ @@ -45,20 +50,18 @@ checkout_and_build() { -DSECP256K1_BUILD_EXAMPLES=OFF cmake --build . -j "$(nproc)" abi-dumper src/libsecp256k1.so -o ABI.dump -lver "$2" + cd "$_orig_dir" } echo "Comparing $base_version (base version) to $new_version (new version)" echo -original_dir="$(pwd)" - -base_source_dir=$(mktemp -d) +base_source_dir="$(mktemp -d)" checkout_and_build "$base_source_dir" "$base_version" -new_source_dir=$(mktemp -d) +new_source_dir="$(mktemp -d)" checkout_and_build "$new_source_dir" "$new_version" -cd "$original_dir" abi-compliance-checker -lib libsecp256k1 -old "${base_source_dir}/build/ABI.dump" -new "${new_source_dir}/build/ABI.dump" git worktree remove "$base_source_dir" git worktree remove "$new_source_dir" diff --git a/src/test/README.md b/src/test/README.md index 0876db7a72..bab1a28f61 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -42,17 +42,18 @@ test_bitcoin --log_level=all --run_test=getarg_tests ``` `log_level` controls the verbosity of the test framework, which logs when a -test case is entered, for example. `test_bitcoin` also accepts the command -line arguments accepted by `bitcoind`. Use `--` to separate both types of -arguments: +test case is entered, for example. + +`test_bitcoin` also accepts some of the command line arguments accepted by +`bitcoind`. Use `--` to separate these sets of arguments: ```bash test_bitcoin --log_level=all --run_test=getarg_tests -- -printtoconsole=1 ``` -The `-printtoconsole=1` after the two dashes redirects the debug log, which -would normally go to a file in the test datadir -(`BasicTestingSetup::m_path_root`), to the standard terminal output. +The `-printtoconsole=1` after the two dashes sends debug logging, which +normally goes only to `debug.log` within the data directory, also to the +standard terminal output. ... or to run just the doubledash test: @@ -60,7 +61,42 @@ would normally go to a file in the test datadir test_bitcoin --run_test=getarg_tests/doubledash ``` -Run `test_bitcoin --help` for the full list. +`test_bitcoin` creates a temporary working (data) directory with a randomly +generated pathname within `test_common_Bitcoin Core/`, which in turn is within +the system's temporary directory (see +[`temp_directory_path`](https://en.cppreference.com/w/cpp/filesystem/temp_directory_path)). +This data directory looks like a simplified form of the standard `bitcoind` data +directory. Its content will vary depending on the test, but it will always +have a `debug.log` file, for example. + +The location of the temporary data directory can be specified with the +`-testdatadir` option. This can make debugging easier. The directory +path used is the argument path appended with +`/test_common_Bitcoin Core/<test-name>/datadir`. +The directory path is created if necessary. +Specifying this argument also causes the data directory +not to be removed after the last test. This is useful for looking at +what the test wrote to `debug.log` after it completes, for example. +(The directory is removed at the start of the next test run, +so no leftover state is used.) + +```bash +$ test_bitcoin --run_test=getarg_tests/doubledash -- -testdatadir=/somewhere/mydatadir +Test directory (will not be deleted): "/somewhere/mydatadir/test_common_Bitcoin Core/getarg_tests/doubledash/datadir +Running 1 test case... + +*** No errors detected +$ ls -l '/somewhere/mydatadir/test_common_Bitcoin Core/getarg_tests/doubledash/datadir' +total 8 +drwxrwxr-x 2 admin admin 4096 Nov 27 22:45 blocks +-rw-rw-r-- 1 admin admin 1003 Nov 27 22:45 debug.log +``` + +If you run an entire test suite, such as `--run_test=getarg_tests`, or all the test suites +(by not specifying `--run_test`), a separate directory +will be created for each individual test. + +Run `test_bitcoin --help` for the full list of tests. ### Adding test cases diff --git a/src/test/argsman_tests.cpp b/src/test/argsman_tests.cpp index 1f46efe464..340208a1c9 100644 --- a/src/test/argsman_tests.cpp +++ b/src/test/argsman_tests.cpp @@ -904,7 +904,7 @@ BOOST_FIXTURE_TEST_CASE(util_ArgsMerge, ArgsMergeTestingSetup) // If check below fails, should manually dump the results with: // - // ARGS_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=util_tests/util_ArgsMerge + // ARGS_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=argsman_tests/util_ArgsMerge // // And verify diff against previous results to make sure the changes are expected. // @@ -1007,7 +1007,7 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) // If check below fails, should manually dump the results with: // - // CHAIN_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=util_tests/util_ChainMerge + // CHAIN_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=argsman_tests/util_ChainMerge // // And verify diff against previous results to make sure the changes are expected. // diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 763f0f897e..05355fb21d 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -14,7 +14,7 @@ #include <boost/test/unit_test.hpp> -std::vector<std::pair<uint256, CTransactionRef>> extra_txn; +std::vector<CTransactionRef> extra_txn; BOOST_FIXTURE_TEST_SUITE(blockencodings_tests, RegTestingSetup) @@ -126,7 +126,7 @@ public: explicit TestHeaderAndShortIDs(const CBlock& block) : TestHeaderAndShortIDs(CBlockHeaderAndShortTxIDs{block}) {} - uint64_t GetShortID(const uint256& txhash) const { + uint64_t GetShortID(const Wtxid& txhash) const { DataStream stream{}; stream << *this; CBlockHeaderAndShortTxIDs base; @@ -155,8 +155,8 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) shortIDs.prefilledtxn.resize(1); shortIDs.prefilledtxn[0] = {1, block.vtx[1]}; shortIDs.shorttxids.resize(2); - shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[0]->GetHash()); - shortIDs.shorttxids[1] = shortIDs.GetShortID(block.vtx[2]->GetHash()); + shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[0]->GetWitnessHash()); + shortIDs.shorttxids[1] = shortIDs.GetShortID(block.vtx[2]->GetWitnessHash()); DataStream stream{}; stream << shortIDs; @@ -226,7 +226,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) shortIDs.prefilledtxn[0] = {0, block.vtx[0]}; shortIDs.prefilledtxn[1] = {1, block.vtx[2]}; // id == 1 as it is 1 after index 1 shortIDs.shorttxids.resize(1); - shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[1]->GetHash()); + shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[1]->GetWitnessHash()); DataStream stream{}; stream << shortIDs; diff --git a/src/test/common_url_tests.cpp b/src/test/common_url_tests.cpp new file mode 100644 index 0000000000..cc893cbed7 --- /dev/null +++ b/src/test/common_url_tests.cpp @@ -0,0 +1,72 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit/. + +#include <common/url.h> + +#include <string> + +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(common_url_tests) + +// These test vectors were ported from test/regress.c in the libevent library +// which used to be a dependency of the UrlDecode function. + +BOOST_AUTO_TEST_CASE(encode_decode_test) { + BOOST_CHECK_EQUAL(UrlDecode("Hello"), "Hello"); + BOOST_CHECK_EQUAL(UrlDecode("99"), "99"); + BOOST_CHECK_EQUAL(UrlDecode(""), ""); + BOOST_CHECK_EQUAL(UrlDecode("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ123456789-.~_"), + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ123456789-.~_"); + BOOST_CHECK_EQUAL(UrlDecode("%20"), " "); + BOOST_CHECK_EQUAL(UrlDecode("%FF%F0%E0"), "\xff\xf0\xe0"); + BOOST_CHECK_EQUAL(UrlDecode("%01%19"), "\x01\x19"); + BOOST_CHECK_EQUAL(UrlDecode("http%3A%2F%2Fwww.ietf.org%2Frfc%2Frfc3986.txt"), + "http://www.ietf.org/rfc/rfc3986.txt"); + BOOST_CHECK_EQUAL(UrlDecode("1%2B2%3D3"), "1+2=3"); +} + +BOOST_AUTO_TEST_CASE(decode_malformed_test) { + BOOST_CHECK_EQUAL(UrlDecode("%%xhello th+ere \xff"), "%%xhello th+ere \xff"); + + BOOST_CHECK_EQUAL(UrlDecode("%"), "%"); + BOOST_CHECK_EQUAL(UrlDecode("%%"), "%%"); + BOOST_CHECK_EQUAL(UrlDecode("%%%"), "%%%"); + BOOST_CHECK_EQUAL(UrlDecode("%%%%"), "%%%%"); + + BOOST_CHECK_EQUAL(UrlDecode("+"), "+"); + BOOST_CHECK_EQUAL(UrlDecode("++"), "++"); + + BOOST_CHECK_EQUAL(UrlDecode("?"), "?"); + BOOST_CHECK_EQUAL(UrlDecode("??"), "??"); + + BOOST_CHECK_EQUAL(UrlDecode("%G1"), "%G1"); + BOOST_CHECK_EQUAL(UrlDecode("%2"), "%2"); + BOOST_CHECK_EQUAL(UrlDecode("%ZX"), "%ZX"); + + BOOST_CHECK_EQUAL(UrlDecode("valid%20string%G1"), "valid string%G1"); + BOOST_CHECK_EQUAL(UrlDecode("%20invalid%ZX"), " invalid%ZX"); + BOOST_CHECK_EQUAL(UrlDecode("%20%G1%ZX"), " %G1%ZX"); + + BOOST_CHECK_EQUAL(UrlDecode("%1 "), "%1 "); + BOOST_CHECK_EQUAL(UrlDecode("% 9"), "% 9"); + BOOST_CHECK_EQUAL(UrlDecode(" %Z "), " %Z "); + BOOST_CHECK_EQUAL(UrlDecode(" % X"), " % X"); + + BOOST_CHECK_EQUAL(UrlDecode("%-1"), "%-1"); + BOOST_CHECK_EQUAL(UrlDecode("%1-"), "%1-"); +} + +BOOST_AUTO_TEST_CASE(decode_lowercase_hex_test) { + BOOST_CHECK_EQUAL(UrlDecode("%f0%a0%b0"), "\xf0\xa0\xb0"); +} + +BOOST_AUTO_TEST_CASE(decode_internal_nulls_test) { + std::string result1{"\0\0x\0\0", 5}; + BOOST_CHECK_EQUAL(UrlDecode("%00%00x%00%00"), result1); + std::string result2{"abc\0\0", 5}; + BOOST_CHECK_EQUAL(UrlDecode("abc%00%00"), result2); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index 264b47b07c..13c2740553 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -4,6 +4,7 @@ #include <compressor.h> #include <script/script.h> +#include <test/util/random.h> #include <test/util/setup_common.h> #include <stdint.h> @@ -131,4 +132,36 @@ BOOST_AUTO_TEST_CASE(compress_script_to_uncompressed_pubkey_id) BOOST_CHECK_EQUAL(out[0], 0x04 | (script[65] & 0x01)); // least significant bit (lsb) of last char of pubkey is mapped into out[0] } +BOOST_AUTO_TEST_CASE(compress_p2pk_scripts_not_on_curve) +{ + XOnlyPubKey x_not_on_curve; + do { + x_not_on_curve = XOnlyPubKey(g_insecure_rand_ctx.randbytes(32)); + } while (x_not_on_curve.IsFullyValid()); + + // Check that P2PK script with uncompressed pubkey [=> OP_PUSH65 <0x04 .....> OP_CHECKSIG] + // which is not fully valid (i.e. point is not on curve) can't be compressed + std::vector<unsigned char> pubkey_raw(65, 0); + pubkey_raw[0] = 4; + std::copy(x_not_on_curve.begin(), x_not_on_curve.end(), &pubkey_raw[1]); + CPubKey pubkey_not_on_curve(pubkey_raw); + assert(pubkey_not_on_curve.IsValid()); + assert(!pubkey_not_on_curve.IsFullyValid()); + CScript script = CScript() << ToByteVector(pubkey_not_on_curve) << OP_CHECKSIG; + BOOST_CHECK_EQUAL(script.size(), 67U); + + CompressedScript out; + bool done = CompressScript(script, out); + BOOST_CHECK_EQUAL(done, false); + + // Check that compressed P2PK script with uncompressed pubkey that is not fully + // valid (i.e. x coordinate of the pubkey is not on curve) can't be decompressed + CompressedScript compressed_script(x_not_on_curve.begin(), x_not_on_curve.end()); + for (unsigned int compression_id : {4, 5}) { + CScript uncompressed_script; + bool success = DecompressScript(uncompressed_script, compression_id, compressed_script); + BOOST_CHECK_EQUAL(success, false); + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/feefrac_tests.cpp b/src/test/feefrac_tests.cpp new file mode 100644 index 0000000000..5af3c3d7ed --- /dev/null +++ b/src/test/feefrac_tests.cpp @@ -0,0 +1,85 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/feefrac.h> +#include <random.h> + +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(feefrac_tests) + +BOOST_AUTO_TEST_CASE(feefrac_operators) +{ + FeeFrac p1{1000, 100}, p2{500, 300}; + FeeFrac sum{1500, 400}; + FeeFrac diff{500, -200}; + FeeFrac empty{0, 0}; + FeeFrac zero_fee{0, 1}; // zero-fee allowed + + BOOST_CHECK(empty == FeeFrac{}); // same as no-args + + BOOST_CHECK(p1 == p1); + BOOST_CHECK(p1 + p2 == sum); + BOOST_CHECK(p1 - p2 == diff); + + FeeFrac p3{2000, 200}; + BOOST_CHECK(p1 != p3); // feefracs only equal if both fee and size are same + BOOST_CHECK(p2 != p3); + + FeeFrac p4{3000, 300}; + BOOST_CHECK(p1 == p4-p3); + BOOST_CHECK(p1 + p3 == p4); + + // Fee-rate comparison + BOOST_CHECK(p1 > p2); + BOOST_CHECK(p1 >= p2); + BOOST_CHECK(p1 >= p4-p3); + BOOST_CHECK(!(p1 >> p3)); // not strictly better + BOOST_CHECK(p1 >> p2); // strictly greater feerate + + BOOST_CHECK(p2 < p1); + BOOST_CHECK(p2 <= p1); + BOOST_CHECK(p1 <= p4-p3); + BOOST_CHECK(!(p3 << p1)); // not strictly worse + BOOST_CHECK(p2 << p1); // strictly lower feerate + + // "empty" comparisons + BOOST_CHECK(!(p1 >> empty)); // << will always result in false + BOOST_CHECK(!(p1 << empty)); + BOOST_CHECK(!(empty >> empty)); + BOOST_CHECK(!(empty << empty)); + + // empty is always bigger than everything else + BOOST_CHECK(empty > p1); + BOOST_CHECK(empty > p2); + BOOST_CHECK(empty > p3); + BOOST_CHECK(empty >= p1); + BOOST_CHECK(empty >= p2); + BOOST_CHECK(empty >= p3); + + // check "max" values for comparison + FeeFrac oversized_1{4611686000000, 4000000}; + FeeFrac oversized_2{184467440000000, 100000}; + + BOOST_CHECK(oversized_1 < oversized_2); + BOOST_CHECK(oversized_1 <= oversized_2); + BOOST_CHECK(oversized_1 << oversized_2); + BOOST_CHECK(oversized_1 != oversized_2); + + // Tests paths that use double arithmetic + FeeFrac busted{(static_cast<int64_t>(INT32_MAX)) + 1, INT32_MAX}; + BOOST_CHECK(!(busted < busted)); + + FeeFrac max_fee{2100000000000000, INT32_MAX}; + BOOST_CHECK(!(max_fee < max_fee)); + BOOST_CHECK(!(max_fee > max_fee)); + BOOST_CHECK(max_fee <= max_fee); + BOOST_CHECK(max_fee >= max_fee); + + FeeFrac max_fee2{1, 1}; + BOOST_CHECK(max_fee >= max_fee2); + +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/feefrac.cpp b/src/test/fuzz/feefrac.cpp new file mode 100644 index 0000000000..2c7553360e --- /dev/null +++ b/src/test/fuzz/feefrac.cpp @@ -0,0 +1,123 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/feefrac.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <compare> +#include <cstdint> +#include <iostream> + +namespace { + +/** Compute a * b, represented in 4x32 bits, highest limb first. */ +std::array<uint32_t, 4> Mul128(uint64_t a, uint64_t b) +{ + std::array<uint32_t, 4> ret{0, 0, 0, 0}; + + /** Perform ret += v << (32 * pos), at 128-bit precision. */ + auto add_fn = [&](uint64_t v, int pos) { + uint64_t accum{0}; + for (int i = 0; i + pos < 4; ++i) { + // Add current value at limb pos in ret. + accum += ret[3 - pos - i]; + // Add low or high half of v. + if (i == 0) accum += v & 0xffffffff; + if (i == 1) accum += v >> 32; + // Store lower half of result in limb pos in ret. + ret[3 - pos - i] = accum & 0xffffffff; + // Leave carry in accum. + accum >>= 32; + } + // Make sure no overflow. + assert(accum == 0); + }; + + // Multiply the 4 individual limbs (schoolbook multiply, with base 2^32). + add_fn((a & 0xffffffff) * (b & 0xffffffff), 0); + add_fn((a >> 32) * (b & 0xffffffff), 1); + add_fn((a & 0xffffffff) * (b >> 32), 1); + add_fn((a >> 32) * (b >> 32), 2); + return ret; +} + +/* comparison helper for std::array */ +std::strong_ordering compare_arrays(const std::array<uint32_t, 4>& a, const std::array<uint32_t, 4>& b) { + for (size_t i = 0; i < a.size(); ++i) { + if (a[i] != b[i]) return a[i] <=> b[i]; + } + return std::strong_ordering::equal; +} + +std::strong_ordering MulCompare(int64_t a1, int64_t a2, int64_t b1, int64_t b2) +{ + // Compute and compare signs. + int sign_a = (a1 == 0 ? 0 : a1 < 0 ? -1 : 1) * (a2 == 0 ? 0 : a2 < 0 ? -1 : 1); + int sign_b = (b1 == 0 ? 0 : b1 < 0 ? -1 : 1) * (b2 == 0 ? 0 : b2 < 0 ? -1 : 1); + if (sign_a != sign_b) return sign_a <=> sign_b; + + // Compute absolute values. + uint64_t abs_a1 = static_cast<uint64_t>(a1), abs_a2 = static_cast<uint64_t>(a2); + uint64_t abs_b1 = static_cast<uint64_t>(b1), abs_b2 = static_cast<uint64_t>(b2); + // Use (~x + 1) instead of the equivalent (-x) to silence the linter; mod 2^64 behavior is + // intentional here. + if (a1 < 0) abs_a1 = ~abs_a1 + 1; + if (a2 < 0) abs_a2 = ~abs_a2 + 1; + if (b1 < 0) abs_b1 = ~abs_b1 + 1; + if (b2 < 0) abs_b2 = ~abs_b2 + 1; + + // Compute products of absolute values. + auto mul_abs_a = Mul128(abs_a1, abs_a2); + auto mul_abs_b = Mul128(abs_b1, abs_b2); + if (sign_a < 0) { + return compare_arrays(mul_abs_b, mul_abs_a); + } else { + return compare_arrays(mul_abs_a, mul_abs_b); + } +} + +} // namespace + +FUZZ_TARGET(feefrac) +{ + FuzzedDataProvider provider(buffer.data(), buffer.size()); + + int64_t f1 = provider.ConsumeIntegral<int64_t>(); + int32_t s1 = provider.ConsumeIntegral<int32_t>(); + if (s1 == 0) f1 = 0; + FeeFrac fr1(f1, s1); + assert(fr1.IsEmpty() == (s1 == 0)); + + int64_t f2 = provider.ConsumeIntegral<int64_t>(); + int32_t s2 = provider.ConsumeIntegral<int32_t>(); + if (s2 == 0) f2 = 0; + FeeFrac fr2(f2, s2); + assert(fr2.IsEmpty() == (s2 == 0)); + + // Feerate comparisons + auto cmp_feerate = MulCompare(f1, s2, f2, s1); + assert(FeeRateCompare(fr1, fr2) == cmp_feerate); + assert((fr1 << fr2) == std::is_lt(cmp_feerate)); + assert((fr1 >> fr2) == std::is_gt(cmp_feerate)); + + // Compare with manual invocation of FeeFrac::Mul. + auto cmp_mul = FeeFrac::Mul(f1, s2) <=> FeeFrac::Mul(f2, s1); + assert(cmp_mul == cmp_feerate); + + // Same, but using FeeFrac::MulFallback. + auto cmp_fallback = FeeFrac::MulFallback(f1, s2) <=> FeeFrac::MulFallback(f2, s1); + assert(cmp_fallback == cmp_feerate); + + // Total order comparisons + auto cmp_total = std::is_eq(cmp_feerate) ? (s2 <=> s1) : cmp_feerate; + assert((fr1 <=> fr2) == cmp_total); + assert((fr1 < fr2) == std::is_lt(cmp_total)); + assert((fr1 > fr2) == std::is_gt(cmp_total)); + assert((fr1 <= fr2) == std::is_lteq(cmp_total)); + assert((fr1 >= fr2) == std::is_gteq(cmp_total)); + assert((fr1 == fr2) == std::is_eq(cmp_total)); + assert((fr1 != fr2) == std::is_neq(cmp_total)); +} diff --git a/src/test/fuzz/feeratediagram.cpp b/src/test/fuzz/feeratediagram.cpp new file mode 100644 index 0000000000..1a9c5ee946 --- /dev/null +++ b/src/test/fuzz/feeratediagram.cpp @@ -0,0 +1,133 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <stdint.h> + +#include <vector> + +#include <util/feefrac.h> +#include <policy/rbf.h> + +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <assert.h> + +namespace { + +/** Takes the pre-computed and topologically-valid chunks and generates a fee diagram which starts at FeeFrac of (0, 0) */ +std::vector<FeeFrac> BuildDiagramFromChunks(const Span<const FeeFrac> chunks) +{ + std::vector<FeeFrac> diagram; + diagram.reserve(chunks.size() + 1); + + diagram.emplace_back(0, 0); + for (auto& chunk : chunks) { + diagram.emplace_back(diagram.back() + chunk); + } + return diagram; +} + + +/** Evaluate a diagram at a specific size, returning the fee as a fraction. + * + * Fees in diagram cannot exceed 2^32, as the returned evaluation could overflow + * the FeeFrac::fee field in the result. */ +FeeFrac EvaluateDiagram(int32_t size, Span<const FeeFrac> diagram) +{ + assert(diagram.size() > 0); + unsigned not_above = 0; + unsigned not_below = diagram.size() - 1; + // If outside the range of diagram, extend begin/end. + if (size < diagram[not_above].size) return {diagram[not_above].fee, 1}; + if (size > diagram[not_below].size) return {diagram[not_below].fee, 1}; + // Perform bisection search to locate the diagram segment that size is in. + while (not_below > not_above + 1) { + unsigned mid = (not_below + not_above) / 2; + if (diagram[mid].size <= size) not_above = mid; + if (diagram[mid].size >= size) not_below = mid; + } + // If the size matches a transition point between segments, return its fee. + if (not_below == not_above) return {diagram[not_below].fee, 1}; + // Otherwise, interpolate. + auto dir_coef = diagram[not_below] - diagram[not_above]; + assert(dir_coef.size > 0); + // Let A = diagram[not_above] and B = diagram[not_below] + const auto& point_a = diagram[not_above]; + // We want to return: + // A.fee + (B.fee - A.fee) / (B.size - A.size) * (size - A.size) + // = A.fee + dir_coef.fee / dir_coef.size * (size - A.size) + // = (A.fee * dir_coef.size + dir_coef.fee * (size - A.size)) / dir_coef.size + assert(size >= point_a.size); + return {point_a.fee * dir_coef.size + dir_coef.fee * (size - point_a.size), dir_coef.size}; +} + +std::weak_ordering CompareFeeFracWithDiagram(const FeeFrac& ff, Span<const FeeFrac> diagram) +{ + return FeeRateCompare(FeeFrac{ff.fee, 1}, EvaluateDiagram(ff.size, diagram)); +} + +std::partial_ordering CompareDiagrams(Span<const FeeFrac> dia1, Span<const FeeFrac> dia2) +{ + bool all_ge = true; + bool all_le = true; + for (const auto p1 : dia1) { + auto cmp = CompareFeeFracWithDiagram(p1, dia2); + if (std::is_lt(cmp)) all_ge = false; + if (std::is_gt(cmp)) all_le = false; + } + for (const auto p2 : dia2) { + auto cmp = CompareFeeFracWithDiagram(p2, dia1); + if (std::is_lt(cmp)) all_le = false; + if (std::is_gt(cmp)) all_ge = false; + } + if (all_ge && all_le) return std::partial_ordering::equivalent; + if (all_ge && !all_le) return std::partial_ordering::greater; + if (!all_ge && all_le) return std::partial_ordering::less; + return std::partial_ordering::unordered; +} + +void PopulateChunks(FuzzedDataProvider& fuzzed_data_provider, std::vector<FeeFrac>& chunks) +{ + chunks.clear(); + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 50) + { + chunks.emplace_back(fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(INT32_MIN>>1, INT32_MAX>>1), fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(1, 1000000)); + } + return; +} + +} // namespace + +FUZZ_TARGET(build_and_compare_feerate_diagram) +{ + // Generate a random set of chunks + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + std::vector<FeeFrac> chunks1, chunks2; + FeeFrac empty{0, 0}; + + PopulateChunks(fuzzed_data_provider, chunks1); + PopulateChunks(fuzzed_data_provider, chunks2); + + std::vector<FeeFrac> diagram1{BuildDiagramFromChunks(chunks1)}; + std::vector<FeeFrac> diagram2{BuildDiagramFromChunks(chunks2)}; + + assert(diagram1.front() == empty); + assert(diagram2.front() == empty); + + auto real = CompareChunks(chunks1, chunks2); + auto sim = CompareDiagrams(diagram1, diagram2); + assert(real == sim); + + // Do explicit evaluation at up to 1000 points, and verify consistency with the result. + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 1000) { + int32_t size = fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(0, diagram2.back().size); + auto eval1 = EvaluateDiagram(size, diagram1); + auto eval2 = EvaluateDiagram(size, diagram2); + auto cmp = FeeRateCompare(eval1, eval2); + if (std::is_lt(cmp)) assert(!std::is_gt(real)); + if (std::is_gt(cmp)) assert(!std::is_lt(real)); + } +} diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index 6de480ff15..a8e490b459 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -35,6 +35,8 @@ __AFL_FUZZ_INIT(); const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; +const std::function<std::string()> G_TEST_GET_FULL_NAME{}; + /** * A copy of the command line arguments that start with `--`. * First `LLVMFuzzerInitialize()` is called, which saves the arguments to `g_args`. @@ -81,7 +83,7 @@ static const TypeTestOneInput* g_test_one_input{nullptr}; void initialize() { // Terminate immediately if a fuzzing harness ever tries to create a TCP socket. - CreateSock = [](const CService&) -> std::unique_ptr<Sock> { std::terminate(); }; + CreateSock = [](const sa_family_t&) -> std::unique_ptr<Sock> { std::terminate(); }; // Terminate immediately if a fuzzing harness ever tries to perform a DNS lookup. g_dns_lookup = [](const std::string& name, bool allow_lookup) { diff --git a/src/test/fuzz/net_permissions.cpp b/src/test/fuzz/net_permissions.cpp index 6ea2139c46..811c0de4b9 100644 --- a/src/test/fuzz/net_permissions.cpp +++ b/src/test/fuzz/net_permissions.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <net_permissions.h> +#include <netbase.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> @@ -31,8 +32,9 @@ FUZZ_TARGET(net_permissions) } NetWhitelistPermissions net_whitelist_permissions; + ConnectionDirection connection_direction; bilingual_str error_net_whitelist_permissions; - if (NetWhitelistPermissions::TryParse(s, net_whitelist_permissions, error_net_whitelist_permissions)) { + if (NetWhitelistPermissions::TryParse(s, net_whitelist_permissions, connection_direction, error_net_whitelist_permissions)) { (void)NetPermissions::ToStrings(net_whitelist_permissions.m_flags); (void)NetPermissions::AddFlag(net_whitelist_permissions.m_flags, net_permission_flags); assert(NetPermissions::HasFlag(net_whitelist_permissions.m_flags, net_permission_flags)); diff --git a/src/test/fuzz/p2p_transport_serialization.cpp b/src/test/fuzz/p2p_transport_serialization.cpp index a205ce19f4..1b7a732260 100644 --- a/src/test/fuzz/p2p_transport_serialization.cpp +++ b/src/test/fuzz/p2p_transport_serialization.cpp @@ -354,6 +354,7 @@ std::unique_ptr<Transport> MakeV2Transport(NodeId nodeid, bool initiator, RNG& r } else { // If it's longer, generate it from the RNG. This avoids having large amounts of // (hopefully) irrelevant data needing to be stored in the fuzzer data. + garb.resize(garb_len); for (auto& v : garb) v = uint8_t(rng()); } // Retrieve entropy diff --git a/src/test/fuzz/package_eval.cpp b/src/test/fuzz/package_eval.cpp index a48ce37bce..c201118bce 100644 --- a/src/test/fuzz/package_eval.cpp +++ b/src/test/fuzz/package_eval.cpp @@ -276,8 +276,14 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) // (the package is a test accept and ATMP is a submission). auto single_submit = txs.size() == 1 && fuzzed_data_provider.ConsumeBool(); + // Exercise client_maxfeerate logic + std::optional<CFeeRate> client_maxfeerate{}; + if (fuzzed_data_provider.ConsumeBool()) { + client_maxfeerate = CFeeRate(fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(-1, 50 * COIN), 100); + } + const auto result_package = WITH_LOCK(::cs_main, - return ProcessNewPackage(chainstate, tx_pool, txs, /*test_accept=*/single_submit)); + return ProcessNewPackage(chainstate, tx_pool, txs, /*test_accept=*/single_submit, client_maxfeerate)); // Always set bypass_limits to false because it is not supported in ProcessNewPackage and // can be a source of divergence. diff --git a/src/test/fuzz/partially_downloaded_block.cpp b/src/test/fuzz/partially_downloaded_block.cpp index 4a4b46da60..ab75afe066 100644 --- a/src/test/fuzz/partially_downloaded_block.cpp +++ b/src/test/fuzz/partially_downloaded_block.cpp @@ -60,7 +60,7 @@ FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) // The coinbase is always available available.insert(0); - std::vector<std::pair<uint256, CTransactionRef>> extra_txn; + std::vector<CTransactionRef> extra_txn; for (size_t i = 1; i < block->vtx.size(); ++i) { auto tx{block->vtx[i]}; @@ -68,7 +68,7 @@ FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) bool add_to_mempool{fuzzed_data_provider.ConsumeBool()}; if (add_to_extra_txn) { - extra_txn.emplace_back(tx->GetWitnessHash(), tx); + extra_txn.emplace_back(tx); available.insert(i); } diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index aa6385d12d..64785948f6 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -23,12 +23,30 @@ namespace { const BasicTestingSetup* g_setup; } // namespace +const int NUM_ITERS = 10000; + +std::vector<COutPoint> g_outpoints; + void initialize_rbf() { static const auto testing_setup = MakeNoLogFileContext<>(); g_setup = testing_setup.get(); } +void initialize_package_rbf() +{ + static const auto testing_setup = MakeNoLogFileContext<>(); + g_setup = testing_setup.get(); + + // Create a fixed set of unique "UTXOs" to source parents from + // to avoid fuzzer giving circular references + for (int i = 0; i < NUM_ITERS; ++i) { + g_outpoints.emplace_back(); + g_outpoints.back().n = i; + } + +} + FUZZ_TARGET(rbf, .init = initialize_rbf) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); @@ -40,7 +58,7 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), NUM_ITERS) { const std::optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!another_mtx) { @@ -63,3 +81,124 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) (void)IsRBFOptIn(tx, pool); } } + +FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + SetMockTime(ConsumeTime(fuzzed_data_provider)); + + std::optional<CMutableTransaction> child = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); + if (!child) return; + + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + + // Add a bunch of parent-child pairs to the mempool, and remember them. + std::vector<CTransaction> mempool_txs; + size_t iter{0}; + + int32_t replacement_vsize = fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(1, 1000000); + + // Keep track of the total vsize of CTxMemPoolEntry's being added to the mempool to avoid overflow + // Add replacement_vsize since this is added to new diagram during RBF check + int64_t running_vsize_total{replacement_vsize}; + + LOCK2(cs_main, pool.cs); + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), NUM_ITERS) + { + // Make sure txns only have one input, and that a unique input is given to avoid circular references + std::optional<CMutableTransaction> parent = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); + if (!parent) { + return; + } + assert(iter <= g_outpoints.size()); + parent->vin.resize(1); + parent->vin[0].prevout = g_outpoints[iter++]; + + mempool_txs.emplace_back(*parent); + const auto parent_entry = ConsumeTxMemPoolEntry(fuzzed_data_provider, mempool_txs.back()); + running_vsize_total += parent_entry.GetTxSize(); + if (running_vsize_total > std::numeric_limits<int32_t>::max()) { + // We aren't adding this final tx to mempool, so we don't want to conflict with it + mempool_txs.pop_back(); + break; + } + pool.addUnchecked(parent_entry); + if (fuzzed_data_provider.ConsumeBool() && !child->vin.empty()) { + child->vin[0].prevout = COutPoint{mempool_txs.back().GetHash(), 0}; + } + mempool_txs.emplace_back(*child); + const auto child_entry = ConsumeTxMemPoolEntry(fuzzed_data_provider, mempool_txs.back()); + running_vsize_total += child_entry.GetTxSize(); + if (running_vsize_total > std::numeric_limits<int32_t>::max()) { + // We aren't adding this final tx to mempool, so we don't want to conflict with it + mempool_txs.pop_back(); + break; + } + pool.addUnchecked(child_entry); + + if (fuzzed_data_provider.ConsumeBool()) { + pool.PrioritiseTransaction(mempool_txs.back().GetHash().ToUint256(), fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(-100000, 100000)); + } + } + + // Pick some transactions at random to be the direct conflicts + CTxMemPool::setEntries direct_conflicts; + for (auto& tx : mempool_txs) { + if (fuzzed_data_provider.ConsumeBool()) { + direct_conflicts.insert(*pool.GetIter(tx.GetHash())); + } + } + + // Calculate all conflicts: + CTxMemPool::setEntries all_conflicts; + for (auto& txiter : direct_conflicts) { + pool.CalculateDescendants(txiter, all_conflicts); + } + + // Calculate the chunks for a replacement. + CAmount replacement_fees = ConsumeMoney(fuzzed_data_provider); + auto calc_results{pool.CalculateChunksForRBF(replacement_fees, replacement_vsize, direct_conflicts, all_conflicts)}; + + if (calc_results.has_value()) { + // Sanity checks on the chunks. + + // Feerates are monotonically decreasing. + FeeFrac first_sum; + for (size_t i = 0; i < calc_results->first.size(); ++i) { + first_sum += calc_results->first[i]; + if (i) assert(!(calc_results->first[i - 1] << calc_results->first[i])); + } + FeeFrac second_sum; + for (size_t i = 0; i < calc_results->second.size(); ++i) { + second_sum += calc_results->second[i]; + if (i) assert(!(calc_results->second[i - 1] << calc_results->second[i])); + } + + FeeFrac replaced; + for (auto txiter : all_conflicts) { + replaced.fee += txiter->GetModifiedFee(); + replaced.size += txiter->GetTxSize(); + } + // The total fee & size of the new diagram minus replaced fee & size should be the total + // fee & size of the old diagram minus replacement fee & size. + assert((first_sum - replaced) == (second_sum - FeeFrac{replacement_fees, replacement_vsize})); + } + + // If internals report error, wrapper should too + auto err_tuple{ImprovesFeerateDiagram(pool, direct_conflicts, all_conflicts, replacement_fees, replacement_vsize)}; + if (!calc_results.has_value()) { + assert(err_tuple.value().first == DiagramCheckError::UNCALCULABLE); + } else { + // Diagram check succeeded + auto old_sum = std::accumulate(calc_results->first.begin(), calc_results->first.end(), FeeFrac{}); + auto new_sum = std::accumulate(calc_results->second.begin(), calc_results->second.end(), FeeFrac{}); + if (!err_tuple.has_value()) { + // New diagram's final fee should always match or exceed old diagram's + assert(old_sum.fee <= new_sum.fee); + } else if (old_sum.fee > new_sum.fee) { + // Or it failed, and if old diagram had higher fees, it should be a failure + assert(err_tuple.value().first == DiagramCheckError::FAILURE); + } + } +} diff --git a/src/test/fuzz/script_bitcoin_consensus.cpp b/src/test/fuzz/script_bitcoin_consensus.cpp deleted file mode 100644 index 846389863d..0000000000 --- a/src/test/fuzz/script_bitcoin_consensus.cpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <script/bitcoinconsensus.h> -#include <script/interpreter.h> -#include <test/fuzz/FuzzedDataProvider.h> -#include <test/fuzz/fuzz.h> -#include <test/fuzz/util.h> - -#include <cstdint> -#include <string> -#include <vector> - -FUZZ_TARGET(script_bitcoin_consensus) -{ - FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const std::vector<uint8_t> random_bytes_1 = ConsumeRandomLengthByteVector(fuzzed_data_provider); - const std::vector<uint8_t> random_bytes_2 = ConsumeRandomLengthByteVector(fuzzed_data_provider); - const CAmount money = ConsumeMoney(fuzzed_data_provider); - bitcoinconsensus_error err; - bitcoinconsensus_error* err_p = fuzzed_data_provider.ConsumeBool() ? &err : nullptr; - const unsigned int n_in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); - const unsigned int flags = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); - assert(bitcoinconsensus_version() == BITCOINCONSENSUS_API_VER); - if ((flags & SCRIPT_VERIFY_WITNESS) != 0 && (flags & SCRIPT_VERIFY_P2SH) == 0) { - return; - } - (void)bitcoinconsensus_verify_script(random_bytes_1.data(), random_bytes_1.size(), random_bytes_2.data(), random_bytes_2.size(), n_in, flags, err_p); - (void)bitcoinconsensus_verify_script_with_amount(random_bytes_1.data(), random_bytes_1.size(), money, random_bytes_2.data(), random_bytes_2.size(), n_in, flags, err_p); - - std::vector<UTXO> spent_outputs; - std::vector<std::vector<unsigned char>> spent_spks; - if (n_in <= 24386) { - spent_outputs.reserve(n_in); - spent_spks.reserve(n_in); - for (size_t i = 0; i < n_in; ++i) { - spent_spks.push_back(ConsumeRandomLengthByteVector(fuzzed_data_provider)); - const CAmount value{ConsumeMoney(fuzzed_data_provider)}; - const auto spk_size{static_cast<unsigned>(spent_spks.back().size())}; - spent_outputs.push_back({.scriptPubKey = spent_spks.back().data(), .scriptPubKeySize = spk_size, .value = value}); - } - } - - const auto spent_outs_size{static_cast<unsigned>(spent_outputs.size())}; - - (void)bitcoinconsensus_verify_script_with_spent_outputs( - random_bytes_1.data(), random_bytes_1.size(), money, random_bytes_2.data(), random_bytes_2.size(), - spent_outputs.data(), spent_outs_size, n_in, flags, err_p); -} diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index e81efac6e0..631da13803 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -90,7 +90,7 @@ FUZZ_TARGET(string) (void)ToUpper(random_string_1); (void)TrimString(random_string_1); (void)TrimString(random_string_1, random_string_2); - (void)urlDecode(random_string_1); + (void)UrlDecode(random_string_1); (void)ContainsNoNUL(random_string_1); (void)_(random_string_1.c_str()); try { diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index b6ba612a84..0b4019d5eb 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -291,7 +291,7 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) // Make sure ProcessNewPackage on one transaction works. // The result is not guaranteed to be the same as what is returned by ATMP. const auto result_package = WITH_LOCK(::cs_main, - return ProcessNewPackage(chainstate, tx_pool, {tx}, true)); + return ProcessNewPackage(chainstate, tx_pool, {tx}, true, /*client_maxfeerate=*/{})); // If something went wrong due to a package-specific policy, it might not return a // validation result for the transaction. if (result_package.m_state.GetResult() != PackageValidationResult::PCKG_POLICY) { diff --git a/src/test/fuzz/util/descriptor.cpp b/src/test/fuzz/util/descriptor.cpp index df78bdf314..0fed2bc5e1 100644 --- a/src/test/fuzz/util/descriptor.cpp +++ b/src/test/fuzz/util/descriptor.cpp @@ -15,7 +15,7 @@ void MockedDescriptorConverter::Init() { // an extended one. if (IdIsCompPubKey(i) || IdIsUnCompPubKey(i) || IdIsXOnlyPubKey(i) || IdIsConstPrivKey(i)) { CKey privkey; - privkey.Set(UCharCast(key_data.begin()), UCharCast(key_data.end()), !IdIsUnCompPubKey(i)); + privkey.Set(key_data.begin(), key_data.end(), !IdIsUnCompPubKey(i)); if (IdIsCompPubKey(i) || IdIsUnCompPubKey(i)) { CPubKey pubkey{privkey.GetPubKey()}; keys_str[i] = HexStr(pubkey); diff --git a/src/test/i2p_tests.cpp b/src/test/i2p_tests.cpp index f80f07d190..d7249d88f4 100644 --- a/src/test/i2p_tests.cpp +++ b/src/test/i2p_tests.cpp @@ -6,6 +6,7 @@ #include <i2p.h> #include <logging.h> #include <netaddress.h> +#include <netbase.h> #include <test/util/logging.h> #include <test/util/net.h> #include <test/util/setup_common.h> @@ -38,7 +39,7 @@ public: private: const BCLog::Level m_prev_log_level; - const std::function<std::unique_ptr<Sock>(const CService&)> m_create_sock_orig; + const std::function<std::unique_ptr<Sock>(const sa_family_t&)> m_create_sock_orig; }; BOOST_FIXTURE_TEST_SUITE(i2p_tests, EnvTestingSetup) @@ -46,12 +47,14 @@ BOOST_FIXTURE_TEST_SUITE(i2p_tests, EnvTestingSetup) BOOST_AUTO_TEST_CASE(unlimited_recv) { // Mock CreateSock() to create MockSock. - CreateSock = [](const CService&) { + CreateSock = [](const sa_family_t&) { return std::make_unique<StaticContentsSock>(std::string(i2p::sam::MAX_MSG_SIZE + 1, 'a')); }; CThreadInterrupt interrupt; - i2p::sam::Session session(gArgs.GetDataDirNet() / "test_i2p_private_key", CService{}, &interrupt); + const std::optional<CService> addr{Lookup("127.0.0.1", 9000, false)}; + const Proxy sam_proxy(addr.value(), false); + i2p::sam::Session session(gArgs.GetDataDirNet() / "test_i2p_private_key", sam_proxy, &interrupt); { ASSERT_DEBUG_LOG("Creating persistent SAM session"); @@ -66,7 +69,7 @@ BOOST_AUTO_TEST_CASE(unlimited_recv) BOOST_AUTO_TEST_CASE(listen_ok_accept_fail) { size_t num_sockets{0}; - CreateSock = [&num_sockets](const CService&) { + CreateSock = [&num_sockets](const sa_family_t&) { // clang-format off ++num_sockets; // First socket is the control socket for creating the session. @@ -111,8 +114,10 @@ BOOST_AUTO_TEST_CASE(listen_ok_accept_fail) }; CThreadInterrupt interrupt; + const CService addr{in6_addr(IN6ADDR_LOOPBACK_INIT), /*port=*/7656}; + const Proxy sam_proxy(addr, false); i2p::sam::Session session(gArgs.GetDataDirNet() / "test_i2p_private_key", - CService{in6_addr(IN6ADDR_LOOPBACK_INIT), /*port=*/7656}, + sam_proxy, &interrupt); i2p::Connection conn; @@ -130,7 +135,7 @@ BOOST_AUTO_TEST_CASE(damaged_private_key) { const auto CreateSockOrig = CreateSock; - CreateSock = [](const CService&) { + CreateSock = [](const sa_family_t&) { return std::make_unique<StaticContentsSock>("HELLO REPLY RESULT=OK VERSION=3.1\n" "SESSION STATUS RESULT=OK DESTINATION=\n"); }; @@ -154,7 +159,9 @@ BOOST_AUTO_TEST_CASE(damaged_private_key) BOOST_REQUIRE(WriteBinaryFile(i2p_private_key_file, file_contents)); CThreadInterrupt interrupt; - i2p::sam::Session session(i2p_private_key_file, CService{}, &interrupt); + const CService addr{in6_addr(IN6ADDR_LOOPBACK_INIT), /*port=*/7656}; + const Proxy sam_proxy{addr, false}; + i2p::sam::Session session(i2p_private_key_file, sam_proxy, &interrupt); { ASSERT_DEBUG_LOG("Creating persistent SAM session"); diff --git a/src/test/main.cpp b/src/test/main.cpp index 0809f83c93..67740ece93 100644 --- a/src/test/main.cpp +++ b/src/test/main.cpp @@ -39,3 +39,10 @@ const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS = } return args; }; + +/** + * Retrieve the boost unit test name. + */ +const std::function<std::string()> G_TEST_GET_FULL_NAME = []() { + return boost::unit_test::framework::current_test_case().full_name(); +}; diff --git a/src/test/miniscript_tests.cpp b/src/test/miniscript_tests.cpp index 1f28e61bc9..a3699f84b6 100644 --- a/src/test/miniscript_tests.cpp +++ b/src/test/miniscript_tests.cpp @@ -297,6 +297,7 @@ using miniscript::operator"" _mst; using Node = miniscript::Node<CPubKey>; /** Compute all challenges (pubkeys, hashes, timelocks) that occur in a given Miniscript. */ +// NOLINTNEXTLINE(misc-no-recursion) std::set<Challenge> FindChallenges(const NodeRef& ref) { std::set<Challenge> chal; for (const auto& key : ref->keys) { diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index fa70f62eb4..3422cb8023 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -366,6 +366,7 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) bilingual_str error; NetWhitebindPermissions whitebindPermissions; NetWhitelistPermissions whitelistPermissions; + ConnectionDirection connection_direction; // Detect invalid white bind BOOST_CHECK(!NetWhitebindPermissions::TryParse("", whitebindPermissions, error)); @@ -435,24 +436,33 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) BOOST_CHECK(NetWhitebindPermissions::TryParse(",,@1.2.3.4:32", whitebindPermissions, error)); BOOST_CHECK_EQUAL(whitebindPermissions.m_flags, NetPermissionFlags::None); + BOOST_CHECK(!NetWhitebindPermissions::TryParse("out,forcerelay@1.2.3.4:32", whitebindPermissions, error)); + BOOST_CHECK(error.original.find("whitebind may only be used for incoming connections (\"out\" was passed)") != std::string::npos); + // Detect invalid flag BOOST_CHECK(!NetWhitebindPermissions::TryParse("bloom,forcerelay,oopsie@1.2.3.4:32", whitebindPermissions, error)); BOOST_CHECK(error.original.find("Invalid P2P permission") != std::string::npos); // Check netmask error - BOOST_CHECK(!NetWhitelistPermissions::TryParse("bloom,forcerelay,noban@1.2.3.4:32", whitelistPermissions, error)); + BOOST_CHECK(!NetWhitelistPermissions::TryParse("bloom,forcerelay,noban@1.2.3.4:32", whitelistPermissions, connection_direction, error)); BOOST_CHECK(error.original.find("Invalid netmask specified in -whitelist") != std::string::npos); // Happy path for whitelist parsing - BOOST_CHECK(NetWhitelistPermissions::TryParse("noban@1.2.3.4", whitelistPermissions, error)); + BOOST_CHECK(NetWhitelistPermissions::TryParse("noban@1.2.3.4", whitelistPermissions, connection_direction, error)); BOOST_CHECK_EQUAL(whitelistPermissions.m_flags, NetPermissionFlags::NoBan); BOOST_CHECK(NetPermissions::HasFlag(whitelistPermissions.m_flags, NetPermissionFlags::NoBan)); - BOOST_CHECK(NetWhitelistPermissions::TryParse("bloom,forcerelay,noban,relay@1.2.3.4/32", whitelistPermissions, error)); + BOOST_CHECK(NetWhitelistPermissions::TryParse("bloom,forcerelay,noban,relay@1.2.3.4/32", whitelistPermissions, connection_direction, error)); BOOST_CHECK_EQUAL(whitelistPermissions.m_flags, NetPermissionFlags::BloomFilter | NetPermissionFlags::ForceRelay | NetPermissionFlags::NoBan | NetPermissionFlags::Relay); BOOST_CHECK(error.empty()); BOOST_CHECK_EQUAL(whitelistPermissions.m_subnet.ToString(), "1.2.3.4/32"); - BOOST_CHECK(NetWhitelistPermissions::TryParse("bloom,forcerelay,noban,relay,mempool@1.2.3.4/32", whitelistPermissions, error)); + BOOST_CHECK(NetWhitelistPermissions::TryParse("bloom,forcerelay,noban,relay,mempool@1.2.3.4/32", whitelistPermissions, connection_direction, error)); + BOOST_CHECK(NetWhitelistPermissions::TryParse("in,relay@1.2.3.4", whitelistPermissions, connection_direction, error)); + BOOST_CHECK_EQUAL(connection_direction, ConnectionDirection::In); + BOOST_CHECK(NetWhitelistPermissions::TryParse("out,bloom@1.2.3.4", whitelistPermissions, connection_direction, error)); + BOOST_CHECK_EQUAL(connection_direction, ConnectionDirection::Out); + BOOST_CHECK(NetWhitelistPermissions::TryParse("in,out,bloom@1.2.3.4", whitelistPermissions, connection_direction, error)); + BOOST_CHECK_EQUAL(connection_direction, ConnectionDirection::Both); const auto strings = NetPermissions::ToStrings(NetPermissionFlags::All); BOOST_CHECK_EQUAL(strings.size(), 7U); diff --git a/src/test/rbf_tests.cpp b/src/test/rbf_tests.cpp index e6c135eed9..19e45c550a 100644 --- a/src/test/rbf_tests.cpp +++ b/src/test/rbf_tests.cpp @@ -37,7 +37,38 @@ static inline CTransactionRef make_tx(const std::vector<CTransactionRef>& inputs return MakeTransactionRef(tx); } -static void add_descendants(const CTransactionRef& tx, int32_t num_descendants, CTxMemPool& pool) +// Make two child transactions from parent (which must have at least 2 outputs). +// Each tx will have the same outputs, using the amounts specified in output_values. +static inline std::pair<CTransactionRef, CTransactionRef> make_two_siblings(const CTransactionRef parent, + const std::vector<CAmount>& output_values) +{ + assert(parent->vout.size() >= 2); + + // First tx takes first parent output + CMutableTransaction tx1 = CMutableTransaction(); + tx1.vin.resize(1); + tx1.vout.resize(output_values.size()); + + tx1.vin[0].prevout.hash = parent->GetHash(); + tx1.vin[0].prevout.n = 0; + // Add a witness so wtxid != txid + CScriptWitness witness; + witness.stack.emplace_back(10); + tx1.vin[0].scriptWitness = witness; + + for (size_t i = 0; i < output_values.size(); ++i) { + tx1.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx1.vout[i].nValue = output_values[i]; + } + + // Second tx takes second parent output + CMutableTransaction tx2 = tx1; + tx2.vin[0].prevout.n = 1; + + return std::make_pair(MakeTransactionRef(tx1), MakeTransactionRef(tx2)); +} + +static CTransactionRef add_descendants(const CTransactionRef& tx, int32_t num_descendants, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, pool.cs) { AssertLockHeld(::cs_main); @@ -50,6 +81,35 @@ static void add_descendants(const CTransactionRef& tx, int32_t num_descendants, pool.addUnchecked(entry.FromTx(next_tx)); tx_to_spend = next_tx; } + // Return last created tx + return tx_to_spend; +} + +static CTransactionRef add_descendant_to_parents(const std::vector<CTransactionRef>& parents, CTxMemPool& pool) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main, pool.cs) +{ + AssertLockHeld(::cs_main); + AssertLockHeld(pool.cs); + TestMemPoolEntryHelper entry; + // Assumes this isn't already spent in mempool + auto child_tx = make_tx(/*inputs=*/parents, /*output_values=*/{50 * CENT}); + pool.addUnchecked(entry.FromTx(child_tx)); + // Return last created tx + return child_tx; +} + +// Makes two children for a single parent +static std::pair<CTransactionRef, CTransactionRef> add_children_to_parent(const CTransactionRef parent, CTxMemPool& pool) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main, pool.cs) +{ + AssertLockHeld(::cs_main); + AssertLockHeld(pool.cs); + TestMemPoolEntryHelper entry; + // Assumes this isn't already spent in mempool + auto children_tx = make_two_siblings(/*parent=*/parent, /*output_values=*/{50 * CENT}); + pool.addUnchecked(entry.FromTx(children_tx.first)); + pool.addUnchecked(entry.FromTx(children_tx.second)); + return children_tx; } BOOST_FIXTURE_TEST_CASE(rbf_helper_functions, TestChain100Setup) @@ -89,28 +149,51 @@ BOOST_FIXTURE_TEST_CASE(rbf_helper_functions, TestChain100Setup) const auto tx8 = make_tx(/*inputs=*/ {m_coinbase_txns[4]}, /*output_values=*/ {999 * CENT}); pool.addUnchecked(entry.Fee(high_fee).FromTx(tx8)); - const auto entry1 = pool.GetIter(tx1->GetHash()).value(); - const auto entry2 = pool.GetIter(tx2->GetHash()).value(); - const auto entry3 = pool.GetIter(tx3->GetHash()).value(); - const auto entry4 = pool.GetIter(tx4->GetHash()).value(); - const auto entry5 = pool.GetIter(tx5->GetHash()).value(); - const auto entry6 = pool.GetIter(tx6->GetHash()).value(); - const auto entry7 = pool.GetIter(tx7->GetHash()).value(); - const auto entry8 = pool.GetIter(tx8->GetHash()).value(); - - BOOST_CHECK_EQUAL(entry1->GetFee(), normal_fee); - BOOST_CHECK_EQUAL(entry2->GetFee(), normal_fee); - BOOST_CHECK_EQUAL(entry3->GetFee(), low_fee); - BOOST_CHECK_EQUAL(entry4->GetFee(), high_fee); - BOOST_CHECK_EQUAL(entry5->GetFee(), low_fee); - BOOST_CHECK_EQUAL(entry6->GetFee(), low_fee); - BOOST_CHECK_EQUAL(entry7->GetFee(), high_fee); - BOOST_CHECK_EQUAL(entry8->GetFee(), high_fee); - - CTxMemPool::setEntries set_12_normal{entry1, entry2}; - CTxMemPool::setEntries set_34_cpfp{entry3, entry4}; - CTxMemPool::setEntries set_56_low{entry5, entry6}; - CTxMemPool::setEntries all_entries{entry1, entry2, entry3, entry4, entry5, entry6, entry7, entry8}; + // Normal txs, will chain txns right before CheckConflictTopology test + const auto tx9 = make_tx(/*inputs=*/ {m_coinbase_txns[5]}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx9)); + const auto tx10 = make_tx(/*inputs=*/ {m_coinbase_txns[6]}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx10)); + + // Will make these two parents of single child + const auto tx11 = make_tx(/*inputs=*/ {m_coinbase_txns[7]}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx11)); + const auto tx12 = make_tx(/*inputs=*/ {m_coinbase_txns[8]}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx12)); + + // Will make two children of this single parent + const auto tx13 = make_tx(/*inputs=*/ {m_coinbase_txns[9]}, /*output_values=*/ {995 * CENT, 995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx13)); + + const auto entry1_normal = pool.GetIter(tx1->GetHash()).value(); + const auto entry2_normal = pool.GetIter(tx2->GetHash()).value(); + const auto entry3_low = pool.GetIter(tx3->GetHash()).value(); + const auto entry4_high = pool.GetIter(tx4->GetHash()).value(); + const auto entry5_low = pool.GetIter(tx5->GetHash()).value(); + const auto entry6_low_prioritised = pool.GetIter(tx6->GetHash()).value(); + const auto entry7_high = pool.GetIter(tx7->GetHash()).value(); + const auto entry8_high = pool.GetIter(tx8->GetHash()).value(); + const auto entry9_unchained = pool.GetIter(tx9->GetHash()).value(); + const auto entry10_unchained = pool.GetIter(tx10->GetHash()).value(); + const auto entry11_unchained = pool.GetIter(tx11->GetHash()).value(); + const auto entry12_unchained = pool.GetIter(tx12->GetHash()).value(); + const auto entry13_unchained = pool.GetIter(tx13->GetHash()).value(); + + BOOST_CHECK_EQUAL(entry1_normal->GetFee(), normal_fee); + BOOST_CHECK_EQUAL(entry2_normal->GetFee(), normal_fee); + BOOST_CHECK_EQUAL(entry3_low->GetFee(), low_fee); + BOOST_CHECK_EQUAL(entry4_high->GetFee(), high_fee); + BOOST_CHECK_EQUAL(entry5_low->GetFee(), low_fee); + BOOST_CHECK_EQUAL(entry6_low_prioritised->GetFee(), low_fee); + BOOST_CHECK_EQUAL(entry7_high->GetFee(), high_fee); + BOOST_CHECK_EQUAL(entry8_high->GetFee(), high_fee); + + CTxMemPool::setEntries set_12_normal{entry1_normal, entry2_normal}; + CTxMemPool::setEntries set_34_cpfp{entry3_low, entry4_high}; + CTxMemPool::setEntries set_56_low{entry5_low, entry6_low_prioritised}; + CTxMemPool::setEntries set_78_high{entry7_high, entry8_high}; + CTxMemPool::setEntries all_entries{entry1_normal, entry2_normal, entry3_low, entry4_high, + entry5_low, entry6_low_prioritised, entry7_high, entry8_high}; CTxMemPool::setEntries empty_set; const auto unused_txid{GetRandHash()}; @@ -118,29 +201,29 @@ BOOST_FIXTURE_TEST_CASE(rbf_helper_functions, TestChain100Setup) // Tests for PaysMoreThanConflicts // These tests use feerate, not absolute fee. BOOST_CHECK(PaysMoreThanConflicts(/*iters_conflicting=*/set_12_normal, - /*replacement_feerate=*/CFeeRate(entry1->GetModifiedFee() + 1, entry1->GetTxSize() + 2), + /*replacement_feerate=*/CFeeRate(entry1_normal->GetModifiedFee() + 1, entry1_normal->GetTxSize() + 2), /*txid=*/unused_txid).has_value()); // Replacement must be strictly greater than the originals. - BOOST_CHECK(PaysMoreThanConflicts(set_12_normal, CFeeRate(entry1->GetModifiedFee(), entry1->GetTxSize()), unused_txid).has_value()); - BOOST_CHECK(PaysMoreThanConflicts(set_12_normal, CFeeRate(entry1->GetModifiedFee() + 1, entry1->GetTxSize()), unused_txid) == std::nullopt); + BOOST_CHECK(PaysMoreThanConflicts(set_12_normal, CFeeRate(entry1_normal->GetModifiedFee(), entry1_normal->GetTxSize()), unused_txid).has_value()); + BOOST_CHECK(PaysMoreThanConflicts(set_12_normal, CFeeRate(entry1_normal->GetModifiedFee() + 1, entry1_normal->GetTxSize()), unused_txid) == std::nullopt); // These tests use modified fees (including prioritisation), not base fees. - BOOST_CHECK(PaysMoreThanConflicts({entry5}, CFeeRate(entry5->GetModifiedFee() + 1, entry5->GetTxSize()), unused_txid) == std::nullopt); - BOOST_CHECK(PaysMoreThanConflicts({entry6}, CFeeRate(entry6->GetFee() + 1, entry6->GetTxSize()), unused_txid).has_value()); - BOOST_CHECK(PaysMoreThanConflicts({entry6}, CFeeRate(entry6->GetModifiedFee() + 1, entry6->GetTxSize()), unused_txid) == std::nullopt); + BOOST_CHECK(PaysMoreThanConflicts({entry5_low}, CFeeRate(entry5_low->GetModifiedFee() + 1, entry5_low->GetTxSize()), unused_txid) == std::nullopt); + BOOST_CHECK(PaysMoreThanConflicts({entry6_low_prioritised}, CFeeRate(entry6_low_prioritised->GetFee() + 1, entry6_low_prioritised->GetTxSize()), unused_txid).has_value()); + BOOST_CHECK(PaysMoreThanConflicts({entry6_low_prioritised}, CFeeRate(entry6_low_prioritised->GetModifiedFee() + 1, entry6_low_prioritised->GetTxSize()), unused_txid) == std::nullopt); // PaysMoreThanConflicts checks individual feerate, not ancestor feerate. This test compares - // replacement_feerate and entry4's feerate, which are the same. The replacement_feerate is - // considered too low even though entry4 has a low ancestor feerate. - BOOST_CHECK(PaysMoreThanConflicts(set_34_cpfp, CFeeRate(entry4->GetModifiedFee(), entry4->GetTxSize()), unused_txid).has_value()); + // replacement_feerate and entry4_high's feerate, which are the same. The replacement_feerate is + // considered too low even though entry4_high has a low ancestor feerate. + BOOST_CHECK(PaysMoreThanConflicts(set_34_cpfp, CFeeRate(entry4_high->GetModifiedFee(), entry4_high->GetTxSize()), unused_txid).has_value()); // Tests for EntriesAndTxidsDisjoint BOOST_CHECK(EntriesAndTxidsDisjoint(empty_set, {tx1->GetHash()}, unused_txid) == std::nullopt); BOOST_CHECK(EntriesAndTxidsDisjoint(set_12_normal, {tx3->GetHash()}, unused_txid) == std::nullopt); - BOOST_CHECK(EntriesAndTxidsDisjoint({entry2}, {tx2->GetHash()}, unused_txid).has_value()); + BOOST_CHECK(EntriesAndTxidsDisjoint({entry2_normal}, {tx2->GetHash()}, unused_txid).has_value()); BOOST_CHECK(EntriesAndTxidsDisjoint(set_12_normal, {tx1->GetHash()}, unused_txid).has_value()); BOOST_CHECK(EntriesAndTxidsDisjoint(set_12_normal, {tx2->GetHash()}, unused_txid).has_value()); // EntriesAndTxidsDisjoint does not calculate descendants of iters_conflicting; it uses whatever - // the caller passed in. As such, no error is returned even though entry2 is a descendant of tx1. - BOOST_CHECK(EntriesAndTxidsDisjoint({entry2}, {tx1->GetHash()}, unused_txid) == std::nullopt); + // the caller passed in. As such, no error is returned even though entry2_normal is a descendant of tx1. + BOOST_CHECK(EntriesAndTxidsDisjoint({entry2_normal}, {tx1->GetHash()}, unused_txid) == std::nullopt); // Tests for PaysForRBF const CFeeRate incremental_relay_feerate{DEFAULT_INCREMENTAL_RELAY_FEE}; @@ -163,8 +246,8 @@ BOOST_FIXTURE_TEST_CASE(rbf_helper_functions, TestChain100Setup) BOOST_CHECK(PaysForRBF(low_fee, high_fee + 99999999, 99999999, incremental_relay_feerate, unused_txid) == std::nullopt); // Tests for GetEntriesForConflicts - CTxMemPool::setEntries all_parents{entry1, entry3, entry5, entry7, entry8}; - CTxMemPool::setEntries all_children{entry2, entry4, entry6}; + CTxMemPool::setEntries all_parents{entry1_normal, entry3_low, entry5_low, entry7_high, entry8_high}; + CTxMemPool::setEntries all_children{entry2_normal, entry4_high, entry6_low_prioritised}; const std::vector<CTransactionRef> parent_inputs({m_coinbase_txns[0], m_coinbase_txns[1], m_coinbase_txns[2], m_coinbase_txns[3], m_coinbase_txns[4]}); const auto conflicts_with_parents = make_tx(parent_inputs, {50 * CENT}); @@ -215,15 +298,328 @@ BOOST_FIXTURE_TEST_CASE(rbf_helper_functions, TestChain100Setup) BOOST_CHECK(HasNoNewUnconfirmed(/*tx=*/ *spends_unconfirmed.get(), /*pool=*/ pool, /*iters_conflicting=*/ all_entries) == std::nullopt); - BOOST_CHECK(HasNoNewUnconfirmed(*spends_unconfirmed.get(), pool, {entry2}) == std::nullopt); + BOOST_CHECK(HasNoNewUnconfirmed(*spends_unconfirmed.get(), pool, {entry2_normal}) == std::nullopt); BOOST_CHECK(HasNoNewUnconfirmed(*spends_unconfirmed.get(), pool, empty_set).has_value()); const auto spends_new_unconfirmed = make_tx({tx1, tx8}, {36 * CENT}); - BOOST_CHECK(HasNoNewUnconfirmed(*spends_new_unconfirmed.get(), pool, {entry2}).has_value()); + BOOST_CHECK(HasNoNewUnconfirmed(*spends_new_unconfirmed.get(), pool, {entry2_normal}).has_value()); BOOST_CHECK(HasNoNewUnconfirmed(*spends_new_unconfirmed.get(), pool, all_entries).has_value()); const auto spends_conflicting_confirmed = make_tx({m_coinbase_txns[0], m_coinbase_txns[1]}, {45 * CENT}); - BOOST_CHECK(HasNoNewUnconfirmed(*spends_conflicting_confirmed.get(), pool, {entry1, entry3}) == std::nullopt); + BOOST_CHECK(HasNoNewUnconfirmed(*spends_conflicting_confirmed.get(), pool, {entry1_normal, entry3_low}) == std::nullopt); + + // Tests for CheckConflictTopology + + // Tx4 has 23 descendants + BOOST_CHECK_EQUAL(pool.CheckConflictTopology(set_34_cpfp).value(), strprintf("%s has 23 descendants, max 1 allowed", entry4_high->GetSharedTx()->GetHash().ToString())); + + // No descendants yet + BOOST_CHECK(pool.CheckConflictTopology({entry9_unchained}) == std::nullopt); + + // Add 1 descendant, still ok + add_descendants(tx9, 1, pool); + BOOST_CHECK(pool.CheckConflictTopology({entry9_unchained}) == std::nullopt); + + // N direct conflicts; ok + BOOST_CHECK(pool.CheckConflictTopology({entry9_unchained, entry10_unchained, entry11_unchained}) == std::nullopt); + + // Add 1 descendant, still ok, even if it's considered a direct conflict as well + const auto child_tx = add_descendants(tx10, 1, pool); + const auto entry10_child = pool.GetIter(child_tx->GetHash()).value(); + BOOST_CHECK(pool.CheckConflictTopology({entry9_unchained, entry10_unchained, entry11_unchained}) == std::nullopt); + BOOST_CHECK(pool.CheckConflictTopology({entry9_unchained, entry10_unchained, entry11_unchained, entry10_child}) == std::nullopt); + + // One more, size 3 cluster too much + const auto grand_child_tx = add_descendants(child_tx, 1, pool); + const auto entry10_grand_child = pool.GetIter(grand_child_tx->GetHash()).value(); + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry9_unchained, entry10_unchained, entry11_unchained}).value(), strprintf("%s has 2 descendants, max 1 allowed", entry10_unchained->GetSharedTx()->GetHash().ToString())); + // even if direct conflict is descendent itself + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry9_unchained, entry10_grand_child, entry11_unchained}).value(), strprintf("%s has 2 ancestors, max 1 allowed", entry10_grand_child->GetSharedTx()->GetHash().ToString())); + + // Make a single child from two singleton parents + const auto two_parent_child_tx = add_descendant_to_parents({tx11, tx12}, pool); + const auto entry_two_parent_child = pool.GetIter(two_parent_child_tx->GetHash()).value(); + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry11_unchained}).value(), strprintf("%s is not the only parent of child %s", entry11_unchained->GetSharedTx()->GetHash().ToString(), entry_two_parent_child->GetSharedTx()->GetHash().ToString())); + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry12_unchained}).value(), strprintf("%s is not the only parent of child %s", entry12_unchained->GetSharedTx()->GetHash().ToString(), entry_two_parent_child->GetSharedTx()->GetHash().ToString())); + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry_two_parent_child}).value(), strprintf("%s has 2 ancestors, max 1 allowed", entry_two_parent_child->GetSharedTx()->GetHash().ToString())); + + // Single parent with two children, we will conflict with the siblings directly only + const auto two_siblings = add_children_to_parent(tx13, pool); + const auto entry_sibling_1 = pool.GetIter(two_siblings.first->GetHash()).value(); + const auto entry_sibling_2 = pool.GetIter(two_siblings.second->GetHash()).value(); + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry_sibling_1}).value(), strprintf("%s is not the only child of parent %s", entry_sibling_1->GetSharedTx()->GetHash().ToString(), entry13_unchained->GetSharedTx()->GetHash().ToString())); + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry_sibling_2}).value(), strprintf("%s is not the only child of parent %s", entry_sibling_2->GetSharedTx()->GetHash().ToString(), entry13_unchained->GetSharedTx()->GetHash().ToString())); + +} + +BOOST_FIXTURE_TEST_CASE(improves_feerate, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(::cs_main, pool.cs); + TestMemPoolEntryHelper entry; + + const CAmount low_fee{CENT/100}; + const CAmount normal_fee{CENT/10}; + + // low feerate parent with normal feerate child + const auto tx1 = make_tx(/*inputs=*/ {m_coinbase_txns[0]}, /*output_values=*/ {10 * COIN}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx1)); + const auto tx2 = make_tx(/*inputs=*/ {tx1}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx2)); + + const auto entry1 = pool.GetIter(tx1->GetHash()).value(); + const auto tx1_fee = entry1->GetModifiedFee(); + const auto tx1_size = entry1->GetTxSize(); + const auto entry2 = pool.GetIter(tx2->GetHash()).value(); + const auto tx2_fee = entry2->GetModifiedFee(); + const auto tx2_size = entry2->GetTxSize(); + + // Now test ImprovesFeerateDiagram with various levels of "package rbf" feerates + + // It doesn't improve itself + const auto res1 = ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee, tx1_size + tx2_size); + BOOST_CHECK(res1.has_value()); + BOOST_CHECK(res1.value().first == DiagramCheckError::FAILURE); + BOOST_CHECK(res1.value().second == "insufficient feerate: does not improve feerate diagram"); + + // With one more satoshi it does + BOOST_CHECK(ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee + 1, tx1_size + tx2_size) == std::nullopt); + + // With prioritisation of in-mempool conflicts, it affects the results of the comparison using the same args as just above + pool.PrioritiseTransaction(entry1->GetSharedTx()->GetHash(), /*nFeeDelta=*/1); + const auto res2 = ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee + 1, tx1_size + tx2_size); + BOOST_CHECK(res2.has_value()); + BOOST_CHECK(res2.value().first == DiagramCheckError::FAILURE); + BOOST_CHECK(res2.value().second == "insufficient feerate: does not improve feerate diagram"); + pool.PrioritiseTransaction(entry1->GetSharedTx()->GetHash(), /*nFeeDelta=*/-1); + + // With one less vB it does + BOOST_CHECK(ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee, tx1_size + tx2_size - 1) == std::nullopt); + + // Adding a grandchild makes the cluster size 3, which is uncalculable + const auto tx3 = make_tx(/*inputs=*/ {tx2}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx3)); + const auto res3 = ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee + 1, tx1_size + tx2_size); + BOOST_CHECK(res3.has_value()); + BOOST_CHECK(res3.value().first == DiagramCheckError::UNCALCULABLE); + BOOST_CHECK(res3.value().second == strprintf("%s has 2 descendants, max 1 allowed", tx1->GetHash().GetHex())); + +} + +BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(::cs_main, pool.cs); + TestMemPoolEntryHelper entry; + + const CAmount low_fee{CENT/100}; + const CAmount normal_fee{CENT/10}; + const CAmount high_fee{CENT}; + + // low -> high -> medium fee transactions that would result in two chunks together since they + // are all same size + const auto low_tx = make_tx(/*inputs=*/ {m_coinbase_txns[0]}, /*output_values=*/ {10 * COIN}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(low_tx)); + + const auto entry_low = pool.GetIter(low_tx->GetHash()).value(); + const auto low_size = entry_low->GetTxSize(); + + // Replacement of size 1 + { + const auto replace_one{pool.CalculateChunksForRBF(/*replacement_fees=*/0, /*replacement_vsize=*/1, {entry_low}, {entry_low})}; + BOOST_CHECK(replace_one.has_value()); + std::vector<FeeFrac> expected_old_chunks{{low_fee, low_size}}; + BOOST_CHECK(replace_one->first == expected_old_chunks); + std::vector<FeeFrac> expected_new_chunks{{0, 1}}; + BOOST_CHECK(replace_one->second == expected_new_chunks); + } + + // Non-zero replacement fee/size + { + const auto replace_one_fee{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_low}, {entry_low})}; + BOOST_CHECK(replace_one_fee.has_value()); + std::vector<FeeFrac> expected_old_diagram{{low_fee, low_size}}; + BOOST_CHECK(replace_one_fee->first == expected_old_diagram); + std::vector<FeeFrac> expected_new_diagram{{high_fee, low_size}}; + BOOST_CHECK(replace_one_fee->second == expected_new_diagram); + } + + // Add a second transaction to the cluster that will make a single chunk, to be evicted in the RBF + const auto high_tx = make_tx(/*inputs=*/ {low_tx}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(high_fee).FromTx(high_tx)); + const auto entry_high = pool.GetIter(high_tx->GetHash()).value(); + const auto high_size = entry_high->GetTxSize(); + + { + const auto replace_single_chunk{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_low}, {entry_low, entry_high})}; + BOOST_CHECK(replace_single_chunk.has_value()); + std::vector<FeeFrac> expected_old_chunks{{low_fee + high_fee, low_size + high_size}}; + BOOST_CHECK(replace_single_chunk->first == expected_old_chunks); + std::vector<FeeFrac> expected_new_chunks{{high_fee, low_size}}; + BOOST_CHECK(replace_single_chunk->second == expected_new_chunks); + } + + // Conflict with the 2nd tx, resulting in new diagram with three entries + { + const auto replace_cpfp_child{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_high}, {entry_high})}; + BOOST_CHECK(replace_cpfp_child.has_value()); + std::vector<FeeFrac> expected_old_chunks{{low_fee + high_fee, low_size + high_size}}; + BOOST_CHECK(replace_cpfp_child->first == expected_old_chunks); + std::vector<FeeFrac> expected_new_chunks{{high_fee, low_size}, {low_fee, low_size}}; + BOOST_CHECK(replace_cpfp_child->second == expected_new_chunks); + } + + // third transaction causes the topology check to fail + const auto normal_tx = make_tx(/*inputs=*/ {high_tx}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(normal_tx)); + const auto entry_normal = pool.GetIter(normal_tx->GetHash()).value(); + const auto normal_size = entry_normal->GetTxSize(); + + { + const auto replace_too_large{pool.CalculateChunksForRBF(/*replacement_fees=*/normal_fee, /*replacement_vsize=*/normal_size, {entry_low}, {entry_low, entry_high, entry_normal})}; + BOOST_CHECK(!replace_too_large.has_value()); + BOOST_CHECK_EQUAL(util::ErrorString(replace_too_large).original, strprintf("%s has 2 descendants, max 1 allowed", low_tx->GetHash().GetHex())); + } + + // Make a size 2 cluster that is itself two chunks; evict both txns + const auto high_tx_2 = make_tx(/*inputs=*/ {m_coinbase_txns[1]}, /*output_values=*/ {10 * COIN}); + pool.addUnchecked(entry.Fee(high_fee).FromTx(high_tx_2)); + const auto entry_high_2 = pool.GetIter(high_tx_2->GetHash()).value(); + const auto high_size_2 = entry_high_2->GetTxSize(); + + const auto low_tx_2 = make_tx(/*inputs=*/ {high_tx_2}, /*output_values=*/ {9 * COIN}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(low_tx_2)); + const auto entry_low_2 = pool.GetIter(low_tx_2->GetHash()).value(); + const auto low_size_2 = entry_low_2->GetTxSize(); + + { + const auto replace_two_chunks_single_cluster{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_high_2}, {entry_high_2, entry_low_2})}; + BOOST_CHECK(replace_two_chunks_single_cluster.has_value()); + std::vector<FeeFrac> expected_old_chunks{{high_fee, high_size_2}, {low_fee, low_size_2}}; + BOOST_CHECK(replace_two_chunks_single_cluster->first == expected_old_chunks); + std::vector<FeeFrac> expected_new_chunks{{high_fee, low_size_2}}; + BOOST_CHECK(replace_two_chunks_single_cluster->second == expected_new_chunks); + } + + // You can have more than two direct conflicts if the there are multiple affected clusters, all of size 2 or less + const auto conflict_1 = make_tx(/*inputs=*/ {m_coinbase_txns[2]}, /*output_values=*/ {10 * COIN}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(conflict_1)); + const auto conflict_1_entry = pool.GetIter(conflict_1->GetHash()).value(); + + const auto conflict_2 = make_tx(/*inputs=*/ {m_coinbase_txns[3]}, /*output_values=*/ {10 * COIN}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(conflict_2)); + const auto conflict_2_entry = pool.GetIter(conflict_2->GetHash()).value(); + + const auto conflict_3 = make_tx(/*inputs=*/ {m_coinbase_txns[4]}, /*output_values=*/ {10 * COIN}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(conflict_3)); + const auto conflict_3_entry = pool.GetIter(conflict_3->GetHash()).value(); + + { + const auto replace_multiple_clusters{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry})}; + BOOST_CHECK(replace_multiple_clusters.has_value()); + BOOST_CHECK(replace_multiple_clusters->first.size() == 3); + BOOST_CHECK(replace_multiple_clusters->second.size() == 1); + } + + // Add a child transaction to conflict_1 and make it cluster size 2, two chunks due to same feerate + const auto conflict_1_child = make_tx(/*inputs=*/{conflict_1}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(conflict_1_child)); + const auto conflict_1_child_entry = pool.GetIter(conflict_1_child->GetHash()).value(); + + { + const auto replace_multiple_clusters_2{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry, conflict_1_child_entry})}; + + BOOST_CHECK(replace_multiple_clusters_2.has_value()); + BOOST_CHECK(replace_multiple_clusters_2->first.size() == 4); + BOOST_CHECK(replace_multiple_clusters_2->second.size() == 1); + } + + // Add another descendant to conflict_1, making the cluster size > 2 should fail at this point. + const auto conflict_1_grand_child = make_tx(/*inputs=*/{conflict_1_child}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(high_fee).FromTx(conflict_1_grand_child)); + const auto conflict_1_grand_child_entry = pool.GetIter(conflict_1_child->GetHash()).value(); + + { + const auto replace_cluster_size_3{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry, conflict_1_child_entry, conflict_1_grand_child_entry})}; + + BOOST_CHECK(!replace_cluster_size_3.has_value()); + BOOST_CHECK_EQUAL(util::ErrorString(replace_cluster_size_3).original, strprintf("%s has 2 descendants, max 1 allowed", conflict_1->GetHash().GetHex())); + } +} + +BOOST_AUTO_TEST_CASE(feerate_chunks_utilities) +{ + // Sanity check the correctness of the feerate chunks comparison. + + // A strictly better case. + std::vector<FeeFrac> old_chunks{{{950, 300}, {100, 100}}}; + std::vector<FeeFrac> new_chunks{{{1000, 300}, {50, 100}}}; + + BOOST_CHECK(std::is_lt(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_gt(CompareChunks(new_chunks, old_chunks))); + + // Incomparable diagrams + old_chunks = {{950, 300}, {100, 100}}; + new_chunks = {{1000, 300}, {0, 100}}; + + BOOST_CHECK(CompareChunks(old_chunks, new_chunks) == std::partial_ordering::unordered); + BOOST_CHECK(CompareChunks(new_chunks, old_chunks) == std::partial_ordering::unordered); + + // Strictly better but smaller size. + old_chunks = {{950, 300}, {100, 100}}; + new_chunks = {{1100, 300}}; + + BOOST_CHECK(std::is_lt(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_gt(CompareChunks(new_chunks, old_chunks))); + + // New diagram is strictly better due to the first chunk, even though + // second chunk contributes no fees + old_chunks = {{950, 300}, {100, 100}}; + new_chunks = {{1100, 100}, {0, 100}}; + + BOOST_CHECK(std::is_lt(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_gt(CompareChunks(new_chunks, old_chunks))); + + // Feerate of first new chunk is better with, but second chunk is worse + old_chunks = {{950, 300}, {100, 100}}; + new_chunks = {{750, 100}, {249, 250}, {151, 650}}; + + BOOST_CHECK(CompareChunks(old_chunks, new_chunks) == std::partial_ordering::unordered); + BOOST_CHECK(CompareChunks(new_chunks, old_chunks) == std::partial_ordering::unordered); + + // If we make the second chunk slightly better, the new diagram now wins. + old_chunks = {{950, 300}, {100, 100}}; + new_chunks = {{750, 100}, {250, 250}, {150, 150}}; + + BOOST_CHECK(std::is_lt(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_gt(CompareChunks(new_chunks, old_chunks))); + + // Identical diagrams, cannot be strictly better + old_chunks = {{950, 300}, {100, 100}}; + new_chunks = {{950, 300}, {100, 100}}; + + BOOST_CHECK(std::is_eq(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_eq(CompareChunks(new_chunks, old_chunks))); + + // Same aggregate fee, but different total size (trigger single tail fee check step) + old_chunks = {{950, 300}, {100, 99}}; + new_chunks = {{950, 300}, {100, 100}}; + + // No change in evaluation when tail check needed. + BOOST_CHECK(std::is_gt(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_lt(CompareChunks(new_chunks, old_chunks))); + + // Trigger multiple tail fee check steps + old_chunks = {{950, 300}, {100, 99}}; + new_chunks = {{950, 300}, {100, 100}, {0, 1}, {0, 1}}; + + BOOST_CHECK(std::is_gt(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_lt(CompareChunks(new_chunks, old_chunks))); + + // Multiple tail fee check steps, unordered result + new_chunks = {{950, 300}, {100, 100}, {0, 1}, {0, 1}, {1, 1}}; + BOOST_CHECK(CompareChunks(old_chunks, new_chunks) == std::partial_ordering::unordered); + BOOST_CHECK(CompareChunks(new_chunks, old_chunks) == std::partial_ordering::unordered); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 0d2460c606..3a1cb45e8d 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -291,6 +291,7 @@ BOOST_AUTO_TEST_CASE(rpc_parse_monetary_values) BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("1e-8")), COIN/100000000); BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.1e-7")), COIN/100000000); BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.01e-6")), COIN/100000000); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.00000000000000000000000000000000000001e+30")), 1); BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.0000000000000000000000000000000000000000000000000000000000000000000000000001e+68")), COIN/100000000); BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("10000000000000000000000000000000000000000000000000000000000000000e-64")), COIN); BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000e64")), COIN); diff --git a/src/test/sanity_tests.cpp b/src/test/sanity_tests.cpp index 451bc99d44..68f82b760c 100644 --- a/src/test/sanity_tests.cpp +++ b/src/test/sanity_tests.cpp @@ -4,7 +4,6 @@ #include <key.h> #include <test/util/setup_common.h> -#include <util/time.h> #include <boost/test/unit_test.hpp> @@ -13,7 +12,6 @@ BOOST_FIXTURE_TEST_SUITE(sanity_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(basic_sanity) { BOOST_CHECK_MESSAGE(ECC_InitSanityCheck() == true, "secp256k1 sanity test"); - BOOST_CHECK_MESSAGE(ChronoSanityCheck() == true, "chrono epoch test"); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index ac457d9c77..e4142e203c 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -2,10 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - #include <test/data/script_tests.json.h> #include <test/data/bip341_wallet_vectors.json.h> @@ -27,10 +23,6 @@ #include <util/fs.h> #include <util/strencodings.h> -#if defined(HAVE_CONSENSUS_LIB) -#include <script/bitcoinconsensus.h> -#endif - #include <cstdint> #include <fstream> #include <string> @@ -143,21 +135,6 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript if (combined_flags & SCRIPT_VERIFY_WITNESS && ~combined_flags & SCRIPT_VERIFY_P2SH) continue; BOOST_CHECK_MESSAGE(VerifyScript(scriptSig, scriptPubKey, &scriptWitness, combined_flags, MutableTransactionSignatureChecker(&tx, 0, txCredit.vout[0].nValue, MissingDataBehavior::ASSERT_FAIL), &err) == expect, message + strprintf(" (with flags %x)", combined_flags)); } - -#if defined(HAVE_CONSENSUS_LIB) - DataStream stream; - stream << TX_WITH_WITNESS(tx2); - uint32_t libconsensus_flags{flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL}; - if (libconsensus_flags == flags) { - int expectedSuccessCode = expect ? 1 : 0; - if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS) { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), txCredit.vout[0].nValue, UCharCast(stream.data()), stream.size(), 0, libconsensus_flags, nullptr) == expectedSuccessCode, message); - } else { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), 0, UCharCast(stream.data()), stream.size(), 0, libconsensus_flags, nullptr) == expectedSuccessCode, message); - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), 0, libconsensus_flags, nullptr) == expectedSuccessCode, message); - } - } -#endif } void static NegateSignatureS(std::vector<unsigned char>& vchSig) { @@ -1277,6 +1254,30 @@ BOOST_AUTO_TEST_CASE(script_combineSigs) BOOST_CHECK(combined.scriptSig == partial3c); } +/** + * Reproduction of an exception incorrectly raised when parsing a public key inside a TapMiniscript. + */ +BOOST_AUTO_TEST_CASE(sign_invalid_miniscript) +{ + FillableSigningProvider keystore; + SignatureData sig_data; + CMutableTransaction prev, curr; + + // Create a Taproot output which contains a leaf in which a non-32 bytes push is used where a public key is expected + // by the Miniscript parser. This offending Script was found by the RPC fuzzer. + const auto invalid_pubkey{ParseHex("173d36c8c9c9c9ffffffffffff0200000000021e1e37373721361818181818181e1e1e1e19000000000000000000b19292929292926b006c9b9b9292")}; + TaprootBuilder builder; + builder.Add(0, {invalid_pubkey}, 0xc0); + XOnlyPubKey nums{ParseHex("50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0")}; + builder.Finalize(nums); + prev.vout.emplace_back(0, GetScriptForDestination(builder.GetOutput())); + curr.vin.emplace_back(COutPoint{prev.GetHash(), 0}); + sig_data.tr_spenddata = builder.GetSpendData(); + + // SignSignature can fail but it shouldn't raise an exception (nor crash). + BOOST_CHECK(!SignSignature(keystore, CTransaction(prev), curr, 0, SIGHASH_ALL, sig_data)); +} + BOOST_AUTO_TEST_CASE(script_standard_push) { ScriptError err; @@ -1498,179 +1499,6 @@ static CScriptWitness ScriptWitnessFromJSON(const UniValue& univalue) return scriptwitness; } -#if defined(HAVE_CONSENSUS_LIB) - -/* Test simple (successful) usage of bitcoinconsensus_verify_script */ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_returns_true) -{ - unsigned int libconsensus_flags = 0; - int nIn = 0; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_1; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << TX_WITH_WITNESS(spendTx); - - bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 1); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_OK); -} - -/* Test bitcoinconsensus_verify_script returns invalid tx index err*/ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_index_err) -{ - unsigned int libconsensus_flags = 0; - int nIn = 3; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_EQUAL; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << TX_WITH_WITNESS(spendTx); - - bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_TX_INDEX); -} - -/* Test bitcoinconsensus_verify_script returns tx size mismatch err*/ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_size) -{ - unsigned int libconsensus_flags = 0; - int nIn = 0; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_EQUAL; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << TX_WITH_WITNESS(spendTx); - - bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size() * 2, nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_TX_SIZE_MISMATCH); -} - -/* Test bitcoinconsensus_verify_script returns invalid tx serialization error */ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_serialization) -{ - unsigned int libconsensus_flags = 0; - int nIn = 0; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_EQUAL; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << 0xffffffff; - - bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_TX_DESERIALIZE); -} - -/* Test bitcoinconsensus_verify_script returns amount required error */ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_amount_required_err) -{ - unsigned int libconsensus_flags = bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS; - int nIn = 0; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_EQUAL; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << TX_WITH_WITNESS(spendTx); - - bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_AMOUNT_REQUIRED); -} - -/* Test bitcoinconsensus_verify_script returns invalid flags err */ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_invalid_flags) -{ - unsigned int libconsensus_flags = 1 << 3; - int nIn = 0; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_EQUAL; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << TX_WITH_WITNESS(spendTx); - - bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_INVALID_FLAGS); -} - -/* Test bitcoinconsensus_verify_script returns spent outputs required err */ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_spent_outputs_required_err) -{ - unsigned int libconsensus_flags{bitcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT}; - const int nIn{0}; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_EQUAL; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << TX_WITH_WITNESS(spendTx); - - bitcoinconsensus_error err; - int result{bitcoinconsensus_verify_script_with_spent_outputs(scriptPubKey.data(), scriptPubKey.size(), creditTx.vout[0].nValue, UCharCast(stream.data()), stream.size(), nullptr, 0, nIn, libconsensus_flags, &err)}; - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_SPENT_OUTPUTS_REQUIRED); - - result = bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), creditTx.vout[0].nValue, UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_SPENT_OUTPUTS_REQUIRED); - - result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_SPENT_OUTPUTS_REQUIRED); -} - -#endif // defined(HAVE_CONSENSUS_LIB) - static std::vector<unsigned int> AllConsensusFlags() { std::vector<unsigned int> ret; @@ -1718,28 +1546,12 @@ static void AssetTest(const UniValue& test) txdata.Init(tx, std::vector<CTxOut>(prevouts)); CachingTransactionSignatureChecker txcheck(&tx, idx, prevouts[idx].nValue, true, txdata); -#if defined(HAVE_CONSENSUS_LIB) - DataStream stream; - stream << TX_WITH_WITNESS(tx); - std::vector<UTXO> utxos; - utxos.resize(prevouts.size()); - for (size_t i = 0; i < prevouts.size(); i++) { - utxos[i].scriptPubKey = prevouts[i].scriptPubKey.data(); - utxos[i].scriptPubKeySize = prevouts[i].scriptPubKey.size(); - utxos[i].value = prevouts[i].nValue; - } -#endif - for (const auto flags : ALL_CONSENSUS_FLAGS) { // "final": true tests are valid for all flags. Others are only valid with flags that are // a subset of test_flags. if (fin || ((flags & test_flags) == flags)) { bool ret = VerifyScript(tx.vin[idx].scriptSig, prevouts[idx].scriptPubKey, &tx.vin[idx].scriptWitness, flags, txcheck, nullptr); BOOST_CHECK(ret); -#if defined(HAVE_CONSENSUS_LIB) - int lib_ret = bitcoinconsensus_verify_script_with_spent_outputs(prevouts[idx].scriptPubKey.data(), prevouts[idx].scriptPubKey.size(), prevouts[idx].nValue, UCharCast(stream.data()), stream.size(), utxos.data(), utxos.size(), idx, flags, nullptr); - BOOST_CHECK(lib_ret == 1); -#endif } } } @@ -1752,27 +1564,11 @@ static void AssetTest(const UniValue& test) txdata.Init(tx, std::vector<CTxOut>(prevouts)); CachingTransactionSignatureChecker txcheck(&tx, idx, prevouts[idx].nValue, true, txdata); -#if defined(HAVE_CONSENSUS_LIB) - DataStream stream; - stream << TX_WITH_WITNESS(tx); - std::vector<UTXO> utxos; - utxos.resize(prevouts.size()); - for (size_t i = 0; i < prevouts.size(); i++) { - utxos[i].scriptPubKey = prevouts[i].scriptPubKey.data(); - utxos[i].scriptPubKeySize = prevouts[i].scriptPubKey.size(); - utxos[i].value = prevouts[i].nValue; - } -#endif - for (const auto flags : ALL_CONSENSUS_FLAGS) { // If a test is supposed to fail with test_flags, it should also fail with any superset thereof. if ((flags & test_flags) == test_flags) { bool ret = VerifyScript(tx.vin[idx].scriptSig, prevouts[idx].scriptPubKey, &tx.vin[idx].scriptWitness, flags, txcheck, nullptr); BOOST_CHECK(!ret); -#if defined(HAVE_CONSENSUS_LIB) - int lib_ret = bitcoinconsensus_verify_script_with_spent_outputs(prevouts[idx].scriptPubKey.data(), prevouts[idx].scriptPubKey.size(), prevouts[idx].nValue, UCharCast(stream.data()), stream.size(), utxos.data(), utxos.size(), idx, flags, nullptr); - BOOST_CHECK(lib_ret == 0); -#endif } } } diff --git a/src/test/serfloat_tests.cpp b/src/test/serfloat_tests.cpp index b36bdc02ca..304541074f 100644 --- a/src/test/serfloat_tests.cpp +++ b/src/test/serfloat_tests.cpp @@ -37,6 +37,7 @@ uint64_t TestDouble(double f) { } // namespace BOOST_AUTO_TEST_CASE(double_serfloat_tests) { + // Test specific values against their expected encoding. BOOST_CHECK_EQUAL(TestDouble(0.0), 0U); BOOST_CHECK_EQUAL(TestDouble(-0.0), 0x8000000000000000); BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::infinity()), 0x7ff0000000000000U); @@ -46,55 +47,76 @@ BOOST_AUTO_TEST_CASE(double_serfloat_tests) { BOOST_CHECK_EQUAL(TestDouble(2.0), 0x4000000000000000ULL); BOOST_CHECK_EQUAL(TestDouble(4.0), 0x4010000000000000ULL); BOOST_CHECK_EQUAL(TestDouble(785.066650390625), 0x4088888880000000ULL); + BOOST_CHECK_EQUAL(TestDouble(3.7243058682384174), 0x400dcb60e0031440); + BOOST_CHECK_EQUAL(TestDouble(91.64070592566159), 0x4056e901536d447a); + BOOST_CHECK_EQUAL(TestDouble(-98.63087668642575), 0xc058a860489c007a); + BOOST_CHECK_EQUAL(TestDouble(4.908737756962054), 0x4013a28c268b2b70); + BOOST_CHECK_EQUAL(TestDouble(77.9247330021754), 0x40537b2ed3547804); + BOOST_CHECK_EQUAL(TestDouble(40.24732825357566), 0x40441fa873c43dfc); + BOOST_CHECK_EQUAL(TestDouble(71.39395607929222), 0x4051d936938f27b6); + BOOST_CHECK_EQUAL(TestDouble(58.80100710817612), 0x404d668766a2bd70); + BOOST_CHECK_EQUAL(TestDouble(-30.10665786964975), 0xc03e1b4dee1e01b8); + BOOST_CHECK_EQUAL(TestDouble(60.15231509068704), 0x404e137f0f969814); + BOOST_CHECK_EQUAL(TestDouble(-48.15848711335961), 0xc04814494e445bc6); + BOOST_CHECK_EQUAL(TestDouble(26.68450101125353), 0x403aaf3b755169b0); + BOOST_CHECK_EQUAL(TestDouble(-65.72071986604303), 0xc0506e2046378ede); + BOOST_CHECK_EQUAL(TestDouble(17.95575825512381), 0x4031f4ac92b0a388); + BOOST_CHECK_EQUAL(TestDouble(-35.27171863226279), 0xc041a2c7ad17a42a); + BOOST_CHECK_EQUAL(TestDouble(-8.58810329425124), 0xc0212d1bdffef538); + BOOST_CHECK_EQUAL(TestDouble(88.51393044338977), 0x405620e43c83b1c8); + BOOST_CHECK_EQUAL(TestDouble(48.07224932612732), 0x4048093f77466ffc); + BOOST_CHECK_EQUAL(TestDouble(9.867348871395659e+117), 0x586f4daeb2459b9f); + BOOST_CHECK_EQUAL(TestDouble(-1.5166424385129721e+206), 0xeabe3bbc484bd458); + BOOST_CHECK_EQUAL(TestDouble(-8.585156555624594e-275), 0x8707c76eee012429); + BOOST_CHECK_EQUAL(TestDouble(2.2794371091628822e+113), 0x5777b2184458f4ee); + BOOST_CHECK_EQUAL(TestDouble(-1.1290476594131867e+163), 0xe1c91893d3488bb0); + BOOST_CHECK_EQUAL(TestDouble(9.143848423979275e-246), 0x0d0ff76e5f2620a3); + BOOST_CHECK_EQUAL(TestDouble(-2.8366718125941117e+81), 0xd0d7ec7e754b394a); + BOOST_CHECK_EQUAL(TestDouble(-1.2754409481684012e+229), 0xef80d32f8ec55342); + BOOST_CHECK_EQUAL(TestDouble(6.000577060053642e-186), 0x197a1be7c8209b6a); + BOOST_CHECK_EQUAL(TestDouble(2.0839423284378986e-302), 0x014c94f8689cb0a5); + BOOST_CHECK_EQUAL(TestDouble(-1.422140051483753e+259), 0xf5bd99271d04bb35); + BOOST_CHECK_EQUAL(TestDouble(-1.0593973991188853e+46), 0xc97db0cdb72d1046); + BOOST_CHECK_EQUAL(TestDouble(2.62945125875249e+190), 0x67779b36366c993b); + BOOST_CHECK_EQUAL(TestDouble(-2.920377657275094e+115), 0xd7e7b7b45908e23b); + BOOST_CHECK_EQUAL(TestDouble(9.790289014855851e-118), 0x27a3c031cc428bcc); + BOOST_CHECK_EQUAL(TestDouble(-4.629317182034961e-114), 0xa866ccf0b753705a); + BOOST_CHECK_EQUAL(TestDouble(-1.7674605603846528e+279), 0xf9e8ed383ffc3e25); + BOOST_CHECK_EQUAL(TestDouble(2.5308171727712605e+120), 0x58ef5cd55f0ec997); + BOOST_CHECK_EQUAL(TestDouble(-1.05034156412799e+54), 0xcb25eea1b9350fa0); - // Roundtrip test on IEC559-compatible systems - if (std::numeric_limits<double>::is_iec559) { - BOOST_CHECK_EQUAL(sizeof(double), 8U); - BOOST_CHECK_EQUAL(sizeof(uint64_t), 8U); - // Test extreme values - TestDouble(std::numeric_limits<double>::min()); - TestDouble(-std::numeric_limits<double>::min()); - TestDouble(std::numeric_limits<double>::max()); - TestDouble(-std::numeric_limits<double>::max()); - TestDouble(std::numeric_limits<double>::lowest()); - TestDouble(-std::numeric_limits<double>::lowest()); - TestDouble(std::numeric_limits<double>::quiet_NaN()); - TestDouble(-std::numeric_limits<double>::quiet_NaN()); - TestDouble(std::numeric_limits<double>::signaling_NaN()); - TestDouble(-std::numeric_limits<double>::signaling_NaN()); - TestDouble(std::numeric_limits<double>::denorm_min()); - TestDouble(-std::numeric_limits<double>::denorm_min()); - // Test exact encoding: on currently supported platforms, EncodeDouble - // should produce exactly the same as the in-memory representation for non-NaN. - for (int j = 0; j < 1000; ++j) { - // Iterate over 9 specific bits exhaustively; the others are chosen randomly. - // These specific bits are the sign bit, and the 2 top and bottom bits of - // exponent and mantissa in the IEEE754 binary64 format. - for (int x = 0; x < 512; ++x) { - uint64_t v = InsecureRandBits(64); - v &= ~(uint64_t{1} << 0); - if (x & 1) v |= (uint64_t{1} << 0); - v &= ~(uint64_t{1} << 1); - if (x & 2) v |= (uint64_t{1} << 1); - v &= ~(uint64_t{1} << 50); - if (x & 4) v |= (uint64_t{1} << 50); - v &= ~(uint64_t{1} << 51); - if (x & 8) v |= (uint64_t{1} << 51); - v &= ~(uint64_t{1} << 52); - if (x & 16) v |= (uint64_t{1} << 52); - v &= ~(uint64_t{1} << 53); - if (x & 32) v |= (uint64_t{1} << 53); - v &= ~(uint64_t{1} << 61); - if (x & 64) v |= (uint64_t{1} << 61); - v &= ~(uint64_t{1} << 62); - if (x & 128) v |= (uint64_t{1} << 62); - v &= ~(uint64_t{1} << 63); - if (x & 256) v |= (uint64_t{1} << 63); - double f; - memcpy(&f, &v, 8); - uint64_t v2 = TestDouble(f); - if (!std::isnan(f)) BOOST_CHECK_EQUAL(v, v2); + // Test extreme values + BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::min()), 0x10000000000000); + BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::min()), 0x8010000000000000); + BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::max()), 0x7fefffffffffffff); + BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::max()), 0xffefffffffffffff); + BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::lowest()), 0xffefffffffffffff); + BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::lowest()), 0x7fefffffffffffff); + BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::denorm_min()), 0x1); + BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::denorm_min()), 0x8000000000000001); + // Note that all NaNs are encoded the same way. + BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::quiet_NaN()), 0x7ff8000000000000); + BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::quiet_NaN()), 0x7ff8000000000000); + BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::signaling_NaN()), 0x7ff8000000000000); + BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::signaling_NaN()), 0x7ff8000000000000); + + // Construct doubles to test from the encoding. + static_assert(sizeof(double) == 8); + static_assert(sizeof(uint64_t) == 8); + for (int j = 0; j < 1000; ++j) { + // Iterate over 9 specific bits exhaustively; the others are chosen randomly. + // These specific bits are the sign bit, and the 2 top and bottom bits of + // exponent and mantissa in the IEEE754 binary64 format. + for (int x = 0; x < 512; ++x) { + uint64_t v = InsecureRandBits(64); + int x_pos = 0; + for (int v_pos : {0, 1, 50, 51, 52, 53, 61, 62, 63}) { + v &= ~(uint64_t{1} << v_pos); + if ((x >> (x_pos++)) & 1) v |= (uint64_t{1} << v_pos); } + double f; + memcpy(&f, &v, 8); + TestDouble(f); } } } diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index 90fce9adf9..2de147deea 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -11,7 +11,7 @@ #include <univalue.h> #ifdef ENABLE_EXTERNAL_SIGNER -#include <boost/process.hpp> +#include <util/subprocess.h> #endif // ENABLE_EXTERNAL_SIGNER #include <boost/test/unit_test.hpp> @@ -34,20 +34,16 @@ BOOST_AUTO_TEST_CASE(run_command) BOOST_CHECK(result.isNull()); } { - const UniValue result = RunCommandParseJSON("echo \"{\"success\": true}\""); + const UniValue result = RunCommandParseJSON("echo {\"success\": true}"); BOOST_CHECK(result.isObject()); const UniValue& success = result.find_value("success"); BOOST_CHECK(!success.isNull()); BOOST_CHECK_EQUAL(success.get_bool(), true); } { - // An invalid command is handled by Boost - const int expected_error{2}; - BOOST_CHECK_EXCEPTION(RunCommandParseJSON("invalid_command"), boost::process::process_error, [&](const boost::process::process_error& e) { - BOOST_CHECK(std::string(e.what()).find("RunCommandParseJSON error:") == std::string::npos); - BOOST_CHECK_EQUAL(e.code().value(), expected_error); - return true; - }); + // An invalid command is handled by cpp-subprocess + const std::string expected{"execve failed: "}; + BOOST_CHECK_EXCEPTION(RunCommandParseJSON("invalid_command"), subprocess::CalledProcessError, HasReason(expected)); } { // Return non-zero exit code, no output to stderr diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index f6456526bb..b948ea8acb 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -132,7 +132,7 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) /*output_amount=*/CAmount(48 * COIN), /*submit=*/false); CTransactionRef tx_child = MakeTransactionRef(mtx_child); Package package_parent_child{tx_parent, tx_child}; - const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_parent_child, /*test_accept=*/true); + const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_parent_child, /*test_accept=*/true, /*client_maxfeerate=*/{}); if (auto err_parent_child{CheckPackageMempoolAcceptResult(package_parent_child, result_parent_child, /*expect_valid=*/true, nullptr)}) { BOOST_ERROR(err_parent_child.value()); } else { @@ -151,7 +151,7 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) CTransactionRef giant_ptx = create_placeholder_tx(999, 999); BOOST_CHECK(GetVirtualTransactionSize(*giant_ptx) > DEFAULT_ANCESTOR_SIZE_LIMIT_KVB * 1000); Package package_single_giant{giant_ptx}; - auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_single_giant, /*test_accept=*/true); + auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_single_giant, /*test_accept=*/true, /*client_maxfeerate=*/{}); if (auto err_single_large{CheckPackageMempoolAcceptResult(package_single_giant, result_single_large, /*expect_valid=*/false, nullptr)}) { BOOST_ERROR(err_single_large.value()); } else { @@ -275,7 +275,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) package_unrelated.emplace_back(MakeTransactionRef(mtx)); } auto result_unrelated_submit = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_unrelated, /*test_accept=*/false); + package_unrelated, /*test_accept=*/false, /*client_maxfeerate=*/{}); // We don't expect m_tx_results for each transaction when basic sanity checks haven't passed. BOOST_CHECK(result_unrelated_submit.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); @@ -315,7 +315,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) // 3 Generations is not allowed. { auto result_3gen_submit = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_3gen, /*test_accept=*/false); + package_3gen, /*test_accept=*/false, /*client_maxfeerate=*/{}); BOOST_CHECK(result_3gen_submit.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_3gen_submit.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_3gen_submit.m_state.GetRejectReason(), "package-not-child-with-parents"); @@ -332,7 +332,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) CTransactionRef tx_parent_invalid = MakeTransactionRef(mtx_parent_invalid); Package package_invalid_parent{tx_parent_invalid, tx_child}; auto result_quit_early = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_invalid_parent, /*test_accept=*/ false); + package_invalid_parent, /*test_accept=*/ false, /*client_maxfeerate=*/{}); if (auto err_parent_invalid{CheckPackageMempoolAcceptResult(package_invalid_parent, result_quit_early, /*expect_valid=*/false, m_node.mempool.get())}) { BOOST_ERROR(err_parent_invalid.value()); } else { @@ -353,7 +353,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) package_missing_parent.push_back(MakeTransactionRef(mtx_child)); { const auto result_missing_parent = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_missing_parent, /*test_accept=*/false); + package_missing_parent, /*test_accept=*/false, /*client_maxfeerate=*/{}); BOOST_CHECK(result_missing_parent.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_missing_parent.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_missing_parent.m_state.GetRejectReason(), "package-not-child-with-unconfirmed-parents"); @@ -363,7 +363,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) // Submit package with parent + child. { const auto submit_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child, /*test_accept=*/false); + package_parent_child, /*test_accept=*/false, /*client_maxfeerate=*/{}); expected_pool_size += 2; BOOST_CHECK_MESSAGE(submit_parent_child.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_parent_child.m_state.GetRejectReason()); @@ -385,7 +385,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) // Already-in-mempool transactions should be detected and de-duplicated. { const auto submit_deduped = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child, /*test_accept=*/false); + package_parent_child, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_deduped{CheckPackageMempoolAcceptResult(package_parent_child, submit_deduped, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_deduped.value()); } else { @@ -456,7 +456,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) { Package package_parent_child1{ptx_parent, ptx_child1}; const auto submit_witness1 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child1, /*test_accept=*/false); + package_parent_child1, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_witness1{CheckPackageMempoolAcceptResult(package_parent_child1, submit_witness1, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_witness1.value()); } @@ -464,7 +464,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // Child2 would have been validated individually. Package package_parent_child2{ptx_parent, ptx_child2}; const auto submit_witness2 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child2, /*test_accept=*/false); + package_parent_child2, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_witness2{CheckPackageMempoolAcceptResult(package_parent_child2, submit_witness2, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_witness2.value()); } else { @@ -478,7 +478,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // Deduplication should work when wtxid != txid. Submit package with the already-in-mempool // transactions again, which should not fail. const auto submit_segwit_dedup = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child1, /*test_accept=*/false); + package_parent_child1, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_segwit_dedup{CheckPackageMempoolAcceptResult(package_parent_child1, submit_segwit_dedup, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_segwit_dedup.value()); } else { @@ -508,7 +508,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) { Package package_child2_grandchild{ptx_child2, ptx_grandchild}; const auto submit_spend_ignored = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_child2_grandchild, /*test_accept=*/false); + package_child2_grandchild, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_spend_ignored{CheckPackageMempoolAcceptResult(package_child2_grandchild, submit_spend_ignored, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_spend_ignored.value()); } else { @@ -606,7 +606,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // parent3 should be accepted // child should be accepted { - const auto mixed_result = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_mixed, false); + const auto mixed_result = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_mixed, false, /*client_maxfeerate=*/{}); if (auto err_mixed{CheckPackageMempoolAcceptResult(package_mixed, mixed_result, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_mixed.value()); } else { @@ -670,7 +670,7 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) { BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_cpfp_deprio = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_cpfp, /*test_accept=*/ false); + package_cpfp, /*test_accept=*/ false, /*client_maxfeerate=*/{}); if (auto err_cpfp_deprio{CheckPackageMempoolAcceptResult(package_cpfp, submit_cpfp_deprio, /*expect_valid=*/false, m_node.mempool.get())}) { BOOST_ERROR(err_cpfp_deprio.value()); } else { @@ -692,7 +692,7 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) { BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_cpfp = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_cpfp, /*test_accept=*/ false); + package_cpfp, /*test_accept=*/ false, /*client_maxfeerate=*/{}); if (auto err_cpfp{CheckPackageMempoolAcceptResult(package_cpfp, submit_cpfp, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_cpfp.value()); } else { @@ -744,7 +744,7 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) // Cheap package should fail for being too low fee. { const auto submit_package_too_low = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_still_too_low, /*test_accept=*/false); + package_still_too_low, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_package_too_low{CheckPackageMempoolAcceptResult(package_still_too_low, submit_package_too_low, /*expect_valid=*/false, m_node.mempool.get())}) { BOOST_ERROR(err_package_too_low.value()); } else { @@ -770,7 +770,7 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) // Now that the child's fees have "increased" by 1 BTC, the cheap package should succeed. { const auto submit_prioritised_package = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_still_too_low, /*test_accept=*/false); + package_still_too_low, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_prioritised{CheckPackageMempoolAcceptResult(package_still_too_low, submit_prioritised_package, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_prioritised.value()); } else { @@ -818,7 +818,7 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) { BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_rich_parent = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_rich_parent, /*test_accept=*/false); + package_rich_parent, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_rich_parent{CheckPackageMempoolAcceptResult(package_rich_parent, submit_rich_parent, /*expect_valid=*/false, m_node.mempool.get())}) { BOOST_ERROR(err_rich_parent.value()); } else { diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index e045949b43..95583b53bf 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -115,7 +115,9 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) const auto expected_error_str{strprintf("non-v3 tx %s (wtxid=%s) cannot spend from v3 tx %s (wtxid=%s)", tx_v2_from_v3->GetHash().ToString(), tx_v2_from_v3->GetWitnessHash().ToString(), mempool_tx_v3->GetHash().ToString(), mempool_tx_v3->GetWitnessHash().ToString())}; - BOOST_CHECK(*SingleV3Checks(tx_v2_from_v3, *ancestors_v2_from_v3, empty_conflicts_set, GetVirtualTransactionSize(*tx_v2_from_v3)) == expected_error_str); + auto result_v2_from_v3{SingleV3Checks(tx_v2_from_v3, *ancestors_v2_from_v3, empty_conflicts_set, GetVirtualTransactionSize(*tx_v2_from_v3))}; + BOOST_CHECK_EQUAL(result_v2_from_v3->first, expected_error_str); + BOOST_CHECK_EQUAL(result_v2_from_v3->second, nullptr); Package package_v3_v2{mempool_tx_v3, tx_v2_from_v3}; BOOST_CHECK_EQUAL(*PackageV3Checks(tx_v2_from_v3, GetVirtualTransactionSize(*tx_v2_from_v3), package_v3_v2, empty_ancestors), expected_error_str); @@ -130,8 +132,9 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) const auto expected_error_str_2{strprintf("non-v3 tx %s (wtxid=%s) cannot spend from v3 tx %s (wtxid=%s)", tx_v2_from_v2_and_v3->GetHash().ToString(), tx_v2_from_v2_and_v3->GetWitnessHash().ToString(), mempool_tx_v3->GetHash().ToString(), mempool_tx_v3->GetWitnessHash().ToString())}; - BOOST_CHECK(*SingleV3Checks(tx_v2_from_v2_and_v3, *ancestors_v2_from_both, empty_conflicts_set, GetVirtualTransactionSize(*tx_v2_from_v2_and_v3)) - == expected_error_str_2); + auto result_v2_from_both{SingleV3Checks(tx_v2_from_v2_and_v3, *ancestors_v2_from_both, empty_conflicts_set, GetVirtualTransactionSize(*tx_v2_from_v2_and_v3))}; + BOOST_CHECK_EQUAL(result_v2_from_both->first, expected_error_str_2); + BOOST_CHECK_EQUAL(result_v2_from_both->second, nullptr); Package package_v3_v2_v2{mempool_tx_v3, mempool_tx_v2, tx_v2_from_v2_and_v3}; BOOST_CHECK_EQUAL(*PackageV3Checks(tx_v2_from_v2_and_v3, GetVirtualTransactionSize(*tx_v2_from_v2_and_v3), package_v3_v2_v2, empty_ancestors), expected_error_str_2); @@ -147,7 +150,9 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) const auto expected_error_str{strprintf("v3 tx %s (wtxid=%s) cannot spend from non-v3 tx %s (wtxid=%s)", tx_v3_from_v2->GetHash().ToString(), tx_v3_from_v2->GetWitnessHash().ToString(), mempool_tx_v2->GetHash().ToString(), mempool_tx_v2->GetWitnessHash().ToString())}; - BOOST_CHECK(*SingleV3Checks(tx_v3_from_v2, *ancestors_v3_from_v2, empty_conflicts_set, GetVirtualTransactionSize(*tx_v3_from_v2)) == expected_error_str); + auto result_v3_from_v2{SingleV3Checks(tx_v3_from_v2, *ancestors_v3_from_v2, empty_conflicts_set, GetVirtualTransactionSize(*tx_v3_from_v2))}; + BOOST_CHECK_EQUAL(result_v3_from_v2->first, expected_error_str); + BOOST_CHECK_EQUAL(result_v3_from_v2->second, nullptr); Package package_v2_v3{mempool_tx_v2, tx_v3_from_v2}; BOOST_CHECK_EQUAL(*PackageV3Checks(tx_v3_from_v2, GetVirtualTransactionSize(*tx_v3_from_v2), package_v2_v3, empty_ancestors), expected_error_str); @@ -162,8 +167,9 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) const auto expected_error_str_2{strprintf("v3 tx %s (wtxid=%s) cannot spend from non-v3 tx %s (wtxid=%s)", tx_v3_from_v2_and_v3->GetHash().ToString(), tx_v3_from_v2_and_v3->GetWitnessHash().ToString(), mempool_tx_v2->GetHash().ToString(), mempool_tx_v2->GetWitnessHash().ToString())}; - BOOST_CHECK(*SingleV3Checks(tx_v3_from_v2_and_v3, *ancestors_v3_from_both, empty_conflicts_set, GetVirtualTransactionSize(*tx_v3_from_v2_and_v3)) - == expected_error_str_2); + auto result_v3_from_both{SingleV3Checks(tx_v3_from_v2_and_v3, *ancestors_v3_from_both, empty_conflicts_set, GetVirtualTransactionSize(*tx_v3_from_v2_and_v3))}; + BOOST_CHECK_EQUAL(result_v3_from_both->first, expected_error_str_2); + BOOST_CHECK_EQUAL(result_v3_from_both->second, nullptr); // tx_v3_from_v2_and_v3 also violates V3_ANCESTOR_LIMIT. const auto expected_error_str_3{strprintf("tx %s (wtxid=%s) would have too many ancestors", @@ -215,8 +221,9 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) BOOST_CHECK_EQUAL(ancestors->size(), 3); const auto expected_error_str{strprintf("tx %s (wtxid=%s) would have too many ancestors", tx_v3_multi_parent->GetHash().ToString(), tx_v3_multi_parent->GetWitnessHash().ToString())}; - BOOST_CHECK_EQUAL(*SingleV3Checks(tx_v3_multi_parent, *ancestors, empty_conflicts_set, GetVirtualTransactionSize(*tx_v3_multi_parent)), - expected_error_str); + auto result{SingleV3Checks(tx_v3_multi_parent, *ancestors, empty_conflicts_set, GetVirtualTransactionSize(*tx_v3_multi_parent))}; + BOOST_CHECK_EQUAL(result->first, expected_error_str); + BOOST_CHECK_EQUAL(result->second, nullptr); BOOST_CHECK_EQUAL(*PackageV3Checks(tx_v3_multi_parent, GetVirtualTransactionSize(*tx_v3_multi_parent), package_multi_parents, empty_ancestors), expected_error_str); @@ -239,8 +246,9 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) auto ancestors{pool.CalculateMemPoolAncestors(entry.FromTx(tx_v3_multi_gen), m_limits)}; const auto expected_error_str{strprintf("tx %s (wtxid=%s) would have too many ancestors", tx_v3_multi_gen->GetHash().ToString(), tx_v3_multi_gen->GetWitnessHash().ToString())}; - BOOST_CHECK_EQUAL(*SingleV3Checks(tx_v3_multi_gen, *ancestors, empty_conflicts_set, GetVirtualTransactionSize(*tx_v3_multi_gen)), - expected_error_str); + auto result{SingleV3Checks(tx_v3_multi_gen, *ancestors, empty_conflicts_set, GetVirtualTransactionSize(*tx_v3_multi_gen))}; + BOOST_CHECK_EQUAL(result->first, expected_error_str); + BOOST_CHECK_EQUAL(result->second, nullptr); // Middle tx is what triggers a failure for the grandchild: BOOST_CHECK_EQUAL(*PackageV3Checks(middle_tx, GetVirtualTransactionSize(*middle_tx), package_multi_gen, empty_ancestors), expected_error_str); @@ -256,8 +264,9 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) auto ancestors{pool.CalculateMemPoolAncestors(entry.FromTx(tx_v3_child_big), m_limits)}; const auto expected_error_str{strprintf("v3 child tx %s (wtxid=%s) is too big: %u > %u virtual bytes", tx_v3_child_big->GetHash().ToString(), tx_v3_child_big->GetWitnessHash().ToString(), vsize, V3_CHILD_MAX_VSIZE)}; - BOOST_CHECK_EQUAL(*SingleV3Checks(tx_v3_child_big, *ancestors, empty_conflicts_set, GetVirtualTransactionSize(*tx_v3_child_big)), - expected_error_str); + auto result{SingleV3Checks(tx_v3_child_big, *ancestors, empty_conflicts_set, GetVirtualTransactionSize(*tx_v3_child_big))}; + BOOST_CHECK_EQUAL(result->first, expected_error_str); + BOOST_CHECK_EQUAL(result->second, nullptr); Package package_child_big{mempool_tx_v3, tx_v3_child_big}; BOOST_CHECK_EQUAL(*PackageV3Checks(tx_v3_child_big, GetVirtualTransactionSize(*tx_v3_child_big), package_child_big, empty_ancestors), @@ -298,9 +307,10 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) const auto expected_error_str{strprintf("v3 child tx %s (wtxid=%s) is too big: %u > %u virtual bytes", tx_many_sigops->GetHash().ToString(), tx_many_sigops->GetWitnessHash().ToString(), total_sigops * DEFAULT_BYTES_PER_SIGOP / WITNESS_SCALE_FACTOR, V3_CHILD_MAX_VSIZE)}; - BOOST_CHECK_EQUAL(*SingleV3Checks(tx_many_sigops, *ancestors, empty_conflicts_set, - GetVirtualTransactionSize(*tx_many_sigops, /*nSigOpCost=*/total_sigops, /*bytes_per_sigop=*/ DEFAULT_BYTES_PER_SIGOP)), - expected_error_str); + auto result{SingleV3Checks(tx_many_sigops, *ancestors, empty_conflicts_set, + GetVirtualTransactionSize(*tx_many_sigops, /*nSigOpCost=*/total_sigops, /*bytes_per_sigop=*/ DEFAULT_BYTES_PER_SIGOP))}; + BOOST_CHECK_EQUAL(result->first, expected_error_str); + BOOST_CHECK_EQUAL(result->second, nullptr); Package package_child_sigops{mempool_tx_v3, tx_many_sigops}; BOOST_CHECK_EQUAL(*PackageV3Checks(tx_many_sigops, total_sigops * DEFAULT_BYTES_PER_SIGOP / WITNESS_SCALE_FACTOR, package_child_sigops, empty_ancestors), @@ -319,22 +329,58 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) BOOST_CHECK(PackageV3Checks(tx_mempool_v3_child, GetVirtualTransactionSize(*tx_mempool_v3_child), package_v3_1p1c, empty_ancestors) == std::nullopt); } - // A v3 transaction cannot have more than 1 descendant. - // Configuration where tx has multiple direct children. + // A v3 transaction cannot have more than 1 descendant. Sibling is returned when exactly 1 exists. { auto tx_v3_child2 = make_tx({COutPoint{mempool_tx_v3->GetHash(), 1}}, /*version=*/3); - auto ancestors{pool.CalculateMemPoolAncestors(entry.FromTx(tx_v3_child2), m_limits)}; + + // Configuration where parent already has 1 other child in mempool + auto ancestors_1sibling{pool.CalculateMemPoolAncestors(entry.FromTx(tx_v3_child2), m_limits)}; const auto expected_error_str{strprintf("tx %s (wtxid=%s) would exceed descendant count limit", mempool_tx_v3->GetHash().ToString(), mempool_tx_v3->GetWitnessHash().ToString())}; - BOOST_CHECK_EQUAL(*SingleV3Checks(tx_v3_child2, *ancestors, empty_conflicts_set, GetVirtualTransactionSize(*tx_v3_child2)), - expected_error_str); - // If replacing the child, make sure there is no double-counting. - BOOST_CHECK(SingleV3Checks(tx_v3_child2, *ancestors, {tx_mempool_v3_child->GetHash()}, GetVirtualTransactionSize(*tx_v3_child2)) + auto result_with_sibling_eviction{SingleV3Checks(tx_v3_child2, *ancestors_1sibling, empty_conflicts_set, GetVirtualTransactionSize(*tx_v3_child2))}; + BOOST_CHECK_EQUAL(result_with_sibling_eviction->first, expected_error_str); + // The other mempool child is returned to allow for sibling eviction. + BOOST_CHECK_EQUAL(result_with_sibling_eviction->second, tx_mempool_v3_child); + + // If directly replacing the child, make sure there is no double-counting. + BOOST_CHECK(SingleV3Checks(tx_v3_child2, *ancestors_1sibling, {tx_mempool_v3_child->GetHash()}, GetVirtualTransactionSize(*tx_v3_child2)) == std::nullopt); Package package_v3_1p2c{mempool_tx_v3, tx_mempool_v3_child, tx_v3_child2}; BOOST_CHECK_EQUAL(*PackageV3Checks(tx_v3_child2, GetVirtualTransactionSize(*tx_v3_child2), package_v3_1p2c, empty_ancestors), expected_error_str); + + // Configuration where parent already has 2 other children in mempool (no sibling eviction allowed). This may happen as the result of a reorg. + pool.addUnchecked(entry.FromTx(tx_v3_child2)); + auto tx_v3_child3 = make_tx({COutPoint{mempool_tx_v3->GetHash(), 24}}, /*version=*/3); + auto entry_mempool_parent = pool.GetIter(mempool_tx_v3->GetHash().ToUint256()).value(); + BOOST_CHECK_EQUAL(entry_mempool_parent->GetCountWithDescendants(), 3); + auto ancestors_2siblings{pool.CalculateMemPoolAncestors(entry.FromTx(tx_v3_child3), m_limits)}; + + auto result_2children{SingleV3Checks(tx_v3_child3, *ancestors_2siblings, empty_conflicts_set, GetVirtualTransactionSize(*tx_v3_child3))}; + BOOST_CHECK_EQUAL(result_2children->first, expected_error_str); + // The other mempool child is not returned because sibling eviction is not allowed. + BOOST_CHECK_EQUAL(result_2children->second, nullptr); + } + + // Sibling eviction: parent already has 1 other child, which also has its own child (no sibling eviction allowed). This may happen as the result of a reorg. + { + auto tx_mempool_grandparent = make_tx(random_outpoints(1), /*version=*/3); + auto tx_mempool_sibling = make_tx({COutPoint{tx_mempool_grandparent->GetHash(), 0}}, /*version=*/3); + auto tx_mempool_nibling = make_tx({COutPoint{tx_mempool_sibling->GetHash(), 0}}, /*version=*/3); + auto tx_to_submit = make_tx({COutPoint{tx_mempool_grandparent->GetHash(), 1}}, /*version=*/3); + + pool.addUnchecked(entry.FromTx(tx_mempool_grandparent)); + pool.addUnchecked(entry.FromTx(tx_mempool_sibling)); + pool.addUnchecked(entry.FromTx(tx_mempool_nibling)); + + auto ancestors_3gen{pool.CalculateMemPoolAncestors(entry.FromTx(tx_to_submit), m_limits)}; + const auto expected_error_str{strprintf("tx %s (wtxid=%s) would exceed descendant count limit", + tx_mempool_grandparent->GetHash().ToString(), tx_mempool_grandparent->GetWitnessHash().ToString())}; + auto result_3gen{SingleV3Checks(tx_to_submit, *ancestors_3gen, empty_conflicts_set, GetVirtualTransactionSize(*tx_to_submit))}; + BOOST_CHECK_EQUAL(result_3gen->first, expected_error_str); + // The other mempool child is not returned because sibling eviction is not allowed. + BOOST_CHECK_EQUAL(result_3gen->second, nullptr); } // Configuration where tx has multiple generations of descendants is not tested because that is diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index e2a88eacdd..ff95e64b7e 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -91,13 +91,16 @@ CreateAndActivateUTXOSnapshot( // these blocks instead CBlockIndex *pindex = orig_tip; while (pindex && pindex != chain.m_chain.Tip()) { - pindex->nStatus &= ~BLOCK_HAVE_DATA; - pindex->nStatus &= ~BLOCK_HAVE_UNDO; - // We have to set the ASSUMED_VALID flag, because otherwise it - // would not be possible to have a block index entry without HAVE_DATA - // and with nTx > 0 (since we aren't setting the pruned flag); - // see CheckBlockIndex(). - pindex->nStatus |= BLOCK_ASSUMED_VALID; + // Remove all data and validity flags by just setting + // BLOCK_VALID_TREE. Also reset transaction counts and sequence + // ids that are set when blocks are received, to make test setup + // more realistic and satisfy consistency checks in + // CheckBlockIndex(). + assert(pindex->IsValid(BlockStatus::BLOCK_VALID_TREE)); + pindex->nStatus = BlockStatus::BLOCK_VALID_TREE; + pindex->nTx = 0; + pindex->nChainTx = 0; + pindex->nSequenceId = 0; pindex = pindex->pprev; } } diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index d26a440074..38350b33cc 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -14,7 +14,6 @@ #include <banman.h> #include <chainparams.h> #include <common/system.h> -#include <common/url.h> #include <consensus/consensus.h> #include <consensus/params.h> #include <consensus/validation.h> @@ -52,6 +51,7 @@ #include <txmempool.h> #include <util/chaintype.h> #include <util/check.h> +#include <util/fs_helpers.h> #include <util/rbf.h> #include <util/strencodings.h> #include <util/string.h> @@ -80,7 +80,6 @@ using node::RegenerateCommitments; using node::VerifyLoadedChainstate; const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; -UrlDecodeFn* const URL_DECODE = nullptr; /** Random context to get unique temp data dirs. Separate from g_insecure_rand_ctx, which can be seeded from a const env var */ static FastRandomContext g_insecure_rand_ctx_temp_path; @@ -100,9 +99,22 @@ struct NetworkSetup }; static NetworkSetup g_networksetup_instance; +/** Register test-only arguments */ +static void SetupUnitTestArgs(ArgsManager& argsman) +{ + argsman.AddArg("-testdatadir", strprintf("Custom data directory (default: %s<random_string>)", fs::PathToString(fs::temp_directory_path() / "test_common_" PACKAGE_NAME / "")), + ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); +} + +/** Test setup failure */ +static void ExitFailure(std::string_view str_err) +{ + std::cerr << str_err << std::endl; + exit(EXIT_FAILURE); +} + BasicTestingSetup::BasicTestingSetup(const ChainType chainType, const std::vector<const char*>& extra_args) - : m_path_root{fs::temp_directory_path() / "test_common_" PACKAGE_NAME / g_insecure_rand_ctx_temp_path.rand256().ToString()}, - m_args{} + : m_args{} { m_node.shutdown = &m_interrupt; m_node.args = &gArgs; @@ -123,18 +135,49 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, const std::vecto arguments = Cat(arguments, G_TEST_COMMAND_LINE_ARGUMENTS()); } util::ThreadRename("test"); - fs::create_directories(m_path_root); - m_args.ForceSetArg("-datadir", fs::PathToString(m_path_root)); - gArgs.ForceSetArg("-datadir", fs::PathToString(m_path_root)); gArgs.ClearPathCache(); { SetupServerArgs(*m_node.args); + SetupUnitTestArgs(*m_node.args); std::string error; if (!m_node.args->ParseParameters(arguments.size(), arguments.data(), error)) { m_node.args->ClearArgs(); throw std::runtime_error{error}; } } + + if (!m_node.args->IsArgSet("-testdatadir")) { + // By default, the data directory has a random name + const auto rand_str{g_insecure_rand_ctx_temp_path.rand256().ToString()}; + m_path_root = fs::temp_directory_path() / "test_common_" PACKAGE_NAME / rand_str; + TryCreateDirectories(m_path_root); + } else { + // Custom data directory + m_has_custom_datadir = true; + fs::path root_dir{m_node.args->GetPathArg("-testdatadir")}; + if (root_dir.empty()) ExitFailure("-testdatadir argument is empty, please specify a path"); + + root_dir = fs::absolute(root_dir); + const std::string test_path{G_TEST_GET_FULL_NAME ? G_TEST_GET_FULL_NAME() : ""}; + m_path_lock = root_dir / "test_common_" PACKAGE_NAME / fs::PathFromString(test_path); + m_path_root = m_path_lock / "datadir"; + + // Try to obtain the lock; if unsuccessful don't disturb the existing test. + TryCreateDirectories(m_path_lock); + if (util::LockDirectory(m_path_lock, ".lock", /*probe_only=*/false) != util::LockResult::Success) { + ExitFailure("Cannot obtain a lock on test data lock directory " + fs::PathToString(m_path_lock) + '\n' + "The test executable is probably already running."); + } + + // Always start with a fresh data directory; this doesn't delete the .lock file located one level above. + fs::remove_all(m_path_root); + if (!TryCreateDirectories(m_path_root)) ExitFailure("Cannot create test data directory"); + + // Print the test directory name if custom. + std::cout << "Test directory (will not be deleted): " << m_path_root << std::endl; + } + m_args.ForceSetArg("-datadir", fs::PathToString(m_path_root)); + gArgs.ForceSetArg("-datadir", fs::PathToString(m_path_root)); + SelectParams(chainType); SeedInsecureRand(); if (G_TEST_LOG_FUN) LogInstance().PushBackCallback(G_TEST_LOG_FUN); @@ -162,7 +205,13 @@ BasicTestingSetup::~BasicTestingSetup() m_node.kernel.reset(); SetMockTime(0s); // Reset mocktime for following tests LogInstance().DisconnectTestLogger(); - fs::remove_all(m_path_root); + if (m_has_custom_datadir) { + // Only remove the lock file, preserve the data directory. + UnlockDirectory(m_path_lock, ".lock"); + fs::remove(m_path_lock / ".lock"); + } else { + fs::remove_all(m_path_root); + } gArgs.ClearArgs(); } diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 9ff4c372a5..8ccf9b571c 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -32,6 +32,9 @@ extern const std::function<void(const std::string&)> G_TEST_LOG_FUN; /** Retrieve the command line arguments. */ extern const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS; +/** Retrieve the unit test name. */ +extern const std::function<std::string()> G_TEST_GET_FULL_NAME; + // Enable BOOST_CHECK_EQUAL for enum class types namespace std { template <typename T> @@ -53,7 +56,9 @@ struct BasicTestingSetup { explicit BasicTestingSetup(const ChainType chainType = ChainType::MAIN, const std::vector<const char*>& extra_args = {}); ~BasicTestingSetup(); - const fs::path m_path_root; + fs::path m_path_root; + fs::path m_path_lock; + bool m_has_custom_datadir{false}; ArgsManager m_args; }; diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 47808a2a58..9a2add748e 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -290,13 +290,18 @@ BOOST_AUTO_TEST_CASE(util_TrimString) BOOST_AUTO_TEST_CASE(util_FormatISO8601DateTime) { + BOOST_CHECK_EQUAL(FormatISO8601DateTime(971890963199), "32767-12-31T23:59:59Z"); + BOOST_CHECK_EQUAL(FormatISO8601DateTime(971890876800), "32767-12-31T00:00:00Z"); BOOST_CHECK_EQUAL(FormatISO8601DateTime(1317425777), "2011-09-30T23:36:17Z"); BOOST_CHECK_EQUAL(FormatISO8601DateTime(0), "1970-01-01T00:00:00Z"); } BOOST_AUTO_TEST_CASE(util_FormatISO8601Date) { + BOOST_CHECK_EQUAL(FormatISO8601Date(971890963199), "32767-12-31"); + BOOST_CHECK_EQUAL(FormatISO8601Date(971890876800), "32767-12-31"); BOOST_CHECK_EQUAL(FormatISO8601Date(1317425777), "2011-09-30"); + BOOST_CHECK_EQUAL(FormatISO8601Date(0), "1970-01-01"); } BOOST_AUTO_TEST_CASE(util_FormatMoney) diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 316ab86c2b..69f4e305ab 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -127,6 +127,7 @@ std::shared_ptr<const CBlock> MinerTestingSetup::BadBlock(const uint256& prev_ha return ret; } +// NOLINTNEXTLINE(misc-no-recursion) void MinerTestingSetup::BuildChain(const uint256& root, int height, const unsigned int invalid_rate, const unsigned int branch_rate, const unsigned int max_size, std::vector<std::shared_ptr<const CBlock>>& blocks) { if (height <= 0 || blocks.size() >= max_size) return; diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 4bbab1cdcd..4bf66a55eb 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -276,9 +276,6 @@ struct SnapshotTestSetup : TestChain100Setup { BOOST_CHECK_EQUAL( *node::ReadSnapshotBaseBlockhash(found), *chainman.SnapshotBlockhash()); - - // Ensure that the genesis block was not marked assumed-valid. - BOOST_CHECK(!chainman.ActiveChain().Genesis()->IsAssumedValid()); } const auto& au_data = ::Params().AssumeutxoForHeight(snapshot_height); @@ -410,7 +407,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, SnapshotTestSetup) //! - First, verify that setBlockIndexCandidates is as expected when using a single, //! fully-validating chainstate. //! -//! - Then mark a region of the chain BLOCK_ASSUMED_VALID and introduce a second chainstate +//! - Then mark a region of the chain as missing data and introduce a second chainstate //! that will tolerate assumed-valid blocks. Run LoadBlockIndex() and ensure that the first //! chainstate only contains fully validated blocks and the other chainstate contains all blocks, //! except those marked assume-valid, because those entries don't HAVE_DATA. @@ -421,7 +418,6 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) Chainstate& cs1 = chainman.ActiveChainstate(); int num_indexes{0}; - int num_assumed_valid{0}; // Blocks in range [assumed_valid_start_idx, last_assumed_valid_idx) will be // marked as assumed-valid and not having data. const int expected_assumed_valid{20}; @@ -456,35 +452,30 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) reload_all_block_indexes(); BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), 1); - // Mark some region of the chain assumed-valid, and remove the HAVE_DATA flag. + // Reset some region of the chain's nStatus, removing the HAVE_DATA flag. for (int i = 0; i <= cs1.m_chain.Height(); ++i) { LOCK(::cs_main); auto index = cs1.m_chain[i]; - // Blocks with heights in range [91, 110] are marked ASSUMED_VALID + // Blocks with heights in range [91, 110] are marked as missing data. if (i < last_assumed_valid_idx && i >= assumed_valid_start_idx) { - index->nStatus = BlockStatus::BLOCK_VALID_TREE | BlockStatus::BLOCK_ASSUMED_VALID; + index->nStatus = BlockStatus::BLOCK_VALID_TREE; + index->nTx = 0; + index->nChainTx = 0; } ++num_indexes; - if (index->IsAssumedValid()) ++num_assumed_valid; // Note the last fully-validated block as the expected validated tip. if (i == (assumed_valid_start_idx - 1)) { validated_tip = index; - BOOST_CHECK(!index->IsAssumedValid()); } // Note the last assumed valid block as the snapshot base if (i == last_assumed_valid_idx - 1) { assumed_base = index; - BOOST_CHECK(index->IsAssumedValid()); - } else if (i == last_assumed_valid_idx) { - BOOST_CHECK(!index->IsAssumedValid()); } } - BOOST_CHECK_EQUAL(expected_assumed_valid, num_assumed_valid); - // Note: cs2's tip is not set when ActivateExistingSnapshot is called. Chainstate& cs2 = WITH_LOCK(::cs_main, return chainman.ActivateExistingSnapshot(*assumed_base->phashBlock)); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 0bee27c2b2..06066e38b2 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -17,6 +17,7 @@ #include <random.h> #include <reverse_iterator.h> #include <util/check.h> +#include <util/feefrac.h> #include <util/moneystr.h> #include <util/overflow.h> #include <util/result.h> @@ -1238,3 +1239,130 @@ std::vector<CTxMemPool::txiter> CTxMemPool::GatherClusters(const std::vector<uin } return clustered_txs; } + +std::optional<std::string> CTxMemPool::CheckConflictTopology(const setEntries& direct_conflicts) +{ + for (const auto& direct_conflict : direct_conflicts) { + // Ancestor and descendant counts are inclusive of the tx itself. + const auto ancestor_count{direct_conflict->GetCountWithAncestors()}; + const auto descendant_count{direct_conflict->GetCountWithDescendants()}; + const bool has_ancestor{ancestor_count > 1}; + const bool has_descendant{descendant_count > 1}; + const auto& txid_string{direct_conflict->GetSharedTx()->GetHash().ToString()}; + // The only allowed configurations are: + // 1 ancestor and 0 descendant + // 0 ancestor and 1 descendant + // 0 ancestor and 0 descendant + if (ancestor_count > 2) { + return strprintf("%s has %u ancestors, max 1 allowed", txid_string, ancestor_count - 1); + } else if (descendant_count > 2) { + return strprintf("%s has %u descendants, max 1 allowed", txid_string, descendant_count - 1); + } else if (has_ancestor && has_descendant) { + return strprintf("%s has both ancestor and descendant, exceeding cluster limit of 2", txid_string); + } + // Additionally enforce that: + // If we have a child, we are its only parent. + // If we have a parent, we are its only child. + if (has_descendant) { + const auto& our_child = direct_conflict->GetMemPoolChildrenConst().begin(); + if (our_child->get().GetCountWithAncestors() > 2) { + return strprintf("%s is not the only parent of child %s", + txid_string, our_child->get().GetSharedTx()->GetHash().ToString()); + } + } else if (has_ancestor) { + const auto& our_parent = direct_conflict->GetMemPoolParentsConst().begin(); + if (our_parent->get().GetCountWithDescendants() > 2) { + return strprintf("%s is not the only child of parent %s", + txid_string, our_parent->get().GetSharedTx()->GetHash().ToString()); + } + } + } + return std::nullopt; +} + +util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CTxMemPool::CalculateChunksForRBF(CAmount replacement_fees, int64_t replacement_vsize, const setEntries& direct_conflicts, const setEntries& all_conflicts) +{ + Assume(replacement_vsize > 0); + + auto err_string{CheckConflictTopology(direct_conflicts)}; + if (err_string.has_value()) { + // Unsupported topology for calculating a feerate diagram + return util::Error{Untranslated(err_string.value())}; + } + + // new diagram will have chunks that consist of each ancestor of + // direct_conflicts that is at its own fee/size, along with the replacement + // tx/package at its own fee/size + + // old diagram will consist of the ancestors and descendants of each element of + // all_conflicts. every such transaction will either be at its own feerate (followed + // by any descendant at its own feerate), or as a single chunk at the descendant's + // ancestor feerate. + + std::vector<FeeFrac> old_chunks; + // Step 1: build the old diagram. + + // The above clusters are all trivially linearized; + // they have a strict topology of 1 or two connected transactions. + + // OLD: Compute existing chunks from all affected clusters + for (auto txiter : all_conflicts) { + // Does this transaction have descendants? + if (txiter->GetCountWithDescendants() > 1) { + // Consider this tx when we consider the descendant. + continue; + } + // Does this transaction have ancestors? + FeeFrac individual{txiter->GetModifiedFee(), txiter->GetTxSize()}; + if (txiter->GetCountWithAncestors() > 1) { + // We'll add chunks for either the ancestor by itself and this tx + // by itself, or for a combined package. + FeeFrac package{txiter->GetModFeesWithAncestors(), static_cast<int32_t>(txiter->GetSizeWithAncestors())}; + if (individual >> package) { + // The individual feerate is higher than the package, and + // therefore higher than the parent's fee. Chunk these + // together. + old_chunks.emplace_back(package); + } else { + // Add two points, one for the parent and one for this child. + old_chunks.emplace_back(package - individual); + old_chunks.emplace_back(individual); + } + } else { + old_chunks.emplace_back(individual); + } + } + + // No topology restrictions post-chunking; sort + std::sort(old_chunks.begin(), old_chunks.end(), std::greater()); + + std::vector<FeeFrac> new_chunks; + + /* Step 2: build the NEW diagram + * CON = Conflicts of proposed chunk + * CNK = Proposed chunk + * NEW = OLD - CON + CNK: New diagram includes all chunks in OLD, minus + * the conflicts, plus the proposed chunk + */ + + // OLD - CON: Add any parents of direct conflicts that are not conflicted themselves + for (auto direct_conflict : direct_conflicts) { + // If a direct conflict has an ancestor that is not in all_conflicts, + // it can be affected by the replacement of the child. + if (direct_conflict->GetMemPoolParentsConst().size() > 0) { + // Grab the parent. + const CTxMemPoolEntry& parent = direct_conflict->GetMemPoolParentsConst().begin()->get(); + if (!all_conflicts.count(mapTx.iterator_to(parent))) { + // This transaction would be left over, so add to the NEW + // diagram. + new_chunks.emplace_back(parent.GetModifiedFee(), parent.GetTxSize()); + } + } + } + // + CNK: Add the proposed chunk itself + new_chunks.emplace_back(replacement_fees, int32_t(replacement_vsize)); + + // No topology restrictions post-chunking; sort + std::sort(new_chunks.begin(), new_chunks.end(), std::greater()); + return std::make_pair(old_chunks, new_chunks); +} diff --git a/src/txmempool.h b/src/txmempool.h index 32f2c024f7..52a3dc2d7d 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -21,6 +21,7 @@ #include <util/epochguard.h> #include <util/hasher.h> #include <util/result.h> +#include <util/feefrac.h> #include <boost/multi_index/hashed_index.hpp> #include <boost/multi_index/identity.hpp> @@ -736,6 +737,28 @@ public: return m_sequence_number; } + /** + * Calculate the sorted chunks for the old and new mempool relating to the + * clusters that would be affected by a potential replacement transaction. + * (replacement_fees, replacement_vsize) values are gathered from a + * proposed set of replacement transactions that are considered as a single + * chunk, and represent their complete cluster. In other words, they have no + * in-mempool ancestors. + * + * @param[in] replacement_fees Package fees + * @param[in] replacement_vsize Package size (must be greater than 0) + * @param[in] direct_conflicts All transactions that would be removed directly by + * having a conflicting input with a proposed transaction + * @param[in] all_conflicts All transactions that would be removed + * @return old and new diagram pair respectively, or an error string if the conflicts don't match a calculable topology + */ + util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CalculateChunksForRBF(CAmount replacement_fees, int64_t replacement_vsize, const setEntries& direct_conflicts, const setEntries& all_conflicts) EXCLUSIVE_LOCKS_REQUIRED(cs); + + /* Check that all direct conflicts are in a cluster size of two or less. Each + * direct conflict may be in a separate cluster. + */ + std::optional<std::string> CheckConflictTopology(const setEntries& direct_conflicts); + private: /** UpdateForDescendants is used by UpdateTransactionsFromBlock to update * the descendants for a single transaction that has been added to the diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index acbce25741..da12157555 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -18,6 +18,7 @@ #include <utility> #include <vector> +// NOLINTNEXTLINE(misc-no-recursion) class UniValue { public: enum VType { VNULL, VOBJ, VARR, VSTR, VNUM, VBOOL, }; diff --git a/src/univalue/lib/univalue_write.cpp b/src/univalue/lib/univalue_write.cpp index 4a3cbba20f..4a2219061a 100644 --- a/src/univalue/lib/univalue_write.cpp +++ b/src/univalue/lib/univalue_write.cpp @@ -27,6 +27,7 @@ static std::string json_escape(const std::string& inS) return outS; } +// NOLINTNEXTLINE(misc-no-recursion) std::string UniValue::write(unsigned int prettyIndent, unsigned int indentLevel) const { @@ -66,6 +67,7 @@ static void indentStr(unsigned int prettyIndent, unsigned int indentLevel, std:: s.append(prettyIndent * indentLevel, ' '); } +// NOLINTNEXTLINE(misc-no-recursion) void UniValue::writeArray(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const { s += "["; @@ -88,6 +90,7 @@ void UniValue::writeArray(unsigned int prettyIndent, unsigned int indentLevel, s s += "]"; } +// NOLINTNEXTLINE(misc-no-recursion) void UniValue::writeObject(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const { s += "{"; diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index 8b90448b36..1c724555f3 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -421,7 +421,7 @@ void univalue_readwrite() // Valid, with leading or trailing whitespace BOOST_CHECK(v.read(" 1.0") && (v.get_real() == 1.0)); BOOST_CHECK(v.read("1.0 ") && (v.get_real() == 1.0)); - BOOST_CHECK(v.read("0.00000000000000000000000000000000000001e+30 ") && v.get_real() == 1e-8); + BOOST_CHECK(v.read("0.00000000000000000000000000000000000001e+30 ")); BOOST_CHECK(!v.read(".19e-6")); //should fail, missing leading 0, therefore invalid JSON // Invalid, initial garbage diff --git a/src/util/feefrac.cpp b/src/util/feefrac.cpp new file mode 100644 index 0000000000..5b6173835c --- /dev/null +++ b/src/util/feefrac.cpp @@ -0,0 +1,73 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/feefrac.h> +#include <algorithm> +#include <array> +#include <vector> + +std::partial_ordering CompareChunks(Span<const FeeFrac> chunks0, Span<const FeeFrac> chunks1) +{ + /** Array to allow indexed access to input diagrams. */ + const std::array<Span<const FeeFrac>, 2> chunk = {chunks0, chunks1}; + /** How many elements we have processed in each input. */ + size_t next_index[2] = {0, 0}; + /** Accumulated fee/sizes in diagrams, up to next_index[i] - 1. */ + FeeFrac accum[2]; + /** Whether the corresponding input is strictly better than the other at least in one place. */ + bool better_somewhere[2] = {false, false}; + /** Get the first unprocessed point in diagram number dia. */ + const auto next_point = [&](int dia) { return chunk[dia][next_index[dia]] + accum[dia]; }; + /** Get the last processed point in diagram number dia. */ + const auto prev_point = [&](int dia) { return accum[dia]; }; + /** Move to the next point in diagram number dia. */ + const auto advance = [&](int dia) { accum[dia] += chunk[dia][next_index[dia]++]; }; + + do { + bool done_0 = next_index[0] == chunk[0].size(); + bool done_1 = next_index[1] == chunk[1].size(); + if (done_0 && done_1) break; + + // Determine which diagram has the first unprocessed point. If a single side is finished, use the + // other one. Only up to one can be done due to check above. + const int unproc_side = (done_0 || done_1) ? done_0 : next_point(0).size > next_point(1).size; + + // Let `P` be the next point on diagram unproc_side, and `A` and `B` the previous and next points + // on the other diagram. We want to know if P lies above or below the line AB. To determine this, we + // compute the slopes of line AB and of line AP, and compare them. These slopes are fee per size, + // and can thus be expressed as FeeFracs. + const FeeFrac& point_p = next_point(unproc_side); + const FeeFrac& point_a = prev_point(!unproc_side); + + const auto slope_ap = point_p - point_a; + Assume(slope_ap.size > 0); + std::weak_ordering cmp = std::weak_ordering::equivalent; + if (done_0 || done_1) { + // If a single side has no points left, act as if AB has slope tail_feerate(of 0). + Assume(!(done_0 && done_1)); + cmp = FeeRateCompare(slope_ap, FeeFrac(0, 1)); + } else { + // If both sides have points left, compute B, and the slope of AB explicitly. + const FeeFrac& point_b = next_point(!unproc_side); + const auto slope_ab = point_b - point_a; + Assume(slope_ab.size >= slope_ap.size); + cmp = FeeRateCompare(slope_ap, slope_ab); + + // If B and P have the same size, B can be marked as processed (in addition to P, see + // below), as we've already performed a comparison at this size. + if (point_b.size == point_p.size) advance(!unproc_side); + } + // If P lies above AB, unproc_side is better in P. If P lies below AB, then !unproc_side is + // better in P. + if (std::is_gt(cmp)) better_somewhere[unproc_side] = true; + if (std::is_lt(cmp)) better_somewhere[!unproc_side] = true; + advance(unproc_side); + + // If both diagrams are better somewhere, they are incomparable. + if (better_somewhere[0] && better_somewhere[1]) return std::partial_ordering::unordered; + } while(true); + + // Otherwise compare the better_somewhere values. + return better_somewhere[0] <=> better_somewhere[1]; +} diff --git a/src/util/feefrac.h b/src/util/feefrac.h new file mode 100644 index 0000000000..9772162010 --- /dev/null +++ b/src/util/feefrac.h @@ -0,0 +1,159 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_FEEFRAC_H +#define BITCOIN_UTIL_FEEFRAC_H + +#include <stdint.h> +#include <compare> +#include <vector> +#include <span.h> +#include <util/check.h> + +/** Data structure storing a fee and size, ordered by increasing fee/size. + * + * The size of a FeeFrac cannot be zero unless the fee is also zero. + * + * FeeFracs have a total ordering, first by increasing feerate (ratio of fee over size), and then + * by decreasing size. The empty FeeFrac (fee and size both 0) sorts last. So for example, the + * following FeeFracs are in sorted order: + * + * - fee=0 size=1 (feerate 0) + * - fee=1 size=2 (feerate 0.5) + * - fee=2 size=3 (feerate 0.667...) + * - fee=2 size=2 (feerate 1) + * - fee=1 size=1 (feerate 1) + * - fee=3 size=2 (feerate 1.5) + * - fee=2 size=1 (feerate 2) + * - fee=0 size=0 (undefined feerate) + * + * A FeeFrac is considered "better" if it sorts after another, by this ordering. All standard + * comparison operators (<=>, ==, !=, >, <, >=, <=) respect this ordering. + * + * The FeeRateCompare, and >> and << operators only compare feerate and treat equal feerate but + * different size as equivalent. The empty FeeFrac is neither lower or higher in feerate than any + * other. + */ +struct FeeFrac +{ + /** Fallback version for Mul (see below). + * + * Separate to permit testing on platforms where it isn't actually needed. + */ + static inline std::pair<int64_t, uint32_t> MulFallback(int64_t a, int32_t b) noexcept + { + // Otherwise, emulate 96-bit multiplication using two 64-bit multiplies. + int64_t low = int64_t{static_cast<uint32_t>(a)} * b; + int64_t high = (a >> 32) * b; + return {high + (low >> 32), static_cast<uint32_t>(low)}; + } + + // Compute a * b, returning an unspecified but totally ordered type. +#ifdef __SIZEOF_INT128__ + static inline __int128 Mul(int64_t a, int32_t b) noexcept + { + // If __int128 is available, use 128-bit wide multiply. + return __int128{a} * b; + } +#else + static constexpr auto Mul = MulFallback; +#endif + + int64_t fee; + int32_t size; + + /** Construct an IsEmpty() FeeFrac. */ + inline FeeFrac() noexcept : fee{0}, size{0} {} + + /** Construct a FeeFrac with specified fee and size. */ + inline FeeFrac(int64_t f, int32_t s) noexcept : fee{f}, size{s} {} + + inline FeeFrac(const FeeFrac&) noexcept = default; + inline FeeFrac& operator=(const FeeFrac&) noexcept = default; + + /** Check if this is empty (size and fee are 0). */ + bool inline IsEmpty() const noexcept { + return size == 0; + } + + /** Add fee and size of another FeeFrac to this one. */ + void inline operator+=(const FeeFrac& other) noexcept + { + fee += other.fee; + size += other.size; + } + + /** Subtract fee and size of another FeeFrac from this one. */ + void inline operator-=(const FeeFrac& other) noexcept + { + fee -= other.fee; + size -= other.size; + } + + /** Sum fee and size. */ + friend inline FeeFrac operator+(const FeeFrac& a, const FeeFrac& b) noexcept + { + return {a.fee + b.fee, a.size + b.size}; + } + + /** Subtract both fee and size. */ + friend inline FeeFrac operator-(const FeeFrac& a, const FeeFrac& b) noexcept + { + return {a.fee - b.fee, a.size - b.size}; + } + + /** Check if two FeeFrac objects are equal (both same fee and same size). */ + friend inline bool operator==(const FeeFrac& a, const FeeFrac& b) noexcept + { + return a.fee == b.fee && a.size == b.size; + } + + /** Compare two FeeFracs just by feerate. */ + friend inline std::weak_ordering FeeRateCompare(const FeeFrac& a, const FeeFrac& b) noexcept + { + auto cross_a = Mul(a.fee, b.size), cross_b = Mul(b.fee, a.size); + return cross_a <=> cross_b; + } + + /** Check if a FeeFrac object has strictly lower feerate than another. */ + friend inline bool operator<<(const FeeFrac& a, const FeeFrac& b) noexcept + { + auto cross_a = Mul(a.fee, b.size), cross_b = Mul(b.fee, a.size); + return cross_a < cross_b; + } + + /** Check if a FeeFrac object has strictly higher feerate than another. */ + friend inline bool operator>>(const FeeFrac& a, const FeeFrac& b) noexcept + { + auto cross_a = Mul(a.fee, b.size), cross_b = Mul(b.fee, a.size); + return cross_a > cross_b; + } + + /** Compare two FeeFracs. <, >, <=, and >= are auto-generated from this. */ + friend inline std::strong_ordering operator<=>(const FeeFrac& a, const FeeFrac& b) noexcept + { + auto cross_a = Mul(a.fee, b.size), cross_b = Mul(b.fee, a.size); + if (cross_a == cross_b) return b.size <=> a.size; + return cross_a <=> cross_b; + } + + /** Swap two FeeFracs. */ + friend inline void swap(FeeFrac& a, FeeFrac& b) noexcept + { + std::swap(a.fee, b.fee); + std::swap(a.size, b.size); + } +}; + +/** Compare the feerate diagrams implied by the provided sorted chunks data. + * + * The implied diagram for each starts at (0, 0), then contains for each chunk the cumulative fee + * and size up to that chunk, and then extends infinitely to the right with a horizontal line. + * + * The caller must guarantee that the sum of the FeeFracs in either of the chunks' data set do not + * overflow (so sum fees < 2^63, and sum sizes < 2^31). + */ +std::partial_ordering CompareChunks(Span<const FeeFrac> chunks0, Span<const FeeFrac> chunks1); + +#endif // BITCOIN_UTIL_FEEFRAC_H diff --git a/src/util/fs_helpers.cpp b/src/util/fs_helpers.cpp index 4de8833a3f..bce5602462 100644 --- a/src/util/fs_helpers.cpp +++ b/src/util/fs_helpers.cpp @@ -69,7 +69,7 @@ LockResult LockDirectory(const fs::path& directory, const fs::path& lockfile_nam } auto lock = std::make_unique<fsbridge::FileLock>(pathLockFile); if (!lock->TryLock()) { - error("Error while attempting to lock directory %s: %s", fs::PathToString(directory), lock->GetReason()); + LogError("Error while attempting to lock directory %s: %s\n", fs::PathToString(directory), lock->GetReason()); return LockResult::ErrorLock; } if (!probe_only) { @@ -249,20 +249,9 @@ fs::path GetSpecialFolderPath(int nFolder, bool fCreate) bool RenameOver(fs::path src, fs::path dest) { -#ifdef __MINGW64__ - // This is a workaround for a bug in libstdc++ which - // implements fs::rename with _wrename function. - // This bug has been fixed in upstream: - // - GCC 10.3: 8dd1c1085587c9f8a21bb5e588dfe1e8cdbba79e - // - GCC 11.1: 1dfd95f0a0ca1d9e6cbc00e6cbfd1fa20a98f312 - // For more details see the commits mentioned above. - return MoveFileExW(src.wstring().c_str(), dest.wstring().c_str(), - MOVEFILE_REPLACE_EXISTING) != 0; -#else std::error_code error; fs::rename(src, dest, error); return !error; -#endif } /** diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index a54f408496..7b5ded2975 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -81,6 +81,8 @@ template <typename Byte> std::optional<std::vector<Byte>> TryParseHex(std::string_view str) { std::vector<Byte> vch; + vch.reserve(str.size() / 2); // two hex characters form a single byte + auto it = str.begin(); while (it != str.end()) { if (IsSpace(*it)) { @@ -444,6 +446,7 @@ bool ParseFixedPoint(std::string_view val, int decimals, int64_t *amount_out) std::string ToLower(std::string_view str) { std::string r; + r.reserve(str.size()); for (auto ch : str) r += ToLower(ch); return r; } @@ -451,6 +454,7 @@ std::string ToLower(std::string_view str) std::string ToUpper(std::string_view str) { std::string r; + r.reserve(str.size()); for (auto ch : str) r += ToUpper(ch); return r; } diff --git a/src/util/string.h b/src/util/string.h index 8b69d6ccae..dab92942fb 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -65,6 +65,7 @@ void ReplaceAll(std::string& in_out, const std::string& search, const std::strin * @param unary_op Apply this operator to each item */ template <typename C, typename S, typename UnaryOp> +// NOLINTNEXTLINE(misc-no-recursion) auto Join(const C& container, const S& separator, UnaryOp unary_op) { decltype(unary_op(*container.begin())) ret; diff --git a/src/util/subprocess.h b/src/util/subprocess.h new file mode 100644 index 0000000000..4acfa8ff83 --- /dev/null +++ b/src/util/subprocess.h @@ -0,0 +1,1614 @@ +// Based on the https://github.com/arun11299/cpp-subprocess project. + +/*! + +Documentation for C++ subprocessing library. + +@copyright The code is licensed under the [MIT + License](http://opensource.org/licenses/MIT): + <br> + Copyright © 2016-2018 Arun Muralidharan. + <br> + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + <br> + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + <br> + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +@author [Arun Muralidharan] +@see https://github.com/arun11299/cpp-subprocess to download the source code + +@version 1.0.0 +*/ + +#ifndef BITCOIN_UTIL_SUBPROCESS_H +#define BITCOIN_UTIL_SUBPROCESS_H + +#include <util/syserror.h> + +#include <algorithm> +#include <cassert> +#include <csignal> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <exception> +#include <future> +#include <initializer_list> +#include <iostream> +#include <locale> +#include <map> +#include <memory> +#include <sstream> +#include <string> +#include <vector> + +#if (defined _MSC_VER) || (defined __MINGW32__) + #define __USING_WINDOWS__ +#endif + +#ifdef __USING_WINDOWS__ + #include <codecvt> +#endif + +extern "C" { +#ifdef __USING_WINDOWS__ + #include <Windows.h> + #include <io.h> + #include <cwchar> + + #define close _close + #define open _open + #define fileno _fileno +#else + #include <sys/wait.h> + #include <unistd.h> +#endif + #include <csignal> + #include <fcntl.h> + #include <sys/types.h> +} + +/*! + * Getting started with reading this source code. + * The source is mainly divided into four parts: + * 1. Exception Classes: + * These are very basic exception classes derived from + * runtime_error exception. + * There are two types of exception thrown from subprocess + * library: OSError and CalledProcessError + * + * 2. Popen Class + * This is the main class the users will deal with. It + * provides with all the API's to deal with processes. + * + * 3. Util namespace + * It includes some helper functions to split/join a string, + * reading from file descriptors, waiting on a process, fcntl + * options on file descriptors etc. + * + * 4. Detail namespace + * This includes some metaprogramming and helper classes. + */ + + +namespace subprocess { + +// Max buffer size allocated on stack for read error +// from pipe +static const size_t SP_MAX_ERR_BUF_SIZ = 1024; + +// Default buffer capacity for OutBuffer and ErrBuffer. +// If the data exceeds this capacity, the buffer size is grown +// by 1.5 times its previous capacity +static const size_t DEFAULT_BUF_CAP_BYTES = 8192; + + +/*----------------------------------------------- + * EXCEPTION CLASSES + *----------------------------------------------- + */ + +/*! + * class: CalledProcessError + * Thrown when there was error executing the command. + * Check Popen class API's to know when this exception + * can be thrown. + * + */ +class CalledProcessError: public std::runtime_error +{ +public: + int retcode; + CalledProcessError(const std::string& error_msg, int retcode): + std::runtime_error(error_msg), retcode(retcode) + {} +}; + + +/*! + * class: OSError + * Thrown when some system call fails to execute or give result. + * The exception message contains the name of the failed system call + * with the stringisized errno code. + * Check Popen class API's to know when this exception would be + * thrown. + * Its usual that the API exception specification would have + * this exception together with CalledProcessError. + */ +class OSError: public std::runtime_error +{ +public: + OSError(const std::string& err_msg, int err_code): + std::runtime_error(err_msg + ": " + SysErrorString(err_code)) + {} +}; + +//-------------------------------------------------------------------- +namespace util +{ + template <typename R> + inline bool is_ready(std::shared_future<R> const &f) + { + return f.wait_for(std::chrono::seconds(0)) == std::future_status::ready; + } + + inline void quote_argument(const std::wstring &argument, std::wstring &command_line, + bool force) + { + // + // Unless we're told otherwise, don't quote unless we actually + // need to do so --- hopefully avoid problems if programs won't + // parse quotes properly + // + + if (force == false && argument.empty() == false && + argument.find_first_of(L" \t\n\v\"") == argument.npos) { + command_line.append(argument); + } + else { + command_line.push_back(L'"'); + + for (auto it = argument.begin();; ++it) { + unsigned number_backslashes = 0; + + while (it != argument.end() && *it == L'\\') { + ++it; + ++number_backslashes; + } + + if (it == argument.end()) { + + // + // Escape all backslashes, but let the terminating + // double quotation mark we add below be interpreted + // as a metacharacter. + // + + command_line.append(number_backslashes * 2, L'\\'); + break; + } + else if (*it == L'"') { + + // + // Escape all backslashes and the following + // double quotation mark. + // + + command_line.append(number_backslashes * 2 + 1, L'\\'); + command_line.push_back(*it); + } + else { + + // + // Backslashes aren't special here. + // + + command_line.append(number_backslashes, L'\\'); + command_line.push_back(*it); + } + } + + command_line.push_back(L'"'); + } + } + +#ifdef __USING_WINDOWS__ + inline std::string get_last_error(DWORD errorMessageID) + { + if (errorMessageID == 0) + return std::string(); + + LPSTR messageBuffer = nullptr; + size_t size = FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_MAX_WIDTH_MASK, + NULL, errorMessageID, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPSTR)&messageBuffer, 0, NULL); + + std::string message(messageBuffer, size); + + LocalFree(messageBuffer); + + return message; + } + + inline FILE *file_from_handle(HANDLE h, const char *mode) + { + int md; + if (!mode) { + throw OSError("invalid_mode", 0); + } + + if (mode[0] == 'w') { + md = _O_WRONLY; + } + else if (mode[0] == 'r') { + md = _O_RDONLY; + } + else { + throw OSError("file_from_handle", 0); + } + + int os_fhandle = _open_osfhandle((intptr_t)h, md); + if (os_fhandle == -1) { + CloseHandle(h); + throw OSError("_open_osfhandle", 0); + } + + FILE *fp = _fdopen(os_fhandle, mode); + if (fp == 0) { + _close(os_fhandle); + throw OSError("_fdopen", 0); + } + + return fp; + } + + inline void configure_pipe(HANDLE* read_handle, HANDLE* write_handle, HANDLE* child_handle) + { + SECURITY_ATTRIBUTES saAttr; + + // Set the bInheritHandle flag so pipe handles are inherited. + saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); + saAttr.bInheritHandle = TRUE; + saAttr.lpSecurityDescriptor = NULL; + + // Create a pipe for the child process's STDIN. + if (!CreatePipe(read_handle, write_handle, &saAttr,0)) + throw OSError("CreatePipe", 0); + + // Ensure the write handle to the pipe for STDIN is not inherited. + if (!SetHandleInformation(*child_handle, HANDLE_FLAG_INHERIT, 0)) + throw OSError("SetHandleInformation", 0); + } +#endif + + /*! + * Function: split + * Parameters: + * [in] str : Input string which needs to be split based upon the + * delimiters provided. + * [in] deleims : Delimiter characters based upon which the string needs + * to be split. Default constructed to ' '(space) and '\t'(tab) + * [out] vector<string> : Vector of strings split at deleimiter. + */ + static inline std::vector<std::string> + split(const std::string& str, const std::string& delims=" \t") + { + std::vector<std::string> res; + size_t init = 0; + + while (true) { + auto pos = str.find_first_of(delims, init); + if (pos == std::string::npos) { + res.emplace_back(str.substr(init, str.length())); + break; + } + res.emplace_back(str.substr(init, pos - init)); + pos++; + init = pos; + } + + return res; + } + + +#ifndef __USING_WINDOWS__ + /*! + * Function: set_clo_on_exec + * Sets/Resets the FD_CLOEXEC flag on the provided file descriptor + * based upon the `set` parameter. + * Parameters: + * [in] fd : The descriptor on which FD_CLOEXEC needs to be set/reset. + * [in] set : If 'true', set FD_CLOEXEC. + * If 'false' unset FD_CLOEXEC. + */ + static inline + void set_clo_on_exec(int fd, bool set = true) + { + int flags = fcntl(fd, F_GETFD, 0); + if (set) flags |= FD_CLOEXEC; + else flags &= ~FD_CLOEXEC; + //TODO: should check for errors + fcntl(fd, F_SETFD, flags); + } + + + /*! + * Function: pipe_cloexec + * Creates a pipe and sets FD_CLOEXEC flag on both + * read and write descriptors of the pipe. + * Parameters: + * [out] : A pair of file descriptors. + * First element of pair is the read descriptor of pipe. + * Second element is the write descriptor of pipe. + */ + static inline + std::pair<int, int> pipe_cloexec() noexcept(false) + { + int pipe_fds[2]; + int res = pipe(pipe_fds); + if (res) { + throw OSError("pipe failure", errno); + } + + set_clo_on_exec(pipe_fds[0]); + set_clo_on_exec(pipe_fds[1]); + + return std::make_pair(pipe_fds[0], pipe_fds[1]); + } +#endif + + + /*! + * Function: write_n + * Writes `length` bytes to the file descriptor `fd` + * from the buffer `buf`. + * Parameters: + * [in] fd : The file descriptotr to write to. + * [in] buf: Buffer from which data needs to be written to fd. + * [in] length: The number of bytes that needs to be written from + * `buf` to `fd`. + * [out] int : Number of bytes written or -1 in case of failure. + */ + static inline + int write_n(int fd, const char* buf, size_t length) + { + size_t nwritten = 0; + while (nwritten < length) { + int written = write(fd, buf + nwritten, length - nwritten); + if (written == -1) return -1; + nwritten += written; + } + return nwritten; + } + + + /*! + * Function: read_atmost_n + * Reads at the most `read_upto` bytes from the + * file object `fp` before returning. + * Parameters: + * [in] fp : The file object from which it needs to read. + * [in] buf : The buffer into which it needs to write the data. + * [in] read_upto: Max number of bytes which must be read from `fd`. + * [out] int : Number of bytes written to `buf` or read from `fd` + * OR -1 in case of error. + * NOTE: In case of EINTR while reading from socket, this API + * will retry to read from `fd`, but only till the EINTR counter + * reaches 50 after which it will return with whatever data it read. + */ + static inline + int read_atmost_n(FILE* fp, char* buf, size_t read_upto) + { +#ifdef __USING_WINDOWS__ + return (int)fread(buf, 1, read_upto, fp); +#else + int fd = fileno(fp); + int rbytes = 0; + int eintr_cnter = 0; + + while (1) { + int read_bytes = read(fd, buf + rbytes, read_upto - rbytes); + if (read_bytes == -1) { + if (errno == EINTR) { + if (eintr_cnter >= 50) return -1; + eintr_cnter++; + continue; + } + return -1; + } + if (read_bytes == 0) return rbytes; + + rbytes += read_bytes; + } + return rbytes; +#endif + } + + + /*! + * Function: read_all + * Reads all the available data from `fp` into + * `buf`. Internally calls read_atmost_n. + * Parameters: + * [in] fp : The file object from which to read from. + * [in] buf : The buffer of type `class Buffer` into which + * the read data is written to. + * [out] int: Number of bytes read OR -1 in case of failure. + * + * NOTE: `class Buffer` is a exposed public class. See below. + */ + + static inline int read_all(FILE* fp, std::vector<char>& buf) + { + auto buffer = buf.data(); + int total_bytes_read = 0; + int fill_sz = buf.size(); + + while (1) { + const int rd_bytes = read_atmost_n(fp, buffer, fill_sz); + + if (rd_bytes == -1) { // Read finished + if (total_bytes_read == 0) return -1; + break; + + } else if (rd_bytes == fill_sz) { // Buffer full + const auto orig_sz = buf.size(); + const auto new_sz = orig_sz * 2; + buf.resize(new_sz); + fill_sz = new_sz - orig_sz; + + //update the buffer pointer + buffer = buf.data(); + total_bytes_read += rd_bytes; + buffer += total_bytes_read; + + } else { // Partial data ? Continue reading + total_bytes_read += rd_bytes; + fill_sz -= rd_bytes; + break; + } + } + buf.erase(buf.begin()+total_bytes_read, buf.end()); // remove extra nulls + return total_bytes_read; + } + +#ifndef __USING_WINDOWS__ + /*! + * Function: wait_for_child_exit + * Waits for the process with pid `pid` to exit + * and returns its status. + * Parameters: + * [in] pid : The pid of the process. + * [out] pair<int, int>: + * pair.first : Return code of the waitpid call. + * pair.second : Exit status of the process. + * + * NOTE: This is a blocking call as in, it will loop + * till the child is exited. + */ + static inline + std::pair<int, int> wait_for_child_exit(int pid) + { + int status = 0; + int ret = -1; + while (1) { + ret = waitpid(pid, &status, 0); + if (ret == -1) break; + if (ret == 0) continue; + return std::make_pair(ret, status); + } + + return std::make_pair(ret, status); + } +#endif + +} // end namespace util + + + +/* ------------------------------- + * Popen Arguments + * ------------------------------- + */ + +/*! + * Base class for all arguments involving string value. + */ +struct string_arg +{ + string_arg(const char* arg): arg_value(arg) {} + string_arg(std::string&& arg): arg_value(std::move(arg)) {} + string_arg(std::string arg): arg_value(std::move(arg)) {} + std::string arg_value; +}; + +/*! + * Option to specify the executable name separately + * from the args sequence. + * In this case the cmd args must only contain the + * options required for this executable. + * + * Eg: executable{"ls"} + */ +struct executable: string_arg +{ + template <typename T> + executable(T&& arg): string_arg(std::forward<T>(arg)) {} +}; + +/*! + * Used for redirecting input/output/error + */ +enum IOTYPE { + STDOUT = 1, + STDERR, + PIPE, +}; + +//TODO: A common base/interface for below stream structures ?? + +/*! + * Option to specify the input channel for the child + * process. It can be: + * 1. An already open file descriptor. + * 2. A file name. + * 3. IOTYPE. Usual a PIPE + * + * Eg: input{PIPE} + * OR in case of redirection, output of another Popen + * input{popen.output()} + */ +struct input +{ + // For an already existing file descriptor. + explicit input(int fd): rd_ch_(fd) {} + + // FILE pointer. + explicit input (FILE* fp):input(fileno(fp)) { assert(fp); } + + explicit input(const char* filename) { + int fd = open(filename, O_RDONLY); + if (fd == -1) throw OSError("File not found: ", errno); + rd_ch_ = fd; + } + explicit input(IOTYPE typ) { + assert (typ == PIPE && "STDOUT/STDERR not allowed"); +#ifndef __USING_WINDOWS__ + std::tie(rd_ch_, wr_ch_) = util::pipe_cloexec(); +#endif + } + + int rd_ch_ = -1; + int wr_ch_ = -1; +}; + + +/*! + * Option to specify the output channel for the child + * process. It can be: + * 1. An already open file descriptor. + * 2. A file name. + * 3. IOTYPE. Usually a PIPE. + * + * Eg: output{PIPE} + * OR output{"output.txt"} + */ +struct output +{ + explicit output(int fd): wr_ch_(fd) {} + + explicit output (FILE* fp):output(fileno(fp)) { assert(fp); } + + explicit output(const char* filename) { + int fd = open(filename, O_APPEND | O_CREAT | O_RDWR, 0640); + if (fd == -1) throw OSError("File not found: ", errno); + wr_ch_ = fd; + } + explicit output(IOTYPE typ) { + assert (typ == PIPE && "STDOUT/STDERR not allowed"); +#ifndef __USING_WINDOWS__ + std::tie(rd_ch_, wr_ch_) = util::pipe_cloexec(); +#endif + } + + int rd_ch_ = -1; + int wr_ch_ = -1; +}; + + +/*! + * Option to specify the error channel for the child + * process. It can be: + * 1. An already open file descriptor. + * 2. A file name. + * 3. IOTYPE. Usually a PIPE or STDOUT + * + */ +struct error +{ + explicit error(int fd): wr_ch_(fd) {} + + explicit error(FILE* fp):error(fileno(fp)) { assert(fp); } + + explicit error(const char* filename) { + int fd = open(filename, O_APPEND | O_CREAT | O_RDWR, 0640); + if (fd == -1) throw OSError("File not found: ", errno); + wr_ch_ = fd; + } + explicit error(IOTYPE typ) { + assert ((typ == PIPE || typ == STDOUT) && "STDERR not allowed"); + if (typ == PIPE) { +#ifndef __USING_WINDOWS__ + std::tie(rd_ch_, wr_ch_) = util::pipe_cloexec(); +#endif + } else { + // Need to defer it till we have checked all arguments + deferred_ = true; + } + } + + bool deferred_ = false; + int rd_ch_ = -1; + int wr_ch_ = -1; +}; + +// ~~~~ End Popen Args ~~~~ + + +/*! + * class: Buffer + * This class is a very thin wrapper around std::vector<char> + * This is basically used to determine the length of the actual + * data stored inside the dynamically resized vector. + * + * This is what is returned as the output to communicate and check_output + * functions, so, users must know about this class. + * + * OutBuffer and ErrBuffer are just different typedefs to this class. + */ +class Buffer +{ +public: + Buffer() {} + explicit Buffer(size_t cap) { buf.resize(cap); } + void add_cap(size_t cap) { buf.resize(cap); } + +#if 0 + Buffer(const Buffer& other): + buf(other.buf), + length(other.length) + { + std::cout << "COPY" << std::endl; + } + + Buffer(Buffer&& other): + buf(std::move(other.buf)), + length(other.length) + { + std::cout << "MOVE" << std::endl; + } +#endif + +public: + std::vector<char> buf; + size_t length = 0; +}; + +// Buffer for storing output written to output fd +using OutBuffer = Buffer; +// Buffer for storing output written to error fd +using ErrBuffer = Buffer; + + +// Fwd Decl. +class Popen; + +/*--------------------------------------------------- + * DETAIL NAMESPACE + *--------------------------------------------------- + */ + +namespace detail { + +// Metaprogram for searching a type within +// a variadic parameter pack +// This is particularly required to do a compile time +// checking of the arguments provided to 'check_output' function +// wherein the user is not expected to provide an 'output' option. + +template <typename... T> struct param_pack{}; + +template <typename F, typename T> struct has_type; + +template <typename F> +struct has_type<F, param_pack<>> { + static constexpr bool value = false; +}; + +template <typename F, typename... T> +struct has_type<F, param_pack<F, T...>> { + static constexpr bool value = true; +}; + +template <typename F, typename H, typename... T> +struct has_type<F, param_pack<H,T...>> { + static constexpr bool value = + std::is_same<F, typename std::decay<H>::type>::value ? true : has_type<F, param_pack<T...>>::value; +}; + +//---- + +/*! + * A helper class to Popen class for setting + * options as provided in the Popen constructor + * or in check_output arguments. + * This design allows us to _not_ have any fixed position + * to any arguments and specify them in a way similar to what + * can be done in python. + */ +struct ArgumentDeducer +{ + ArgumentDeducer(Popen* p): popen_(p) {} + + void set_option(executable&& exe); + void set_option(input&& inp); + void set_option(output&& out); + void set_option(error&& err); + +private: + Popen* popen_ = nullptr; +}; + +/*! + * A helper class to Popen. + * This takes care of all the fork-exec logic + * in the execute_child API. + */ +class Child +{ +public: + Child(Popen* p, int err_wr_pipe): + parent_(p), + err_wr_pipe_(err_wr_pipe) + {} + + void execute_child(); + +private: + // Lets call it parent even though + // technically a bit incorrect + Popen* parent_ = nullptr; + int err_wr_pipe_ = -1; +}; + +// Fwd Decl. +class Streams; + +/*! + * A helper class to Streams. + * This takes care of management of communicating + * with the child process with the means of the correct + * file descriptor. + */ +class Communication +{ +public: + Communication(Streams* stream): stream_(stream) + {} + void operator=(const Communication&) = delete; +public: + int send(const char* msg, size_t length); + int send(const std::vector<char>& msg); + + std::pair<OutBuffer, ErrBuffer> communicate(const char* msg, size_t length); + std::pair<OutBuffer, ErrBuffer> communicate(const std::vector<char>& msg) + { return communicate(msg.data(), msg.size()); } + + void set_out_buf_cap(size_t cap) { out_buf_cap_ = cap; } + void set_err_buf_cap(size_t cap) { err_buf_cap_ = cap; } + +private: + std::pair<OutBuffer, ErrBuffer> communicate_threaded( + const char* msg, size_t length); + +private: + Streams* stream_; + size_t out_buf_cap_ = DEFAULT_BUF_CAP_BYTES; + size_t err_buf_cap_ = DEFAULT_BUF_CAP_BYTES; +}; + + + +/*! + * This is a helper class to Popen. + * It takes care of management of all the file descriptors + * and file pointers. + * It dispatches of the communication aspects to the + * Communication class. + * Read through the data members to understand about the + * various file descriptors used. + */ +class Streams +{ +public: + Streams():comm_(this) {} + void operator=(const Streams&) = delete; + +public: + void setup_comm_channels(); + + void cleanup_fds() + { + if (write_to_child_ != -1 && read_from_parent_ != -1) { + close(write_to_child_); + } + if (write_to_parent_ != -1 && read_from_child_ != -1) { + close(read_from_child_); + } + if (err_write_ != -1 && err_read_ != -1) { + close(err_read_); + } + } + + void close_parent_fds() + { + if (write_to_child_ != -1) close(write_to_child_); + if (read_from_child_ != -1) close(read_from_child_); + if (err_read_ != -1) close(err_read_); + } + + void close_child_fds() + { + if (write_to_parent_ != -1) close(write_to_parent_); + if (read_from_parent_ != -1) close(read_from_parent_); + if (err_write_ != -1) close(err_write_); + } + + FILE* input() { return input_.get(); } + FILE* output() { return output_.get(); } + FILE* error() { return error_.get(); } + + void input(FILE* fp) { input_.reset(fp, fclose); } + void output(FILE* fp) { output_.reset(fp, fclose); } + void error(FILE* fp) { error_.reset(fp, fclose); } + + void set_out_buf_cap(size_t cap) { comm_.set_out_buf_cap(cap); } + void set_err_buf_cap(size_t cap) { comm_.set_err_buf_cap(cap); } + +public: /* Communication forwarding API's */ + int send(const char* msg, size_t length) + { return comm_.send(msg, length); } + + int send(const std::vector<char>& msg) + { return comm_.send(msg); } + + std::pair<OutBuffer, ErrBuffer> communicate(const char* msg, size_t length) + { return comm_.communicate(msg, length); } + + std::pair<OutBuffer, ErrBuffer> communicate(const std::vector<char>& msg) + { return comm_.communicate(msg); } + + +public:// Yes they are public + + std::shared_ptr<FILE> input_ = nullptr; + std::shared_ptr<FILE> output_ = nullptr; + std::shared_ptr<FILE> error_ = nullptr; + +#ifdef __USING_WINDOWS__ + HANDLE g_hChildStd_IN_Rd = nullptr; + HANDLE g_hChildStd_IN_Wr = nullptr; + HANDLE g_hChildStd_OUT_Rd = nullptr; + HANDLE g_hChildStd_OUT_Wr = nullptr; + HANDLE g_hChildStd_ERR_Rd = nullptr; + HANDLE g_hChildStd_ERR_Wr = nullptr; +#endif + + // Pipes for communicating with child + + // Emulates stdin + int write_to_child_ = -1; // Parent owned descriptor + int read_from_parent_ = -1; // Child owned descriptor + + // Emulates stdout + int write_to_parent_ = -1; // Child owned descriptor + int read_from_child_ = -1; // Parent owned descriptor + + // Emulates stderr + int err_write_ = -1; // Write error to parent (Child owned) + int err_read_ = -1; // Read error from child (Parent owned) + +private: + Communication comm_; +}; + +} // end namespace detail + + + +/*! + * class: Popen + * This is the single most important class in the whole library + * and glues together all the helper classes to provide a common + * interface to the client. + * + * API's provided by the class: + * 1. Popen({"cmd"}, output{..}, error{..}, ....) + * Command provided as a sequence. + * 2. Popen("cmd arg1"m output{..}, error{..}, ....) + * Command provided in a single string. + * 3. wait() - Wait for the child to exit. + * 4. retcode() - The return code of the exited child. + * 5. pid() - PID of the spawned child. + * 6. poll() - Check the status of the running child. + * 7. kill(sig_num) - Kill the child. SIGTERM used by default. + * 8. send(...) - Send input to the input channel of the child. + * 9. communicate(...) - Get the output/error from the child and close the channels + * from the parent side. + *10. input() - Get the input channel/File pointer. Can be used for + * customizing the way of sending input to child. + *11. output() - Get the output channel/File pointer. Usually used + in case of redirection. See piping examples. + *12. error() - Get the error channel/File pointer. Usually used + in case of redirection. + */ +class Popen +{ +public: + friend struct detail::ArgumentDeducer; + friend class detail::Child; + + template <typename... Args> + Popen(const std::string& cmd_args, Args&& ...args): + args_(cmd_args) + { + vargs_ = util::split(cmd_args); + init_args(std::forward<Args>(args)...); + + // Setup the communication channels of the Popen class + stream_.setup_comm_channels(); + + execute_process(); + } + + template <typename... Args> + Popen(std::initializer_list<const char*> cmd_args, Args&& ...args) + { + vargs_.insert(vargs_.end(), cmd_args.begin(), cmd_args.end()); + init_args(std::forward<Args>(args)...); + + // Setup the communication channels of the Popen class + stream_.setup_comm_channels(); + + execute_process(); + } + + template <typename... Args> + Popen(std::vector<std::string> vargs_, Args &&... args) : vargs_(vargs_) + { + init_args(std::forward<Args>(args)...); + + // Setup the communication channels of the Popen class + stream_.setup_comm_channels(); + + execute_process(); + } + +/* + ~Popen() + { +#ifdef __USING_WINDOWS__ + CloseHandle(this->process_handle_); +#endif + } +*/ + + int pid() const noexcept { return child_pid_; } + + int retcode() const noexcept { return retcode_; } + + int wait() noexcept(false); + + int poll() noexcept(false); + + // Does not fail, Caller is expected to recheck the + // status with a call to poll() + void kill(int sig_num = 9); + + void set_out_buf_cap(size_t cap) { stream_.set_out_buf_cap(cap); } + + void set_err_buf_cap(size_t cap) { stream_.set_err_buf_cap(cap); } + + int send(const char* msg, size_t length) + { return stream_.send(msg, length); } + + int send(const std::string& msg) + { return send(msg.c_str(), msg.size()); } + + int send(const std::vector<char>& msg) + { return stream_.send(msg); } + + std::pair<OutBuffer, ErrBuffer> communicate(const char* msg, size_t length) + { + auto res = stream_.communicate(msg, length); + retcode_ = wait(); + return res; + } + + std::pair<OutBuffer, ErrBuffer> communicate(const std::string& msg) + { + return communicate(msg.c_str(), msg.size()); + } + + std::pair<OutBuffer, ErrBuffer> communicate(const std::vector<char>& msg) + { + auto res = stream_.communicate(msg); + retcode_ = wait(); + return res; + } + + std::pair<OutBuffer, ErrBuffer> communicate() + { + return communicate(nullptr, 0); + } + + FILE* input() { return stream_.input(); } + FILE* output() { return stream_.output();} + FILE* error() { return stream_.error(); } + + /// Stream close APIs + void close_input() { stream_.input_.reset(); } + void close_output() { stream_.output_.reset(); } + void close_error() { stream_.error_.reset(); } + +private: + template <typename F, typename... Args> + void init_args(F&& farg, Args&&... args); + void init_args(); + void populate_c_argv(); + void execute_process() noexcept(false); + +private: + detail::Streams stream_; + +#ifdef __USING_WINDOWS__ + HANDLE process_handle_; + std::future<void> cleanup_future_; +#endif + + std::string exe_name_; + + // Command in string format + std::string args_; + // Command provided as sequence + std::vector<std::string> vargs_; + std::vector<char*> cargv_; + + bool child_created_ = false; + // Pid of the child process + int child_pid_ = -1; + + int retcode_ = -1; +}; + +inline void Popen::init_args() { + populate_c_argv(); +} + +template <typename F, typename... Args> +inline void Popen::init_args(F&& farg, Args&&... args) +{ + detail::ArgumentDeducer argd(this); + argd.set_option(std::forward<F>(farg)); + init_args(std::forward<Args>(args)...); +} + +inline void Popen::populate_c_argv() +{ + cargv_.clear(); + cargv_.reserve(vargs_.size() + 1); + for (auto& arg : vargs_) cargv_.push_back(&arg[0]); + cargv_.push_back(nullptr); +} + +inline int Popen::wait() noexcept(false) +{ +#ifdef __USING_WINDOWS__ + int ret = WaitForSingleObject(process_handle_, INFINITE); + + return 0; +#else + int ret, status; + std::tie(ret, status) = util::wait_for_child_exit(pid()); + if (ret == -1) { + if (errno != ECHILD) throw OSError("waitpid failed", errno); + return 0; + } + if (WIFEXITED(status)) return WEXITSTATUS(status); + if (WIFSIGNALED(status)) return WTERMSIG(status); + else return 255; + + return 0; +#endif +} + +inline int Popen::poll() noexcept(false) +{ +#ifdef __USING_WINDOWS__ + int ret = WaitForSingleObject(process_handle_, 0); + if (ret != WAIT_OBJECT_0) return -1; + + DWORD dretcode_; + if (FALSE == GetExitCodeProcess(process_handle_, &dretcode_)) + throw OSError("GetExitCodeProcess", 0); + + retcode_ = (int)dretcode_; + CloseHandle(process_handle_); + + return retcode_; +#else + if (!child_created_) return -1; // TODO: ?? + + int status; + + // Returns zero if child is still running + int ret = waitpid(child_pid_, &status, WNOHANG); + if (ret == 0) return -1; + + if (ret == child_pid_) { + if (WIFSIGNALED(status)) { + retcode_ = WTERMSIG(status); + } else if (WIFEXITED(status)) { + retcode_ = WEXITSTATUS(status); + } else { + retcode_ = 255; + } + return retcode_; + } + + if (ret == -1) { + // From subprocess.py + // This happens if SIGCHLD is set to be ignored + // or waiting for child process has otherwise been disabled + // for our process. This child is dead, we cannot get the + // status. + if (errno == ECHILD) retcode_ = 0; + else throw OSError("waitpid failed", errno); + } else { + retcode_ = ret; + } + + return retcode_; +#endif +} + +inline void Popen::kill(int sig_num) +{ +#ifdef __USING_WINDOWS__ + if (!TerminateProcess(this->process_handle_, (UINT)sig_num)) { + throw OSError("TerminateProcess", 0); + } +#else + ::kill(child_pid_, sig_num); +#endif +} + + +inline void Popen::execute_process() noexcept(false) +{ +#ifdef __USING_WINDOWS__ + if (exe_name_.length()) { + this->vargs_.insert(this->vargs_.begin(), this->exe_name_); + this->populate_c_argv(); + } + this->exe_name_ = vargs_[0]; + + std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter; + std::wstring argument; + std::wstring command_line; + + for (auto arg : this->vargs_) { + argument = converter.from_bytes(arg); + util::quote_argument(argument, command_line, false); + command_line += L" "; + } + + // CreateProcessW can modify szCmdLine so we allocate needed memory + wchar_t *szCmdline = new wchar_t[command_line.size() + 1]; + wcscpy_s(szCmdline, command_line.size() + 1, command_line.c_str()); + PROCESS_INFORMATION piProcInfo; + STARTUPINFOW siStartInfo; + BOOL bSuccess = FALSE; + DWORD creation_flags = CREATE_UNICODE_ENVIRONMENT | CREATE_NO_WINDOW; + + // Set up members of the PROCESS_INFORMATION structure. + ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION)); + + // Set up members of the STARTUPINFOW structure. + // This structure specifies the STDIN and STDOUT handles for redirection. + + ZeroMemory(&siStartInfo, sizeof(STARTUPINFOW)); + siStartInfo.cb = sizeof(STARTUPINFOW); + + siStartInfo.hStdError = this->stream_.g_hChildStd_ERR_Wr; + siStartInfo.hStdOutput = this->stream_.g_hChildStd_OUT_Wr; + siStartInfo.hStdInput = this->stream_.g_hChildStd_IN_Rd; + + siStartInfo.dwFlags |= STARTF_USESTDHANDLES; + + // Create the child process. + bSuccess = CreateProcessW(NULL, + szCmdline, // command line + NULL, // process security attributes + NULL, // primary thread security attributes + TRUE, // handles are inherited + creation_flags, // creation flags + NULL, // use parent's environment + NULL, // use parent's current directory + &siStartInfo, // STARTUPINFOW pointer + &piProcInfo); // receives PROCESS_INFORMATION + + // If an error occurs, exit the application. + if (!bSuccess) { + DWORD errorMessageID = ::GetLastError(); + throw CalledProcessError("CreateProcess failed: " + util::get_last_error(errorMessageID), errorMessageID); + } + + CloseHandle(piProcInfo.hThread); + + /* + TODO: use common apis to close linux handles + */ + + this->process_handle_ = piProcInfo.hProcess; + + this->cleanup_future_ = std::async(std::launch::async, [this] { + WaitForSingleObject(this->process_handle_, INFINITE); + + CloseHandle(this->stream_.g_hChildStd_ERR_Wr); + CloseHandle(this->stream_.g_hChildStd_OUT_Wr); + CloseHandle(this->stream_.g_hChildStd_IN_Rd); + }); + +/* + NOTE: In the linux version, there is a check to make sure that the process + has been started. Here, we do nothing because CreateProcess will throw + if we fail to create the process. +*/ + + +#else + + int err_rd_pipe, err_wr_pipe; + std::tie(err_rd_pipe, err_wr_pipe) = util::pipe_cloexec(); + + if (exe_name_.length()) { + vargs_.insert(vargs_.begin(), exe_name_); + populate_c_argv(); + } + exe_name_ = vargs_[0]; + + child_pid_ = fork(); + + if (child_pid_ < 0) { + close(err_rd_pipe); + close(err_wr_pipe); + throw OSError("fork failed", errno); + } + + child_created_ = true; + + if (child_pid_ == 0) + { + // Close descriptors belonging to parent + stream_.close_parent_fds(); + + //Close the read end of the error pipe + close(err_rd_pipe); + + detail::Child chld(this, err_wr_pipe); + chld.execute_child(); + } + else + { + close (err_wr_pipe);// close child side of pipe, else get stuck in read below + + stream_.close_child_fds(); + + try { + char err_buf[SP_MAX_ERR_BUF_SIZ] = {0,}; + + int read_bytes = util::read_atmost_n( + fdopen(err_rd_pipe, "r"), + err_buf, + SP_MAX_ERR_BUF_SIZ); + close(err_rd_pipe); + + if (read_bytes || strlen(err_buf)) { + // Call waitpid to reap the child process + // waitpid suspends the calling process until the + // child terminates. + int retcode = wait(); + + // Throw whatever information we have about child failure + throw CalledProcessError(err_buf, retcode); + } + } catch (std::exception& exp) { + stream_.cleanup_fds(); + throw; + } + + } +#endif +} + +namespace detail { + + inline void ArgumentDeducer::set_option(executable&& exe) { + popen_->exe_name_ = std::move(exe.arg_value); + } + + inline void ArgumentDeducer::set_option(input&& inp) { + if (inp.rd_ch_ != -1) popen_->stream_.read_from_parent_ = inp.rd_ch_; + if (inp.wr_ch_ != -1) popen_->stream_.write_to_child_ = inp.wr_ch_; + } + + inline void ArgumentDeducer::set_option(output&& out) { + if (out.wr_ch_ != -1) popen_->stream_.write_to_parent_ = out.wr_ch_; + if (out.rd_ch_ != -1) popen_->stream_.read_from_child_ = out.rd_ch_; + } + + inline void ArgumentDeducer::set_option(error&& err) { + if (err.deferred_) { + if (popen_->stream_.write_to_parent_) { + popen_->stream_.err_write_ = popen_->stream_.write_to_parent_; + } else { + throw std::runtime_error("Set output before redirecting error to output"); + } + } + if (err.wr_ch_ != -1) popen_->stream_.err_write_ = err.wr_ch_; + if (err.rd_ch_ != -1) popen_->stream_.err_read_ = err.rd_ch_; + } + + + inline void Child::execute_child() { +#ifndef __USING_WINDOWS__ + int sys_ret = -1; + auto& stream = parent_->stream_; + + try { + if (stream.write_to_parent_ == 0) + stream.write_to_parent_ = dup(stream.write_to_parent_); + + if (stream.err_write_ == 0 || stream.err_write_ == 1) + stream.err_write_ = dup(stream.err_write_); + + // Make the child owned descriptors as the + // stdin, stdout and stderr for the child process + auto _dup2_ = [](int fd, int to_fd) { + if (fd == to_fd) { + // dup2 syscall does not reset the + // CLOEXEC flag if the descriptors + // provided to it are same. + // But, we need to reset the CLOEXEC + // flag as the provided descriptors + // are now going to be the standard + // input, output and error + util::set_clo_on_exec(fd, false); + } else if(fd != -1) { + int res = dup2(fd, to_fd); + if (res == -1) throw OSError("dup2 failed", errno); + } + }; + + // Create the standard streams + _dup2_(stream.read_from_parent_, 0); // Input stream + _dup2_(stream.write_to_parent_, 1); // Output stream + _dup2_(stream.err_write_, 2); // Error stream + + // Close the duped descriptors + if (stream.read_from_parent_ != -1 && stream.read_from_parent_ > 2) + close(stream.read_from_parent_); + + if (stream.write_to_parent_ != -1 && stream.write_to_parent_ > 2) + close(stream.write_to_parent_); + + if (stream.err_write_ != -1 && stream.err_write_ > 2) + close(stream.err_write_); + + // Replace the current image with the executable + sys_ret = execvp(parent_->exe_name_.c_str(), parent_->cargv_.data()); + + if (sys_ret == -1) throw OSError("execve failed", errno); + + } catch (const OSError& exp) { + // Just write the exception message + // TODO: Give back stack trace ? + std::string err_msg(exp.what()); + //ATTN: Can we do something on error here ? + util::write_n(err_wr_pipe_, err_msg.c_str(), err_msg.length()); + } + + // Calling application would not get this + // exit failure + _exit (EXIT_FAILURE); +#endif + } + + + inline void Streams::setup_comm_channels() + { +#ifdef __USING_WINDOWS__ + util::configure_pipe(&this->g_hChildStd_IN_Rd, &this->g_hChildStd_IN_Wr, &this->g_hChildStd_IN_Wr); + this->input(util::file_from_handle(this->g_hChildStd_IN_Wr, "w")); + this->write_to_child_ = _fileno(this->input()); + + util::configure_pipe(&this->g_hChildStd_OUT_Rd, &this->g_hChildStd_OUT_Wr, &this->g_hChildStd_OUT_Rd); + this->output(util::file_from_handle(this->g_hChildStd_OUT_Rd, "r")); + this->read_from_child_ = _fileno(this->output()); + + util::configure_pipe(&this->g_hChildStd_ERR_Rd, &this->g_hChildStd_ERR_Wr, &this->g_hChildStd_ERR_Rd); + this->error(util::file_from_handle(this->g_hChildStd_ERR_Rd, "r")); + this->err_read_ = _fileno(this->error()); +#else + + if (write_to_child_ != -1) input(fdopen(write_to_child_, "wb")); + if (read_from_child_ != -1) output(fdopen(read_from_child_, "rb")); + if (err_read_ != -1) error(fdopen(err_read_, "rb")); + + auto handles = {input(), output(), error()}; + + for (auto& h : handles) { + if (h == nullptr) continue; + setvbuf(h, nullptr, _IONBF, BUFSIZ); + } + #endif + } + + inline int Communication::send(const char* msg, size_t length) + { + if (stream_->input() == nullptr) return -1; + return std::fwrite(msg, sizeof(char), length, stream_->input()); + } + + inline int Communication::send(const std::vector<char>& msg) + { + return send(msg.data(), msg.size()); + } + + inline std::pair<OutBuffer, ErrBuffer> + Communication::communicate(const char* msg, size_t length) + { + // Optimization from subprocess.py + // If we are using one pipe, or no pipe + // at all, using select() or threads is unnecessary. + auto hndls = {stream_->input(), stream_->output(), stream_->error()}; + int count = std::count(std::begin(hndls), std::end(hndls), nullptr); + const int len_conv = length; + + if (count >= 2) { + OutBuffer obuf; + ErrBuffer ebuf; + if (stream_->input()) { + if (msg) { + int wbytes = std::fwrite(msg, sizeof(char), length, stream_->input()); + if (wbytes < len_conv) { + if (errno != EPIPE && errno != EINVAL) { + throw OSError("fwrite error", errno); + } + } + } + // Close the input stream + stream_->input_.reset(); + } else if (stream_->output()) { + // Read till EOF + // ATTN: This could be blocking, if the process + // at the other end screws up, we get screwed as well + obuf.add_cap(out_buf_cap_); + + int rbytes = util::read_all( + stream_->output(), + obuf.buf); + + if (rbytes == -1) { + throw OSError("read to obuf failed", errno); + } + + obuf.length = rbytes; + // Close the output stream + stream_->output_.reset(); + + } else if (stream_->error()) { + // Same screwness applies here as well + ebuf.add_cap(err_buf_cap_); + + int rbytes = util::read_atmost_n( + stream_->error(), + ebuf.buf.data(), + ebuf.buf.size()); + + if (rbytes == -1) { + throw OSError("read to ebuf failed", errno); + } + + ebuf.length = rbytes; + // Close the error stream + stream_->error_.reset(); + } + return std::make_pair(std::move(obuf), std::move(ebuf)); + } + + return communicate_threaded(msg, length); + } + + + inline std::pair<OutBuffer, ErrBuffer> + Communication::communicate_threaded(const char* msg, size_t length) + { + OutBuffer obuf; + ErrBuffer ebuf; + std::future<int> out_fut, err_fut; + const int length_conv = length; + + if (stream_->output()) { + obuf.add_cap(out_buf_cap_); + + out_fut = std::async(std::launch::async, + [&obuf, this] { + return util::read_all(this->stream_->output(), obuf.buf); + }); + } + if (stream_->error()) { + ebuf.add_cap(err_buf_cap_); + + err_fut = std::async(std::launch::async, + [&ebuf, this] { + return util::read_all(this->stream_->error(), ebuf.buf); + }); + } + if (stream_->input()) { + if (msg) { + int wbytes = std::fwrite(msg, sizeof(char), length, stream_->input()); + if (wbytes < length_conv) { + if (errno != EPIPE && errno != EINVAL) { + throw OSError("fwrite error", errno); + } + } + } + stream_->input_.reset(); + } + + if (out_fut.valid()) { + int res = out_fut.get(); + if (res != -1) obuf.length = res; + else obuf.length = 0; + } + if (err_fut.valid()) { + int res = err_fut.get(); + if (res != -1) ebuf.length = res; + else ebuf.length = 0; + } + + return std::make_pair(std::move(obuf), std::move(ebuf)); + } + +} // end namespace detail + +} + +#endif // BITCOIN_UTIL_SUBPROCESS_H diff --git a/src/util/time.cpp b/src/util/time.cpp index 5ca9d21f8d..456662bd84 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -3,70 +3,21 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif +#include <util/time.h> #include <compat/compat.h> #include <tinyformat.h> -#include <util/time.h> #include <util/check.h> #include <atomic> #include <chrono> -#include <ctime> -#include <locale> -#include <thread> -#include <sstream> #include <string> +#include <thread> void UninterruptibleSleep(const std::chrono::microseconds& n) { std::this_thread::sleep_for(n); } static std::atomic<int64_t> nMockTime(0); //!< For testing -bool ChronoSanityCheck() -{ - // std::chrono::system_clock.time_since_epoch and time_t(0) are not guaranteed - // to use the Unix epoch timestamp, prior to C++20, but in practice they almost - // certainly will. Any differing behavior will be assumed to be an error, unless - // certain platforms prove to consistently deviate, at which point we'll cope - // with it by adding offsets. - - // Create a new clock from time_t(0) and make sure that it represents 0 - // seconds from the system_clock's time_since_epoch. Then convert that back - // to a time_t and verify that it's the same as before. - const time_t time_t_epoch{}; - auto clock = std::chrono::system_clock::from_time_t(time_t_epoch); - if (std::chrono::duration_cast<std::chrono::seconds>(clock.time_since_epoch()).count() != 0) { - return false; - } - - time_t time_val = std::chrono::system_clock::to_time_t(clock); - if (time_val != time_t_epoch) { - return false; - } - - // Check that the above zero time is actually equal to the known unix timestamp. - struct tm epoch; -#ifdef HAVE_GMTIME_R - if (gmtime_r(&time_val, &epoch) == nullptr) { -#else - if (gmtime_s(&epoch, &time_val) != 0) { -#endif - return false; - } - - if ((epoch.tm_sec != 0) || - (epoch.tm_min != 0) || - (epoch.tm_hour != 0) || - (epoch.tm_mday != 1) || - (epoch.tm_mon != 0) || - (epoch.tm_year != 70)) { - return false; - } - return true; -} - NodeClock::time_point NodeClock::now() noexcept { const std::chrono::seconds mocktime{nMockTime.load(std::memory_order_relaxed)}; @@ -96,30 +47,21 @@ std::chrono::seconds GetMockTime() int64_t GetTime() { return GetTime<std::chrono::seconds>().count(); } -std::string FormatISO8601DateTime(int64_t nTime) { - struct tm ts; - time_t time_val = nTime; -#ifdef HAVE_GMTIME_R - if (gmtime_r(&time_val, &ts) == nullptr) { -#else - if (gmtime_s(&ts, &time_val) != 0) { -#endif - return {}; - } - return strprintf("%04i-%02i-%02iT%02i:%02i:%02iZ", ts.tm_year + 1900, ts.tm_mon + 1, ts.tm_mday, ts.tm_hour, ts.tm_min, ts.tm_sec); +std::string FormatISO8601DateTime(int64_t nTime) +{ + const std::chrono::sys_seconds secs{std::chrono::seconds{nTime}}; + const auto days{std::chrono::floor<std::chrono::days>(secs)}; + const std::chrono::year_month_day ymd{days}; + const std::chrono::hh_mm_ss hms{secs - days}; + return strprintf("%04i-%02u-%02uT%02i:%02i:%02iZ", signed{ymd.year()}, unsigned{ymd.month()}, unsigned{ymd.day()}, hms.hours().count(), hms.minutes().count(), hms.seconds().count()); } -std::string FormatISO8601Date(int64_t nTime) { - struct tm ts; - time_t time_val = nTime; -#ifdef HAVE_GMTIME_R - if (gmtime_r(&time_val, &ts) == nullptr) { -#else - if (gmtime_s(&ts, &time_val) != 0) { -#endif - return {}; - } - return strprintf("%04i-%02i-%02i", ts.tm_year + 1900, ts.tm_mon + 1, ts.tm_mday); +std::string FormatISO8601Date(int64_t nTime) +{ + const std::chrono::sys_seconds secs{std::chrono::seconds{nTime}}; + const auto days{std::chrono::floor<std::chrono::days>(secs)}; + const std::chrono::year_month_day ymd{days}; + return strprintf("%04i-%02u-%02u", signed{ymd.year()}, unsigned{ymd.month()}, unsigned{ymd.day()}); } struct timeval MillisToTimeval(int64_t nTimeout) diff --git a/src/util/time.h b/src/util/time.h index 6aa776137c..108560e0e0 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -116,7 +116,4 @@ struct timeval MillisToTimeval(int64_t nTimeout); */ struct timeval MillisToTimeval(std::chrono::milliseconds ms); -/** Sanity check epoch match normal Unix epoch */ -bool ChronoSanityCheck(); - #endif // BITCOIN_UTIL_TIME_H diff --git a/src/validation.cpp b/src/validation.cpp index c15e660499..903f9caf13 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -472,6 +472,11 @@ public: * policies such as mempool min fee and min relay fee. */ const bool m_package_feerates; + /** Used for local submission of transactions to catch "absurd" fees + * due to fee miscalculation by wallets. std:nullopt implies unset, allowing any feerates. + * Any individual transaction failing this check causes immediate failure. + */ + const std::optional<CFeeRate> m_client_maxfeerate; /** Parameters for single transaction mempool validation. */ static ATMPArgs SingleAccept(const CChainParams& chainparams, int64_t accept_time, @@ -485,6 +490,7 @@ public: /* m_allow_replacement */ true, /* m_package_submission */ false, /* m_package_feerates */ false, + /* m_client_maxfeerate */ {}, // checked by caller }; } @@ -499,12 +505,13 @@ public: /* m_allow_replacement */ false, /* m_package_submission */ false, // not submitting to mempool /* m_package_feerates */ false, + /* m_client_maxfeerate */ {}, // checked by caller }; } /** Parameters for child-with-unconfirmed-parents package validation. */ static ATMPArgs PackageChildWithParents(const CChainParams& chainparams, int64_t accept_time, - std::vector<COutPoint>& coins_to_uncache) { + std::vector<COutPoint>& coins_to_uncache, const std::optional<CFeeRate>& client_maxfeerate) { return ATMPArgs{/* m_chainparams */ chainparams, /* m_accept_time */ accept_time, /* m_bypass_limits */ false, @@ -513,6 +520,7 @@ public: /* m_allow_replacement */ false, /* m_package_submission */ true, /* m_package_feerates */ true, + /* m_client_maxfeerate */ client_maxfeerate, }; } @@ -526,6 +534,7 @@ public: /* m_allow_replacement */ true, /* m_package_submission */ true, // do not LimitMempoolSize in Finalize() /* m_package_feerates */ false, // only 1 transaction + /* m_client_maxfeerate */ package_args.m_client_maxfeerate, }; } @@ -539,7 +548,8 @@ public: bool test_accept, bool allow_replacement, bool package_submission, - bool package_feerates) + bool package_feerates, + std::optional<CFeeRate> client_maxfeerate) : m_chainparams{chainparams}, m_accept_time{accept_time}, m_bypass_limits{bypass_limits}, @@ -547,7 +557,8 @@ public: m_test_accept{test_accept}, m_allow_replacement{allow_replacement}, m_package_submission{package_submission}, - m_package_feerates{package_feerates} + m_package_feerates{package_feerates}, + m_client_maxfeerate{client_maxfeerate} { } }; @@ -589,12 +600,14 @@ private: // of checking a given transaction. struct Workspace { explicit Workspace(const CTransactionRef& ptx) : m_ptx(ptx), m_hash(ptx->GetHash()) {} - /** Txids of mempool transactions that this transaction directly conflicts with. */ + /** Txids of mempool transactions that this transaction directly conflicts with or may + * replace via sibling eviction. */ std::set<Txid> m_conflicts; - /** Iterators to mempool entries that this transaction directly conflicts with. */ + /** Iterators to mempool entries that this transaction directly conflicts with or may + * replace via sibling eviction. */ CTxMemPool::setEntries m_iters_conflicting; /** Iterators to all mempool entries that would be replaced by this transaction, including - * those it directly conflicts with and their descendants. */ + * m_conflicts and their descendants. */ CTxMemPool::setEntries m_all_conflicting; /** All mempool ancestors of this transaction. */ CTxMemPool::setEntries m_ancestors; @@ -602,9 +615,12 @@ private: * inserted into the mempool until Finalize(). */ std::unique_ptr<CTxMemPoolEntry> m_entry; /** Pointers to the transactions that have been removed from the mempool and replaced by - * this transaction, used to return to the MemPoolAccept caller. Only populated if + * this transaction (everything in m_all_conflicting), used to return to the MemPoolAccept caller. Only populated if * validation is successful and the original transactions are removed. */ std::list<CTransactionRef> m_replaced_transactions; + /** Whether RBF-related data structures (m_conflicts, m_iters_conflicting, m_all_conflicting, + * m_replaced_transactions) include a sibling in addition to txns with conflicting inputs. */ + bool m_sibling_eviction{false}; /** Virtual size of the transaction as used by the mempool, calculated using serialized size * of the transaction and sigops. */ @@ -694,7 +710,8 @@ private: Chainstate& m_active_chainstate; - /** Whether the transaction(s) would replace any mempool transactions. If so, RBF rules apply. */ + /** Whether the transaction(s) would replace any mempool transactions and/or evict any siblings. + * If so, RBF rules apply. */ bool m_rbf{false}; }; @@ -958,8 +975,27 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } ws.m_ancestors = *ancestors; - if (const auto err_string{SingleV3Checks(ws.m_ptx, ws.m_ancestors, ws.m_conflicts, ws.m_vsize)}) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "v3-rule-violation", *err_string); + // Even though just checking direct mempool parents for inheritance would be sufficient, we + // check using the full ancestor set here because it's more convenient to use what we have + // already calculated. + if (const auto err{SingleV3Checks(ws.m_ptx, ws.m_ancestors, ws.m_conflicts, ws.m_vsize)}) { + // Disabled within package validation. + if (err->second != nullptr && args.m_allow_replacement) { + // Potential sibling eviction. Add the sibling to our list of mempool conflicts to be + // included in RBF checks. + ws.m_conflicts.insert(err->second->GetHash()); + // Adding the sibling to m_iters_conflicting here means that it doesn't count towards + // RBF Carve Out above. This is correct, since removing to-be-replaced transactions from + // the descendant count is done separately in SingleV3Checks for v3 transactions. + ws.m_iters_conflicting.insert(m_pool.GetIter(err->second->GetHash()).value()); + ws.m_sibling_eviction = true; + // The sibling will be treated as part of the to-be-replaced set in ReplacementChecks. + // Note that we are not checking whether it opts in to replaceability via BIP125 or v3 + // (which is normally done in PreChecks). However, the only way a v3 transaction can + // have a non-v3 and non-BIP125 descendant is due to a reorg. + } else { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "v3-rule-violation", err->first); + } } // A transaction that spends outputs that would be replaced by it is invalid. Now @@ -999,18 +1035,21 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. // This must be changed if package RBF is enabled. - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, + strprintf("insufficient fee%s", ws.m_sibling_eviction ? " (including sibling eviction)" : ""), *err_string); } // Calculate all conflicting entries and enforce Rule #5. if (const auto err_string{GetEntriesForConflicts(tx, m_pool, ws.m_iters_conflicting, ws.m_all_conflicting)}) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, - "too many potential replacements", *err_string); + strprintf("too many potential replacements%s", ws.m_sibling_eviction ? " (including sibling eviction)" : ""), *err_string); } // Enforce Rule #2. if (const auto err_string{HasNoNewUnconfirmed(tx, m_pool, ws.m_iters_conflicting)}) { + // Sibling eviction is only done for v3 transactions, which cannot have multiple ancestors. + Assume(!ws.m_sibling_eviction); return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, - "replacement-adds-unconfirmed", *err_string); + strprintf("replacement-adds-unconfirmed%s", ws.m_sibling_eviction ? " (including sibling eviction)" : ""), *err_string); } // Check if it's economically rational to mine this transaction rather than the ones it // replaces and pays for its own relay fees. Enforce Rules #3 and #4. @@ -1023,7 +1062,8 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. // This must be changed if package RBF is enabled. - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, + strprintf("insufficient fee%s", ws.m_sibling_eviction ? " (including sibling eviction)" : ""), *err_string); } return true; } @@ -1256,6 +1296,12 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef return MempoolAcceptResult::Failure(ws.m_state); } + // Individual modified feerate exceeded caller-defined max; abort + if (args.m_client_maxfeerate && CFeeRate(ws.m_modified_fees, ws.m_vsize) > args.m_client_maxfeerate.value()) { + ws.m_state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "max feerate exceeded", ""); + return MempoolAcceptResult::Failure(ws.m_state); + } + if (m_rbf && !ReplacementChecks(ws)) return MempoolAcceptResult::Failure(ws.m_state); // Perform the inexpensive checks first and avoid hashing and signature verification unless @@ -1316,6 +1362,18 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); return PackageMempoolAcceptResult(package_state, std::move(results)); } + + // Individual modified feerate exceeded caller-defined max; abort + // N.B. this doesn't take into account CPFPs. Chunk-aware validation may be more robust. + if (args.m_client_maxfeerate && CFeeRate(ws.m_modified_fees, ws.m_vsize) > args.m_client_maxfeerate.value()) { + // Need to set failure here both individually and at package level + ws.m_state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "max feerate exceeded", ""); + package_state.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); + // Exit early to avoid doing pointless work. Update the failed tx result; the rest are unfinished. + results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); + return PackageMempoolAcceptResult(package_state, std::move(results)); + } + // Make the coins created by this transaction available for subsequent transactions in the // package to spend. Since we already checked conflicts in the package and we don't allow // replacements, we don't need to track the coins spent. Note that this logic will need to be @@ -1660,7 +1718,7 @@ MempoolAcceptResult AcceptToMemoryPool(Chainstate& active_chainstate, const CTra } PackageMempoolAcceptResult ProcessNewPackage(Chainstate& active_chainstate, CTxMemPool& pool, - const Package& package, bool test_accept) + const Package& package, bool test_accept, const std::optional<CFeeRate>& client_maxfeerate) { AssertLockHeld(cs_main); assert(!package.empty()); @@ -1674,7 +1732,7 @@ PackageMempoolAcceptResult ProcessNewPackage(Chainstate& active_chainstate, CTxM auto args = MemPoolAccept::ATMPArgs::PackageTestAccept(chainparams, GetTime(), coins_to_uncache); return MemPoolAccept(pool, active_chainstate).AcceptMultipleTransactions(package, args); } else { - auto args = MemPoolAccept::ATMPArgs::PackageChildWithParents(chainparams, GetTime(), coins_to_uncache); + auto args = MemPoolAccept::ATMPArgs::PackageChildWithParents(chainparams, GetTime(), coins_to_uncache, client_maxfeerate); return MemPoolAccept(pool, active_chainstate).AcceptPackage(package, args); } }(); @@ -1995,10 +2053,10 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, return true; } -bool FatalError(Notifications& notifications, BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage) +bool FatalError(Notifications& notifications, BlockValidationState& state, const bilingual_str& message) { - notifications.fatalError(strMessage, userMessage); - return state.Error(strMessage); + notifications.fatalError(message); + return state.Error(message.original); } /** @@ -2045,12 +2103,12 @@ DisconnectResult Chainstate::DisconnectBlock(const CBlock& block, const CBlockIn CBlockUndo blockUndo; if (!m_blockman.UndoReadFromDisk(blockUndo, *pindex)) { - error("DisconnectBlock(): failure reading undo data"); + LogError("DisconnectBlock(): failure reading undo data\n"); return DISCONNECT_FAILED; } if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) { - error("DisconnectBlock(): block and undo data inconsistent"); + LogError("DisconnectBlock(): block and undo data inconsistent\n"); return DISCONNECT_FAILED; } @@ -2089,7 +2147,7 @@ DisconnectResult Chainstate::DisconnectBlock(const CBlock& block, const CBlockIn if (i > 0) { // not coinbases CTxUndo &txundo = blockUndo.vtxundo[i-1]; if (txundo.vprevout.size() != tx.vin.size()) { - error("DisconnectBlock(): transaction and undo data inconsistent"); + LogError("DisconnectBlock(): transaction and undo data inconsistent\n"); return DISCONNECT_FAILED; } for (unsigned int j = tx.vin.size(); j > 0;) { @@ -2220,9 +2278,10 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // We don't write down blocks to disk if they may have been // corrupted, so this should be impossible unless we're having hardware // problems. - return FatalError(m_chainman.GetNotifications(), state, "Corrupt block found indicating potential hardware failure; shutting down"); + return FatalError(m_chainman.GetNotifications(), state, _("Corrupt block found indicating potential hardware failure.")); } - return error("%s: Consensus::CheckBlock: %s", __func__, state.ToString()); + LogError("%s: Consensus::CheckBlock: %s\n", __func__, state.ToString()); + return false; } // verify that the view's current state corresponds to the previous block @@ -2408,7 +2467,8 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // Any transaction validation failure in ConnectBlock is a block consensus failure state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, tx_state.GetRejectReason(), tx_state.GetDebugMessage()); - return error("%s: Consensus::CheckTxInputs: %s, %s", __func__, tx.GetHash().ToString(), state.ToString()); + LogError("%s: Consensus::CheckTxInputs: %s, %s\n", __func__, tx.GetHash().ToString(), state.ToString()); + return false; } nFees += txfee; if (!MoneyRange(nFees)) { @@ -2449,8 +2509,9 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // Any transaction validation failure in ConnectBlock is a block consensus failure state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, tx_state.GetRejectReason(), tx_state.GetDebugMessage()); - return error("ConnectBlock(): CheckInputScripts on %s failed with %s", + LogError("ConnectBlock(): CheckInputScripts on %s failed with %s\n", tx.GetHash().ToString(), state.ToString()); + return false; } control.Add(std::move(vChecks)); } @@ -2643,7 +2704,7 @@ bool Chainstate::FlushStateToDisk( if (fDoFullFlush || fPeriodicWrite) { // Ensure we can write block index if (!CheckDiskSpace(m_blockman.m_opts.blocks_dir)) { - return FatalError(m_chainman.GetNotifications(), state, "Disk space is too low!", _("Disk space is too low!")); + return FatalError(m_chainman.GetNotifications(), state, _("Disk space is too low!")); } { LOG_TIME_MILLIS_WITH_CATEGORY("write block and undo data to disk", BCLog::BENCH); @@ -2661,7 +2722,7 @@ bool Chainstate::FlushStateToDisk( LOG_TIME_MILLIS_WITH_CATEGORY("write block index to disk", BCLog::BENCH); if (!m_blockman.WriteBlockIndexDB()) { - return FatalError(m_chainman.GetNotifications(), state, "Failed to write to block index database"); + return FatalError(m_chainman.GetNotifications(), state, _("Failed to write to block index database.")); } } // Finally remove any pruned files @@ -2683,11 +2744,11 @@ bool Chainstate::FlushStateToDisk( // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. if (!CheckDiskSpace(m_chainman.m_options.datadir, 48 * 2 * 2 * CoinsTip().GetCacheSize())) { - return FatalError(m_chainman.GetNotifications(), state, "Disk space is too low!", _("Disk space is too low!")); + return FatalError(m_chainman.GetNotifications(), state, _("Disk space is too low!")); } // Flush the chainstate (which may refer to block index entries). if (!CoinsTip().Flush()) - return FatalError(m_chainman.GetNotifications(), state, "Failed to write to coin database"); + return FatalError(m_chainman.GetNotifications(), state, _("Failed to write to coin database.")); m_last_flush = nNow; full_flush_completed = true; TRACE5(utxocache, flush, @@ -2703,7 +2764,7 @@ bool Chainstate::FlushStateToDisk( m_chainman.m_options.signals->ChainStateFlushed(this->GetRole(), m_chain.GetLocator()); } } catch (const std::runtime_error& e) { - return FatalError(m_chainman.GetNotifications(), state, std::string("System error while flushing: ") + e.what()); + return FatalError(m_chainman.GetNotifications(), state, strprintf(_("System error while flushing: %s"), e.what())); } return true; } @@ -2823,15 +2884,18 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); CBlock& block = *pblock; if (!m_blockman.ReadBlockFromDisk(block, *pindexDelete)) { - return error("DisconnectTip(): Failed to read block"); + LogError("DisconnectTip(): Failed to read block\n"); + return false; } // Apply the block atomically to the chain state. const auto time_start{SteadyClock::now()}; { CCoinsViewCache view(&CoinsTip()); assert(view.GetBestBlock() == pindexDelete->GetBlockHash()); - if (DisconnectBlock(block, pindexDelete, view) != DISCONNECT_OK) - return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); + if (DisconnectBlock(block, pindexDelete, view) != DISCONNECT_OK) { + LogError("DisconnectTip(): DisconnectBlock %s failed\n", pindexDelete->GetBlockHash().ToString()); + return false; + } bool flushed = view.Flush(); assert(flushed); } @@ -2936,7 +3000,7 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, if (!pblock) { std::shared_ptr<CBlock> pblockNew = std::make_shared<CBlock>(); if (!m_blockman.ReadBlockFromDisk(*pblockNew, *pindexNew)) { - return FatalError(m_chainman.GetNotifications(), state, "Failed to read block"); + return FatalError(m_chainman.GetNotifications(), state, _("Failed to read block.")); } pthisBlock = pblockNew; } else { @@ -2960,7 +3024,8 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, if (!rv) { if (state.IsInvalid()) InvalidBlockFound(pindexNew, state); - return error("%s: ConnectBlock %s failed, %s", __func__, pindexNew->GetBlockHash().ToString(), state.ToString()); + LogError("%s: ConnectBlock %s failed, %s\n", __func__, pindexNew->GetBlockHash().ToString(), state.ToString()); + return false; } time_3 = SteadyClock::now(); time_connect_total += time_3 - time_2; @@ -3122,7 +3187,7 @@ bool Chainstate::ActivateBestChainStep(BlockValidationState& state, CBlockIndex* // If we're unable to disconnect a block during normal operation, // then that is a failure of our local system -- we should abort // rather than stay on a less work chain. - FatalError(m_chainman.GetNotifications(), state, "Failed to disconnect block; see debug.log for details"); + FatalError(m_chainman.GetNotifications(), state, _("Failed to disconnect block.")); return false; } fBlocksDisconnected = true; @@ -3624,7 +3689,18 @@ void ChainstateManager::ReceivedBlockTransactions(const CBlock& block, CBlockInd { AssertLockHeld(cs_main); pindexNew->nTx = block.vtx.size(); - pindexNew->nChainTx = 0; + // Typically nChainTx will be 0 at this point, but it can be nonzero if this + // is a pruned block which is being downloaded again, or if this is an + // assumeutxo snapshot block which has a hardcoded nChainTx value from the + // snapshot metadata. If the pindex is not the snapshot block and the + // nChainTx value is not zero, assert that value is actually correct. + auto prev_tx_sum = [](CBlockIndex& block) { return block.nTx + (block.pprev ? block.pprev->nChainTx : 0); }; + if (!Assume(pindexNew->nChainTx == 0 || pindexNew->nChainTx == prev_tx_sum(*pindexNew) || + pindexNew == GetSnapshotBaseBlock())) { + LogWarning("Internal bug detected: block %d has unexpected nChainTx %i that should be %i (%s %s). Please report this issue here: %s\n", + pindexNew->nHeight, pindexNew->nChainTx, prev_tx_sum(*pindexNew), PACKAGE_NAME, FormatFullVersion(), PACKAGE_BUGREPORT); + pindexNew->nChainTx = 0; + } pindexNew->nFile = pos.nFile; pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; @@ -3644,7 +3720,15 @@ void ChainstateManager::ReceivedBlockTransactions(const CBlock& block, CBlockInd while (!queue.empty()) { CBlockIndex *pindex = queue.front(); queue.pop_front(); - pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; + // Before setting nChainTx, assert that it is 0 or already set to + // the correct value. This assert will fail after receiving the + // assumeutxo snapshot block if assumeutxo snapshot metadata has an + // incorrect hardcoded AssumeutxoData::nChainTx value. + if (!Assume(pindex->nChainTx == 0 || pindex->nChainTx == prev_tx_sum(*pindex))) { + LogWarning("Internal bug detected: block %d has unexpected nChainTx %i that should be %i (%s %s). Please report this issue here: %s\n", + pindex->nHeight, pindex->nChainTx, prev_tx_sum(*pindex), PACKAGE_NAME, FormatFullVersion(), PACKAGE_BUGREPORT); + } + pindex->nChainTx = prev_tx_sum(*pindex); pindex->nSequenceId = nBlockSequenceId++; for (Chainstate *c : GetAll()) { c->TryAddBlockIndexCandidate(pindex); @@ -4152,9 +4236,10 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& if (NotifyHeaderTip(*this)) { if (IsInitialBlockDownload() && ppindex && *ppindex) { const CBlockIndex& last_accepted{**ppindex}; - const int64_t blocks_left{(GetTime() - last_accepted.GetBlockTime()) / GetConsensus().nPowTargetSpacing}; + int64_t blocks_left{(NodeClock::now() - last_accepted.Time()) / GetConsensus().PowTargetSpacing()}; + blocks_left = std::max<int64_t>(0, blocks_left); const double progress{100.0 * last_accepted.nHeight / (last_accepted.nHeight + blocks_left)}; - LogPrintf("Synchronizing blockheaders, height: %d (~%.2f%%)\n", last_accepted.nHeight, progress); + LogInfo("Synchronizing blockheaders, height: %d (~%.2f%%)\n", last_accepted.nHeight, progress); } } return true; @@ -4178,9 +4263,10 @@ void ChainstateManager::ReportHeadersPresync(const arith_uint256& work, int64_t bool initial_download = IsInitialBlockDownload(); GetNotifications().headerTip(GetSynchronizationState(initial_download), height, timestamp, /*presync=*/true); if (initial_download) { - const int64_t blocks_left{(GetTime() - timestamp) / GetConsensus().nPowTargetSpacing}; + int64_t blocks_left{(NodeClock::now() - NodeSeconds{std::chrono::seconds{timestamp}}) / GetConsensus().PowTargetSpacing()}; + blocks_left = std::max<int64_t>(0, blocks_left); const double progress{100.0 * height / (height + blocks_left)}; - LogPrintf("Pre-synchronizing blockheaders, height: %d (~%.2f%%)\n", height, progress); + LogInfo("Pre-synchronizing blockheaders, height: %d (~%.2f%%)\n", height, progress); } } @@ -4243,7 +4329,8 @@ bool ChainstateManager::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, pindex->nStatus |= BLOCK_FAILED_VALID; m_blockman.m_dirty_blockindex.insert(pindex); } - return error("%s: %s", __func__, state.ToString()); + LogError("%s: %s\n", __func__, state.ToString()); + return false; } // Header is valid/has work, merkle tree and segwit merkle tree are good...RELAY NOW @@ -4262,7 +4349,7 @@ bool ChainstateManager::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, } ReceivedBlockTransactions(block, pindex, blockPos); } catch (const std::runtime_error& e) { - return FatalError(GetNotifications(), state, std::string("System error: ") + e.what()); + return FatalError(GetNotifications(), state, strprintf(_("System error while saving block to disk: %s"), e.what())); } // TODO: FlushStateToDisk() handles flushing of both block and chainstate @@ -4306,7 +4393,8 @@ bool ChainstateManager::ProcessNewBlock(const std::shared_ptr<const CBlock>& blo if (m_options.signals) { m_options.signals->BlockChecked(*block, state); } - return error("%s: AcceptBlock FAILED (%s)", __func__, state.ToString()); + LogError("%s: AcceptBlock FAILED (%s)\n", __func__, state.ToString()); + return false; } } @@ -4314,13 +4402,15 @@ bool ChainstateManager::ProcessNewBlock(const std::shared_ptr<const CBlock>& blo BlockValidationState state; // Only used to report errors, not invalidity - ignore it if (!ActiveChainstate().ActivateBestChain(state, block)) { - return error("%s: ActivateBestChain failed (%s)", __func__, state.ToString()); + LogError("%s: ActivateBestChain failed (%s)\n", __func__, state.ToString()); + return false; } Chainstate* bg_chain{WITH_LOCK(cs_main, return BackgroundSyncInProgress() ? m_ibd_chainstate.get() : nullptr)}; BlockValidationState bg_state; if (bg_chain && !bg_chain->ActivateBestChain(bg_state, block)) { - return error("%s: [background] ActivateBestChain failed (%s)", __func__, bg_state.ToString()); + LogError("%s: [background] ActivateBestChain failed (%s)\n", __func__, bg_state.ToString()); + return false; } return true; @@ -4358,12 +4448,18 @@ bool TestBlockValidity(BlockValidationState& state, indexDummy.phashBlock = &block_hash; // NOTE: CheckBlockHeader is called by CheckBlock - if (!ContextualCheckBlockHeader(block, state, chainstate.m_blockman, chainstate.m_chainman, pindexPrev)) - return error("%s: Consensus::ContextualCheckBlockHeader: %s", __func__, state.ToString()); - if (!CheckBlock(block, state, chainparams.GetConsensus(), fCheckPOW, fCheckMerkleRoot)) - return error("%s: Consensus::CheckBlock: %s", __func__, state.ToString()); - if (!ContextualCheckBlock(block, state, chainstate.m_chainman, pindexPrev)) - return error("%s: Consensus::ContextualCheckBlock: %s", __func__, state.ToString()); + if (!ContextualCheckBlockHeader(block, state, chainstate.m_blockman, chainstate.m_chainman, pindexPrev)) { + LogError("%s: Consensus::ContextualCheckBlockHeader: %s\n", __func__, state.ToString()); + return false; + } + if (!CheckBlock(block, state, chainparams.GetConsensus(), fCheckPOW, fCheckMerkleRoot)) { + LogError("%s: Consensus::CheckBlock: %s\n", __func__, state.ToString()); + return false; + } + if (!ContextualCheckBlock(block, state, chainstate.m_chainman, pindexPrev)) { + LogError("%s: Consensus::ContextualCheckBlock: %s\n", __func__, state.ToString()); + return false; + } if (!chainstate.ConnectBlock(block, state, &indexDummy, viewNew, true)) { return false; } @@ -4567,7 +4663,8 @@ bool Chainstate::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& in // TODO: merge with ConnectBlock CBlock block; if (!m_blockman.ReadBlockFromDisk(block, *pindex)) { - return error("ReplayBlock(): ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + LogError("ReplayBlock(): ReadBlockFromDisk failed at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + return false; } for (const CTransactionRef& tx : block.vtx) { @@ -4591,7 +4688,10 @@ bool Chainstate::ReplayBlocks() std::vector<uint256> hashHeads = db.GetHeadBlocks(); if (hashHeads.empty()) return true; // We're already in a consistent state. - if (hashHeads.size() != 2) return error("ReplayBlocks(): unknown inconsistent state"); + if (hashHeads.size() != 2) { + LogError("ReplayBlocks(): unknown inconsistent state\n"); + return false; + } m_chainman.GetNotifications().progress(_("Replaying blocks…"), 0, false); LogPrintf("Replaying blocks\n"); @@ -4601,13 +4701,15 @@ bool Chainstate::ReplayBlocks() const CBlockIndex* pindexFork = nullptr; // Latest block common to both the old and the new tip. if (m_blockman.m_block_index.count(hashHeads[0]) == 0) { - return error("ReplayBlocks(): reorganization to unknown block requested"); + LogError("ReplayBlocks(): reorganization to unknown block requested\n"); + return false; } pindexNew = &(m_blockman.m_block_index[hashHeads[0]]); if (!hashHeads[1].IsNull()) { // The old tip is allowed to be 0, indicating it's the first flush. if (m_blockman.m_block_index.count(hashHeads[1]) == 0) { - return error("ReplayBlocks(): reorganization from unknown block requested"); + LogError("ReplayBlocks(): reorganization from unknown block requested\n"); + return false; } pindexOld = &(m_blockman.m_block_index[hashHeads[1]]); pindexFork = LastCommonAncestor(pindexOld, pindexNew); @@ -4619,12 +4721,14 @@ bool Chainstate::ReplayBlocks() if (pindexOld->nHeight > 0) { // Never disconnect the genesis block. CBlock block; if (!m_blockman.ReadBlockFromDisk(block, *pindexOld)) { - return error("RollbackBlock(): ReadBlockFromDisk() failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); + LogError("RollbackBlock(): ReadBlockFromDisk() failed at %d, hash=%s\n", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); + return false; } LogPrintf("Rolling back %s (%i)\n", pindexOld->GetBlockHash().ToString(), pindexOld->nHeight); DisconnectResult res = DisconnectBlock(block, pindexOld, cache); if (res == DISCONNECT_FAILED) { - return error("RollbackBlock(): DisconnectBlock failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); + LogError("RollbackBlock(): DisconnectBlock failed at %d, hash=%s\n", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); + return false; } // If DISCONNECT_UNCLEAN is returned, it means a non-existing UTXO was deleted, or an existing UTXO was // overwritten. It corresponds to cases where the block-to-be-disconnect never had all its operations @@ -4743,12 +4847,14 @@ bool Chainstate::LoadGenesisBlock() const CBlock& block = params.GenesisBlock(); FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, nullptr)}; if (blockPos.IsNull()) { - return error("%s: writing genesis block to disk failed", __func__); + LogError("%s: writing genesis block to disk failed\n", __func__); + return false; } CBlockIndex* pindex = m_blockman.AddToBlockIndex(block, m_chainman.m_best_header); m_chainman.ReceivedBlockTransactions(block, pindex, blockPos); } catch (const std::runtime_error& e) { - return error("%s: failed to write genesis block: %s", __func__, e.what()); + LogError("%s: failed to write genesis block: %s\n", __func__, e.what()); + return false; } return true; @@ -4927,7 +5033,7 @@ void ChainstateManager::LoadExternalBlockFile( } } } catch (const std::runtime_error& e) { - GetNotifications().fatalError(std::string("System error: ") + e.what()); + GetNotifications().fatalError(strprintf(_("System error while loading external block file: %s"), e.what())); } LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, Ticks<std::chrono::milliseconds>(SteadyClock::now() - start)); } @@ -4967,16 +5073,31 @@ void ChainstateManager::CheckBlockIndex() size_t nNodes = 0; int nHeight = 0; CBlockIndex* pindexFirstInvalid = nullptr; // Oldest ancestor of pindex which is invalid. - CBlockIndex* pindexFirstMissing = nullptr; // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA. - CBlockIndex* pindexFirstNeverProcessed = nullptr; // Oldest ancestor of pindex for which nTx == 0. + CBlockIndex* pindexFirstMissing = nullptr; // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA, since assumeutxo snapshot if used. + CBlockIndex* pindexFirstNeverProcessed = nullptr; // Oldest ancestor of pindex for which nTx == 0, since assumeutxo snapshot if used. CBlockIndex* pindexFirstNotTreeValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_TREE (regardless of being valid or not). - CBlockIndex* pindexFirstNotTransactionsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not). - CBlockIndex* pindexFirstNotChainValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not). - CBlockIndex* pindexFirstNotScriptsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_SCRIPTS (regardless of being valid or not). - CBlockIndex* pindexFirstAssumeValid = nullptr; // Oldest ancestor of pindex which has BLOCK_ASSUMED_VALID + CBlockIndex* pindexFirstNotTransactionsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not), since assumeutxo snapshot if used. + CBlockIndex* pindexFirstNotChainValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not), since assumeutxo snapshot if used. + CBlockIndex* pindexFirstNotScriptsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_SCRIPTS (regardless of being valid or not), since assumeutxo snapshot if used. + + // After checking an assumeutxo snapshot block, reset pindexFirst pointers + // to earlier blocks that have not been downloaded or validated yet, so + // checks for later blocks can assume the earlier blocks were validated and + // be stricter, testing for more requirements. + const CBlockIndex* snap_base{GetSnapshotBaseBlock()}; + CBlockIndex *snap_first_missing{}, *snap_first_notx{}, *snap_first_notv{}, *snap_first_nocv{}, *snap_first_nosv{}; + auto snap_update_firsts = [&] { + if (pindex == snap_base) { + std::swap(snap_first_missing, pindexFirstMissing); + std::swap(snap_first_notx, pindexFirstNeverProcessed); + std::swap(snap_first_notv, pindexFirstNotTransactionsValid); + std::swap(snap_first_nocv, pindexFirstNotChainValid); + std::swap(snap_first_nosv, pindexFirstNotScriptsValid); + } + }; + while (pindex != nullptr) { nNodes++; - if (pindexFirstAssumeValid == nullptr && pindex->nStatus & BLOCK_ASSUMED_VALID) pindexFirstAssumeValid = pindex; if (pindexFirstInvalid == nullptr && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA)) { pindexFirstMissing = pindex; @@ -4984,10 +5105,7 @@ void ChainstateManager::CheckBlockIndex() if (pindexFirstNeverProcessed == nullptr && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; if (pindex->pprev != nullptr && pindexFirstNotTreeValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex; - if (pindex->pprev != nullptr && !pindex->IsAssumedValid()) { - // Skip validity flag checks for BLOCK_ASSUMED_VALID index entries, since these - // *_VALID_MASK flags will not be present for index entries we are temporarily assuming - // valid. + if (pindex->pprev != nullptr) { if (pindexFirstNotTransactionsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) { pindexFirstNotTransactionsValid = pindex; @@ -5017,36 +5135,26 @@ void ChainstateManager::CheckBlockIndex() if (!pindex->HaveNumChainTxs()) assert(pindex->nSequenceId <= 0); // nSequenceId can't be set positive for blocks that aren't linked (negative is used for preciousblock) // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred). // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred. - // Unless these indexes are assumed valid and pending block download on a - // background chainstate. - if (!m_blockman.m_have_pruned && !pindex->IsAssumedValid()) { + if (!m_blockman.m_have_pruned) { // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0 assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); - if (pindexFirstAssumeValid == nullptr) { - // If we've got some assume valid blocks, then we might have - // missing blocks (not HAVE_DATA) but still treat them as - // having been processed (with a fake nTx value). Otherwise, we - // can assert that these are the same. - assert(pindexFirstMissing == pindexFirstNeverProcessed); - } + assert(pindexFirstMissing == pindexFirstNeverProcessed); } else { // If we have pruned, then we can only say that HAVE_DATA implies nTx > 0 if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0); } if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA); - if (pindex->IsAssumedValid()) { - // Assumed-valid blocks should have some nTx value. - assert(pindex->nTx > 0); + if (snap_base && snap_base->GetAncestor(pindex->nHeight) == pindex) { // Assumed-valid blocks should connect to the main chain. assert((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TREE); - } else { - // Otherwise there should only be an nTx value if we have - // actually seen a block's transactions. - assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. } + // There should only be an nTx value if we have + // actually seen a block's transactions. + assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to HaveNumChainTxs(). - assert((pindexFirstNeverProcessed == nullptr) == pindex->HaveNumChainTxs()); - assert((pindexFirstNotTransactionsValid == nullptr) == pindex->HaveNumChainTxs()); + // HaveNumChainTxs will also be set in the assumeutxo snapshot block from snapshot metadata. + assert((pindexFirstNeverProcessed == nullptr || pindex == snap_base) == pindex->HaveNumChainTxs()); + assert((pindexFirstNotTransactionsValid == nullptr || pindex == snap_base) == pindex->HaveNumChainTxs()); assert(pindex->nHeight == nHeight); // nHeight must be consistent. assert(pindex->pprev == nullptr || pindex->nChainWork >= pindex->pprev->nChainWork); // For every block except the genesis block, the chainwork must be larger than the parent's. assert(nHeight < 2 || (pindex->pskip && (pindex->pskip->nHeight < nHeight))); // The pskip pointer must point back for all but the first 2 blocks. @@ -5059,30 +5167,64 @@ void ChainstateManager::CheckBlockIndex() assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); // The failed mask cannot be set for blocks without invalid parents. } // Make sure nChainTx sum is correctly computed. - unsigned int prev_chain_tx = pindex->pprev ? pindex->pprev->nChainTx : 0; - assert((pindex->nChainTx == pindex->nTx + prev_chain_tx) - // Transaction may be completely unset - happens if only the header was accepted but the block hasn't been processed. - || (pindex->nChainTx == 0 && pindex->nTx == 0) - // nChainTx may be unset, but nTx set (if a block has been accepted, but one of its predecessors hasn't been processed yet) - || (pindex->nChainTx == 0 && prev_chain_tx == 0 && pindex->pprev) - // Transaction counts prior to snapshot are unknown. - || pindex->IsAssumedValid()); + if (!pindex->pprev) { + // If no previous block, nTx and nChainTx must be the same. + assert(pindex->nChainTx == pindex->nTx); + } else if (pindex->pprev->nChainTx > 0 && pindex->nTx > 0) { + // If previous nChainTx is set and number of transactions in block is known, sum must be set. + assert(pindex->nChainTx == pindex->nTx + pindex->pprev->nChainTx); + } else { + // Otherwise nChainTx should only be set if this is a snapshot + // block, and must be set if it is. + assert((pindex->nChainTx != 0) == (pindex == snap_base)); + } + // Chainstate-specific checks on setBlockIndexCandidates for (auto c : GetAll()) { if (c->m_chain.Tip() == nullptr) continue; - if (!CBlockIndexWorkComparator()(pindex, c->m_chain.Tip()) && pindexFirstNeverProcessed == nullptr) { + // Two main factors determine whether pindex is a candidate in + // setBlockIndexCandidates: + // + // - If pindex has less work than the chain tip, it should not be a + // candidate, and this will be asserted below. Otherwise it is a + // potential candidate. + // + // - If pindex or one of its parent blocks back to the genesis block + // or an assumeutxo snapshot never downloaded transactions + // (pindexFirstNeverProcessed is non-null), it should not be a + // candidate, and this will be asserted below. The only exception + // is if pindex itself is an assumeutxo snapshot block. Then it is + // also a potential candidate. + if (!CBlockIndexWorkComparator()(pindex, c->m_chain.Tip()) && (pindexFirstNeverProcessed == nullptr || pindex == snap_base)) { + // If pindex was detected as invalid (pindexFirstInvalid is + // non-null), it is not required to be in + // setBlockIndexCandidates. if (pindexFirstInvalid == nullptr) { - const bool is_active = c == &ActiveChainstate(); - // If this block sorts at least as good as the current tip and - // is valid and we have all data for its parents, it must be in - // setBlockIndexCandidates. m_chain.Tip() must also be there - // even if some data has been pruned. + // If pindex and all its parents back to the genesis block + // or an assumeutxo snapshot block downloaded transactions, + // and the transactions were not pruned (pindexFirstMissing + // is null), it is a potential candidate. The check + // excludes pruned blocks, because if any blocks were + // pruned between pindex the current chain tip, pindex will + // only temporarily be added to setBlockIndexCandidates, + // before being moved to m_blocks_unlinked. This check + // could be improved to verify that if all blocks between + // the chain tip and pindex have data, pindex must be a + // candidate. + // + // If pindex is the chain tip, it also is a potential + // candidate. // - if ((pindexFirstMissing == nullptr || pindex == c->m_chain.Tip())) { - // The active chainstate should always have this block - // as a candidate, but a background chainstate should - // only have it if it is an ancestor of the snapshot base. - if (is_active || GetSnapshotBaseBlock()->GetAncestor(pindex->nHeight) == pindex) { + // If the chainstate was loaded from a snapshot and pindex + // is the base of the snapshot, pindex is also a potential + // candidate. + if (pindexFirstMissing == nullptr || pindex == c->m_chain.Tip() || pindex == c->SnapshotBase()) { + // If this chainstate is the active chainstate, pindex + // must be in setBlockIndexCandidates. Otherwise, this + // chainstate is a background validation chainstate, and + // pindex only needs to be added if it is an ancestor of + // the snapshot that is being validated. + if (c == &ActiveChainstate() || snap_base->GetAncestor(pindex->nHeight) == pindex) { assert(c->setBlockIndexCandidates.count(pindex)); } } @@ -5113,7 +5255,7 @@ void ChainstateManager::CheckBlockIndex() if (pindexFirstMissing == nullptr) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in m_blocks_unlinked. if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == nullptr && pindexFirstMissing != nullptr) { // We HAVE_DATA for this block, have received data for all parents at some point, but we're currently missing data for some parent. - assert(m_blockman.m_have_pruned || pindexFirstAssumeValid != nullptr); // We must have pruned, or else we're using a snapshot (causing us to have faked the received data for some parent(s)). + assert(m_blockman.m_have_pruned); // This block may have entered m_blocks_unlinked if: // - it has a descendant that at some point had more work than the // tip, and @@ -5126,7 +5268,7 @@ void ChainstateManager::CheckBlockIndex() const bool is_active = c == &ActiveChainstate(); if (!CBlockIndexWorkComparator()(pindex, c->m_chain.Tip()) && c->setBlockIndexCandidates.count(pindex) == 0) { if (pindexFirstInvalid == nullptr) { - if (is_active || GetSnapshotBaseBlock()->GetAncestor(pindex->nHeight) == pindex) { + if (is_active || snap_base->GetAncestor(pindex->nHeight) == pindex) { assert(foundInUnlinked); } } @@ -5137,6 +5279,7 @@ void ChainstateManager::CheckBlockIndex() // End: actual consistency checks. // Try descending into the first subnode. + snap_update_firsts(); std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> range = forward.equal_range(pindex); if (range.first != range.second) { // A subnode was found. @@ -5148,6 +5291,7 @@ void ChainstateManager::CheckBlockIndex() // Move upwards until we reach a node of which we have not yet visited the last child. while (pindex) { // We are going to either move to a parent or a sibling of pindex. + snap_update_firsts(); // If pindex was the first with a certain property, unset the corresponding variable. if (pindex == pindexFirstInvalid) pindexFirstInvalid = nullptr; if (pindex == pindexFirstMissing) pindexFirstMissing = nullptr; @@ -5156,7 +5300,6 @@ void ChainstateManager::CheckBlockIndex() if (pindex == pindexFirstNotTransactionsValid) pindexFirstNotTransactionsValid = nullptr; if (pindex == pindexFirstNotChainValid) pindexFirstNotChainValid = nullptr; if (pindex == pindexFirstNotScriptsValid) pindexFirstNotScriptsValid = nullptr; - if (pindex == pindexFirstAssumeValid) pindexFirstAssumeValid = nullptr; // Find our parent. CBlockIndex* pindexPar = pindex->pprev; // Find which child we just visited. @@ -5230,6 +5373,12 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pin if (pindex == nullptr) return 0.0; + if (!Assume(pindex->nChainTx > 0)) { + LogWarning("Internal bug detected: block %d has unset nChainTx (%s %s). Please report this issue here: %s\n", + pindex->nHeight, PACKAGE_NAME, FormatFullVersion(), PACKAGE_BUGREPORT); + return 0.0; + } + int64_t nNow = time(nullptr); double fTxTotal; @@ -5394,8 +5543,8 @@ bool ChainstateManager::ActivateSnapshot( snapshot_chainstate.reset(); bool removed = DeleteCoinsDBFromDisk(*snapshot_datadir, /*is_snapshot=*/true); if (!removed) { - GetNotifications().fatalError(strprintf("Failed to remove snapshot chainstate dir (%s). " - "Manually remove it before restarting.\n", fs::PathToString(*snapshot_datadir))); + GetNotifications().fatalError(strprintf(_("Failed to remove snapshot chainstate dir (%s). " + "Manually remove it before restarting.\n"), fs::PathToString(*snapshot_datadir))); } } return false; @@ -5636,30 +5785,14 @@ bool ChainstateManager::PopulateAndValidateSnapshot( // Fake various pieces of CBlockIndex state: CBlockIndex* index = nullptr; - // Don't make any modifications to the genesis block. - // This is especially important because we don't want to erroneously - // apply BLOCK_ASSUMED_VALID to genesis, which would happen if we didn't skip - // it here (since it apparently isn't BLOCK_VALID_SCRIPTS). + // Don't make any modifications to the genesis block since it shouldn't be + // necessary, and since the genesis block doesn't have normal flags like + // BLOCK_VALID_SCRIPTS set. constexpr int AFTER_GENESIS_START{1}; for (int i = AFTER_GENESIS_START; i <= snapshot_chainstate.m_chain.Height(); ++i) { index = snapshot_chainstate.m_chain[i]; - // Fake nTx so that LoadBlockIndex() loads assumed-valid CBlockIndex - // entries (among other things) - if (!index->nTx) { - index->nTx = 1; - } - // Fake nChainTx so that GuessVerificationProgress reports accurately - index->nChainTx = index->pprev->nChainTx + index->nTx; - - // Mark unvalidated block index entries beneath the snapshot base block as assumed-valid. - if (!index->IsValid(BLOCK_VALID_SCRIPTS)) { - // This flag will be removed once the block is fully validated by a - // background chainstate. - index->nStatus |= BLOCK_ASSUMED_VALID; - } - // Fake BLOCK_OPT_WITNESS so that Chainstate::NeedsRedownload() // won't ask to rewind the entire assumed-valid chain on startup. if (DeploymentActiveAt(*index, *this, Consensus::DEPLOYMENT_SEGWIT)) { @@ -5675,6 +5808,7 @@ bool ChainstateManager::PopulateAndValidateSnapshot( } assert(index); + assert(index == snapshot_start_block); index->nChainTx = au_data.nChainTx; snapshot_chainstate.setBlockIndexCandidates.insert(snapshot_start_block); @@ -5749,7 +5883,7 @@ SnapshotCompletionResult ChainstateManager::MaybeCompleteSnapshotValidation() user_error = strprintf(Untranslated("%s\n%s"), user_error, util::ErrorString(rename_result)); } - GetNotifications().fatalError(user_error.original, user_error); + GetNotifications().fatalError(user_error); }; if (index_new.GetBlockHash() != snapshot_blockhash) { @@ -6008,13 +6142,14 @@ bool ChainstateManager::DeleteSnapshotChainstate() Assert(m_snapshot_chainstate); Assert(m_ibd_chainstate); - fs::path snapshot_datadir = GetSnapshotCoinsDBPath(*m_snapshot_chainstate); + fs::path snapshot_datadir = Assert(node::FindSnapshotChainstateDir(m_options.datadir)).value(); if (!DeleteCoinsDBFromDisk(snapshot_datadir, /*is_snapshot=*/ true)) { LogPrintf("Deletion of %s failed. Please remove it manually to continue reindexing.\n", fs::PathToString(snapshot_datadir)); return false; } m_active_chainstate = m_ibd_chainstate.get(); + m_active_chainstate->m_mempool = m_snapshot_chainstate->m_mempool; m_snapshot_chainstate.reset(); return true; } @@ -6090,9 +6225,9 @@ bool ChainstateManager::ValidatedSnapshotCleanup() const fs::filesystem_error& err) { LogPrintf("Error renaming path (%s) -> (%s): %s\n", fs::PathToString(p_old), fs::PathToString(p_new), err.what()); - GetNotifications().fatalError(strprintf( + GetNotifications().fatalError(strprintf(_( "Rename of '%s' -> '%s' failed. " - "Cannot clean up the background chainstate leveldb directory.", + "Cannot clean up the background chainstate leveldb directory."), fs::PathToString(p_old), fs::PathToString(p_new))); }; diff --git a/src/validation.h b/src/validation.h index 71aac46f81..e3b2a2d59b 100644 --- a/src/validation.h +++ b/src/validation.h @@ -93,7 +93,7 @@ extern const std::vector<std::string> CHECKLEVEL_DOC; CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); -bool FatalError(kernel::Notifications& notifications, BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = {}); +bool FatalError(kernel::Notifications& notifications, BlockValidationState& state, const bilingual_str& message); /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pindex); @@ -274,13 +274,15 @@ MempoolAcceptResult AcceptToMemoryPool(Chainstate& active_chainstate, const CTra /** * Validate (and maybe submit) a package to the mempool. See doc/policy/packages.md for full details * on package validation rules. -* @param[in] test_accept When true, run validation checks but don't submit to mempool. +* @param[in] test_accept When true, run validation checks but don't submit to mempool. +* @param[in] client_maxfeerate If exceeded by an individual transaction, rest of (sub)package evaluation is aborted. +* Only for sanity checks against local submission of transactions. * @returns a PackageMempoolAcceptResult which includes a MempoolAcceptResult for each transaction. * If a transaction fails, validation will exit early and some results may be missing. It is also * possible for the package to be partially submitted. */ PackageMempoolAcceptResult ProcessNewPackage(Chainstate& active_chainstate, CTxMemPool& pool, - const Package& txns, bool test_accept) + const Package& txns, bool test_accept, const std::optional<CFeeRate>& client_maxfeerate) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /* Mempool validation helper functions */ @@ -476,7 +478,7 @@ enum class CoinsCacheSizeState * current best chain. * * Eventually, the API here is targeted at being exposed externally as a - * consumable libconsensus library, so any functions added must only call + * consumable library, so any functions added must only call * other class member functions, pure functions in other parts of the consensus * library, callbacks via the validation interface, or read/write-to-disk * functions (eventually this will also be via callbacks). @@ -583,9 +585,10 @@ public: const CBlockIndex* SnapshotBase() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** - * The set of all CBlockIndex entries with either BLOCK_VALID_TRANSACTIONS (for - * itself and all ancestors) *or* BLOCK_ASSUMED_VALID (if using background - * chainstates) and as good as our current tip or better. Entries may be failed, + * The set of all CBlockIndex entries that have as much work as our current + * tip or more, and transaction data needed to be validated (with + * BLOCK_VALID_TRANSACTIONS for each block and its parents back to the + * genesis block or an assumeutxo snapshot block). Entries may be failed, * though, and pruning nodes may be missing the data for the block. */ std::set<CBlockIndex*, node::CBlockIndexWorkComparator> setBlockIndexCandidates; diff --git a/src/wallet/external_signer_scriptpubkeyman.cpp b/src/wallet/external_signer_scriptpubkeyman.cpp index a71f8f9fbc..b5703fa54a 100644 --- a/src/wallet/external_signer_scriptpubkeyman.cpp +++ b/src/wallet/external_signer_scriptpubkeyman.cpp @@ -9,9 +9,11 @@ #include <wallet/external_signer_scriptpubkeyman.h> #include <iostream> +#include <key_io.h> #include <memory> #include <stdexcept> #include <string> +#include <univalue.h> #include <utility> #include <vector> @@ -51,15 +53,26 @@ ExternalSigner ExternalSignerScriptPubKeyMan::GetExternalSigner() { return signers[0]; } -bool ExternalSignerScriptPubKeyMan::DisplayAddress(const CScript scriptPubKey, const ExternalSigner &signer) const +util::Result<void> ExternalSignerScriptPubKeyMan::DisplayAddress(const CTxDestination& dest, const ExternalSigner &signer) const { // TODO: avoid the need to infer a descriptor from inside a descriptor wallet + const CScript& scriptPubKey = GetScriptForDestination(dest); auto provider = GetSolvingProvider(scriptPubKey); auto descriptor = InferDescriptor(scriptPubKey, *provider); - signer.DisplayAddress(descriptor->ToString()); - // TODO inspect result - return true; + const UniValue& result = signer.DisplayAddress(descriptor->ToString()); + + const UniValue& error = result.find_value("error"); + if (error.isStr()) return util::Error{strprintf(_("Signer returned error: %s"), error.getValStr())}; + + const UniValue& ret_address = result.find_value("address"); + if (!ret_address.isStr()) return util::Error{_("Signer did not echo address")}; + + if (ret_address.getValStr() != EncodeDestination(dest)) { + return util::Error{strprintf(_("Signer echoed unexpected address %s"), ret_address.getValStr())}; + } + + return util::Result<void>(); } // If sign is true, transaction must previously have been filled diff --git a/src/wallet/external_signer_scriptpubkeyman.h b/src/wallet/external_signer_scriptpubkeyman.h index c052ce6129..44286456b6 100644 --- a/src/wallet/external_signer_scriptpubkeyman.h +++ b/src/wallet/external_signer_scriptpubkeyman.h @@ -9,6 +9,8 @@ #include <memory> +struct bilingual_str; + namespace wallet { class ExternalSignerScriptPubKeyMan : public DescriptorScriptPubKeyMan { @@ -27,7 +29,11 @@ class ExternalSignerScriptPubKeyMan : public DescriptorScriptPubKeyMan static ExternalSigner GetExternalSigner(); - bool DisplayAddress(const CScript scriptPubKey, const ExternalSigner &signer) const; + /** + * Display address on the device and verify that the returned value matches. + * @returns nothing or an error message + */ + util::Result<void> DisplayAddress(const CTxDestination& dest, const ExternalSigner& signer) const; TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; }; diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index d15273dfc9..0c1cae7253 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -92,7 +92,7 @@ WalletTxStatus MakeWalletTxStatus(const CWallet& wallet, const CWalletTx& wtx) WalletTxStatus result; result.block_height = wtx.state<TxStateConfirmed>() ? wtx.state<TxStateConfirmed>()->confirmed_block_height : - wtx.state<TxStateConflicted>() ? wtx.state<TxStateConflicted>()->conflicting_block_height : + wtx.state<TxStateBlockConflicted>() ? wtx.state<TxStateBlockConflicted>()->conflicting_block_height : std::numeric_limits<int>::max(); result.blocks_to_maturity = wallet.GetTxBlocksToMaturity(wtx); result.depth_in_main_chain = wallet.GetTxDepthInMainChain(wtx); @@ -101,7 +101,7 @@ WalletTxStatus MakeWalletTxStatus(const CWallet& wallet, const CWalletTx& wtx) result.is_trusted = CachedTxIsTrusted(wallet, wtx); result.is_abandoned = wtx.isAbandoned(); result.is_coinbase = wtx.IsCoinBase(); - result.is_in_main_chain = wallet.IsTxInMainChain(wtx); + result.is_in_main_chain = wtx.isConfirmed(); return result; } @@ -247,7 +247,7 @@ public: return value.empty() ? m_wallet->EraseAddressReceiveRequest(batch, dest, id) : m_wallet->SetAddressReceiveRequest(batch, dest, id, value); } - bool displayAddress(const CTxDestination& dest) override + util::Result<void> displayAddress(const CTxDestination& dest) override { LOCK(m_wallet->cs_wallet); return m_wallet->DisplayAddress(dest); @@ -286,7 +286,7 @@ public: if (!res) return util::Error{util::ErrorString(res)}; const auto& txr = *res; fee = txr.fee; - change_pos = txr.change_pos ? *txr.change_pos : -1; + change_pos = txr.change_pos ? int(*txr.change_pos) : -1; return txr.tx; } diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index b9d8d9abc9..c164266f80 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -149,7 +149,7 @@ CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, c { AssertLockHeld(wallet.cs_wallet); - if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { + if (wallet.IsTxImmatureCoinBase(wtx) && wtx.isConfirmed()) { return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, filter); } @@ -253,12 +253,12 @@ bool CachedTxIsFromMe(const CWallet& wallet, const CWalletTx& wtx, const isminef return (CachedTxGetDebit(wallet, wtx, filter) > 0); } +// NOLINTNEXTLINE(misc-no-recursion) bool CachedTxIsTrusted(const CWallet& wallet, const CWalletTx& wtx, std::set<uint256>& trusted_parents) { AssertLockHeld(wallet.cs_wallet); - int nDepth = wallet.GetTxDepthInMainChain(wtx); - if (nDepth >= 1) return true; - if (nDepth < 0) return false; + if (wtx.isConfirmed()) return true; + if (wtx.isBlockConflicted()) return false; // using wtx's cached debit if (!wallet.m_spend_zero_conf_change || !CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)) return false; diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index acaa2d8b15..bed9ec029a 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -395,6 +395,7 @@ class DescribeWalletAddressVisitor public: const SigningProvider * const provider; + // NOLINTNEXTLINE(misc-no-recursion) void ProcessSubScript(const CScript& subscript, UniValue& obj) const { // Always present: script type and redeemscript @@ -445,6 +446,7 @@ public: return obj; } + // NOLINTNEXTLINE(misc-no-recursion) UniValue operator()(const ScriptHash& scripthash) const { UniValue obj(UniValue::VOBJ); @@ -465,6 +467,7 @@ public: return obj; } + // NOLINTNEXTLINE(misc-no-recursion) UniValue operator()(const WitnessV0ScriptHash& id) const { UniValue obj(UniValue::VOBJ); @@ -786,9 +789,8 @@ RPCHelpMan walletdisplayaddress() throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } - if (!pwallet->DisplayAddress(dest)) { - throw JSONRPCError(RPC_MISC_ERROR, "Failed to display address"); - } + util::Result<void> res = pwallet->DisplayAddress(dest); + if (!res) throw JSONRPCError(RPC_MISC_ERROR, util::ErrorString(res).original); UniValue result(UniValue::VOBJ); result.pushKV("address", request.params[0].get_str()); diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 5167e986b1..ae2dfe5795 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -854,6 +854,7 @@ enum class ScriptContext // Analyse the provided scriptPubKey, determining which keys and which redeem scripts from the ImportData struct are needed to spend it, and mark them as used. // Returns an error string, or the empty string for success. +// NOLINTNEXTLINE(misc-no-recursion) static std::string RecurseImportData(const CScript& script, ImportData& import_data, const ScriptContext script_ctx) { // Use Solver to obtain script type and parsed pubkeys or hashes: diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index e6c021d426..05b340995d 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -40,6 +40,10 @@ static void WalletTxToJSON(const CWallet& wallet, const CWalletTx& wtx, UniValue for (const uint256& conflict : wallet.GetTxConflicts(wtx)) conflicts.push_back(conflict.GetHex()); entry.pushKV("walletconflicts", conflicts); + UniValue mempool_conflicts(UniValue::VARR); + for (const Txid& mempool_conflict : wtx.mempool_conflicts) + mempool_conflicts.push_back(mempool_conflict.GetHex()); + entry.pushKV("mempoolconflicts", mempool_conflicts); entry.pushKV("time", wtx.GetTxTime()); entry.pushKV("timereceived", int64_t{wtx.nTimeReceived}); @@ -417,6 +421,10 @@ static std::vector<RPCResult> TransactionDescriptionString() }}, {RPCResult::Type::STR_HEX, "replaced_by_txid", /*optional=*/true, "Only if 'category' is 'send'. The txid if this tx was replaced."}, {RPCResult::Type::STR_HEX, "replaces_txid", /*optional=*/true, "Only if 'category' is 'send'. The txid if this tx replaces another."}, + {RPCResult::Type::ARR, "mempoolconflicts", "Transactions that directly conflict with either this transaction or an ancestor transaction", + { + {RPCResult::Type::STR_HEX, "txid", "The transaction id."}, + }}, {RPCResult::Type::STR, "to", /*optional=*/true, "If a comment to is associated with the transaction."}, {RPCResult::Type::NUM_TIME, "time", "The transaction time expressed in " + UNIX_EPOCH_TIME + "."}, {RPCResult::Type::NUM_TIME, "timereceived", "The time received expressed in " + UNIX_EPOCH_TIME + "."}, diff --git a/src/wallet/rpc/util.cpp b/src/wallet/rpc/util.cpp index 06ec7db1bc..1252843e9d 100644 --- a/src/wallet/rpc/util.cpp +++ b/src/wallet/rpc/util.cpp @@ -11,6 +11,7 @@ #include <wallet/context.h> #include <wallet/wallet.h> +#include <string_view> #include <univalue.h> #include <boost/date_time/posix_time/posix_time.hpp> @@ -61,9 +62,9 @@ bool ParseIncludeWatchonly(const UniValue& include_watchonly, const CWallet& wal bool GetWalletNameFromJSONRPCRequest(const JSONRPCRequest& request, std::string& wallet_name) { - if (URL_DECODE && request.URI.substr(0, WALLET_ENDPOINT_BASE.size()) == WALLET_ENDPOINT_BASE) { + if (request.URI.starts_with(WALLET_ENDPOINT_BASE)) { // wallet endpoint was used - wallet_name = URL_DECODE(request.URI.substr(WALLET_ENDPOINT_BASE.size())); + wallet_name = UrlDecode(std::string_view{request.URI}.substr(WALLET_ENDPOINT_BASE.size())); return true; } return false; diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index 6a8ce954fb..a684d4e191 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -817,6 +817,217 @@ static RPCHelpMan migratewallet() }; } +RPCHelpMan gethdkeys() +{ + return RPCHelpMan{ + "gethdkeys", + "\nList all BIP 32 HD keys in the wallet and which descriptors use them.\n", + { + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { + {"active_only", RPCArg::Type::BOOL, RPCArg::Default{false}, "Show the keys for only active descriptors"}, + {"private", RPCArg::Type::BOOL, RPCArg::Default{false}, "Show private keys"} + }}, + }, + RPCResult{RPCResult::Type::ARR, "", "", { + { + {RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "xpub", "The extended public key"}, + {RPCResult::Type::BOOL, "has_private", "Whether the wallet has the private key for this xpub"}, + {RPCResult::Type::STR, "xprv", /*optional=*/true, "The extended private key if \"private\" is true"}, + {RPCResult::Type::ARR, "descriptors", "Array of descriptor objects that use this HD key", + { + {RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "desc", "Descriptor string representation"}, + {RPCResult::Type::BOOL, "active", "Whether this descriptor is currently used to generate new addresses"}, + }}, + }}, + }}, + } + }}, + RPCExamples{ + HelpExampleCli("gethdkeys", "") + HelpExampleRpc("gethdkeys", "") + + HelpExampleCliNamed("gethdkeys", {{"active_only", "true"}, {"private", "true"}}) + HelpExampleRpcNamed("gethdkeys", {{"active_only", "true"}, {"private", "true"}}) + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + const std::shared_ptr<const CWallet> wallet = GetWalletForJSONRPCRequest(request); + if (!wallet) return UniValue::VNULL; + + if (!wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + throw JSONRPCError(RPC_WALLET_ERROR, "gethdkeys is not available for non-descriptor wallets"); + } + + LOCK(wallet->cs_wallet); + + UniValue options{request.params[0].isNull() ? UniValue::VOBJ : request.params[0]}; + const bool active_only{options.exists("active_only") ? options["active_only"].get_bool() : false}; + const bool priv{options.exists("private") ? options["private"].get_bool() : false}; + if (priv) { + EnsureWalletIsUnlocked(*wallet); + } + + + std::set<ScriptPubKeyMan*> spkms; + if (active_only) { + spkms = wallet->GetActiveScriptPubKeyMans(); + } else { + spkms = wallet->GetAllScriptPubKeyMans(); + } + + std::map<CExtPubKey, std::set<std::tuple<std::string, bool, bool>>> wallet_xpubs; + std::map<CExtPubKey, CExtKey> wallet_xprvs; + for (auto* spkm : spkms) { + auto* desc_spkm{dynamic_cast<DescriptorScriptPubKeyMan*>(spkm)}; + CHECK_NONFATAL(desc_spkm); + LOCK(desc_spkm->cs_desc_man); + WalletDescriptor w_desc = desc_spkm->GetWalletDescriptor(); + + // Retrieve the pubkeys from the descriptor + std::set<CPubKey> desc_pubkeys; + std::set<CExtPubKey> desc_xpubs; + w_desc.descriptor->GetPubKeys(desc_pubkeys, desc_xpubs); + for (const CExtPubKey& xpub : desc_xpubs) { + std::string desc_str; + bool ok = desc_spkm->GetDescriptorString(desc_str, false); + CHECK_NONFATAL(ok); + wallet_xpubs[xpub].emplace(desc_str, wallet->IsActiveScriptPubKeyMan(*spkm), desc_spkm->HasPrivKey(xpub.pubkey.GetID())); + if (std::optional<CKey> key = priv ? desc_spkm->GetKey(xpub.pubkey.GetID()) : std::nullopt) { + wallet_xprvs[xpub] = CExtKey(xpub, *key); + } + } + } + + UniValue response(UniValue::VARR); + for (const auto& [xpub, descs] : wallet_xpubs) { + bool has_xprv = false; + UniValue descriptors(UniValue::VARR); + for (const auto& [desc, active, has_priv] : descs) { + UniValue d(UniValue::VOBJ); + d.pushKV("desc", desc); + d.pushKV("active", active); + has_xprv |= has_priv; + + descriptors.push_back(std::move(d)); + } + UniValue xpub_info(UniValue::VOBJ); + xpub_info.pushKV("xpub", EncodeExtPubKey(xpub)); + xpub_info.pushKV("has_private", has_xprv); + if (priv) { + xpub_info.pushKV("xprv", EncodeExtKey(wallet_xprvs.at(xpub))); + } + xpub_info.pushKV("descriptors", std::move(descriptors)); + + response.push_back(std::move(xpub_info)); + } + + return response; + }, + }; +} + +static RPCHelpMan createwalletdescriptor() +{ + return RPCHelpMan{"createwalletdescriptor", + "Creates the wallet's descriptor for the given address type. " + "The address type must be one that the wallet does not already have a descriptor for." + + HELP_REQUIRING_PASSPHRASE, + { + {"type", RPCArg::Type::STR, RPCArg::Optional::NO, "The address type the descriptor will produce. Options are \"legacy\", \"p2sh-segwit\", \"bech32\", and \"bech32m\"."}, + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { + {"internal", RPCArg::Type::BOOL, RPCArg::DefaultHint{"Both external and internal will be generated unless this parameter is specified"}, "Whether to only make one descriptor that is internal (if parameter is true) or external (if parameter is false)"}, + {"hdkey", RPCArg::Type::STR, RPCArg::DefaultHint{"The HD key used by all other active descriptors"}, "The HD key that the wallet knows the private key of, listed using 'gethdkeys', to use for this descriptor's key"}, + }}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::ARR, "descs", "The public descriptors that were added to the wallet", + {{RPCResult::Type::STR, "", ""}} + } + }, + }, + RPCExamples{ + HelpExampleCli("createwalletdescriptor", "bech32m") + + HelpExampleRpc("createwalletdescriptor", "bech32m") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + if (!pwallet) return UniValue::VNULL; + + // Make sure wallet is a descriptor wallet + if (!pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + throw JSONRPCError(RPC_WALLET_ERROR, "createwalletdescriptor is not available for non-descriptor wallets"); + } + + std::optional<OutputType> output_type = ParseOutputType(request.params[0].get_str()); + if (!output_type) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[0].get_str())); + } + + UniValue options{request.params[1].isNull() ? UniValue::VOBJ : request.params[1]}; + UniValue internal_only{options["internal"]}; + UniValue hdkey{options["hdkey"]}; + + std::vector<bool> internals; + if (internal_only.isNull()) { + internals.push_back(false); + internals.push_back(true); + } else { + internals.push_back(internal_only.get_bool()); + } + + LOCK(pwallet->cs_wallet); + EnsureWalletIsUnlocked(*pwallet); + + CExtPubKey xpub; + if (hdkey.isNull()) { + std::set<CExtPubKey> active_xpubs = pwallet->GetActiveHDPubKeys(); + if (active_xpubs.size() != 1) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unable to determine which HD key to use from active descriptors. Please specify with 'hdkey'"); + } + xpub = *active_xpubs.begin(); + } else { + xpub = DecodeExtPubKey(hdkey.get_str()); + if (!xpub.pubkey.IsValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unable to parse HD key. Please provide a valid xpub"); + } + } + + std::optional<CKey> key = pwallet->GetKey(xpub.pubkey.GetID()); + if (!key) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Private key for %s is not known", EncodeExtPubKey(xpub))); + } + CExtKey active_hdkey(xpub, *key); + + std::vector<std::reference_wrapper<DescriptorScriptPubKeyMan>> spkms; + WalletBatch batch{pwallet->GetDatabase()}; + for (bool internal : internals) { + WalletDescriptor w_desc = GenerateWalletDescriptor(xpub, *output_type, internal); + uint256 w_id = DescriptorID(*w_desc.descriptor); + if (!pwallet->GetScriptPubKeyMan(w_id)) { + spkms.emplace_back(pwallet->SetupDescriptorScriptPubKeyMan(batch, active_hdkey, *output_type, internal)); + } + } + if (spkms.empty()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Descriptor already exists"); + } + + // Fetch each descspkm from the wallet in order to get the descriptor strings + UniValue descs{UniValue::VARR}; + for (const auto& spkm : spkms) { + std::string desc_str; + bool ok = spkm.get().GetDescriptorString(desc_str, false); + CHECK_NONFATAL(ok); + descs.push_back(desc_str); + } + UniValue out{UniValue::VOBJ}; + out.pushKV("descs", std::move(descs)); + return out; + } + }; +} + // addresses RPCHelpMan getaddressinfo(); RPCHelpMan getnewaddress(); @@ -900,6 +1111,7 @@ Span<const CRPCCommand> GetWalletRPCCommands() {"wallet", &bumpfee}, {"wallet", &psbtbumpfee}, {"wallet", &createwallet}, + {"wallet", &createwalletdescriptor}, {"wallet", &restorewallet}, {"wallet", &dumpprivkey}, {"wallet", &dumpwallet}, @@ -907,6 +1119,7 @@ Span<const CRPCCommand> GetWalletRPCCommands() {"wallet", &getaddressesbylabel}, {"wallet", &getaddressinfo}, {"wallet", &getbalance}, + {"wallet", &gethdkeys}, {"wallet", &getnewaddress}, {"wallet", &getrawchangeaddress}, {"wallet", &getreceivedbyaddress}, diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index e10a17f003..b42275fe4b 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -11,6 +11,7 @@ #include <script/sign.h> #include <script/solver.h> #include <util/bip32.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> @@ -96,6 +97,7 @@ bool HaveKeys(const std::vector<valtype>& pubkeys, const LegacyScriptPubKeyMan& //! @param recurse_scripthash whether to recurse into nested p2sh and p2wsh //! scripts or simply treat any script that has been //! stored in the keystore as spendable +// NOLINTNEXTLINE(misc-no-recursion) IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& scriptPubKey, IsMineSigVersion sigversion, bool recurse_scripthash=true) { IsMineResult ret = IsMineResult::NO; @@ -2143,6 +2145,36 @@ std::map<CKeyID, CKey> DescriptorScriptPubKeyMan::GetKeys() const return m_map_keys; } +bool DescriptorScriptPubKeyMan::HasPrivKey(const CKeyID& keyid) const +{ + AssertLockHeld(cs_desc_man); + return m_map_keys.contains(keyid) || m_map_crypted_keys.contains(keyid); +} + +std::optional<CKey> DescriptorScriptPubKeyMan::GetKey(const CKeyID& keyid) const +{ + AssertLockHeld(cs_desc_man); + if (m_storage.HasEncryptionKeys() && !m_storage.IsLocked()) { + const auto& it = m_map_crypted_keys.find(keyid); + if (it == m_map_crypted_keys.end()) { + return std::nullopt; + } + const std::vector<unsigned char>& crypted_secret = it->second.second; + CKey key; + if (!Assume(m_storage.WithEncryptionKey([&](const CKeyingMaterial& encryption_key) { + return DecryptKey(encryption_key, crypted_secret, it->second.first, key); + }))) { + return std::nullopt; + } + return key; + } + const auto& it = m_map_keys.find(keyid); + if (it == m_map_keys.end()) { + return std::nullopt; + } + return it->second; +} + bool DescriptorScriptPubKeyMan::TopUp(unsigned int size) { WalletBatch batch(m_storage.GetDatabase()); @@ -2296,55 +2328,7 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(WalletBatch& batch, co return false; } - int64_t creation_time = GetTime(); - - std::string xpub = EncodeExtPubKey(master_key.Neuter()); - - // Build descriptor string - std::string desc_prefix; - std::string desc_suffix = "/*)"; - switch (addr_type) { - case OutputType::LEGACY: { - desc_prefix = "pkh(" + xpub + "/44h"; - break; - } - case OutputType::P2SH_SEGWIT: { - desc_prefix = "sh(wpkh(" + xpub + "/49h"; - desc_suffix += ")"; - break; - } - case OutputType::BECH32: { - desc_prefix = "wpkh(" + xpub + "/84h"; - break; - } - case OutputType::BECH32M: { - desc_prefix = "tr(" + xpub + "/86h"; - break; - } - case OutputType::UNKNOWN: { - // We should never have a DescriptorScriptPubKeyMan for an UNKNOWN OutputType, - // so if we get to this point something is wrong - assert(false); - } - } // no default case, so the compiler can warn about missing cases - assert(!desc_prefix.empty()); - - // Mainnet derives at 0', testnet and regtest derive at 1' - if (Params().IsTestChain()) { - desc_prefix += "/1h"; - } else { - desc_prefix += "/0h"; - } - - std::string internal_path = internal ? "/1" : "/0"; - std::string desc_str = desc_prefix + "/0h" + internal_path + desc_suffix; - - // Make the descriptor - FlatSigningProvider keys; - std::string error; - std::unique_ptr<Descriptor> desc = Parse(desc_str, keys, error, false); - WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0); - m_wallet_descriptor = w_desc; + m_wallet_descriptor = GenerateWalletDescriptor(master_key.Neuter(), addr_type, internal); // Store the master private key, and descriptor if (!AddDescriptorKeyWithDB(batch, master_key.key, master_key.key.GetPubKey())) { diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 2d83ae556f..2c1ab8d44a 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -27,7 +27,6 @@ #include <unordered_map> enum class OutputType; -struct bilingual_str; namespace wallet { struct MigrationData; @@ -633,6 +632,9 @@ public: bool SetupDescriptorGeneration(WalletBatch& batch, const CExtKey& master_key, OutputType addr_type, bool internal); bool HavePrivateKeys() const override; + bool HasPrivKey(const CKeyID& keyid) const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); + //! Retrieve the particular key if it is available. Returns nullopt if the key is not in the wallet, or if the wallet is locked. + std::optional<CKey> GetKey(const CKeyID& keyid) const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); std::optional<int64_t> GetOldestKeyPoolTime() const override; unsigned int GetKeyPoolSize() const override; @@ -669,7 +671,7 @@ public: std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys(int32_t minimum_index) const; int32_t GetEndRange() const; - bool GetDescriptorString(std::string& out, const bool priv) const; + [[nodiscard]] bool GetDescriptorString(std::string& out, const bool priv) const; void UpgradeDescriptorCache(); }; diff --git a/src/wallet/test/walletload_tests.cpp b/src/wallet/test/walletload_tests.cpp index 3dba2231f0..2e43eda582 100644 --- a/src/wallet/test/walletload_tests.cpp +++ b/src/wallet/test/walletload_tests.cpp @@ -34,6 +34,7 @@ public: std::optional<int64_t> ScriptSize() const override { return {}; } std::optional<int64_t> MaxSatisfactionWeight(bool) const override { return {}; } std::optional<int64_t> MaxSatisfactionElems() const override { return {}; } + void GetPubKeys(std::set<CPubKey>& pubkeys, std::set<CExtPubKey>& ext_pubs) const override {} }; BOOST_FIXTURE_TEST_CASE(wallet_load_descriptors, TestingSetup) diff --git a/src/wallet/transaction.cpp b/src/wallet/transaction.cpp index 6777257e53..561880482f 100644 --- a/src/wallet/transaction.cpp +++ b/src/wallet/transaction.cpp @@ -45,7 +45,7 @@ void CWalletTx::updateState(interfaces::Chain& chain) }; if (auto* conf = state<TxStateConfirmed>()) { lookup_block(conf->confirmed_block_hash, conf->confirmed_block_height, m_state); - } else if (auto* conf = state<TxStateConflicted>()) { + } else if (auto* conf = state<TxStateBlockConflicted>()) { lookup_block(conf->conflicting_block_hash, conf->conflicting_block_height, m_state); } } diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index ddeb931112..9c27574103 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -43,12 +43,12 @@ struct TxStateInMempool { }; //! State of rejected transaction that conflicts with a confirmed block. -struct TxStateConflicted { +struct TxStateBlockConflicted { uint256 conflicting_block_hash; int conflicting_block_height; - explicit TxStateConflicted(const uint256& block_hash, int height) : conflicting_block_hash(block_hash), conflicting_block_height(height) {} - std::string toString() const { return strprintf("Conflicted (block=%s, height=%i)", conflicting_block_hash.ToString(), conflicting_block_height); } + explicit TxStateBlockConflicted(const uint256& block_hash, int height) : conflicting_block_hash(block_hash), conflicting_block_height(height) {} + std::string toString() const { return strprintf("BlockConflicted (block=%s, height=%i)", conflicting_block_hash.ToString(), conflicting_block_height); } }; //! State of transaction not confirmed or conflicting with a known block and @@ -75,7 +75,7 @@ struct TxStateUnrecognized { }; //! All possible CWalletTx states -using TxState = std::variant<TxStateConfirmed, TxStateInMempool, TxStateConflicted, TxStateInactive, TxStateUnrecognized>; +using TxState = std::variant<TxStateConfirmed, TxStateInMempool, TxStateBlockConflicted, TxStateInactive, TxStateUnrecognized>; //! Subset of states transaction sync logic is implemented to handle. using SyncTxState = std::variant<TxStateConfirmed, TxStateInMempool, TxStateInactive>; @@ -90,7 +90,7 @@ static inline TxState TxStateInterpretSerialized(TxStateUnrecognized data) } else if (data.index >= 0) { return TxStateConfirmed{data.block_hash, /*height=*/-1, data.index}; } else if (data.index == -1) { - return TxStateConflicted{data.block_hash, /*height=*/-1}; + return TxStateBlockConflicted{data.block_hash, /*height=*/-1}; } return data; } @@ -102,7 +102,7 @@ static inline uint256 TxStateSerializedBlockHash(const TxState& state) [](const TxStateInactive& inactive) { return inactive.abandoned ? uint256::ONE : uint256::ZERO; }, [](const TxStateInMempool& in_mempool) { return uint256::ZERO; }, [](const TxStateConfirmed& confirmed) { return confirmed.confirmed_block_hash; }, - [](const TxStateConflicted& conflicted) { return conflicted.conflicting_block_hash; }, + [](const TxStateBlockConflicted& conflicted) { return conflicted.conflicting_block_hash; }, [](const TxStateUnrecognized& unrecognized) { return unrecognized.block_hash; } }, state); } @@ -114,7 +114,7 @@ static inline int TxStateSerializedIndex(const TxState& state) [](const TxStateInactive& inactive) { return inactive.abandoned ? -1 : 0; }, [](const TxStateInMempool& in_mempool) { return 0; }, [](const TxStateConfirmed& confirmed) { return confirmed.position_in_block; }, - [](const TxStateConflicted& conflicted) { return -1; }, + [](const TxStateBlockConflicted& conflicted) { return -1; }, [](const TxStateUnrecognized& unrecognized) { return unrecognized.index; } }, state); } @@ -258,6 +258,14 @@ public: CTransactionRef tx; TxState m_state; + // Set of mempool transactions that conflict + // directly with the transaction, or that conflict + // with an ancestor transaction. This set will be + // empty if state is InMempool or Confirmed, but + // can be nonempty if state is Inactive or + // BlockConflicted. + std::set<Txid> mempool_conflicts; + template<typename Stream> void Serialize(Stream& s) const { @@ -335,9 +343,10 @@ public: void updateState(interfaces::Chain& chain); bool isAbandoned() const { return state<TxStateInactive>() && state<TxStateInactive>()->abandoned; } - bool isConflicted() const { return state<TxStateConflicted>(); } + bool isMempoolConflicted() const { return !mempool_conflicts.empty(); } + bool isBlockConflicted() const { return state<TxStateBlockConflicted>(); } bool isInactive() const { return state<TxStateInactive>(); } - bool isUnconfirmed() const { return !isAbandoned() && !isConflicted() && !isConfirmed(); } + bool isUnconfirmed() const { return !isAbandoned() && !isBlockConflicted() && !isMempoolConflicted() && !isConfirmed(); } bool isConfirmed() const { return state<TxStateConfirmed>(); } const Txid& GetHash() const LIFETIMEBOUND { return tx->GetHash(); } const Wtxid& GetWitnessHash() const LIFETIMEBOUND { return tx->GetWitnessHash(); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 3ac09430d8..8f4171eb15 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -752,8 +752,8 @@ bool CWallet::IsSpent(const COutPoint& outpoint) const const uint256& wtxid = it->second; const auto mit = mapWallet.find(wtxid); if (mit != mapWallet.end()) { - int depth = GetTxDepthInMainChain(mit->second); - if (depth > 0 || (depth == 0 && !mit->second.isAbandoned())) + const auto& wtx = mit->second; + if (!wtx.isAbandoned() && !wtx.isBlockConflicted() && !wtx.isMempoolConflicted()) return true; // Spent } } @@ -1197,7 +1197,7 @@ bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx auto it = mapWallet.find(txin.prevout.hash); if (it != mapWallet.end()) { CWalletTx& prevtx = it->second; - if (auto* prev = prevtx.state<TxStateConflicted>()) { + if (auto* prev = prevtx.state<TxStateBlockConflicted>()) { MarkConflicted(prev->conflicting_block_hash, prev->conflicting_block_height, wtx.GetHash()); } } @@ -1309,7 +1309,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) assert(!wtx.isConfirmed()); assert(!wtx.InMempool()); // If already conflicted or abandoned, no need to set abandoned - if (!wtx.isConflicted() && !wtx.isAbandoned()) { + if (!wtx.isBlockConflicted() && !wtx.isAbandoned()) { wtx.m_state = TxStateInactive{/*abandoned=*/true}; return TxUpdate::NOTIFY_CHANGED; } @@ -1346,7 +1346,7 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c if (conflictconfirms < GetTxDepthInMainChain(wtx)) { // Block is 'more conflicted' than current confirm; update. // Mark transaction as conflicted with this block. - wtx.m_state = TxStateConflicted{hashBlock, conflicting_height}; + wtx.m_state = TxStateBlockConflicted{hashBlock, conflicting_height}; return TxUpdate::CHANGED; } return TxUpdate::UNCHANGED; @@ -1360,7 +1360,10 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c void CWallet::RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { // Do not flush the wallet here for performance reasons WalletBatch batch(GetDatabase(), false); + RecursiveUpdateTxState(&batch, tx_hash, try_updating_state); +} +void CWallet::RecursiveUpdateTxState(WalletBatch* batch, const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { std::set<uint256> todo; std::set<uint256> done; @@ -1377,7 +1380,7 @@ void CWallet::RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingSt TxUpdate update_state = try_updating_state(wtx); if (update_state != TxUpdate::UNCHANGED) { wtx.MarkDirty(); - batch.WriteTx(wtx); + if (batch) batch->WriteTx(wtx); // Iterate over all its outputs, and update those tx states as well (if applicable) for (unsigned int i = 0; i < wtx.tx->vout.size(); ++i) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(COutPoint(Txid::FromUint256(now), i)); @@ -1418,6 +1421,20 @@ void CWallet::transactionAddedToMempool(const CTransactionRef& tx) { if (it != mapWallet.end()) { RefreshMempoolStatus(it->second, chain()); } + + const Txid& txid = tx->GetHash(); + + for (const CTxIn& tx_in : tx->vin) { + // For each wallet transaction spending this prevout.. + for (auto range = mapTxSpends.equal_range(tx_in.prevout); range.first != range.second; range.first++) { + const uint256& spent_id = range.first->second; + // Skip the recently added tx + if (spent_id == txid) continue; + RecursiveUpdateTxState(/*batch=*/nullptr, spent_id, [&txid](CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { + return wtx.mempool_conflicts.insert(txid).second ? TxUpdate::CHANGED : TxUpdate::UNCHANGED; + }); + } + } } void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) { @@ -1455,6 +1472,21 @@ void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRe // https://github.com/bitcoin-core/bitcoin-devwiki/wiki/Wallet-Transaction-Conflict-Tracking SyncTransaction(tx, TxStateInactive{}); } + + const Txid& txid = tx->GetHash(); + + for (const CTxIn& tx_in : tx->vin) { + // Iterate over all wallet transactions spending txin.prev + // and recursively mark them as no longer conflicting with + // txid + for (auto range = mapTxSpends.equal_range(tx_in.prevout); range.first != range.second; range.first++) { + const uint256& spent_id = range.first->second; + + RecursiveUpdateTxState(/*batch=*/nullptr, spent_id, [&txid](CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { + return wtx.mempool_conflicts.erase(txid) ? TxUpdate::CHANGED : TxUpdate::UNCHANGED; + }); + } + } } void CWallet::blockConnected(ChainstateRole role, const interfaces::BlockInfo& block) @@ -1506,11 +1538,11 @@ void CWallet::blockDisconnected(const interfaces::BlockInfo& block) for (TxSpends::const_iterator _it = range.first; _it != range.second; ++_it) { CWalletTx& wtx = mapWallet.find(_it->second)->second; - if (!wtx.isConflicted()) continue; + if (!wtx.isBlockConflicted()) continue; auto try_updating_state = [&](CWalletTx& tx) { - if (!tx.isConflicted()) return TxUpdate::UNCHANGED; - if (tx.state<TxStateConflicted>()->conflicting_block_height >= disconnect_height) { + if (!tx.isBlockConflicted()) return TxUpdate::UNCHANGED; + if (tx.state<TxStateBlockConflicted>()->conflicting_block_height >= disconnect_height) { tx.m_state = TxStateInactive{}; return TxUpdate::CHANGED; } @@ -2635,7 +2667,7 @@ void ReserveDestination::ReturnDestination() address = CNoDestination(); } -bool CWallet::DisplayAddress(const CTxDestination& dest) +util::Result<void> CWallet::DisplayAddress(const CTxDestination& dest) { CScript scriptPubKey = GetScriptForDestination(dest); for (const auto& spk_man : GetScriptPubKeyMans(scriptPubKey)) { @@ -2644,9 +2676,9 @@ bool CWallet::DisplayAddress(const CTxDestination& dest) continue; } ExternalSigner signer = ExternalSignerScriptPubKeyMan::GetExternalSigner(); - return signer_spk_man->DisplayAddress(scriptPubKey, signer); + return signer_spk_man->DisplayAddress(dest, signer); } - return false; + return util::Error{_("There is no ScriptPubKeyManager for this address")}; } bool CWallet::LockCoin(const COutPoint& output, WalletBatch* batch) @@ -2787,7 +2819,7 @@ unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx, bool rescanning_old std::optional<uint256> block_hash; if (auto* conf = wtx.state<TxStateConfirmed>()) { block_hash = conf->confirmed_block_hash; - } else if (auto* conf = wtx.state<TxStateConflicted>()) { + } else if (auto* conf = wtx.state<TxStateBlockConflicted>()) { block_hash = conf->conflicting_block_hash; } @@ -3377,7 +3409,7 @@ int CWallet::GetTxDepthInMainChain(const CWalletTx& wtx) const if (auto* conf = wtx.state<TxStateConfirmed>()) { assert(conf->confirmed_block_height >= 0); return GetLastBlockHeight() - conf->confirmed_block_height + 1; - } else if (auto* conf = wtx.state<TxStateConflicted>()) { + } else if (auto* conf = wtx.state<TxStateBlockConflicted>()) { assert(conf->conflicting_block_height >= 0); return -1 * (GetLastBlockHeight() - conf->conflicting_block_height + 1); } else { @@ -3465,6 +3497,17 @@ std::set<ScriptPubKeyMan*> CWallet::GetActiveScriptPubKeyMans() const return spk_mans; } +bool CWallet::IsActiveScriptPubKeyMan(const ScriptPubKeyMan& spkm) const +{ + for (const auto& [_, ext_spkm] : m_external_spk_managers) { + if (ext_spkm == &spkm) return true; + } + for (const auto& [_, int_spkm] : m_internal_spk_managers) { + if (int_spkm == &spkm) return true; + } + return false; +} + std::set<ScriptPubKeyMan*> CWallet::GetAllScriptPubKeyMans() const { std::set<ScriptPubKeyMan*> spk_mans; @@ -3619,6 +3662,26 @@ DescriptorScriptPubKeyMan& CWallet::LoadDescriptorScriptPubKeyMan(uint256 id, Wa return *spk_manager; } +DescriptorScriptPubKeyMan& CWallet::SetupDescriptorScriptPubKeyMan(WalletBatch& batch, const CExtKey& master_key, const OutputType& output_type, bool internal) +{ + AssertLockHeld(cs_wallet); + auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, m_keypool_size)); + if (IsCrypted()) { + if (IsLocked()) { + throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); + } + if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, &batch)) { + throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); + } + } + spk_manager->SetupDescriptorGeneration(batch, master_key, output_type, internal); + DescriptorScriptPubKeyMan* out = spk_manager.get(); + uint256 id = spk_manager->GetID(); + AddScriptPubKeyMan(id, std::move(spk_manager)); + AddActiveScriptPubKeyManWithDb(batch, id, output_type, internal); + return *out; +} + void CWallet::SetupDescriptorScriptPubKeyMans(const CExtKey& master_key) { AssertLockHeld(cs_wallet); @@ -3629,19 +3692,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans(const CExtKey& master_key) for (bool internal : {false, true}) { for (OutputType t : OUTPUT_TYPES) { - auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, m_keypool_size)); - if (IsCrypted()) { - if (IsLocked()) { - throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); - } - if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, &batch)) { - throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); - } - } - spk_manager->SetupDescriptorGeneration(batch, master_key, t, internal); - uint256 id = spk_manager->GetID(); - AddScriptPubKeyMan(id, std::move(spk_manager)); - AddActiveScriptPubKeyManWithDb(batch, id, t, internal); + SetupDescriptorScriptPubKeyMan(batch, master_key, t, internal); } } @@ -4336,7 +4387,7 @@ util::Result<MigrationResult> MigrateLegacyToDescriptor(const std::string& walle // Make a backup of the DB fs::path this_wallet_dir = fs::absolute(fs::PathFromString(local_wallet->GetDatabase().Filename())).parent_path(); - fs::path backup_filename = fs::PathFromString(strprintf("%s-%d.legacy.bak", wallet_name, GetTime())); + fs::path backup_filename = fs::PathFromString(strprintf("%s_%d.legacy.bak", (wallet_name.empty() ? "default_wallet" : wallet_name), GetTime())); fs::path backup_path = this_wallet_dir / backup_filename; if (!local_wallet->BackupWallet(fs::PathToString(backup_path))) { if (was_loaded) { @@ -4469,4 +4520,40 @@ void CWallet::TopUpCallback(const std::set<CScript>& spks, ScriptPubKeyMan* spkm // Update scriptPubKey cache CacheNewScriptPubKeys(spks, spkm); } + +std::set<CExtPubKey> CWallet::GetActiveHDPubKeys() const +{ + AssertLockHeld(cs_wallet); + + Assert(IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); + + std::set<CExtPubKey> active_xpubs; + for (const auto& spkm : GetActiveScriptPubKeyMans()) { + const DescriptorScriptPubKeyMan* desc_spkm = dynamic_cast<DescriptorScriptPubKeyMan*>(spkm); + assert(desc_spkm); + LOCK(desc_spkm->cs_desc_man); + WalletDescriptor w_desc = desc_spkm->GetWalletDescriptor(); + + std::set<CPubKey> desc_pubkeys; + std::set<CExtPubKey> desc_xpubs; + w_desc.descriptor->GetPubKeys(desc_pubkeys, desc_xpubs); + active_xpubs.merge(std::move(desc_xpubs)); + } + return active_xpubs; +} + +std::optional<CKey> CWallet::GetKey(const CKeyID& keyid) const +{ + Assert(IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); + + for (const auto& spkm : GetAllScriptPubKeyMans()) { + const DescriptorScriptPubKeyMan* desc_spkm = dynamic_cast<DescriptorScriptPubKeyMan*>(spkm); + assert(desc_spkm); + LOCK(desc_spkm->cs_desc_man); + if (std::optional<CKey> key = desc_spkm->GetKey(keyid)) { + return key; + } + } + return std::nullopt; +} } // namespace wallet diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 8b0ee22276..6a998fa398 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -364,6 +364,7 @@ private: /** Mark a transaction (and its in-wallet descendants) as a particular tx state. */ void RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void RecursiveUpdateTxState(WalletBatch* batch, const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** Mark a transaction's inputs dirty, thus forcing the outputs to be recomputed */ void MarkInputsDirty(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -518,11 +519,6 @@ public: * referenced in transaction, and might cause assert failures. */ int GetTxDepthInMainChain(const CWalletTx& wtx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool IsTxInMainChain(const CWalletTx& wtx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) - { - AssertLockHeld(cs_wallet); - return GetTxDepthInMainChain(wtx) > 0; - } /** * @return number of blocks to maturity for this transaction: @@ -541,8 +537,8 @@ public: bool IsSpentKey(const CScript& scriptPubKey) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SetSpentKeyState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used, std::set<CTxDestination>& tx_destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - /** Display address on an external signer. Returns false if external signer support is not compiled */ - bool DisplayAddress(const CTxDestination& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** Display address on an external signer. */ + util::Result<void> DisplayAddress(const CTxDestination& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool IsLockedCoin(const COutPoint& output) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool LockCoin(const COutPoint& output, WalletBatch* batch = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -942,6 +938,7 @@ public: //! Returns all unique ScriptPubKeyMans in m_internal_spk_managers and m_external_spk_managers std::set<ScriptPubKeyMan*> GetActiveScriptPubKeyMans() const; + bool IsActiveScriptPubKeyMan(const ScriptPubKeyMan& spkm) const; //! Returns all unique ScriptPubKeyMans std::set<ScriptPubKeyMan*> GetAllScriptPubKeyMans() const; @@ -1017,6 +1014,8 @@ public: //! @param[in] internal Whether this ScriptPubKeyMan provides change addresses void DeactivateScriptPubKeyMan(uint256 id, OutputType type, bool internal); + //! Create new DescriptorScriptPubKeyMan and add it to the wallet + DescriptorScriptPubKeyMan& SetupDescriptorScriptPubKeyMan(WalletBatch& batch, const CExtKey& master_key, const OutputType& output_type, bool internal) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Create new DescriptorScriptPubKeyMans and add them to the wallet void SetupDescriptorScriptPubKeyMans(const CExtKey& master_key) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SetupDescriptorScriptPubKeyMans() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -1053,6 +1052,13 @@ public: void CacheNewScriptPubKeys(const std::set<CScript>& spks, ScriptPubKeyMan* spkm); void TopUpCallback(const std::set<CScript>& spks, ScriptPubKeyMan* spkm) override; + + //! Retrieve the xpubs in use by the active descriptors + std::set<CExtPubKey> GetActiveHDPubKeys() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + //! Find the private key for the given key id from the wallet's descriptors, if available + //! Returns nullopt when no descriptor has the key or if the wallet is locked. + std::optional<CKey> GetKey(const CKeyID& keyid) const; }; /** diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index fdd5bc36d8..0de2617d45 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -4,7 +4,9 @@ #include <wallet/walletutil.h> +#include <chainparams.h> #include <common/args.h> +#include <key_io.h> #include <logging.h> namespace wallet { @@ -43,4 +45,58 @@ WalletFeature GetClosestWalletFeature(int version) } return static_cast<WalletFeature>(0); } + +WalletDescriptor GenerateWalletDescriptor(const CExtPubKey& master_key, const OutputType& addr_type, bool internal) +{ + int64_t creation_time = GetTime(); + + std::string xpub = EncodeExtPubKey(master_key); + + // Build descriptor string + std::string desc_prefix; + std::string desc_suffix = "/*)"; + switch (addr_type) { + case OutputType::LEGACY: { + desc_prefix = "pkh(" + xpub + "/44h"; + break; + } + case OutputType::P2SH_SEGWIT: { + desc_prefix = "sh(wpkh(" + xpub + "/49h"; + desc_suffix += ")"; + break; + } + case OutputType::BECH32: { + desc_prefix = "wpkh(" + xpub + "/84h"; + break; + } + case OutputType::BECH32M: { + desc_prefix = "tr(" + xpub + "/86h"; + break; + } + case OutputType::UNKNOWN: { + // We should never have a DescriptorScriptPubKeyMan for an UNKNOWN OutputType, + // so if we get to this point something is wrong + assert(false); + } + } // no default case, so the compiler can warn about missing cases + assert(!desc_prefix.empty()); + + // Mainnet derives at 0', testnet and regtest derive at 1' + if (Params().IsTestChain()) { + desc_prefix += "/1h"; + } else { + desc_prefix += "/0h"; + } + + std::string internal_path = internal ? "/1" : "/0"; + std::string desc_str = desc_prefix + "/0h" + internal_path + desc_suffix; + + // Make the descriptor + FlatSigningProvider keys; + std::string error; + std::unique_ptr<Descriptor> desc = Parse(desc_str, keys, error, false); + WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0); + return w_desc; +} + } // namespace wallet diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index 7ad3ffe9e4..38926c1eb8 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -114,6 +114,8 @@ public: WalletDescriptor() {} WalletDescriptor(std::shared_ptr<Descriptor> descriptor, uint64_t creation_time, int32_t range_start, int32_t range_end, int32_t next_index) : descriptor(descriptor), id(DescriptorID(*descriptor)), creation_time(creation_time), range_start(range_start), range_end(range_end), next_index(next_index) { } }; + +WalletDescriptor GenerateWalletDescriptor(const CExtPubKey& master_key, const OutputType& output_type, bool internal); } // namespace wallet #endif // BITCOIN_WALLET_WALLETUTIL_H diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index 63c2737706..536e471053 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -8,6 +8,7 @@ #include <kernel/chain.h> #include <kernel/mempool_entry.h> #include <logging.h> +#include <netbase.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <validationinterface.h> @@ -41,7 +42,7 @@ std::list<const CZMQAbstractNotifier*> CZMQNotificationInterface::GetActiveNotif return result; } -std::unique_ptr<CZMQNotificationInterface> CZMQNotificationInterface::Create(std::function<bool(CBlock&, const CBlockIndex&)> get_block_by_index) +std::unique_ptr<CZMQNotificationInterface> CZMQNotificationInterface::Create(std::function<bool(std::vector<uint8_t>&, const CBlockIndex&)> get_block_by_index) { std::map<std::string, CZMQNotifierFactory> factories; factories["pubhashblock"] = CZMQAbstractNotifier::Create<CZMQPublishHashBlockNotifier>; @@ -57,7 +58,12 @@ std::unique_ptr<CZMQNotificationInterface> CZMQNotificationInterface::Create(std { std::string arg("-zmq" + entry.first); const auto& factory = entry.second; - for (const std::string& address : gArgs.GetArgs(arg)) { + for (std::string& address : gArgs.GetArgs(arg)) { + // libzmq uses prefix "ipc://" for UNIX domain sockets + if (address.substr(0, ADDR_PREFIX_UNIX.length()) == ADDR_PREFIX_UNIX) { + address.replace(0, ADDR_PREFIX_UNIX.length(), ADDR_PREFIX_IPC); + } + std::unique_ptr<CZMQAbstractNotifier> notifier = factory(); notifier->SetType(entry.first); notifier->SetAddress(address); diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index 45d0982bd3..c879fdd0dd 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -12,6 +12,7 @@ #include <functional> #include <list> #include <memory> +#include <vector> class CBlock; class CBlockIndex; @@ -25,7 +26,7 @@ public: std::list<const CZMQAbstractNotifier*> GetActiveNotifiers() const; - static std::unique_ptr<CZMQNotificationInterface> Create(std::function<bool(CBlock&, const CBlockIndex&)> get_block_by_index); + static std::unique_ptr<CZMQNotificationInterface> Create(std::function<bool(std::vector<uint8_t>&, const CBlockIndex&)> get_block_by_index); protected: bool Initialize(); diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 0f20706364..608870c489 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -243,16 +243,13 @@ bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) { LogPrint(BCLog::ZMQ, "Publish rawblock %s to %s\n", pindex->GetBlockHash().GetHex(), this->address); - DataStream ss; - CBlock block; + std::vector<uint8_t> block{}; if (!m_get_block_by_index(block, *pindex)) { zmqError("Can't read block from disk"); return false; } - ss << TX_WITH_WITNESS(block); - - return SendZmqMessage(MSG_RAWBLOCK, &(*ss.begin()), ss.size()); + return SendZmqMessage(MSG_RAWBLOCK, block.data(), block.size()); } bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &transaction) diff --git a/src/zmq/zmqpublishnotifier.h b/src/zmq/zmqpublishnotifier.h index a5cd433761..cc941a899c 100644 --- a/src/zmq/zmqpublishnotifier.h +++ b/src/zmq/zmqpublishnotifier.h @@ -10,8 +10,8 @@ #include <cstddef> #include <cstdint> #include <functional> +#include <vector> -class CBlock; class CBlockIndex; class CTransaction; @@ -49,10 +49,10 @@ public: class CZMQPublishRawBlockNotifier : public CZMQAbstractPublishNotifier { private: - const std::function<bool(CBlock&, const CBlockIndex&)> m_get_block_by_index; + const std::function<bool(std::vector<uint8_t>&, const CBlockIndex&)> m_get_block_by_index; public: - CZMQPublishRawBlockNotifier(std::function<bool(CBlock&, const CBlockIndex&)> get_block_by_index) + CZMQPublishRawBlockNotifier(std::function<bool(std::vector<uint8_t>&, const CBlockIndex&)> get_block_by_index) : m_get_block_by_index{std::move(get_block_by_index)} {} bool NotifyBlock(const CBlockIndex *pindex) override; }; diff --git a/src/zmq/zmqutil.h b/src/zmq/zmqutil.h index 334b51aa91..bec48c0a56 100644 --- a/src/zmq/zmqutil.h +++ b/src/zmq/zmqutil.h @@ -9,4 +9,7 @@ void zmqError(const std::string& str); +/** Prefix for unix domain socket addresses (which are local filesystem paths) */ +const std::string ADDR_PREFIX_IPC = "ipc://"; // used by libzmq, example "ipc:///root/path/to/file" + #endif // BITCOIN_ZMQ_ZMQUTIL_H diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index 740d3b7f0e..01ba2834c4 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -36,7 +36,7 @@ class AbortNodeTest(BitcoinTestFramework): # Check that node0 aborted self.log.info("Waiting for crash") - self.nodes[0].wait_until_stopped(timeout=5, expect_error=True, expected_stderr="Error: A fatal internal error occurred, see debug.log for details") + self.nodes[0].wait_until_stopped(timeout=5, expect_error=True, expected_stderr="Error: A fatal internal error occurred, see debug.log for details: Failed to disconnect block.") self.log.info("Node crashed - now verifying restart fails") self.nodes[0].assert_start_raises_init_error() diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py index a7ce864fde..95d33d62ea 100755 --- a/test/functional/feature_addrman.py +++ b/test/functional/feature_addrman.py @@ -156,12 +156,7 @@ class AddrmanTest(BitcoinTestFramework): ) self.log.info("Check that missing addrman is recreated") - self.stop_node(0) - os.remove(peers_dat) - with self.nodes[0].assert_debug_log([ - f'Creating peers.dat because the file was not found ("{peers_dat}")', - ]): - self.start_node(0) + self.restart_node(0, clear_addrman=True) assert_equal(self.nodes[0].getnodeaddresses(), []) diff --git a/test/functional/feature_asmap.py b/test/functional/feature_asmap.py index ae483fe449..024a8fa18c 100755 --- a/test/functional/feature_asmap.py +++ b/test/functional/feature_asmap.py @@ -39,11 +39,12 @@ def expected_messages(filename): class AsmapTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [["-checkaddrman=1"]] # Do addrman checks on all operations. + # Do addrman checks on all operations and use deterministic addrman + self.extra_args = [["-checkaddrman=1", "-test=addrman"]] def fill_addrman(self, node_id): - """Add 1 tried address to the addrman, followed by 1 new address.""" - for addr, tried in [[0, True], [1, False]]: + """Add 2 tried addresses to the addrman, followed by 2 new addresses.""" + for addr, tried in [[0, True], [1, True], [2, False], [3, False]]: self.nodes[node_id].addpeeraddress(address=f"101.{addr}.0.0", tried=tried, port=8333) def test_without_asmap_arg(self): @@ -84,12 +85,12 @@ class AsmapTest(BitcoinTestFramework): self.log.info("Test bitcoind -asmap restart with addrman containing new and tried entries") self.stop_node(0) shutil.copyfile(self.asmap_raw, self.default_asmap) - self.start_node(0, ["-asmap", "-checkaddrman=1"]) + self.start_node(0, ["-asmap", "-checkaddrman=1", "-test=addrman"]) self.fill_addrman(node_id=0) - self.restart_node(0, ["-asmap", "-checkaddrman=1"]) + self.restart_node(0, ["-asmap", "-checkaddrman=1", "-test=addrman"]) with self.node.assert_debug_log( expected_msgs=[ - "CheckAddrman: new 1, tried 1, total 2 started", + "CheckAddrman: new 2, tried 2, total 4 started", "CheckAddrman: completed", ] ): @@ -114,7 +115,7 @@ class AsmapTest(BitcoinTestFramework): def test_asmap_health_check(self): self.log.info('Test bitcoind -asmap logs ASMap Health Check with basic stats') shutil.copyfile(self.asmap_raw, self.default_asmap) - msg = "ASMap Health Check: 2 clearnet peers are mapped to 1 ASNs with 0 peers being unmapped" + msg = "ASMap Health Check: 4 clearnet peers are mapped to 3 ASNs with 0 peers being unmapped" with self.node.assert_debug_log(expected_msgs=[msg]): self.start_node(0, extra_args=['-asmap']) os.remove(self.default_asmap) diff --git a/test/functional/feature_assumeutxo.py b/test/functional/feature_assumeutxo.py index 60dd751ff8..19cbbcffdb 100755 --- a/test/functional/feature_assumeutxo.py +++ b/test/functional/feature_assumeutxo.py @@ -11,9 +11,6 @@ The assumeutxo value generated and used here is committed to in ## Possible test improvements -- TODO: test what happens with -reindex and -reindex-chainstate before the - snapshot is validated, and make sure it's deleted successfully. - Interesting test cases could be loading an assumeutxo snapshot file with: - TODO: Valid hash but invalid snapshot file (bad coin height or @@ -34,6 +31,7 @@ Interesting starting states could be loading a snapshot when the current chain t """ from shutil import rmtree +from dataclasses import dataclass from test_framework.messages import tx_from_hex from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -113,6 +111,12 @@ class AssumeutxoTest(BitcoinTestFramework): f.write(valid_snapshot_contents[(32 + 8 + offset + len(content)):]) expected_error(log_msg=f"[snapshot] bad snapshot content hash: expected a4bf3407ccb2cc0145c49ebba8fa91199f8a3903daf0883875941497d2493c27, got {wrong_hash}") + def test_headers_not_synced(self, valid_snapshot_path): + for node in self.nodes[1:]: + assert_raises_rpc_error(-32603, "The base block header (3bb7ce5eba0be48939b7a521ac1ba9316afee2c7bada3a0cca24188e6d7d96c0) must appear in the headers chain. Make sure all headers are syncing, and call this RPC again.", + node.loadtxoutset, + valid_snapshot_path) + def test_invalid_chainstate_scenarios(self): self.log.info("Test different scenarios of invalid snapshot chainstate in datadir") @@ -127,7 +131,7 @@ class AssumeutxoTest(BitcoinTestFramework): with self.nodes[0].assert_debug_log([log_msg]): self.nodes[0].assert_start_raises_init_error(expected_msg=error_msg) - expected_error_msg = f"Error: A fatal internal error occurred, see debug.log for details" + expected_error_msg = f"Error: A fatal internal error occurred, see debug.log for details: Assumeutxo data not found for the given blockhash '7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a'." error_details = f"Assumeutxo data not found for the given blockhash" expected_error(log_msg=error_details, error_msg=expected_error_msg) @@ -166,26 +170,28 @@ class AssumeutxoTest(BitcoinTestFramework): for n in self.nodes: n.setmocktime(n.getblockheader(n.getbestblockhash())['time']) - self.sync_blocks() - # Generate a series of blocks that `n0` will have in the snapshot, - # but that n1 doesn't yet see. In order for the snapshot to activate, - # though, we have to ferry over the new headers to n1 so that it - # isn't waiting forever to see the header of the snapshot's base block - # while disconnected from n0. + # but that n1 and n2 don't yet see. + assert n0.getblockcount() == START_HEIGHT + blocks = {START_HEIGHT: Block(n0.getbestblockhash(), 1, START_HEIGHT + 1)} for i in range(100): + block_tx = 1 if i % 3 == 0: self.mini_wallet.send_self_transfer(from_node=n0) + block_tx += 1 self.generate(n0, nblocks=1, sync_fun=self.no_op) - newblock = n0.getblock(n0.getbestblockhash(), 0) - - # make n1 aware of the new header, but don't give it the block. - n1.submitheader(newblock) - n2.submitheader(newblock) + height = n0.getblockcount() + hash = n0.getbestblockhash() + blocks[height] = Block(hash, block_tx, blocks[height-1].chain_tx + block_tx) + if i == 4: + # Create a stale block that forks off the main chain before the snapshot. + temp_invalid = n0.getbestblockhash() + n0.invalidateblock(temp_invalid) + stale_hash = self.generateblock(n0, output="raw(aaaa)", transactions=[], sync_fun=self.no_op)["hash"] + n0.invalidateblock(stale_hash) + n0.reconsiderblock(temp_invalid) + stale_block = n0.getblock(stale_hash, 0) - # Ensure everyone is seeing the same headers. - for n in self.nodes: - assert_equal(n.getblockchaininfo()["headers"], SNAPSHOT_BASE_HEIGHT) self.log.info("-- Testing assumeutxo + some indexes + pruning") @@ -195,10 +201,27 @@ class AssumeutxoTest(BitcoinTestFramework): self.log.info(f"Creating a UTXO snapshot at height {SNAPSHOT_BASE_HEIGHT}") dump_output = n0.dumptxoutset('utxos.dat') + self.log.info("Test loading snapshot when headers are not synced") + self.test_headers_not_synced(dump_output['path']) + + # In order for the snapshot to activate, we have to ferry over the new + # headers to n1 and n2 so that they see the header of the snapshot's + # base block while disconnected from n0. + for i in range(1, 300): + block = n0.getblock(n0.getblockhash(i), 0) + # make n1 and n2 aware of the new header, but don't give them the + # block. + n1.submitheader(block) + n2.submitheader(block) + + # Ensure everyone is seeing the same headers. + for n in self.nodes: + assert_equal(n.getblockchaininfo()["headers"], SNAPSHOT_BASE_HEIGHT) + assert_equal( dump_output['txoutset_hash'], "a4bf3407ccb2cc0145c49ebba8fa91199f8a3903daf0883875941497d2493c27") - assert_equal(dump_output["nchaintx"], 334) + assert_equal(dump_output["nchaintx"], blocks[SNAPSHOT_BASE_HEIGHT].chain_tx) assert_equal(n0.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT) # Mine more blocks on top of the snapshot that n1 hasn't yet seen. This @@ -219,6 +242,29 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(loaded['coins_loaded'], SNAPSHOT_BASE_HEIGHT) assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT) + def check_tx_counts(final: bool) -> None: + """Check nTx and nChainTx intermediate values right after loading + the snapshot, and final values after the snapshot is validated.""" + for height, block in blocks.items(): + tx = n1.getblockheader(block.hash)["nTx"] + chain_tx = n1.getchaintxstats(nblocks=1, blockhash=block.hash)["txcount"] + + # Intermediate nTx of the starting block should be set, but nTx of + # later blocks should be 0 before they are downloaded. + if final or height == START_HEIGHT: + assert_equal(tx, block.tx) + else: + assert_equal(tx, 0) + + # Intermediate nChainTx of the starting block and snapshot block + # should be set, but others should be 0 until they are downloaded. + if final or height in (START_HEIGHT, SNAPSHOT_BASE_HEIGHT): + assert_equal(chain_tx, block.chain_tx) + else: + assert_equal(chain_tx, 0) + + check_tx_counts(final=False) + normal, snapshot = n1.getchainstates()["chainstates"] assert_equal(normal['blocks'], START_HEIGHT) assert_equal(normal.get('snapshot_blockhash'), None) @@ -229,6 +275,15 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(n1.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT) + self.log.info("Submit a stale block that forked off the chain before the snapshot") + # Normally a block like this would not be downloaded, but if it is + # submitted early before the background chain catches up to the fork + # point, it winds up in m_blocks_unlinked and triggers a corner case + # that previously crashed CheckBlockIndex. + n1.submitblock(stale_block) + n1.getchaintips() + n1.getblock(stale_hash) + self.log.info("Submit a spending transaction for a snapshot chainstate coin to the mempool") # spend the coinbase output of the first block that is not available on node1 spend_coin_blockhash = n1.getblockhash(START_HEIGHT + 1) @@ -266,6 +321,16 @@ class AssumeutxoTest(BitcoinTestFramework): self.log.info("Restarted node before snapshot validation completed, reloading...") self.restart_node(1, extra_args=self.extra_args[1]) + + # Send snapshot block to n1 out of order. This makes the test less + # realistic because normally the snapshot block is one of the last + # blocks downloaded, but its useful to test because it triggers more + # corner cases in ReceivedBlockTransactions() and CheckBlockIndex() + # setting and testing nChainTx values, and it exposed previous bugs. + snapshot_hash = n0.getblockhash(SNAPSHOT_BASE_HEIGHT) + snapshot_block = n0.getblock(snapshot_hash, 0) + n1.submitblock(snapshot_block) + self.connect_nodes(0, 1) self.log.info(f"Ensuring snapshot chain syncs to tip. ({FINAL_HEIGHT})") @@ -282,6 +347,8 @@ class AssumeutxoTest(BitcoinTestFramework): } self.wait_until(lambda: n1.getindexinfo() == completed_idx_state) + self.log.info("Re-check nTx and nChainTx values") + check_tx_counts(final=True) for i in (0, 1): n = self.nodes[i] @@ -309,6 +376,17 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(loaded['coins_loaded'], SNAPSHOT_BASE_HEIGHT) assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT) + for reindex_arg in ['-reindex=1', '-reindex-chainstate=1']: + self.log.info(f"Check that restarting with {reindex_arg} will delete the snapshot chainstate") + self.restart_node(2, extra_args=[reindex_arg, *self.extra_args[2]]) + assert_equal(1, len(n2.getchainstates()["chainstates"])) + for i in range(1, 300): + block = n0.getblock(n0.getblockhash(i), 0) + n2.submitheader(block) + loaded = n2.loadtxoutset(dump_output['path']) + assert_equal(loaded['coins_loaded'], SNAPSHOT_BASE_HEIGHT) + assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT) + normal, snapshot = n2.getchainstates()['chainstates'] assert_equal(normal['blocks'], START_HEIGHT) assert_equal(normal.get('snapshot_blockhash'), None) @@ -356,6 +434,11 @@ class AssumeutxoTest(BitcoinTestFramework): self.connect_nodes(0, 2) self.wait_until(lambda: n2.getblockcount() == FINAL_HEIGHT) +@dataclass +class Block: + hash: str + tx: int + chain_tx: int if __name__ == '__main__': AssumeutxoTest().main() diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index 613d2eab14..982fa79915 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -159,7 +159,7 @@ class AssumeValidTest(BitcoinTestFramework): for i in range(2202): p2p1.send_message(msg_block(self.blocks[i])) # Syncing 2200 blocks can take a while on slow systems. Give it plenty of time to sync. - p2p1.sync_with_ping(960) + p2p1.sync_with_ping(timeout=960) assert_equal(self.nodes[1].getblock(self.nodes[1].getbestblockhash())['height'], 2202) p2p2 = self.nodes[2].add_p2p_connection(BaseNode()) diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 8c45fb5a4d..fb3f662271 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -83,9 +83,10 @@ CLTV_HEIGHT = 111 class BIP65Test(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.extra_args = [[ f'-testactivationheight=cltv@{CLTV_HEIGHT}', - '-whitelist=noban@127.0.0.1', '-par=1', # Use only one script thread to get the exact reject reason for testing '-acceptnonstdtxn=1', # cltv_invalidate is nonstandard ]] diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index 92e4187f3c..bc1f9e8f2f 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -95,8 +95,9 @@ class BIP68_112_113Test(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.extra_args = [[ - '-whitelist=noban@127.0.0.1', f'-testactivationheight=csv@{CSV_ACTIVATION_HEIGHT}', '-par=1', # Use only one script thread to get the exact reject reason for testing ]] diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 44c12b2a59..035e7151ca 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -47,9 +47,10 @@ DERSIG_HEIGHT = 102 class BIP66Test(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.extra_args = [[ f'-testactivationheight=dersig@{DERSIG_HEIGHT}', - '-whitelist=noban@127.0.0.1', '-par=1', # Use only one script thread to get the exact log msg for testing ]] self.setup_clean_chain = True diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 4f56d585d3..ffc87f8b8b 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -132,11 +132,12 @@ def make_tx(wallet, utxo, feerate): class EstimateFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 3 - # Force fSendTrickle to true (via whitelist.noban) + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.extra_args = [ - ["-whitelist=noban@127.0.0.1"], - ["-whitelist=noban@127.0.0.1", "-blockmaxweight=68000"], - ["-whitelist=noban@127.0.0.1", "-blockmaxweight=32000"], + [], + ["-blockmaxweight=68000"], + ["-blockmaxweight=32000"], ] def setup_network(self): diff --git a/test/functional/feature_index_prune.py b/test/functional/feature_index_prune.py index d6e802b399..66c0a4f615 100755 --- a/test/functional/feature_index_prune.py +++ b/test/functional/feature_index_prune.py @@ -31,7 +31,7 @@ class FeatureIndexPruneTest(BitcoinTestFramework): expected_stats = { 'coinstatsindex': {'synced': True, 'best_block_height': height} } - self.wait_until(lambda: self.nodes[1].getindexinfo() == expected_stats) + self.wait_until(lambda: self.nodes[1].getindexinfo() == expected_stats, timeout=150) expected = {**expected_filter, **expected_stats} self.wait_until(lambda: self.nodes[2].getindexinfo() == expected) @@ -128,7 +128,7 @@ class FeatureIndexPruneTest(BitcoinTestFramework): self.log.info("make sure we get an init error when starting the nodes again with the indices") filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" - end_msg = f"{os.linesep}Error: Failed to start indexes, shutting down.." + end_msg = f"{os.linesep}Error: A fatal internal error occurred, see debug.log for details: Failed to start indexes, shutting down.." for i, msg in enumerate([filter_msg, stats_msg, filter_msg]): self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=msg+end_msg) diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index 662007d65e..7a6f639021 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -17,6 +17,7 @@ Test plan: - support no authentication (other proxy) - support no authentication + user/pass authentication (Tor) - proxy on IPv6 + - proxy over unix domain sockets - Create various proxies (as threads) - Create nodes that connect to them @@ -39,7 +40,9 @@ addnode connect to a CJDNS address - Test passing unknown -onlynet """ +import os import socket +import tempfile from test_framework.socks5 import Socks5Configuration, Socks5Command, Socks5Server, AddressType from test_framework.test_framework import BitcoinTestFramework @@ -47,7 +50,7 @@ from test_framework.util import ( assert_equal, p2p_port, ) -from test_framework.netutil import test_ipv6_local +from test_framework.netutil import test_ipv6_local, test_unix_socket # Networks returned by RPC getpeerinfo. NET_UNROUTABLE = "not_publicly_routable" @@ -60,14 +63,17 @@ NET_CJDNS = "cjdns" # Networks returned by RPC getnetworkinfo, defined in src/rpc/net.cpp::GetNetworksInfo() NETWORKS = frozenset({NET_IPV4, NET_IPV6, NET_ONION, NET_I2P, NET_CJDNS}) +# Use the shortest temp path possible since UNIX sockets may have as little as 92-char limit +socket_path = tempfile.NamedTemporaryFile().name class ProxyTest(BitcoinTestFramework): def set_test_params(self): - self.num_nodes = 5 + self.num_nodes = 7 self.setup_clean_chain = True def setup_nodes(self): self.have_ipv6 = test_ipv6_local() + self.have_unix_sockets = test_unix_socket() # Create two proxies on different ports # ... one unauthenticated self.conf1 = Socks5Configuration() @@ -89,6 +95,15 @@ class ProxyTest(BitcoinTestFramework): else: self.log.warning("Testing without local IPv6 support") + if self.have_unix_sockets: + self.conf4 = Socks5Configuration() + self.conf4.af = socket.AF_UNIX + self.conf4.addr = socket_path + self.conf4.unauth = True + self.conf4.auth = True + else: + self.log.warning("Testing without local unix domain sockets support") + self.serv1 = Socks5Server(self.conf1) self.serv1.start() self.serv2 = Socks5Server(self.conf2) @@ -96,6 +111,9 @@ class ProxyTest(BitcoinTestFramework): if self.have_ipv6: self.serv3 = Socks5Server(self.conf3) self.serv3.start() + if self.have_unix_sockets: + self.serv4 = Socks5Server(self.conf4) + self.serv4.start() # We will not try to connect to this. self.i2p_sam = ('127.0.0.1', 7656) @@ -109,10 +127,15 @@ class ProxyTest(BitcoinTestFramework): ['-listen', f'-proxy={self.conf2.addr[0]}:{self.conf2.addr[1]}','-proxyrandomize=1'], [], ['-listen', f'-proxy={self.conf1.addr[0]}:{self.conf1.addr[1]}','-proxyrandomize=1', - '-cjdnsreachable'] + '-cjdnsreachable'], + [], + [] ] if self.have_ipv6: args[3] = ['-listen', f'-proxy=[{self.conf3.addr[0]}]:{self.conf3.addr[1]}','-proxyrandomize=0', '-noonion'] + if self.have_unix_sockets: + args[5] = ['-listen', f'-proxy=unix:{socket_path}'] + args[6] = ['-listen', f'-onion=unix:{socket_path}'] self.add_nodes(self.num_nodes, extra_args=args) self.start_nodes() @@ -124,7 +147,7 @@ class ProxyTest(BitcoinTestFramework): def node_test(self, node, *, proxies, auth, test_onion, test_cjdns): rv = [] addr = "15.61.23.23:1234" - self.log.debug(f"Test: outgoing IPv4 connection through node for address {addr}") + self.log.debug(f"Test: outgoing IPv4 connection through node {node.index} for address {addr}") node.addnode(addr, "onetry") cmd = proxies[0].queue.get() assert isinstance(cmd, Socks5Command) @@ -140,7 +163,7 @@ class ProxyTest(BitcoinTestFramework): if self.have_ipv6: addr = "[1233:3432:2434:2343:3234:2345:6546:4534]:5443" - self.log.debug(f"Test: outgoing IPv6 connection through node for address {addr}") + self.log.debug(f"Test: outgoing IPv6 connection through node {node.index} for address {addr}") node.addnode(addr, "onetry") cmd = proxies[1].queue.get() assert isinstance(cmd, Socks5Command) @@ -156,7 +179,7 @@ class ProxyTest(BitcoinTestFramework): if test_onion: addr = "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion:8333" - self.log.debug(f"Test: outgoing onion connection through node for address {addr}") + self.log.debug(f"Test: outgoing onion connection through node {node.index} for address {addr}") node.addnode(addr, "onetry") cmd = proxies[2].queue.get() assert isinstance(cmd, Socks5Command) @@ -171,7 +194,7 @@ class ProxyTest(BitcoinTestFramework): if test_cjdns: addr = "[fc00:1:2:3:4:5:6:7]:8888" - self.log.debug(f"Test: outgoing CJDNS connection through node for address {addr}") + self.log.debug(f"Test: outgoing CJDNS connection through node {node.index} for address {addr}") node.addnode(addr, "onetry") cmd = proxies[1].queue.get() assert isinstance(cmd, Socks5Command) @@ -185,7 +208,7 @@ class ProxyTest(BitcoinTestFramework): self.network_test(node, addr, network=NET_CJDNS) addr = "node.noumenon:8333" - self.log.debug(f"Test: outgoing DNS name connection through node for address {addr}") + self.log.debug(f"Test: outgoing DNS name connection through node {node.index} for address {addr}") node.addnode(addr, "onetry") cmd = proxies[3].queue.get() assert isinstance(cmd, Socks5Command) @@ -230,6 +253,12 @@ class ProxyTest(BitcoinTestFramework): proxies=[self.serv1, self.serv1, self.serv1, self.serv1], auth=False, test_onion=True, test_cjdns=True) + if self.have_unix_sockets: + self.node_test(self.nodes[5], + proxies=[self.serv4, self.serv4, self.serv4, self.serv4], + auth=True, test_onion=True, test_cjdns=False) + + def networks_dict(d): r = {} for x in d['networks']: @@ -315,6 +344,37 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n4['i2p']['reachable'], False) assert_equal(n4['cjdns']['reachable'], True) + if self.have_unix_sockets: + n5 = networks_dict(nodes_network_info[5]) + assert_equal(NETWORKS, n5.keys()) + for net in NETWORKS: + if net == NET_I2P: + expected_proxy = '' + expected_randomize = False + else: + expected_proxy = 'unix:' + self.conf4.addr # no port number + expected_randomize = True + assert_equal(n5[net]['proxy'], expected_proxy) + assert_equal(n5[net]['proxy_randomize_credentials'], expected_randomize) + assert_equal(n5['onion']['reachable'], True) + assert_equal(n5['i2p']['reachable'], False) + assert_equal(n5['cjdns']['reachable'], False) + + n6 = networks_dict(nodes_network_info[6]) + assert_equal(NETWORKS, n6.keys()) + for net in NETWORKS: + if net != NET_ONION: + expected_proxy = '' + expected_randomize = False + else: + expected_proxy = 'unix:' + self.conf4.addr # no port number + expected_randomize = True + assert_equal(n6[net]['proxy'], expected_proxy) + assert_equal(n6[net]['proxy_randomize_credentials'], expected_randomize) + assert_equal(n6['onion']['reachable'], True) + assert_equal(n6['i2p']['reachable'], False) + assert_equal(n6['cjdns']['reachable'], False) + self.stop_node(1) self.log.info("Test passing invalid -proxy hostname raises expected init error") @@ -383,6 +443,18 @@ class ProxyTest(BitcoinTestFramework): msg = "Error: Unknown network specified in -onlynet: 'abc'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + self.log.info("Test passing too-long unix path to -proxy raises init error") + self.nodes[1].extra_args = [f"-proxy=unix:{'x' * 1000}"] + if self.have_unix_sockets: + msg = f"Error: Invalid -proxy address or hostname: 'unix:{'x' * 1000}'" + else: + # If unix sockets are not supported, the file path is incorrectly interpreted as host:port + msg = f"Error: Invalid port specified in -proxy: 'unix:{'x' * 1000}'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + # Cleanup socket path we established outside the individual test directory. + if self.have_unix_sockets: + os.unlink(socket_path) if __name__ == '__main__': ProxyTest().main() diff --git a/test/functional/feature_reindex_readonly.py b/test/functional/feature_reindex_readonly.py index dd99c3c4fa..25cff87a3b 100755 --- a/test/functional/feature_reindex_readonly.py +++ b/test/functional/feature_reindex_readonly.py @@ -24,6 +24,7 @@ class BlockstoreReindexTest(BitcoinTestFramework): opreturn = "6a" nulldata = fastprune_blockfile_size * "ff" self.generateblock(self.nodes[0], output=f"raw({opreturn}{nulldata})", transactions=[]) + block_count = self.nodes[0].getblockcount() self.stop_node(0) assert (self.nodes[0].chain_path / "blocks" / "blk00000.dat").exists() @@ -73,10 +74,10 @@ class BlockstoreReindexTest(BitcoinTestFramework): pass if undo_immutable: - self.log.info("Attempt to restart and reindex the node with the unwritable block file") - with self.nodes[0].assert_debug_log(expected_msgs=['FlushStateToDisk', 'failed to open file'], unexpected_msgs=[]): - self.nodes[0].assert_start_raises_init_error(extra_args=['-reindex', '-fastprune'], - expected_msg="Error: A fatal internal error occurred, see debug.log for details") + self.log.debug("Attempt to restart and reindex the node with the unwritable block file") + with self.nodes[0].wait_for_debug_log([b"Reindexing finished"]): + self.start_node(0, extra_args=['-reindex', '-fastprune']) + assert block_count == self.nodes[0].getblockcount() undo_immutable() filename.chmod(0o777) diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index 05aa40bbfe..ae8d6b226d 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -53,8 +53,7 @@ class RESTTest (BitcoinTestFramework): self.num_nodes = 2 self.extra_args = [["-rest", "-blockfilterindex=1"], []] # whitelist peers to speed up tx relay / mempool sync - for args in self.extra_args: - args.append("-whitelist=noban@127.0.0.1") + self.noban_tx_relay = True self.supports_cli = False def test_rest_request( diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 2358dd4387..9f6f8919de 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -3,8 +3,11 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the ZMQ notification interface.""" +import os import struct +import tempfile from time import sleep +from io import BytesIO from test_framework.address import ( ADDRESS_BCRT1_P2WSH_OP_TRUE, @@ -17,6 +20,7 @@ from test_framework.blocktools import ( ) from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import ( + CBlock, hash256, tx_from_hex, ) @@ -28,7 +32,7 @@ from test_framework.util import ( from test_framework.wallet import ( MiniWallet, ) -from test_framework.netutil import test_ipv6_local +from test_framework.netutil import test_ipv6_local, test_unix_socket # Test may be skipped and not have zmq installed @@ -104,9 +108,8 @@ class ZMQTestSetupBlock: class ZMQTest (BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - # This test isn't testing txn relay/timing, so set whitelist on the - # peers for instant txn relay. This speeds up the test run time 2-3x. - self.extra_args = [["-whitelist=noban@127.0.0.1"]] * self.num_nodes + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.zmq_port_base = p2p_port(self.num_nodes + 1) def skip_test_if_missing_module(self): @@ -118,6 +121,10 @@ class ZMQTest (BitcoinTestFramework): self.ctx = zmq.Context() try: self.test_basic() + if test_unix_socket(): + self.test_basic(unix=True) + else: + self.log.info("Skipping ipc test, because UNIX sockets are not supported.") self.test_sequence() self.test_mempool_sync() self.test_reorg() @@ -138,8 +145,7 @@ class ZMQTest (BitcoinTestFramework): socket.setsockopt(zmq.IPV6, 1) subscribers.append(ZMQSubscriber(socket, topic.encode())) - self.restart_node(0, [f"-zmqpub{topic}={address}" for topic, address in services] + - self.extra_args[0]) + self.restart_node(0, [f"-zmqpub{topic}={address.replace('ipc://', 'unix:')}" for topic, address in services]) for i, sub in enumerate(subscribers): sub.socket.connect(services[i][1]) @@ -176,12 +182,19 @@ class ZMQTest (BitcoinTestFramework): return subscribers - def test_basic(self): + def test_basic(self, unix = False): + self.log.info(f"Running basic test with {'ipc' if unix else 'tcp'} protocol") # Invalid zmq arguments don't take down the node, see #17185. self.restart_node(0, ["-zmqpubrawtx=foo", "-zmqpubhashtx=bar"]) address = f"tcp://127.0.0.1:{self.zmq_port_base}" + + if unix: + # Use the shortest temp path possible since paths may have as little as 92-char limit + socket_path = tempfile.NamedTemporaryFile().name + address = f"ipc://{socket_path}" + subs = self.setup_zmq_test([(topic, address) for topic in ["hashblock", "hashtx", "rawblock", "rawtx"]]) hashblock = subs[0] @@ -203,8 +216,13 @@ class ZMQTest (BitcoinTestFramework): assert_equal(tx.hash, txid.hex()) # Should receive the generated raw block. - block = rawblock.receive() - assert_equal(genhashes[x], hash256_reversed(block[:80]).hex()) + hex = rawblock.receive() + block = CBlock() + block.deserialize(BytesIO(hex)) + assert block.is_valid() + assert_equal(block.vtx[0].hash, tx.hash) + assert_equal(len(block.vtx), 1) + assert_equal(genhashes[x], hash256_reversed(hex[:80]).hex()) # Should receive the generated block hash. hash = hashblock.receive().hex() @@ -242,6 +260,8 @@ class ZMQTest (BitcoinTestFramework): ]) assert_equal(self.nodes[1].getzmqnotifications(), []) + if unix: + os.unlink(socket_path) def test_reorg(self): diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 538e1fe053..272e932fcc 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -96,6 +96,12 @@ class MempoolAcceptanceTest(BitcoinTestFramework): rawtxs=[raw_tx_in_block], maxfeerate=1, )) + # Check negative feerate + assert_raises_rpc_error(-3, "Amount out of range", lambda: self.check_mempool_result( + result_expected=None, + rawtxs=[raw_tx_in_block], + maxfeerate=-0.01, + )) # ... 0.99 passes self.check_mempool_result( result_expected=[{'txid': txid_in_block, 'allowed': False, 'reject-reason': 'txn-already-known'}], diff --git a/test/functional/mempool_accept_v3.py b/test/functional/mempool_accept_v3.py index 7ac3c97c4b..1b55cd0a0d 100755 --- a/test/functional/mempool_accept_v3.py +++ b/test/functional/mempool_accept_v3.py @@ -15,10 +15,13 @@ from test_framework.util import ( assert_raises_rpc_error, ) from test_framework.wallet import ( + COIN, DEFAULT_FEE, MiniWallet, ) +MAX_REPLACEMENT_CANDIDATES = 100 + def cleanup(extra_args=None): def decorator(func): def wrapper(self): @@ -290,8 +293,13 @@ class MempoolAcceptV3(BitcoinTestFramework): self.check_mempool([tx_in_mempool["txid"]]) @cleanup(extra_args=["-acceptnonstdtxn=1"]) - def test_mempool_sibling(self): - self.log.info("Test that v3 transaction cannot have mempool siblings") + def test_sibling_eviction_package(self): + """ + When a transaction has a mempool sibling, it may be eligible for sibling eviction. + However, this option is only available in single transaction acceptance. It doesn't work in + a multi-testmempoolaccept (where RBF is disabled) or when doing package CPFP. + """ + self.log.info("Test v3 sibling eviction in submitpackage and multi-testmempoolaccept") node = self.nodes[0] # Add a parent + child to mempool tx_mempool_parent = self.wallet.send_self_transfer_multi( @@ -307,26 +315,57 @@ class MempoolAcceptV3(BitcoinTestFramework): ) self.check_mempool([tx_mempool_parent["txid"], tx_mempool_sibling["txid"]]) - tx_has_mempool_sibling = self.wallet.create_self_transfer( + tx_sibling_1 = self.wallet.create_self_transfer( utxo_to_spend=tx_mempool_parent["new_utxos"][1], - version=3 + version=3, + fee_rate=DEFAULT_FEE*100, ) - expected_error_mempool_sibling = f"v3-rule-violation, tx {tx_mempool_parent['txid']} (wtxid={tx_mempool_parent['wtxid']}) would exceed descendant count limit" - assert_raises_rpc_error(-26, expected_error_mempool_sibling, node.sendrawtransaction, tx_has_mempool_sibling["hex"]) + tx_has_mempool_uncle = self.wallet.create_self_transfer(utxo_to_spend=tx_sibling_1["new_utxo"], version=3) - tx_has_mempool_uncle = self.wallet.create_self_transfer(utxo_to_spend=tx_has_mempool_sibling["new_utxo"], version=3) + tx_sibling_2 = self.wallet.create_self_transfer( + utxo_to_spend=tx_mempool_parent["new_utxos"][0], + version=3, + fee_rate=DEFAULT_FEE*200, + ) + + tx_sibling_3 = self.wallet.create_self_transfer( + utxo_to_spend=tx_mempool_parent["new_utxos"][1], + version=3, + fee_rate=0, + ) + tx_bumps_parent_with_sibling = self.wallet.create_self_transfer( + utxo_to_spend=tx_sibling_3["new_utxo"], + version=3, + fee_rate=DEFAULT_FEE*300, + ) - # Also fails with another non-related transaction via testmempoolaccept + # Fails with another non-related transaction via testmempoolaccept tx_unrelated = self.wallet.create_self_transfer(version=3) - result_test_unrelated = node.testmempoolaccept([tx_has_mempool_sibling["hex"], tx_unrelated["hex"]]) + result_test_unrelated = node.testmempoolaccept([tx_sibling_1["hex"], tx_unrelated["hex"]]) assert_equal(result_test_unrelated[0]["reject-reason"], "v3-rule-violation") - result_test_1p1c = node.testmempoolaccept([tx_has_mempool_sibling["hex"], tx_has_mempool_uncle["hex"]]) + # Fails in a package via testmempoolaccept + result_test_1p1c = node.testmempoolaccept([tx_sibling_1["hex"], tx_has_mempool_uncle["hex"]]) assert_equal(result_test_1p1c[0]["reject-reason"], "v3-rule-violation") - # Also fails with a child via submitpackage - result_submitpackage = node.submitpackage([tx_has_mempool_sibling["hex"], tx_has_mempool_uncle["hex"]]) - assert_equal(result_submitpackage["tx-results"][tx_has_mempool_sibling['wtxid']]['error'], expected_error_mempool_sibling) + # Allowed when tx is submitted in a package and evaluated individually. + # Note that the child failed since it would be the 3rd generation. + result_package_indiv = node.submitpackage([tx_sibling_1["hex"], tx_has_mempool_uncle["hex"]]) + self.check_mempool([tx_mempool_parent["txid"], tx_sibling_1["txid"]]) + expected_error_gen3 = f"v3-rule-violation, tx {tx_has_mempool_uncle['txid']} (wtxid={tx_has_mempool_uncle['wtxid']}) would have too many ancestors" + + assert_equal(result_package_indiv["tx-results"][tx_has_mempool_uncle['wtxid']]['error'], expected_error_gen3) + + # Allowed when tx is submitted in a package with in-mempool parent (which is deduplicated). + node.submitpackage([tx_mempool_parent["hex"], tx_sibling_2["hex"]]) + self.check_mempool([tx_mempool_parent["txid"], tx_sibling_2["txid"]]) + + # Child cannot pay for sibling eviction for parent, as it violates v3 topology limits + result_package_cpfp = node.submitpackage([tx_sibling_3["hex"], tx_bumps_parent_with_sibling["hex"]]) + self.check_mempool([tx_mempool_parent["txid"], tx_sibling_2["txid"]]) + expected_error_cpfp = f"v3-rule-violation, tx {tx_mempool_parent['txid']} (wtxid={tx_mempool_parent['wtxid']}) would exceed descendant count limit" + + assert_equal(result_package_cpfp["tx-results"][tx_sibling_3['wtxid']]['error'], expected_error_cpfp) @cleanup(extra_args=["-datacarriersize=1000", "-acceptnonstdtxn=1"]) @@ -429,11 +468,123 @@ class MempoolAcceptV3(BitcoinTestFramework): self.check_mempool([ancestor_tx["txid"], child_1_conflict["txid"], child_2["txid"]]) assert_equal(node.getmempoolentry(ancestor_tx["txid"])["descendantcount"], 3) + @cleanup(extra_args=["-acceptnonstdtxn=1"]) + def test_v3_sibling_eviction(self): + self.log.info("Test sibling eviction for v3") + node = self.nodes[0] + tx_v3_parent = self.wallet.send_self_transfer_multi(from_node=node, num_outputs=2, version=3) + # This is the sibling to replace + tx_v3_child_1 = self.wallet.send_self_transfer( + from_node=node, utxo_to_spend=tx_v3_parent["new_utxos"][0], fee_rate=DEFAULT_FEE * 2, version=3 + ) + assert tx_v3_child_1["txid"] in node.getrawmempool() + + self.log.info("Test tx must be higher feerate than sibling to evict it") + tx_v3_child_2_rule6 = self.wallet.create_self_transfer( + utxo_to_spend=tx_v3_parent["new_utxos"][1], fee_rate=DEFAULT_FEE, version=3 + ) + rule6_str = f"insufficient fee (including sibling eviction), rejecting replacement {tx_v3_child_2_rule6['txid']}; new feerate" + assert_raises_rpc_error(-26, rule6_str, node.sendrawtransaction, tx_v3_child_2_rule6["hex"]) + self.check_mempool([tx_v3_parent['txid'], tx_v3_child_1['txid']]) + + self.log.info("Test tx must meet absolute fee rules to evict sibling") + tx_v3_child_2_rule4 = self.wallet.create_self_transfer( + utxo_to_spend=tx_v3_parent["new_utxos"][1], fee_rate=2 * DEFAULT_FEE + Decimal("0.00000001"), version=3 + ) + rule4_str = f"insufficient fee (including sibling eviction), rejecting replacement {tx_v3_child_2_rule4['txid']}, not enough additional fees to relay" + assert_raises_rpc_error(-26, rule4_str, node.sendrawtransaction, tx_v3_child_2_rule4["hex"]) + self.check_mempool([tx_v3_parent['txid'], tx_v3_child_1['txid']]) + + self.log.info("Test tx cannot cause more than 100 evictions including RBF and sibling eviction") + # First add 4 groups of 25 transactions. + utxos_for_conflict = [] + txids_v2_100 = [] + for _ in range(4): + confirmed_utxo = self.wallet.get_utxo(confirmed_only=True) + utxos_for_conflict.append(confirmed_utxo) + # 25 is within descendant limits + chain_length = int(MAX_REPLACEMENT_CANDIDATES / 4) + chain = self.wallet.create_self_transfer_chain(chain_length=chain_length, utxo_to_spend=confirmed_utxo) + for item in chain: + txids_v2_100.append(item["txid"]) + node.sendrawtransaction(item["hex"]) + self.check_mempool(txids_v2_100 + [tx_v3_parent["txid"], tx_v3_child_1["txid"]]) + + # Replacing 100 transactions is fine + tx_v3_replacement_only = self.wallet.create_self_transfer_multi(utxos_to_spend=utxos_for_conflict, fee_per_output=4000000) + # Override maxfeerate - it costs a lot to replace these 100 transactions. + assert node.testmempoolaccept([tx_v3_replacement_only["hex"]], maxfeerate=0)[0]["allowed"] + # Adding another one exceeds the limit. + utxos_for_conflict.append(tx_v3_parent["new_utxos"][1]) + tx_v3_child_2_rule5 = self.wallet.create_self_transfer_multi(utxos_to_spend=utxos_for_conflict, fee_per_output=4000000, version=3) + rule5_str = f"too many potential replacements (including sibling eviction), rejecting replacement {tx_v3_child_2_rule5['txid']}; too many potential replacements (101 > 100)" + assert_raises_rpc_error(-26, rule5_str, node.sendrawtransaction, tx_v3_child_2_rule5["hex"]) + self.check_mempool(txids_v2_100 + [tx_v3_parent["txid"], tx_v3_child_1["txid"]]) + + self.log.info("Test sibling eviction is successful if it meets all RBF rules") + tx_v3_child_2 = self.wallet.create_self_transfer( + utxo_to_spend=tx_v3_parent["new_utxos"][1], fee_rate=DEFAULT_FEE*10, version=3 + ) + node.sendrawtransaction(tx_v3_child_2["hex"]) + self.check_mempool(txids_v2_100 + [tx_v3_parent["txid"], tx_v3_child_2["txid"]]) + + self.log.info("Test that it's possible to do a sibling eviction and RBF at the same time") + utxo_unrelated_conflict = self.wallet.get_utxo(confirmed_only=True) + tx_unrelated_replacee = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=utxo_unrelated_conflict) + assert tx_unrelated_replacee["txid"] in node.getrawmempool() + + fee_to_beat_child2 = int(tx_v3_child_2["fee"] * COIN) + + tx_v3_child_3 = self.wallet.create_self_transfer_multi( + utxos_to_spend=[tx_v3_parent["new_utxos"][0], utxo_unrelated_conflict], fee_per_output=fee_to_beat_child2*5, version=3 + ) + node.sendrawtransaction(tx_v3_child_3["hex"]) + self.check_mempool(txids_v2_100 + [tx_v3_parent["txid"], tx_v3_child_3["txid"]]) + + @cleanup(extra_args=["-acceptnonstdtxn=1"]) + def test_reorg_sibling_eviction_1p2c(self): + node = self.nodes[0] + self.log.info("Test that sibling eviction is not allowed when multiple siblings exist") + + tx_with_multi_children = self.wallet.send_self_transfer_multi(from_node=node, num_outputs=3, version=3, confirmed_only=True) + self.check_mempool([tx_with_multi_children["txid"]]) + + block_to_disconnect = self.generate(node, 1)[0] + self.check_mempool([]) + + tx_with_sibling1 = self.wallet.send_self_transfer(from_node=node, version=3, utxo_to_spend=tx_with_multi_children["new_utxos"][0]) + tx_with_sibling2 = self.wallet.send_self_transfer(from_node=node, version=3, utxo_to_spend=tx_with_multi_children["new_utxos"][1]) + self.check_mempool([tx_with_sibling1["txid"], tx_with_sibling2["txid"]]) + + # Create a reorg, bringing tx_with_multi_children back into the mempool with a descendant count of 3. + node.invalidateblock(block_to_disconnect) + self.check_mempool([tx_with_multi_children["txid"], tx_with_sibling1["txid"], tx_with_sibling2["txid"]]) + assert_equal(node.getmempoolentry(tx_with_multi_children["txid"])["descendantcount"], 3) + + # Sibling eviction is not allowed because there are two siblings + tx_with_sibling3 = self.wallet.create_self_transfer( + version=3, + utxo_to_spend=tx_with_multi_children["new_utxos"][2], + fee_rate=DEFAULT_FEE*50 + ) + expected_error_2siblings = f"v3-rule-violation, tx {tx_with_multi_children['txid']} (wtxid={tx_with_multi_children['wtxid']}) would exceed descendant count limit" + assert_raises_rpc_error(-26, expected_error_2siblings, node.sendrawtransaction, tx_with_sibling3["hex"]) + + # However, an RBF (with conflicting inputs) is possible even if the resulting cluster size exceeds 2 + tx_with_sibling3_rbf = self.wallet.send_self_transfer( + from_node=node, + version=3, + utxo_to_spend=tx_with_multi_children["new_utxos"][0], + fee_rate=DEFAULT_FEE*50 + ) + self.check_mempool([tx_with_multi_children["txid"], tx_with_sibling3_rbf["txid"], tx_with_sibling2["txid"]]) + + def run_test(self): self.log.info("Generate blocks to create UTXOs") node = self.nodes[0] self.wallet = MiniWallet(node) - self.generate(self.wallet, 110) + self.generate(self.wallet, 120) self.test_v3_acceptance() self.test_v3_replacement() self.test_v3_bip125() @@ -441,10 +592,12 @@ class MempoolAcceptV3(BitcoinTestFramework): self.test_nondefault_package_limits() self.test_v3_ancestors_package() self.test_v3_ancestors_package_and_mempool() - self.test_mempool_sibling() + self.test_sibling_eviction_package() self.test_v3_package_inheritance() self.test_v3_in_testmempoolaccept() self.test_reorg_2child_rbf() + self.test_v3_sibling_eviction() + self.test_reorg_sibling_eviction_1p2c() if __name__ == "__main__": diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index 6215610c31..e8a568f7ab 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -6,7 +6,6 @@ from decimal import Decimal -from test_framework.blocktools import COINBASE_MATURITY from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -14,8 +13,7 @@ from test_framework.util import ( assert_fee_amount, assert_greater_than, assert_raises_rpc_error, - create_lots_of_big_transactions, - gen_return_txouts, + fill_mempool, ) from test_framework.wallet import ( COIN, @@ -34,50 +32,6 @@ class MempoolLimitTest(BitcoinTestFramework): ]] self.supports_cli = False - def fill_mempool(self): - """Fill mempool until eviction.""" - self.log.info("Fill the mempool until eviction is triggered and the mempoolminfee rises") - txouts = gen_return_txouts() - node = self.nodes[0] - miniwallet = self.wallet - relayfee = node.getnetworkinfo()['relayfee'] - - tx_batch_size = 1 - num_of_batches = 75 - # Generate UTXOs to flood the mempool - # 1 to create a tx initially that will be evicted from the mempool later - # 75 transactions each with a fee rate higher than the previous one - # And 1 more to verify that this tx does not get added to the mempool with a fee rate less than the mempoolminfee - # And 2 more for the package cpfp test - self.generate(miniwallet, 1 + (num_of_batches * tx_batch_size)) - - # Mine 99 blocks so that the UTXOs are allowed to be spent - self.generate(node, COINBASE_MATURITY - 1) - - self.log.debug("Create a mempool tx that will be evicted") - tx_to_be_evicted_id = miniwallet.send_self_transfer(from_node=node, fee_rate=relayfee)["txid"] - - # Increase the tx fee rate to give the subsequent transactions a higher priority in the mempool - # The tx has an approx. vsize of 65k, i.e. multiplying the previous fee rate (in sats/kvB) - # by 130 should result in a fee that corresponds to 2x of that fee rate - base_fee = relayfee * 130 - - self.log.debug("Fill up the mempool with txs with higher fee rate") - with node.assert_debug_log(["rolling minimum fee bumped"]): - for batch_of_txid in range(num_of_batches): - fee = (batch_of_txid + 1) * base_fee - create_lots_of_big_transactions(miniwallet, node, fee, tx_batch_size, txouts) - - self.log.debug("The tx should be evicted by now") - # The number of transactions created should be greater than the ones present in the mempool - assert_greater_than(tx_batch_size * num_of_batches, len(node.getrawmempool())) - # Initial tx created should not be present in the mempool anymore as it had a lower fee rate - assert tx_to_be_evicted_id not in node.getrawmempool() - - self.log.debug("Check that mempoolminfee is larger than minrelaytxfee") - assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) - assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) - def test_rbf_carveout_disallowed(self): node = self.nodes[0] @@ -139,7 +93,7 @@ class MempoolLimitTest(BitcoinTestFramework): assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) - self.fill_mempool() + fill_mempool(self, node, self.wallet) current_info = node.getmempoolinfo() mempoolmin_feerate = current_info["mempoolminfee"] @@ -229,7 +183,7 @@ class MempoolLimitTest(BitcoinTestFramework): assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) - self.fill_mempool() + fill_mempool(self, node, self.wallet) current_info = node.getmempoolinfo() mempoolmin_feerate = current_info["mempoolminfee"] @@ -303,7 +257,7 @@ class MempoolLimitTest(BitcoinTestFramework): assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) - self.fill_mempool() + fill_mempool(self, node, self.wallet) # Deliberately try to create a tx with a fee less than the minimum mempool fee to assert that it does not get added to the mempool self.log.info('Create a mempool tx that will not pass mempoolminfee') diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 95f7939412..dcb66b2ca1 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -27,10 +27,11 @@ assert CUSTOM_DESCENDANT_LIMIT >= CUSTOM_ANCESTOR_LIMIT class MempoolPackagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.extra_args = [ [ "-maxorphantx=1000", - "-whitelist=noban@127.0.0.1", # immediate tx relay ], [ "-maxorphantx=1000", diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index da796d3f70..5f2dde8eac 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -308,7 +308,7 @@ class MiningTest(BitcoinTestFramework): # Should ask for the block from a p2p node, if they announce the header as well: peer = node.add_p2p_connection(P2PDataStore()) - peer.wait_for_getheaders(timeout=5) # Drop the first getheaders + peer.wait_for_getheaders(timeout=5, block_hash=block.hashPrevBlock) peer.send_blocks_and_test(blocks=[block], node=node) # Must be active now: assert chain_tip(block.hash, status='active', branchlen=0) in node.getchaintips() diff --git a/test/functional/mocks/signer.py b/test/functional/mocks/signer.py index 5f4fad6380..23d163aac3 100755 --- a/test/functional/mocks/signer.py +++ b/test/functional/mocks/signer.py @@ -25,35 +25,36 @@ def getdescriptors(args): sys.stdout.write(json.dumps({ "receive": [ - "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/0/*)#vt6w3l3j", - "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/0/*))#r0grqw5x", - "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/0/*)#x30uthjs", - "tr([00000001/86'/1'/" + args.account + "']" + xpub + "/0/*)#sng9rd4t" + "pkh([00000001/44h/1h/" + args.account + "']" + xpub + "/0/*)#aqllu46s", + "sh(wpkh([00000001/49h/1h/" + args.account + "']" + xpub + "/0/*))#5dh56mgg", + "wpkh([00000001/84h/1h/" + args.account + "']" + xpub + "/0/*)#h62dxaej", + "tr([00000001/86h/1h/" + args.account + "']" + xpub + "/0/*)#pcd5w87f" ], "internal": [ - "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/1/*)#all0v2p2", - "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/1/*))#kwx4c3pe", - "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/1/*)#h92akzzg", - "tr([00000001/86'/1'/" + args.account + "']" + xpub + "/1/*)#p8dy7c9n" + "pkh([00000001/44h/1h/" + args.account + "']" + xpub + "/1/*)#v567pq2g", + "sh(wpkh([00000001/49h/1h/" + args.account + "']" + xpub + "/1/*))#pvezzyah", + "wpkh([00000001/84h/1h/" + args.account + "']" + xpub + "/1/*)#xw0vmgf2", + "tr([00000001/86h/1h/" + args.account + "']" + xpub + "/1/*)#svg4njw3" ] })) def displayaddress(args): - # Several descriptor formats are acceptable, so allowing for potential - # changes to InferDescriptor: if args.fingerprint != "00000001": return sys.stdout.write(json.dumps({"error": "Unexpected fingerprint", "fingerprint": args.fingerprint})) - expected_desc = [ - "wpkh([00000001/84'/1'/0'/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#0yneg42r", - "tr([00000001/86'/1'/0'/0/0]c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#4vdj9jqk", - ] + expected_desc = { + "wpkh([00000001/84h/1h/0h/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#3te6hhy7": "bcrt1qm90ugl4d48jv8n6e5t9ln6t9zlpm5th68x4f8g", + "sh(wpkh([00000001/49h/1h/0h/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7))#kz9y5w82": "2N2gQKzjUe47gM8p1JZxaAkTcoHPXV6YyVp", + "pkh([00000001/44h/1h/0h/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#q3pqd8wh": "n1LKejAadN6hg2FrBXoU1KrwX4uK16mco9", + "tr([00000001/86h/1h/0h/0/0]c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#puqqa90m": "tb1phw4cgpt6cd30kz9k4wkpwm872cdvhss29jga2xpmftelhqll62mscq0k4g", + "wpkh([00000001/84h/1h/0h/0/1]03a20a46308be0b8ded6dff0a22b10b4245c587ccf23f3b4a303885be3a524f172)#aqpjv5xr": "wrong_address", + } if args.desc not in expected_desc: return sys.stdout.write(json.dumps({"error": "Unexpected descriptor", "desc": args.desc})) - return sys.stdout.write(json.dumps({"address": "bcrt1qm90ugl4d48jv8n6e5t9ln6t9zlpm5th68x4f8g"})) + return sys.stdout.write(json.dumps({"address": expected_desc[args.desc]})) def signtx(args): if args.fingerprint != "00000001": diff --git a/test/functional/p2p_addrv2_relay.py b/test/functional/p2p_addrv2_relay.py index f9a8c44be2..ea114e7d70 100755 --- a/test/functional/p2p_addrv2_relay.py +++ b/test/functional/p2p_addrv2_relay.py @@ -11,6 +11,7 @@ import time from test_framework.messages import ( CAddress, msg_addrv2, + msg_sendaddrv2, ) from test_framework.p2p import ( P2PInterface, @@ -75,6 +76,12 @@ class AddrTest(BitcoinTestFramework): self.extra_args = [["-whitelist=addr@127.0.0.1"]] def run_test(self): + self.log.info('Check disconnection when sending sendaddrv2 after verack') + conn = self.nodes[0].add_p2p_connection(P2PInterface()) + with self.nodes[0].assert_debug_log(['sendaddrv2 received after verack from peer=0; disconnecting']): + conn.send_message(msg_sendaddrv2()) + conn.wait_for_disconnect() + self.log.info('Create connection that sends addrv2 messages') addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) msg = msg_addrv2() @@ -89,8 +96,8 @@ class AddrTest(BitcoinTestFramework): msg.addrs = ADDRS msg_size = calc_addrv2_msg_size(ADDRS) with self.nodes[0].assert_debug_log([ - f'received: addrv2 ({msg_size} bytes) peer=0', - f'sending addrv2 ({msg_size} bytes) peer=1', + f'received: addrv2 ({msg_size} bytes) peer=1', + f'sending addrv2 ({msg_size} bytes) peer=2', ]): addr_source.send_and_ping(msg) self.nodes[0].setmocktime(int(time.time()) + 30 * 60) diff --git a/test/functional/p2p_block_sync.py b/test/functional/p2p_block_sync.py index d821edc1b1..6c7f08364e 100755 --- a/test/functional/p2p_block_sync.py +++ b/test/functional/p2p_block_sync.py @@ -22,7 +22,7 @@ class BlockSyncTest(BitcoinTestFramework): # node0 -> node1 -> node2 # So node1 has both an inbound and outbound peer. # In our test, we will mine a block on node0, and ensure that it makes - # to to both node1 and node2. + # to both node1 and node2. self.connect_nodes(0, 1) self.connect_nodes(1, 2) diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index d6c06fdeed..9e314db110 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -139,7 +139,7 @@ class TestP2PConn(P2PInterface): This is used when we want to send a message into the node that we expect will get us disconnected, eg an invalid block.""" self.send_message(message) - self.wait_for_disconnect(timeout) + self.wait_for_disconnect(timeout=timeout) class CompactBlocksTest(BitcoinTestFramework): def set_test_params(self): @@ -387,7 +387,7 @@ class CompactBlocksTest(BitcoinTestFramework): if announce == "inv": test_node.send_message(msg_inv([CInv(MSG_BLOCK, block.sha256)])) - test_node.wait_until(lambda: "getheaders" in test_node.last_message, timeout=30) + test_node.wait_for_getheaders(timeout=30) test_node.send_header_for_blocks([block]) else: test_node.send_header_for_blocks([block]) diff --git a/test/functional/p2p_compactblocks_hb.py b/test/functional/p2p_compactblocks_hb.py index c985a1f98d..023b33ff6d 100755 --- a/test/functional/p2p_compactblocks_hb.py +++ b/test/functional/p2p_compactblocks_hb.py @@ -32,10 +32,15 @@ class CompactBlocksConnectionTest(BitcoinTestFramework): self.connect_nodes(peer, 0) self.generate(self.nodes[0], 1) self.disconnect_nodes(peer, 0) - status_to = [self.peer_info(1, i)['bip152_hb_to'] for i in range(2, 6)] - status_from = [self.peer_info(i, 1)['bip152_hb_from'] for i in range(2, 6)] - assert_equal(status_to, status_from) - return status_to + + def status_to(): + return [self.peer_info(1, i)['bip152_hb_to'] for i in range(2, 6)] + + def status_from(): + return [self.peer_info(i, 1)['bip152_hb_from'] for i in range(2, 6)] + + self.wait_until(lambda: status_to() == status_from()) + return status_to() def run_test(self): self.log.info("Testing reserved high-bandwidth mode slot for outbound peer...") diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index c389ff732f..678b006886 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -77,6 +77,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.nodes[1].setmocktime(old_time) self.nodes[1].setban("127.0.0.0/32", "add") self.nodes[1].setban("127.0.0.0/24", "add") + self.nodes[1].setban("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion", "add") self.nodes[1].setban("192.168.0.1", "add", 1) # ban for 1 seconds self.nodes[1].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) # ban for 1000 seconds listBeforeShutdown = self.nodes[1].listbanned() @@ -85,13 +86,13 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("setban: test banning with absolute timestamp") self.nodes[1].setban("192.168.0.2", "add", old_time + 120, True) - # Move time forward by 3 seconds so the third ban has expired + # Move time forward by 3 seconds so the fourth ban has expired self.nodes[1].setmocktime(old_time + 3) - assert_equal(len(self.nodes[1].listbanned()), 4) + assert_equal(len(self.nodes[1].listbanned()), 5) self.log.info("Test ban_duration and time_remaining") for ban in self.nodes[1].listbanned(): - if ban["address"] in ["127.0.0.0/32", "127.0.0.0/24"]: + if ban["address"] in ["127.0.0.0/32", "127.0.0.0/24", "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion"]: assert_equal(ban["ban_duration"], 86400) assert_equal(ban["time_remaining"], 86397) elif ban["address"] == "2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19": @@ -108,6 +109,7 @@ class DisconnectBanTest(BitcoinTestFramework): assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) assert_equal("192.168.0.2/32", listAfterShutdown[2]['address']) assert_equal("/19" in listAfterShutdown[3]['address'], True) + assert_equal("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion", listAfterShutdown[4]['address']) # Clear ban lists self.nodes[1].clearbanned() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index 6b03cdf877..bcba534f9a 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -46,16 +46,16 @@ class TestP2PConn(P2PInterface): class FeeFilterTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True # We lower the various required feerates for this test # to catch a corner-case where feefilter used to slightly undercut # mempool and wallet feerate calculation based on GetFee # rounding down 3 places, leading to stranded transactions. # See issue #16499 - # grant noban permission to all peers to speed up tx relay / mempool sync self.extra_args = [[ "-minrelaytxfee=0.00000100", - "-mintxfee=0.00000100", - "-whitelist=noban@127.0.0.1", + "-mintxfee=0.00000100" ]] * self.num_nodes def run_test(self): diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 62d55cc101..7c8ed58e51 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -94,9 +94,10 @@ class P2PBloomFilter(P2PInterface): class FilterTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.extra_args = [[ '-peerbloomfilters', - '-whitelist=noban@127.0.0.1', # immediate tx relay ]] def generatetoscriptpubkey(self, scriptpubkey): diff --git a/test/functional/p2p_handshake.py b/test/functional/p2p_handshake.py new file mode 100755 index 0000000000..dd19fe9333 --- /dev/null +++ b/test/functional/p2p_handshake.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test P2P behaviour during the handshake phase (VERSION, VERACK messages). +""" +import itertools +import time + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.messages import ( + NODE_NETWORK, + NODE_NETWORK_LIMITED, + NODE_NONE, + NODE_P2P_V2, + NODE_WITNESS, +) +from test_framework.p2p import P2PInterface + + +# Desirable service flags for outbound non-pruned and pruned peers. Note that +# the desirable service flags for pruned peers are dynamic and only apply if +# 1. the peer's service flag NODE_NETWORK_LIMITED is set *and* +# 2. the local chain is close to the tip (<24h) +DESIRABLE_SERVICE_FLAGS_FULL = NODE_NETWORK | NODE_WITNESS +DESIRABLE_SERVICE_FLAGS_PRUNED = NODE_NETWORK_LIMITED | NODE_WITNESS + + +class P2PHandshakeTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def add_outbound_connection(self, node, connection_type, services, wait_for_disconnect): + peer = node.add_outbound_p2p_connection( + P2PInterface(), p2p_idx=0, wait_for_disconnect=wait_for_disconnect, + connection_type=connection_type, services=services, + supports_v2_p2p=self.options.v2transport, advertise_v2_p2p=self.options.v2transport) + if not wait_for_disconnect: + # check that connection is alive past the version handshake and disconnect manually + peer.sync_with_ping() + peer.peer_disconnect() + peer.wait_for_disconnect() + self.wait_until(lambda: len(node.getpeerinfo()) == 0) + + def test_desirable_service_flags(self, node, service_flag_tests, desirable_service_flags, expect_disconnect): + """Check that connecting to a peer either fails or succeeds depending on its offered + service flags in the VERSION message. The test is exercised for all relevant + outbound connection types where the desirable service flags check is done.""" + CONNECTION_TYPES = ["outbound-full-relay", "block-relay-only", "addr-fetch"] + for conn_type, services in itertools.product(CONNECTION_TYPES, service_flag_tests): + if self.options.v2transport: + services |= NODE_P2P_V2 + expected_result = "disconnect" if expect_disconnect else "connect" + self.log.info(f' - services 0x{services:08x}, type "{conn_type}" [{expected_result}]') + if expect_disconnect: + assert (services & desirable_service_flags) != desirable_service_flags + expected_debug_log = f'does not offer the expected services ' \ + f'({services:08x} offered, {desirable_service_flags:08x} expected)' + with node.assert_debug_log([expected_debug_log]): + self.add_outbound_connection(node, conn_type, services, wait_for_disconnect=True) + else: + assert (services & desirable_service_flags) == desirable_service_flags + self.add_outbound_connection(node, conn_type, services, wait_for_disconnect=False) + + def generate_at_mocktime(self, time): + self.nodes[0].setmocktime(time) + self.generate(self.nodes[0], 1) + self.nodes[0].setmocktime(0) + + def run_test(self): + node = self.nodes[0] + self.log.info("Check that lacking desired service flags leads to disconnect (non-pruned peers)") + self.test_desirable_service_flags(node, [NODE_NONE, NODE_NETWORK, NODE_WITNESS], + DESIRABLE_SERVICE_FLAGS_FULL, expect_disconnect=True) + self.test_desirable_service_flags(node, [NODE_NETWORK | NODE_WITNESS], + DESIRABLE_SERVICE_FLAGS_FULL, expect_disconnect=False) + + self.log.info("Check that limited peers are only desired if the local chain is close to the tip (<24h)") + self.generate_at_mocktime(int(time.time()) - 25 * 3600) # tip outside the 24h window, should fail + self.test_desirable_service_flags(node, [NODE_NETWORK_LIMITED | NODE_WITNESS], + DESIRABLE_SERVICE_FLAGS_FULL, expect_disconnect=True) + self.generate_at_mocktime(int(time.time()) - 23 * 3600) # tip inside the 24h window, should succeed + self.test_desirable_service_flags(node, [NODE_NETWORK_LIMITED | NODE_WITNESS], + DESIRABLE_SERVICE_FLAGS_PRUNED, expect_disconnect=False) + + self.log.info("Check that feeler connections get disconnected immediately") + with node.assert_debug_log([f"feeler connection completed"]): + self.add_outbound_connection(node, "feeler", NODE_NONE, wait_for_disconnect=True) + + +if __name__ == '__main__': + P2PHandshakeTest().main() diff --git a/test/functional/p2p_initial_headers_sync.py b/test/functional/p2p_initial_headers_sync.py index e67c384da7..bc6e0fb355 100755 --- a/test/functional/p2p_initial_headers_sync.py +++ b/test/functional/p2p_initial_headers_sync.py @@ -38,9 +38,10 @@ class HeadersSyncTest(BitcoinTestFramework): def run_test(self): self.log.info("Adding a peer to node0") peer1 = self.nodes[0].add_p2p_connection(P2PInterface()) + best_block_hash = int(self.nodes[0].getbestblockhash(), 16) # Wait for peer1 to receive a getheaders - peer1.wait_for_getheaders() + peer1.wait_for_getheaders(block_hash=best_block_hash) # An empty reply will clear the outstanding getheaders request, # allowing additional getheaders requests to be sent to this peer in # the future. @@ -60,17 +61,12 @@ class HeadersSyncTest(BitcoinTestFramework): assert "getheaders" not in peer2.last_message assert "getheaders" not in peer3.last_message - with p2p_lock: - peer1.last_message.pop("getheaders", None) - self.log.info("Have all peers announce a new block") self.announce_random_block(all_peers) self.log.info("Check that peer1 receives a getheaders in response") - peer1.wait_for_getheaders() + peer1.wait_for_getheaders(block_hash=best_block_hash) peer1.send_message(msg_headers()) # Send empty response, see above - with p2p_lock: - peer1.last_message.pop("getheaders", None) self.log.info("Check that exactly 1 of {peer2, peer3} received a getheaders in response") count = 0 @@ -80,7 +76,6 @@ class HeadersSyncTest(BitcoinTestFramework): if "getheaders" in p.last_message: count += 1 peer_receiving_getheaders = p - p.last_message.pop("getheaders", None) p.send_message(msg_headers()) # Send empty response, see above assert_equal(count, 1) @@ -89,14 +84,14 @@ class HeadersSyncTest(BitcoinTestFramework): self.announce_random_block(all_peers) self.log.info("Check that peer1 receives a getheaders in response") - peer1.wait_for_getheaders() + peer1.wait_for_getheaders(block_hash=best_block_hash) self.log.info("Check that the remaining peer received a getheaders as well") expected_peer = peer2 if peer2 == peer_receiving_getheaders: expected_peer = peer3 - expected_peer.wait_for_getheaders() + expected_peer.wait_for_getheaders(block_hash=best_block_hash) self.log.info("Success!") diff --git a/test/functional/p2p_invalid_block.py b/test/functional/p2p_invalid_block.py index 806fd9c6cb..8ec62ae5ee 100755 --- a/test/functional/p2p_invalid_block.py +++ b/test/functional/p2p_invalid_block.py @@ -32,7 +32,8 @@ class InvalidBlockRequestTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True - self.extra_args = [["-whitelist=noban@127.0.0.1"]] + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True def run_test(self): # Add p2p connection to node0 diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index 89c35e943b..8b63d8ee26 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -15,14 +15,17 @@ from test_framework.messages import ( NODE_P2P_V2, NODE_WITNESS, msg_getdata, - msg_verack, ) from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + assert_raises_rpc_error, + try_rpc ) +# Minimum blocks required to signal NODE_NETWORK_LIMITED # +NODE_NETWORK_LIMITED_MIN_BLOCKS = 288 class P2PIgnoreInv(P2PInterface): firstAddrnServices = 0 @@ -43,7 +46,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 - self.extra_args = [['-prune=550', '-addrmantest'], [], []] + self.extra_args = [['-prune=550'], [], []] def disconnect_all(self): self.disconnect_nodes(0, 1) @@ -54,6 +57,64 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): self.add_nodes(self.num_nodes, self.extra_args) self.start_nodes() + def test_avoid_requesting_historical_blocks(self): + self.log.info("Test full node does not request blocks beyond the limited peer threshold") + pruned_node = self.nodes[0] + miner = self.nodes[1] + full_node = self.nodes[2] + + # Connect and generate block to ensure IBD=false + self.connect_nodes(1, 0) + self.connect_nodes(1, 2) + self.generate(miner, 1) + + # Verify peers are out of IBD + for node in self.nodes: + assert not node.getblockchaininfo()['initialblockdownload'] + + # Isolate full_node (the node will remain out of IBD) + full_node.setnetworkactive(False) + self.wait_until(lambda: len(full_node.getpeerinfo()) == 0) + + # Mine blocks and sync the pruned node. Surpass the NETWORK_NODE_LIMITED threshold. + # Blocks deeper than the threshold are considered "historical blocks" + num_historial_blocks = 12 + self.generate(miner, NODE_NETWORK_LIMITED_MIN_BLOCKS + num_historial_blocks, sync_fun=self.no_op) + self.sync_blocks([miner, pruned_node]) + + # Connect full_node to prune_node and check peers don't disconnect right away. + # (they will disconnect if full_node, which is chain-wise behind, request blocks + # older than NODE_NETWORK_LIMITED_MIN_BLOCKS) + start_height_full_node = full_node.getblockcount() + full_node.setnetworkactive(True) + self.connect_nodes(2, 0) + assert_equal(len(full_node.getpeerinfo()), 1) + + # Wait until the full_node is headers-wise sync + best_block_hash = pruned_node.getbestblockhash() + default_value = {'status': ''} # No status + self.wait_until(lambda: next(filter(lambda x: x['hash'] == best_block_hash, full_node.getchaintips()), default_value)['status'] == "headers-only") + + # Now, since the node aims to download a window of 1024 blocks, + # ensure it requests the blocks below the threshold only (with a + # 2-block buffer). And also, ensure it does not request any + # historical block. + tip_height = pruned_node.getblockcount() + limit_buffer = 2 + # Prevent races by waiting for the tip to arrive first + self.wait_until(lambda: not try_rpc(-1, "Block not found", full_node.getblock, pruned_node.getbestblockhash())) + for height in range(start_height_full_node + 1, tip_height + 1): + if height <= tip_height - (NODE_NETWORK_LIMITED_MIN_BLOCKS - limit_buffer): + assert_raises_rpc_error(-1, "Block not found on disk", full_node.getblock, pruned_node.getblockhash(height)) + else: + full_node.getblock(pruned_node.getblockhash(height)) # just assert it does not throw an exception + + # Lastly, ensure the full_node is not sync and verify it can get synced by + # establishing a connection with another full node capable of providing them. + assert_equal(full_node.getblockcount(), start_height_full_node) + self.connect_nodes(2, 1) + self.sync_blocks([miner, full_node]) + def run_test(self): node = self.nodes[0].add_p2p_connection(P2PIgnoreInv()) @@ -77,17 +138,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): self.log.info("Requesting block at height 2 (tip-289) must fail (ignored).") node.send_getdata_for_block(blocks[0]) # first block outside of the 288+2 limit - node.wait_for_disconnect(5) - - self.log.info("Check local address relay, do a fresh connection.") - self.nodes[0].disconnect_p2ps() - node1 = self.nodes[0].add_p2p_connection(P2PIgnoreInv()) - node1.send_message(msg_verack()) - - node1.wait_for_addr() - #must relay address with NODE_NETWORK_LIMITED - assert_equal(node1.firstAddrnServices, expected_services) - + node.wait_for_disconnect(timeout=5) self.nodes[0].disconnect_p2ps() # connect unsynced node 2 with pruned NODE_NETWORK_LIMITED peer @@ -118,5 +169,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): # sync must be possible, node 1 is no longer in IBD and should therefore connect to node 0 (NODE_NETWORK_LIMITED) self.sync_blocks([self.nodes[0], self.nodes[1]]) + self.test_avoid_requesting_historical_blocks() + if __name__ == '__main__': NodeNetworkLimitedTest().main() diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index 6153e4a156..80a27943fd 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -83,7 +83,14 @@ class P2PPermissionsTests(BitcoinTestFramework): ["-whitelist=all@127.0.0.1"], ["forcerelay", "noban", "mempool", "bloomfilter", "relay", "download", "addr"]) + for flag, permissions in [(["-whitelist=noban,out@127.0.0.1"], ["noban", "download"]), (["-whitelist=noban@127.0.0.1"], [])]: + self.restart_node(0, flag) + self.connect_nodes(0, 1) + peerinfo = self.nodes[0].getpeerinfo()[0] + assert_equal(peerinfo['permissions'], permissions) + self.stop_node(1) + self.nodes[1].assert_start_raises_init_error(["-whitelist=in,out@127.0.0.1"], "Only direction was set, no permissions", match=ErrorMatch.PARTIAL_REGEX) self.nodes[1].assert_start_raises_init_error(["-whitelist=oopsie@127.0.0.1"], "Invalid P2P permission", match=ErrorMatch.PARTIAL_REGEX) self.nodes[1].assert_start_raises_init_error(["-whitelist=noban@127.0.0.1:230"], "Invalid netmask specified in", match=ErrorMatch.PARTIAL_REGEX) self.nodes[1].assert_start_raises_init_error(["-whitebind=noban@127.0.0.1/10"], "Cannot resolve -whitebind address", match=ErrorMatch.PARTIAL_REGEX) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index d316c4b602..213c748eda 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -191,31 +191,32 @@ class TestP2PConn(P2PInterface): def announce_block_and_wait_for_getdata(self, block, use_header, timeout=60): with p2p_lock: self.last_message.pop("getdata", None) - self.last_message.pop("getheaders", None) msg = msg_headers() msg.headers = [CBlockHeader(block)] if use_header: self.send_message(msg) else: self.send_message(msg_inv(inv=[CInv(MSG_BLOCK, block.sha256)])) - self.wait_for_getheaders() + self.wait_for_getheaders(block_hash=block.hashPrevBlock, timeout=timeout) self.send_message(msg) - self.wait_for_getdata([block.sha256]) + self.wait_for_getdata([block.sha256], timeout=timeout) def request_block(self, blockhash, inv_type, timeout=60): with p2p_lock: self.last_message.pop("block", None) self.send_message(msg_getdata(inv=[CInv(inv_type, blockhash)])) - self.wait_for_block(blockhash, timeout) + self.wait_for_block(blockhash, timeout=timeout) return self.last_message["block"].block class SegWitTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. self.extra_args = [ - ["-acceptnonstdtxn=1", f"-testactivationheight=segwit@{SEGWIT_HEIGHT}", "-whitelist=noban@127.0.0.1", "-par=1"], + ["-acceptnonstdtxn=1", f"-testactivationheight=segwit@{SEGWIT_HEIGHT}", "-par=1"], ["-acceptnonstdtxn=0", f"-testactivationheight=segwit@{SEGWIT_HEIGHT}"], ] self.supports_cli = False @@ -2054,7 +2055,7 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[0], self.wtx_node, tx2, with_witness=True, accepted=False) # Expect a request for parent (tx) by txid despite use of WTX peer - self.wtx_node.wait_for_getdata([tx.sha256], 60) + self.wtx_node.wait_for_getdata([tx.sha256], timeout=60) with p2p_lock: lgd = self.wtx_node.lastgetdata[:] assert_equal(lgd, [CInv(MSG_WITNESS_TX, tx.sha256)]) diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 508d6fe403..27a3aa8fb9 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -311,6 +311,7 @@ class SendHeadersTest(BitcoinTestFramework): # Now that we've synced headers, headers announcements should work tip = self.mine_blocks(1) + expected_hash = tip inv_node.check_last_inv_announcement(inv=[tip]) test_node.check_last_headers_announcement(headers=[tip]) @@ -334,7 +335,10 @@ class SendHeadersTest(BitcoinTestFramework): if j == 0: # Announce via inv test_node.send_block_inv(tip) - test_node.wait_for_getheaders() + if i == 0: + test_node.wait_for_getheaders(block_hash=expected_hash) + else: + assert "getheaders" not in test_node.last_message # Should have received a getheaders now test_node.send_header_for_blocks(blocks) # Test that duplicate inv's won't result in duplicate @@ -521,6 +525,7 @@ class SendHeadersTest(BitcoinTestFramework): self.log.info("Part 5: Testing handling of unconnecting headers") # First we test that receipt of an unconnecting header doesn't prevent # chain sync. + expected_hash = tip for i in range(10): self.log.debug("Part 5.{}: starting...".format(i)) test_node.last_message.pop("getdata", None) @@ -533,15 +538,14 @@ class SendHeadersTest(BitcoinTestFramework): block_time += 1 height += 1 # Send the header of the second block -> this won't connect. - with p2p_lock: - test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[1]]) - test_node.wait_for_getheaders() + test_node.wait_for_getheaders(block_hash=expected_hash) test_node.send_header_for_blocks(blocks) test_node.wait_for_getdata([x.sha256 for x in blocks]) [test_node.send_message(msg_block(x)) for x in blocks] test_node.sync_with_ping() assert_equal(int(self.nodes[0].getbestblockhash(), 16), blocks[1].sha256) + expected_hash = blocks[1].sha256 blocks = [] # Now we test that if we repeatedly don't send connecting headers, we @@ -556,13 +560,12 @@ class SendHeadersTest(BitcoinTestFramework): for i in range(1, MAX_NUM_UNCONNECTING_HEADERS_MSGS): # Send a header that doesn't connect, check that we get a getheaders. - with p2p_lock: - test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i]]) - test_node.wait_for_getheaders() + test_node.wait_for_getheaders(block_hash=expected_hash) # Next header will connect, should re-set our count: test_node.send_header_for_blocks([blocks[0]]) + expected_hash = blocks[0].sha256 # Remove the first two entries (blocks[1] would connect): blocks = blocks[2:] @@ -571,10 +574,8 @@ class SendHeadersTest(BitcoinTestFramework): # before we get disconnected. Should be 5*MAX_NUM_UNCONNECTING_HEADERS_MSGS for i in range(5 * MAX_NUM_UNCONNECTING_HEADERS_MSGS - 1): # Send a header that doesn't connect, check that we get a getheaders. - with p2p_lock: - test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i % len(blocks)]]) - test_node.wait_for_getheaders() + test_node.wait_for_getheaders(block_hash=expected_hash) # Eventually this stops working. test_node.send_header_for_blocks([blocks[-1]]) diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 0e463c5072..7a50f1e605 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -5,6 +5,7 @@ """ Test transaction download behavior """ +from decimal import Decimal import time from test_framework.messages import ( @@ -14,6 +15,7 @@ from test_framework.messages import ( MSG_WTX, msg_inv, msg_notfound, + msg_tx, ) from test_framework.p2p import ( P2PInterface, @@ -22,6 +24,7 @@ from test_framework.p2p import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + fill_mempool, ) from test_framework.wallet import MiniWallet @@ -54,6 +57,7 @@ MAX_GETDATA_INBOUND_WAIT = GETDATA_TX_INTERVAL + INBOUND_PEER_TX_DELAY + TXID_RE class TxDownloadTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + self.extra_args= [['-datacarriersize=100000', '-maxmempool=5', '-persistmempool=0']] * self.num_nodes def test_tx_requests(self): self.log.info("Test that we request transactions from all our peers, eventually") @@ -241,6 +245,29 @@ class TxDownloadTest(BitcoinTestFramework): self.log.info('Check that spurious notfound is ignored') self.nodes[0].p2ps[0].send_message(msg_notfound(vec=[CInv(MSG_TX, 1)])) + def test_rejects_filter_reset(self): + self.log.info('Check that rejected tx is not requested again') + node = self.nodes[0] + fill_mempool(self, node, self.wallet) + self.wallet.rescan_utxos() + mempoolminfee = node.getmempoolinfo()['mempoolminfee'] + peer = node.add_p2p_connection(TestP2PConn()) + low_fee_tx = self.wallet.create_self_transfer(fee_rate=Decimal("0.9")*mempoolminfee) + assert_equal(node.testmempoolaccept([low_fee_tx['hex']])[0]["reject-reason"], "mempool min fee not met") + peer.send_and_ping(msg_tx(low_fee_tx['tx'])) + peer.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=int(low_fee_tx['wtxid'], 16))])) + node.setmocktime(int(time.time())) + node.bumpmocktime(MAX_GETDATA_INBOUND_WAIT) + peer.sync_with_ping() + assert_equal(peer.tx_getdata_count, 0) + + self.log.info('Check that rejection filter is cleared after new block comes in') + self.generate(self.wallet, 1, sync_fun=self.no_op) + peer.sync_with_ping() + peer.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=int(low_fee_tx['wtxid'], 16))])) + node.bumpmocktime(MAX_GETDATA_INBOUND_WAIT) + peer.wait_for_getdata([int(low_fee_tx['wtxid'], 16)]) + def run_test(self): self.wallet = MiniWallet(self.nodes[0]) @@ -257,16 +284,22 @@ class TxDownloadTest(BitcoinTestFramework): # Run each test against new bitcoind instances, as setting mocktimes has long-term effects on when # the next trickle relay event happens. - for test in [self.test_in_flight_max, self.test_inv_block, self.test_tx_requests]: + for test, with_inbounds in [ + (self.test_in_flight_max, True), + (self.test_inv_block, True), + (self.test_tx_requests, True), + (self.test_rejects_filter_reset, False), + ]: self.stop_nodes() self.start_nodes() self.connect_nodes(1, 0) # Setup the p2p connections self.peers = [] - for node in self.nodes: - for _ in range(NUM_INBOUND): - self.peers.append(node.add_p2p_connection(TestP2PConn())) - self.log.info("Nodes are setup with {} incoming connections each".format(NUM_INBOUND)) + if with_inbounds: + for node in self.nodes: + for _ in range(NUM_INBOUND): + self.peers.append(node.add_p2p_connection(TestP2PConn())) + self.log.info("Nodes are setup with {} incoming connections each".format(NUM_INBOUND)) test() diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index accb2439f2..2701d2471d 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -13,7 +13,6 @@ import platform import time import test_framework.messages -from test_framework.netutil import ADDRMAN_NEW_BUCKET_COUNT, ADDRMAN_TRIED_BUCKET_COUNT, ADDRMAN_BUCKET_SIZE from test_framework.p2p import ( P2PInterface, P2P_SERVICES, @@ -42,6 +41,24 @@ def assert_net_servicesnames(servicesflag, servicenames): assert servicesflag_generated == servicesflag +def seed_addrman(node): + """ Populate the addrman with addresses from different networks. + Here 2 ipv4, 2 ipv6, 1 cjdns, 2 onion and 1 i2p addresses are added. + """ + # These addresses currently don't collide with a deterministic addrman. + # If the addrman positioning/bucketing is changed, these might collide + # and adding them fails. + success = { "success": True } + assert_equal(node.addpeeraddress(address="1.2.3.4", tried=True, port=8333), success) + assert_equal(node.addpeeraddress(address="2.0.0.0", port=8333), success) + assert_equal(node.addpeeraddress(address="1233:3432:2434:2343:3234:2345:6546:4534", tried=True, port=8333), success) + assert_equal(node.addpeeraddress(address="2803:0:1234:abcd::1", port=45324), success) + assert_equal(node.addpeeraddress(address="fc00:1:2:3:4:5:6:7", port=8333), success) + assert_equal(node.addpeeraddress(address="pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion", tried=True, port=8333), success) + assert_equal(node.addpeeraddress(address="nrfj6inpyf73gpkyool35hcmne5zwfmse3jl3aw23vk7chdemalyaqad.onion", port=45324, tried=True), success) + assert_equal(node.addpeeraddress(address="c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p", port=8333), success) + + class NetTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 @@ -305,22 +322,16 @@ class NetTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "Network not recognized: Foo", self.nodes[0].getnodeaddresses, 1, "Foo") def test_addpeeraddress(self): - """RPC addpeeraddress sets the source address equal to the destination address. - If an address with the same /16 as an existing new entry is passed, it will be - placed in the same new bucket and have a 1/64 chance of the bucket positions - colliding (depending on the value of nKey in the addrman), in which case the - new address won't be added. The probability of collision can be reduced to - 1/2^16 = 1/65536 by using an address from a different /16. We avoid this here - by first testing adding a tried table entry before testing adding a new table one. - """ self.log.info("Test addpeeraddress") - self.restart_node(1, ["-checkaddrman=1"]) + # The node has an existing, non-deterministic addrman from a previous test. + # Clear it to have a deterministic addrman. + self.restart_node(1, ["-checkaddrman=1", "-test=addrman"], clear_addrman=True) node = self.nodes[1] - self.log.debug("Test that addpeerinfo is a hidden RPC") + self.log.debug("Test that addpeeraddress is a hidden RPC") # It is hidden from general help, but its detailed help may be called directly. - assert "addpeerinfo" not in node.help() - assert "addpeerinfo" in node.help("addpeerinfo") + assert "addpeeraddress" not in node.help() + assert "unknown command: addpeeraddress" not in node.help("addpeeraddress") self.log.debug("Test that adding an empty address fails") assert_equal(node.addpeeraddress(address="", port=8333), {"success": False}) @@ -333,26 +344,50 @@ class NetTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "JSON integer out of range", self.nodes[0].addpeeraddress, address="1.2.3.4", port=-1) assert_raises_rpc_error(-1, "JSON integer out of range", self.nodes[0].addpeeraddress, address="1.2.3.4", port=65536) + self.log.debug("Test that adding a valid address to the new table succeeds") + assert_equal(node.addpeeraddress(address="1.0.0.0", tried=False, port=8333), {"success": True}) + addrman = node.getrawaddrman() + assert_equal(len(addrman["tried"]), 0) + new_table = list(addrman["new"].values()) + assert_equal(len(new_table), 1) + assert_equal(new_table[0]["address"], "1.0.0.0") + assert_equal(new_table[0]["port"], 8333) + + self.log.debug("Test that adding an already-present new address to the new and tried tables fails") + for value in [True, False]: + assert_equal(node.addpeeraddress(address="1.0.0.0", tried=value, port=8333), {"success": False, "error": "failed-adding-to-new"}) + assert_equal(len(node.getnodeaddresses(count=0)), 1) + self.log.debug("Test that adding a valid address to the tried table succeeds") - self.addr_time = int(time.time()) - node.setmocktime(self.addr_time) assert_equal(node.addpeeraddress(address="1.2.3.4", tried=True, port=8333), {"success": True}) - with node.assert_debug_log(expected_msgs=["CheckAddrman: new 0, tried 1, total 1 started"]): - addrs = node.getnodeaddresses(count=0) # getnodeaddresses re-runs the addrman checks - assert_equal(len(addrs), 1) - assert_equal(addrs[0]["address"], "1.2.3.4") - assert_equal(addrs[0]["port"], 8333) + addrman = node.getrawaddrman() + assert_equal(len(addrman["new"]), 1) + tried_table = list(addrman["tried"].values()) + assert_equal(len(tried_table), 1) + assert_equal(tried_table[0]["address"], "1.2.3.4") + assert_equal(tried_table[0]["port"], 8333) + node.getnodeaddresses(count=0) # getnodeaddresses re-runs the addrman checks self.log.debug("Test that adding an already-present tried address to the new and tried tables fails") for value in [True, False]: - assert_equal(node.addpeeraddress(address="1.2.3.4", tried=value, port=8333), {"success": False}) - assert_equal(len(node.getnodeaddresses(count=0)), 1) - - self.log.debug("Test that adding a second address, this time to the new table, succeeds") + assert_equal(node.addpeeraddress(address="1.2.3.4", tried=value, port=8333), {"success": False, "error": "failed-adding-to-new"}) + assert_equal(len(node.getnodeaddresses(count=0)), 2) + + self.log.debug("Test that adding an address, which collides with the address in tried table, fails") + colliding_address = "1.2.5.45" # grinded address that produces a tried-table collision + assert_equal(node.addpeeraddress(address=colliding_address, tried=True, port=8333), {"success": False, "error": "failed-adding-to-tried"}) + # When adding an address to the tried table, it's first added to the new table. + # As we fail to move it to the tried table, it remains in the new table. + addrman_info = node.getaddrmaninfo() + assert_equal(addrman_info["all_networks"]["tried"], 1) + assert_equal(addrman_info["all_networks"]["new"], 2) + + self.log.debug("Test that adding an another address to the new table succeeds") assert_equal(node.addpeeraddress(address="2.0.0.0", port=8333), {"success": True}) - with node.assert_debug_log(expected_msgs=["CheckAddrman: new 1, tried 1, total 2 started"]): - addrs = node.getnodeaddresses(count=0) # getnodeaddresses re-runs the addrman checks - assert_equal(len(addrs), 2) + addrman_info = node.getaddrmaninfo() + assert_equal(addrman_info["all_networks"]["tried"], 1) + assert_equal(addrman_info["all_networks"]["new"], 3) + node.getnodeaddresses(count=0) # getnodeaddresses re-runs the addrman checks def test_sendmsgtopeer(self): node = self.nodes[0] @@ -390,30 +425,38 @@ class NetTest(BitcoinTestFramework): def test_getaddrmaninfo(self): self.log.info("Test getaddrmaninfo") + self.restart_node(1, extra_args=["-cjdnsreachable", "-test=addrman"], clear_addrman=True) node = self.nodes[1] + seed_addrman(node) + + expected_network_count = { + 'all_networks': {'new': 4, 'tried': 4, 'total': 8}, + 'ipv4': {'new': 1, 'tried': 1, 'total': 2}, + 'ipv6': {'new': 1, 'tried': 1, 'total': 2}, + 'onion': {'new': 0, 'tried': 2, 'total': 2}, + 'i2p': {'new': 1, 'tried': 0, 'total': 1}, + 'cjdns': {'new': 1, 'tried': 0, 'total': 1}, + } - # current count of ipv4 addresses in addrman is {'new':1, 'tried':1} - self.log.info("Test that count of addresses in addrman match expected values") + self.log.debug("Test that count of addresses in addrman match expected values") res = node.getaddrmaninfo() - assert_equal(res["ipv4"]["new"], 1) - assert_equal(res["ipv4"]["tried"], 1) - assert_equal(res["ipv4"]["total"], 2) - assert_equal(res["all_networks"]["new"], 1) - assert_equal(res["all_networks"]["tried"], 1) - assert_equal(res["all_networks"]["total"], 2) - for net in ["ipv6", "onion", "i2p", "cjdns"]: - assert_equal(res[net]["new"], 0) - assert_equal(res[net]["tried"], 0) - assert_equal(res[net]["total"], 0) + for network, count in expected_network_count.items(): + assert_equal(res[network]['new'], count['new']) + assert_equal(res[network]['tried'], count['tried']) + assert_equal(res[network]['total'], count['total']) def test_getrawaddrman(self): self.log.info("Test getrawaddrman") + self.restart_node(1, extra_args=["-cjdnsreachable", "-test=addrman"], clear_addrman=True) node = self.nodes[1] + self.addr_time = int(time.time()) + node.setmocktime(self.addr_time) + seed_addrman(node) self.log.debug("Test that getrawaddrman is a hidden RPC") # It is hidden from general help, but its detailed help may be called directly. assert "getrawaddrman" not in node.help() - assert "getrawaddrman" in node.help("getrawaddrman") + assert "unknown command: getrawaddrman" not in node.help("getrawaddrman") def check_addr_information(result, expected): """Utility to compare a getrawaddrman result entry with an expected entry""" @@ -430,88 +473,96 @@ class NetTest(BitcoinTestFramework): getrawaddrman = node.getrawaddrman() getaddrmaninfo = node.getaddrmaninfo() for (table_name, table_info) in expected.items(): - assert_equal(len(getrawaddrman[table_name]), len(table_info["entries"])) + assert_equal(len(getrawaddrman[table_name]), len(table_info)) assert_equal(len(getrawaddrman[table_name]), getaddrmaninfo["all_networks"][table_name]) for bucket_position in getrawaddrman[table_name].keys(): - bucket = int(bucket_position.split("/")[0]) - position = int(bucket_position.split("/")[1]) - - # bucket and position only be sanity checked here as the - # test-addrman isn't deterministic - assert 0 <= int(bucket) < table_info["bucket_count"] - assert 0 <= int(position) < ADDRMAN_BUCKET_SIZE - entry = getrawaddrman[table_name][bucket_position] - expected_entry = list(filter(lambda e: e["address"] == entry["address"], table_info["entries"]))[0] + expected_entry = list(filter(lambda e: e["address"] == entry["address"], table_info))[0] + assert bucket_position == expected_entry["bucket_position"] check_addr_information(entry, expected_entry) - # we expect one addrman new and tried table entry, which were added in a previous test + # we expect 4 new and 4 tried table entries in the addrman which were added using seed_addrman() expected = { - "new": { - "bucket_count": ADDRMAN_NEW_BUCKET_COUNT, - "entries": [ + "new": [ { + "bucket_position": "82/8", "address": "2.0.0.0", "port": 8333, "services": 9, "network": "ipv4", "source": "2.0.0.0", "source_network": "ipv4", + }, + { + "bucket_position": "336/24", + "address": "fc00:1:2:3:4:5:6:7", + "port": 8333, + "services": 9, + "network": "cjdns", + "source": "fc00:1:2:3:4:5:6:7", + "source_network": "cjdns", + }, + { + "bucket_position": "963/46", + "address": "c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p", + "port": 8333, + "services": 9, + "network": "i2p", + "source": "c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p", + "source_network": "i2p", + }, + { + "bucket_position": "613/6", + "address": "2803:0:1234:abcd::1", + "services": 9, + "network": "ipv6", + "source": "2803:0:1234:abcd::1", + "source_network": "ipv6", + "port": 45324, } - ] - }, - "tried": { - "bucket_count": ADDRMAN_TRIED_BUCKET_COUNT, - "entries": [ + ], + "tried": [ { + "bucket_position": "6/33", "address": "1.2.3.4", "port": 8333, "services": 9, "network": "ipv4", "source": "1.2.3.4", "source_network": "ipv4", + }, + { + "bucket_position": "197/34", + "address": "1233:3432:2434:2343:3234:2345:6546:4534", + "port": 8333, + "services": 9, + "network": "ipv6", + "source": "1233:3432:2434:2343:3234:2345:6546:4534", + "source_network": "ipv6", + }, + { + "bucket_position": "72/61", + "address": "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion", + "port": 8333, + "services": 9, + "network": "onion", + "source": "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion", + "source_network": "onion" + }, + { + "bucket_position": "139/46", + "address": "nrfj6inpyf73gpkyool35hcmne5zwfmse3jl3aw23vk7chdemalyaqad.onion", + "services": 9, + "network": "onion", + "source": "nrfj6inpyf73gpkyool35hcmne5zwfmse3jl3aw23vk7chdemalyaqad.onion", + "source_network": "onion", + "port": 45324, } - ] - } + ] } - self.log.debug("Test that the getrawaddrman contains information about the addresses added in a previous test") - check_getrawaddrman_entries(expected) - - self.log.debug("Add one new address to each addrman table") - expected["new"]["entries"].append({ - "address": "2803:0:1234:abcd::1", - "services": 9, - "network": "ipv6", - "source": "2803:0:1234:abcd::1", - "source_network": "ipv6", - "port": -1, # set once addpeeraddress is successful - }) - expected["tried"]["entries"].append({ - "address": "nrfj6inpyf73gpkyool35hcmne5zwfmse3jl3aw23vk7chdemalyaqad.onion", - "services": 9, - "network": "onion", - "source": "nrfj6inpyf73gpkyool35hcmne5zwfmse3jl3aw23vk7chdemalyaqad.onion", - "source_network": "onion", - "port": -1, # set once addpeeraddress is successful - }) - - port = 0 - for (table_name, table_info) in expected.items(): - # There's a slight chance that the to-be-added address collides with an already - # present table entry. To avoid this, we increment the port until an address has been - # added. Incrementing the port changes the position in the new table bucket (bucket - # stays the same) and changes both the bucket and the position in the tried table. - while True: - if node.addpeeraddress(address=table_info["entries"][1]["address"], port=port, tried=table_name == "tried")["success"]: - table_info["entries"][1]["port"] = port - self.log.debug(f"Added {table_info['entries'][1]['address']} to {table_name} table") - break - else: - port += 1 - - self.log.debug("Test that the newly added addresses appear in getrawaddrman") + self.log.debug("Test that getrawaddrman contains information about newly added addresses in each addrman table") check_getrawaddrman_entries(expected) diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index 664f2df3f1..37c42f2533 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -18,6 +18,7 @@ from test_framework.util import ( assert_equal, assert_fee_amount, assert_raises_rpc_error, + fill_mempool, ) from test_framework.wallet import ( DEFAULT_FEE, @@ -29,7 +30,8 @@ class RPCPackagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True - self.extra_args = [["-whitelist=noban@127.0.0.1"]] # noban speeds up tx relay + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True def assert_testres_equal(self, package_hex, testres_expected): """Shuffle package_hex and assert that the testmempoolaccept result matches testres_expected. This should only @@ -81,6 +83,8 @@ class RPCPackagesTest(BitcoinTestFramework): self.test_conflicting() self.test_rbf() self.test_submitpackage() + self.test_maxfeerate_submitpackage() + self.test_maxburn_submitpackage() def test_independent(self, coin): self.log.info("Test multiple independent transactions in a package") @@ -356,5 +360,89 @@ class RPCPackagesTest(BitcoinTestFramework): assert_equal(res["tx-results"][sec_wtxid]["error"], "version") peer.wait_for_broadcast([first_wtxid]) + def test_maxfeerate_submitpackage(self): + node = self.nodes[0] + # clear mempool + deterministic_address = node.get_deterministic_priv_key().address + self.generatetoaddress(node, 1, deterministic_address) + + self.log.info("Submitpackage maxfeerate arg testing") + chained_txns = self.wallet.create_self_transfer_chain(chain_length=2) + minrate_btc_kvb = min([chained_txn["fee"] / chained_txn["tx"].get_vsize() * 1000 for chained_txn in chained_txns]) + chain_hex = [t["hex"] for t in chained_txns] + pkg_result = node.submitpackage(chain_hex, maxfeerate=minrate_btc_kvb - Decimal("0.00000001")) + + # First tx failed in single transaction evaluation, so package message is generic + assert_equal(pkg_result["package_msg"], "transaction failed") + assert_equal(pkg_result["tx-results"][chained_txns[0]["wtxid"]]["error"], "max feerate exceeded") + assert_equal(pkg_result["tx-results"][chained_txns[1]["wtxid"]]["error"], "bad-txns-inputs-missingorspent") + assert_equal(node.getrawmempool(), []) + + # Make chain of two transactions where parent doesn't make minfee threshold + # but child is too high fee + # Lower mempool limit to make it easier to fill_mempool + self.restart_node(0, extra_args=[ + "-datacarriersize=100000", + "-maxmempool=5", + "-persistmempool=0", + ]) + self.wallet.rescan_utxos() + + fill_mempool(self, node, self.wallet) + + minrelay = node.getmempoolinfo()["minrelaytxfee"] + parent = self.wallet.create_self_transfer( + fee_rate=minrelay, + confirmed_only=True, + ) + + child = self.wallet.create_self_transfer( + fee_rate=DEFAULT_FEE, + utxo_to_spend=parent["new_utxo"], + ) + + pkg_result = node.submitpackage([parent["hex"], child["hex"]], maxfeerate=DEFAULT_FEE - Decimal("0.00000001")) + + # Child is connected even though parent is invalid and still reports fee exceeded + # this implies sub-package evaluation of both entries together. + assert_equal(pkg_result["package_msg"], "transaction failed") + assert "mempool min fee not met" in pkg_result["tx-results"][parent["wtxid"]]["error"] + assert_equal(pkg_result["tx-results"][child["wtxid"]]["error"], "max feerate exceeded") + assert parent["txid"] not in node.getrawmempool() + assert child["txid"] not in node.getrawmempool() + + # Reset maxmempool, datacarriersize, reset dynamic mempool minimum feerate, and empty mempool. + self.restart_node(0) + self.wallet.rescan_utxos() + + assert_equal(node.getrawmempool(), []) + + def test_maxburn_submitpackage(self): + node = self.nodes[0] + + assert_equal(node.getrawmempool(), []) + + self.log.info("Submitpackage maxburnamount arg testing") + chained_txns_burn = self.wallet.create_self_transfer_chain( + chain_length=2, + utxo_to_spend=self.wallet.get_utxo(confirmed_only=True), + ) + chained_burn_hex = [t["hex"] for t in chained_txns_burn] + + tx = tx_from_hex(chained_burn_hex[1]) + tx.vout[-1].scriptPubKey = b'a' * 10001 # scriptPubKey bigger than 10k IsUnspendable + chained_burn_hex = [chained_burn_hex[0], tx.serialize().hex()] + # burn test is run before any package evaluation; nothing makes it in and we get broader exception + assert_raises_rpc_error(-25, "Unspendable output exceeds maximum configured by user", node.submitpackage, chained_burn_hex, 0, chained_txns_burn[1]["new_utxo"]["value"] - Decimal("0.00000001")) + assert_equal(node.getrawmempool(), []) + + minrate_btc_kvb_burn = min([chained_txn_burn["fee"] / chained_txn_burn["tx"].get_vsize() * 1000 for chained_txn_burn in chained_txns_burn]) + + # Relax the restrictions for both and send it; parent gets through as own subpackage + pkg_result = node.submitpackage(chained_burn_hex, maxfeerate=minrate_btc_kvb_burn, maxburnamount=chained_txns_burn[1]["new_utxo"]["value"]) + assert "error" not in pkg_result["tx-results"][chained_txns_burn[0]["wtxid"]] + assert_equal(pkg_result["tx-results"][tx.getwtxid()]["error"], "scriptpubkey") + assert_equal(node.getrawmempool(), [chained_txns_burn[0]["txid"]]) + if __name__ == "__main__": RPCPackagesTest().main() diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 016aa3ba11..6ee7e56886 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -16,8 +16,6 @@ from test_framework.messages import ( CTxIn, CTxOut, MAX_BIP125_RBF_SEQUENCE, - WITNESS_SCALE_FACTOR, - ser_compact_size, ) from test_framework.psbt import ( PSBT, @@ -42,6 +40,7 @@ from test_framework.util import ( find_vout_for_address, ) from test_framework.wallet_util import ( + calculate_input_weight, generate_keypair, get_generate_key, ) @@ -752,17 +751,9 @@ class PSBTTest(BitcoinTestFramework): input_idx = i break psbt_in = dec["inputs"][input_idx] - # Calculate the input weight - # (prevout + sequence + length of scriptSig + scriptsig) * WITNESS_SCALE_FACTOR + len of num scriptWitness stack items + (length of stack item + stack item) * N stack items - # Note that occasionally this weight estimate may be slightly larger or smaller than the real weight - # as sometimes ECDSA signatures are one byte shorter than expected with a probability of 1/128 - len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptsig += len(ser_compact_size(len_scriptsig)) - len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(ser_compact_size(len(psbt_in["final_scriptwitness"])))) if "final_scriptwitness" in psbt_in else 0 - len_prevout_txid = 32 - len_prevout_index = 4 - len_sequence = 4 - input_weight = ((len_prevout_txid + len_prevout_index + len_sequence + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness + scriptsig_hex = psbt_in["final_scriptSig"]["hex"] if "final_scriptSig" in psbt_in else "" + witness_stack_hex = psbt_in["final_scriptwitness"] if "final_scriptwitness" in psbt_in else None + input_weight = calculate_input_weight(scriptsig_hex, witness_stack_hex) low_input_weight = input_weight // 2 high_input_weight = input_weight * 2 @@ -886,7 +877,7 @@ class PSBTTest(BitcoinTestFramework): assert_equal(comb_psbt, psbt) self.log.info("Test walletprocesspsbt raises if an invalid sighashtype is passed") - assert_raises_rpc_error(-8, "all is not a valid sighash parameter.", self.nodes[0].walletprocesspsbt, psbt, sighashtype="all") + assert_raises_rpc_error(-8, "'all' is not a valid sighash parameter.", self.nodes[0].walletprocesspsbt, psbt, sighashtype="all") self.log.info("Test decoding PSBT with per-input preimage types") # note that the decodepsbt RPC doesn't check whether preimages and hashes match @@ -992,7 +983,7 @@ class PSBTTest(BitcoinTestFramework): self.nodes[2].sendrawtransaction(processed_psbt['hex']) self.log.info("Test descriptorprocesspsbt raises if an invalid sighashtype is passed") - assert_raises_rpc_error(-8, "all is not a valid sighash parameter.", self.nodes[2].descriptorprocesspsbt, psbt, [descriptor], sighashtype="all") + assert_raises_rpc_error(-8, "'all' is not a valid sighash parameter.", self.nodes[2].descriptorprocesspsbt, psbt, [descriptor], sighashtype="all") if __name__ == '__main__': diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index c12865b5e3..12697e9d0c 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -73,9 +73,8 @@ class RawTransactionsTest(BitcoinTestFramework): ["-txindex"], ["-fastprune", "-prune=1"], ] - # whitelist all peers to speed up tx relay / mempool sync - for args in self.extra_args: - args.append("-whitelist=noban@127.0.0.1") + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.supports_cli = False def setup_network(self): diff --git a/test/functional/rpc_setban.py b/test/functional/rpc_setban.py index bc426d7371..ba86b278bd 100755 --- a/test/functional/rpc_setban.py +++ b/test/functional/rpc_setban.py @@ -64,20 +64,10 @@ class SetBanTests(BitcoinTestFramework): assert self.is_banned(node, tor_addr) assert not self.is_banned(node, ip_addr) - self.log.info("Test the ban list is preserved through restart") - - self.restart_node(1) - assert self.is_banned(node, tor_addr) - assert not self.is_banned(node, ip_addr) - node.setban(tor_addr, "remove") assert not self.is_banned(self.nodes[1], tor_addr) assert not self.is_banned(node, ip_addr) - self.restart_node(1) - assert not self.is_banned(node, tor_addr) - assert not self.is_banned(node, ip_addr) - self.log.info("Test -bantime") self.restart_node(1, ["-bantime=1234"]) self.nodes[1].setban("127.0.0.1", "add") diff --git a/test/functional/rpc_signrawtransactionwithkey.py b/test/functional/rpc_signrawtransactionwithkey.py index 0913f5057e..268584331e 100755 --- a/test/functional/rpc_signrawtransactionwithkey.py +++ b/test/functional/rpc_signrawtransactionwithkey.py @@ -124,7 +124,7 @@ class SignRawTransactionWithKeyTest(BitcoinTestFramework): self.log.info("Test signing transaction with invalid sighashtype") tx = self.nodes[0].createrawtransaction(INPUTS, OUTPUTS) privkeys = [self.nodes[0].get_deterministic_priv_key().key] - assert_raises_rpc_error(-8, "all is not a valid sighash parameter.", self.nodes[0].signrawtransactionwithkey, tx, privkeys, sighashtype="all") + assert_raises_rpc_error(-8, "'all' is not a valid sighash parameter.", self.nodes[0].signrawtransactionwithkey, tx, privkeys, sighashtype="all") def run_test(self): self.successful_signing_test() diff --git a/test/functional/rpc_uptime.py b/test/functional/rpc_uptime.py index cb99e483ec..f8df59d02a 100755 --- a/test/functional/rpc_uptime.py +++ b/test/functional/rpc_uptime.py @@ -23,7 +23,7 @@ class UptimeTest(BitcoinTestFramework): self._test_uptime() def _test_negative_time(self): - assert_raises_rpc_error(-8, "Mocktime cannot be negative: -1.", self.nodes[0].setmocktime, -1) + assert_raises_rpc_error(-8, "Mocktime must be in the range [0, 9223372036], not -1.", self.nodes[0].setmocktime, -1) def _test_uptime(self): wait_time = 10 diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 1780678de1..4e496a9275 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -46,6 +46,7 @@ MAX_PROTOCOL_MESSAGE_LENGTH = 4000000 # Maximum length of incoming protocol mes MAX_HEADERS_RESULTS = 2000 # Number of headers sent in one getheaders result MAX_INV_SIZE = 50000 # Maximum number of entries in an 'inv' protocol message +NODE_NONE = 0 NODE_NETWORK = (1 << 0) NODE_BLOOM = (1 << 2) NODE_WITNESS = (1 << 3) diff --git a/test/functional/test_framework/netutil.py b/test/functional/test_framework/netutil.py index 30a4a58d6f..08d41fe97f 100644 --- a/test/functional/test_framework/netutil.py +++ b/test/functional/test_framework/netutil.py @@ -158,3 +158,12 @@ def test_ipv6_local(): except socket.error: have_ipv6 = False return have_ipv6 + +def test_unix_socket(): + '''Return True if UNIX sockets are available on this platform.''' + try: + socket.AF_UNIX + except AttributeError: + return False + else: + return True diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index dc04696114..00bd1e4017 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -585,22 +585,22 @@ class P2PInterface(P2PConnection): wait_until_helper_internal(test_function, timeout=timeout, lock=p2p_lock, timeout_factor=self.timeout_factor) - def wait_for_connect(self, timeout=60): + def wait_for_connect(self, *, timeout=60): test_function = lambda: self.is_connected self.wait_until(test_function, timeout=timeout, check_connected=False) - def wait_for_disconnect(self, timeout=60): + def wait_for_disconnect(self, *, timeout=60): test_function = lambda: not self.is_connected self.wait_until(test_function, timeout=timeout, check_connected=False) - def wait_for_reconnect(self, timeout=60): + def wait_for_reconnect(self, *, timeout=60): def test_function(): return self.is_connected and self.last_message.get('version') and not self.supports_v2_p2p self.wait_until(test_function, timeout=timeout, check_connected=False) # Message receiving helper methods - def wait_for_tx(self, txid, timeout=60): + def wait_for_tx(self, txid, *, timeout=60): def test_function(): if not self.last_message.get('tx'): return False @@ -608,13 +608,13 @@ class P2PInterface(P2PConnection): self.wait_until(test_function, timeout=timeout) - def wait_for_block(self, blockhash, timeout=60): + def wait_for_block(self, blockhash, *, timeout=60): def test_function(): return self.last_message.get("block") and self.last_message["block"].block.rehash() == blockhash self.wait_until(test_function, timeout=timeout) - def wait_for_header(self, blockhash, timeout=60): + def wait_for_header(self, blockhash, *, timeout=60): def test_function(): last_headers = self.last_message.get('headers') if not last_headers: @@ -623,7 +623,7 @@ class P2PInterface(P2PConnection): self.wait_until(test_function, timeout=timeout) - def wait_for_merkleblock(self, blockhash, timeout=60): + def wait_for_merkleblock(self, blockhash, *, timeout=60): def test_function(): last_filtered_block = self.last_message.get('merkleblock') if not last_filtered_block: @@ -632,7 +632,7 @@ class P2PInterface(P2PConnection): self.wait_until(test_function, timeout=timeout) - def wait_for_getdata(self, hash_list, timeout=60): + def wait_for_getdata(self, hash_list, *, timeout=60): """Waits for a getdata message. The object hashes in the inventory vector must match the provided hash_list.""" @@ -644,19 +644,21 @@ class P2PInterface(P2PConnection): self.wait_until(test_function, timeout=timeout) - def wait_for_getheaders(self, timeout=60): - """Waits for a getheaders message. + def wait_for_getheaders(self, block_hash=None, *, timeout=60): + """Waits for a getheaders message containing a specific block hash. - Receiving any getheaders message will satisfy the predicate. the last_message["getheaders"] - value must be explicitly cleared before calling this method, or this will return - immediately with success. TODO: change this method to take a hash value and only - return true if the correct block header has been requested.""" + If no block hash is provided, checks whether any getheaders message has been received by the node.""" def test_function(): - return self.last_message.get("getheaders") + last_getheaders = self.last_message.pop("getheaders", None) + if block_hash is None: + return last_getheaders + if last_getheaders is None: + return False + return block_hash == last_getheaders.locator.vHave[0] self.wait_until(test_function, timeout=timeout) - def wait_for_inv(self, expected_inv, timeout=60): + def wait_for_inv(self, expected_inv, *, timeout=60): """Waits for an INV message and checks that the first inv object in the message was as expected.""" if len(expected_inv) > 1: raise NotImplementedError("wait_for_inv() will only verify the first inv object") @@ -668,7 +670,7 @@ class P2PInterface(P2PConnection): self.wait_until(test_function, timeout=timeout) - def wait_for_verack(self, timeout=60): + def wait_for_verack(self, *, timeout=60): def test_function(): return "verack" in self.last_message @@ -681,11 +683,11 @@ class P2PInterface(P2PConnection): self.send_message(self.on_connection_send_msg) self.on_connection_send_msg = None # Never used again - def send_and_ping(self, message, timeout=60): + def send_and_ping(self, message, *, timeout=60): self.send_message(message) self.sync_with_ping(timeout=timeout) - def sync_with_ping(self, timeout=60): + def sync_with_ping(self, *, timeout=60): """Ensure ProcessMessages and SendMessages is called on this connection""" # Sending two pings back-to-back, requires that the node calls # `ProcessMessage` twice, and thus ensures `SendMessages` must have @@ -726,7 +728,7 @@ class NetworkThread(threading.Thread): """Start the network thread.""" self.network_event_loop.run_forever() - def close(self, timeout=10): + def close(self, *, timeout=10): """Close the connections and network event loop.""" self.network_event_loop.call_soon_threadsafe(self.network_event_loop.stop) wait_until_helper_internal(lambda: not self.network_event_loop.is_running(), timeout=timeout) @@ -933,7 +935,7 @@ class P2PTxInvStore(P2PInterface): with p2p_lock: return list(self.tx_invs_received.keys()) - def wait_for_broadcast(self, txns, timeout=60): + def wait_for_broadcast(self, txns, *, timeout=60): """Waits for the txns (list of txids) to complete initial broadcast. The mempool should mark unbroadcast=False for these transactions. """ diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index d8ae20981d..a2f767cc98 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -96,6 +96,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method""" self.chain: str = 'regtest' self.setup_clean_chain: bool = False + self.noban_tx_relay: bool = False self.nodes: list[TestNode] = [] self.extra_args = None self.network_thread = None @@ -163,7 +164,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="Don't stop bitcoinds after the test execution") parser.add_argument("--cachedir", dest="cachedir", default=os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../../cache"), help="Directory for caching pregenerated datadirs (default: %(default)s)") - parser.add_argument("--tmpdir", dest="tmpdir", help="Root directory for datadirs") + parser.add_argument("--tmpdir", dest="tmpdir", help="Root directory for datadirs (must not exist)") parser.add_argument("-l", "--loglevel", dest="loglevel", default="INFO", help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.") parser.add_argument("--tracerpc", dest="trace_rpc", default=False, action="store_true", @@ -191,6 +192,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): parser.add_argument("--timeout-factor", dest="timeout_factor", type=float, help="adjust test timeouts by a factor. Setting it to 0 disables all timeouts") parser.add_argument("--v2transport", dest="v2transport", default=False, action="store_true", help="use BIP324 v2 connections between all nodes by default") + parser.add_argument("--v1transport", dest="v1transport", default=False, action="store_true", + help="Explicitly use v1 transport (can be used to overwrite global --v2transport option)") self.add_options(parser) # Running TestShell in a Jupyter notebook causes an additional -f argument @@ -206,6 +209,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): config = configparser.ConfigParser() config.read_file(open(self.options.configfile)) self.config = config + if self.options.v1transport: + self.options.v2transport=False if "descriptors" not in self.options: # Wallet is not required by the test at all and the value of self.options.descriptors won't matter. @@ -494,6 +499,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): extra_confs = [[]] * num_nodes if extra_args is None: extra_args = [[]] * num_nodes + # Whitelist peers to speed up tx relay / mempool sync. Don't use it if testing tx relay or timing. + if self.noban_tx_relay: + for i in range(len(extra_args)): + extra_args[i] = extra_args[i] + ["-whitelist=noban,in,out@127.0.0.1"] if versions is None: versions = [None] * num_nodes if binary is None: @@ -577,10 +586,16 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # Wait for nodes to stop node.wait_until_stopped() - def restart_node(self, i, extra_args=None): + def restart_node(self, i, extra_args=None, clear_addrman=False): """Stop and start a test node""" self.stop_node(i) - self.start_node(i, extra_args) + if clear_addrman: + peers_dat = self.nodes[i].chain_path / "peers.dat" + os.remove(peers_dat) + with self.nodes[i].assert_debug_log(expected_msgs=[f'Creating peers.dat because the file was not found ("{peers_dat}")']): + self.start_node(i, extra_args) + else: + self.start_node(i, extra_args) def wait_for_node_exit(self, i, timeout): self.nodes[i].process.wait(timeout) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 3baa78fd79..67e0be5280 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -136,9 +136,7 @@ class TestNode(): self.args.append("-v2transport=1") else: self.args.append("-v2transport=0") - else: - # v2transport requested but not supported for node - assert not v2transport + # if v2transport is requested via global flag but not supported for node version, ignore it self.cli = TestNodeCLI(bitcoin_cli, self.datadir_path) self.use_cli = use_cli @@ -726,7 +724,7 @@ class TestNode(): return p2p_conn - def add_outbound_p2p_connection(self, p2p_conn, *, wait_for_verack=True, p2p_idx, connection_type="outbound-full-relay", supports_v2_p2p=None, advertise_v2_p2p=None, **kwargs): + def add_outbound_p2p_connection(self, p2p_conn, *, wait_for_verack=True, wait_for_disconnect=False, p2p_idx, connection_type="outbound-full-relay", supports_v2_p2p=None, advertise_v2_p2p=None, **kwargs): """Add an outbound p2p connection from node. Must be an "outbound-full-relay", "block-relay-only", "addr-fetch" or "feeler" connection. @@ -773,7 +771,7 @@ class TestNode(): if reconnect: p2p_conn.wait_for_reconnect() - if connection_type == "feeler": + if connection_type == "feeler" or wait_for_disconnect: # feeler connections are closed as soon as the node receives a `version` message p2p_conn.wait_until(lambda: p2p_conn.message_count["version"] == 1, check_connected=False) p2p_conn.wait_until(lambda: not p2p_conn.is_connected, check_connected=False) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index b4b05b1597..0de09b6440 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -52,7 +52,24 @@ def assert_fee_amount(fee, tx_size, feerate_BTC_kvB): raise AssertionError("Fee of %s BTC too high! (Should be %s BTC)" % (str(fee), str(target_fee))) +def summarise_dict_differences(thing1, thing2): + if not isinstance(thing1, dict) or not isinstance(thing2, dict): + return thing1, thing2 + d1, d2 = {}, {} + for k in sorted(thing1.keys()): + if k not in thing2: + d1[k] = thing1[k] + elif thing1[k] != thing2[k]: + d1[k], d2[k] = summarise_dict_differences(thing1[k], thing2[k]) + for k in sorted(thing2.keys()): + if k not in thing1: + d2[k] = thing2[k] + return d1, d2 + def assert_equal(thing1, thing2, *args): + if thing1 != thing2 and not args and isinstance(thing1, dict) and isinstance(thing2, dict): + d1,d2 = summarise_dict_differences(thing1, thing2) + raise AssertionError("not(%s == %s)\n in particular not(%s == %s)" % (thing1, thing2, d1, d2)) if thing1 != thing2 or any(thing1 != arg for arg in args): raise AssertionError("not(%s)" % " == ".join(str(arg) for arg in (thing1, thing2) + args)) @@ -479,6 +496,65 @@ def check_node_connections(*, node, num_in, num_out): assert_equal(info["connections_in"], num_in) assert_equal(info["connections_out"], num_out) +def fill_mempool(test_framework, node, miniwallet): + """Fill mempool until eviction. + + Allows for simpler testing of scenarios with floating mempoolminfee > minrelay + Requires -datacarriersize=100000 and + -maxmempool=5. + It will not ensure mempools become synced as it + is based on a single node and assumes -minrelaytxfee + is 1 sat/vbyte. + To avoid unintentional tx dependencies, it is recommended to use separate miniwallets for + mempool filling vs transactions in tests. + """ + test_framework.log.info("Fill the mempool until eviction is triggered and the mempoolminfee rises") + txouts = gen_return_txouts() + relayfee = node.getnetworkinfo()['relayfee'] + + assert_equal(relayfee, Decimal('0.00001000')) + + tx_batch_size = 1 + num_of_batches = 75 + # Generate UTXOs to flood the mempool + # 1 to create a tx initially that will be evicted from the mempool later + # 75 transactions each with a fee rate higher than the previous one + test_framework.generate(miniwallet, 1 + (num_of_batches * tx_batch_size)) + + # Mine COINBASE_MATURITY - 1 blocks so that the UTXOs are allowed to be spent + test_framework.generate(node, 100 - 1) + + # Get all UTXOs up front to ensure none of the transactions spend from each other, as that may + # change their effective feerate and thus the order in which they are selected for eviction. + confirmed_utxos = [miniwallet.get_utxo(confirmed_only=True) for _ in range(num_of_batches * tx_batch_size + 1)] + assert_equal(len(confirmed_utxos), num_of_batches * tx_batch_size + 1) + + test_framework.log.debug("Create a mempool tx that will be evicted") + tx_to_be_evicted_id = miniwallet.send_self_transfer(from_node=node, utxo_to_spend=confirmed_utxos[0], fee_rate=relayfee)["txid"] + del confirmed_utxos[0] + + # Increase the tx fee rate to give the subsequent transactions a higher priority in the mempool + # The tx has an approx. vsize of 65k, i.e. multiplying the previous fee rate (in sats/kvB) + # by 130 should result in a fee that corresponds to 2x of that fee rate + base_fee = relayfee * 130 + + test_framework.log.debug("Fill up the mempool with txs with higher fee rate") + with node.assert_debug_log(["rolling minimum fee bumped"]): + for batch_of_txid in range(num_of_batches): + fee = (batch_of_txid + 1) * base_fee + utxos = confirmed_utxos[:tx_batch_size] + create_lots_of_big_transactions(miniwallet, node, fee, tx_batch_size, txouts, utxos) + del confirmed_utxos[:tx_batch_size] + + test_framework.log.debug("The tx should be evicted by now") + # The number of transactions created should be greater than the ones present in the mempool + assert_greater_than(tx_batch_size * num_of_batches, len(node.getrawmempool())) + # Initial tx created should not be present in the mempool anymore as it had a lower fee rate + assert tx_to_be_evicted_id not in node.getrawmempool() + + test_framework.log.debug("Check that mempoolminfee is larger than minrelaytxfee") + assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) + assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) # Transaction/Block functions ############################# diff --git a/test/functional/test_framework/wallet_util.py b/test/functional/test_framework/wallet_util.py index 44811918bf..2168e607b2 100755 --- a/test/functional/test_framework/wallet_util.py +++ b/test/functional/test_framework/wallet_util.py @@ -4,6 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Useful util functions for testing the wallet""" from collections import namedtuple +import unittest from test_framework.address import ( byte_to_base58, @@ -15,6 +16,11 @@ from test_framework.address import ( script_to_p2wsh, ) from test_framework.key import ECKey +from test_framework.messages import ( + CTxIn, + CTxInWitness, + WITNESS_SCALE_FACTOR, +) from test_framework.script_util import ( key_to_p2pkh_script, key_to_p2wpkh_script, @@ -123,6 +129,19 @@ def generate_keypair(compressed=True, wif=False): privkey = bytes_to_wif(privkey.get_bytes(), compressed) return privkey, pubkey +def calculate_input_weight(scriptsig_hex, witness_stack_hex=None): + """Given a scriptSig and a list of witness stack items for an input in hex format, + calculate the total input weight. If the input has no witness data, + `witness_stack_hex` can be set to None.""" + tx_in = CTxIn(scriptSig=bytes.fromhex(scriptsig_hex)) + witness_size = 0 + if witness_stack_hex is not None: + tx_inwit = CTxInWitness() + for witness_item_hex in witness_stack_hex: + tx_inwit.scriptWitness.stack.append(bytes.fromhex(witness_item_hex)) + witness_size = len(tx_inwit.serialize()) + return len(tx_in.serialize()) * WITNESS_SCALE_FACTOR + witness_size + class WalletUnlock(): """ A context manager for unlocking a wallet with a passphrase and automatically locking it afterward. @@ -141,3 +160,42 @@ class WalletUnlock(): def __exit__(self, *args): _ = args self.wallet.walletlock() + + +class TestFrameworkWalletUtil(unittest.TestCase): + def test_calculate_input_weight(self): + SKELETON_BYTES = 32 + 4 + 4 # prevout-txid, prevout-index, sequence + SMALL_LEN_BYTES = 1 # bytes needed for encoding scriptSig / witness item lengths < 253 + LARGE_LEN_BYTES = 3 # bytes needed for encoding scriptSig / witness item lengths >= 253 + + # empty scriptSig, no witness + self.assertEqual(calculate_input_weight(""), + (SKELETON_BYTES + SMALL_LEN_BYTES) * WITNESS_SCALE_FACTOR) + self.assertEqual(calculate_input_weight("", None), + (SKELETON_BYTES + SMALL_LEN_BYTES) * WITNESS_SCALE_FACTOR) + # small scriptSig, no witness + scriptSig_small = "00"*252 + self.assertEqual(calculate_input_weight(scriptSig_small, None), + (SKELETON_BYTES + SMALL_LEN_BYTES + 252) * WITNESS_SCALE_FACTOR) + # small scriptSig, empty witness stack + self.assertEqual(calculate_input_weight(scriptSig_small, []), + (SKELETON_BYTES + SMALL_LEN_BYTES + 252) * WITNESS_SCALE_FACTOR + SMALL_LEN_BYTES) + # large scriptSig, no witness + scriptSig_large = "00"*253 + self.assertEqual(calculate_input_weight(scriptSig_large, None), + (SKELETON_BYTES + LARGE_LEN_BYTES + 253) * WITNESS_SCALE_FACTOR) + # large scriptSig, empty witness stack + self.assertEqual(calculate_input_weight(scriptSig_large, []), + (SKELETON_BYTES + LARGE_LEN_BYTES + 253) * WITNESS_SCALE_FACTOR + SMALL_LEN_BYTES) + # empty scriptSig, 5 small witness stack items + self.assertEqual(calculate_input_weight("", ["00", "11", "22", "33", "44"]), + ((SKELETON_BYTES + SMALL_LEN_BYTES) * WITNESS_SCALE_FACTOR) + SMALL_LEN_BYTES + 5 * SMALL_LEN_BYTES + 5) + # empty scriptSig, 253 small witness stack items + self.assertEqual(calculate_input_weight("", ["00"]*253), + ((SKELETON_BYTES + SMALL_LEN_BYTES) * WITNESS_SCALE_FACTOR) + LARGE_LEN_BYTES + 253 * SMALL_LEN_BYTES + 253) + # small scriptSig, 3 large witness stack items + self.assertEqual(calculate_input_weight(scriptSig_small, ["00"*253]*3), + ((SKELETON_BYTES + SMALL_LEN_BYTES + 252) * WITNESS_SCALE_FACTOR) + SMALL_LEN_BYTES + 3 * LARGE_LEN_BYTES + 3*253) + # large scriptSig, 3 large witness stack items + self.assertEqual(calculate_input_weight(scriptSig_large, ["00"*253]*3), + ((SKELETON_BYTES + LARGE_LEN_BYTES + 253) * WITNESS_SCALE_FACTOR) + SMALL_LEN_BYTES + 3 * LARGE_LEN_BYTES + 3*253) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 4d66ea97c8..32b55813a8 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -85,6 +85,7 @@ TEST_FRAMEWORK_MODULES = [ "crypto.ripemd160", "script", "segwit_addr", + "wallet_util", ] EXTENDED_SCRIPTS = [ @@ -120,7 +121,7 @@ BASE_SCRIPTS = [ 'wallet_backup.py --legacy-wallet', 'wallet_backup.py --descriptors', 'feature_segwit.py --legacy-wallet', - 'feature_segwit.py --descriptors', + 'feature_segwit.py --descriptors --v1transport', 'feature_segwit.py --descriptors --v2transport', 'p2p_tx_download.py', 'wallet_avoidreuse.py --legacy-wallet', @@ -156,7 +157,7 @@ BASE_SCRIPTS = [ # vv Tests less than 30s vv 'p2p_invalid_messages.py', 'rpc_createmultisig.py', - 'p2p_timeouts.py', + 'p2p_timeouts.py --v1transport', 'p2p_timeouts.py --v2transport', 'wallet_dump.py --legacy-wallet', 'rpc_signer.py', @@ -181,6 +182,8 @@ BASE_SCRIPTS = [ 'wallet_keypool_topup.py --legacy-wallet', 'wallet_keypool_topup.py --descriptors', 'wallet_fast_rescan.py --descriptors', + 'wallet_gethdkeys.py --descriptors', + 'wallet_createwalletdescriptor.py --descriptors', 'interface_zmq.py', 'rpc_invalid_address_message.py', 'rpc_validateaddress.py', @@ -201,7 +204,7 @@ BASE_SCRIPTS = [ 'mempool_spend_coinbase.py', 'wallet_avoid_mixing_output_types.py --descriptors', 'mempool_reorg.py', - 'p2p_block_sync.py', + 'p2p_block_sync.py --v1transport', 'p2p_block_sync.py --v2transport', 'wallet_createwallet.py --legacy-wallet', 'wallet_createwallet.py --usecli', @@ -230,13 +233,13 @@ BASE_SCRIPTS = [ 'wallet_transactiontime_rescan.py --descriptors', 'wallet_transactiontime_rescan.py --legacy-wallet', 'p2p_addrv2_relay.py', - 'p2p_compactblocks_hb.py', + 'p2p_compactblocks_hb.py --v1transport', 'p2p_compactblocks_hb.py --v2transport', - 'p2p_disconnect_ban.py', + 'p2p_disconnect_ban.py --v1transport', 'p2p_disconnect_ban.py --v2transport', 'feature_posix_fs_permissions.py', 'rpc_decodescript.py', - 'rpc_blockchain.py', + 'rpc_blockchain.py --v1transport', 'rpc_blockchain.py --v2transport', 'rpc_deprecated.py', 'wallet_disable.py', @@ -246,21 +249,21 @@ BASE_SCRIPTS = [ 'p2p_getaddr_caching.py', 'p2p_getdata.py', 'p2p_addrfetch.py', - 'rpc_net.py', + 'rpc_net.py --v1transport', 'rpc_net.py --v2transport', 'wallet_keypool.py --legacy-wallet', 'wallet_keypool.py --descriptors', 'wallet_descriptor.py --descriptors', 'p2p_nobloomfilter_messages.py', 'p2p_filter.py', - 'rpc_setban.py', + 'rpc_setban.py --v1transport', 'rpc_setban.py --v2transport', 'p2p_blocksonly.py', 'mining_prioritisetransaction.py', 'p2p_invalid_locator.py', - 'p2p_invalid_block.py', + 'p2p_invalid_block.py --v1transport', 'p2p_invalid_block.py --v2transport', - 'p2p_invalid_tx.py', + 'p2p_invalid_tx.py --v1transport', 'p2p_invalid_tx.py --v2transport', 'p2p_v2_transport.py', 'p2p_v2_encrypted.py', @@ -286,12 +289,12 @@ BASE_SCRIPTS = [ 'rpc_preciousblock.py', 'wallet_importprunedfunds.py --legacy-wallet', 'wallet_importprunedfunds.py --descriptors', - 'p2p_leak_tx.py', + 'p2p_leak_tx.py --v1transport', 'p2p_leak_tx.py --v2transport', 'p2p_eviction.py', - 'p2p_ibd_stalling.py', + 'p2p_ibd_stalling.py --v1transport', 'p2p_ibd_stalling.py --v2transport', - 'p2p_net_deadlock.py', + 'p2p_net_deadlock.py --v1transport', 'p2p_net_deadlock.py --v2transport', 'wallet_signmessagewithaddress.py', 'rpc_signmessagewithprivkey.py', @@ -381,7 +384,7 @@ BASE_SCRIPTS = [ 'feature_coinstatsindex.py', 'wallet_orphanedreward.py', 'wallet_timelock.py', - 'p2p_node_network_limited.py', + 'p2p_node_network_limited.py --v1transport', 'p2p_node_network_limited.py --v2transport', 'p2p_permissions.py', 'feature_blocksdir.py', @@ -395,6 +398,8 @@ BASE_SCRIPTS = [ 'rpc_getdescriptorinfo.py', 'rpc_mempool_info.py', 'rpc_help.py', + 'p2p_handshake.py', + 'p2p_handshake.py --v2transport', 'feature_dirsymlinks.py', 'feature_help.py', 'feature_shutdown.py', @@ -614,14 +619,12 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= max_len_name = len(max(test_list, key=len)) test_count = len(test_list) all_passed = True - i = 0 - while i < test_count: + while not job_queue.done(): if failfast and not all_passed: break for test_result, testdir, stdout, stderr, skip_reason in job_queue.get_next(): test_results.append(test_result) - i += 1 - done_str = "{}/{} - {}{}{}".format(i, test_count, BOLD[1], test_result.name, BOLD[0]) + done_str = f"{len(test_results)}/{test_count} - {BOLD[1]}{test_result.name}{BOLD[0]}" if test_result.status == "Passed": logging.debug("%s passed, Duration: %s s" % (done_str, test_result.time)) elif test_result.status == "Skipped": @@ -706,14 +709,15 @@ class TestHandler: self.tmpdir = tmpdir self.test_list = test_list self.flags = flags - self.num_running = 0 self.jobs = [] self.use_term_control = use_term_control + def done(self): + return not (self.jobs or self.test_list) + def get_next(self): - while self.num_running < self.num_jobs and self.test_list: + while len(self.jobs) < self.num_jobs and self.test_list: # Add tests - self.num_running += 1 test = self.test_list.pop(0) portseed = len(self.test_list) portseed_arg = ["--portseed={}".format(portseed)] @@ -757,7 +761,6 @@ class TestHandler: skip_reason = re.search(r"Test Skipped: (.*)", stdout).group(1) else: status = "Failed" - self.num_running -= 1 self.jobs.remove(job) if self.use_term_control: clearline = '\r' + (' ' * dot_count) + '\r' diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index 2691507773..dda48aae1b 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -28,8 +28,7 @@ class AbandonConflictTest(BitcoinTestFramework): self.num_nodes = 2 self.extra_args = [["-minrelaytxfee=0.00001"], []] # whitelist peers to speed up tx relay / mempool sync - for args in self.extra_args: - args.append("-whitelist=noban@127.0.0.1") + self.noban_tx_relay = True def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -232,7 +231,11 @@ class AbandonConflictTest(BitcoinTestFramework): balance = newbalance # Invalidate the block with the double spend. B & C's 10 BTC outputs should no longer be available - self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) + blk = self.nodes[0].getbestblockhash() + # mine 10 blocks so that when the blk is invalidated, the transactions are not + # returned to the mempool + self.generate(self.nodes[1], 10) + self.nodes[0].invalidateblock(blk) assert_equal(alice.gettransaction(txAB1)["confirmations"], 0) newbalance = alice.getbalance() assert_equal(newbalance, balance - Decimal("20")) diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index be5b3ebadb..6b27b32dea 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -79,9 +79,8 @@ class AddressTypeTest(BitcoinTestFramework): ["-changetype=p2sh-segwit"], [], ] - # whitelist all peers to speed up tx relay / mempool sync - for args in self.extra_args: - args.append("-whitelist=noban@127.0.0.1") + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.supports_cli = False def skip_test_if_missing_module(self): diff --git a/test/functional/wallet_assumeutxo.py b/test/functional/wallet_assumeutxo.py index 3c1a997bd1..30396da015 100755 --- a/test/functional/wallet_assumeutxo.py +++ b/test/functional/wallet_assumeutxo.py @@ -62,8 +62,6 @@ class AssumeutxoTest(BitcoinTestFramework): for n in self.nodes: n.setmocktime(n.getblockheader(n.getbestblockhash())['time']) - self.sync_blocks() - n0.createwallet('w') w = n0.get_wallet_rpc("w") diff --git a/test/functional/wallet_avoid_mixing_output_types.py b/test/functional/wallet_avoid_mixing_output_types.py index 861765f452..66fbf780e5 100755 --- a/test/functional/wallet_avoid_mixing_output_types.py +++ b/test/functional/wallet_avoid_mixing_output_types.py @@ -112,15 +112,15 @@ class AddressInputTypeGrouping(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.extra_args = [ [ "-addresstype=bech32", - "-whitelist=noban@127.0.0.1", "-txindex", ], [ "-addresstype=p2sh-segwit", - "-whitelist=noban@127.0.0.1", "-txindex", ], ] diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 9d3c55d6b6..4983bfda7f 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -69,9 +69,8 @@ class AvoidReuseTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - # This test isn't testing txn relay/timing, so set whitelist on the - # peers for instant txn relay. This speeds up the test run time 2-3x. - self.extra_args = [["-whitelist=noban@127.0.0.1"]] * self.num_nodes + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index eb3e0ae728..d03b08bcc4 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -50,13 +50,14 @@ class WalletBackupTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.setup_clean_chain = True - # nodes 1, 2,3 are spenders, let's give them a keypool=100 - # whitelist all peers to speed up tx relay / mempool sync + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True + # nodes 1, 2, 3 are spenders, let's give them a keypool=100 self.extra_args = [ - ["-whitelist=noban@127.0.0.1", "-keypool=100"], - ["-whitelist=noban@127.0.0.1", "-keypool=100"], - ["-whitelist=noban@127.0.0.1", "-keypool=100"], - ["-whitelist=noban@127.0.0.1"], + ["-keypool=100"], + ["-keypool=100"], + ["-keypool=100"], + [], ] self.rpc_timeout = 120 diff --git a/test/functional/wallet_backwards_compatibility.py b/test/functional/wallet_backwards_compatibility.py index 4d6e6024c5..ab008a40cd 100755 --- a/test/functional/wallet_backwards_compatibility.py +++ b/test/functional/wallet_backwards_compatibility.py @@ -355,6 +355,25 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): down_wallet_name = f"re_down_{node.version}" down_backup_path = os.path.join(self.options.tmpdir, f"{down_wallet_name}.dat") wallet.backupwallet(down_backup_path) + + # Check that taproot descriptors can be added to 0.21 wallets + # This must be done after the backup is created so that 0.21 can still load + # the backup + if self.options.descriptors and self.major_version_equals(node, 21): + assert_raises_rpc_error(-12, "No bech32m addresses available", wallet.getnewaddress, address_type="bech32m") + xpubs = wallet.gethdkeys(active_only=True) + assert_equal(len(xpubs), 1) + assert_equal(len(xpubs[0]["descriptors"]), 6) + wallet.createwalletdescriptor("bech32m") + xpubs = wallet.gethdkeys(active_only=True) + assert_equal(len(xpubs), 1) + assert_equal(len(xpubs[0]["descriptors"]), 8) + tr_descs = [desc["desc"] for desc in xpubs[0]["descriptors"] if desc["desc"].startswith("tr(")] + assert_equal(len(tr_descs), 2) + for desc in tr_descs: + assert info["hdmasterfingerprint"] in desc + wallet.getnewaddress(address_type="bech32m") + wallet.unloadwallet() # Check that no automatic upgrade broke the downgrading the wallet diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index af9270a321..c322ae52c1 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -53,15 +53,14 @@ class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.extra_args = [ # Limit mempool descendants as a hack to have wallet txs rejected from the mempool. # Set walletrejectlongchains=0 so the wallet still creates the transactions. ['-limitdescendantcount=3', '-walletrejectlongchains=0'], [], ] - # whitelist peers to speed up tx relay / mempool sync - for args in self.extra_args: - args.append("-whitelist=noban@127.0.0.1") def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index f798eee365..56228d2bad 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -32,8 +32,10 @@ class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.extra_args = [[ - "-dustrelayfee=0", "-walletrejectlongchains=0", "-whitelist=noban@127.0.0.1" + "-dustrelayfee=0", "-walletrejectlongchains=0" ]] * self.num_nodes self.setup_clean_chain = True self.supports_cli = False @@ -679,7 +681,7 @@ class WalletTest(BitcoinTestFramework): "category": baz["category"], "vout": baz["vout"]} expected_fields = frozenset({'amount', 'bip125-replaceable', 'confirmations', 'details', 'fee', - 'hex', 'lastprocessedblock', 'time', 'timereceived', 'trusted', 'txid', 'wtxid', 'walletconflicts'}) + 'hex', 'lastprocessedblock', 'time', 'timereceived', 'trusted', 'txid', 'wtxid', 'walletconflicts', 'mempoolconflicts'}) verbose_field = "decoded" expected_verbose_fields = expected_fields | {verbose_field} diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index fea933a93b..5b7db55f45 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -55,11 +55,12 @@ class BumpFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.extra_args = [[ "-walletrbf={}".format(i), "-mintxfee=0.00002", "-addresstype=bech32", - "-whitelist=noban@127.0.0.1", ] for i in range(self.num_nodes)] def skip_test_if_missing_module(self): diff --git a/test/functional/wallet_conflicts.py b/test/functional/wallet_conflicts.py index 802b718cd5..e5739a6a59 100755 --- a/test/functional/wallet_conflicts.py +++ b/test/functional/wallet_conflicts.py @@ -9,6 +9,7 @@ Test that wallet correctly tracks transactions that have been conflicted by bloc from decimal import Decimal +from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -28,6 +29,20 @@ class TxConflicts(BitcoinTestFramework): return next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(from_tx_id)["details"] if tx_out["amount"] == Decimal(f"{search_value}")) def run_test(self): + """ + The following tests check the behavior of the wallet when + transaction conflicts are created. These conflicts are created + using raw transaction RPCs that double-spend UTXOs and have more + fees, replacing the original transaction. + """ + + self.test_block_conflicts() + self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 7, self.nodes[2].getnewaddress()) + self.test_mempool_conflict() + self.test_mempool_and_block_conflicts() + self.test_descendants_with_mempool_conflicts() + + def test_block_conflicts(self): self.log.info("Send tx from which to conflict outputs later") txid_conflict_from_1 = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) txid_conflict_from_2 = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) @@ -123,5 +138,291 @@ class TxConflicts(BitcoinTestFramework): assert_equal(former_conflicted["confirmations"], 1) assert_equal(former_conflicted["blockheight"], 217) + def test_mempool_conflict(self): + self.nodes[0].createwallet("alice") + alice = self.nodes[0].get_wallet_rpc("alice") + + bob = self.nodes[1] + + self.nodes[2].send(outputs=[{alice.getnewaddress() : 25} for _ in range(3)]) + self.generate(self.nodes[2], 1) + + self.log.info("Test a scenario where a transaction has a mempool conflict") + + unspents = alice.listunspent() + assert_equal(len(unspents), 3) + assert all([tx["amount"] == 25 for tx in unspents]) + + # tx1 spends unspent[0] and unspent[1] + raw_tx = alice.createrawtransaction(inputs=[unspents[0], unspents[1]], outputs=[{bob.getnewaddress() : 49.9999}]) + tx1 = alice.signrawtransactionwithwallet(raw_tx)['hex'] + + # tx2 spends unspent[1] and unspent[2], conflicts with tx1 + raw_tx = alice.createrawtransaction(inputs=[unspents[1], unspents[2]], outputs=[{bob.getnewaddress() : 49.99}]) + tx2 = alice.signrawtransactionwithwallet(raw_tx)['hex'] + + # tx3 spends unspent[2], conflicts with tx2 + raw_tx = alice.createrawtransaction(inputs=[unspents[2]], outputs=[{bob.getnewaddress() : 24.9899}]) + tx3 = alice.signrawtransactionwithwallet(raw_tx)['hex'] + + # broadcast tx1 + tx1_txid = alice.sendrawtransaction(tx1) + + assert_equal(alice.listunspent(), [unspents[2]]) + assert_equal(alice.getbalance(), 25) + + # broadcast tx2, replaces tx1 in mempool + tx2_txid = alice.sendrawtransaction(tx2) + + # Check that unspent[0] is now available because the transaction spending it has been replaced in the mempool + assert_equal(alice.listunspent(), [unspents[0]]) + assert_equal(alice.getbalance(), 25) + + assert_equal(alice.gettransaction(tx1_txid)["mempoolconflicts"], [tx2_txid]) + + self.log.info("Test scenario where a mempool conflict is removed") + + # broadcast tx3, replaces tx2 in mempool + # Now that tx1's conflict has been removed, tx1 is now + # not conflicted, and instead is inactive until it is + # rebroadcasted. Now unspent[0] is not available, because + # tx1 is no longer conflicted. + alice.sendrawtransaction(tx3) + + assert_equal(alice.gettransaction(tx1_txid)["mempoolconflicts"], []) + assert tx1_txid not in self.nodes[0].getrawmempool() + + # now all of alice's outputs should be considered spent + # unspent[0]: spent by inactive tx1 + # unspent[1]: spent by inactive tx1 + # unspent[2]: spent by active tx3 + assert_equal(alice.listunspent(), []) + assert_equal(alice.getbalance(), 0) + + # Clean up for next test + bob.sendall([self.nodes[2].getnewaddress()]) + self.generate(self.nodes[2], 1) + + alice.unloadwallet() + + def test_mempool_and_block_conflicts(self): + self.nodes[0].createwallet("alice_2") + alice = self.nodes[0].get_wallet_rpc("alice_2") + bob = self.nodes[1] + + self.nodes[2].send(outputs=[{alice.getnewaddress() : 25} for _ in range(3)]) + self.generate(self.nodes[2], 1) + + self.log.info("Test a scenario where a transaction has both a block conflict and a mempool conflict") + unspents = [{"txid" : element["txid"], "vout" : element["vout"]} for element in alice.listunspent()] + + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + + # alice and bob nodes are disconnected so that transactions can be + # created by alice, but broadcasted from bob so that alice's wallet + # doesn't know about them + self.disconnect_nodes(0, 1) + + # Sends funds to bob + raw_tx = alice.createrawtransaction(inputs=[unspents[0]], outputs=[{bob.getnewaddress() : 24.99999}]) + raw_tx1 = alice.signrawtransactionwithwallet(raw_tx)['hex'] + tx1_txid = bob.sendrawtransaction(raw_tx1) # broadcast original tx spending unspents[0] only to bob + + # create a conflict to previous tx (also spends unspents[0]), but don't broadcast, sends funds back to alice + raw_tx = alice.createrawtransaction(inputs=[unspents[0], unspents[2]], outputs=[{alice.getnewaddress() : 49.999}]) + tx1_conflict = alice.signrawtransactionwithwallet(raw_tx)['hex'] + + # Sends funds to bob + raw_tx = alice.createrawtransaction(inputs=[unspents[1]], outputs=[{bob.getnewaddress() : 24.9999}]) + raw_tx2 = alice.signrawtransactionwithwallet(raw_tx)['hex'] + tx2_txid = bob.sendrawtransaction(raw_tx2) # broadcast another original tx spending unspents[1] only to bob + + # create a conflict to previous tx (also spends unspents[1]), but don't broadcast, sends funds to alice + raw_tx = alice.createrawtransaction(inputs=[unspents[1]], outputs=[{alice.getnewaddress() : 24.9999}]) + tx2_conflict = alice.signrawtransactionwithwallet(raw_tx)['hex'] + + bob_unspents = [{"txid" : element, "vout" : 0} for element in [tx1_txid, tx2_txid]] + + # tx1 and tx2 are now in bob's mempool, and they are unconflicted, so bob has these funds + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], Decimal("49.99989000")) + + # spend both of bob's unspents, child tx of tx1 and tx2 + raw_tx = bob.createrawtransaction(inputs=[bob_unspents[0], bob_unspents[1]], outputs=[{bob.getnewaddress() : 49.999}]) + raw_tx3 = bob.signrawtransactionwithwallet(raw_tx)['hex'] + tx3_txid = bob.sendrawtransaction(raw_tx3) # broadcast tx only to bob + + # alice knows about 0 txs, bob knows about 3 + assert_equal(len(alice.getrawmempool()), 0) + assert_equal(len(bob.getrawmempool()), 3) + + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], Decimal("49.99900000")) + + # bob broadcasts tx_1 conflict + tx1_conflict_txid = bob.sendrawtransaction(tx1_conflict) + assert_equal(len(alice.getrawmempool()), 0) + assert_equal(len(bob.getrawmempool()), 2) # tx1_conflict kicks out both tx1, and its child tx3 + + assert tx2_txid in bob.getrawmempool() + assert tx1_conflict_txid in bob.getrawmempool() + + assert_equal(bob.gettransaction(tx1_txid)["mempoolconflicts"], [tx1_conflict_txid]) + assert_equal(bob.gettransaction(tx2_txid)["mempoolconflicts"], []) + assert_equal(bob.gettransaction(tx3_txid)["mempoolconflicts"], [tx1_conflict_txid]) + + # check that tx3 is now conflicted, so the output from tx2 can now be spent + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], Decimal("24.99990000")) + + # we will be disconnecting this block in the future + alice.sendrawtransaction(tx2_conflict) + assert_equal(len(alice.getrawmempool()), 1) # currently alice's mempool is only aware of tx2_conflict + # 11 blocks are mined so that when they are invalidated, tx_2 + # does not get put back into the mempool + blk = self.generate(self.nodes[0], 11, sync_fun=self.no_op)[0] + assert_equal(len(alice.getrawmempool()), 0) # tx2_conflict is now mined + + self.connect_nodes(0, 1) + self.sync_blocks() + assert_equal(alice.getbestblockhash(), bob.getbestblockhash()) + + # now that tx2 has a block conflict, tx1_conflict should be the only tx in bob's mempool + assert tx1_conflict_txid in bob.getrawmempool() + assert_equal(len(bob.getrawmempool()), 1) + + # tx3 should now also be block-conflicted by tx2_conflict + assert_equal(bob.gettransaction(tx3_txid)["confirmations"], -11) + # bob has no pending funds, since tx1, tx2, and tx3 are all conflicted + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + bob.invalidateblock(blk) # remove tx2_conflict + # bob should still have no pending funds because tx1 and tx3 are still conflicted, and tx2 has not been re-broadcast + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + assert_equal(len(bob.getrawmempool()), 1) + # check that tx3 is no longer block-conflicted + assert_equal(bob.gettransaction(tx3_txid)["confirmations"], 0) + + bob.sendrawtransaction(raw_tx2) + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], Decimal("24.99990000")) + + # create a conflict to previous tx (also spends unspents[2]), but don't broadcast, sends funds back to alice + raw_tx = alice.createrawtransaction(inputs=[unspents[2]], outputs=[{alice.getnewaddress() : 24.99}]) + tx1_conflict_conflict = alice.signrawtransactionwithwallet(raw_tx)['hex'] + + bob.sendrawtransaction(tx1_conflict_conflict) # kick tx1_conflict out of the mempool + bob.sendrawtransaction(raw_tx1) #re-broadcast tx1 because it is no longer conflicted + + # Now bob has no pending funds because tx1 and tx2 are spent by tx3, which hasn't been re-broadcast yet + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + + bob.sendrawtransaction(raw_tx3) + assert_equal(len(bob.getrawmempool()), 4) # The mempool contains: tx1, tx2, tx1_conflict_conflict, tx3 + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], Decimal("49.99900000")) + + # Clean up for next test + bob.reconsiderblock(blk) + assert_equal(alice.getbestblockhash(), bob.getbestblockhash()) + self.sync_mempools() + self.generate(self.nodes[2], 1) + + alice.unloadwallet() + + def test_descendants_with_mempool_conflicts(self): + self.nodes[0].createwallet("alice_3") + alice = self.nodes[0].get_wallet_rpc("alice_3") + + self.nodes[2].send(outputs=[{alice.getnewaddress() : 25} for _ in range(2)]) + self.generate(self.nodes[2], 1) + + self.nodes[1].createwallet("bob_1") + bob = self.nodes[1].get_wallet_rpc("bob_1") + + self.nodes[2].createwallet("carol") + carol = self.nodes[2].get_wallet_rpc("carol") + + self.log.info("Test a scenario where a transaction's parent has a mempool conflict") + + unspents = alice.listunspent() + assert_equal(len(unspents), 2) + assert all([tx["amount"] == 25 for tx in unspents]) + + assert_equal(alice.getrawmempool(), []) + + # Alice spends first utxo to bob in tx1 + raw_tx = alice.createrawtransaction(inputs=[unspents[0]], outputs=[{bob.getnewaddress() : 24.9999}]) + tx1 = alice.signrawtransactionwithwallet(raw_tx)['hex'] + tx1_txid = alice.sendrawtransaction(tx1) + + self.sync_mempools() + + assert_equal(alice.getbalance(), 25) + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], Decimal("24.99990000")) + + assert_equal(bob.gettransaction(tx1_txid)["mempoolconflicts"], []) + + raw_tx = bob.createrawtransaction(inputs=[bob.listunspent(minconf=0)[0]], outputs=[{carol.getnewaddress() : 24.999}]) + # Bob creates a child to tx1 + tx1_child = bob.signrawtransactionwithwallet(raw_tx)['hex'] + tx1_child_txid = bob.sendrawtransaction(tx1_child) + + self.sync_mempools() + + # Currently neither tx1 nor tx1_child should have any conflicts + assert_equal(bob.gettransaction(tx1_txid)["mempoolconflicts"], []) + assert_equal(bob.gettransaction(tx1_child_txid)["mempoolconflicts"], []) + assert tx1_txid in bob.getrawmempool() + assert tx1_child_txid in bob.getrawmempool() + assert_equal(len(bob.getrawmempool()), 2) + + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + assert_equal(carol.getbalances()["mine"]["untrusted_pending"], Decimal("24.99900000")) + + # Alice spends first unspent again, conflicting with tx1 + raw_tx = alice.createrawtransaction(inputs=[unspents[0], unspents[1]], outputs=[{carol.getnewaddress() : 49.99}]) + tx1_conflict = alice.signrawtransactionwithwallet(raw_tx)['hex'] + tx1_conflict_txid = alice.sendrawtransaction(tx1_conflict) + + self.sync_mempools() + + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + assert_equal(carol.getbalances()["mine"]["untrusted_pending"], Decimal("49.99000000")) + + assert tx1_txid not in bob.getrawmempool() + assert tx1_child_txid not in bob.getrawmempool() + assert tx1_conflict_txid in bob.getrawmempool() + assert_equal(len(bob.getrawmempool()), 1) + + # Now both tx1 and tx1_child are conflicted by tx1_conflict + assert_equal(bob.gettransaction(tx1_txid)["mempoolconflicts"], [tx1_conflict_txid]) + assert_equal(bob.gettransaction(tx1_child_txid)["mempoolconflicts"], [tx1_conflict_txid]) + + # Now create a conflict to tx1_conflict, so that it gets kicked out of the mempool + raw_tx = alice.createrawtransaction(inputs=[unspents[1]], outputs=[{carol.getnewaddress() : 24.9895}]) + tx1_conflict_conflict = alice.signrawtransactionwithwallet(raw_tx)['hex'] + tx1_conflict_conflict_txid = alice.sendrawtransaction(tx1_conflict_conflict) + + self.sync_mempools() + + # Now that tx1_conflict has been removed, both tx1 and tx1_child + assert_equal(bob.gettransaction(tx1_txid)["mempoolconflicts"], []) + assert_equal(bob.gettransaction(tx1_child_txid)["mempoolconflicts"], []) + + # Both tx1 and tx1_child are still not in the mempool because they have not be re-broadcasted + assert tx1_txid not in bob.getrawmempool() + assert tx1_child_txid not in bob.getrawmempool() + assert tx1_conflict_txid not in bob.getrawmempool() + assert tx1_conflict_conflict_txid in bob.getrawmempool() + assert_equal(len(bob.getrawmempool()), 1) + + assert_equal(alice.getbalance(), 0) + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + assert_equal(carol.getbalances()["mine"]["untrusted_pending"], Decimal("24.98950000")) + + # Both tx1 and tx1_child can now be re-broadcasted + bob.sendrawtransaction(tx1) + bob.sendrawtransaction(tx1_child) + assert_equal(len(bob.getrawmempool()), 3) + + alice.unloadwallet() + bob.unloadwallet() + carol.unloadwallet() + if __name__ == '__main__': TxConflicts().main() diff --git a/test/functional/wallet_createwalletdescriptor.py b/test/functional/wallet_createwalletdescriptor.py new file mode 100755 index 0000000000..18e1703da3 --- /dev/null +++ b/test/functional/wallet_createwalletdescriptor.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test wallet createwalletdescriptor RPC.""" + +from test_framework.descriptors import descsum_create +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) +from test_framework.wallet_util import WalletUnlock + + +class WalletCreateDescriptorTest(BitcoinTestFramework): + def add_options(self, parser): + self.add_wallet_options(parser, descriptors=True, legacy=False) + + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self.test_basic() + self.test_imported_other_keys() + self.test_encrypted() + + def test_basic(self): + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet("blank", blank=True) + wallet = self.nodes[0].get_wallet_rpc("blank") + + xpub_info = def_wallet.gethdkeys(private=True) + xpub = xpub_info[0]["xpub"] + xprv = xpub_info[0]["xprv"] + expected_descs = [] + for desc in def_wallet.listdescriptors()["descriptors"]: + if desc["desc"].startswith("wpkh("): + expected_descs.append(desc["desc"]) + + assert_raises_rpc_error(-5, "Unable to determine which HD key to use from active descriptors. Please specify with 'hdkey'", wallet.createwalletdescriptor, "bech32") + assert_raises_rpc_error(-5, f"Private key for {xpub} is not known", wallet.createwalletdescriptor, type="bech32", hdkey=xpub) + + self.log.info("Test createwalletdescriptor after importing active descriptor to blank wallet") + # Import one active descriptor + assert_equal(wallet.importdescriptors([{"desc": descsum_create(f"pkh({xprv}/44h/2h/0h/0/0/*)"), "timestamp": "now", "active": True}])[0]["success"], True) + assert_equal(len(wallet.listdescriptors()["descriptors"]), 1) + assert_equal(len(wallet.gethdkeys()), 1) + + new_descs = wallet.createwalletdescriptor("bech32")["descs"] + assert_equal(len(new_descs), 2) + assert_equal(len(wallet.gethdkeys()), 1) + assert_equal(new_descs, expected_descs) + + self.log.info("Test descriptor creation options") + old_descs = set([(d["desc"], d["active"], d["internal"]) for d in wallet.listdescriptors(private=True)["descriptors"]]) + wallet.createwalletdescriptor(type="bech32m", internal=False) + curr_descs = set([(d["desc"], d["active"], d["internal"]) for d in wallet.listdescriptors(private=True)["descriptors"]]) + new_descs = list(curr_descs - old_descs) + assert_equal(len(new_descs), 1) + assert_equal(len(wallet.gethdkeys()), 1) + assert_equal(new_descs[0][0], descsum_create(f"tr({xprv}/86h/1h/0h/0/*)")) + assert_equal(new_descs[0][1], True) + assert_equal(new_descs[0][2], False) + + old_descs = curr_descs + wallet.createwalletdescriptor(type="bech32m", internal=True) + curr_descs = set([(d["desc"], d["active"], d["internal"]) for d in wallet.listdescriptors(private=True)["descriptors"]]) + new_descs = list(curr_descs - old_descs) + assert_equal(len(new_descs), 1) + assert_equal(len(wallet.gethdkeys()), 1) + assert_equal(new_descs[0][0], descsum_create(f"tr({xprv}/86h/1h/0h/1/*)")) + assert_equal(new_descs[0][1], True) + assert_equal(new_descs[0][2], True) + + def test_imported_other_keys(self): + self.log.info("Test createwalletdescriptor with multiple keys in active descriptors") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet("multiple_keys") + wallet = self.nodes[0].get_wallet_rpc("multiple_keys") + + wallet_xpub = wallet.gethdkeys()[0]["xpub"] + + xpub_info = def_wallet.gethdkeys(private=True) + xpub = xpub_info[0]["xpub"] + xprv = xpub_info[0]["xprv"] + + assert_equal(wallet.importdescriptors([{"desc": descsum_create(f"wpkh({xprv}/0/0/*)"), "timestamp": "now", "active": True}])[0]["success"], True) + assert_equal(len(wallet.gethdkeys()), 2) + + assert_raises_rpc_error(-5, "Unable to determine which HD key to use from active descriptors. Please specify with 'hdkey'", wallet.createwalletdescriptor, "bech32") + assert_raises_rpc_error(-4, "Descriptor already exists", wallet.createwalletdescriptor, type="bech32m", hdkey=wallet_xpub) + assert_raises_rpc_error(-5, "Unable to parse HD key. Please provide a valid xpub", wallet.createwalletdescriptor, type="bech32m", hdkey=xprv) + + # Able to replace tr() descriptor with other hd key + wallet.createwalletdescriptor(type="bech32m", hdkey=xpub) + + def test_encrypted(self): + self.log.info("Test createwalletdescriptor with encrypted wallets") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet("encrypted", blank=True, passphrase="pass") + wallet = self.nodes[0].get_wallet_rpc("encrypted") + + xpub_info = def_wallet.gethdkeys(private=True) + xprv = xpub_info[0]["xprv"] + + with WalletUnlock(wallet, "pass"): + assert_equal(wallet.importdescriptors([{"desc": descsum_create(f"wpkh({xprv}/0/0/*)"), "timestamp": "now", "active": True}])[0]["success"], True) + assert_equal(len(wallet.gethdkeys()), 1) + + assert_raises_rpc_error(-13, "Error: Please enter the wallet passphrase with walletpassphrase first.", wallet.createwalletdescriptor, type="bech32m") + + with WalletUnlock(wallet, "pass"): + wallet.createwalletdescriptor(type="bech32m") + + + +if __name__ == '__main__': + WalletCreateDescriptorTest().main() diff --git a/test/functional/wallet_fundrawtransaction.py b/test/functional/wallet_fundrawtransaction.py index d886a59ac1..ff4648e638 100755 --- a/test/functional/wallet_fundrawtransaction.py +++ b/test/functional/wallet_fundrawtransaction.py @@ -45,9 +45,8 @@ class RawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.setup_clean_chain = True - # This test isn't testing tx relay. Set whitelist on the peers for - # instant tx relay. - self.extra_args = [['-whitelist=noban@127.0.0.1']] * self.num_nodes + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.rpc_timeout = 90 # to prevent timeouts in `test_transaction_too_large` def skip_test_if_missing_module(self): diff --git a/test/functional/wallet_gethdkeys.py b/test/functional/wallet_gethdkeys.py new file mode 100755 index 0000000000..f09b8c875a --- /dev/null +++ b/test/functional/wallet_gethdkeys.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test wallet gethdkeys RPC.""" + +from test_framework.descriptors import descsum_create +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) +from test_framework.wallet_util import WalletUnlock + + +class WalletGetHDKeyTest(BitcoinTestFramework): + def add_options(self, parser): + self.add_wallet_options(parser, descriptors=True, legacy=False) + + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self.test_basic_gethdkeys() + self.test_ranged_imports() + self.test_lone_key_imports() + self.test_ranged_multisig() + self.test_mixed_multisig() + + def test_basic_gethdkeys(self): + self.log.info("Test gethdkeys basics") + self.nodes[0].createwallet("basic") + wallet = self.nodes[0].get_wallet_rpc("basic") + xpub_info = wallet.gethdkeys() + assert_equal(len(xpub_info), 1) + assert_equal(xpub_info[0]["has_private"], True) + + assert "xprv" not in xpub_info[0] + xpub = xpub_info[0]["xpub"] + + xpub_info = wallet.gethdkeys(private=True) + xprv = xpub_info[0]["xprv"] + assert_equal(xpub_info[0]["xpub"], xpub) + assert_equal(xpub_info[0]["has_private"], True) + + descs = wallet.listdescriptors(True) + for desc in descs["descriptors"]: + assert xprv in desc["desc"] + + self.log.info("HD pubkey can be retrieved from encrypted wallets") + prev_xprv = xprv + wallet.encryptwallet("pass") + # HD key is rotated on encryption, there should now be 2 HD keys + assert_equal(len(wallet.gethdkeys()), 2) + # New key is active, should be able to get only that one and its descriptors + xpub_info = wallet.gethdkeys(active_only=True) + assert_equal(len(xpub_info), 1) + assert xpub_info[0]["xpub"] != xpub + assert "xprv" not in xpub_info[0] + assert_equal(xpub_info[0]["has_private"], True) + + self.log.info("HD privkey can be retrieved from encrypted wallets") + assert_raises_rpc_error(-13, "Error: Please enter the wallet passphrase with walletpassphrase first", wallet.gethdkeys, private=True) + with WalletUnlock(wallet, "pass"): + xpub_info = wallet.gethdkeys(active_only=True, private=True)[0] + assert xpub_info["xprv"] != xprv + for desc in wallet.listdescriptors(True)["descriptors"]: + if desc["active"]: + # After encrypting, HD key was rotated and should appear in all active descriptors + assert xpub_info["xprv"] in desc["desc"] + else: + # Inactive descriptors should have the previous HD key + assert prev_xprv in desc["desc"] + + def test_ranged_imports(self): + self.log.info("Keys of imported ranged descriptors appear in gethdkeys") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet("imports") + wallet = self.nodes[0].get_wallet_rpc("imports") + + xpub_info = wallet.gethdkeys() + assert_equal(len(xpub_info), 1) + active_xpub = xpub_info[0]["xpub"] + + import_xpub = def_wallet.gethdkeys(active_only=True)[0]["xpub"] + desc_import = def_wallet.listdescriptors(True)["descriptors"] + for desc in desc_import: + desc["active"] = False + wallet.importdescriptors(desc_import) + assert_equal(wallet.gethdkeys(active_only=True), xpub_info) + + xpub_info = wallet.gethdkeys() + assert_equal(len(xpub_info), 2) + for x in xpub_info: + if x["xpub"] == active_xpub: + for desc in x["descriptors"]: + assert_equal(desc["active"], True) + elif x["xpub"] == import_xpub: + for desc in x["descriptors"]: + assert_equal(desc["active"], False) + else: + assert False + + + def test_lone_key_imports(self): + self.log.info("Non-HD keys do not appear in gethdkeys") + self.nodes[0].createwallet("lonekey", blank=True) + wallet = self.nodes[0].get_wallet_rpc("lonekey") + + assert_equal(wallet.gethdkeys(), []) + wallet.importdescriptors([{"desc": descsum_create("wpkh(cTe1f5rdT8A8DFgVWTjyPwACsDPJM9ff4QngFxUixCSvvbg1x6sh)"), "timestamp": "now"}]) + assert_equal(wallet.gethdkeys(), []) + + self.log.info("HD keys of non-ranged descriptors should appear in gethdkeys") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + xpub_info = def_wallet.gethdkeys(private=True) + xpub = xpub_info[0]["xpub"] + xprv = xpub_info[0]["xprv"] + prv_desc = descsum_create(f"wpkh({xprv})") + pub_desc = descsum_create(f"wpkh({xpub})") + assert_equal(wallet.importdescriptors([{"desc": prv_desc, "timestamp": "now"}])[0]["success"], True) + xpub_info = wallet.gethdkeys() + assert_equal(len(xpub_info), 1) + assert_equal(xpub_info[0]["xpub"], xpub) + assert_equal(len(xpub_info[0]["descriptors"]), 1) + assert_equal(xpub_info[0]["descriptors"][0]["desc"], pub_desc) + assert_equal(xpub_info[0]["descriptors"][0]["active"], False) + + def test_ranged_multisig(self): + self.log.info("HD keys of a multisig appear in gethdkeys") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet("ranged_multisig") + wallet = self.nodes[0].get_wallet_rpc("ranged_multisig") + + xpub1 = wallet.gethdkeys()[0]["xpub"] + xprv1 = wallet.gethdkeys(private=True)[0]["xprv"] + xpub2 = def_wallet.gethdkeys()[0]["xpub"] + + prv_multi_desc = descsum_create(f"wsh(multi(2,{xprv1}/*,{xpub2}/*))") + pub_multi_desc = descsum_create(f"wsh(multi(2,{xpub1}/*,{xpub2}/*))") + assert_equal(wallet.importdescriptors([{"desc": prv_multi_desc, "timestamp": "now"}])[0]["success"], True) + + xpub_info = wallet.gethdkeys() + assert_equal(len(xpub_info), 2) + for x in xpub_info: + if x["xpub"] == xpub1: + found_desc = next((d for d in xpub_info[0]["descriptors"] if d["desc"] == pub_multi_desc), None) + assert found_desc is not None + assert_equal(found_desc["active"], False) + elif x["xpub"] == xpub2: + assert_equal(len(x["descriptors"]), 1) + assert_equal(x["descriptors"][0]["desc"], pub_multi_desc) + assert_equal(x["descriptors"][0]["active"], False) + else: + assert False + + def test_mixed_multisig(self): + self.log.info("Non-HD keys of a multisig do not appear in gethdkeys") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet("single_multisig") + wallet = self.nodes[0].get_wallet_rpc("single_multisig") + + xpub = wallet.gethdkeys()[0]["xpub"] + xprv = wallet.gethdkeys(private=True)[0]["xprv"] + pub = def_wallet.getaddressinfo(def_wallet.getnewaddress())["pubkey"] + + prv_multi_desc = descsum_create(f"wsh(multi(2,{xprv},{pub}))") + pub_multi_desc = descsum_create(f"wsh(multi(2,{xpub},{pub}))") + import_res = wallet.importdescriptors([{"desc": prv_multi_desc, "timestamp": "now"}]) + assert_equal(import_res[0]["success"], True) + + xpub_info = wallet.gethdkeys() + assert_equal(len(xpub_info), 1) + assert_equal(xpub_info[0]["xpub"], xpub) + found_desc = next((d for d in xpub_info[0]["descriptors"] if d["desc"] == pub_multi_desc), None) + assert found_desc is not None + assert_equal(found_desc["active"], False) + + +if __name__ == '__main__': + WalletGetHDKeyTest().main() diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index bdb9081261..26477131cf 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -22,6 +22,8 @@ class WalletGroupTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 5 + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.extra_args = [ [], [], @@ -31,7 +33,6 @@ class WalletGroupTest(BitcoinTestFramework): ] for args in self.extra_args: - args.append("-whitelist=noban@127.0.0.1") # whitelist peers to speed up tx relay / mempool sync args.append(f"-paytxfee={20 * 1e3 / 1e8}") # apply feerate of 20 sats/vB across all nodes self.rpc_timeout = 480 @@ -41,11 +42,6 @@ class WalletGroupTest(BitcoinTestFramework): def run_test(self): self.log.info("Setting up") - # To take full use of immediate tx relay, all nodes need to be reachable - # via inbound peers, i.e. connect first to last to close the circle - # (the default test network topology looks like this: - # node0 <-- node1 <-- node2 <-- node3 <-- node4 <-- node5) - self.connect_nodes(0, self.num_nodes - 1) # Mine some coins self.generate(self.nodes[0], COINBASE_MATURITY + 1) diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 0f4b7cfcb1..52161043ea 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -23,8 +23,7 @@ class WalletHDTest(BitcoinTestFramework): self.num_nodes = 2 self.extra_args = [[], ['-keypool=0']] # whitelist peers to speed up tx relay / mempool sync - for args in self.extra_args: - args.append("-whitelist=noban@127.0.0.1") + self.noban_tx_relay = True self.supports_cli = False diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index e647fb2d5c..2a9435b370 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -160,6 +160,8 @@ class ImportRescanTest(BitcoinTestFramework): self.num_nodes = 2 + len(IMPORT_NODES) self.supports_cli = False self.rpc_timeout = 120 + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -177,7 +179,7 @@ class ImportRescanTest(BitcoinTestFramework): self.import_deterministic_coinbase_privkeys() self.stop_nodes() - self.start_nodes(extra_args=[["-whitelist=noban@127.0.0.1"]] * self.num_nodes) + self.start_nodes() for i in range(1, self.num_nodes): self.connect_nodes(i, 0) diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index 1f1f92589c..f9d05a2fe4 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -36,12 +36,11 @@ class ImportDescriptorsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.extra_args = [["-addresstype=legacy"], ["-addresstype=bech32", "-keypool=5"] ] - # whitelist peers to speed up tx relay / mempool sync - for args in self.extra_args: - args.append("-whitelist=noban@127.0.0.1") self.setup_clean_chain = True self.wallet_names = [] @@ -689,7 +688,7 @@ class ImportDescriptorsTest(BitcoinTestFramework): encrypted_wallet.walletpassphrase("passphrase", 99999) with concurrent.futures.ThreadPoolExecutor(max_workers=1) as thread: - with self.nodes[0].assert_debug_log(expected_msgs=["Rescan started from block 0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206... (slow variant inspecting all blocks)"], timeout=5): + with self.nodes[0].assert_debug_log(expected_msgs=["Rescan started from block 0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206... (slow variant inspecting all blocks)"], timeout=10): importing = thread.submit(encrypted_wallet.importdescriptors, requests=[descriptor]) # Set the passphrase timeout to 1 to test that the wallet remains unlocked during the rescan diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index 48180e8294..e1bd85d8a9 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -25,8 +25,10 @@ class KeypoolRestoreTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 4 - self.extra_args = [[], ['-keypool=100'], ['-keypool=100'], ['-keypool=100']] + self.num_nodes = 5 + self.extra_args = [[]] + for _ in range(self.num_nodes - 1): + self.extra_args.append(['-keypool=100']) def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -40,12 +42,13 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.stop_node(1) shutil.copyfile(wallet_path, wallet_backup_path) self.start_node(1, self.extra_args[1]) - self.connect_nodes(0, 1) - self.connect_nodes(0, 2) - self.connect_nodes(0, 3) - - for i, output_type in enumerate(["legacy", "p2sh-segwit", "bech32"]): + for i in [1, 2, 3, 4]: + self.connect_nodes(0, i) + output_types = ["legacy", "p2sh-segwit", "bech32"] + if self.options.descriptors: + output_types.append("bech32m") + for i, output_type in enumerate(output_types): self.log.info("Generate keys for wallet with address type: {}".format(output_type)) idx = i+1 for _ in range(90): @@ -59,9 +62,10 @@ class KeypoolRestoreTest(BitcoinTestFramework): assert not address_details["isscript"] and not address_details["iswitness"] elif i == 1: assert address_details["isscript"] and not address_details["iswitness"] - else: + elif i == 2: assert not address_details["isscript"] and address_details["iswitness"] - + elif i == 3: + assert address_details["isscript"] and address_details["iswitness"] self.log.info("Send funds to wallet") self.nodes[0].sendtoaddress(addr_oldpool, 10) @@ -87,6 +91,8 @@ class KeypoolRestoreTest(BitcoinTestFramework): assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/49h/1h/0h/0/110") elif output_type == 'bech32': assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/84h/1h/0h/0/110") + elif output_type == 'bech32m': + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/86h/1h/0h/0/110") else: assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/0'/0'/110'") diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index 8ec21484d1..d0f1336a5e 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -22,7 +22,7 @@ class ReceivedByTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 # whitelist peers to speed up tx relay / mempool sync - self.extra_args = [["-whitelist=noban@127.0.0.1"]] * self.num_nodes + self.noban_tx_relay = True def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index a19a3ac2cb..fd586d546e 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -26,7 +26,7 @@ class ListSinceBlockTest(BitcoinTestFramework): self.num_nodes = 4 self.setup_clean_chain = True # whitelist peers to speed up tx relay / mempool sync - self.extra_args = [["-whitelist=noban@127.0.0.1"]] * self.num_nodes + self.noban_tx_relay = True def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 064ce12108..c820eaa6f6 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -26,9 +26,9 @@ class ListTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 3 - # This test isn't testing txn relay/timing, so set whitelist on the - # peers for instant txn relay. This speeds up the test run time 2-3x. - self.extra_args = [["-whitelist=noban@127.0.0.1", "-walletrbf=0"]] * self.num_nodes + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True + self.extra_args = [["-walletrbf=0"]] * self.num_nodes def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_migration.py b/test/functional/wallet_migration.py index f9919716be..890b6a5c1b 100755 --- a/test/functional/wallet_migration.py +++ b/test/functional/wallet_migration.py @@ -529,11 +529,20 @@ class WalletMigrationTest(BitcoinTestFramework): self.log.info("Test migration of the wallet named as the empty string") wallet = self.create_legacy_wallet("") - self.migrate_wallet(wallet) + # Set time to verify backup existence later + curr_time = int(time.time()) + wallet.setmocktime(curr_time) + + res = self.migrate_wallet(wallet) info = wallet.getwalletinfo() assert_equal(info["descriptors"], True) assert_equal(info["format"], "sqlite") + # Check backup existence and its non-empty wallet filename + backup_path = self.nodes[0].wallets_path / f'default_wallet_{curr_time}.legacy.bak' + assert backup_path.exists() + assert_equal(str(backup_path), res['backup_path']) + def test_direct_file(self): self.log.info("Test migration of a wallet that is not in a wallet directory") wallet = self.create_legacy_wallet("plainfile") diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 580a9d2b22..0a0a8dba0d 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -9,10 +9,6 @@ from itertools import product from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create -from test_framework.messages import ( - ser_compact_size, - WITNESS_SCALE_FACTOR, -) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -21,7 +17,10 @@ from test_framework.util import ( assert_raises_rpc_error, count_bytes, ) -from test_framework.wallet_util import generate_keypair +from test_framework.wallet_util import ( + calculate_input_weight, + generate_keypair, +) class WalletSendTest(BitcoinTestFramework): @@ -30,10 +29,11 @@ class WalletSendTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - # whitelist all peers to speed up tx relay / mempool sync + # whitelist peers to speed up tx relay / mempool sync + self.noban_tx_relay = True self.extra_args = [ - ["-whitelist=127.0.0.1","-walletrbf=1"], - ["-whitelist=127.0.0.1","-walletrbf=1"], + ["-walletrbf=1"], + ["-walletrbf=1"] ] getcontext().prec = 8 # Satoshi precision for Decimal @@ -542,17 +542,9 @@ class WalletSendTest(BitcoinTestFramework): input_idx = i break psbt_in = dec["inputs"][input_idx] - # Calculate the input weight - # (prevout + sequence + length of scriptSig + scriptsig) * WITNESS_SCALE_FACTOR + len of num scriptWitness stack items + (length of stack item + stack item) * N stack items - # Note that occasionally this weight estimate may be slightly larger or smaller than the real weight - # as sometimes ECDSA signatures are one byte shorter than expected with a probability of 1/128 - len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptsig += len(ser_compact_size(len_scriptsig)) - len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(ser_compact_size(len(psbt_in["final_scriptwitness"])))) if "final_scriptwitness" in psbt_in else 0 - len_prevout_txid = 32 - len_prevout_index = 4 - len_sequence = 4 - input_weight = ((len_prevout_txid + len_prevout_index + len_sequence + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness + scriptsig_hex = psbt_in["final_scriptSig"]["hex"] if "final_scriptSig" in psbt_in else "" + witness_stack_hex = psbt_in["final_scriptwitness"] if "final_scriptwitness" in psbt_in else None + input_weight = calculate_input_weight(scriptsig_hex, witness_stack_hex) # Input weight error conditions assert_raises_rpc_error( diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 32a1887153..abfc3c1ba1 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -130,8 +130,9 @@ class WalletSignerTest(BitcoinTestFramework): assert_equal(address_info['hdkeypath'], "m/86h/1h/0h/0/0") self.log.info('Test walletdisplayaddress') - result = hww.walletdisplayaddress(address1) - assert_equal(result, {"address": address1}) + for address in [address1, address2, address3]: + result = hww.walletdisplayaddress(address) + assert_equal(result, {"address": address}) # Handle error thrown by script self.set_mock_result(self.nodes[1], "2") @@ -140,6 +141,13 @@ class WalletSignerTest(BitcoinTestFramework): ) self.clear_mock_result(self.nodes[1]) + # Returned address MUST match: + address_fail = hww.getnewaddress(address_type="bech32") + assert_equal(address_fail, "bcrt1ql7zg7ukh3dwr25ex2zn9jse926f27xy2jz58tm") + assert_raises_rpc_error(-1, 'Signer echoed unexpected address wrong_address', + hww.walletdisplayaddress, address_fail + ) + self.log.info('Prepare mock PSBT') self.nodes[0].sendtoaddress(address4, 1) self.generate(self.nodes[0], 1) diff --git a/test/functional/wallet_signrawtransactionwithwallet.py b/test/functional/wallet_signrawtransactionwithwallet.py index b0517f951d..612a2542e7 100755 --- a/test/functional/wallet_signrawtransactionwithwallet.py +++ b/test/functional/wallet_signrawtransactionwithwallet.py @@ -55,7 +55,7 @@ class SignRawTransactionWithWalletTest(BitcoinTestFramework): def test_with_invalid_sighashtype(self): self.log.info("Test signrawtransactionwithwallet raises if an invalid sighashtype is passed") - assert_raises_rpc_error(-8, "all is not a valid sighash parameter.", self.nodes[0].signrawtransactionwithwallet, hexstring=RAW_TX, sighashtype="all") + assert_raises_rpc_error(-8, "'all' is not a valid sighash parameter.", self.nodes[0].signrawtransactionwithwallet, hexstring=RAW_TX, sighashtype="all") def script_verification_error_test(self): """Create and sign a raw transaction with valid (vin 0), invalid (vin 1) and one missing (vin 2) input script. diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index b3edb0e253..558d63e85c 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -104,9 +104,11 @@ def main(): logging.error("Must have fuzz executable built") sys.exit(1) + fuzz_bin=os.getenv("BITCOINFUZZ", default=os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz')) + # Build list of tests test_list_all = parse_test_list( - fuzz_bin=os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'), + fuzz_bin=fuzz_bin, source_dir=config['environment']['SRCDIR'], ) @@ -151,7 +153,7 @@ def main(): try: help_output = subprocess.run( args=[ - os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'), + fuzz_bin, '-help=1', ], env=get_fuzz_env(target=test_list_selection[0], source_dir=config['environment']['SRCDIR']), @@ -173,7 +175,7 @@ def main(): return generate_corpus( fuzz_pool=fuzz_pool, src_dir=config['environment']['SRCDIR'], - build_dir=config["environment"]["BUILDDIR"], + fuzz_bin=fuzz_bin, corpus_dir=args.corpus_dir, targets=test_list_selection, ) @@ -184,7 +186,7 @@ def main(): corpus=args.corpus_dir, test_list=test_list_selection, src_dir=config['environment']['SRCDIR'], - build_dir=config["environment"]["BUILDDIR"], + fuzz_bin=fuzz_bin, merge_dirs=[Path(m_dir) for m_dir in args.m_dir], ) return @@ -194,7 +196,7 @@ def main(): corpus=args.corpus_dir, test_list=test_list_selection, src_dir=config['environment']['SRCDIR'], - build_dir=config["environment"]["BUILDDIR"], + fuzz_bin=fuzz_bin, using_libfuzzer=using_libfuzzer, use_valgrind=args.valgrind, empty_min_time=args.empty_min_time, @@ -237,7 +239,7 @@ def transform_rpc_target(targets, src_dir): return targets -def generate_corpus(*, fuzz_pool, src_dir, build_dir, corpus_dir, targets): +def generate_corpus(*, fuzz_pool, src_dir, fuzz_bin, corpus_dir, targets): """Generates new corpus. Run {targets} without input, and outputs the generated corpus to @@ -270,7 +272,7 @@ def generate_corpus(*, fuzz_pool, src_dir, build_dir, corpus_dir, targets): os.makedirs(target_corpus_dir, exist_ok=True) use_value_profile = int(random.random() < .3) command = [ - os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), + fuzz_bin, "-rss_limit_mb=8000", "-max_total_time=6000", "-reload=0", @@ -283,12 +285,12 @@ def generate_corpus(*, fuzz_pool, src_dir, build_dir, corpus_dir, targets): future.result() -def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, build_dir, merge_dirs): +def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, merge_dirs): logging.info(f"Merge the inputs from the passed dir into the corpus_dir. Passed dirs {merge_dirs}") jobs = [] for t in test_list: args = [ - os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), + fuzz_bin, '-rss_limit_mb=8000', '-set_cover_merge=1', # set_cover_merge is used instead of -merge=1 to reduce the overall @@ -325,13 +327,13 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, build_dir, merge_dirs future.result() -def run_once(*, fuzz_pool, corpus, test_list, src_dir, build_dir, using_libfuzzer, use_valgrind, empty_min_time): +def run_once(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, using_libfuzzer, use_valgrind, empty_min_time): jobs = [] for t in test_list: corpus_path = corpus / t os.makedirs(corpus_path, exist_ok=True) args = [ - os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), + fuzz_bin, ] empty_dir = not any(corpus_path.iterdir()) if using_libfuzzer: diff --git a/test/lint/README.md b/test/lint/README.md index 9cb61b484c..9d167bac72 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -16,7 +16,11 @@ result is cached and it prevents issues when the image changes. test runner =========== -To run all the lint checks in the test runner outside the docker, use: +To run all the lint checks in the test runner outside the docker you first need +to install the rust toolchain using your package manager of choice or +[rustup](https://www.rust-lang.org/tools/install). + +Then you can use: ```sh ( cd ./test/lint/test_runner/ && cargo fmt && cargo clippy && RUST_BACKTRACE=1 cargo run ) @@ -83,3 +87,7 @@ To do so, add the upstream repository as remote: ``` git remote add --fetch secp256k1 https://github.com/bitcoin-core/secp256k1.git ``` + +lint_ignore_dirs.py +=================== +Add list of common directories to ignore when running tests diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index 55c9528dea..fe845ed19e 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -22,6 +22,11 @@ if ! sed --help 2>&1 | grep -q 'GNU'; then exit 1; fi +if ! grep --help 2>&1 | grep -q 'GNU'; then + echo "Error: the installed grep package is not compatible. Please make sure you have GNU grep installed in your system."; + exit 1; +fi + RET=0 PREV_BRANCH=$(git name-rev --name-only HEAD) PREV_HEAD=$(git rev-parse HEAD) diff --git a/test/lint/lint-git-commit-check.py b/test/lint/lint-git-commit-check.py index 5897a17e70..5dc30cc755 100755 --- a/test/lint/lint-git-commit-check.py +++ b/test/lint/lint-git-commit-check.py @@ -23,31 +23,18 @@ def parse_args(): """, epilog=f""" You can manually set the commit-range with the COMMIT_RANGE - environment variable (e.g. "COMMIT_RANGE='47ba2c3...ee50c9e' - {sys.argv[0]}"). Defaults to current merge base when neither - prev-commits nor the environment variable is set. + environment variable (e.g. "COMMIT_RANGE='HEAD~n..HEAD' + {sys.argv[0]}") for the last 'n' commits. """) - - parser.add_argument("--prev-commits", "-p", required=False, help="The previous n commits to check") - return parser.parse_args() def main(): - args = parse_args() + parse_args() exit_code = 0 - if not os.getenv("COMMIT_RANGE"): - if args.prev_commits: - commit_range = "HEAD~" + args.prev_commits + "...HEAD" - else: - # This assumes that the target branch of the pull request will be master. - merge_base = check_output(["git", "merge-base", "HEAD", "master"], text=True, encoding="utf8").rstrip("\n") - commit_range = merge_base + "..HEAD" - else: - commit_range = os.getenv("COMMIT_RANGE") - if commit_range == "SKIP_EMPTY_NOT_A_PR": - sys.exit(0) + assert os.getenv("COMMIT_RANGE") # E.g. COMMIT_RANGE='HEAD~n..HEAD' + commit_range = os.getenv("COMMIT_RANGE") commit_hashes = check_output(["git", "log", commit_range, "--format=%H"], text=True, encoding="utf8").splitlines() diff --git a/test/lint/lint-include-guards.py b/test/lint/lint-include-guards.py index 291e528c1d..77af05c1c2 100755 --- a/test/lint/lint-include-guards.py +++ b/test/lint/lint-include-guards.py @@ -12,19 +12,17 @@ import re import sys from subprocess import check_output +from lint_ignore_dirs import SHARED_EXCLUDED_SUBTREES + HEADER_ID_PREFIX = 'BITCOIN_' HEADER_ID_SUFFIX = '_H' EXCLUDE_FILES_WITH_PREFIX = ['contrib/devtools/bitcoin-tidy', 'src/crypto/ctaes', - 'src/leveldb', - 'src/crc32c', - 'src/secp256k1', - 'src/minisketch', 'src/tinyformat.h', 'src/bench/nanobench.h', - 'src/test/fuzz/FuzzedDataProvider.h'] + 'src/test/fuzz/FuzzedDataProvider.h'] + SHARED_EXCLUDED_SUBTREES def _get_header_file_lst() -> list[str]: diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py index 6386a92c0f..90884299d5 100755 --- a/test/lint/lint-includes.py +++ b/test/lint/lint-includes.py @@ -14,13 +14,11 @@ import sys from subprocess import check_output, CalledProcessError +from lint_ignore_dirs import SHARED_EXCLUDED_SUBTREES + EXCLUDED_DIRS = ["contrib/devtools/bitcoin-tidy/", - "src/leveldb/", - "src/crc32c/", - "src/secp256k1/", - "src/minisketch/", - ] + ] + SHARED_EXCLUDED_SUBTREES EXPECTED_BOOST_INCLUDES = ["boost/date_time/posix_time/posix_time.hpp", "boost/multi_index/detail/hash_index_iterator.hpp", @@ -32,7 +30,6 @@ EXPECTED_BOOST_INCLUDES = ["boost/date_time/posix_time/posix_time.hpp", "boost/multi_index/tag.hpp", "boost/multi_index_container.hpp", "boost/operators.hpp", - "boost/process.hpp", "boost/signals2/connection.hpp", "boost/signals2/optional_last_value.hpp", "boost/signals2/signal.hpp", diff --git a/test/lint/lint-spelling.py b/test/lint/lint-spelling.py index ac0bddeaa6..3e578b218f 100755 --- a/test/lint/lint-spelling.py +++ b/test/lint/lint-spelling.py @@ -11,8 +11,11 @@ Note: Will exit successfully regardless of spelling errors. from subprocess import check_output, STDOUT, CalledProcessError +from lint_ignore_dirs import SHARED_EXCLUDED_SUBTREES + IGNORE_WORDS_FILE = 'test/lint/spelling.ignore-words.txt' -FILES_ARGS = ['git', 'ls-files', '--', ":(exclude)build-aux/m4/", ":(exclude)contrib/seeds/*.txt", ":(exclude)depends/", ":(exclude)doc/release-notes/", ":(exclude)src/leveldb/", ":(exclude)src/crc32c/", ":(exclude)src/qt/locale/", ":(exclude)src/qt/*.qrc", ":(exclude)src/secp256k1/", ":(exclude)src/minisketch/", ":(exclude)contrib/guix/patches"] +FILES_ARGS = ['git', 'ls-files', '--', ":(exclude)build-aux/m4/", ":(exclude)contrib/seeds/*.txt", ":(exclude)depends/", ":(exclude)doc/release-notes/", ":(exclude)src/qt/locale/", ":(exclude)src/qt/*.qrc", ":(exclude)contrib/guix/patches"] +FILES_ARGS += [f":(exclude){dir}" for dir in SHARED_EXCLUDED_SUBTREES] def check_codespell_install(): diff --git a/test/lint/lint-whitespace.py b/test/lint/lint-whitespace.py deleted file mode 100755 index f5e4a776d0..0000000000 --- a/test/lint/lint-whitespace.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) 2017-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for new lines in diff that introduce trailing whitespace or -# tab characters instead of spaces. - -# We can't run this check unless we know the commit range for the PR. - -import argparse -import os -import re -import sys - -from subprocess import check_output - -EXCLUDED_DIRS = ["depends/patches/", - "contrib/guix/patches/", - "src/leveldb/", - "src/crc32c/", - "src/secp256k1/", - "src/minisketch/", - "doc/release-notes/", - "src/qt/locale"] - -def parse_args(): - """Parse command line arguments.""" - parser = argparse.ArgumentParser( - description=""" - Check for new lines in diff that introduce trailing whitespace - or tab characters instead of spaces in unstaged changes, the - previous n commits, or a commit-range. - """, - epilog=f""" - You can manually set the commit-range with the COMMIT_RANGE - environment variable (e.g. "COMMIT_RANGE='47ba2c3...ee50c9e' - {sys.argv[0]}"). Defaults to current merge base when neither - prev-commits nor the environment variable is set. - """) - - parser.add_argument("--prev-commits", "-p", required=False, help="The previous n commits to check") - - return parser.parse_args() - - -def report_diff(selection): - filename = "" - seen = False - seenln = False - - print("The following changes were suspected:") - - for line in selection: - if re.match(r"^diff", line): - filename = line - seen = False - elif re.match(r"^@@", line): - linenumber = line - seenln = False - else: - if not seen: - # The first time a file is seen with trailing whitespace or a tab character, we print the - # filename (preceded by a newline). - print("") - print(filename) - seen = True - if not seenln: - print(linenumber) - seenln = True - print(line) - - -def get_diff(commit_range, check_only_code): - exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] - - if check_only_code: - what_files = ["*.cpp", "*.h", "*.md", "*.py", "*.sh"] - else: - what_files = ["."] - - diff = check_output(["git", "diff", "-U0", commit_range, "--"] + what_files + exclude_args, text=True, encoding="utf8") - - return diff - - -def main(): - args = parse_args() - - if not os.getenv("COMMIT_RANGE"): - if args.prev_commits: - commit_range = "HEAD~" + args.prev_commits + "...HEAD" - else: - # This assumes that the target branch of the pull request will be master. - merge_base = check_output(["git", "merge-base", "HEAD", "master"], text=True, encoding="utf8").rstrip("\n") - commit_range = merge_base + "..HEAD" - else: - commit_range = os.getenv("COMMIT_RANGE") - if commit_range == "SKIP_EMPTY_NOT_A_PR": - sys.exit(0) - - whitespace_selection = [] - tab_selection = [] - - # Check if trailing whitespace was found in the diff. - for line in get_diff(commit_range, check_only_code=False).splitlines(): - if re.match(r"^(diff --git|\@@|^\+.*\s+$)", line): - whitespace_selection.append(line) - - whitespace_additions = [i for i in whitespace_selection if i.startswith("+")] - - # Check if tab characters were found in the diff. - for line in get_diff(commit_range, check_only_code=True).splitlines(): - if re.match(r"^(diff --git|\@@|^\+.*\t)", line): - tab_selection.append(line) - - tab_additions = [i for i in tab_selection if i.startswith("+")] - - ret = 0 - - if len(whitespace_additions) > 0: - print("This diff appears to have added new lines with trailing whitespace.") - report_diff(whitespace_selection) - ret = 1 - - if len(tab_additions) > 0: - print("This diff appears to have added new lines with tab characters instead of spaces.") - report_diff(tab_selection) - ret = 1 - - sys.exit(ret) - - -if __name__ == "__main__": - main() diff --git a/test/lint/lint_ignore_dirs.py b/test/lint/lint_ignore_dirs.py new file mode 100644 index 0000000000..af9ee7ef6b --- /dev/null +++ b/test/lint/lint_ignore_dirs.py @@ -0,0 +1,5 @@ +SHARED_EXCLUDED_SUBTREES = ["src/leveldb/", + "src/crc32c/", + "src/secp256k1/", + "src/minisketch/", + ] diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index b97e822484..e22e047e4b 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -14,7 +14,9 @@ type LintFn = fn() -> LintResult; /// Return the git command fn git() -> Command { - Command::new("git") + let mut git = Command::new("git"); + git.arg("--no-pager"); + git } /// Return stdout @@ -95,6 +97,85 @@ fs:: namespace, which has unsafe filesystem functions marked as deleted. } } +/// Return the pathspecs for whitespace related excludes +fn get_pathspecs_exclude_whitespace() -> Vec<String> { + let mut list = get_pathspecs_exclude_subtrees(); + list.extend( + [ + // Permanent excludes + "*.patch", + "src/qt/locale", + "contrib/windeploy/win-codesign.cert", + "doc/README_windows.txt", + // Temporary excludes, or existing violations + "doc/release-notes/release-notes-0.*", + "contrib/init/bitcoind.openrc", + "contrib/macdeploy/macdeployqtplus", + "src/crypto/sha256_sse4.cpp", + "src/qt/res/src/*.svg", + "test/functional/test_framework/crypto/ellswift_decode_test_vectors.csv", + "test/functional/test_framework/crypto/xswiftec_inv_test_vectors.csv", + "contrib/qos/tc.sh", + "contrib/verify-commits/gpg.sh", + "src/univalue/include/univalue_escapes.h", + "src/univalue/test/object.cpp", + "test/lint/git-subtree-check.sh", + ] + .iter() + .map(|s| format!(":(exclude){}", s)), + ); + list +} + +fn lint_trailing_whitespace() -> LintResult { + let trailing_space = git() + .args(["grep", "-I", "--line-number", "\\s$", "--"]) + .args(get_pathspecs_exclude_whitespace()) + .status() + .expect("command error") + .success(); + if trailing_space { + Err(r#" +^^^ +Trailing whitespace is problematic, because git may warn about it, or editors may remove it by +default, forcing developers in the future to either undo the changes manually or spend time on +review. + +Thus, it is best to remove the trailing space now. + +Please add any false positives, such as subtrees, Windows-related files, patch files, or externally +sourced files to the exclude list. + "# + .to_string()) + } else { + Ok(()) + } +} + +fn lint_tabs_whitespace() -> LintResult { + let tabs = git() + .args(["grep", "-I", "--line-number", "--perl-regexp", "^\\t", "--"]) + .args(["*.cpp", "*.h", "*.md", "*.py", "*.sh"]) + .args(get_pathspecs_exclude_whitespace()) + .status() + .expect("command error") + .success(); + if tabs { + Err(r#" +^^^ +Use of tabs in this codebase is problematic, because existing code uses spaces and tabs will cause +display issues and conflict with editor settings. + +Please remove the tabs. + +Please add any false positives, such as subtrees, or externally sourced files to the exclude list. + "# + .to_string()) + } else { + Ok(()) + } +} + fn lint_includes_build_config() -> LintResult { let config_path = "./src/config/bitcoin-config.h.in"; let include_directive = "#include <config/bitcoin-config.h>"; @@ -232,6 +313,8 @@ fn main() -> ExitCode { let test_list: Vec<(&str, LintFn)> = vec![ ("subtree check", lint_subtree), ("std::filesystem check", lint_std_filesystem), + ("trailing whitespace check", lint_trailing_whitespace), + ("no-tabs check", lint_tabs_whitespace), ("build config includes check", lint_includes_build_config), ("-help=1 documentation check", lint_doc), ("lint-*.py scripts", lint_all), diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 2a2f7ca470..482667a26a 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -51,6 +51,7 @@ unsigned-integer-overflow:CCoinsViewCache::Uncache unsigned-integer-overflow:CompressAmount unsigned-integer-overflow:DecompressAmount unsigned-integer-overflow:crypto/ +unsigned-integer-overflow:getchaintxstats* unsigned-integer-overflow:MurmurHash3 unsigned-integer-overflow:CBlockPolicyEstimator::processBlockTx unsigned-integer-overflow:TxConfirmStats::EstimateMedianVal @@ -61,6 +62,7 @@ implicit-integer-sign-change:CBlockPolicyEstimator::processBlockTx implicit-integer-sign-change:SetStdinEcho implicit-integer-sign-change:compressor.h implicit-integer-sign-change:crypto/ +implicit-integer-sign-change:getchaintxstats* implicit-integer-sign-change:TxConfirmStats::removeTx implicit-integer-sign-change:prevector.h implicit-integer-sign-change:verify_flags |