diff options
236 files changed, 6172 insertions, 2311 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b2f4f2913..b92e3c66d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD ^$(git rev-list -n1 --merges HEAD)^@ | head -1)" >> "$GITHUB_ENV" - run: | sudo apt-get update - sudo apt-get install clang-15 ccache build-essential libtool autotools-dev automake pkg-config bsdmainutils python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libnatpmp-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y + sudo apt-get install clang-15 ccache build-essential libtool autotools-dev automake pkg-config bsdmainutils python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libnatpmp-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y - name: Compile and run tests run: | # Run tests on commits after the last merge commit and before the PR head commit diff --git a/.gitignore b/.gitignore index 3fe36aba89..f7bcbd1459 100644 --- a/.gitignore +++ b/.gitignore @@ -136,7 +136,6 @@ test/lint/test_runner/target/ /doc/doxygen/ -libbitcoinconsensus.pc contrib/devtools/split-debug.sh # Output from running db4 installation @@ -145,7 +144,6 @@ db4/ # clang-check *.plist -osx_volname dist/ /guix-build-* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ae4ff1a92..0ac6db76ed 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,9 +66,10 @@ Discussion about codebase improvements happens in GitHub issues and pull requests. The developer -[mailing list](https://lists.linuxfoundation.org/mailman/listinfo/bitcoin-dev) +[mailing list](https://groups.google.com/g/bitcoindev) should be used to discuss complicated or controversial consensus or P2P protocol changes before working on a patch set. +Archives can be found on [https://gnusha.org/pi/bitcoindev/](https://gnusha.org/pi/bitcoindev/). Contributor Workflow diff --git a/Makefile.am b/Makefile.am index 5ea690dec8..cd1509a0ff 100644 --- a/Makefile.am +++ b/Makefile.am @@ -14,11 +14,6 @@ endif .PHONY: deploy FORCE .INTERMEDIATE: $(COVERAGE_INFO) -if BUILD_BITCOIN_LIBS -pkgconfigdir = $(libdir)/pkgconfig -pkgconfig_DATA = libbitcoinconsensus.pc -endif - BITCOIND_BIN=$(top_builddir)/src/$(BITCOIN_DAEMON_NAME)$(EXEEXT) BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT) BITCOIN_TEST_BIN=$(top_builddir)/src/test/$(BITCOIN_TEST_NAME)$(EXEEXT) @@ -118,9 +113,6 @@ OSX_APP_BUILT=$(OSX_APP)/Contents/PkgInfo $(OSX_APP)/Contents/Resources/empty.lp $(OSX_APP)/Contents/Resources/bitcoin.icns $(OSX_APP)/Contents/Info.plist \ $(OSX_APP)/Contents/MacOS/Bitcoin-Qt $(OSX_APP)/Contents/Resources/Base.lproj/InfoPlist.strings -osx_volname: - echo $(OSX_VOLNAME) >$@ - if BUILD_DARWIN $(OSX_ZIP): $(OSX_APP_BUILT) $(OSX_PACKAGING) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -zip @@ -338,7 +330,7 @@ clean-docs: clean-local: clean-docs rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ fuzz.coverage/ test/tmp/ cache/ $(OSX_APP) rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__ - rm -rf osx_volname dist/ test/lint/test_runner/target/ test/lint/__pycache__ + rm -rf dist/ test/lint/test_runner/target/ test/lint/__pycache__ test-security-check: if TARGET_DARWIN diff --git a/build-aux/m4/l_atomic.m4 b/build-aux/m4/l_atomic.m4 index aa00168fce..859ddaabbb 100644 --- a/build-aux/m4/l_atomic.m4 +++ b/build-aux/m4/l_atomic.m4 @@ -7,7 +7,7 @@ dnl warranty. # Clang, when building for 32-bit, # and linking against libstdc++, requires linking with # -latomic if using the C++ atomic library. -# Can be tested with: clang++ test.cpp -m32 +# Can be tested with: clang++ -std=c++20 test.cpp -m32 # # Sourced from http://bugs.debian.org/797228 @@ -27,8 +27,11 @@ m4_define([_CHECK_ATOMIC_testbody], [[ auto t1 = t.load(); t.compare_exchange_strong(t1, 3s); - std::atomic<int64_t> a{}; + std::atomic<double> d{}; + d.store(3.14); + auto d1 = d.load(); + std::atomic<int64_t> a{}; int64_t v = 5; int64_t r = a.fetch_add(v); return static_cast<int>(r); diff --git a/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj b/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj index 95fdcdb79b..a34ef41d16 100644 --- a/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj +++ b/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj @@ -15,7 +15,6 @@ <ClCompile Include="..\..\src\primitives\block.cpp" /> <ClCompile Include="..\..\src\primitives\transaction.cpp" /> <ClCompile Include="..\..\src\pubkey.cpp" /> - <ClCompile Include="..\..\src\script\bitcoinconsensus.cpp" /> <ClCompile Include="..\..\src\script\interpreter.cpp" /> <ClCompile Include="..\..\src\script\script.cpp" /> <ClCompile Include="..\..\src\script\script_error.cpp" /> diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 47cb8864c2..6b12c53f2a 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -46,7 +46,7 @@ if [ ! -d "${LINT_RUNNER_PATH}" ]; then fi ${CI_RETRY_EXE} pip3 install \ - codespell==2.2.5 \ + codespell==2.2.6 \ flake8==6.1.0 \ lief==0.13.2 \ mypy==1.4.1 \ diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index 318b2bb819..cdf0f60147 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -8,21 +8,24 @@ export LC_ALL=C set -ex -if [ -n "$LOCAL_BRANCH" ]; then - # To faithfully recreate CI linting locally, specify all commits on the current - # branch. - COMMIT_RANGE="$(git merge-base HEAD master)..HEAD" -elif [ -n "$CIRRUS_PR" ]; then +if [ -n "$CIRRUS_PR" ]; then COMMIT_RANGE="HEAD~..HEAD" - echo - git log --no-merges --oneline "$COMMIT_RANGE" - echo - test/lint/commit-script-check.sh "$COMMIT_RANGE" + if [ "$(git rev-list -1 HEAD)" != "$(git rev-list -1 --merges HEAD)" ]; then + echo "Error: The top commit must be a merge commit, usually the remote 'pull/${PR_NUMBER}/merge' branch." + false + fi else - COMMIT_RANGE="SKIP_EMPTY_NOT_A_PR" + # Otherwise, assume that a merge commit exists. This merge commit is assumed + # to be the base, after which linting will be done. If the merge commit is + # HEAD, the range will be empty. + COMMIT_RANGE="$( git rev-list --max-count=1 --merges HEAD )..HEAD" fi export COMMIT_RANGE +echo +git log --no-merges --oneline "$COMMIT_RANGE" +echo +test/lint/commit-script-check.sh "$COMMIT_RANGE" RUST_BACKTRACE=1 "${LINT_RUNNER_PATH}/test_runner" if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; then diff --git a/ci/lint/container-entrypoint.sh b/ci/lint/container-entrypoint.sh index a403f923a2..c8519a3912 100755 --- a/ci/lint/container-entrypoint.sh +++ b/ci/lint/container-entrypoint.sh @@ -14,7 +14,7 @@ export PATH="/python_build/bin:${PATH}" export LINT_RUNNER_PATH="/lint_test_runner" if [ -z "$1" ]; then - LOCAL_BRANCH=1 bash -ic "./ci/lint/06_script.sh" + bash -ic "./ci/lint/06_script.sh" else exec "$@" fi diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index a5f3682c36..00a4d781c2 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_multiprocess -export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:22.04" +export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:24.04" export PACKAGES="llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 840daf9708..668e9ecc8a 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -17,10 +17,10 @@ else fi export CONTAINER_NAME=ci_native_asan -export PACKAGES="systemtap-sdt-dev clang-17 llvm-17 libclang-rt-17-dev python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" +export PACKAGES="systemtap-sdt-dev clang-18 llvm-18 libclang-rt-18-dev python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" export NO_DEPENDS=1 export GOAL="install" export BITCOIN_CONFIG="--enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \ CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \ --with-sanitizers=address,float-divide-by-zero,integer,undefined \ -CC='clang-17 -ftrivial-auto-var-init=pattern' CXX='clang++-17 -ftrivial-auto-var-init=pattern'" +CC='clang-18 -ftrivial-auto-var-init=pattern' CXX='clang++-18 -ftrivial-auto-var-init=pattern'" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index abee3c1541..f50561f875 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_fuzz -export PACKAGES="clang-17 llvm-17 libclang-rt-17-dev libevent-dev libboost-dev libsqlite3-dev" +export PACKAGES="clang-18 llvm-18 libclang-rt-18-dev libevent-dev libboost-dev libsqlite3-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false @@ -16,6 +16,6 @@ export RUN_FUZZ_TESTS=true export GOAL="install" export CI_CONTAINER_CAP="--cap-add SYS_PTRACE" # If run with (ASan + LSan), the container needs access to ptrace (https://github.com/google/sanitizers/issues/764) export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,float-divide-by-zero,integer \ -CC='clang-17 -ftrivial-auto-var-init=pattern' CXX='clang++-17 -ftrivial-auto-var-init=pattern'" +CC='clang-18 -ftrivial-auto-var-init=pattern' CXX='clang++-18 -ftrivial-auto-var-init=pattern'" export CCACHE_MAXSIZE=200M -export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-17" +export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-18" diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 0a9dee2ed8..36a172e42c 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -17,7 +17,7 @@ export PACKAGES="ninja-build" # BDB generates false-positives and will be removed in future export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE'" export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 4f80d7ed42..bf4d1573e3 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -6,15 +6,14 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="docker.io/debian:bookworm" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="clang llvm libclang-rt-dev libevent-dev libboost-dev libsqlite3-dev valgrind" +export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libevent-dev libboost-dev libsqlite3-dev valgrind" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" -# Temporarily pin dwarf 4, until using Valgrind 3.20 or later -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CFLAGS=-gdwarf-4 CXXFLAGS=-gdwarf-4" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang-16 CXX=clang++-16" export CCACHE_MAXSIZE=200M diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index cbcd51e218..431e4aba49 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -17,7 +17,7 @@ export PACKAGES="ninja-build" # BDB generates false-positives and will be removed in future export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" export CCACHE_MAXSIZE=250M diff --git a/ci/test/00_setup_env_native_previous_releases.sh b/ci/test/00_setup_env_native_previous_releases.sh index 3166686d9a..9da3b18999 100755 --- a/ci/test/00_setup_env_native_previous_releases.sh +++ b/ci/test/00_setup_env_native_previous_releases.sh @@ -16,5 +16,5 @@ export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" export DOWNLOAD_PREVIOUS_RELEASES="true" -export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports --enable-debug \ +export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports --enable-debug \ CFLAGS=\"-g0 -O2 -funsigned-char\" CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS=\"-g0 -O2 -funsigned-char\"" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index a5ba64da15..4c8658479b 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_tidy export TIDY_LLVM_V="18" -export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq bear libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" +export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq bear libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev qtbase5-dev qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index aa23bad809..3fcaa8c6c6 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" -export PACKAGES="clang-17 llvm-17 libclang-rt-17-dev libc++abi-17-dev libc++-17-dev python3-zmq" -export DEP_OPTS="CC=clang-17 CXX='clang++-17 -stdlib=libc++'" +export PACKAGES="clang-18 llvm-18 libclang-rt-18-dev libc++abi-18-dev libc++-18-dev python3-zmq" +export DEP_OPTS="CC=clang-18 CXX='clang++-18 -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' CXXFLAGS='-g' --with-sanitizers=thread" +export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' --with-sanitizers=thread" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 9bdb2b7860..720e522a6a 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -6,12 +6,11 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="docker.io/debian:bookworm" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang llvm libclang-rt-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" +export PACKAGES="valgrind clang-16 llvm-16 libclang-rt-16-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" -# Temporarily pin dwarf 4, until using Valgrind 3.20 or later -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++ CFLAGS=-gdwarf-4 CXXFLAGS=-gdwarf-4" # TODO enable GUI +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang-16 CXX=clang++-16" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index ca84ecce51..2fd94e253c 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -9,8 +9,8 @@ export LC_ALL=C.UTF-8 export HOST=s390x-linux-gnu export PACKAGES="python3-zmq" export CONTAINER_NAME=ci_s390x -export CI_IMAGE_NAME_TAG="docker.io/s390x/debian:bookworm" -export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export CI_IMAGE_NAME_TAG="docker.io/s390x/ubuntu:24.04" +export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true export GOAL="install" export BITCOIN_CONFIG="--enable-reduce-exports" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 6f1498963a..25962a53e5 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -36,7 +36,7 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-18.1.1" /msan/llvm-project + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-18.1.3" /msan/llvm-project cmake -G Ninja -B /msan/clang_build/ \ -DLLVM_ENABLE_PROJECTS="clang" \ diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index d85e24a5c0..f5da7bc55d 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -10,7 +10,7 @@ set -ex export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" -export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" +export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" if [ "$CI_OS_NAME" == "macos" ]; then @@ -71,8 +71,6 @@ elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ] fi fi -mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" - if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) diff --git a/configure.ac b/configure.ac index 03b5ff1d69..febb352cdb 100644 --- a/configure.ac +++ b/configure.ac @@ -307,9 +307,9 @@ AC_ARG_ENABLE([werror], [enable_werror=no]) AC_ARG_ENABLE([external-signer], - [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is auto, requires Boost::Process)])], + [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is yes)])], [use_external_signer=$enableval], - [use_external_signer=auto]) + [use_external_signer=yes]) AC_LANG_PUSH([C++]) @@ -626,15 +626,9 @@ AC_ARG_ENABLE([experimental-util-chainstate], [build_bitcoin_chainstate=$enableval], [build_bitcoin_chainstate=no]) -AC_ARG_WITH([libs], - [AS_HELP_STRING([--with-libs], - [build libraries (default=yes)])], - [build_bitcoin_libs=$withval], - [build_bitcoin_libs=yes]) - AC_ARG_WITH([experimental-kernel-lib], [AS_HELP_STRING([--with-experimental-kernel-lib], - [build experimental bitcoinkernel library (default is to build if we're building libraries and the experimental build-chainstate executable)])], + [build experimental bitcoinkernel library (default is to build if we're building the experimental build-chainstate executable)])], [build_experimental_kernel_lib=$withval], [build_experimental_kernel_lib=auto]) @@ -974,24 +968,6 @@ AC_CHECK_DECLS([setsid]) AC_CHECK_DECLS([pipe2]) -AC_CHECK_FUNCS([timingsafe_bcmp]) - -AC_MSG_CHECKING([for __builtin_clzl]) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ - (void) __builtin_clzl(0); - ]])], - [ AC_MSG_RESULT([yes]); have_clzl=yes; AC_DEFINE([HAVE_BUILTIN_CLZL], [1], [Define this symbol if you have __builtin_clzl])], - [ AC_MSG_RESULT([no]); have_clzl=no;] -) - -AC_MSG_CHECKING([for __builtin_clzll]) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ - (void) __builtin_clzll(0); - ]])], - [ AC_MSG_RESULT([yes]); have_clzll=yes; AC_DEFINE([HAVE_BUILTIN_CLZLL], [1], [Define this symbol if you have __builtin_clzll])], - [ AC_MSG_RESULT([no]); have_clzll=no;] -) - dnl Check for malloc_info (for memory statistics information in getmemoryinfo) AC_MSG_CHECKING([for getmemoryinfo]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <malloc.h>]], @@ -1092,22 +1068,6 @@ if test "$use_thread_local" = "yes" || test "$use_thread_local" = "auto"; then LDFLAGS="$TEMP_LDFLAGS" fi -dnl check for gmtime_r(), fallback to gmtime_s() if that is unavailable -dnl fail if neither are available. -AC_MSG_CHECKING([for gmtime_r]) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ctime>]], - [[ gmtime_r((const time_t *) nullptr, (struct tm *) nullptr); ]])], - [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GMTIME_R], [1], [Define this symbol if gmtime_r is available]) ], - [ AC_MSG_RESULT([no]); - AC_MSG_CHECKING([for gmtime_s]); - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ctime>]], - [[ gmtime_s((struct tm *) nullptr, (const time_t *) nullptr); ]])], - [ AC_MSG_RESULT([yes])], - [ AC_MSG_RESULT([no]); AC_MSG_ERROR([Both gmtime_r and gmtime_s are unavailable]) ] - ) - ] -) - dnl Check for different ways of gathering OS randomness AC_MSG_CHECKING([for Linux getrandom function]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ @@ -1262,7 +1222,6 @@ if test "$enable_fuzz" = "yes"; then build_bitcoin_chainstate=no build_bitcoin_wallet=no build_bitcoind=no - build_bitcoin_libs=no bitcoin_enable_qt=no bitcoin_enable_qt_test=no bitcoin_enable_qt_dbus=no @@ -1421,7 +1380,7 @@ if test "$use_boost" = "yes"; then dnl Check for Boost headers AX_BOOST_BASE([1.73.0],[],[AC_MSG_ERROR([Boost is not available!])]) if test "$want_boost" = "no"; then - AC_MSG_ERROR([only libbitcoinconsensus can be built without Boost]) + AC_MSG_ERROR([Boost is required]) fi dnl we don't use multi_index serialization @@ -1439,56 +1398,14 @@ if test "$use_boost" = "yes"; then fi fi -if test "$use_external_signer" != "no"; then - AC_MSG_CHECKING([whether Boost.Process can be used]) - TEMP_CXXFLAGS="$CXXFLAGS" - dnl Boost 1.78 requires the following workaround. - dnl See: https://github.com/boostorg/process/issues/235 - CXXFLAGS="$CXXFLAGS -Wno-error=narrowing" - TEMP_CPPFLAGS="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - TEMP_LDFLAGS="$LDFLAGS" - dnl Boost 1.73 and older require the following workaround. - LDFLAGS="$LDFLAGS $PTHREAD_CFLAGS" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[ - #define BOOST_PROCESS_USE_STD_FS - #include <boost/process.hpp> - ]],[[ - namespace bp = boost::process; - bp::opstream stdin_stream; - bp::ipstream stdout_stream; - bp::child c("dummy", bp::std_out > stdout_stream, bp::std_err > stdout_stream, bp::std_in < stdin_stream); - stdin_stream << std::string{"test"} << std::endl; - if (c.running()) c.terminate(); - c.wait(); - c.exit_code(); - ]])], - [have_boost_process="yes"], - [have_boost_process="no"]) - LDFLAGS="$TEMP_LDFLAGS" - CPPFLAGS="$TEMP_CPPFLAGS" - CXXFLAGS="$TEMP_CXXFLAGS" - AC_MSG_RESULT([$have_boost_process]) - if test "$have_boost_process" = "yes"; then - case $host in - dnl Boost Process for Windows uses Boost ASIO. Boost ASIO performs - dnl pre-main init of Windows networking libraries, which we do not - dnl want. - *mingw*) - use_external_signer="no" - ;; - *) - use_external_signer="yes" - AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled]) - AC_DEFINE([BOOST_PROCESS_USE_STD_FS], [1], [Defined to avoid Boost::Process trying to use Boost Filesystem]) - ;; - esac - else - if test "$use_external_signer" = "yes"; then - AC_MSG_ERROR([External signing is not supported for this Boost version]) - fi - use_external_signer="no"; - fi +case $host in + dnl Re-enable it after enabling Windows support in cpp-subprocess. + *mingw*) + use_external_signer="no" + ;; +esac +if test "$use_external_signer" = "yes"; then + AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled]) fi AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "$use_external_signer" = "yes"]) @@ -1635,18 +1552,8 @@ fi AM_CONDITIONAL([BUILD_BITCOIN_CHAINSTATE], [test $build_bitcoin_chainstate = "yes"]) AC_MSG_RESULT($build_bitcoin_chainstate) -AC_MSG_CHECKING([whether to build libraries]) -AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test $build_bitcoin_libs = "yes"]) - -if test "$build_bitcoin_libs" = "yes"; then - AC_DEFINE([HAVE_CONSENSUS_LIB], [1], [Define this symbol if the consensus lib has been built]) - AC_CONFIG_FILES([libbitcoinconsensus.pc:libbitcoinconsensus.pc.in]) -fi - AM_CONDITIONAL([BUILD_BITCOIN_KERNEL_LIB], [test "$build_experimental_kernel_lib" != "no" && ( test "$build_experimental_kernel_lib" = "yes" || test "$build_bitcoin_chainstate" = "yes" )]) -AC_MSG_RESULT($build_bitcoin_libs) - AC_LANG_POP if test "$use_ccache" != "no"; then @@ -1780,8 +1687,8 @@ else AC_MSG_RESULT([no]) fi -if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_bench$use_tests" = "nononononononononono"; then - AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-fuzz(-binary) --enable-bench or --enable-tests]) +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_bench$use_tests" = "nonononononononono"; then + AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-daemon --with-gui --enable-fuzz(-binary) --enable-bench or --enable-tests]) fi AM_CONDITIONAL([TARGET_DARWIN], [test "$TARGET_OS" = "darwin"]) @@ -1813,7 +1720,6 @@ AM_CONDITIONAL([USE_UPNP], [test "$use_upnp" = "yes"]) dnl for minisketch AM_CONDITIONAL([ENABLE_CLMUL], [test "$enable_clmul" = "yes"]) -AM_CONDITIONAL([HAVE_CLZ], [test "$have_clzl$have_clzll" = "yesyes"]) AC_DEFINE([CLIENT_VERSION_MAJOR], [_CLIENT_VERSION_MAJOR], [Major version]) AC_DEFINE([CLIENT_VERSION_MINOR], [_CLIENT_VERSION_MINOR], [Minor version]) @@ -1879,7 +1785,6 @@ AC_SUBST(MINIUPNPC_CPPFLAGS) AC_SUBST(MINIUPNPC_LIBS) AC_SUBST(NATPMP_CPPFLAGS) AC_SUBST(NATPMP_LIBS) -AC_SUBST(HAVE_GMTIME_R) AC_SUBST(HAVE_FDATASYNC) AC_SUBST(HAVE_FULLFSYNC) AC_SUBST(HAVE_O_CLOEXEC) @@ -1941,7 +1846,6 @@ echo echo "Options used to compile and link:" echo " external signer = $use_external_signer" echo " multiprocess = $build_multiprocess" -echo " with libs = $build_bitcoin_libs" echo " with wallet = $enable_wallet" if test "$enable_wallet" != "no"; then echo " with sqlite = $use_sqlite" diff --git a/contrib/devtools/bitcoin-tidy/CMakeLists.txt b/contrib/devtools/bitcoin-tidy/CMakeLists.txt index 35e60d1d87..1260c71423 100644 --- a/contrib/devtools/bitcoin-tidy/CMakeLists.txt +++ b/contrib/devtools/bitcoin-tidy/CMakeLists.txt @@ -1,14 +1,25 @@ -cmake_minimum_required(VERSION 3.9) +cmake_minimum_required(VERSION 3.22) -project(bitcoin-tidy VERSION 1.0.0 DESCRIPTION "clang-tidy checks for Bitcoin Core") +project(bitcoin-tidy + VERSION + 1.0.0 + DESCRIPTION "clang-tidy checks for Bitcoin Core" + LANGUAGES CXX) include(GNUInstallDirs) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_CXX_EXTENSIONS False) -# TODO: Figure out how to avoid the terminfo check +set(CMAKE_DISABLE_FIND_PACKAGE_CURL ON) +set(CMAKE_DISABLE_FIND_PACKAGE_FFI ON) +set(CMAKE_DISABLE_FIND_PACKAGE_LibEdit ON) +set(CMAKE_DISABLE_FIND_PACKAGE_LibXml2 ON) +set(CMAKE_DISABLE_FIND_PACKAGE_Terminfo ON) +set(CMAKE_DISABLE_FIND_PACKAGE_ZLIB ON) +set(CMAKE_DISABLE_FIND_PACKAGE_zstd ON) + find_package(LLVM REQUIRED CONFIG) find_program(CLANG_TIDY_EXE NAMES "clang-tidy-${LLVM_VERSION_MAJOR}" "clang-tidy" HINTS ${LLVM_TOOLS_BINARY_DIR}) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") diff --git a/contrib/devtools/bitcoin-tidy/README b/contrib/devtools/bitcoin-tidy/README.md index c15e07c4ed..c15e07c4ed 100644 --- a/contrib/devtools/bitcoin-tidy/README +++ b/contrib/devtools/bitcoin-tidy/README.md diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index b301369ad9..1e9b682f3f 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -61,7 +61,6 @@ store_path() { # Set environment variables to point the NATIVE toolchain to the right # includes/libs NATIVE_GCC="$(store_path gcc-toolchain)" -NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" unset LIBRARY_PATH unset CPATH @@ -70,12 +69,20 @@ unset CPLUS_INCLUDE_PATH unset OBJC_INCLUDE_PATH unset OBJCPLUS_INCLUDE_PATH -export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC_STATIC}/lib" export C_INCLUDE_PATH="${NATIVE_GCC}/include" export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include" export OBJCPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" +case "$HOST" in + *darwin*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; + *mingw*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; + *) + NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" + export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC_STATIC}/lib" + ;; +esac + # Set environment variables to point the CROSS toolchain to the right # includes/libs for $HOST case "$HOST" in @@ -300,11 +307,9 @@ mkdir -p "$DISTSRC" case "$HOST" in *darwin*) - make osx_volname ${V:+V=1} make deploydir ${V:+V=1} mkdir -p "unsigned-app-${HOST}" cp --target-directory="unsigned-app-${HOST}" \ - osx_volname \ contrib/macdeploy/detached-sig-create.sh mv --target-directory="unsigned-app-${HOST}" dist ( @@ -321,26 +326,16 @@ mkdir -p "$DISTSRC" ( cd installed - case "$HOST" in - *mingw*) - mv --target-directory="$DISTNAME"/lib/ "$DISTNAME"/bin/*.dll - ;; - esac - # Prune libtool and object archives find . -name "lib*.la" -delete find . -name "lib*.a" -delete - # Prune pkg-config files - rm -rf "${DISTNAME}/lib/pkgconfig" - case "$HOST" in *darwin*) ;; *) - # Split binaries and libraries from their debug symbols + # Split binaries from their debug symbols { find "${DISTNAME}/bin" -type f -executable -print0 - find "${DISTNAME}/lib" -type f -print0 } | xargs -0 -P"$JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg ;; esac diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 6c912ca748..ce6a9562b4 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -51,7 +51,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=d5ca4d4fd713a9f7e17e074a1e37dda99bbb09fc \ + --commit=dc4842797bfdc5f9f3f5f725bf189c2b68bd6b5a \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 3353c8a874..8f13c642d3 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -110,12 +110,15 @@ desirable for building Bitcoin Core release binaries." (define (gcc-mingw-patches gcc) (package-with-extra-patches gcc - (search-our-patches "gcc-remap-guix-store.patch" - "vmov-alignment.patch"))) + (search-our-patches "gcc-remap-guix-store.patch"))) + +(define (binutils-mingw-patches binutils) + (package-with-extra-patches binutils + (search-our-patches "binutils-unaligned-default.patch"))) (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (cross-binutils target)) + (let* ((xbinutils (binutils-mingw-patches (cross-binutils target))) (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (cross-gcc target #:xgcc (gcc-mingw-patches mingw-w64-base-gcc) @@ -423,6 +426,7 @@ inspecting signatures in Mach-O binaries.") (list "--enable-initfini-array=yes", "--enable-default-ssp=yes", "--enable-default-pie=yes", + "--enable-standard-branch-protection=yes", building-on))) ((#:phases phases) `(modify-phases ,phases @@ -499,6 +503,7 @@ inspecting signatures in Mach-O binaries.") gzip xz ;; Build tools + cmake-minimal gnu-make libtool autoconf-2.71 @@ -515,7 +520,6 @@ inspecting signatures in Mach-O binaries.") (cond ((string-suffix? "-mingw32" target) (list ;; Native GCC 12 toolchain gcc-toolchain-12 - (list gcc-toolchain-12 "static") zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") nsis-x86_64 @@ -527,12 +531,10 @@ inspecting signatures in Mach-O binaries.") (list gcc-toolchain-12 "static") (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list ;; Native GCC 10 toolchain - gcc-toolchain-10 - (list gcc-toolchain-10 "static") + (list ;; Native GCC 11 toolchain + gcc-toolchain-11 binutils clang-toolchain-17 - cmake-minimal python-signapple zip)) (else '()))))) diff --git a/contrib/guix/patches/binutils-unaligned-default.patch b/contrib/guix/patches/binutils-unaligned-default.patch new file mode 100644 index 0000000000..d1bc71aee1 --- /dev/null +++ b/contrib/guix/patches/binutils-unaligned-default.patch @@ -0,0 +1,22 @@ +commit 6537181f59ed186a341db621812a6bc35e22eaf6 +Author: fanquake <fanquake@gmail.com> +Date: Wed Apr 10 12:15:52 2024 +0200 + + build: turn on -muse-unaligned-vector-move by default + + This allows us to avoid (more invasively) patching GCC, to avoid + unaligned instruction use. + +diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c +index e0632681477..14a9653abdf 100644 +--- a/gas/config/tc-i386.c ++++ b/gas/config/tc-i386.c +@@ -801,7 +801,7 @@ static unsigned int no_cond_jump_promotion = 0; + static unsigned int sse2avx; + + /* Encode aligned vector move as unaligned vector move. */ +-static unsigned int use_unaligned_vector_move; ++static unsigned int use_unaligned_vector_move = 1; + + /* Encode scalar AVX instructions with specific vector length. */ + static enum diff --git a/contrib/guix/patches/glibc-2.27-fcommon.patch b/contrib/guix/patches/glibc-2.27-fcommon.patch index 817aa85bb9..f8d14837fc 100644 --- a/contrib/guix/patches/glibc-2.27-fcommon.patch +++ b/contrib/guix/patches/glibc-2.27-fcommon.patch @@ -5,7 +5,7 @@ Date: Fri May 6 11:03:04 2022 +0100 build: use -fcommon to retain legacy behaviour with GCC 10 GCC 10 started using -fno-common by default, which causes issues with - the powerpc builds using gibc 2.27. A patch was commited to glibc to fix + the powerpc builds using gibc 2.27. A patch was committed to glibc to fix the issue, 18363b4f010da9ba459b13310b113ac0647c2fcc but is non-trvial to backport, and was broken in at least one way, see the followup in commit 7650321ce037302bfc2f026aa19e0213b8d02fe6. diff --git a/contrib/guix/patches/vmov-alignment.patch b/contrib/guix/patches/vmov-alignment.patch deleted file mode 100644 index 96e1cb7cd1..0000000000 --- a/contrib/guix/patches/vmov-alignment.patch +++ /dev/null @@ -1,288 +0,0 @@ -Description: Use unaligned VMOV instructions -Author: Stephen Kitt <skitt@debian.org> -Bug-Debian: https://bugs.debian.org/939559 -See also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412 - -Based on a patch originally by Claude Heiland-Allen <claude@mathr.co.uk> - ---- a/gcc/config/i386/sse.md -+++ b/gcc/config/i386/sse.md -@@ -1058,17 +1058,11 @@ - { - if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode))) - { -- if (misaligned_operand (operands[1], <MODE>mode)) -- return "vmovu<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; -- else -- return "vmova<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; -+ return "vmovu<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; - } - else - { -- if (misaligned_operand (operands[1], <MODE>mode)) -- return "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; -- else -- return "vmovdqa<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; -+ return "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; - } - } - [(set_attr "type" "ssemov") -@@ -1184,17 +1178,11 @@ - { - if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode))) - { -- if (misaligned_operand (operands[0], <MODE>mode)) -- return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; -- else -- return "vmova<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; -+ return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; - } - else - { -- if (misaligned_operand (operands[0], <MODE>mode)) -- return "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; -- else -- return "vmovdqa<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; -+ return "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; - } - } - [(set_attr "type" "ssemov") -@@ -7806,7 +7794,7 @@ - "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))" - "@ - %vmovlps\t{%1, %0|%q0, %1} -- %vmovaps\t{%1, %0|%0, %1} -+ %vmovups\t{%1, %0|%0, %1} - %vmovlps\t{%1, %d0|%d0, %q1}" - [(set_attr "type" "ssemov") - (set_attr "prefix" "maybe_vex") -@@ -13997,29 +13985,15 @@ - switch (<MODE>mode) - { - case E_V8DFmode: -- if (misaligned_operand (operands[2], <ssequartermode>mode)) -- return "vmovupd\t{%2, %x0|%x0, %2}"; -- else -- return "vmovapd\t{%2, %x0|%x0, %2}"; -+ return "vmovupd\t{%2, %x0|%x0, %2}"; - case E_V16SFmode: -- if (misaligned_operand (operands[2], <ssequartermode>mode)) -- return "vmovups\t{%2, %x0|%x0, %2}"; -- else -- return "vmovaps\t{%2, %x0|%x0, %2}"; -+ return "vmovups\t{%2, %x0|%x0, %2}"; - case E_V8DImode: -- if (misaligned_operand (operands[2], <ssequartermode>mode)) -- return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" -+ return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" - : "vmovdqu\t{%2, %x0|%x0, %2}"; -- else -- return which_alternative == 2 ? "vmovdqa64\t{%2, %x0|%x0, %2}" -- : "vmovdqa\t{%2, %x0|%x0, %2}"; - case E_V16SImode: -- if (misaligned_operand (operands[2], <ssequartermode>mode)) -- return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" -+ return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" - : "vmovdqu\t{%2, %x0|%x0, %2}"; -- else -- return which_alternative == 2 ? "vmovdqa32\t{%2, %x0|%x0, %2}" -- : "vmovdqa\t{%2, %x0|%x0, %2}"; - default: - gcc_unreachable (); - } -@@ -21225,63 +21199,27 @@ - switch (get_attr_mode (insn)) - { - case MODE_V16SF: -- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) -- return "vmovups\t{%1, %t0|%t0, %1}"; -- else -- return "vmovaps\t{%1, %t0|%t0, %1}"; -+ return "vmovups\t{%1, %t0|%t0, %1}"; - case MODE_V8DF: -- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) -- return "vmovupd\t{%1, %t0|%t0, %1}"; -- else -- return "vmovapd\t{%1, %t0|%t0, %1}"; -+ return "vmovupd\t{%1, %t0|%t0, %1}"; - case MODE_V8SF: -- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) -- return "vmovups\t{%1, %x0|%x0, %1}"; -- else -- return "vmovaps\t{%1, %x0|%x0, %1}"; -+ return "vmovups\t{%1, %x0|%x0, %1}"; - case MODE_V4DF: -- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) -- return "vmovupd\t{%1, %x0|%x0, %1}"; -- else -- return "vmovapd\t{%1, %x0|%x0, %1}"; -+ return "vmovupd\t{%1, %x0|%x0, %1}"; - case MODE_XI: -- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) -- { -- if (which_alternative == 2) -- return "vmovdqu\t{%1, %t0|%t0, %1}"; -- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) -- return "vmovdqu64\t{%1, %t0|%t0, %1}"; -- else -- return "vmovdqu32\t{%1, %t0|%t0, %1}"; -- } -+ if (which_alternative == 2) -+ return "vmovdqu\t{%1, %t0|%t0, %1}"; -+ else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) -+ return "vmovdqu64\t{%1, %t0|%t0, %1}"; - else -- { -- if (which_alternative == 2) -- return "vmovdqa\t{%1, %t0|%t0, %1}"; -- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) -- return "vmovdqa64\t{%1, %t0|%t0, %1}"; -- else -- return "vmovdqa32\t{%1, %t0|%t0, %1}"; -- } -+ return "vmovdqu32\t{%1, %t0|%t0, %1}"; - case MODE_OI: -- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) -- { -- if (which_alternative == 2) -- return "vmovdqu\t{%1, %x0|%x0, %1}"; -- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) -- return "vmovdqu64\t{%1, %x0|%x0, %1}"; -- else -- return "vmovdqu32\t{%1, %x0|%x0, %1}"; -- } -+ if (which_alternative == 2) -+ return "vmovdqu\t{%1, %x0|%x0, %1}"; -+ else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) -+ return "vmovdqu64\t{%1, %x0|%x0, %1}"; - else -- { -- if (which_alternative == 2) -- return "vmovdqa\t{%1, %x0|%x0, %1}"; -- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) -- return "vmovdqa64\t{%1, %x0|%x0, %1}"; -- else -- return "vmovdqa32\t{%1, %x0|%x0, %1}"; -- } -+ return "vmovdqu32\t{%1, %x0|%x0, %1}"; - default: - gcc_unreachable (); - } ---- a/gcc/config/i386/i386.cc -+++ b/gcc/config/i386/i386.cc -@@ -5418,17 +5418,15 @@ ix86_get_ssemov (rtx *operands, unsigned size, - { - case opcode_int: - if (scalar_mode == E_HFmode) -- opcode = (misaligned_p -- ? (TARGET_AVX512BW ? "vmovdqu16" : "vmovdqu64") -- : "vmovdqa64"); -+ opcode = TARGET_AVX512BW ? "vmovdqu16" : "vmovdqu64"; - else -- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; -+ opcode = "vmovdqu32"; - break; - case opcode_float: -- opcode = misaligned_p ? "vmovups" : "vmovaps"; -+ opcode = "vmovups"; - break; - case opcode_double: -- opcode = misaligned_p ? "vmovupd" : "vmovapd"; -+ opcode = "vmovupd"; - break; - } - } -@@ -5438,29 +5436,21 @@ ix86_get_ssemov (rtx *operands, unsigned size, - { - case E_HFmode: - if (evex_reg_p) -- opcode = (misaligned_p -- ? (TARGET_AVX512BW -- ? "vmovdqu16" -- : "vmovdqu64") -- : "vmovdqa64"); -+ opcode = TARGET_AVX512BW ? "vmovdqu16" : "vmovdqu64"; - else -- opcode = (misaligned_p -- ? (TARGET_AVX512BW -- ? "vmovdqu16" -- : "%vmovdqu") -- : "%vmovdqa"); -+ opcode = TARGET_AVX512BW ? "vmovdqu16" : "%vmovdqu"; - break; - case E_SFmode: -- opcode = misaligned_p ? "%vmovups" : "%vmovaps"; -+ opcode = "%vmovups"; - break; - case E_DFmode: -- opcode = misaligned_p ? "%vmovupd" : "%vmovapd"; -+ opcode = "%vmovupd"; - break; - case E_TFmode: - if (evex_reg_p) -- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; -+ opcode = "vmovdqu64"; - else -- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; -+ opcode = "%vmovdqu"; - break; - default: - gcc_unreachable (); -@@ -5472,48 +5462,32 @@ ix86_get_ssemov (rtx *operands, unsigned size, - { - case E_QImode: - if (evex_reg_p) -- opcode = (misaligned_p -- ? (TARGET_AVX512BW -- ? "vmovdqu8" -- : "vmovdqu64") -- : "vmovdqa64"); -+ opcode = TARGET_AVX512BW ? "vmovdqu8" : "vmovdqu64"; - else -- opcode = (misaligned_p -- ? (TARGET_AVX512BW -- ? "vmovdqu8" -- : "%vmovdqu") -- : "%vmovdqa"); -+ opcode = TARGET_AVX512BW ? "vmovdqu8" : "%vmovdqu"; - break; - case E_HImode: - if (evex_reg_p) -- opcode = (misaligned_p -- ? (TARGET_AVX512BW -- ? "vmovdqu16" -- : "vmovdqu64") -- : "vmovdqa64"); -+ opcode = TARGET_AVX512BW ? "vmovdqu16" : "vmovdqu64"; - else -- opcode = (misaligned_p -- ? (TARGET_AVX512BW -- ? "vmovdqu16" -- : "%vmovdqu") -- : "%vmovdqa"); -+ opcode = TARGET_AVX512BW ? "vmovdqu16" : "%vmovdqu"; - break; - case E_SImode: - if (evex_reg_p) -- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; -+ opcode = "vmovdqu32"; - else -- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; -+ opcode = "%vmovdqu"; - break; - case E_DImode: - case E_TImode: - case E_OImode: - if (evex_reg_p) -- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; -+ opcode = "vmovdqu64"; - else -- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; -+ opcode = "%vmovdqu"; - break; - case E_XImode: -- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; -+ opcode = "vmovdqu64"; - break; - default: - gcc_unreachable (); diff --git a/contrib/init/bitcoind.service b/contrib/init/bitcoind.service index 87da17f955..ade8a05926 100644 --- a/contrib/init/bitcoind.service +++ b/contrib/init/bitcoind.service @@ -81,5 +81,8 @@ PrivateDevices=true # Deny the creation of writable and executable memory mappings. MemoryDenyWriteExecute=true +# Restrict ABIs to help ensure MemoryDenyWriteExecute is enforced +SystemCallArchitectures=native + [Install] WantedBy=multi-user.target diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index ee91acd5ef..c537f9e7ec 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -13,8 +13,8 @@ # # Note that suppressions may depend on OS and/or library versions. # Tested on: -# * aarch64 (Debian Bookworm system libs, clang, without gui) -# * x86_64 (Debian Bookworm system libs, clang, without gui) +# * aarch64 (Ubuntu Noble system libs, clang, without gui) +# * x86_64 (Ubuntu Noble system libs, clang, without gui) { Suppress libdb warning - https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=662917 Memcheck:Cond diff --git a/depends/funcs.mk b/depends/funcs.mk index 7b5c3d0c59..494ed5d324 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -147,7 +147,7 @@ $(1)_stage_env+=PATH="$(build_prefix)/bin:$(PATH)" # config.guess, which is what we set it too here. This also quells autoconf # warnings, "If you wanted to set the --build type, don't use --host.", # when using versions older than 2.70. -$(1)_autoconf=./configure --build=$(BUILD) --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" +$(1)_autoconf=./configure --build=$(BUILD) --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) --with-pic $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" ifneq ($($(1)_nm),) $(1)_autoconf += NM="$$($(1)_nm)" endif @@ -170,12 +170,19 @@ ifneq ($($(1)_ldflags),) $(1)_autoconf += LDFLAGS="$$($(1)_ldflags)" endif +# We hardcode the library install path to "lib" to match the PKG_CONFIG_PATH +# setting in depends/config.site.in, which also hardcodes "lib". +# Without this setting, CMake by default would use the OS library +# directory, which might be "lib64" or something else, not "lib", on multiarch systems. $(1)_cmake=env CC="$$($(1)_cc)" \ CFLAGS="$$($(1)_cppflags) $$($(1)_cflags)" \ CXX="$$($(1)_cxx)" \ CXXFLAGS="$$($(1)_cppflags) $$($(1)_cxxflags)" \ LDFLAGS="$$($(1)_ldflags)" \ - cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" $$($(1)_config_opts) + cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" \ + -DCMAKE_INSTALL_LIBDIR=lib/ \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + $$($(1)_config_opts) ifeq ($($(1)_type),build) $(1)_cmake += -DCMAKE_INSTALL_RPATH:PATH="$$($($(1)_type)_prefix)/lib" else diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 29ad7ef252..8beedcc98a 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -105,7 +105,7 @@ endif darwin_release_CFLAGS=-O2 darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) -darwin_debug_CFLAGS=-O1 +darwin_debug_CFLAGS=-O1 -g darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) darwin_cmake_system=Darwin diff --git a/depends/hosts/linux.mk b/depends/hosts/linux.mk index 8be23be57d..f5ce2bb0b8 100644 --- a/depends/hosts/linux.mk +++ b/depends/hosts/linux.mk @@ -10,10 +10,14 @@ endif linux_release_CFLAGS=-O2 linux_release_CXXFLAGS=$(linux_release_CFLAGS) -linux_debug_CFLAGS=-O1 +linux_debug_CFLAGS=-O1 -g linux_debug_CXXFLAGS=$(linux_debug_CFLAGS) -linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -D_LIBCPP_ENABLE_DEBUG_MODE=1 +# https://gcc.gnu.org/onlinedocs/libstdc++/manual/debug_mode.html +linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC + +# https://libcxx.llvm.org/Hardening.html +linux_debug_CPPFLAGS+=-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG ifeq (86,$(findstring 86,$(build_arch))) i686_linux_CC=gcc -m32 diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index 15aa7cd25a..4c657358f6 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -14,7 +14,7 @@ endif mingw32_release_CFLAGS=-O2 mingw32_release_CXXFLAGS=$(mingw32_release_CFLAGS) -mingw32_debug_CFLAGS=-O1 +mingw32_debug_CFLAGS=-O1 -g mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS) mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC diff --git a/depends/packages.md b/depends/packages.md index ad91eaffee..7a7a42afa1 100644 --- a/depends/packages.md +++ b/depends/packages.md @@ -162,6 +162,9 @@ From the [Gentoo Wiki entry](https://wiki.gentoo.org/wiki/Project:Quality_Assura > creates. This leads to massive overlinking, which is toxic to the Gentoo > ecosystem, as it leads to a massive number of unnecessary rebuilds. +Where possible, packages are built with Position Independent Code. Either using +the Autotools `--with-pic` flag, or `CMAKE_POSITION_INDEPENDENT_CODE` with CMake. + ## Secondary dependencies: Secondary dependency packages relative to the bitcoin binaries/libraries (i.e. diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index 1a21238152..be82b0d309 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -9,11 +9,6 @@ $(package)_patches=clang_cxx_11.patch define $(package)_set_vars $(package)_config_opts=--disable-shared --enable-cxx --disable-replication --enable-option-checking $(package)_config_opts_mingw32=--enable-mingw -$(package)_config_opts_linux=--with-pic -$(package)_config_opts_freebsd=--with-pic -$(package)_config_opts_netbsd=--with-pic -$(package)_config_opts_openbsd=--with-pic -$(package)_config_opts_android=--with-pic $(package)_cflags+=-Wno-error=implicit-function-declaration -Wno-error=format-security -Wno-error=implicit-int $(package)_cppflags_freebsd=-D_XOPEN_SOURCE=600 -D__BSD_VISIBLE=1 $(package)_cppflags_netbsd=-D_XOPEN_SOURCE=600 diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index ab43764b38..ebc097d686 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -3,11 +3,6 @@ $(package)_version=1.81.0 $(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$($(package)_version)/source/ $(package)_file_name=boost_$(subst .,_,$($(package)_version)).tar.bz2 $(package)_sha256_hash=71feeed900fbccca04a3b4f2f84a7c217186f28a940ed8b7ed4725986baf99fa -$(package)_patches=process_macos_sdk.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/process_macos_sdk.patch -endef define $(package)_stage_cmds mkdir -p $($(package)_staging_prefix_dir)/include && \ diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index 2465c8091b..6d792db711 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -5,15 +5,10 @@ $(package)_download_file=$(native_$(package)_download_file) $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) -# Hardcode library install path to "lib" to match the PKG_CONFIG_PATH -# setting in depends/config.site.in, which also hardcodes "lib". -# Without this setting, cmake by default would use the OS library -# directory, which might be "lib64" or something else, not "lib", on multiarch systems. define $(package)_set_vars := $(package)_config_opts := -DBUILD_TESTING=OFF $(package)_config_opts += -DWITH_OPENSSL=OFF $(package)_config_opts += -DWITH_ZLIB=OFF - $(package)_config_opts += -DCMAKE_INSTALL_LIBDIR=lib/ endef define $(package)_config_cmds diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index bb203d06f8..2ec660109c 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -6,12 +6,11 @@ $(package)_sha256_hash=f79b8f904b749e3e0d20afeadecf8249c55b2e32d4ebb089ae378df47 # -D_DEFAULT_SOURCE defines __USE_MISC, which exposes additional # definitions in endian.h, which are required for a working -# endianess check in configure when building with -flto. +# endianness check in configure when building with -flto. define $(package)_set_vars $(package)_config_opts=--disable-shared --without-docbook --without-tests --without-examples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts += --without-xmlwf - $(package)_config_opts_linux=--with-pic $(package)_cppflags += -D_DEFAULT_SOURCE endef diff --git a/depends/packages/freetype.mk b/depends/packages/freetype.mk index 6f5dbe0f01..c28259ed67 100644 --- a/depends/packages/freetype.mk +++ b/depends/packages/freetype.mk @@ -7,7 +7,6 @@ $(package)_sha256_hash=8bee39bd3968c4804b70614a0a3ad597299ad0e824bc8aad5ce8aaf48 define $(package)_set_vars $(package)_config_opts=--without-zlib --without-png --without-harfbuzz --without-bzip2 --disable-static $(package)_config_opts += --enable-option-checking --without-brotli - $(package)_config_opts_linux=--with-pic endef define $(package)_config_cmds diff --git a/depends/packages/libXau.mk b/depends/packages/libXau.mk index b7e032c0b2..aeb14dcd6e 100644 --- a/depends/packages/libXau.mk +++ b/depends/packages/libXau.mk @@ -10,7 +10,6 @@ $(package)_dependencies=xproto define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-lint-library --without-lint $(package)_config_opts += --disable-dependency-tracking --enable-option-checking - $(package)_config_opts_linux=--with-pic endef define $(package)_preprocess_cmds diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 9650f77db9..d764be5d0a 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -11,11 +11,6 @@ define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress --disable-samples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_release=--disable-debug-mode - $(package)_config_opts_linux=--with-pic - $(package)_config_opts_freebsd=--with-pic - $(package)_config_opts_netbsd=--with-pic - $(package)_config_opts_openbsd=--with-pic - $(package)_config_opts_android=--with-pic $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601 ifeq ($(NO_HARDEN),) diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk index d237f52dbb..c292c49bfb 100644 --- a/depends/packages/libmultiprocess.mk +++ b/depends/packages/libmultiprocess.mk @@ -8,13 +8,7 @@ ifneq ($(host),$(build)) $(package)_dependencies += native_capnp endif -# Hardcode library install path to "lib" to match the PKG_CONFIG_PATH -# setting in depends/config.site.in, which also hardcodes "lib". -# Without this setting, cmake by default would use the OS library -# directory, which might be "lib64" or something else, not "lib", on multiarch systems. define $(package)_set_vars := -$(package)_config_opts += -DCMAKE_INSTALL_LIBDIR=lib/ -$(package)_config_opts += -DCMAKE_POSITION_INDEPENDENT_CODE=ON ifneq ($(host),$(build)) $(package)_config_opts := -DCAPNP_EXECUTABLE="$$(native_capnp_prefixbin)/capnp" $(package)_config_opts += -DCAPNPC_CXX_EXECUTABLE="$$(native_capnp_prefixbin)/capnpc-c++" @@ -26,7 +20,7 @@ define $(package)_config_cmds endef define $(package)_build_cmds - $(MAKE) + $(MAKE) multiprocess endef define $(package)_stage_cmds diff --git a/depends/packages/libxcb_util.mk b/depends/packages/libxcb_util.mk index 6f1b9cd7c6..6e4c7359b2 100644 --- a/depends/packages/libxcb_util.mk +++ b/depends/packages/libxcb_util.mk @@ -8,7 +8,6 @@ $(package)_dependencies=libxcb define $(package)_set_vars $(package)_config_opts = --disable-shared --disable-devel-docs --without-doxygen $(package)_config_opts += --disable-dependency-tracking --enable-option-checking -$(package)_config_opts += --with-pic endef define $(package)_preprocess_cmds diff --git a/depends/packages/qrencode.mk b/depends/packages/qrencode.mk index 2afd95d7c4..4d852d833d 100644 --- a/depends/packages/qrencode.mk +++ b/depends/packages/qrencode.mk @@ -3,22 +3,22 @@ $(package)_version=4.1.1 $(package)_download_path=https://fukuchi.org/works/qrencode/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=e455d9732f8041cf5b9c388e345a641fd15707860f928e94507b1961256a6923 +$(package)_patches=cmake_fixups.patch define $(package)_set_vars -$(package)_config_opts=--disable-shared --without-tools --without-tests --without-png -$(package)_config_opts += --disable-gprof --disable-gcov --disable-mudflap -$(package)_config_opts += --disable-dependency-tracking --enable-option-checking -$(package)_config_opts_linux=--with-pic -$(package)_config_opts_android=--with-pic +$(package)_config_opts := -DWITH_TOOLS=NO -DWITH_TESTS=NO -DGPROF=OFF -DCOVERAGE=OFF +$(package)_config_opts += -DCMAKE_DISABLE_FIND_PACKAGE_PNG=TRUE -DWITHOUT_PNG=ON +$(package)_config_opts += -DCMAKE_DISABLE_FIND_PACKAGE_ICONV=TRUE $(package)_cflags += -Wno-int-conversion -Wno-implicit-function-declaration endef define $(package)_preprocess_cmds - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub use + patch -p1 < $($(package)_patch_dir)/cmake_fixups.patch endef + define $(package)_config_cmds - $($(package)_autoconf) + $($(package)_cmake) -S . -B . endef define $(package)_build_cmds @@ -28,7 +28,3 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef - -define $(package)_postprocess_cmds - rm lib/*.la -endef diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index d09ad75eec..0acf4cf565 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,9 +1,9 @@ package=qt -$(package)_version=5.15.11 +$(package)_version=5.15.13 $(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules $(package)_suffix=everywhere-opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=425ad301acd91ca66c10c0dabee0704e2d0cd2801a6b670115800cbb95f84846 +$(package)_sha256_hash=4cca51dcc1f22ceeee6b3e33cd1c3a60b14e85e24644dca3af89a2c2989ab809 $(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_linguist_tools = lrelease lupdate lconvert @@ -15,7 +15,6 @@ $(package)_patches += no-xlib.patch $(package)_patches += fix_android_jni_static.patch $(package)_patches += dont_hardcode_pwd.patch $(package)_patches += qtbase-moc-ignore-gcc-macro.patch -$(package)_patches += use_android_ndk23.patch $(package)_patches += rcc_hardcode_timestamp.patch $(package)_patches += duplicate_lcqpafonts.patch $(package)_patches += guix_cross_lib_path.patch @@ -25,10 +24,10 @@ $(package)_patches += utc_from_string_no_optimize.patch $(package)_patches += windows_lto.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=a31785948c640b7c66d9fe2db4993728ca07f64e41c560b3625ad191b276ff20 +$(package)_qttranslations_sha256_hash=24d4c58bc2a40c0f44f59ee64af4192c7d0038c1e45af61646cfc5b65058f271 $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=7cd847ae6ff09416df617136eadcaf0eb98e3bc9b89979219a3ea8111fb8d339 +$(package)_qttools_sha256_hash=57c9794c572c4e02871f2e7581525752b0cf85ea16cfab23a4ac9ba7b39a5d34 $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) @@ -177,6 +176,7 @@ $(package)_config_opts_mingw32 += -xplatform win32-g++ $(package)_config_opts_mingw32 += "QMAKE_CFLAGS = '$($(package)_cflags) $($(package)_cppflags)'" $(package)_config_opts_mingw32 += "QMAKE_CXX = '$($(package)_cxx)'" $(package)_config_opts_mingw32 += "QMAKE_CXXFLAGS = '$($(package)_cxxflags) $($(package)_cppflags)'" +$(package)_config_opts_mingw32 += "QMAKE_LINK = '$($(package)_cxx)'" $(package)_config_opts_mingw32 += "QMAKE_LFLAGS = '$($(package)_ldflags)'" $(package)_config_opts_mingw32 += "QMAKE_LIB = '$($(package)_ar) rc'" $(package)_config_opts_mingw32 += -device-option CROSS_COMPILE="$(host)-" @@ -245,7 +245,6 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ - patch -p1 -i $($(package)_patch_dir)/use_android_ndk23.patch && \ patch -p1 -i $($(package)_patch_dir)/memory_resource.patch && \ patch -p1 -i $($(package)_patch_dir)/rcc_hardcode_timestamp.patch && \ patch -p1 -i $($(package)_patch_dir)/duplicate_lcqpafonts.patch && \ diff --git a/depends/packages/sqlite.mk b/depends/packages/sqlite.mk index 6809b39113..15bc0f4d7a 100644 --- a/depends/packages/sqlite.mk +++ b/depends/packages/sqlite.mk @@ -7,12 +7,7 @@ $(package)_sha256_hash=5af07de982ba658fd91a03170c945f99c971f6955bc79df3266544373 define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-readline --disable-dynamic-extensions --enable-option-checking $(package)_config_opts+= --disable-rtree --disable-fts4 --disable-fts5 -$(package)_config_opts_linux=--with-pic -$(package)_config_opts_freebsd=--with-pic -$(package)_config_opts_netbsd=--with-pic -$(package)_config_opts_openbsd=--with-pic # We avoid using `--enable-debug` because it overrides CFLAGS, a behavior we want to prevent. -$(package)_cflags_debug += -g $(package)_cppflags_debug += -DSQLITE_DEBUG $(package)_cppflags+=-DSQLITE_DQS=0 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED $(package)_cppflags+=-DSQLITE_OMIT_SHARED_CACHE -DSQLITE_OMIT_JSON -DSQLITE_LIKE_DOESNT_MATCH_BLOBS diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index cc78999dbb..bfa5e97c60 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -11,11 +11,6 @@ define $(package)_set_vars $(package)_config_opts += --without-libsodium --without-libgssapi_krb5 --without-pgm --without-norm --without-vmci $(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov --disable-dependency-tracking $(package)_config_opts += --disable-Werror --disable-drafts --enable-option-checking - $(package)_config_opts_linux=--with-pic - $(package)_config_opts_freebsd=--with-pic - $(package)_config_opts_netbsd=--with-pic - $(package)_config_opts_openbsd=--with-pic - $(package)_config_opts_android=--with-pic endef define $(package)_preprocess_cmds diff --git a/depends/patches/boost/process_macos_sdk.patch b/depends/patches/boost/process_macos_sdk.patch deleted file mode 100644 index ebc556d972..0000000000 --- a/depends/patches/boost/process_macos_sdk.patch +++ /dev/null @@ -1,27 +0,0 @@ -Fix Boost Process compilation with macOS 14 SDK. -Can be dropped with Boost 1.84.0. -https://github.com/boostorg/process/pull/343. -https://github.com/boostorg/process/issues/342. - -diff --git a/boost/process/detail/posix/handles.hpp b/boost/process/detail/posix/handles.hpp -index cd9e1ce5a..304e77b1c 100644 ---- a/boost/process/detail/posix/handles.hpp -+++ b/boost/process/detail/posix/handles.hpp -@@ -33,7 +33,7 @@ inline std::vector<native_handle_type> get_handles(std::error_code & ec) - else - ec.clear(); - -- auto my_fd = ::dirfd(dir.get()); -+ auto my_fd = dirfd(dir.get()); - - struct ::dirent * ent_p; - -@@ -117,7 +117,7 @@ struct limit_handles_ : handler_base_ext - return; - } - -- auto my_fd = ::dirfd(dir); -+ auto my_fd = dirfd(dir); - struct ::dirent * ent_p; - - while ((ent_p = readdir(dir)) != nullptr) diff --git a/depends/patches/qrencode/cmake_fixups.patch b/depends/patches/qrencode/cmake_fixups.patch new file mode 100644 index 0000000000..7518d756cb --- /dev/null +++ b/depends/patches/qrencode/cmake_fixups.patch @@ -0,0 +1,23 @@ +cmake: set minimum version to 3.5 + +Correct some dev warning output. + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 773e037..a558145 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -1,4 +1,4 @@ +-cmake_minimum_required(VERSION 3.1.0) ++cmake_minimum_required(VERSION 3.5) + + project(QRencode VERSION 4.1.1 LANGUAGES C) + +@@ -20,7 +20,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + set(CMAKE_THREAD_PREFER_PTHREAD ON) + find_package(Threads) + find_package(PNG) +-find_package(Iconv) ++find_package(ICONV) + + if(CMAKE_USE_PTHREADS_INIT) + add_definitions(-DHAVE_LIBPTHREAD=1) diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch index 89c96026fb..79824f244a 100644 --- a/depends/patches/qt/fix_android_jni_static.patch +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -15,4 +15,3 @@ QWindowSystemInterfacePrivate::TabletEvent::setPlatformSynthesizesMouse(false); m_javaVM = vm; - diff --git a/depends/patches/qt/memory_resource.patch b/depends/patches/qt/memory_resource.patch index 650c328528..312f0669f6 100644 --- a/depends/patches/qt/memory_resource.patch +++ b/depends/patches/qt/memory_resource.patch @@ -17,7 +17,7 @@ and https://bugreports.qt.io/browse/QTBUG-114316 --- a/qtbase/src/corelib/global/qcompilerdetection.h +++ b/qtbase/src/corelib/global/qcompilerdetection.h -@@ -1050,16 +1050,22 @@ +@@ -1055,16 +1055,22 @@ # endif // !_HAS_CONSTEXPR # endif // !__GLIBCXX__ && !_LIBCPP_VERSION # endif // Q_OS_QNX diff --git a/depends/patches/qt/use_android_ndk23.patch b/depends/patches/qt/use_android_ndk23.patch deleted file mode 100644 index f22367d527..0000000000 --- a/depends/patches/qt/use_android_ndk23.patch +++ /dev/null @@ -1,13 +0,0 @@ -Use Android NDK r23 LTS - ---- old/qtbase/mkspecs/features/android/default_pre.prf -+++ new/qtbase/mkspecs/features/android/default_pre.prf -@@ -76,7 +76,7 @@ else: equals(QT_ARCH, x86_64): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/x86_64-linux- - else: equals(QT_ARCH, arm64-v8a): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/aarch64-linux-android- - else: CROSS_COMPILE = $$NDK_LLVM_PATH/bin/arm-linux-androideabi- - --QMAKE_RANLIB = $${CROSS_COMPILE}ranlib -+QMAKE_RANLIB = $$NDK_LLVM_PATH/bin/llvm-ranlib - QMAKE_LINK_SHLIB = $$QMAKE_LINK - QMAKE_LFLAGS = - diff --git a/doc/README.md b/doc/README.md index 446684b482..7b6dacaf4f 100644 --- a/doc/README.md +++ b/doc/README.md @@ -59,7 +59,6 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Translation Strings Policy](translation_strings_policy.md) - [JSON-RPC Interface](JSON-RPC-interface.md) - [Unauthenticated REST Interface](REST-interface.md) -- [Shared Libraries](shared-libraries.md) - [BIPS](bips.md) - [Dnsseed Policy](dnsseed-policy.md) - [Benchmarking](benchmarking.md) diff --git a/doc/build-unix.md b/doc/build-unix.md index bf367fc421..de54fb4eeb 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -30,7 +30,7 @@ tuned to conserve memory with additional CXXFLAGS: Alternatively, or in addition, debugging information can be skipped for compilation. The default compile flags are `-g -O2`, and can be changed with: - ./configure CXXFLAGS="-O2" + ./configure CXXFLAGS="-g0" Finally, clang (often less resource hungry) can be used instead of gcc, which is used by default: @@ -81,7 +81,7 @@ To build without GUI pass `--without-gui`. To build with Qt 5 you need the following: - sudo apt-get install libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools + sudo apt-get install qtbase5-dev qttools5-dev qttools5-dev-tools Additionally, to support Wayland protocol for modern desktop environments: diff --git a/doc/dependencies.md b/doc/dependencies.md index 237f617c02..1e48d225b0 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -30,7 +30,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Fontconfig](../depends/packages/fontconfig.mk) | [link](https://www.freedesktop.org/wiki/Software/fontconfig/) | [2.12.6](https://github.com/bitcoin/bitcoin/pull/23495) | 2.6 | Yes | | [FreeType](../depends/packages/freetype.mk) | [link](https://freetype.org) | [2.11.0](https://github.com/bitcoin/bitcoin/commit/01544dd78ccc0b0474571da854e27adef97137fb) | 2.3.0 | Yes | | [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [4.1.1](https://github.com/bitcoin/bitcoin/pull/27312) | | No | -| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.11](https://github.com/bitcoin/bitcoin/pull/28769) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | +| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.13](https://github.com/bitcoin/bitcoin/pull/29732) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | ### Networking | Dependency | Releases | Version used | Minimum required | Runtime | diff --git a/doc/descriptors.md b/doc/descriptors.md index 1baf652f30..3b94ec03e4 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -243,7 +243,18 @@ Often it is useful to communicate a description of scripts along with the necessary private keys. For this reason, anywhere a public key or xpub is supported, a private key in WIF format or xprv may be provided instead. This is useful when private keys are necessary for hardened derivation -steps, or for dumping wallet descriptors including private key material. +steps, for signing transactions, or for dumping wallet descriptors +including private key material. + +For example, after importing the following 2-of-3 multisig descriptor +into a wallet, one could use `signrawtransactionwithwallet` +to sign a transaction with the first key: +``` +sh(multi(2,xprv.../84'/0'/0'/0/0,xpub1...,xpub2...)) +``` +Note how the first key is an xprv private key with a specific derivation path, +while the other two are public keys. + ### Compatibility with old wallets diff --git a/doc/design/assumeutxo.md b/doc/design/assumeutxo.md index f527ac0f2d..a4980729d0 100644 --- a/doc/design/assumeutxo.md +++ b/doc/design/assumeutxo.md @@ -16,7 +16,7 @@ load it. A pruned node can load a snapshot. To save space, it's possible to delete the snapshot file as soon as `loadtxoutset` finishes. -The minimum `-dbcache` setting is 550 MiB, but this functionality ignores that +The minimum `-prune` setting is 550 MiB, but this functionality ignores that minimum and uses at least 1100 MiB. As the background sync continues there will be temporarily two chainstate diff --git a/doc/design/libraries.md b/doc/design/libraries.md index 7cda64e713..3346c8e81b 100644 --- a/doc/design/libraries.md +++ b/doc/design/libraries.md @@ -4,10 +4,9 @@ |--------------------------|-------------| | *libbitcoin_cli* | RPC client functionality used by *bitcoin-cli* executable | | *libbitcoin_common* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_util*, but higher-level (see [Dependencies](#dependencies)). | -| *libbitcoin_consensus* | Stable, backwards-compatible consensus functionality used by *libbitcoin_node* and *libbitcoin_wallet* and also exposed as a [shared library](../shared-libraries.md). | -| *libbitcoinconsensus* | Shared library build of static *libbitcoin_consensus* library | -| *libbitcoin_kernel* | Consensus engine and support library used for validation by *libbitcoin_node* and also exposed as a [shared library](../shared-libraries.md). | -| *libbitcoinqt* | GUI functionality used by *bitcoin-qt* and *bitcoin-gui* executables | +| *libbitcoin_consensus* | Stable, backwards-compatible consensus functionality used by *libbitcoin_node* and *libbitcoin_wallet*. | +| *libbitcoin_kernel* | Consensus engine and support library used for validation by *libbitcoin_node*. | +| *libbitcoinqt* | GUI functionality used by *bitcoin-qt* and *bitcoin-gui* executables. | | *libbitcoin_ipc* | IPC functionality used by *bitcoin-node*, *bitcoin-wallet*, *bitcoin-gui* executables to communicate when [`--enable-multiprocess`](multiprocess.md) is used. | | *libbitcoin_node* | P2P and RPC server functionality used by *bitcoind* and *bitcoin-qt* executables. | | *libbitcoin_util* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_common*, but lower-level (see [Dependencies](#dependencies)). | @@ -17,7 +16,7 @@ ## Conventions -- Most libraries are internal libraries and have APIs which are completely unstable! There are few or no restrictions on backwards compatibility or rules about external dependencies. Exceptions are *libbitcoin_consensus* and *libbitcoin_kernel* which have external interfaces documented at [../shared-libraries.md](../shared-libraries.md). +- Most libraries are internal libraries and have APIs which are completely unstable! There are few or no restrictions on backwards compatibility or rules about external dependencies. An exception is *libbitcoin_kernel*, which, at some future point, will have a documented external interface. - Generally each library should have a corresponding source directory and namespace. Source code organization is a work in progress, so it is true that some namespaces are applied inconsistently, and if you look at [`libbitcoin_*_SOURCES`](../../src/Makefile.am) lists you can see that many libraries pull in files from outside their source directory. But when working with libraries, it is good to follow a consistent pattern like: diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 89c13600eb..cc3f0518e5 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -115,6 +115,8 @@ code. Use `reinterpret_cast` and `const_cast` as appropriate. - Prefer [`list initialization ({})`](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Res-list) where possible. For example `int x{0};` instead of `int x = 0;` or `int x(0);` + - Recursion is checked by clang-tidy and thus must be made explicit. Use + `NOLINTNEXTLINE(misc-no-recursion)` to suppress the check. For function calls a namespace should be specified explicitly, unless such functions have been declared within it. Otherwise, [argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl), also known as ADL, could be diff --git a/doc/dnsseed-policy.md b/doc/dnsseed-policy.md index 55a5c28258..99f48f2670 100644 --- a/doc/dnsseed-policy.md +++ b/doc/dnsseed-policy.md @@ -44,7 +44,7 @@ related to the DNS seed operation. If these expectations cannot be satisfied the operator should discontinue providing services and contact the active Bitcoin Core development team as well as posting on -[bitcoin-dev](https://lists.linuxfoundation.org/mailman/listinfo/bitcoin-dev). +[bitcoin-dev](https://groups.google.com/g/bitcoindev). Behavior outside of these expectations may be reasonable in some situations but should be discussed in public in advance. diff --git a/doc/release-notes/release-notes-25.2.md b/doc/release-notes/release-notes-25.2.md new file mode 100644 index 0000000000..3f050ebaef --- /dev/null +++ b/doc/release-notes/release-notes-25.2.md @@ -0,0 +1,74 @@ +25.2 Release Notes +================== + +Bitcoin Core version 25.2 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-25.2> + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +### Gui + +- gui#774 Fix crash on selecting "Mask values" in transaction view + +### RPC + +- #29003 rpc: fix getrawtransaction segfault + +### Wallet + +- #29176 wallet: Fix use-after-free in WalletBatch::EraseRecords +- #29510 wallet: `getrawchangeaddress` and `getnewaddress` failures should not affect keypools for descriptor wallets + +### P2P and network changes + +- #29412 p2p: Don't process mutated blocks +- #29524 p2p: Don't consider blocks mutated if they don't connect to known prev block + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Martin Zumsande +- Sebastian Falbesoner +- MarcoFalke +- UdjinM6 +- dergoegge +- Greg Sanders + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-26.1.md b/doc/release-notes/release-notes-26.1.md new file mode 100644 index 0000000000..b5c6e63007 --- /dev/null +++ b/doc/release-notes/release-notes-26.1.md @@ -0,0 +1,106 @@ +26.1 Release Notes +================== + +Bitcoin Core version 26.1 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-26.1/> + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 11.0+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +### Wallet + +- #28994 wallet: skip BnB when SFFO is enabled +- #28920 wallet: birth time update during tx scanning +- #29176 wallet: Fix use-after-free in WalletBatch::EraseRecords +- #29510 wallet: getrawchangeaddress and getnewaddress failures should not affect keypools for descriptor wallets + +### RPC + +- #29003 rpc: fix getrawtransaction segfault +- #28784 rpc: keep .cookie file if it was not generated + +### Logs + +- #29227 log mempool loading progress + +### P2P and network changes + +- #29200 net: create I2P sessions using both ECIES-X25519 and ElGamal encryption +- #29412 p2p: Don't process mutated blocks +- #29524 p2p: Don't consider blocks mutated if they don't connect to known prev block + +### Build + +- #29127 Use hardened runtime on macOS release builds. +- #29195 build: Fix -Xclang -internal-isystem option + +### CI + +- #28992 ci: Use Ubuntu 24.04 Noble for asan,tsan,tidy,fuzz +- #29080 ci: Set HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK to avoid unrelated failures +- #29610 ci: Fix "macOS native" job + +### Miscellaneous + +- #28391 refactor: Simplify CTxMempool/BlockAssembler fields, remove some external mapTx access +- #29179 test: wallet rescan with reorged parent + IsFromMe child in mempool +- #28791 snapshots: don't core dump when running -checkblockindex after loadtxoutset +- #29357 test: Drop x modifier in fsbridge::fopen call for MinGW builds +- #29529 fuzz: restrict fopencookie usage to Linux & FreeBSD + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- dergoegge +- fanquake +- furszy +- glozow +- Greg Sanders +- Hennadii Stepanov +- Jon Atack +- MarcoFalke +- Mark Friedenbach +- Martin Zumsande +- Murch +- Roman Zeyde +- stickies-v +- UdjinM6 + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). + diff --git a/doc/release-notes/release-notes-27.0.md b/doc/release-notes/release-notes-27.0.md new file mode 100644 index 0000000000..5060068328 --- /dev/null +++ b/doc/release-notes/release-notes-27.0.md @@ -0,0 +1,217 @@ +Bitcoin Core version 27.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-27.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux Kernel 3.17+, macOS 11.0+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +libbitcoinconsensus +------------------- + +- libbitcoinconsensus is deprecated and will be removed for v28. This library has + existed for nearly 10 years with very little known uptake or impact. It has + become a maintenance burden. + + The underlying functionality does not change between versions, so any users of + the library can continue to use the final release indefinitely, with the + understanding that Taproot is its final consensus update. + + In the future, libbitcoinkernel will provide a much more useful API that is + aware of the UTXO set, and therefore be able to fully validate transactions and + blocks. (#29189) + +mempool.dat compatibility +------------------------- + +- The `mempool.dat` file created by -persistmempool or the savemempool RPC will + be written in a new format. This new format includes the XOR'ing of transaction + contents to mitigate issues where external programs (such as anti-virus) attempt + to interpret and potentially modify the file. + + This new format can not be read by previous software releases. To allow for a + downgrade, a temporary setting `-persistmempoolv1` has been added to fall back + to the legacy format. (#28207) + +P2P and network changes +----------------------- + +- BIP324 v2 transport is now enabled by default. It remains possible to disable v2 + by running with `-v2transport=0`. (#29347) +- Manual connection options (`-connect`, `-addnode` and `-seednode`) will + now follow `-v2transport` to connect with v2 by default. They will retry with + v1 on failure. (#29058) + +- Network-adjusted time has been removed from consensus code. It is replaced + with (unadjusted) system time. The warning for a large median time offset + (70 minutes or more) is kept. This removes the implicit security assumption of + requiring an honest majority of outbound peers, and increases the importance + of the node operator ensuring their system time is (and stays) correct to not + fall out of consensus with the network. (#28956) + +Mempool Policy Changes +---------------------- + +- Opt-in Topologically Restricted Until Confirmation (TRUC) Transactions policy + (aka v3 transaction policy) is available for use on test networks when + `-acceptnonstdtxn=1` is set. By setting the transaction version number to 3, TRUC transactions + request the application of limits on spending of their unconfirmed outputs. These + restrictions simplify the assessment of incentive compatibility of accepting or + replacing TRUC transactions, thus ensuring any replacements are more profitable for + the node and making fee-bumping more reliable. TRUC transactions are currently + nonstandard and can only be used on test networks where the standardness rules are + relaxed or disabled (e.g. with `-acceptnonstdtxn=1`). (#28948) + +External Signing +---------------- + +- Support for external signing on Windows has been disabled. It will be re-enabled + once the underlying dependency (Boost Process), has been replaced with a different + library. (#28967) + +Updated RPCs +------------ + +- The addnode RPC now follows the `-v2transport` option (now on by default, see above) for making connections. + It remains possible to specify the transport type manually with the v2transport argument of addnode. (#29239) + +Build System +------------ + +- A C++20 capable compiler is now required to build Bitcoin Core. (#28349) +- MacOS releases are configured to use the hardened runtime libraries (#29127) + +Wallet +------ + +- The CoinGrinder coin selection algorithm has been introduced to mitigate unnecessary + large input sets and lower transaction costs at high feerates. CoinGrinder + searches for the input set with minimal weight. Solutions found by + CoinGrinder will produce a change output. CoinGrinder is only active at + elevated feerates (default: 30+ sat/vB, based on `-consolidatefeerate`×3). (#27877) +- The Branch And Bound coin selection algorithm will be disabled when the subtract fee + from outputs feature is used. (#28994) +- If the birth time of a descriptor is detected to be later than the first transaction + involving that descriptor, the birth time will be reset to the earlier time. (#28920) + +Low-level changes +================= + +Pruning +------- + +- When pruning during initial block download, more blocks will be pruned at each + flush in order to speed up the syncing of such nodes. (#20827) + +Init +---- + +- Various fixes to prevent issues where subsequent instances of Bitcoin Core would + result in deletion of files in use by an existing instance. (#28784, #28946) +- Improved handling of empty `settings.json` files. (#29144) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 22388o⚡️ +- Aaron Clauson +- Amiti Uttarwar +- Andrew Toth +- Anthony Towns +- Antoine Poinsot +- Ava Chow +- Brandon Odiwuor +- brunoerg +- Chris Stewart +- Cory Fields +- dergoegge +- djschnei21 +- Fabian Jahr +- fanquake +- furszy +- Gloria Zhao +- Greg Sanders +- Hennadii Stepanov +- Hernan Marino +- iamcarlos94 +- ismaelsadeeq +- Jameson Lopp +- Jesse Barton +- John Moffett +- Jon Atack +- josibake +- jrakibi +- Justin Dhillon +- Kashif Smith +- kevkevin +- Kristaps Kaupe +- L0la L33tz +- Luke Dashjr +- Lőrinc +- marco +- MarcoFalke +- Mark Friedenbach +- Marnix +- Martin Leitner-Ankerl +- Martin Zumsande +- Max Edwards +- Murch +- muxator +- naiyoma +- Nikodemas Tuckus +- ns-xvrn +- pablomartin4btc +- Peter Todd +- Pieter Wuille +- Richard Myers +- Roman Zeyde +- Russell Yanofsky +- Ryan Ofsky +- Sebastian Falbesoner +- Sergi Delgado Segura +- Sjors Provoost +- stickies-v +- stratospher +- Supachai Kheawjuy +- TheCharlatan +- UdjinM6 +- Vasil Dimov +- w0xlt +- willcl-ark + + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/shared-libraries.md b/doc/shared-libraries.md deleted file mode 100644 index 3a448c6556..0000000000 --- a/doc/shared-libraries.md +++ /dev/null @@ -1,78 +0,0 @@ -Shared Libraries -================ - -## bitcoinconsensus -***This library is deprecated and will be removed in v28*** - -The purpose of this library is to make the verification functionality that is critical to Bitcoin's consensus available to other applications, e.g. to language bindings. - -### API - -The interface is defined in the C header `bitcoinconsensus.h` located in `src/script/bitcoinconsensus.h`. - -#### Version - -`bitcoinconsensus_version` returns an `unsigned int` with the API version *(currently `2`)*. - -#### Script Validation - -`bitcoinconsensus_verify_script`, `bitcoinconsensus_verify_script_with_amount` and `bitcoinconsensus_verify_script_with_spent_outputs` return an `int` with the status of the verification. It will be `1` if the input script correctly spends the previous output `scriptPubKey`. - -##### Parameters -###### bitcoinconsensus_verify_script -- `const unsigned char *scriptPubKey` - The previous output script that encumbers spending. -- `unsigned int scriptPubKeyLen` - The number of bytes for the `scriptPubKey`. -- `const unsigned char *txTo` - The transaction with the input that is spending the previous output. -- `unsigned int txToLen` - The number of bytes for the `txTo`. -- `unsigned int nIn` - The index of the input in `txTo` that spends the `scriptPubKey`. -- `unsigned int flags` - The script validation flags *(see below)*. -- `bitcoinconsensus_error* err` - Will have the error/success code for the operation *(see below)*. - -###### bitcoinconsensus_verify_script_with_amount -- `const unsigned char *scriptPubKey` - The previous output script that encumbers spending. -- `unsigned int scriptPubKeyLen` - The number of bytes for the `scriptPubKey`. -- `int64_t amount` - The amount spent in the input -- `const unsigned char *txTo` - The transaction with the input that is spending the previous output. -- `unsigned int txToLen` - The number of bytes for the `txTo`. -- `unsigned int nIn` - The index of the input in `txTo` that spends the `scriptPubKey`. -- `unsigned int flags` - The script validation flags *(see below)*. -- `bitcoinconsensus_error* err` - Will have the error/success code for the operation *(see below)*. - -###### bitcoinconsensus_verify_script_with_spent_outputs -- `const unsigned char *scriptPubKey` - The previous output script that encumbers spending. -- `unsigned int scriptPubKeyLen` - The number of bytes for the `scriptPubKey`. -- `int64_t amount` - The amount spent in the input -- `const unsigned char *txTo` - The transaction with the input that is spending the previous output. -- `unsigned int txToLen` - The number of bytes for the `txTo`. -- `UTXO *spentOutputs` - Previous outputs spent in the transaction. `UTXO` is a struct composed by `const unsigned char *scriptPubKey`, `unsigned int scriptPubKeySize` (the number of bytes for the `scriptPubKey`) and `unsigned int value`. -- `unsigned int spentOutputsLen` - The number of bytes for the `spentOutputs`. -- `unsigned int nIn` - The index of the input in `txTo` that spends the `scriptPubKey`. -- `unsigned int flags` - The script validation flags *(see below)*. -- `bitcoinconsensus_error* err` - Will have the error/success code for the operation *(see below)*. - -##### Script Flags -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NONE` -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_P2SH` - Evaluate P2SH ([BIP16](https://github.com/bitcoin/bips/blob/master/bip-0016.mediawiki)) subscripts -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_DERSIG` - Enforce strict DER ([BIP66](https://github.com/bitcoin/bips/blob/master/bip-0066.mediawiki)) compliance -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NULLDUMMY` - Enforce NULLDUMMY ([BIP147](https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki)) -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY` - Enable CHECKLOCKTIMEVERIFY ([BIP65](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki)) -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKSEQUENCEVERIFY` - Enable CHECKSEQUENCEVERIFY ([BIP112](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki)) -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS` - Enable WITNESS ([BIP141](https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki)) -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT` - Enable TAPROOT ([BIP340](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki), [BIP341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki), [BIP342](https://github.com/bitcoin/bips/blob/master/bip-0342.mediawiki)) - -##### Errors -- `bitcoinconsensus_ERR_OK` - No errors with input parameters *(see the return value of `bitcoinconsensus_verify_script` for the verification status)* -- `bitcoinconsensus_ERR_TX_INDEX` - An invalid index for `txTo` -- `bitcoinconsensus_ERR_TX_SIZE_MISMATCH` - `txToLen` did not match with the size of `txTo` -- `bitcoinconsensus_ERR_DESERIALIZE` - An error deserializing `txTo` -- `bitcoinconsensus_ERR_AMOUNT_REQUIRED` - Input amount is required if WITNESS is used -- `bitcoinconsensus_ERR_INVALID_FLAGS` - Script verification `flags` are invalid (i.e. not part of the libconsensus interface) -- `bitcoinconsensus_ERR_SPENT_OUTPUTS_REQUIRED` - Spent outputs are required if TAPROOT is used -- `bitcoinconsensus_ERR_SPENT_OUTPUTS_MISMATCH` - Spent outputs size doesn't match tx inputs size - -### Example Implementations -- [NBitcoin](https://github.com/MetacoSA/NBitcoin/blob/5e1055cd7c4186dee4227c344af8892aea54faec/NBitcoin/Script.cs#L979-#L1031) (.NET Bindings) -- [node-libbitcoinconsensus](https://github.com/bitpay/node-libbitcoinconsensus) (Node.js Bindings) -- [java-libbitcoinconsensus](https://github.com/dexX7/java-libbitcoinconsensus) (Java Bindings) -- [bitcoinconsensus-php](https://github.com/Bit-Wasp/bitcoinconsensus-php) (PHP Bindings) -- [rust-bitcoinconsensus](https://github.com/rust-bitcoin/rust-bitcoinconsensus) (Rust Bindings)
\ No newline at end of file diff --git a/libbitcoinconsensus.pc.in b/libbitcoinconsensus.pc.in deleted file mode 100644 index 1ceab280bb..0000000000 --- a/libbitcoinconsensus.pc.in +++ /dev/null @@ -1,10 +0,0 @@ -prefix=@prefix@ -exec_prefix=@exec_prefix@ -libdir=@libdir@ -includedir=@includedir@ - -Name: @PACKAGE_NAME@ consensus library -Description: Library for the Bitcoin consensus protocol. -Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lbitcoinconsensus -Cflags: -I${includedir} diff --git a/src/.clang-tidy b/src/.clang-tidy index e4b789dcaa..a00400f083 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -6,6 +6,7 @@ bugprone-string-constructor, bugprone-use-after-move, bugprone-lambda-function-name, misc-unused-using-decls, +misc-no-recursion, modernize-use-default-member-init, modernize-use-emplace, modernize-use-noexcept, diff --git a/src/Makefile.am b/src/Makefile.am index 1f55bfbafd..7673d2f545 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -39,9 +39,6 @@ LIBSECP256K1=secp256k1/libsecp256k1.la if ENABLE_ZMQ LIBBITCOIN_ZMQ=libbitcoin_zmq.a endif -if BUILD_BITCOIN_LIBS -LIBBITCOINCONSENSUS=libbitcoinconsensus.la -endif if BUILD_BITCOIN_KERNEL_LIB LIBBITCOINKERNEL=libbitcoinkernel.la endif @@ -302,6 +299,7 @@ BITCOIN_CORE_H = \ util/error.h \ util/exception.h \ util/fastrange.h \ + util/feefrac.h \ util/fees.h \ util/fs.h \ util/fs_helpers.h \ @@ -322,6 +320,7 @@ BITCOIN_CORE_H = \ util/sock.h \ util/spanparsing.h \ util/string.h \ + util/subprocess.hpp \ util/syserror.h \ util/task_runner.h \ util/thread.h \ @@ -648,7 +647,6 @@ libbitcoin_consensus_a_SOURCES = \ primitives/transaction.h \ pubkey.cpp \ pubkey.h \ - script/bitcoinconsensus.cpp \ script/interpreter.cpp \ script/interpreter.h \ script/script.cpp \ @@ -741,6 +739,7 @@ libbitcoin_util_a_SOURCES = \ util/check.cpp \ util/error.cpp \ util/exception.cpp \ + util/feefrac.cpp \ util/fees.cpp \ util/fs.cpp \ util/fs_helpers.cpp \ @@ -983,6 +982,7 @@ libbitcoinkernel_la_SOURCES = \ util/batchpriority.cpp \ util/chaintype.cpp \ util/check.cpp \ + util/feefrac.cpp \ util/fs.cpp \ util/fs_helpers.cpp \ util/hasher.cpp \ @@ -1007,21 +1007,6 @@ libbitcoinkernel_la-clientversion.l$(OBJEXT): obj/build.h endif # BUILD_BITCOIN_KERNEL_LIB # -# bitcoinconsensus library # -if BUILD_BITCOIN_LIBS -lib_LTLIBRARIES += $(LIBBITCOINCONSENSUS) - -include_HEADERS = script/bitcoinconsensus.h -libbitcoinconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_la_SOURCES) $(libbitcoin_consensus_a_SOURCES) - -libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) -libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) -libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL -DDISABLE_OPTIMIZED_SHA256 -libbitcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) - -endif -# - CTAES_DIST = crypto/ctaes/bench.c CTAES_DIST += crypto/ctaes/ctaes.c CTAES_DIST += crypto/ctaes/ctaes.h diff --git a/src/Makefile.minisketch.include b/src/Makefile.minisketch.include index 1363bec34e..c6f894f0ca 100644 --- a/src/Makefile.minisketch.include +++ b/src/Makefile.minisketch.include @@ -12,10 +12,6 @@ LIBMINISKETCH_CPPFLAGS += -DHAVE_CLMUL MINISKETCH_LIBS += $(LIBMINISKETCH_CLMUL) endif -if HAVE_CLZ -LIBMINISKETCH_CPPFLAGS += -DHAVE_CLZ -endif - EXTRA_LIBRARIES += $(MINISKETCH_LIBS) minisketch_libminisketch_clmul_a_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 9f9bdbbd0c..cf88a02b95 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -93,6 +93,7 @@ BITCOIN_TESTS =\ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ test/disconnected_transactions.cpp \ + test/feefrac_tests.cpp \ test/flatfile_tests.cpp \ test/fs_tests.cpp \ test/getarg_tests.cpp \ @@ -313,7 +314,9 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/descriptor_parse.cpp \ test/fuzz/deserialize.cpp \ test/fuzz/eval_script.cpp \ + test/fuzz/feefrac.cpp \ test/fuzz/fee_rate.cpp \ + test/fuzz/feeratediagram.cpp \ test/fuzz/fees.cpp \ test/fuzz/flatfile.cpp \ test/fuzz/float.cpp \ @@ -362,7 +365,6 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/rpc.cpp \ test/fuzz/script.cpp \ test/fuzz/script_assets_test_minimizer.cpp \ - test/fuzz/script_bitcoin_consensus.cpp \ test/fuzz/script_descriptor_cache.cpp \ test/fuzz/script_flags.cpp \ test/fuzz/script_format.cpp \ diff --git a/src/addrdb.cpp b/src/addrdb.cpp index f8d4240f3f..14dc314c36 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -195,7 +195,7 @@ util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgro auto check_addrman = std::clamp<int32_t>(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); bool deterministic = HasTestOption(args, "addrman"); // use a deterministic addrman only for tests - auto addrman{std::make_unique<AddrMan>(netgroupman, /*deterministic=*/deterministic, /*consistency_check_ratio=*/check_addrman)}; + auto addrman{std::make_unique<AddrMan>(netgroupman, deterministic, /*consistency_check_ratio=*/check_addrman)}; const auto start{SteadyClock::now()}; const auto path_addr{args.GetDataDirNet() / "peers.dat"}; @@ -204,7 +204,7 @@ util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgro LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->Size(), Ticks<std::chrono::milliseconds>(SteadyClock::now() - start)); } catch (const DbNotFoundError&) { // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/deterministic, /*consistency_check_ratio=*/check_addrman); + addrman = std::make_unique<AddrMan>(netgroupman, deterministic, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating peers.dat because the file was not found (%s)\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const InvalidAddrManVersionError&) { @@ -212,7 +212,7 @@ util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgro return util::Error{strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again."))}; } // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/deterministic, /*consistency_check_ratio=*/check_addrman); + addrman = std::make_unique<AddrMan>(netgroupman, deterministic, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { diff --git a/src/bench/bip324_ecdh.cpp b/src/bench/bip324_ecdh.cpp index 659da0f08e..fb10c2957e 100644 --- a/src/bench/bip324_ecdh.cpp +++ b/src/bench/bip324_ecdh.cpp @@ -27,7 +27,7 @@ static void BIP324_ECDH(benchmark::Bench& bench) bench.batch(1).unit("ecdh").run([&] { CKey key; - key.Set(UCharCast(key_data.data()), UCharCast(key_data.data()) + 32, true); + key.Set(key_data.data(), key_data.data() + 32, true); EllSwiftPubKey our_ellswift(our_ellswift_data); EllSwiftPubKey their_ellswift(their_ellswift_data); diff --git a/src/bench/ellswift.cpp b/src/bench/ellswift.cpp index 82e8dec982..9441b4863e 100644 --- a/src/bench/ellswift.cpp +++ b/src/bench/ellswift.cpp @@ -17,7 +17,7 @@ static void EllSwiftCreate(benchmark::Bench& bench) bench.batch(1).unit("pubkey").run([&] { auto ret = key.EllSwiftCreate(MakeByteSpan(entropy)); /* Use the first 32 bytes of the ellswift encoded public key as next private key. */ - key.Set(UCharCast(ret.data()), UCharCast(ret.data()) + 32, true); + key.Set(ret.data(), ret.data() + 32, true); assert(key.IsValid()); /* Use the last 32 bytes of the ellswift encoded public key as next entropy. */ std::copy(ret.begin() + 32, ret.begin() + 64, MakeWritableByteSpan(entropy).begin()); diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index e7166a91cf..ee750bc1f8 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -2,15 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - #include <bench/bench.h> #include <key.h> -#if defined(HAVE_CONSENSUS_LIB) -#include <script/bitcoinconsensus.h> -#endif #include <script/script.h> #include <script/interpreter.h> #include <streams.h> @@ -63,17 +56,6 @@ static void VerifyScriptBench(benchmark::Bench& bench) &err); assert(err == SCRIPT_ERR_OK); assert(success); - -#if defined(HAVE_CONSENSUS_LIB) - DataStream stream; - stream << TX_WITH_WITNESS(txSpend); - int csuccess = bitcoinconsensus_verify_script_with_amount( - txCredit.vout[0].scriptPubKey.data(), - txCredit.vout[0].scriptPubKey.size(), - txCredit.vout[0].nValue, - (const unsigned char*)stream.data(), stream.size(), 0, flags, nullptr); - assert(csuccess == 1); -#endif }); ECC_Stop(); } diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 3eb64aa344..642af06e82 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -89,14 +89,13 @@ int main(int argc, char* argv[]) { std::cout << "Warning: " << warning.original << std::endl; } - void flushError(const std::string& debug_message) override + void flushError(const bilingual_str& message) override { - std::cerr << "Error flushing block data to disk: " << debug_message << std::endl; + std::cerr << "Error flushing block data to disk: " << message.original << std::endl; } - void fatalError(const std::string& debug_message, const bilingual_str& user_message) override + void fatalError(const bilingual_str& message) override { - std::cerr << "Error: " << debug_message << std::endl; - std::cerr << (user_message.empty() ? "A fatal internal error occurred." : user_message.original) << std::endl; + std::cerr << "Error: " << message.original << std::endl; } }; auto notifications = std::make_unique<KernelNotifications>(); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index f0e27cb675..129deeec60 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -827,7 +827,10 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co if (response.error != -1) { responseErrorMessage = strprintf(" (error code %d - \"%s\")", response.error, http_errorstring(response.error)); } - throw CConnectionFailed(strprintf("Could not connect to the server %s:%d%s\n\nMake sure the bitcoind server is running and that you are connecting to the correct RPC port.", host, port, responseErrorMessage)); + throw CConnectionFailed(strprintf("Could not connect to the server %s:%d%s\n\n" + "Make sure the bitcoind server is running and that you are connecting to the correct RPC port.\n" + "Use \"bitcoin-cli -help\" for more info.", + host, port, responseErrorMessage)); } else if (response.status == HTTP_UNAUTHORIZED) { if (failedToGetAuthCookie) { throw std::runtime_error(strprintf( diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 1e940e8f03..5a4513d281 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -40,14 +40,14 @@ void CBlockHeaderAndShortTxIDs::FillShortTxIDSelector() const { shorttxidk1 = shorttxidhash.GetUint64(1); } -uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const uint256& txhash) const { +uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const Wtxid& wtxid) const { static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids calculation assumes 6-byte shorttxids"); - return SipHashUint256(shorttxidk0, shorttxidk1, txhash) & 0xffffffffffffL; + return SipHashUint256(shorttxidk0, shorttxidk1, wtxid) & 0xffffffffffffL; } -ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector<std::pair<uint256, CTransactionRef>>& extra_txn) { +ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector<CTransactionRef>& extra_txn) { if (cmpctblock.header.IsNull() || (cmpctblock.shorttxids.empty() && cmpctblock.prefilledtxn.empty())) return READ_STATUS_INVALID; if (cmpctblock.shorttxids.size() + cmpctblock.prefilledtxn.size() > MAX_BLOCK_WEIGHT / MIN_SERIALIZABLE_TRANSACTION_WEIGHT) @@ -134,11 +134,14 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c } for (size_t i = 0; i < extra_txn.size(); i++) { - uint64_t shortid = cmpctblock.GetShortID(extra_txn[i].first); + if (extra_txn[i] == nullptr) { + continue; + } + uint64_t shortid = cmpctblock.GetShortID(extra_txn[i]->GetWitnessHash()); std::unordered_map<uint64_t, uint16_t>::iterator idit = shorttxids.find(shortid); if (idit != shorttxids.end()) { if (!have_txn[idit->second]) { - txn_available[idit->second] = extra_txn[i].second; + txn_available[idit->second] = extra_txn[i]; have_txn[idit->second] = true; mempool_count++; extra_count++; @@ -150,7 +153,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c // Note that we don't want duplication between extra_txn and mempool to // trigger this case, so we compare witness hashes first if (txn_available[idit->second] && - txn_available[idit->second]->GetWitnessHash() != extra_txn[i].second->GetWitnessHash()) { + txn_available[idit->second]->GetWitnessHash() != extra_txn[i]->GetWitnessHash()) { txn_available[idit->second].reset(); mempool_count--; extra_count--; diff --git a/src/blockencodings.h b/src/blockencodings.h index fb0f734ff8..2b1fabadd6 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -111,7 +111,7 @@ public: CBlockHeaderAndShortTxIDs(const CBlock& block); - uint64_t GetShortID(const uint256& txhash) const; + uint64_t GetShortID(const Wtxid& wtxid) const; size_t BlockTxCount() const { return shorttxids.size() + prefilledtxn.size(); } @@ -142,7 +142,7 @@ public: explicit PartiallyDownloadedBlock(CTxMemPool* poolIn) : pool(poolIn) {} // extra_txn is a list of extra transactions to look at, in <witness hash, reference> form - ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector<std::pair<uint256, CTransactionRef>>& extra_txn); + ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector<CTransactionRef>& extra_txn); bool IsTxAvailable(size_t index) const; ReadStatus FillBlock(CBlock& block, const std::vector<CTransactionRef>& vtx_missing); }; diff --git a/src/common/run_command.cpp b/src/common/run_command.cpp index 8bd5febd53..e5356490ef 100644 --- a/src/common/run_command.cpp +++ b/src/common/run_command.cpp @@ -12,39 +12,34 @@ #include <univalue.h> #ifdef ENABLE_EXTERNAL_SIGNER -#include <boost/process.hpp> +#include <util/subprocess.hpp> #endif // ENABLE_EXTERNAL_SIGNER UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in) { #ifdef ENABLE_EXTERNAL_SIGNER - namespace bp = boost::process; + namespace sp = subprocess; UniValue result_json; - bp::opstream stdin_stream; - bp::ipstream stdout_stream; - bp::ipstream stderr_stream; + std::istringstream stdout_stream; + std::istringstream stderr_stream; if (str_command.empty()) return UniValue::VNULL; - bp::child c( - str_command, - bp::std_out > stdout_stream, - bp::std_err > stderr_stream, - bp::std_in < stdin_stream - ); + auto c = sp::Popen(str_command, sp::input{sp::PIPE}, sp::output{sp::PIPE}, sp::error{sp::PIPE}); if (!str_std_in.empty()) { - stdin_stream << str_std_in << std::endl; + c.send(str_std_in); } - stdin_stream.pipe().close(); + auto [out_res, err_res] = c.communicate(); + stdout_stream.str(std::string{out_res.buf.begin(), out_res.buf.end()}); + stderr_stream.str(std::string{err_res.buf.begin(), err_res.buf.end()}); std::string result; std::string error; std::getline(stdout_stream, result); std::getline(stderr_stream, error); - c.wait(); - const int n_error = c.exit_code(); + const int n_error = c.retcode(); if (n_error) throw std::runtime_error(strprintf("RunCommandParseJSON error: process(%s) returned %d: %s\n", str_command, n_error, error)); if (!result_json.read(result)) throw std::runtime_error("Unable to parse JSON: " + result); diff --git a/src/crypto/chacha20poly1305.cpp b/src/crypto/chacha20poly1305.cpp index 3e8051c2dc..b969bb1a29 100644 --- a/src/crypto/chacha20poly1305.cpp +++ b/src/crypto/chacha20poly1305.cpp @@ -2,10 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - #include <crypto/chacha20poly1305.h> #include <crypto/common.h> @@ -30,10 +26,7 @@ void AEADChaCha20Poly1305::SetKey(Span<const std::byte> key) noexcept namespace { -#ifndef HAVE_TIMINGSAFE_BCMP -#define HAVE_TIMINGSAFE_BCMP - -int timingsafe_bcmp(const unsigned char* b1, const unsigned char* b2, size_t n) noexcept +int timingsafe_bcmp_internal(const unsigned char* b1, const unsigned char* b2, size_t n) noexcept { const unsigned char *p1 = b1, *p2 = b2; int ret = 0; @@ -42,8 +35,6 @@ int timingsafe_bcmp(const unsigned char* b1, const unsigned char* b2, size_t n) return (ret != 0); } -#endif - /** Compute poly1305 tag. chacha20 must be set to the right nonce, block 0. Will be at block 1 after. */ void ComputeTag(ChaCha20& chacha20, Span<const std::byte> aad, Span<const std::byte> cipher, Span<std::byte> tag) noexcept { @@ -97,7 +88,7 @@ bool AEADChaCha20Poly1305::Decrypt(Span<const std::byte> cipher, Span<const std: m_chacha20.Seek(nonce, 0); std::byte expected_tag[EXPANSION]; ComputeTag(m_chacha20, aad, cipher.first(cipher.size() - EXPANSION), expected_tag); - if (timingsafe_bcmp(UCharCast(expected_tag), UCharCast(cipher.last(EXPANSION).data()), EXPANSION)) return false; + if (timingsafe_bcmp_internal(UCharCast(expected_tag), UCharCast(cipher.last(EXPANSION).data()), EXPANSION)) return false; // Decrypt (starting at block 1). m_chacha20.Crypt(cipher.first(plain1.size()), plain1); diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 4c7bb6f20f..301f22a248 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -20,7 +20,7 @@ #include <asm/hwcap.h> #endif -#if defined(MAC_OSX) && defined(ENABLE_ARM_SHANI) +#if defined(__APPLE__) && defined(ENABLE_ARM_SHANI) #include <sys/types.h> #include <sys/sysctl.h> #endif @@ -670,7 +670,7 @@ std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implem #endif #endif -#if defined(MAC_OSX) +#if defined(__APPLE__) int val = 0; size_t len = sizeof(val); if (sysctlbyname("hw.optional.arm.FEAT_SHA256", &val, &len, nullptr, 0) == 0) { diff --git a/src/cuckoocache.h b/src/cuckoocache.h index cb0b362143..df320ed465 100644 --- a/src/cuckoocache.h +++ b/src/cuckoocache.h @@ -359,7 +359,7 @@ public: * @param bytes the approximate number of bytes to use for this data * structure * @returns A pair of the maximum number of elements storable (see setup() - * documentation for more detail) and the approxmiate total size of these + * documentation for more detail) and the approximate total size of these * elements in bytes or std::nullopt if the size requested is too large. */ std::optional<std::pair<uint32_t, size_t>> setup_bytes(size_t bytes) diff --git a/src/external_signer.cpp b/src/external_signer.cpp index 749bb5f74f..ff159a2aa5 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -62,12 +62,12 @@ bool ExternalSigner::Enumerate(const std::string& command, std::vector<ExternalS UniValue ExternalSigner::DisplayAddress(const std::string& descriptor) const { - return RunCommandParseJSON(m_command + " --fingerprint \"" + m_fingerprint + "\"" + NetworkArg() + " displayaddress --desc \"" + descriptor + "\""); + return RunCommandParseJSON(m_command + " --fingerprint " + m_fingerprint + NetworkArg() + " displayaddress --desc " + descriptor); } UniValue ExternalSigner::GetDescriptors(const int account) { - return RunCommandParseJSON(m_command + " --fingerprint \"" + m_fingerprint + "\"" + NetworkArg() + " getdescriptors --account " + strprintf("%d", account)); + return RunCommandParseJSON(m_command + " --fingerprint " + m_fingerprint + NetworkArg() + " getdescriptors --account " + strprintf("%d", account)); } bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::string& error) @@ -93,8 +93,8 @@ bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::str return false; } - const std::string command = m_command + " --stdin --fingerprint \"" + m_fingerprint + "\"" + NetworkArg(); - const std::string stdinStr = "signtx \"" + EncodeBase64(ssTx.str()) + "\""; + const std::string command = m_command + " --stdin --fingerprint " + m_fingerprint + NetworkArg(); + const std::string stdinStr = "signtx " + EncodeBase64(ssTx.str()); const UniValue signer_result = RunCommandParseJSON(command, stdinStr); diff --git a/src/index/base.cpp b/src/index/base.cpp index eb7de2ea0a..a203ce4a9f 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -31,7 +31,7 @@ template <typename... Args> void BaseIndex::FatalErrorf(const char* fmt, const Args&... args) { auto message = tfm::format(fmt, args...); - node::AbortNode(m_chain->context()->shutdown, m_chain->context()->exit_status, message); + node::AbortNode(m_chain->context()->shutdown, m_chain->context()->exit_status, Untranslated(message)); } CBlockLocator GetLocator(interfaces::Chain& chain, const uint256& block_hash) @@ -162,9 +162,9 @@ void BaseIndex::Sync() const CBlockIndex* pindex_next = WITH_LOCK(cs_main, return NextSyncBlock(pindex, m_chainstate->m_chain)); if (!pindex_next) { SetBestBlockIndex(pindex); - m_synced = true; // No need to handle errors in Commit. See rationale above. Commit(); + m_synced = true; break; } if (pindex_next->pprev != pindex && !Rewind(pindex, pindex_next->pprev)) { diff --git a/src/init.cpp b/src/init.cpp index 0aa04755cb..e8e6391af7 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -553,16 +553,12 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control host and port to use if onion listening enabled (default: %s). If no port is specified, the default port of %i will be used.", DEFAULT_TOR_CONTROL, DEFAULT_TOR_CONTROL_PORT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::CONNECTION); #ifdef USE_UPNP -#if USE_UPNP - argsman.AddArg("-upnp", "Use UPnP to map the listening port (default: 1 when listening and no -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); -#else - argsman.AddArg("-upnp", strprintf("Use UPnP to map the listening port (default: %u)", 0), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); -#endif + argsman.AddArg("-upnp", strprintf("Use UPnP to map the listening port (default: %u)", DEFAULT_UPNP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); #else hidden_args.emplace_back("-upnp"); #endif #ifdef USE_NATPMP - argsman.AddArg("-natpmp", strprintf("Use NAT-PMP to map the listening port (default: %s)", DEFAULT_NATPMP ? "1 when listening and no -proxy" : "0"), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-natpmp", strprintf("Use NAT-PMP to map the listening port (default: %u)", DEFAULT_NATPMP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); #else hidden_args.emplace_back("-natpmp"); #endif // USE_NATPMP @@ -1031,7 +1027,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) if (args.IsArgSet("-test")) { if (chainparams.GetChainType() != ChainType::REGTEST) { - return InitError(Untranslated("-test=<option> should only be used in functional tests")); + return InitError(Untranslated("-test=<option> can only be used with regtest")); } const std::vector<std::string> options = args.GetArgs("-test"); for (const std::string& option : options) { @@ -1757,7 +1753,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Start indexes initial sync if (!StartIndexBackgroundSync(node)) { bilingual_str err_str = _("Failed to start indexes, shutting down.."); - chainman.GetNotifications().fatalError(err_str.original, err_str); + chainman.GetNotifications().fatalError(err_str); return; } // Load mempool from disk diff --git a/src/kernel/chainparams.cpp b/src/kernel/chainparams.cpp index 264a2fd681..26c261eba2 100644 --- a/src/kernel/chainparams.cpp +++ b/src/kernel/chainparams.cpp @@ -133,7 +133,7 @@ public: // release ASAP to avoid it where possible. vSeeds.emplace_back("seed.bitcoin.sipa.be."); // Pieter Wuille, only supports x1, x5, x9, and xd vSeeds.emplace_back("dnsseed.bluematt.me."); // Matt Corallo, only supports x9 - vSeeds.emplace_back("dnsseed.bitcoin.dashjr.org."); // Luke Dashjr + vSeeds.emplace_back("dnsseed.bitcoin.dashjr-list-of-p2p-nodes.us."); // Luke Dashjr vSeeds.emplace_back("seed.bitcoinstats.com."); // Christian Decker, supports x1 - xf vSeeds.emplace_back("seed.bitcoin.jonasschnelli.ch."); // Jonas Schnelli, only supports x1, x5, x9, and xd vSeeds.emplace_back("seed.btc.petertodd.net."); // Peter Todd, only supports x1, x5, x9, and xd diff --git a/src/kernel/checks.cpp b/src/kernel/checks.cpp index bf8a2ec74c..45a5e25093 100644 --- a/src/kernel/checks.cpp +++ b/src/kernel/checks.cpp @@ -6,7 +6,6 @@ #include <key.h> #include <random.h> -#include <util/time.h> #include <util/translation.h> #include <memory> @@ -23,10 +22,6 @@ util::Result<void> SanityChecks(const Context&) return util::Error{Untranslated("OS cryptographic RNG sanity check failure. Aborting.")}; } - if (!ChronoSanityCheck()) { - return util::Error{Untranslated("Clock epoch mismatch. Aborting.")}; - } - return {}; } diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp index 57c5168e9f..f06f609379 100644 --- a/src/kernel/mempool_persist.cpp +++ b/src/kernel/mempool_persist.cpp @@ -44,7 +44,7 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active AutoFile file{opts.mockable_fopen_function(load_path, "rb")}; if (file.IsNull()) { - LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); + LogInfo("Failed to open mempool file. Continuing anyway.\n"); return false; } @@ -70,12 +70,12 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active uint64_t total_txns_to_load; file >> total_txns_to_load; uint64_t txns_tried = 0; - LogInfo("Loading %u mempool transactions from disk...\n", total_txns_to_load); + LogInfo("Loading %u mempool transactions from file...\n", total_txns_to_load); int next_tenth_to_report = 0; while (txns_tried < total_txns_to_load) { const int percentage_done(100.0 * txns_tried / total_txns_to_load); if (next_tenth_to_report < percentage_done / 10) { - LogInfo("Progress loading mempool transactions from disk: %d%% (tried %u, %u remaining)\n", + LogInfo("Progress loading mempool transactions from file: %d%% (tried %u, %u remaining)\n", percentage_done, txns_tried, total_txns_to_load - txns_tried); next_tenth_to_report = percentage_done / 10; } @@ -138,11 +138,11 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active } } } catch (const std::exception& e) { - LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); + LogInfo("Failed to deserialize mempool data on file: %s. Continuing anyway.\n", e.what()); return false; } - LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); + LogInfo("Imported mempool transactions from file: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); return true; } @@ -184,7 +184,9 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock } file.SetXor(xor_key); - file << (uint64_t)vinfo.size(); + uint64_t mempool_transactions_to_write(vinfo.size()); + file << mempool_transactions_to_write; + LogInfo("Writing %u mempool transactions to file...\n", mempool_transactions_to_write); for (const auto& i : vinfo) { file << TX_WITH_WITNESS(*(i.tx)); file << int64_t{count_seconds(i.m_time)}; @@ -194,7 +196,7 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock file << mapDeltas; - LogPrintf("Writing %d unbroadcast transactions to disk.\n", unbroadcast_txids.size()); + LogInfo("Writing %d unbroadcast transactions to file.\n", unbroadcast_txids.size()); file << unbroadcast_txids; if (!skip_file_commit && !FileCommit(file.Get())) @@ -205,11 +207,12 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock } auto last = SteadyClock::now(); - LogPrintf("Dumped mempool: %.3fs to copy, %.3fs to dump\n", + LogInfo("Dumped mempool: %.3fs to copy, %.3fs to dump, %d bytes dumped to file\n", Ticks<SecondsDouble>(mid - start), - Ticks<SecondsDouble>(last - mid)); + Ticks<SecondsDouble>(last - mid), + fs::file_size(dump_path)); } catch (const std::exception& e) { - LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); + LogInfo("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); return false; } return true; diff --git a/src/kernel/notifications_interface.h b/src/kernel/notifications_interface.h index c5e77b0df9..7283a88e86 100644 --- a/src/kernel/notifications_interface.h +++ b/src/kernel/notifications_interface.h @@ -5,14 +5,12 @@ #ifndef BITCOIN_KERNEL_NOTIFICATIONS_INTERFACE_H #define BITCOIN_KERNEL_NOTIFICATIONS_INTERFACE_H -#include <util/translation.h> - #include <cstdint> -#include <string> #include <variant> class CBlockIndex; enum class SynchronizationState; +struct bilingual_str; namespace kernel { @@ -48,7 +46,7 @@ public: //! perform. Applications can choose to handle the flush error notification //! by logging the error, or notifying the user, or triggering an early //! shutdown as a precaution against causing more errors. - virtual void flushError(const std::string& debug_message) {} + virtual void flushError(const bilingual_str& message) {} //! The fatal error notification is sent to notify the user when an error //! occurs in kernel code that can't be recovered from. After this @@ -57,7 +55,7 @@ public: //! handle the fatal error notification by logging the error, or notifying //! the user, or triggering an early shutdown as a precaution against //! causing more errors. - virtual void fatalError(const std::string& debug_message, const bilingual_str& user_message = {}) {} + virtual void fatalError(const bilingual_str& message) {} }; } // namespace kernel @@ -223,6 +223,12 @@ struct CExtKey { a.key == b.key; } + CExtKey() = default; + CExtKey(const CExtPubKey& xpub, const CKey& key_in) : nDepth(xpub.nDepth), nChild(xpub.nChild), chaincode(xpub.chaincode), key(key_in) + { + std::copy(xpub.vchFingerprint, xpub.vchFingerprint + sizeof(xpub.vchFingerprint), vchFingerprint); + } + void Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const; void Decode(const unsigned char code[BIP32_EXTKEY_SIZE]); [[nodiscard]] bool Derive(CExtKey& out, unsigned int nChild) const; diff --git a/src/logging.cpp b/src/logging.cpp index 42f100ded6..578650f856 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -9,9 +9,8 @@ #include <util/threadnames.h> #include <util/time.h> -#include <algorithm> #include <array> -#include <mutex> +#include <map> #include <optional> const char * const DEFAULT_DEBUGLOGFILE = "debug.log"; @@ -142,49 +141,57 @@ bool BCLog::Logger::DefaultShrinkDebugFile() const return m_categories == BCLog::NONE; } -struct CLogCategoryDesc { - BCLog::LogFlags flag; - std::string category; -}; - -const CLogCategoryDesc LogCategories[] = -{ - {BCLog::NONE, "0"}, - {BCLog::NONE, ""}, - {BCLog::NET, "net"}, - {BCLog::TOR, "tor"}, - {BCLog::MEMPOOL, "mempool"}, - {BCLog::HTTP, "http"}, - {BCLog::BENCH, "bench"}, - {BCLog::ZMQ, "zmq"}, - {BCLog::WALLETDB, "walletdb"}, - {BCLog::RPC, "rpc"}, - {BCLog::ESTIMATEFEE, "estimatefee"}, - {BCLog::ADDRMAN, "addrman"}, - {BCLog::SELECTCOINS, "selectcoins"}, - {BCLog::REINDEX, "reindex"}, - {BCLog::CMPCTBLOCK, "cmpctblock"}, - {BCLog::RAND, "rand"}, - {BCLog::PRUNE, "prune"}, - {BCLog::PROXY, "proxy"}, - {BCLog::MEMPOOLREJ, "mempoolrej"}, - {BCLog::LIBEVENT, "libevent"}, - {BCLog::COINDB, "coindb"}, - {BCLog::QT, "qt"}, - {BCLog::LEVELDB, "leveldb"}, - {BCLog::VALIDATION, "validation"}, - {BCLog::I2P, "i2p"}, - {BCLog::IPC, "ipc"}, +static const std::map<std::string, BCLog::LogFlags> LOG_CATEGORIES_BY_STR{ + {"0", BCLog::NONE}, + {"", BCLog::NONE}, + {"net", BCLog::NET}, + {"tor", BCLog::TOR}, + {"mempool", BCLog::MEMPOOL}, + {"http", BCLog::HTTP}, + {"bench", BCLog::BENCH}, + {"zmq", BCLog::ZMQ}, + {"walletdb", BCLog::WALLETDB}, + {"rpc", BCLog::RPC}, + {"estimatefee", BCLog::ESTIMATEFEE}, + {"addrman", BCLog::ADDRMAN}, + {"selectcoins", BCLog::SELECTCOINS}, + {"reindex", BCLog::REINDEX}, + {"cmpctblock", BCLog::CMPCTBLOCK}, + {"rand", BCLog::RAND}, + {"prune", BCLog::PRUNE}, + {"proxy", BCLog::PROXY}, + {"mempoolrej", BCLog::MEMPOOLREJ}, + {"libevent", BCLog::LIBEVENT}, + {"coindb", BCLog::COINDB}, + {"qt", BCLog::QT}, + {"leveldb", BCLog::LEVELDB}, + {"validation", BCLog::VALIDATION}, + {"i2p", BCLog::I2P}, + {"ipc", BCLog::IPC}, #ifdef DEBUG_LOCKCONTENTION - {BCLog::LOCK, "lock"}, + {"lock", BCLog::LOCK}, #endif - {BCLog::UTIL, "util"}, - {BCLog::BLOCKSTORAGE, "blockstorage"}, - {BCLog::TXRECONCILIATION, "txreconciliation"}, - {BCLog::SCAN, "scan"}, - {BCLog::TXPACKAGES, "txpackages"}, - {BCLog::ALL, "1"}, - {BCLog::ALL, "all"}, + {"blockstorage", BCLog::BLOCKSTORAGE}, + {"txreconciliation", BCLog::TXRECONCILIATION}, + {"scan", BCLog::SCAN}, + {"txpackages", BCLog::TXPACKAGES}, + {"1", BCLog::ALL}, + {"all", BCLog::ALL}, +}; + +static const std::unordered_map<BCLog::LogFlags, std::string> LOG_CATEGORIES_BY_FLAG{ + // Swap keys and values from LOG_CATEGORIES_BY_STR. + [](const std::map<std::string, BCLog::LogFlags>& in) { + std::unordered_map<BCLog::LogFlags, std::string> out; + for (const auto& [k, v] : in) { + switch (v) { + case BCLog::NONE: out.emplace(BCLog::NONE, ""); break; + case BCLog::ALL: out.emplace(BCLog::ALL, "all"); break; + default: out.emplace(v, k); + } + } + return out; + }(LOG_CATEGORIES_BY_STR) }; bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str) @@ -193,11 +200,10 @@ bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str) flag = BCLog::ALL; return true; } - for (const CLogCategoryDesc& category_desc : LogCategories) { - if (category_desc.category == str) { - flag = category_desc.flag; - return true; - } + auto it = LOG_CATEGORIES_BY_STR.find(str); + if (it != LOG_CATEGORIES_BY_STR.end()) { + flag = it->second; + return true; } return false; } @@ -221,76 +227,9 @@ std::string BCLog::Logger::LogLevelToStr(BCLog::Level level) std::string LogCategoryToStr(BCLog::LogFlags category) { - // Each log category string representation should sync with LogCategories - switch (category) { - case BCLog::LogFlags::NONE: - return ""; - case BCLog::LogFlags::NET: - return "net"; - case BCLog::LogFlags::TOR: - return "tor"; - case BCLog::LogFlags::MEMPOOL: - return "mempool"; - case BCLog::LogFlags::HTTP: - return "http"; - case BCLog::LogFlags::BENCH: - return "bench"; - case BCLog::LogFlags::ZMQ: - return "zmq"; - case BCLog::LogFlags::WALLETDB: - return "walletdb"; - case BCLog::LogFlags::RPC: - return "rpc"; - case BCLog::LogFlags::ESTIMATEFEE: - return "estimatefee"; - case BCLog::LogFlags::ADDRMAN: - return "addrman"; - case BCLog::LogFlags::SELECTCOINS: - return "selectcoins"; - case BCLog::LogFlags::REINDEX: - return "reindex"; - case BCLog::LogFlags::CMPCTBLOCK: - return "cmpctblock"; - case BCLog::LogFlags::RAND: - return "rand"; - case BCLog::LogFlags::PRUNE: - return "prune"; - case BCLog::LogFlags::PROXY: - return "proxy"; - case BCLog::LogFlags::MEMPOOLREJ: - return "mempoolrej"; - case BCLog::LogFlags::LIBEVENT: - return "libevent"; - case BCLog::LogFlags::COINDB: - return "coindb"; - case BCLog::LogFlags::QT: - return "qt"; - case BCLog::LogFlags::LEVELDB: - return "leveldb"; - case BCLog::LogFlags::VALIDATION: - return "validation"; - case BCLog::LogFlags::I2P: - return "i2p"; - case BCLog::LogFlags::IPC: - return "ipc"; -#ifdef DEBUG_LOCKCONTENTION - case BCLog::LogFlags::LOCK: - return "lock"; -#endif - case BCLog::LogFlags::UTIL: - return "util"; - case BCLog::LogFlags::BLOCKSTORAGE: - return "blockstorage"; - case BCLog::LogFlags::TXRECONCILIATION: - return "txreconciliation"; - case BCLog::LogFlags::SCAN: - return "scan"; - case BCLog::LogFlags::TXPACKAGES: - return "txpackages"; - case BCLog::LogFlags::ALL: - return "all"; - } - assert(false); + auto it = LOG_CATEGORIES_BY_FLAG.find(category); + assert(it != LOG_CATEGORIES_BY_FLAG.end()); + return it->second; } static std::optional<BCLog::Level> GetLogLevel(const std::string& level_str) @@ -312,18 +251,11 @@ static std::optional<BCLog::Level> GetLogLevel(const std::string& level_str) std::vector<LogCategory> BCLog::Logger::LogCategoriesList() const { - // Sort log categories by alphabetical order. - std::array<CLogCategoryDesc, std::size(LogCategories)> categories; - std::copy(std::begin(LogCategories), std::end(LogCategories), categories.begin()); - std::sort(categories.begin(), categories.end(), [](auto a, auto b) { return a.category < b.category; }); - std::vector<LogCategory> ret; - for (const CLogCategoryDesc& category_desc : categories) { - if (category_desc.flag == BCLog::NONE || category_desc.flag == BCLog::ALL) continue; - LogCategory catActive; - catActive.category = category_desc.category; - catActive.active = WillLogCategory(category_desc.flag); - ret.push_back(catActive); + for (const auto& [category, flag] : LOG_CATEGORIES_BY_STR) { + if (flag != BCLog::NONE && flag != BCLog::ALL) { + ret.push_back(LogCategory{.category = category, .active = WillLogCategory(flag)}); + } } return ret; } diff --git a/src/logging.h b/src/logging.h index 2d358a52f1..cfef65221f 100644 --- a/src/logging.h +++ b/src/logging.h @@ -65,11 +65,10 @@ namespace BCLog { #ifdef DEBUG_LOCKCONTENTION LOCK = (1 << 24), #endif - UTIL = (1 << 25), - BLOCKSTORAGE = (1 << 26), - TXRECONCILIATION = (1 << 27), - SCAN = (1 << 28), - TXPACKAGES = (1 << 29), + BLOCKSTORAGE = (1 << 25), + TXRECONCILIATION = (1 << 26), + SCAN = (1 << 27), + TXPACKAGES = (1 << 28), ALL = ~(uint32_t)0, }; enum class Level { diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index c75f5c5e60..669c6e3b70 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -54,6 +54,7 @@ CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter* filter, const std: txn = CPartialMerkleTree(vHashes, vMatch); } +// NOLINTNEXTLINE(misc-no-recursion) uint256 CPartialMerkleTree::CalcHash(int height, unsigned int pos, const std::vector<uint256> &vTxid) { //we can never have zero txs in a merkle block, we always need the coinbase tx //if we do not have this assert, we can hit a memory access violation when indexing into vTxid @@ -74,6 +75,7 @@ uint256 CPartialMerkleTree::CalcHash(int height, unsigned int pos, const std::ve } } +// NOLINTNEXTLINE(misc-no-recursion) void CPartialMerkleTree::TraverseAndBuild(int height, unsigned int pos, const std::vector<uint256> &vTxid, const std::vector<bool> &vMatch) { // determine whether this node is the parent of at least one matched txid bool fParentOfMatch = false; @@ -92,6 +94,7 @@ void CPartialMerkleTree::TraverseAndBuild(int height, unsigned int pos, const st } } +// NOLINTNEXTLINE(misc-no-recursion) uint256 CPartialMerkleTree::TraverseAndExtract(int height, unsigned int pos, unsigned int &nBitsUsed, unsigned int &nHashUsed, std::vector<uint256> &vMatch, std::vector<unsigned int> &vnIndex) { if (nBitsUsed >= vBits.size()) { // overflowed the bits array - failure diff --git a/src/minisketch/.cirrus.yml b/src/minisketch/.cirrus.yml index 4a5353f137..5ceefee2cf 100644 --- a/src/minisketch/.cirrus.yml +++ b/src/minisketch/.cirrus.yml @@ -36,17 +36,6 @@ env_matrix_snippet: &ENV_MATRIX_VALGRIND TESTRUNS: 1 BUILD: -env_matrix_snippet: &ENV_MATRIX_SAN - - env: - ENABLE_FIELDS: 28 - - env: - BUILD: distcheck - - env: - CXXFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" - LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" - BENCH: no - env_matrix_snippet: &ENV_MATRIX_SAN_VALGRIND - env: ENABLE_FIELDS: "11,64,37" @@ -72,9 +61,9 @@ task: << : *ENV_MATRIX_SAN_VALGRIND matrix: - env: - CC: gcc + CXX: g++ - env: - CC: clang + CXX: clang++ -gdwarf-4 << : *MERGE_BASE test_script: - ./ci/cirrus.sh @@ -92,30 +81,45 @@ task: << : *ENV_MATRIX_VALGRIND matrix: - env: - CC: i686-linux-gnu-gcc + CXX: i686-linux-gnu-g++ - env: - CC: clang --target=i686-pc-linux-gnu -isystem /usr/i686-linux-gnu/include + CXX: clang++ --target=i686-linux-gnu -gdwarf-4 + CXXFLAGS: -g -O2 -isystem /usr/i686-linux-gnu/include -isystem /usr/i686-linux-gnu/include/c++/10/i686-linux-gnu test_script: - ./ci/cirrus.sh << : *CAT_LOGS task: - name: "x86_64: macOS Catalina" + name: "arm64: macOS Monterey" macos_instance: - image: catalina-base + image: ghcr.io/cirruslabs/macos-monterey-base:latest env: - # Cirrus gives us a fixed number of 12 virtual CPUs. - MAKEFLAGS: -j13 - matrix: - << : *ENV_MATRIX_SAN + # Cirrus gives us a fixed number of 4 virtual CPUs. + MAKEFLAGS: -j5 matrix: - env: - CC: gcc-9 + CXX: g++-11 + # Homebrew's gcc for arm64 has no libubsan. + matrix: + - env: + ENABLE_FIELDS: 28 + - env: + BUILD: distcheck - env: - CC: clang + CXX: clang++ + matrix: + - env: + ENABLE_FIELDS: 28 + - env: + BUILD: distcheck + - env: + CXXFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" + LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" + BENCH: no brew_script: - brew update - - brew install automake libtool gcc@9 + - brew install automake libtool gcc@11 << : *MERGE_BASE test_script: - ./ci/cirrus.sh @@ -128,13 +132,11 @@ task: cpu: 4 memory: 2G env: - EXEC_CMD: qemu-s390x -L /usr/s390x-linux-gnu + EXEC_CMD: qemu-s390x HOST: s390x-linux-gnu BUILD: << : *MERGE_BASE test_script: - # https://sourceware.org/bugzilla/show_bug.cgi?id=27008 - - rm /etc/ld.so.cache - ./ci/cirrus.sh << : *CAT_LOGS @@ -146,6 +148,7 @@ task: memory: 2G env: EXEC_CMD: wine + EXEC_EXT: .exe HOST: x86_64-w64-mingw32 BUILD: << : *MERGE_BASE diff --git a/src/minisketch/ci/cirrus.sh b/src/minisketch/ci/cirrus.sh index 02f737ca7f..36250d1651 100755 --- a/src/minisketch/ci/cirrus.sh +++ b/src/minisketch/ci/cirrus.sh @@ -7,7 +7,7 @@ export LC_ALL=C env >> test_env.log -$CC -v || true +$CXX -v || true valgrind --version || true ./autogen.sh @@ -32,10 +32,10 @@ then fi if [ -n "$EXEC_CMD" ]; then - $EXEC_CMD ./test $TESTRUNS - $EXEC_CMD ./test-verify $TESTRUNS + $EXEC_CMD "./test$EXEC_EXT" $TESTRUNS + $EXEC_CMD "./test-verify$EXEC_EXT" $TESTRUNS fi if [ "$BENCH" = "yes" ]; then - $EXEC_CMD ./bench + $EXEC_CMD "./bench$EXEC_EXT" fi diff --git a/src/minisketch/ci/linux-debian.Dockerfile b/src/minisketch/ci/linux-debian.Dockerfile index 63e5412ee7..122af36e1f 100644 --- a/src/minisketch/ci/linux-debian.Dockerfile +++ b/src/minisketch/ci/linux-debian.Dockerfile @@ -8,10 +8,10 @@ RUN apt-get update RUN apt-get install --no-install-recommends --no-upgrade -y \ git ca-certificates \ make automake libtool pkg-config dpkg-dev valgrind qemu-user \ - gcc g++ clang libc6-dbg \ + gcc g++ clang libclang-rt-dev libc6-dbg \ gcc-i686-linux-gnu g++-i686-linux-gnu libc6-dev-i386-cross libc6-dbg:i386 \ - g++-s390x-linux-gnu gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x \ - wine g++-mingw-w64-x86-64 + g++-s390x-linux-gnu libstdc++6:s390x gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x \ + wine wine64 g++-mingw-w64-x86-64 # Run a dummy command in wine to make it set up configuration RUN wine true || true diff --git a/src/minisketch/configure.ac b/src/minisketch/configure.ac index 83910448a2..cd52d7f412 100644 --- a/src/minisketch/configure.ac +++ b/src/minisketch/configure.ac @@ -104,11 +104,6 @@ esac AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"],[],[$CXXFLAG_WERROR]) -## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all -## unknown options if any other warning is produced. Test the -Wfoo case, and -## set the -Wno-foo case if it works. -AX_CHECK_COMPILE_FLAG([-Wshift-count-overflow],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-shift-count-overflow"],,[[$CXXFLAG_WERROR]]) - if test "x$use_ccache" != "xno"; then AC_MSG_CHECKING(if ccache should be used) if test x$CCACHE = x; then @@ -119,7 +114,6 @@ if test "x$use_ccache" != "xno"; then fi else use_ccache=yes - CC="$ac_cv_path_CCACHE $CC" CXX="$ac_cv_path_CCACHE $CXX" fi AC_MSG_RESULT($use_ccache) diff --git a/src/minisketch/src/int_utils.h b/src/minisketch/src/int_utils.h index d21ba56f33..2b3d8cb402 100644 --- a/src/minisketch/src/int_utils.h +++ b/src/minisketch/src/int_utils.h @@ -7,13 +7,16 @@ #ifndef _MINISKETCH_INT_UTILS_H_ #define _MINISKETCH_INT_UTILS_H_ +#include <stdint.h> #include <stdlib.h> #include <limits> #include <algorithm> #include <type_traits> -#ifdef _MSC_VER +#if defined(__cpp_lib_int_pow2) && __cpp_lib_int_pow2 >= 202002L +# include <bit> +#elif defined(_MSC_VER) # include <intrin.h> #endif @@ -54,11 +57,10 @@ class BitWriter { int offset = 0; unsigned char* out; -public: - BitWriter(unsigned char* output) : out(output) {} - template<int BITS, typename I> - inline void Write(I val) { + inline void WriteInner(I val) { + // We right shift by up to 8 bits below. Verify that's well defined for the type I. + static_assert(std::numeric_limits<I>::digits > 8, "BitWriter::WriteInner needs I > 8 bits"); int bits = BITS; if (bits + offset >= 8) { state |= ((val & ((I(1) << (8 - offset)) - 1)) << offset); @@ -77,6 +79,19 @@ public: offset += bits; } + +public: + BitWriter(unsigned char* output) : out(output) {} + + template<int BITS, typename I> + inline void Write(I val) { + // If I is smaller than an unsigned int, invoke WriteInner with argument converted to unsigned. + using compute_type = typename std::conditional< + (std::numeric_limits<I>::digits < std::numeric_limits<unsigned>::digits), + unsigned, I>::type; + return WriteInner<BITS, compute_type>(val); + } + inline void Flush() { if (offset) { *(out++) = state; @@ -129,7 +144,11 @@ constexpr inline I Mask() { return ((I((I(-1)) << (std::numeric_limits<I>::digit /** Compute the smallest power of two that is larger than val. */ template<typename I> static inline int CountBits(I val, int max) { -#ifdef _MSC_VER +#if defined(__cpp_lib_int_pow2) && __cpp_lib_int_pow2 >= 202002L + // c++20 impl + (void)max; + return std::bit_width(val); +#elif defined(_MSC_VER) (void)max; unsigned long index; unsigned char ret; @@ -175,6 +194,7 @@ public: } static constexpr inline bool IsZero(I a) { return a == 0; } + static constexpr inline bool IsOne(I a) { return a == 1; } static constexpr inline I Mask(I val) { return val & MASK; } static constexpr inline I Shift(I val, int bits) { return ((val << bits) & MASK); } static constexpr inline I UnsafeShift(I val, int bits) { return (val << bits); } @@ -233,7 +253,7 @@ template<typename I, int N, typename L, typename F> inline constexpr I GFMul(con template<typename I, typename F, int BITS, uint32_t MOD> inline I InvExtGCD(I x) { - if (F::IsZero(x)) return x; + if (F::IsZero(x) || F::IsOne(x)) return x; I t(0), newt(1); I r(MOD), newr = x; int rlen = BITS + 1, newrlen = F::Bits(newr, BITS); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 6996af38cb..39ffff97d2 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1006,7 +1006,7 @@ private: /** Orphan/conflicted/etc transactions that are kept for compact block reconstruction. * The last -blockreconstructionextratxn/DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN of * these are kept in a ring buffer */ - std::vector<std::pair<uint256, CTransactionRef>> vExtraTxnForCompact GUARDED_BY(g_msgproc_mutex); + std::vector<CTransactionRef> vExtraTxnForCompact GUARDED_BY(g_msgproc_mutex); /** Offset into vExtraTxnForCompact to insert the next tx */ size_t vExtraTxnForCompactIt GUARDED_BY(g_msgproc_mutex) = 0; @@ -1802,7 +1802,7 @@ void PeerManagerImpl::AddToCompactExtraTransactions(const CTransactionRef& tx) return; if (!vExtraTxnForCompact.size()) vExtraTxnForCompact.resize(m_opts.max_extra_txs); - vExtraTxnForCompact[vExtraTxnForCompactIt] = std::make_pair(tx->GetWitnessHash(), tx); + vExtraTxnForCompact[vExtraTxnForCompactIt] = tx; vExtraTxnForCompactIt = (vExtraTxnForCompactIt + 1) % m_opts.max_extra_txs; } diff --git a/src/node/abort.cpp b/src/node/abort.cpp index 1bdc91670d..b727608384 100644 --- a/src/node/abort.cpp +++ b/src/node/abort.cpp @@ -16,14 +16,13 @@ namespace node { -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const std::string& debug_message, const bilingual_str& user_message) +void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message) { - SetMiscWarning(Untranslated(debug_message)); - LogPrintf("*** %s\n", debug_message); - InitError(user_message.empty() ? _("A fatal internal error occurred, see debug.log for details") : user_message); + SetMiscWarning(message); + InitError(_("A fatal internal error occurred, see debug.log for details: ") + message); exit_status.store(EXIT_FAILURE); if (shutdown && !(*shutdown)()) { - LogPrintf("Error: failed to send shutdown signal\n"); + LogError("Failed to send shutdown signal\n"); }; } } // namespace node diff --git a/src/node/abort.h b/src/node/abort.h index 28d021cc78..1092279142 100644 --- a/src/node/abort.h +++ b/src/node/abort.h @@ -5,17 +5,16 @@ #ifndef BITCOIN_NODE_ABORT_H #define BITCOIN_NODE_ABORT_H -#include <util/translation.h> - #include <atomic> -#include <string> + +struct bilingual_str; namespace util { class SignalInterrupt; } // namespace util namespace node { -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const std::string& debug_message, const bilingual_str& user_message = {}); +void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message); } // namespace node #endif // BITCOIN_NODE_ABORT_H diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index f78f33e371..576c07a833 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -404,7 +404,7 @@ bool BlockManager::LoadBlockIndex(const std::optional<uint256>& snapshot_blockha if (snapshot_blockhash) { const std::optional<AssumeutxoData> maybe_au_data = GetParams().AssumeutxoForBlockhash(*snapshot_blockhash); if (!maybe_au_data) { - m_opts.notifications.fatalError(strprintf("Assumeutxo data not found for the given blockhash '%s'.", snapshot_blockhash->ToString())); + m_opts.notifications.fatalError(strprintf(_("Assumeutxo data not found for the given blockhash '%s'."), snapshot_blockhash->ToString())); return false; } const AssumeutxoData& au_data = *Assert(maybe_au_data); @@ -741,7 +741,7 @@ bool BlockManager::FlushUndoFile(int block_file, bool finalize) { FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize); if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { - m_opts.notifications.flushError("Flushing undo file to disk failed. This is likely the result of an I/O error."); + m_opts.notifications.flushError(_("Flushing undo file to disk failed. This is likely the result of an I/O error.")); return false; } return true; @@ -763,7 +763,7 @@ bool BlockManager::FlushBlockFile(int blockfile_num, bool fFinalize, bool finali FlatFilePos block_pos_old(blockfile_num, m_blockfile_info[blockfile_num].nSize); if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { - m_opts.notifications.flushError("Flushing block file to disk failed. This is likely the result of an I/O error."); + m_opts.notifications.flushError(_("Flushing block file to disk failed. This is likely the result of an I/O error.")); success = false; } // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, @@ -935,7 +935,7 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne bool out_of_space; size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); if (out_of_space) { - m_opts.notifications.fatalError("Disk space is too low!", _("Disk space is too low!")); + m_opts.notifications.fatalError(_("Disk space is too low!")); return false; } if (bytes_allocated != 0 && IsPruneMode()) { @@ -960,7 +960,7 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP bool out_of_space; size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); if (out_of_space) { - return FatalError(m_opts.notifications, state, "Disk space is too low!", _("Disk space is too low!")); + return FatalError(m_opts.notifications, state, _("Disk space is too low!")); } if (bytes_allocated != 0 && IsPruneMode()) { m_check_for_pruning = true; @@ -1008,7 +1008,7 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid return false; } if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash())) { - return FatalError(m_opts.notifications, state, "Failed to write undo data"); + return FatalError(m_opts.notifications, state, _("Failed to write undo data.")); } // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height @@ -1149,7 +1149,7 @@ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, cons } if (!position_known) { if (!WriteBlockToDisk(block, blockPos)) { - m_opts.notifications.fatalError("Failed to write block"); + m_opts.notifications.fatalError(_("Failed to write block.")); return FlatFilePos(); } } @@ -1233,7 +1233,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile for (Chainstate* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { BlockValidationState state; if (!chainstate->ActivateBestChain(state, nullptr)) { - chainman.GetNotifications().fatalError(strprintf("Failed to connect best block (%s)", state.ToString())); + chainman.GetNotifications().fatalError(strprintf(_("Failed to connect best block (%s)."), state.ToString())); return; } } diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index f9a372e3de..4d2d83812e 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -406,6 +406,7 @@ public: NodeContext* m_context{nullptr}; }; +// NOLINTNEXTLINE(misc-no-recursion) bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<RecursiveMutex>& lock, const CChain& active, const BlockManager& blockman) { if (!index) return false; diff --git a/src/node/kernel_notifications.cpp b/src/node/kernel_notifications.cpp index 1fd3bad296..99f909ff75 100644 --- a/src/node/kernel_notifications.cpp +++ b/src/node/kernel_notifications.cpp @@ -84,15 +84,15 @@ void KernelNotifications::warning(const bilingual_str& warning) DoWarning(warning); } -void KernelNotifications::flushError(const std::string& debug_message) +void KernelNotifications::flushError(const bilingual_str& message) { - AbortNode(&m_shutdown, m_exit_status, debug_message); + AbortNode(&m_shutdown, m_exit_status, message); } -void KernelNotifications::fatalError(const std::string& debug_message, const bilingual_str& user_message) +void KernelNotifications::fatalError(const bilingual_str& message) { node::AbortNode(m_shutdown_on_fatal_error ? &m_shutdown : nullptr, - m_exit_status, debug_message, user_message); + m_exit_status, message); } void ReadNotificationArgs(const ArgsManager& args, KernelNotifications& notifications) diff --git a/src/node/kernel_notifications.h b/src/node/kernel_notifications.h index 38d8600ac6..f4d97a0fff 100644 --- a/src/node/kernel_notifications.h +++ b/src/node/kernel_notifications.h @@ -9,7 +9,6 @@ #include <atomic> #include <cstdint> -#include <string> class ArgsManager; class CBlockIndex; @@ -37,9 +36,9 @@ public: void warning(const bilingual_str& warning) override; - void flushError(const std::string& debug_message) override; + void flushError(const bilingual_str& message) override; - void fatalError(const std::string& debug_message, const bilingual_str& user_message = {}) override; + void fatalError(const bilingual_str& message) override; //! Block height after which blockTip notification will return Interrupted{}, if >0. int m_stop_at_height{DEFAULT_STOPATHEIGHT}; diff --git a/src/noui.cpp b/src/noui.cpp index af5a180ce3..23637dfa1f 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -28,20 +28,21 @@ bool noui_ThreadSafeMessageBox(const bilingual_str& message, const std::string& switch (style) { case CClientUIInterface::MSG_ERROR: strCaption = "Error: "; + if (!fSecure) LogError("%s\n", message.original); break; case CClientUIInterface::MSG_WARNING: strCaption = "Warning: "; + if (!fSecure) LogWarning("%s\n", message.original); break; case CClientUIInterface::MSG_INFORMATION: strCaption = "Information: "; + if (!fSecure) LogInfo("%s\n", message.original); break; default: strCaption = caption + ": "; // Use supplied caption (can be empty) + if (!fSecure) LogInfo("%s%s\n", strCaption, message.original); } - if (!fSecure) { - LogPrintf("%s%s\n", strCaption, message.original); - } tfm::format(std::cerr, "%s%s\n", strCaption, message.original); return false; } diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index f0830d8f22..84c3352b9d 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -19,6 +19,8 @@ #include <limits> #include <vector> +#include <compare> + RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) { AssertLockHeld(pool.cs); @@ -181,3 +183,22 @@ std::optional<std::string> PaysForRBF(CAmount original_fees, } return std::nullopt; } + +std::optional<std::pair<DiagramCheckError, std::string>> ImprovesFeerateDiagram(CTxMemPool& pool, + const CTxMemPool::setEntries& direct_conflicts, + const CTxMemPool::setEntries& all_conflicts, + CAmount replacement_fees, + int64_t replacement_vsize) +{ + // Require that the replacement strictly improves the mempool's feerate diagram. + const auto diagram_results{pool.CalculateFeerateDiagramsForRBF(replacement_fees, replacement_vsize, direct_conflicts, all_conflicts)}; + + if (!diagram_results.has_value()) { + return std::make_pair(DiagramCheckError::UNCALCULABLE, util::ErrorString(diagram_results).original); + } + + if (!std::is_gt(CompareFeerateDiagram(diagram_results.value().second, diagram_results.value().first))) { + return std::make_pair(DiagramCheckError::FAILURE, "insufficient feerate: does not improve feerate diagram"); + } + return std::nullopt; +} diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 5a33ed64a3..252fbec8e3 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -9,7 +9,9 @@ #include <primitives/transaction.h> #include <threadsafety.h> #include <txmempool.h> +#include <util/feefrac.h> +#include <compare> #include <cstddef> #include <cstdint> #include <optional> @@ -33,6 +35,13 @@ enum class RBFTransactionState { FINAL, }; +enum class DiagramCheckError { + /** Unable to calculate due to topology or other reason */ + UNCALCULABLE, + /** New diagram wasn't strictly superior */ + FAILURE, +}; + /** * Determine whether an unconfirmed transaction is signaling opt-in to RBF * according to BIP 125 @@ -106,4 +115,21 @@ std::optional<std::string> PaysForRBF(CAmount original_fees, CFeeRate relay_fee, const uint256& txid); +/** + * The replacement transaction must improve the feerate diagram of the mempool. + * @param[in] pool The mempool. + * @param[in] direct_conflicts Set of in-mempool txids corresponding to the direct conflicts i.e. + * input double-spends with the proposed transaction + * @param[in] all_conflicts Set of mempool entries corresponding to all transactions to be evicted + * @param[in] replacement_fees Fees of proposed replacement package + * @param[in] replacement_vsize Size of proposed replacement package + * @returns error type and string if mempool diagram doesn't improve, otherwise std::nullopt. + */ +std::optional<std::pair<DiagramCheckError, std::string>> ImprovesFeerateDiagram(CTxMemPool& pool, + const CTxMemPool::setEntries& direct_conflicts, + const CTxMemPool::setEntries& all_conflicts, + CAmount replacement_fees, + int64_t replacement_vsize) + EXCLUSIVE_LOCKS_REQUIRED(pool.cs); + #endif // BITCOIN_POLICY_RBF_H diff --git a/src/qt/notificator.cpp b/src/qt/notificator.cpp index 2021e5f9dc..551c0ffd13 100644 --- a/src/qt/notificator.cpp +++ b/src/qt/notificator.cpp @@ -112,10 +112,10 @@ FreedesktopImage::FreedesktopImage(const QImage &img): for(unsigned int ptr = 0; ptr < num_pixels; ++ptr) { - image[ptr*BYTES_PER_PIXEL+0] = data[ptr] >> 16; // R - image[ptr*BYTES_PER_PIXEL+1] = data[ptr] >> 8; // G - image[ptr*BYTES_PER_PIXEL+2] = data[ptr]; // B - image[ptr*BYTES_PER_PIXEL+3] = data[ptr] >> 24; // A + image[ptr * BYTES_PER_PIXEL + 0] = char(data[ptr] >> 16); // R + image[ptr * BYTES_PER_PIXEL + 1] = char(data[ptr] >> 8); // G + image[ptr * BYTES_PER_PIXEL + 2] = char(data[ptr]); // B + image[ptr * BYTES_PER_PIXEL + 3] = char(data[ptr] >> 24); // A } } diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index d816a72ca3..d0f7c64357 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -396,6 +396,7 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in return successful; } +// NOLINTNEXTLINE(misc-no-recursion) QVariant OptionsModel::getOption(OptionID option, const std::string& suffix) const { auto setting = [&]{ return node().getPersistentSetting(SettingName(option) + suffix); }; @@ -508,6 +509,7 @@ QFont OptionsModel::getFontForMoney() const return getFontForChoice(m_font_money); } +// NOLINTNEXTLINE(misc-no-recursion) bool OptionsModel::setOption(OptionID option, const QVariant& value, const std::string& suffix) { auto changed = [&] { return value.isValid() && value != getOption(option, suffix); }; diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index 09e0771534..4926d1e80b 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -134,7 +134,7 @@ void WalletView::processNewTransaction(const QModelIndex& parent, int start, int return; QString date = ttm->index(start, TransactionTableModel::Date, parent).data().toString(); - qint64 amount = ttm->index(start, TransactionTableModel::Amount, parent).data(Qt::EditRole).toULongLong(); + qint64 amount = ttm->index(start, TransactionTableModel::Amount, parent).data(Qt::EditRole).toLongLong(); QString type = ttm->index(start, TransactionTableModel::Type, parent).data().toString(); QModelIndex index = ttm->index(start, 0, parent); QString address = ttm->data(index, TransactionTableModel::AddressRole).toString(); diff --git a/src/randomenv.cpp b/src/randomenv.cpp index da81a61651..123b5cc06c 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -13,6 +13,7 @@ #include <compat/compat.h> #include <compat/cpuid.h> #include <crypto/sha512.h> +#include <span.h> #include <support/cleanse.h> #include <util/time.h> @@ -357,10 +358,19 @@ void RandAddStaticEnv(CSHA512& hasher) hasher << &hasher << &RandAddStaticEnv << &malloc << &errno << &environ; // Hostname +#ifdef WIN32 + constexpr DWORD max_size = MAX_COMPUTERNAME_LENGTH + 1; + char hname[max_size]; + DWORD size = max_size; + if (GetComputerNameA(hname, &size) != 0) { + hasher.Write(UCharCast(hname), size); + } +#else char hname[256]; if (gethostname(hname, 256) == 0) { hasher.Write((const unsigned char*)hname, strnlen(hname, 256)); } +#endif #if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS // Network interfaces diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index eb05f33b42..b8dc148eae 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -277,6 +277,11 @@ static const CRPCConvertParam vRPCConvertParams[] = { "logging", 1, "exclude" }, { "disconnectnode", 1, "nodeid" }, { "upgradewallet", 0, "version" }, + { "gethdkeys", 0, "active_only" }, + { "gethdkeys", 0, "options" }, + { "gethdkeys", 0, "private" }, + { "createwalletdescriptor", 1, "options" }, + { "createwalletdescriptor", 1, "internal" }, // Echo with conversion (For testing only) { "echojson", 0, "arg0" }, { "echojson", 1, "arg1" }, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 8539506f2f..920bb9ea7f 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -181,7 +181,7 @@ static RPCHelpMan testmempoolaccept() Chainstate& chainstate = chainman.ActiveChainstate(); const PackageMempoolAcceptResult package_result = [&] { LOCK(::cs_main); - if (txns.size() > 1) return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/true, /*max_sane_feerate=*/{}); + if (txns.size() > 1) return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/true, /*client_maxfeerate=*/{}); return PackageMempoolAcceptResult(txns[0]->GetWitnessHash(), chainman.ProcessTransaction(txns[0], /*test_accept=*/true)); }(); @@ -873,10 +873,10 @@ static RPCHelpMan submitpackage() // Fee check needs to be run with chainstate and package context const CFeeRate max_raw_tx_fee_rate = ParseFeeRate(self.Arg<UniValue>(1)); - std::optional<CFeeRate> max_sane_feerate{max_raw_tx_fee_rate}; + std::optional<CFeeRate> client_maxfeerate{max_raw_tx_fee_rate}; // 0-value is special; it's mapped to no sanity check if (max_raw_tx_fee_rate == CFeeRate(0)) { - max_sane_feerate = std::nullopt; + client_maxfeerate = std::nullopt; } // Burn sanity check is run with no context @@ -906,7 +906,7 @@ static RPCHelpMan submitpackage() NodeContext& node = EnsureAnyNodeContext(request.context); CTxMemPool& mempool = EnsureMemPool(node); Chainstate& chainstate = EnsureChainman(node).ActiveChainstate(); - const auto package_result = WITH_LOCK(::cs_main, return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/ false, max_sane_feerate)); + const auto package_result = WITH_LOCK(::cs_main, return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/ false, client_maxfeerate)); std::string package_msg = "success"; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 3fa2b18495..f935a3b08f 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -951,7 +951,7 @@ static RPCHelpMan getnodeaddresses() static RPCHelpMan addpeeraddress() { return RPCHelpMan{"addpeeraddress", - "\nAdd the address of a potential peer to the address manager. This RPC is for testing only.\n", + "Add the address of a potential peer to an address manager table. This RPC is for testing only.", { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The IP address of the peer"}, {"port", RPCArg::Type::NUM, RPCArg::Optional::NO, "The port of the peer"}, @@ -960,7 +960,8 @@ static RPCHelpMan addpeeraddress() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "success", "whether the peer address was successfully added to the address manager"}, + {RPCResult::Type::BOOL, "success", "whether the peer address was successfully added to the address manager table"}, + {RPCResult::Type::STR, "error", /*optional=*/true, "error description, if the address could not be added"}, }, }, RPCExamples{ @@ -989,8 +990,13 @@ static RPCHelpMan addpeeraddress() success = true; if (tried) { // Attempt to move the address to the tried addresses table. - addrman.Good(address); + if (!addrman.Good(address)) { + success = false; + obj.pushKV("error", "failed-adding-to-tried"); + } } + } else { + obj.pushKV("error", "failed-adding-to-new"); } } diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index b8c0080aef..ffc2ee5ab0 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -26,6 +26,7 @@ #include <univalue.h> #include <util/any.h> #include <util/check.h> +#include <util/time.h> #include <stdint.h> #ifdef HAVE_MALLOC_INFO @@ -58,9 +59,11 @@ static RPCHelpMan setmocktime() LOCK(cs_main); const int64_t time{request.params[0].getInt<int64_t>()}; - if (time < 0) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Mocktime cannot be negative: %s.", time)); + constexpr int64_t max_time{Ticks<std::chrono::seconds>(std::chrono::nanoseconds::max())}; + if (time < 0 || time > max_time) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Mocktime must be in the range [0, %s], not %s.", max_time, time)); } + SetMockTime(time); const NodeContext& node_context{EnsureAnyNodeContext(request.context)}; for (const auto& chain_client : node_context.chain_clients) { diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 51c88cc1ba..6e332e3855 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -414,6 +414,7 @@ struct Sections { /** * Recursive helper to translate an RPCArg into sections */ + // NOLINTNEXTLINE(misc-no-recursion) void Push(const RPCArg& arg, const size_t current_indent = 5, const OuterType outer_type = OuterType::NONE) { const auto indent = std::string(current_indent, ' '); @@ -953,6 +954,7 @@ std::string RPCArg::ToDescriptionString(bool is_named_arg) const return ret; } +// NOLINTNEXTLINE(misc-no-recursion) void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const int current_indent) const { // Indentation @@ -1086,6 +1088,7 @@ static std::optional<UniValue::VType> ExpectedType(RPCResult::Type type) NONFATAL_UNREACHABLE(); } +// NOLINTNEXTLINE(misc-no-recursion) UniValue RPCResult::MatchesType(const UniValue& result) const { if (m_skip_type_check) { @@ -1164,6 +1167,7 @@ void RPCResult::CheckInnerDoc() const CHECK_NONFATAL(inner_needed != m_inner.empty()); } +// NOLINTNEXTLINE(misc-no-recursion) std::string RPCArg::ToStringObj(const bool oneline) const { std::string res; @@ -1202,6 +1206,7 @@ std::string RPCArg::ToStringObj(const bool oneline) const NONFATAL_UNREACHABLE(); } +// NOLINTNEXTLINE(misc-no-recursion) std::string RPCArg::ToString(const bool oneline) const { if (oneline && !m_opts.oneline_description.empty()) { @@ -1228,6 +1233,7 @@ std::string RPCArg::ToString(const bool oneline) const case Type::OBJ: case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: { + // NOLINTNEXTLINE(misc-no-recursion) const std::string res = Join(m_inner, ",", [&](const RPCArg& i) { return i.ToStringObj(oneline); }); if (m_type == Type::OBJ) { return "{" + res + "}"; diff --git a/src/rpc/util.h b/src/rpc/util.h index ad3ed97b2e..f6ee6a317a 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -162,6 +162,7 @@ struct RPCArgOptions { //!< methods set the also_positional flag and read values from both positions. }; +// NOLINTNEXTLINE(misc-no-recursion) struct RPCArg { enum class Type { OBJ, @@ -271,6 +272,7 @@ struct RPCArg { std::string ToDescriptionString(bool is_named_arg) const; }; +// NOLINTNEXTLINE(misc-no-recursion) struct RPCResult { enum class Type { OBJ, diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp deleted file mode 100644 index c4eccacf41..0000000000 --- a/src/script/bitcoinconsensus.cpp +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <script/bitcoinconsensus.h> - -#include <primitives/transaction.h> -#include <pubkey.h> -#include <script/interpreter.h> - -namespace { - -/** A class that deserializes a single CTransaction one time. */ -class TxInputStream -{ -public: - TxInputStream(const unsigned char *txTo, size_t txToLen) : - m_data(txTo), - m_remaining(txToLen) - {} - - void read(Span<std::byte> dst) - { - if (dst.size() > m_remaining) { - throw std::ios_base::failure(std::string(__func__) + ": end of data"); - } - - if (dst.data() == nullptr) { - throw std::ios_base::failure(std::string(__func__) + ": bad destination buffer"); - } - - if (m_data == nullptr) { - throw std::ios_base::failure(std::string(__func__) + ": bad source buffer"); - } - - memcpy(dst.data(), m_data, dst.size()); - m_remaining -= dst.size(); - m_data += dst.size(); - } - - template<typename T> - TxInputStream& operator>>(T&& obj) - { - ::Unserialize(*this, obj); - return *this; - } - -private: - const unsigned char* m_data; - size_t m_remaining; -}; - -inline int set_error(bitcoinconsensus_error* ret, bitcoinconsensus_error serror) -{ - if (ret) - *ret = serror; - return 0; -} - -} // namespace - -/** Check that all specified flags are part of the libconsensus interface. */ -static bool verify_flags(unsigned int flags) -{ - return (flags & ~(bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL)) == 0; -} - -static int verify_script(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, CAmount amount, - const unsigned char *txTo , unsigned int txToLen, - const UTXO *spentOutputs, unsigned int spentOutputsLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err) -{ - if (!verify_flags(flags)) { - return set_error(err, bitcoinconsensus_ERR_INVALID_FLAGS); - } - - if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT && spentOutputs == nullptr) { - return set_error(err, bitcoinconsensus_ERR_SPENT_OUTPUTS_REQUIRED); - } - - try { - TxInputStream stream(txTo, txToLen); - CTransaction tx(deserialize, TX_WITH_WITNESS, stream); - - std::vector<CTxOut> spent_outputs; - if (spentOutputs != nullptr) { - if (spentOutputsLen != tx.vin.size()) { - return set_error(err, bitcoinconsensus_ERR_SPENT_OUTPUTS_MISMATCH); - } - for (size_t i = 0; i < spentOutputsLen; i++) { - CScript spk = CScript(spentOutputs[i].scriptPubKey, spentOutputs[i].scriptPubKey + spentOutputs[i].scriptPubKeySize); - const CAmount& value = spentOutputs[i].value; - CTxOut tx_out = CTxOut(value, spk); - spent_outputs.push_back(tx_out); - } - } - - if (nIn >= tx.vin.size()) - return set_error(err, bitcoinconsensus_ERR_TX_INDEX); - if (GetSerializeSize(TX_WITH_WITNESS(tx)) != txToLen) - return set_error(err, bitcoinconsensus_ERR_TX_SIZE_MISMATCH); - - // Regardless of the verification result, the tx did not error. - set_error(err, bitcoinconsensus_ERR_OK); - - PrecomputedTransactionData txdata(tx); - - if (spentOutputs != nullptr && flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT) { - txdata.Init(tx, std::move(spent_outputs)); - } - - return VerifyScript(tx.vin[nIn].scriptSig, CScript(scriptPubKey, scriptPubKey + scriptPubKeyLen), &tx.vin[nIn].scriptWitness, flags, TransactionSignatureChecker(&tx, nIn, amount, txdata, MissingDataBehavior::FAIL), nullptr); - } catch (const std::exception&) { - return set_error(err, bitcoinconsensus_ERR_TX_DESERIALIZE); // Error deserializing - } -} - -int bitcoinconsensus_verify_script_with_spent_outputs(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, int64_t amount, - const unsigned char *txTo , unsigned int txToLen, - const UTXO *spentOutputs, unsigned int spentOutputsLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err) -{ - CAmount am(amount); - return ::verify_script(scriptPubKey, scriptPubKeyLen, am, txTo, txToLen, spentOutputs, spentOutputsLen, nIn, flags, err); -} - -int bitcoinconsensus_verify_script_with_amount(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, int64_t amount, - const unsigned char *txTo , unsigned int txToLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err) -{ - CAmount am(amount); - UTXO *spentOutputs = nullptr; - unsigned int spentOutputsLen = 0; - return ::verify_script(scriptPubKey, scriptPubKeyLen, am, txTo, txToLen, spentOutputs, spentOutputsLen, nIn, flags, err); -} - - -int bitcoinconsensus_verify_script(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, - const unsigned char *txTo , unsigned int txToLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err) -{ - if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS) { - return set_error(err, bitcoinconsensus_ERR_AMOUNT_REQUIRED); - } - - CAmount am(0); - UTXO *spentOutputs = nullptr; - unsigned int spentOutputsLen = 0; - return ::verify_script(scriptPubKey, scriptPubKeyLen, am, txTo, txToLen, spentOutputs, spentOutputsLen, nIn, flags, err); -} - -unsigned int bitcoinconsensus_version() -{ - // Just use the API version for now - return BITCOINCONSENSUS_API_VER; -} diff --git a/src/script/bitcoinconsensus.h b/src/script/bitcoinconsensus.h deleted file mode 100644 index a202b5ba06..0000000000 --- a/src/script/bitcoinconsensus.h +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_SCRIPT_BITCOINCONSENSUS_H -#define BITCOIN_SCRIPT_BITCOINCONSENSUS_H - -#include <stdint.h> - -#if defined(BUILD_BITCOIN_INTERNAL) && defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> - #if defined(_WIN32) - #if defined(HAVE_DLLEXPORT_ATTRIBUTE) - #define EXPORT_SYMBOL __declspec(dllexport) - #else - #define EXPORT_SYMBOL - #endif - #elif defined(HAVE_DEFAULT_VISIBILITY_ATTRIBUTE) - #define EXPORT_SYMBOL __attribute__ ((visibility ("default"))) - #endif -#elif defined(MSC_VER) && !defined(STATIC_LIBBITCOINCONSENSUS) - #define EXPORT_SYMBOL __declspec(dllimport) -#endif - -#ifndef EXPORT_SYMBOL - #define EXPORT_SYMBOL -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#define BITCOINCONSENSUS_API_VER 2 - -typedef enum bitcoinconsensus_error_t -{ - bitcoinconsensus_ERR_OK = 0, - bitcoinconsensus_ERR_TX_INDEX, - bitcoinconsensus_ERR_TX_SIZE_MISMATCH, - bitcoinconsensus_ERR_TX_DESERIALIZE, - bitcoinconsensus_ERR_AMOUNT_REQUIRED, - bitcoinconsensus_ERR_INVALID_FLAGS, - bitcoinconsensus_ERR_SPENT_OUTPUTS_REQUIRED, - bitcoinconsensus_ERR_SPENT_OUTPUTS_MISMATCH -} bitcoinconsensus_error; - -/** Script verification flags */ -enum -{ - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NONE = 0, - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_P2SH = (1U << 0), // evaluate P2SH (BIP16) subscripts - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_DERSIG = (1U << 2), // enforce strict DER (BIP66) compliance - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NULLDUMMY = (1U << 4), // enforce NULLDUMMY (BIP147) - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY = (1U << 9), // enable CHECKLOCKTIMEVERIFY (BIP65) - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKSEQUENCEVERIFY = (1U << 10), // enable CHECKSEQUENCEVERIFY (BIP112) - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS = (1U << 11), // enable WITNESS (BIP141) - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT = (1U << 17), // enable TAPROOT (BIPs 341 & 342) - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL = bitcoinconsensus_SCRIPT_FLAGS_VERIFY_P2SH | bitcoinconsensus_SCRIPT_FLAGS_VERIFY_DERSIG | - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NULLDUMMY | bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY | - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKSEQUENCEVERIFY | bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS | - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT -}; - -typedef struct { - const unsigned char *scriptPubKey; - unsigned int scriptPubKeySize; - int64_t value; -} UTXO; - -/// Returns 1 if the input nIn of the serialized transaction pointed to by -/// txTo correctly spends the scriptPubKey pointed to by scriptPubKey under -/// the additional constraints specified by flags. -/// If not nullptr, err will contain an error/success code for the operation -EXPORT_SYMBOL int bitcoinconsensus_verify_script(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, - const unsigned char *txTo , unsigned int txToLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err); - -EXPORT_SYMBOL int bitcoinconsensus_verify_script_with_amount(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, int64_t amount, - const unsigned char *txTo , unsigned int txToLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err); - -EXPORT_SYMBOL int bitcoinconsensus_verify_script_with_spent_outputs(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, int64_t amount, - const unsigned char *txTo , unsigned int txToLen, - const UTXO *spentOutputs, unsigned int spentOutputsLen, - unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err); - -EXPORT_SYMBOL unsigned int bitcoinconsensus_version(); - -#ifdef __cplusplus -} // extern "C" -#endif - -#undef EXPORT_SYMBOL - -#endif // BITCOIN_SCRIPT_BITCOINCONSENSUS_H diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index c6bc5f8f1d..a11d4dcbd5 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -212,6 +212,11 @@ public: /** Derive a private key, if private data is available in arg. */ virtual bool GetPrivKey(int pos, const SigningProvider& arg, CKey& key) const = 0; + + /** Return the non-extended public key for this PubkeyProvider, if it has one. */ + virtual std::optional<CPubKey> GetRootPubKey() const = 0; + /** Return the extended public key for this PubkeyProvider, if it has one. */ + virtual std::optional<CExtPubKey> GetRootExtPubKey() const = 0; }; class OriginPubkeyProvider final : public PubkeyProvider @@ -265,6 +270,14 @@ public: { return m_provider->GetPrivKey(pos, arg, key); } + std::optional<CPubKey> GetRootPubKey() const override + { + return m_provider->GetRootPubKey(); + } + std::optional<CExtPubKey> GetRootExtPubKey() const override + { + return m_provider->GetRootExtPubKey(); + } }; /** An object representing a parsed constant public key in a descriptor. */ @@ -310,6 +323,14 @@ public: { return arg.GetKey(m_pubkey.GetID(), key); } + std::optional<CPubKey> GetRootPubKey() const override + { + return m_pubkey; + } + std::optional<CExtPubKey> GetRootExtPubKey() const override + { + return std::nullopt; + } }; enum class DeriveType { @@ -525,6 +546,14 @@ public: key = extkey.key; return true; } + std::optional<CPubKey> GetRootPubKey() const override + { + return std::nullopt; + } + std::optional<CExtPubKey> GetRootExtPubKey() const override + { + return m_root_extkey; + } }; /** Base class for all Descriptor implementations. */ @@ -570,6 +599,7 @@ public: COMPAT, // string calculation that mustn't change over time to stay compatible with previous software versions }; + // NOLINTNEXTLINE(misc-no-recursion) bool IsSolvable() const override { for (const auto& arg : m_subdescriptor_args) { @@ -578,6 +608,7 @@ public: return true; } + // NOLINTNEXTLINE(misc-no-recursion) bool IsRange() const final { for (const auto& pubkey : m_pubkey_args) { @@ -589,6 +620,7 @@ public: return false; } + // NOLINTNEXTLINE(misc-no-recursion) virtual bool ToStringSubScriptHelper(const SigningProvider* arg, std::string& ret, const StringType type, const DescriptorCache* cache = nullptr) const { size_t pos = 0; @@ -601,6 +633,7 @@ public: return true; } + // NOLINTNEXTLINE(misc-no-recursion) virtual bool ToStringHelper(const SigningProvider* arg, std::string& out, const StringType type, const DescriptorCache* cache = nullptr) const { std::string extra = ToStringExtra(); @@ -653,6 +686,7 @@ public: return ret; } + // NOLINTNEXTLINE(misc-no-recursion) bool ExpandHelper(int pos, const SigningProvider& arg, const DescriptorCache* read_cache, std::vector<CScript>& output_scripts, FlatSigningProvider& out, DescriptorCache* write_cache) const { std::vector<std::pair<CPubKey, KeyOriginInfo>> entries; @@ -694,6 +728,7 @@ public: return ExpandHelper(pos, DUMMY_SIGNING_PROVIDER, &read_cache, output_scripts, out, nullptr); } + // NOLINTNEXTLINE(misc-no-recursion) void ExpandPrivate(int pos, const SigningProvider& provider, FlatSigningProvider& out) const final { for (const auto& p : m_pubkey_args) { @@ -720,6 +755,20 @@ public: std::optional<int64_t> MaxSatisfactionWeight(bool) const override { return {}; } std::optional<int64_t> MaxSatisfactionElems() const override { return {}; } + + // NOLINTNEXTLINE(misc-no-recursion) + void GetPubKeys(std::set<CPubKey>& pubkeys, std::set<CExtPubKey>& ext_pubs) const override + { + for (const auto& p : m_pubkey_args) { + std::optional<CPubKey> pub = p->GetRootPubKey(); + if (pub) pubkeys.insert(*pub); + std::optional<CExtPubKey> ext_pub = p->GetRootExtPubKey(); + if (ext_pub) ext_pubs.insert(*ext_pub); + } + for (const auto& arg : m_subdescriptor_args) { + arg->GetPubKeys(pubkeys, ext_pubs); + } + } }; /** A parsed addr(A) descriptor. */ @@ -1537,6 +1586,7 @@ struct KeyParser { }; /** Parse a script in a particular context. */ +// NOLINTNEXTLINE(misc-no-recursion) std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, std::string& error) { using namespace spanparsing; @@ -1844,6 +1894,7 @@ std::unique_ptr<DescriptorImpl> InferMultiA(const CScript& script, ParseScriptCo return std::make_unique<MultiADescriptor>(match->first, std::move(keys)); } +// NOLINTNEXTLINE(misc-no-recursion) std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptContext ctx, const SigningProvider& provider) { if (ctx == ParseScriptContext::P2TR && script.size() == 34 && script[0] == 32 && script[33] == OP_CHECKSIG) { diff --git a/src/script/descriptor.h b/src/script/descriptor.h index caa5d1608d..e78a775330 100644 --- a/src/script/descriptor.h +++ b/src/script/descriptor.h @@ -158,6 +158,13 @@ struct Descriptor { /** Get the maximum size number of stack elements for satisfying this descriptor. */ virtual std::optional<int64_t> MaxSatisfactionElems() const = 0; + + /** Return all (extended) public keys for this descriptor, including any from subdescriptors. + * + * @param[out] pubkeys Any public keys + * @param[out] ext_pubs Any extended public keys + */ + virtual void GetPubKeys(std::set<CPubKey>& pubkeys, std::set<CExtPubKey>& ext_pubs) const = 0; }; /** Parse a `descriptor` string. Included private keys are put in `out`. diff --git a/src/script/miniscript.h b/src/script/miniscript.h index 76b952350b..f635fa7340 100644 --- a/src/script/miniscript.h +++ b/src/script/miniscript.h @@ -1617,7 +1617,7 @@ public: //! Produce a witness for this script, if possible and given the information available in the context. //! The non-malleable satisfaction is guaranteed to be valid if it exists, and ValidSatisfaction() //! is true. If IsSane() holds, this satisfaction is guaranteed to succeed in case the node's - //! conditions are satisfied (private keys and hash preimages available, locktimes satsified). + //! conditions are satisfied (private keys and hash preimages available, locktimes satisfied). template<typename Ctx> Availability Satisfy(const Ctx& ctx, std::vector<std::vector<unsigned char>>& stack, bool nonmalleable = true) const { auto ret = ProduceInput(ctx); diff --git a/src/secp256k1/.github/actions/install-homebrew-valgrind/action.yml b/src/secp256k1/.github/actions/install-homebrew-valgrind/action.yml index 094ff891f7..ce10eb2686 100644 --- a/src/secp256k1/.github/actions/install-homebrew-valgrind/action.yml +++ b/src/secp256k1/.github/actions/install-homebrew-valgrind/action.yml @@ -16,7 +16,7 @@ runs: cat valgrind_fingerprint shell: bash - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: cache with: path: ${{ env.CI_HOMEBREW_CELLAR_VALGRIND }} diff --git a/src/secp256k1/.github/actions/run-in-docker-action/action.yml b/src/secp256k1/.github/actions/run-in-docker-action/action.yml index dbfaa4fece..74933686a0 100644 --- a/src/secp256k1/.github/actions/run-in-docker-action/action.yml +++ b/src/secp256k1/.github/actions/run-in-docker-action/action.yml @@ -36,6 +36,11 @@ runs: load: true cache-from: type=gha + - # Workaround for https://github.com/google/sanitizers/issues/1614 . + # The underlying issue has been fixed in clang 18.1.3. + run: sudo sysctl -w vm.mmap_rnd_bits=28 + shell: bash + - # Tell Docker to pass environment variables in `env` into the container. run: > docker run \ diff --git a/src/secp256k1/CMakeLists.txt b/src/secp256k1/CMakeLists.txt index cf0dc3ba93..9ef7defe51 100644 --- a/src/secp256k1/CMakeLists.txt +++ b/src/secp256k1/CMakeLists.txt @@ -51,29 +51,40 @@ endif() option(SECP256K1_INSTALL "Enable installation." ${PROJECT_IS_TOP_LEVEL}) -option(SECP256K1_ENABLE_MODULE_ECDH "Enable ECDH module." ON) -if(SECP256K1_ENABLE_MODULE_ECDH) - add_compile_definitions(ENABLE_MODULE_ECDH=1) -endif() +## Modules +# We declare all options before processing them, to make sure we can express +# dependendencies while processing. +option(SECP256K1_ENABLE_MODULE_ECDH "Enable ECDH module." ON) option(SECP256K1_ENABLE_MODULE_RECOVERY "Enable ECDSA pubkey recovery module." OFF) -if(SECP256K1_ENABLE_MODULE_RECOVERY) - add_compile_definitions(ENABLE_MODULE_RECOVERY=1) -endif() - option(SECP256K1_ENABLE_MODULE_EXTRAKEYS "Enable extrakeys module." ON) option(SECP256K1_ENABLE_MODULE_SCHNORRSIG "Enable schnorrsig module." ON) +option(SECP256K1_ENABLE_MODULE_ELLSWIFT "Enable ElligatorSwift module." ON) + +# Processing must be done in a topological sorting of the dependency graph +# (dependent module first). +if(SECP256K1_ENABLE_MODULE_ELLSWIFT) + add_compile_definitions(ENABLE_MODULE_ELLSWIFT=1) +endif() + if(SECP256K1_ENABLE_MODULE_SCHNORRSIG) + if(DEFINED SECP256K1_ENABLE_MODULE_EXTRAKEYS AND NOT SECP256K1_ENABLE_MODULE_EXTRAKEYS) + message(FATAL_ERROR "Module dependency error: You have disabled the extrakeys module explicitly, but it is required by the schnorrsig module.") + endif() set(SECP256K1_ENABLE_MODULE_EXTRAKEYS ON) add_compile_definitions(ENABLE_MODULE_SCHNORRSIG=1) endif() + if(SECP256K1_ENABLE_MODULE_EXTRAKEYS) add_compile_definitions(ENABLE_MODULE_EXTRAKEYS=1) endif() -option(SECP256K1_ENABLE_MODULE_ELLSWIFT "Enable ElligatorSwift module." ON) -if(SECP256K1_ENABLE_MODULE_ELLSWIFT) - add_compile_definitions(ENABLE_MODULE_ELLSWIFT=1) +if(SECP256K1_ENABLE_MODULE_RECOVERY) + add_compile_definitions(ENABLE_MODULE_RECOVERY=1) +endif() + +if(SECP256K1_ENABLE_MODULE_ECDH) + add_compile_definitions(ENABLE_MODULE_ECDH=1) endif() option(SECP256K1_USE_EXTERNAL_DEFAULT_CALLBACKS "Enable external default callback functions." OFF) @@ -254,9 +265,14 @@ if(SECP256K1_BUILD_BENCHMARK OR SECP256K1_BUILD_TESTS OR SECP256K1_BUILD_EXHAUST enable_testing() endif() +set(SECP256K1_LATE_CFLAGS "" CACHE STRING "Compiler flags that are added to the command line after all other flags added by the build system.") +include(AllTargetsCompileOptions) + add_subdirectory(src) +all_targets_compile_options(src "${SECP256K1_LATE_CFLAGS}") if(SECP256K1_BUILD_EXAMPLES) add_subdirectory(examples) + all_targets_compile_options(examples "${SECP256K1_LATE_CFLAGS}") endif() message("\n") @@ -330,6 +346,9 @@ else() message(" - LDFLAGS for executables ............ ${CMAKE_EXE_LINKER_FLAGS_DEBUG}") message(" - LDFLAGS for shared libraries ....... ${CMAKE_SHARED_LINKER_FLAGS_DEBUG}") endif() +if(SECP256K1_LATE_CFLAGS) + message("SECP256K1_LATE_CFLAGS ................. ${SECP256K1_LATE_CFLAGS}") +endif() message("\n") if(SECP256K1_EXPERIMENTAL) message( diff --git a/src/secp256k1/CONTRIBUTING.md b/src/secp256k1/CONTRIBUTING.md index a5e457913a..5fbf7332c9 100644 --- a/src/secp256k1/CONTRIBUTING.md +++ b/src/secp256k1/CONTRIBUTING.md @@ -44,7 +44,7 @@ The Contributor Workflow & Peer Review in libsecp256k1 are similar to Bitcoin Co In addition, libsecp256k1 tries to maintain the following coding conventions: -* No runtime heap allocation (e.g., no `malloc`) unless explicitly requested by the caller (via `secp256k1_context_create` or `secp256k1_scratch_space_create`, for example). Morever, it should be possible to use the library without any heap allocations. +* No runtime heap allocation (e.g., no `malloc`) unless explicitly requested by the caller (via `secp256k1_context_create` or `secp256k1_scratch_space_create`, for example). Moreover, it should be possible to use the library without any heap allocations. * The tests should cover all lines and branches of the library (see [Test coverage](#coverage)). * Operations involving secret data should be tested for being constant time with respect to the secrets (see [src/ctime_tests.c](src/ctime_tests.c)). * Local variables containing secret data should be cleared explicitly to try to delete secrets from memory. diff --git a/src/secp256k1/README.md b/src/secp256k1/README.md index 4013e6a93b..6e88eb4ecb 100644 --- a/src/secp256k1/README.md +++ b/src/secp256k1/README.md @@ -79,9 +79,9 @@ To maintain a pristine source tree, CMake encourages to perform an out-of-source $ mkdir build && cd build $ cmake .. - $ make - $ make check # run the test suite - $ sudo make install # optional + $ cmake --build . + $ ctest # run the test suite + $ sudo cmake --build . --target install # optional To compile optional modules (such as Schnorr signatures), you need to run `cmake` with additional flags (such as `-DSECP256K1_ENABLE_MODULE_SCHNORRSIG=ON`). Run `cmake .. -LH` to see the full list of available flags. diff --git a/src/secp256k1/ci/ci.sh b/src/secp256k1/ci/ci.sh index 9cc715955e..3999af4f1c 100755 --- a/src/secp256k1/ci/ci.sh +++ b/src/secp256k1/ci/ci.sh @@ -17,7 +17,8 @@ print_environment() { SECP256K1_TEST_ITERS BENCH SECP256K1_BENCH_ITERS CTIMETESTS\ EXAMPLES \ HOST WRAPPER_CMD \ - CC CFLAGS CPPFLAGS AR NM + CC CFLAGS CPPFLAGS AR NM \ + UBSAN_OPTIONS ASAN_OPTIONS LSAN_OPTIONS do eval "isset=\${$var+x}" if [ -n "$isset" ]; then diff --git a/src/secp256k1/cmake/AllTargetsCompileOptions.cmake b/src/secp256k1/cmake/AllTargetsCompileOptions.cmake new file mode 100644 index 0000000000..6e420e0fde --- /dev/null +++ b/src/secp256k1/cmake/AllTargetsCompileOptions.cmake @@ -0,0 +1,12 @@ +# Add compile options to all targets added in the subdirectory. +function(all_targets_compile_options dir options) + get_directory_property(targets DIRECTORY ${dir} BUILDSYSTEM_TARGETS) + separate_arguments(options) + set(compiled_target_types STATIC_LIBRARY SHARED_LIBRARY OBJECT_LIBRARY EXECUTABLE) + foreach(target ${targets}) + get_target_property(type ${target} TYPE) + if(type IN_LIST compiled_target_types) + target_compile_options(${target} PRIVATE ${options}) + endif() + endforeach() +endfunction() diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index 2c1596775e..158ed5d769 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -387,29 +387,32 @@ SECP_CFLAGS="$SECP_CFLAGS $WERROR_CFLAGS" ### Handle module options ### -if test x"$enable_module_ecdh" = x"yes"; then - SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ECDH=1" -fi - -if test x"$enable_module_recovery" = x"yes"; then - SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_RECOVERY=1" +# Processing must be done in a reverse topological sorting of the dependency graph +# (dependent module first). +if test x"$enable_module_ellswift" = x"yes"; then + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ELLSWIFT=1" fi if test x"$enable_module_schnorrsig" = x"yes"; then - SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_SCHNORRSIG=1" + if test x"$enable_module_extrakeys" = x"no"; then + AC_MSG_ERROR([Module dependency error: You have disabled the extrakeys module explicitly, but it is required by the schnorrsig module.]) + fi enable_module_extrakeys=yes + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_SCHNORRSIG=1" fi -if test x"$enable_module_ellswift" = x"yes"; then - SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ELLSWIFT=1" -fi - -# Test if extrakeys is set after the schnorrsig module to allow the schnorrsig -# module to set enable_module_extrakeys=yes if test x"$enable_module_extrakeys" = x"yes"; then SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_EXTRAKEYS=1" fi +if test x"$enable_module_recovery" = x"yes"; then + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_RECOVERY=1" +fi + +if test x"$enable_module_ecdh" = x"yes"; then + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ECDH=1" +fi + if test x"$enable_external_default_callbacks" = x"yes"; then SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DUSE_EXTERNAL_DEFAULT_CALLBACKS=1" fi diff --git a/src/secp256k1/contrib/lax_der_parsing.h b/src/secp256k1/contrib/lax_der_parsing.h index 034a38e6a0..37c8c691f2 100644 --- a/src/secp256k1/contrib/lax_der_parsing.h +++ b/src/secp256k1/contrib/lax_der_parsing.h @@ -67,8 +67,8 @@ extern "C" { * * Returns: 1 when the signature could be parsed, 0 otherwise. * Args: ctx: a secp256k1 context object - * Out: sig: a pointer to a signature object - * In: input: a pointer to the signature to be parsed + * Out: sig: pointer to a signature object + * In: input: pointer to the signature to be parsed * inputlen: the length of the array pointed to be input * * This function will accept any valid DER encoded signature, even if the diff --git a/src/secp256k1/doc/release-process.md b/src/secp256k1/doc/release-process.md index 51e337a5ab..cdf62430df 100644 --- a/src/secp256k1/doc/release-process.md +++ b/src/secp256k1/doc/release-process.md @@ -1,4 +1,4 @@ -# Release Process +# Release process This document outlines the process for releasing versions of the form `$MAJOR.$MINOR.$PATCH`. @@ -14,31 +14,30 @@ This process also assumes that there will be no minor releases for old major rel We aim to cut a regular release every 3-4 months, approximately twice as frequent as major Bitcoin Core releases. Every second release should be published one month before the feature freeze of the next major Bitcoin Core release, allowing sufficient time to update the library in Core. -## Sanity Checks -Perform these checks before creating a release: +## Sanity checks +Perform these checks when reviewing the release PR (see below): 1. Ensure `make distcheck` doesn't fail. -```shell -./autogen.sh && ./configure --enable-dev-mode && make distcheck -``` + ```shell + ./autogen.sh && ./configure --enable-dev-mode && make distcheck + ``` 2. Check installation with autotools: -```shell -dir=$(mktemp -d) -./autogen.sh && ./configure --prefix=$dir && make clean && make install && ls -RlAh $dir -gcc -o ecdsa examples/ecdsa.c $(PKG_CONFIG_PATH=$dir/lib/pkgconfig pkg-config --cflags --libs libsecp256k1) -Wl,-rpath,"$dir/lib" && ./ecdsa -``` + ```shell + dir=$(mktemp -d) + ./autogen.sh && ./configure --prefix=$dir && make clean && make install && ls -RlAh $dir + gcc -o ecdsa examples/ecdsa.c $(PKG_CONFIG_PATH=$dir/lib/pkgconfig pkg-config --cflags --libs libsecp256k1) -Wl,-rpath,"$dir/lib" && ./ecdsa + ``` 3. Check installation with CMake: -```shell -dir=$(mktemp -d) -build=$(mktemp -d) -cmake -B $build -DCMAKE_INSTALL_PREFIX=$dir && cmake --build $build --target install && ls -RlAh $dir -gcc -o ecdsa examples/ecdsa.c -I $dir/include -L $dir/lib*/ -l secp256k1 -Wl,-rpath,"$dir/lib",-rpath,"$dir/lib64" && ./ecdsa -``` -4. Use the [`check-abi.sh`](/tools/check-abi.sh) tool to ensure there are no unexpected ABI incompatibilities and that the version number and release notes accurately reflect all potential ABI changes. To run this tool, the `abi-dumper` and `abi-compliance-checker` packages are required. - -```shell -tools/check-abi.sh -``` + ```shell + dir=$(mktemp -d) + build=$(mktemp -d) + cmake -B $build -DCMAKE_INSTALL_PREFIX=$dir && cmake --build $build --target install && ls -RlAh $dir + gcc -o ecdsa examples/ecdsa.c -I $dir/include -L $dir/lib*/ -l secp256k1 -Wl,-rpath,"$dir/lib",-rpath,"$dir/lib64" && ./ecdsa + ``` +4. Use the [`check-abi.sh`](/tools/check-abi.sh) tool to verify that there are no unexpected ABI incompatibilities and that the version number and the release notes accurately reflect all potential ABI changes. To run this tool, the `abi-dumper` and `abi-compliance-checker` packages are required. + ```shell + tools/check-abi.sh + ``` ## Regular release @@ -47,27 +46,29 @@ tools/check-abi.sh * adding a section for the release (make sure that the version number is a link to a diff between the previous and new version), * removing the `[Unreleased]` section header, and * including an entry for `### ABI Compatibility` if it doesn't exist, - * sets `_PKG_VERSION_IS_RELEASE` to `true` in `configure.ac`, and - * if this is not a patch release - * updates `_PKG_VERSION_*` and `_LIB_VERSION_*` in `configure.ac` and + * sets `_PKG_VERSION_IS_RELEASE` to `true` in `configure.ac`, and, + * if this is not a patch release, + * updates `_PKG_VERSION_*` and `_LIB_VERSION_*` in `configure.ac`, and * updates `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_*` in `CMakeLists.txt`. -2. After the PR is merged, tag the commit and push it: +2. Perform the [sanity checks](#sanity-checks) on the PR branch. +3. After the PR is merged, tag the commit, and push the tag: ``` RELEASE_COMMIT=<merge commit of step 1> git tag -s v$MAJOR.$MINOR.$PATCH -m "libsecp256k1 $MAJOR.$MINOR.$PATCH" $RELEASE_COMMIT git push git@github.com:bitcoin-core/secp256k1.git v$MAJOR.$MINOR.$PATCH ``` -3. Open a PR to the master branch with a commit (using message `"release cleanup: bump version after $MAJOR.$MINOR.$PATCH"`, for example) that +4. Open a PR to the master branch with a commit (using message `"release cleanup: bump version after $MAJOR.$MINOR.$PATCH"`, for example) that * sets `_PKG_VERSION_IS_RELEASE` to `false` and increments `_PKG_VERSION_PATCH` and `_LIB_VERSION_REVISION` in `configure.ac`, * increments the `$PATCH` component of `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_REVISION` in `CMakeLists.txt`, and * adds an `[Unreleased]` section header to the [CHANGELOG.md](../CHANGELOG.md). If other maintainers are not present to approve the PR, it can be merged without ACKs. -4. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md). +5. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md). +6. Send an announcement email to the bitcoin-dev mailing list. ## Maintenance release -Note that bugfixes only need to be backported to releases for which no compatible release without the bug exists. +Note that bug fixes need to be backported only to releases for which no compatible release without the bug exists. 1. If there's no maintenance branch `$MAJOR.$MINOR`, create one: ``` @@ -75,19 +76,18 @@ Note that bugfixes only need to be backported to releases for which no compatibl git push git@github.com:bitcoin-core/secp256k1.git $MAJOR.$MINOR ``` 2. Open a pull request to the `$MAJOR.$MINOR` branch that - * includes the bugfixes, + * includes the bug fixes, * finalizes the release notes similar to a regular release, * increments `_PKG_VERSION_PATCH` and `_LIB_VERSION_REVISION` in `configure.ac` and the `$PATCH` component of `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_REVISION` in `CMakeLists.txt` (with commit message `"release: bump versions for $MAJOR.$MINOR.$PATCH"`, for example). -3. After the PRs are merged, update the release branch and tag the commit: +3. Perform the [sanity checks](#sanity-checks) on the PR branch. +4. After the PRs are merged, update the release branch, tag the commit, and push the tag: ``` git checkout $MAJOR.$MINOR && git pull git tag -s v$MAJOR.$MINOR.$PATCH -m "libsecp256k1 $MAJOR.$MINOR.$PATCH" - ``` -4. Push tag: - ``` git push git@github.com:bitcoin-core/secp256k1.git v$MAJOR.$MINOR.$PATCH ``` -5. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md). -6. Open PR to the master branch that includes a commit (with commit message `"release notes: add $MAJOR.$MINOR.$PATCH"`, for example) that adds release notes to [CHANGELOG.md](../CHANGELOG.md). +6. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md). +7. Send an announcement email to the bitcoin-dev mailing list. +8. Open PR to the master branch that includes a commit (with commit message `"release notes: add $MAJOR.$MINOR.$PATCH"`, for example) that adds release notes to [CHANGELOG.md](../CHANGELOG.md). diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index 936f0b42b7..f4053f2a93 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -265,7 +265,7 @@ SECP256K1_API void secp256k1_selftest(void); * memory allocation entirely, see secp256k1_context_static and the functions in * secp256k1_preallocated.h. * - * Returns: a newly created context object. + * Returns: pointer to a newly created context object. * In: flags: Always set to SECP256K1_CONTEXT_NONE (see below). * * The only valid non-deprecated flag in recent library versions is @@ -296,8 +296,8 @@ SECP256K1_API secp256k1_context *secp256k1_context_create( * Cloning secp256k1_context_static is not possible, and should not be emulated by * the caller (e.g., using memcpy). Create a new context instead. * - * Returns: a newly created context object. - * Args: ctx: an existing context to copy (not secp256k1_context_static) + * Returns: pointer to a newly created context object. + * Args: ctx: pointer to a context to copy (not secp256k1_context_static). */ SECP256K1_API secp256k1_context *secp256k1_context_clone( const secp256k1_context *ctx @@ -313,7 +313,7 @@ SECP256K1_API secp256k1_context *secp256k1_context_clone( * behaviour is undefined. In that case, secp256k1_context_preallocated_destroy must * be used instead. * - * Args: ctx: an existing context to destroy, constructed using + * Args: ctx: pointer to a context to destroy, constructed using * secp256k1_context_create or secp256k1_context_clone * (i.e., not secp256k1_context_static). */ @@ -350,8 +350,8 @@ SECP256K1_API void secp256k1_context_destroy( * fails. In this case, the corresponding default handler will be called with * the data pointer argument set to NULL. * - * Args: ctx: an existing context object. - * In: fun: a pointer to a function to call when an illegal argument is + * Args: ctx: pointer to a context object. + * In: fun: pointer to a function to call when an illegal argument is * passed to the API, taking a message and an opaque pointer. * (NULL restores the default handler.) * data: the opaque pointer to pass to fun above, must be NULL for the default handler. @@ -377,8 +377,8 @@ SECP256K1_API void secp256k1_context_set_illegal_callback( * for that). After this callback returns, anything may happen, including * crashing. * - * Args: ctx: an existing context object. - * In: fun: a pointer to a function to call when an internal error occurs, + * Args: ctx: pointer to a context object. + * In: fun: pointer to a function to call when an internal error occurs, * taking a message and an opaque pointer (NULL restores the * default handler, see secp256k1_context_set_illegal_callback * for details). @@ -395,7 +395,7 @@ SECP256K1_API void secp256k1_context_set_error_callback( /** Create a secp256k1 scratch space object. * * Returns: a newly created scratch space. - * Args: ctx: an existing context object. + * Args: ctx: pointer to a context object. * In: size: amount of memory to be available as scratch space. Some extra * (<100 bytes) will be allocated for extra accounting. */ @@ -407,7 +407,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT secp256k1_scratch_space *secp256k1_sc /** Destroy a secp256k1 scratch space. * * The pointer may not be used afterwards. - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object. * scratch: space to destroy */ SECP256K1_API void secp256k1_scratch_space_destroy( @@ -419,7 +419,7 @@ SECP256K1_API void secp256k1_scratch_space_destroy( * * Returns: 1 if the public key was fully valid. * 0 if the public key could not be parsed or is invalid. - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object. * Out: pubkey: pointer to a pubkey object. If 1 is returned, it is set to a * parsed version of input. If not, its value is undefined. * In: input: pointer to a serialized public key @@ -439,14 +439,14 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_parse( /** Serialize a pubkey object into a serialized byte sequence. * * Returns: 1 always. - * Args: ctx: a secp256k1 context object. - * Out: output: a pointer to a 65-byte (if compressed==0) or 33-byte (if + * Args: ctx: pointer to a context object. + * Out: output: pointer to a 65-byte (if compressed==0) or 33-byte (if * compressed==1) byte array to place the serialized key * in. - * In/Out: outputlen: a pointer to an integer which is initially set to the + * In/Out: outputlen: pointer to an integer which is initially set to the * size of output, and is overwritten with the written * size. - * In: pubkey: a pointer to a secp256k1_pubkey containing an + * In: pubkey: pointer to a secp256k1_pubkey containing an * initialized public key. * flags: SECP256K1_EC_COMPRESSED if serialization should be in * compressed format, otherwise SECP256K1_EC_UNCOMPRESSED. @@ -464,7 +464,7 @@ SECP256K1_API int secp256k1_ec_pubkey_serialize( * Returns: <0 if the first public key is less than the second * >0 if the first public key is greater than the second * 0 if the two public keys are equal - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object * In: pubkey1: first public key to compare * pubkey2: second public key to compare */ @@ -477,9 +477,9 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_cmp( /** Parse an ECDSA signature in compact (64 bytes) format. * * Returns: 1 when the signature could be parsed, 0 otherwise. - * Args: ctx: a secp256k1 context object - * Out: sig: a pointer to a signature object - * In: input64: a pointer to the 64-byte array to parse + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: input64: pointer to the 64-byte array to parse * * The signature must consist of a 32-byte big endian R value, followed by a * 32-byte big endian S value. If R or S fall outside of [0..order-1], the @@ -498,9 +498,9 @@ SECP256K1_API int secp256k1_ecdsa_signature_parse_compact( /** Parse a DER ECDSA signature. * * Returns: 1 when the signature could be parsed, 0 otherwise. - * Args: ctx: a secp256k1 context object - * Out: sig: a pointer to a signature object - * In: input: a pointer to the signature to be parsed + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: input: pointer to the signature to be parsed * inputlen: the length of the array pointed to be input * * This function will accept any valid DER encoded signature, even if the @@ -520,13 +520,13 @@ SECP256K1_API int secp256k1_ecdsa_signature_parse_der( /** Serialize an ECDSA signature in DER format. * * Returns: 1 if enough space was available to serialize, 0 otherwise - * Args: ctx: a secp256k1 context object - * Out: output: a pointer to an array to store the DER serialization - * In/Out: outputlen: a pointer to a length integer. Initially, this integer + * Args: ctx: pointer to a context object + * Out: output: pointer to an array to store the DER serialization + * In/Out: outputlen: pointer to a length integer. Initially, this integer * should be set to the length of output. After the call * it will be set to the length of the serialization (even * if 0 was returned). - * In: sig: a pointer to an initialized signature object + * In: sig: pointer to an initialized signature object */ SECP256K1_API int secp256k1_ecdsa_signature_serialize_der( const secp256k1_context *ctx, @@ -538,9 +538,9 @@ SECP256K1_API int secp256k1_ecdsa_signature_serialize_der( /** Serialize an ECDSA signature in compact (64 byte) format. * * Returns: 1 - * Args: ctx: a secp256k1 context object - * Out: output64: a pointer to a 64-byte array to store the compact serialization - * In: sig: a pointer to an initialized signature object + * Args: ctx: pointer to a context object + * Out: output64: pointer to a 64-byte array to store the compact serialization + * In: sig: pointer to an initialized signature object * * See secp256k1_ecdsa_signature_parse_compact for details about the encoding. */ @@ -554,7 +554,7 @@ SECP256K1_API int secp256k1_ecdsa_signature_serialize_compact( * * Returns: 1: correct signature * 0: incorrect or unparseable signature - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object * In: sig: the signature being verified. * msghash32: the 32-byte message hash being verified. * The verifier must make sure to apply a cryptographic @@ -585,12 +585,12 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ecdsa_verify( /** Convert a signature to a normalized lower-S form. * * Returns: 1 if sigin was not normalized, 0 if it already was. - * Args: ctx: a secp256k1 context object - * Out: sigout: a pointer to a signature to fill with the normalized form, + * Args: ctx: pointer to a context object + * Out: sigout: pointer to a signature to fill with the normalized form, * or copy if the input was already normalized. (can be NULL if * you're only interested in whether the input was already * normalized). - * In: sigin: a pointer to a signature to check/normalize (can be identical to sigout) + * In: sigin: pointer to a signature to check/normalize (can be identical to sigout) * * With ECDSA a third-party can forge a second distinct signature of the same * message, given a single initial signature, but without knowing the key. This diff --git a/src/secp256k1/include/secp256k1_ecdh.h b/src/secp256k1/include/secp256k1_ecdh.h index 515e174299..4d9da3461d 100644 --- a/src/secp256k1/include/secp256k1_ecdh.h +++ b/src/secp256k1/include/secp256k1_ecdh.h @@ -39,7 +39,7 @@ SECP256K1_API const secp256k1_ecdh_hash_function secp256k1_ecdh_hash_function_de * 0: scalar was invalid (zero or overflow) or hashfp returned 0 * Args: ctx: pointer to a context object. * Out: output: pointer to an array to be filled by hashfp. - * In: pubkey: a pointer to a secp256k1_pubkey containing an initialized public key. + * In: pubkey: pointer to a secp256k1_pubkey containing an initialized public key. * seckey: a 32-byte scalar with which to multiply the point. * hashfp: pointer to a hash function. If NULL, * secp256k1_ecdh_hash_function_sha256 is used diff --git a/src/secp256k1/include/secp256k1_ellswift.h b/src/secp256k1/include/secp256k1_ellswift.h index f79bd88396..ae37287f82 100644 --- a/src/secp256k1/include/secp256k1_ellswift.h +++ b/src/secp256k1/include/secp256k1_ellswift.h @@ -87,7 +87,7 @@ SECP256K1_API const secp256k1_ellswift_xdh_hash_function secp256k1_ellswift_xdh_ * Returns: 1 always. * Args: ctx: pointer to a context object * Out: ell64: pointer to a 64-byte array to be filled - * In: pubkey: a pointer to a secp256k1_pubkey containing an + * In: pubkey: pointer to a secp256k1_pubkey containing an * initialized public key * rnd32: pointer to 32 bytes of randomness * @@ -169,7 +169,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ellswift_create( * (will not be NULL) * ell_b64: pointer to the 64-byte encoded public key of party B * (will not be NULL) - * seckey32: a pointer to our 32-byte secret key + * seckey32: pointer to our 32-byte secret key * party: boolean indicating which party we are: zero if we are * party A, non-zero if we are party B. seckey32 must be * the private key corresponding to that party's ell_?64. diff --git a/src/secp256k1/include/secp256k1_extrakeys.h b/src/secp256k1/include/secp256k1_extrakeys.h index 7fcce68e68..ad70b92f95 100644 --- a/src/secp256k1/include/secp256k1_extrakeys.h +++ b/src/secp256k1/include/secp256k1_extrakeys.h @@ -39,7 +39,7 @@ typedef struct { * Returns: 1 if the public key was fully valid. * 0 if the public key could not be parsed or is invalid. * - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object. * Out: pubkey: pointer to a pubkey object. If 1 is returned, it is set to a * parsed version of input. If not, it's set to an invalid value. * In: input32: pointer to a serialized xonly_pubkey. @@ -54,9 +54,9 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_parse( * * Returns: 1 always. * - * Args: ctx: a secp256k1 context object. - * Out: output32: a pointer to a 32-byte array to place the serialized key in. - * In: pubkey: a pointer to a secp256k1_xonly_pubkey containing an initialized public key. + * Args: ctx: pointer to a context object. + * Out: output32: pointer to a 32-byte array to place the serialized key in. + * In: pubkey: pointer to a secp256k1_xonly_pubkey containing an initialized public key. */ SECP256K1_API int secp256k1_xonly_pubkey_serialize( const secp256k1_context *ctx, @@ -69,7 +69,7 @@ SECP256K1_API int secp256k1_xonly_pubkey_serialize( * Returns: <0 if the first public key is less than the second * >0 if the first public key is greater than the second * 0 if the two public keys are equal - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object. * In: pubkey1: first public key to compare * pubkey2: second public key to compare */ diff --git a/src/secp256k1/include/secp256k1_preallocated.h b/src/secp256k1/include/secp256k1_preallocated.h index f37744777b..f2d95c245e 100644 --- a/src/secp256k1/include/secp256k1_preallocated.h +++ b/src/secp256k1/include/secp256k1_preallocated.h @@ -52,8 +52,8 @@ SECP256K1_API size_t secp256k1_context_preallocated_size( * in the memory. In simpler words, the prealloc pointer (or any pointer derived * from it) should not be used during the lifetime of the context object. * - * Returns: a newly created context object. - * In: prealloc: a pointer to a rewritable contiguous block of memory of + * Returns: pointer to newly created context object. + * In: prealloc: pointer to a rewritable contiguous block of memory of * size at least secp256k1_context_preallocated_size(flags) * bytes, as detailed above. * flags: which parts of the context to initialize. @@ -72,7 +72,7 @@ SECP256K1_API secp256k1_context *secp256k1_context_preallocated_create( * caller-provided memory. * * Returns: the required size of the caller-provided memory block. - * In: ctx: an existing context to copy. + * In: ctx: pointer to a context to copy. */ SECP256K1_API size_t secp256k1_context_preallocated_clone_size( const secp256k1_context *ctx @@ -91,9 +91,9 @@ SECP256K1_API size_t secp256k1_context_preallocated_clone_size( * Cloning secp256k1_context_static is not possible, and should not be emulated by * the caller (e.g., using memcpy). Create a new context instead. * - * Returns: a newly created context object. - * Args: ctx: an existing context to copy (not secp256k1_context_static). - * In: prealloc: a pointer to a rewritable contiguous block of memory of + * Returns: pointer to a newly created context object. + * Args: ctx: pointer to a context to copy (not secp256k1_context_static). + * In: prealloc: pointer to a rewritable contiguous block of memory of * size at least secp256k1_context_preallocated_size(flags) * bytes, as detailed above. */ @@ -118,7 +118,7 @@ SECP256K1_API secp256k1_context *secp256k1_context_preallocated_clone( * preallocated pointer given to secp256k1_context_preallocated_create or * secp256k1_context_preallocated_clone. * - * Args: ctx: an existing context to destroy, constructed using + * Args: ctx: pointer to a context to destroy, constructed using * secp256k1_context_preallocated_create or * secp256k1_context_preallocated_clone * (i.e., not secp256k1_context_static). diff --git a/src/secp256k1/include/secp256k1_recovery.h b/src/secp256k1/include/secp256k1_recovery.h index b12ca4d972..341b8bac63 100644 --- a/src/secp256k1/include/secp256k1_recovery.h +++ b/src/secp256k1/include/secp256k1_recovery.h @@ -28,9 +28,9 @@ typedef struct { /** Parse a compact ECDSA signature (64 bytes + recovery id). * * Returns: 1 when the signature could be parsed, 0 otherwise - * Args: ctx: a secp256k1 context object - * Out: sig: a pointer to a signature object - * In: input64: a pointer to a 64-byte compact signature + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: input64: pointer to a 64-byte compact signature * recid: the recovery id (0, 1, 2 or 3) */ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_parse_compact( @@ -43,9 +43,9 @@ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_parse_compact( /** Convert a recoverable signature into a normal signature. * * Returns: 1 - * Args: ctx: a secp256k1 context object. - * Out: sig: a pointer to a normal signature. - * In: sigin: a pointer to a recoverable signature. + * Args: ctx: pointer to a context object. + * Out: sig: pointer to a normal signature. + * In: sigin: pointer to a recoverable signature. */ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_convert( const secp256k1_context *ctx, @@ -56,10 +56,10 @@ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_convert( /** Serialize an ECDSA signature in compact format (64 bytes + recovery id). * * Returns: 1 - * Args: ctx: a secp256k1 context object. - * Out: output64: a pointer to a 64-byte array of the compact signature. - * recid: a pointer to an integer to hold the recovery id. - * In: sig: a pointer to an initialized signature object. + * Args: ctx: pointer to a context object. + * Out: output64: pointer to a 64-byte array of the compact signature. + * recid: pointer to an integer to hold the recovery id. + * In: sig: pointer to an initialized signature object. */ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_serialize_compact( const secp256k1_context *ctx, diff --git a/src/secp256k1/include/secp256k1_schnorrsig.h b/src/secp256k1/include/secp256k1_schnorrsig.h index 26358533f6..23163de2fb 100644 --- a/src/secp256k1/include/secp256k1_schnorrsig.h +++ b/src/secp256k1/include/secp256k1_schnorrsig.h @@ -169,11 +169,11 @@ SECP256K1_API int secp256k1_schnorrsig_sign_custom( * * Returns: 1: correct signature * 0: incorrect signature - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object. * In: sig64: pointer to the 64-byte signature to verify. * msg: the message being verified. Can only be NULL if msglen is 0. * msglen: length of the message - * pubkey: pointer to an x-only public key to verify with (cannot be NULL) + * pubkey: pointer to an x-only public key to verify with */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorrsig_verify( const secp256k1_context *ctx, diff --git a/src/secp256k1/src/assumptions.h b/src/secp256k1/src/assumptions.h index 8ed04209e9..7961005350 100644 --- a/src/secp256k1/src/assumptions.h +++ b/src/secp256k1/src/assumptions.h @@ -19,65 +19,69 @@ reduce the odds of experiencing an unwelcome surprise. */ -struct secp256k1_assumption_checker { - /* This uses a trick to implement a static assertion in C89: a type with an array of negative size is not - allowed. */ - int dummy_array[( - /* Bytes are 8 bits. */ - (CHAR_BIT == 8) && +#if defined(__has_attribute) +# if __has_attribute(__unavailable__) +__attribute__((__unavailable__("Don't call this function. It only exists because STATIC_ASSERT cannot be used outside a function."))) +# endif +#endif +static void secp256k1_assumption_checker(void) { + /* Bytes are 8 bits. */ + STATIC_ASSERT(CHAR_BIT == 8); - /* No integer promotion for uint32_t. This ensures that we can multiply uintXX_t values where XX >= 32 - without signed overflow, which would be undefined behaviour. */ - (UINT_MAX <= UINT32_MAX) && + /* No integer promotion for uint32_t. This ensures that we can multiply uintXX_t values where XX >= 32 + without signed overflow, which would be undefined behaviour. */ + STATIC_ASSERT(UINT_MAX <= UINT32_MAX); - /* Conversions from unsigned to signed outside of the bounds of the signed type are - implementation-defined. Verify that they function as reinterpreting the lower - bits of the input in two's complement notation. Do this for conversions: - - from uint(N)_t to int(N)_t with negative result - - from uint(2N)_t to int(N)_t with negative result - - from int(2N)_t to int(N)_t with negative result - - from int(2N)_t to int(N)_t with positive result */ + /* Conversions from unsigned to signed outside of the bounds of the signed type are + implementation-defined. Verify that they function as reinterpreting the lower + bits of the input in two's complement notation. Do this for conversions: + - from uint(N)_t to int(N)_t with negative result + - from uint(2N)_t to int(N)_t with negative result + - from int(2N)_t to int(N)_t with negative result + - from int(2N)_t to int(N)_t with positive result */ - /* To int8_t. */ - ((int8_t)(uint8_t)0xAB == (int8_t)-(int8_t)0x55) && - ((int8_t)(uint16_t)0xABCD == (int8_t)-(int8_t)0x33) && - ((int8_t)(int16_t)(uint16_t)0xCDEF == (int8_t)(uint8_t)0xEF) && - ((int8_t)(int16_t)(uint16_t)0x9234 == (int8_t)(uint8_t)0x34) && + /* To int8_t. */ + STATIC_ASSERT(((int8_t)(uint8_t)0xAB == (int8_t)-(int8_t)0x55)); + STATIC_ASSERT((int8_t)(uint16_t)0xABCD == (int8_t)-(int8_t)0x33); + STATIC_ASSERT((int8_t)(int16_t)(uint16_t)0xCDEF == (int8_t)(uint8_t)0xEF); + STATIC_ASSERT((int8_t)(int16_t)(uint16_t)0x9234 == (int8_t)(uint8_t)0x34); - /* To int16_t. */ - ((int16_t)(uint16_t)0xBCDE == (int16_t)-(int16_t)0x4322) && - ((int16_t)(uint32_t)0xA1B2C3D4 == (int16_t)-(int16_t)0x3C2C) && - ((int16_t)(int32_t)(uint32_t)0xC1D2E3F4 == (int16_t)(uint16_t)0xE3F4) && - ((int16_t)(int32_t)(uint32_t)0x92345678 == (int16_t)(uint16_t)0x5678) && + /* To int16_t. */ + STATIC_ASSERT((int16_t)(uint16_t)0xBCDE == (int16_t)-(int16_t)0x4322); + STATIC_ASSERT((int16_t)(uint32_t)0xA1B2C3D4 == (int16_t)-(int16_t)0x3C2C); + STATIC_ASSERT((int16_t)(int32_t)(uint32_t)0xC1D2E3F4 == (int16_t)(uint16_t)0xE3F4); + STATIC_ASSERT((int16_t)(int32_t)(uint32_t)0x92345678 == (int16_t)(uint16_t)0x5678); - /* To int32_t. */ - ((int32_t)(uint32_t)0xB2C3D4E5 == (int32_t)-(int32_t)0x4D3C2B1B) && - ((int32_t)(uint64_t)0xA123B456C789D012ULL == (int32_t)-(int32_t)0x38762FEE) && - ((int32_t)(int64_t)(uint64_t)0xC1D2E3F4A5B6C7D8ULL == (int32_t)(uint32_t)0xA5B6C7D8) && - ((int32_t)(int64_t)(uint64_t)0xABCDEF0123456789ULL == (int32_t)(uint32_t)0x23456789) && + /* To int32_t. */ + STATIC_ASSERT((int32_t)(uint32_t)0xB2C3D4E5 == (int32_t)-(int32_t)0x4D3C2B1B); + STATIC_ASSERT((int32_t)(uint64_t)0xA123B456C789D012ULL == (int32_t)-(int32_t)0x38762FEE); + STATIC_ASSERT((int32_t)(int64_t)(uint64_t)0xC1D2E3F4A5B6C7D8ULL == (int32_t)(uint32_t)0xA5B6C7D8); + STATIC_ASSERT((int32_t)(int64_t)(uint64_t)0xABCDEF0123456789ULL == (int32_t)(uint32_t)0x23456789); - /* To int64_t. */ - ((int64_t)(uint64_t)0xB123C456D789E012ULL == (int64_t)-(int64_t)0x4EDC3BA928761FEEULL) && + /* To int64_t. */ + STATIC_ASSERT((int64_t)(uint64_t)0xB123C456D789E012ULL == (int64_t)-(int64_t)0x4EDC3BA928761FEEULL); #if defined(SECP256K1_INT128_NATIVE) - ((int64_t)(((uint128_t)0xA1234567B8901234ULL << 64) + 0xC5678901D2345678ULL) == (int64_t)-(int64_t)0x3A9876FE2DCBA988ULL) && - (((int64_t)(int128_t)(((uint128_t)0xB1C2D3E4F5A6B7C8ULL << 64) + 0xD9E0F1A2B3C4D5E6ULL)) == (int64_t)(uint64_t)0xD9E0F1A2B3C4D5E6ULL) && - (((int64_t)(int128_t)(((uint128_t)0xABCDEF0123456789ULL << 64) + 0x0123456789ABCDEFULL)) == (int64_t)(uint64_t)0x0123456789ABCDEFULL) && + STATIC_ASSERT((int64_t)(((uint128_t)0xA1234567B8901234ULL << 64) + 0xC5678901D2345678ULL) == (int64_t)-(int64_t)0x3A9876FE2DCBA988ULL); + STATIC_ASSERT(((int64_t)(int128_t)(((uint128_t)0xB1C2D3E4F5A6B7C8ULL << 64) + 0xD9E0F1A2B3C4D5E6ULL)) == (int64_t)(uint64_t)0xD9E0F1A2B3C4D5E6ULL); + STATIC_ASSERT(((int64_t)(int128_t)(((uint128_t)0xABCDEF0123456789ULL << 64) + 0x0123456789ABCDEFULL)) == (int64_t)(uint64_t)0x0123456789ABCDEFULL); - /* To int128_t. */ - ((int128_t)(((uint128_t)0xB1234567C8901234ULL << 64) + 0xD5678901E2345678ULL) == (int128_t)(-(int128_t)0x8E1648B3F50E80DCULL * 0x8E1648B3F50E80DDULL + 0x5EA688D5482F9464ULL)) && + /* To int128_t. */ + STATIC_ASSERT((int128_t)(((uint128_t)0xB1234567C8901234ULL << 64) + 0xD5678901E2345678ULL) == (int128_t)(-(int128_t)0x8E1648B3F50E80DCULL * 0x8E1648B3F50E80DDULL + 0x5EA688D5482F9464ULL)); #endif - /* Right shift on negative signed values is implementation defined. Verify that it - acts as a right shift in two's complement with sign extension (i.e duplicating - the top bit into newly added bits). */ - ((((int8_t)0xE8) >> 2) == (int8_t)(uint8_t)0xFA) && - ((((int16_t)0xE9AC) >> 4) == (int16_t)(uint16_t)0xFE9A) && - ((((int32_t)0x937C918A) >> 9) == (int32_t)(uint32_t)0xFFC9BE48) && - ((((int64_t)0xA8B72231DF9CF4B9ULL) >> 19) == (int64_t)(uint64_t)0xFFFFF516E4463BF3ULL) && + /* Right shift on negative signed values is implementation defined. Verify that it + acts as a right shift in two's complement with sign extension (i.e duplicating + the top bit into newly added bits). */ + STATIC_ASSERT((((int8_t)0xE8) >> 2) == (int8_t)(uint8_t)0xFA); + STATIC_ASSERT((((int16_t)0xE9AC) >> 4) == (int16_t)(uint16_t)0xFE9A); + STATIC_ASSERT((((int32_t)0x937C918A) >> 9) == (int32_t)(uint32_t)0xFFC9BE48); + STATIC_ASSERT((((int64_t)0xA8B72231DF9CF4B9ULL) >> 19) == (int64_t)(uint64_t)0xFFFFF516E4463BF3ULL); #if defined(SECP256K1_INT128_NATIVE) - ((((int128_t)(((uint128_t)0xCD833A65684A0DBCULL << 64) + 0xB349312F71EA7637ULL)) >> 39) == (int128_t)(((uint128_t)0xFFFFFFFFFF9B0674ULL << 64) + 0xCAD0941B79669262ULL)) && + STATIC_ASSERT((((int128_t)(((uint128_t)0xCD833A65684A0DBCULL << 64) + 0xB349312F71EA7637ULL)) >> 39) == (int128_t)(((uint128_t)0xFFFFFFFFFF9B0674ULL << 64) + 0xCAD0941B79669262ULL)); #endif - 1) * 2 - 1]; -}; + + /* This function is not supposed to be called. */ + VERIFY_CHECK(0); +} #endif /* SECP256K1_ASSUMPTIONS_H */ diff --git a/src/secp256k1/src/checkmem.h b/src/secp256k1/src/checkmem.h index f2169decfc..7e333ce5f3 100644 --- a/src/secp256k1/src/checkmem.h +++ b/src/secp256k1/src/checkmem.h @@ -30,6 +30,8 @@ * - SECP256K1_CHECKMEM_DEFINE(p, len): * - marks the len-byte memory pointed to by p as defined data (public data, in the * context of constant-time checking). + * - SECP256K1_CHECKMEM_MSAN_DEFINE(p, len): + * - Like SECP256K1_CHECKMEM_DEFINE, but applies only to memory_sanitizer. * */ @@ -48,11 +50,16 @@ # define SECP256K1_CHECKMEM_ENABLED 1 # define SECP256K1_CHECKMEM_UNDEFINE(p, len) __msan_allocated_memory((p), (len)) # define SECP256K1_CHECKMEM_DEFINE(p, len) __msan_unpoison((p), (len)) +# define SECP256K1_CHECKMEM_MSAN_DEFINE(p, len) __msan_unpoison((p), (len)) # define SECP256K1_CHECKMEM_CHECK(p, len) __msan_check_mem_is_initialized((p), (len)) # define SECP256K1_CHECKMEM_RUNNING() (1) # endif #endif +#if !defined SECP256K1_CHECKMEM_MSAN_DEFINE +# define SECP256K1_CHECKMEM_MSAN_DEFINE(p, len) SECP256K1_CHECKMEM_NOOP((p), (len)) +#endif + /* If valgrind integration is desired (through the VALGRIND define), implement the * SECP256K1_CHECKMEM_* macros using valgrind. */ #if !defined SECP256K1_CHECKMEM_ENABLED diff --git a/src/secp256k1/src/field.h b/src/secp256k1/src/field.h index bd589bf8a8..8c65a3aff6 100644 --- a/src/secp256k1/src/field.h +++ b/src/secp256k1/src/field.h @@ -255,8 +255,8 @@ static void secp256k1_fe_add(secp256k1_fe *r, const secp256k1_fe *a); /** Multiply two field elements. * * On input, a and b must be valid field elements; r does not need to be initialized. - * r and a may point to the same object, but neither can be equal to b. The magnitudes - * of a and b must not exceed 8. + * r and a may point to the same object, but neither may point to the object pointed + * to by b. The magnitudes of a and b must not exceed 8. * Performs {r = a * b} * On output, r will have magnitude 1, but won't be normalized. */ diff --git a/src/secp256k1/src/modules/ellswift/tests_impl.h b/src/secp256k1/src/modules/ellswift/tests_impl.h index 7d1efbc492..f96e3a1268 100644 --- a/src/secp256k1/src/modules/ellswift/tests_impl.h +++ b/src/secp256k1/src/modules/ellswift/tests_impl.h @@ -188,9 +188,9 @@ void run_ellswift_tests(void) { CHECK(ret == ((testcase->enc_bitmap >> c) & 1)); if (ret) { secp256k1_fe x2; - CHECK(check_fe_equal(&t, &testcase->encs[c])); + CHECK(fe_equal(&t, &testcase->encs[c])); secp256k1_ellswift_xswiftec_var(&x2, &testcase->u, &testcase->encs[c]); - CHECK(check_fe_equal(&testcase->x, &x2)); + CHECK(fe_equal(&testcase->x, &x2)); } } } @@ -203,7 +203,7 @@ void run_ellswift_tests(void) { CHECK(ret); ret = secp256k1_pubkey_load(CTX, &ge, &pubkey); CHECK(ret); - CHECK(check_fe_equal(&testcase->x, &ge.x)); + CHECK(fe_equal(&testcase->x, &ge.x)); CHECK(secp256k1_fe_is_odd(&ge.y) == testcase->odd_y); } for (i = 0; (unsigned)i < sizeof(ellswift_xdh_tests_bip324) / sizeof(ellswift_xdh_tests_bip324[0]); ++i) { @@ -290,7 +290,7 @@ void run_ellswift_tests(void) { secp256k1_ecmult(&resj, &decj, &sec, NULL); secp256k1_ge_set_gej(&res, &resj); /* Compare. */ - CHECK(check_fe_equal(&res.x, &share_x)); + CHECK(fe_equal(&res.x, &share_x)); } /* Verify the joint behavior of secp256k1_ellswift_xdh */ for (i = 0; i < 200 * COUNT; i++) { diff --git a/src/secp256k1/src/scalar_4x64_impl.h b/src/secp256k1/src/scalar_4x64_impl.h index 7b9c542f07..82cd957f16 100644 --- a/src/secp256k1/src/scalar_4x64_impl.h +++ b/src/secp256k1/src/scalar_4x64_impl.h @@ -462,6 +462,14 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) : "S"(l), "i"(SECP256K1_N_C_0), "i"(SECP256K1_N_C_1) : "rax", "rdx", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "cc"); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m0, sizeof(m0)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m1, sizeof(m1)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m2, sizeof(m2)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m3, sizeof(m3)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m4, sizeof(m4)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m5, sizeof(m5)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&m6, sizeof(m6)); + /* Reduce 385 bits into 258. */ __asm__ __volatile__( /* Preload */ @@ -541,6 +549,12 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) : "g"(m0), "g"(m1), "g"(m2), "g"(m3), "g"(m4), "g"(m5), "g"(m6), "i"(SECP256K1_N_C_0), "i"(SECP256K1_N_C_1) : "rax", "rdx", "r8", "r9", "r10", "r11", "r12", "r13", "cc"); + SECP256K1_CHECKMEM_MSAN_DEFINE(&p0, sizeof(p0)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&p1, sizeof(p1)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&p2, sizeof(p2)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&p3, sizeof(p3)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&p4, sizeof(p4)); + /* Reduce 258 bits into 256. */ __asm__ __volatile__( /* Preload */ @@ -586,6 +600,10 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) : "=g"(c) : "g"(p0), "g"(p1), "g"(p2), "g"(p3), "g"(p4), "D"(r), "i"(SECP256K1_N_C_0), "i"(SECP256K1_N_C_1) : "rax", "rdx", "r8", "r9", "r10", "cc", "memory"); + + SECP256K1_CHECKMEM_MSAN_DEFINE(r, sizeof(*r)); + SECP256K1_CHECKMEM_MSAN_DEFINE(&c, sizeof(c)); + #else secp256k1_uint128 c128; uint64_t c, c0, c1, c2; @@ -663,7 +681,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) secp256k1_scalar_reduce(r, c + secp256k1_scalar_check_overflow(r)); } -static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, const secp256k1_scalar *b) { +static void secp256k1_scalar_mul_512(uint64_t *l8, const secp256k1_scalar *a, const secp256k1_scalar *b) { #ifdef USE_ASM_X86_64 const uint64_t *pb = b->d; __asm__ __volatile__( @@ -678,7 +696,7 @@ static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, c /* (rax,rdx) = a0 * b0 */ "movq %%r15, %%rax\n" "mulq %%r11\n" - /* Extract l0 */ + /* Extract l8[0] */ "movq %%rax, 0(%%rsi)\n" /* (r8,r9,r10) = (rdx) */ "movq %%rdx, %%r8\n" @@ -696,7 +714,7 @@ static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, c "addq %%rax, %%r8\n" "adcq %%rdx, %%r9\n" "adcq $0, %%r10\n" - /* Extract l1 */ + /* Extract l8[1] */ "movq %%r8, 8(%%rsi)\n" "xorq %%r8, %%r8\n" /* (r9,r10,r8) += a0 * b2 */ @@ -717,7 +735,7 @@ static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, c "addq %%rax, %%r9\n" "adcq %%rdx, %%r10\n" "adcq $0, %%r8\n" - /* Extract l2 */ + /* Extract l8[2] */ "movq %%r9, 16(%%rsi)\n" "xorq %%r9, %%r9\n" /* (r10,r8,r9) += a0 * b3 */ @@ -746,7 +764,7 @@ static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, c "addq %%rax, %%r10\n" "adcq %%rdx, %%r8\n" "adcq $0, %%r9\n" - /* Extract l3 */ + /* Extract l8[3] */ "movq %%r10, 24(%%rsi)\n" "xorq %%r10, %%r10\n" /* (r8,r9,r10) += a1 * b3 */ @@ -767,7 +785,7 @@ static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, c "addq %%rax, %%r8\n" "adcq %%rdx, %%r9\n" "adcq $0, %%r10\n" - /* Extract l4 */ + /* Extract l8[4] */ "movq %%r8, 32(%%rsi)\n" "xorq %%r8, %%r8\n" /* (r9,r10,r8) += a2 * b3 */ @@ -782,51 +800,54 @@ static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, c "addq %%rax, %%r9\n" "adcq %%rdx, %%r10\n" "adcq $0, %%r8\n" - /* Extract l5 */ + /* Extract l8[5] */ "movq %%r9, 40(%%rsi)\n" /* (r10,r8) += a3 * b3 */ "movq %%r15, %%rax\n" "mulq %%r14\n" "addq %%rax, %%r10\n" "adcq %%rdx, %%r8\n" - /* Extract l6 */ + /* Extract l8[6] */ "movq %%r10, 48(%%rsi)\n" - /* Extract l7 */ + /* Extract l8[7] */ "movq %%r8, 56(%%rsi)\n" : "+d"(pb) - : "S"(l), "D"(a->d) + : "S"(l8), "D"(a->d) : "rax", "rbx", "rcx", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", "cc", "memory"); + + SECP256K1_CHECKMEM_MSAN_DEFINE(l8, sizeof(*l8) * 8); + #else /* 160 bit accumulator. */ uint64_t c0 = 0, c1 = 0; uint32_t c2 = 0; - /* l[0..7] = a[0..3] * b[0..3]. */ + /* l8[0..7] = a[0..3] * b[0..3]. */ muladd_fast(a->d[0], b->d[0]); - extract_fast(l[0]); + extract_fast(l8[0]); muladd(a->d[0], b->d[1]); muladd(a->d[1], b->d[0]); - extract(l[1]); + extract(l8[1]); muladd(a->d[0], b->d[2]); muladd(a->d[1], b->d[1]); muladd(a->d[2], b->d[0]); - extract(l[2]); + extract(l8[2]); muladd(a->d[0], b->d[3]); muladd(a->d[1], b->d[2]); muladd(a->d[2], b->d[1]); muladd(a->d[3], b->d[0]); - extract(l[3]); + extract(l8[3]); muladd(a->d[1], b->d[3]); muladd(a->d[2], b->d[2]); muladd(a->d[3], b->d[1]); - extract(l[4]); + extract(l8[4]); muladd(a->d[2], b->d[3]); muladd(a->d[3], b->d[2]); - extract(l[5]); + extract(l8[5]); muladd_fast(a->d[3], b->d[3]); - extract_fast(l[6]); + extract_fast(l8[6]); VERIFY_CHECK(c1 == 0); - l[7] = c0; + l8[7] = c0; #endif } diff --git a/src/secp256k1/src/scalar_impl.h b/src/secp256k1/src/scalar_impl.h index bbba83e937..972d8041b0 100644 --- a/src/secp256k1/src/scalar_impl.h +++ b/src/secp256k1/src/scalar_impl.h @@ -229,7 +229,7 @@ static void secp256k1_scalar_split_lambda(secp256k1_scalar * SECP256K1_RESTRICT * <= {triangle inequality} * a1*|k*b2/n - c1| + a2*|k*(-b1)/n - c2| * < {Lemma 1 and Lemma 2} - * a1*(2^-1 + epslion1) + a2*(2^-1 + epsilon2) + * a1*(2^-1 + epsilon1) + a2*(2^-1 + epsilon2) * < {rounding up to an integer} * (a1 + a2 + 1)/2 * < {rounding up to a power of 2} @@ -247,7 +247,7 @@ static void secp256k1_scalar_split_lambda(secp256k1_scalar * SECP256K1_RESTRICT * <= {triangle inequality} * (-b1)*|k*b2/n - c1| + b2*|k*(-b1)/n - c2| * < {Lemma 1 and Lemma 2} - * (-b1)*(2^-1 + epslion1) + b2*(2^-1 + epsilon2) + * (-b1)*(2^-1 + epsilon1) + b2*(2^-1 + epsilon2) * < {rounding up to an integer} * (-b1 + b2)/2 + 1 * < {rounding up to a power of 2} diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index 4c11e7f0b8..15a5eede67 100644 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -237,36 +237,25 @@ static SECP256K1_INLINE void secp256k1_declassify(const secp256k1_context* ctx, } static int secp256k1_pubkey_load(const secp256k1_context* ctx, secp256k1_ge* ge, const secp256k1_pubkey* pubkey) { - if (sizeof(secp256k1_ge_storage) == 64) { - /* When the secp256k1_ge_storage type is exactly 64 byte, use its - * representation inside secp256k1_pubkey, as conversion is very fast. - * Note that secp256k1_pubkey_save must use the same representation. */ - secp256k1_ge_storage s; - memcpy(&s, &pubkey->data[0], sizeof(s)); - secp256k1_ge_from_storage(ge, &s); - } else { - /* Otherwise, fall back to 32-byte big endian for X and Y. */ - secp256k1_fe x, y; - ARG_CHECK(secp256k1_fe_set_b32_limit(&x, pubkey->data)); - ARG_CHECK(secp256k1_fe_set_b32_limit(&y, pubkey->data + 32)); - secp256k1_ge_set_xy(ge, &x, &y); - } + secp256k1_ge_storage s; + + /* We require that the secp256k1_ge_storage type is exactly 64 bytes. + * This is formally not guaranteed by the C standard, but should hold on any + * sane compiler in the real world. */ + STATIC_ASSERT(sizeof(secp256k1_ge_storage) == 64); + memcpy(&s, &pubkey->data[0], 64); + secp256k1_ge_from_storage(ge, &s); ARG_CHECK(!secp256k1_fe_is_zero(&ge->x)); return 1; } static void secp256k1_pubkey_save(secp256k1_pubkey* pubkey, secp256k1_ge* ge) { - if (sizeof(secp256k1_ge_storage) == 64) { - secp256k1_ge_storage s; - secp256k1_ge_to_storage(&s, ge); - memcpy(&pubkey->data[0], &s, sizeof(s)); - } else { - VERIFY_CHECK(!secp256k1_ge_is_infinity(ge)); - secp256k1_fe_normalize_var(&ge->x); - secp256k1_fe_normalize_var(&ge->y); - secp256k1_fe_get_b32(pubkey->data, &ge->x); - secp256k1_fe_get_b32(pubkey->data + 32, &ge->y); - } + secp256k1_ge_storage s; + + STATIC_ASSERT(sizeof(secp256k1_ge_storage) == 64); + VERIFY_CHECK(!secp256k1_ge_is_infinity(ge)); + secp256k1_ge_to_storage(&s, ge); + memcpy(&pubkey->data[0], &s, 64); } int secp256k1_ec_pubkey_parse(const secp256k1_context* ctx, secp256k1_pubkey* pubkey, const unsigned char *input, size_t inputlen) { diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index bec1c45585..85b4881295 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -2927,20 +2927,18 @@ static void run_scalar_tests(void) { secp256k1_scalar_set_b32(&r2, res[i][1], &overflow); CHECK(!overflow); secp256k1_scalar_mul(&z, &x, &y); - CHECK(!secp256k1_scalar_check_overflow(&z)); CHECK(secp256k1_scalar_eq(&r1, &z)); if (!secp256k1_scalar_is_zero(&y)) { secp256k1_scalar_inverse(&zz, &y); - CHECK(!secp256k1_scalar_check_overflow(&zz)); secp256k1_scalar_inverse_var(&zzv, &y); CHECK(secp256k1_scalar_eq(&zzv, &zz)); secp256k1_scalar_mul(&z, &z, &zz); - CHECK(!secp256k1_scalar_check_overflow(&z)); CHECK(secp256k1_scalar_eq(&x, &z)); secp256k1_scalar_mul(&zz, &zz, &y); - CHECK(!secp256k1_scalar_check_overflow(&zz)); CHECK(secp256k1_scalar_eq(&secp256k1_scalar_one, &zz)); } + secp256k1_scalar_mul(&z, &x, &x); + CHECK(secp256k1_scalar_eq(&r2, &z)); } } } @@ -2955,7 +2953,7 @@ static void random_fe_non_square(secp256k1_fe *ns) { } } -static int check_fe_equal(const secp256k1_fe *a, const secp256k1_fe *b) { +static int fe_equal(const secp256k1_fe *a, const secp256k1_fe *b) { secp256k1_fe an = *a; secp256k1_fe bn = *b; secp256k1_fe_normalize_weak(&an); @@ -3092,7 +3090,7 @@ static void run_field_half(void) { #endif secp256k1_fe_normalize_weak(&u); secp256k1_fe_add(&u, &u); - CHECK(check_fe_equal(&t, &u)); + CHECK(fe_equal(&t, &u)); /* Check worst-case input: ensure the LSB is 1 so that P will be added, * which will also cause all carries to be 1, since all limbs that can @@ -3111,7 +3109,7 @@ static void run_field_half(void) { #endif secp256k1_fe_normalize_weak(&u); secp256k1_fe_add(&u, &u); - CHECK(check_fe_equal(&t, &u)); + CHECK(fe_equal(&t, &u)); } } @@ -3138,7 +3136,7 @@ static void run_field_misc(void) { secp256k1_fe_add(&z, &q); /* z = x+v */ q = x; /* q = x */ secp256k1_fe_add_int(&q, v); /* q = x+v */ - CHECK(check_fe_equal(&q, &z)); + CHECK(fe_equal(&q, &z)); /* Test the fe equality and comparison operations. */ CHECK(secp256k1_fe_cmp_var(&x, &x) == 0); CHECK(secp256k1_fe_equal(&x, &x)); @@ -3198,27 +3196,27 @@ static void run_field_misc(void) { secp256k1_fe_add(&y, &x); z = x; secp256k1_fe_mul_int(&z, 3); - CHECK(check_fe_equal(&y, &z)); + CHECK(fe_equal(&y, &z)); secp256k1_fe_add(&y, &x); secp256k1_fe_add(&z, &x); - CHECK(check_fe_equal(&z, &y)); + CHECK(fe_equal(&z, &y)); z = x; secp256k1_fe_mul_int(&z, 5); secp256k1_fe_mul(&q, &x, &fe5); - CHECK(check_fe_equal(&z, &q)); + CHECK(fe_equal(&z, &q)); secp256k1_fe_negate(&x, &x, 1); secp256k1_fe_add(&z, &x); secp256k1_fe_add(&q, &x); - CHECK(check_fe_equal(&y, &z)); - CHECK(check_fe_equal(&q, &y)); + CHECK(fe_equal(&y, &z)); + CHECK(fe_equal(&q, &y)); /* Check secp256k1_fe_half. */ z = x; secp256k1_fe_half(&z); secp256k1_fe_add(&z, &z); - CHECK(check_fe_equal(&x, &z)); + CHECK(fe_equal(&x, &z)); secp256k1_fe_add(&z, &z); secp256k1_fe_half(&z); - CHECK(check_fe_equal(&x, &z)); + CHECK(fe_equal(&x, &z)); } } @@ -3287,18 +3285,31 @@ static void run_fe_mul(void) { } static void run_sqr(void) { - secp256k1_fe x, s; + int i; + secp256k1_fe x, y, lhs, rhs, tmp; - { - int i; - secp256k1_fe_set_int(&x, 1); - secp256k1_fe_negate(&x, &x, 1); + secp256k1_fe_set_int(&x, 1); + secp256k1_fe_negate(&x, &x, 1); - for (i = 1; i <= 512; ++i) { - secp256k1_fe_mul_int(&x, 2); - secp256k1_fe_normalize(&x); - secp256k1_fe_sqr(&s, &x); - } + for (i = 1; i <= 512; ++i) { + secp256k1_fe_mul_int(&x, 2); + secp256k1_fe_normalize(&x); + + /* Check that (x+y)*(x-y) = x^2 - y*2 for some random values y */ + random_fe_test(&y); + + lhs = x; + secp256k1_fe_add(&lhs, &y); /* lhs = x+y */ + secp256k1_fe_negate(&tmp, &y, 1); /* tmp = -y */ + secp256k1_fe_add(&tmp, &x); /* tmp = x-y */ + secp256k1_fe_mul(&lhs, &lhs, &tmp); /* lhs = (x+y)*(x-y) */ + + secp256k1_fe_sqr(&rhs, &x); /* rhs = x^2 */ + secp256k1_fe_sqr(&tmp, &y); /* tmp = y^2 */ + secp256k1_fe_negate(&tmp, &tmp, 1); /* tmp = -y^2 */ + secp256k1_fe_add(&rhs, &tmp); /* rhs = x^2 - y^2 */ + + CHECK(fe_equal(&lhs, &rhs)); } } @@ -3620,9 +3631,9 @@ static void run_inverse_tests(void) for (i = 0; (size_t)i < sizeof(fe_cases)/sizeof(fe_cases[0]); ++i) { for (var = 0; var <= 1; ++var) { test_inverse_field(&x_fe, &fe_cases[i][0], var); - check_fe_equal(&x_fe, &fe_cases[i][1]); + CHECK(fe_equal(&x_fe, &fe_cases[i][1])); test_inverse_field(&x_fe, &fe_cases[i][1], var); - check_fe_equal(&x_fe, &fe_cases[i][0]); + CHECK(fe_equal(&x_fe, &fe_cases[i][0])); } } for (i = 0; (size_t)i < sizeof(scalar_cases)/sizeof(scalar_cases[0]); ++i) { @@ -4558,7 +4569,7 @@ static void ecmult_const_mult_xonly(void) { /* Check that resj's X coordinate corresponds with resx. */ secp256k1_fe_sqr(&v, &resj.z); secp256k1_fe_mul(&v, &v, &resx); - CHECK(check_fe_equal(&v, &resj.x)); + CHECK(fe_equal(&v, &resj.x)); } /* Test that secp256k1_ecmult_const_xonly correctly rejects X coordinates not on curve. */ diff --git a/src/secp256k1/src/util.h b/src/secp256k1/src/util.h index 187bf1c5e0..154d9ebcf1 100644 --- a/src/secp256k1/src/util.h +++ b/src/secp256k1/src/util.h @@ -51,13 +51,27 @@ static void print_buf_plain(const unsigned char *buf, size_t len) { # define SECP256K1_INLINE inline # endif +/** Assert statically that expr is true. + * + * This is a statement-like macro and can only be used inside functions. + */ +#define STATIC_ASSERT(expr) do { \ + switch(0) { \ + case 0: \ + /* If expr evaluates to 0, we have two case labels "0", which is illegal. */ \ + case /* ERROR: static assertion failed */ (expr): \ + ; \ + } \ +} while(0) + /** Assert statically that expr is an integer constant expression, and run stmt. * * Useful for example to enforce that magnitude arguments are constant. */ #define ASSERT_INT_CONST_AND_DO(expr, stmt) do { \ switch(42) { \ - case /* ERROR: integer argument is not constant */ expr: \ + /* C allows only integer constant expressions as case labels. */ \ + case /* ERROR: integer argument is not constant */ (expr): \ break; \ default: ; \ } \ diff --git a/src/secp256k1/tools/check-abi.sh b/src/secp256k1/tools/check-abi.sh index 8f6119cd8e..55c945ac16 100755 --- a/src/secp256k1/tools/check-abi.sh +++ b/src/secp256k1/tools/check-abi.sh @@ -3,17 +3,19 @@ set -eu default_base_version="$(git describe --match "v*.*.*" --abbrev=0)" -default_new_version="master" +default_new_version="HEAD" display_help_and_exit() { - echo "Usage: $0 <base_ver> <new_ver>" + echo "Usage: $0 [<base_ver> [<new_ver>]]" echo "" echo "Description: This script uses the ABI Compliance Checker tool to determine if the ABI" echo " of a new version of libsecp256k1 has changed in a backward-incompatible way." echo "" echo "Options:" - echo " base_ver Specify the base version (default: $default_base_version)" - echo " new_ver Specify the new version (default: $default_new_version)" + echo " base_ver Specify the base version as a git commit-ish" + echo " (default: most recent reachable tag matching \"v.*.*\", currently \"$default_base_version\")" + echo " new_ver Specify the new version as a git commit-ish" + echo " (default: $default_new_version)" echo " -h, --help Display this help message" exit 0 } @@ -23,9 +25,11 @@ if [ "$#" -eq 0 ]; then new_version="$default_new_version" elif [ "$#" -eq 1 ] && { [ "$1" = "-h" ] || [ "$1" = "--help" ]; }; then display_help_and_exit -elif [ "$#" -eq 2 ]; then +elif [ "$#" -eq 1 ] || [ "$#" -eq 2 ]; then base_version="$1" - new_version="$2" + if [ "$#" -eq 2 ]; then + new_version="$2" + fi else echo "Invalid usage. See help:" echo "" @@ -33,7 +37,8 @@ else fi checkout_and_build() { - git worktree add -d "$1" "$2" + _orig_dir="$(pwd)" + git worktree add --detach "$1" "$2" cd "$1" mkdir build && cd build cmake -S .. --preset dev-mode \ @@ -45,20 +50,18 @@ checkout_and_build() { -DSECP256K1_BUILD_EXAMPLES=OFF cmake --build . -j "$(nproc)" abi-dumper src/libsecp256k1.so -o ABI.dump -lver "$2" + cd "$_orig_dir" } echo "Comparing $base_version (base version) to $new_version (new version)" echo -original_dir="$(pwd)" - -base_source_dir=$(mktemp -d) +base_source_dir="$(mktemp -d)" checkout_and_build "$base_source_dir" "$base_version" -new_source_dir=$(mktemp -d) +new_source_dir="$(mktemp -d)" checkout_and_build "$new_source_dir" "$new_version" -cd "$original_dir" abi-compliance-checker -lib libsecp256k1 -old "${base_source_dir}/build/ABI.dump" -new "${new_source_dir}/build/ABI.dump" git worktree remove "$base_source_dir" git worktree remove "$new_source_dir" diff --git a/src/test/argsman_tests.cpp b/src/test/argsman_tests.cpp index 1f46efe464..340208a1c9 100644 --- a/src/test/argsman_tests.cpp +++ b/src/test/argsman_tests.cpp @@ -904,7 +904,7 @@ BOOST_FIXTURE_TEST_CASE(util_ArgsMerge, ArgsMergeTestingSetup) // If check below fails, should manually dump the results with: // - // ARGS_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=util_tests/util_ArgsMerge + // ARGS_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=argsman_tests/util_ArgsMerge // // And verify diff against previous results to make sure the changes are expected. // @@ -1007,7 +1007,7 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) // If check below fails, should manually dump the results with: // - // CHAIN_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=util_tests/util_ChainMerge + // CHAIN_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=argsman_tests/util_ChainMerge // // And verify diff against previous results to make sure the changes are expected. // diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 763f0f897e..05355fb21d 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -14,7 +14,7 @@ #include <boost/test/unit_test.hpp> -std::vector<std::pair<uint256, CTransactionRef>> extra_txn; +std::vector<CTransactionRef> extra_txn; BOOST_FIXTURE_TEST_SUITE(blockencodings_tests, RegTestingSetup) @@ -126,7 +126,7 @@ public: explicit TestHeaderAndShortIDs(const CBlock& block) : TestHeaderAndShortIDs(CBlockHeaderAndShortTxIDs{block}) {} - uint64_t GetShortID(const uint256& txhash) const { + uint64_t GetShortID(const Wtxid& txhash) const { DataStream stream{}; stream << *this; CBlockHeaderAndShortTxIDs base; @@ -155,8 +155,8 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) shortIDs.prefilledtxn.resize(1); shortIDs.prefilledtxn[0] = {1, block.vtx[1]}; shortIDs.shorttxids.resize(2); - shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[0]->GetHash()); - shortIDs.shorttxids[1] = shortIDs.GetShortID(block.vtx[2]->GetHash()); + shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[0]->GetWitnessHash()); + shortIDs.shorttxids[1] = shortIDs.GetShortID(block.vtx[2]->GetWitnessHash()); DataStream stream{}; stream << shortIDs; @@ -226,7 +226,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) shortIDs.prefilledtxn[0] = {0, block.vtx[0]}; shortIDs.prefilledtxn[1] = {1, block.vtx[2]}; // id == 1 as it is 1 after index 1 shortIDs.shorttxids.resize(1); - shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[1]->GetHash()); + shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[1]->GetWitnessHash()); DataStream stream{}; stream << shortIDs; diff --git a/src/test/feefrac_tests.cpp b/src/test/feefrac_tests.cpp new file mode 100644 index 0000000000..2e015b382e --- /dev/null +++ b/src/test/feefrac_tests.cpp @@ -0,0 +1,124 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/feefrac.h> +#include <random.h> + +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(feefrac_tests) + +BOOST_AUTO_TEST_CASE(feefrac_operators) +{ + FeeFrac p1{1000, 100}, p2{500, 300}; + FeeFrac sum{1500, 400}; + FeeFrac diff{500, -200}; + FeeFrac empty{0, 0}; + FeeFrac zero_fee{0, 1}; // zero-fee allowed + + BOOST_CHECK(empty == FeeFrac{}); // same as no-args + + BOOST_CHECK(p1 == p1); + BOOST_CHECK(p1 + p2 == sum); + BOOST_CHECK(p1 - p2 == diff); + + FeeFrac p3{2000, 200}; + BOOST_CHECK(p1 != p3); // feefracs only equal if both fee and size are same + BOOST_CHECK(p2 != p3); + + FeeFrac p4{3000, 300}; + BOOST_CHECK(p1 == p4-p3); + BOOST_CHECK(p1 + p3 == p4); + + // Fee-rate comparison + BOOST_CHECK(p1 > p2); + BOOST_CHECK(p1 >= p2); + BOOST_CHECK(p1 >= p4-p3); + BOOST_CHECK(!(p1 >> p3)); // not strictly better + BOOST_CHECK(p1 >> p2); // strictly greater feerate + + BOOST_CHECK(p2 < p1); + BOOST_CHECK(p2 <= p1); + BOOST_CHECK(p1 <= p4-p3); + BOOST_CHECK(!(p3 << p1)); // not strictly worse + BOOST_CHECK(p2 << p1); // strictly lower feerate + + // "empty" comparisons + BOOST_CHECK(!(p1 >> empty)); // << will always result in false + BOOST_CHECK(!(p1 << empty)); + BOOST_CHECK(!(empty >> empty)); + BOOST_CHECK(!(empty << empty)); + + // empty is always bigger than everything else + BOOST_CHECK(empty > p1); + BOOST_CHECK(empty > p2); + BOOST_CHECK(empty > p3); + BOOST_CHECK(empty >= p1); + BOOST_CHECK(empty >= p2); + BOOST_CHECK(empty >= p3); + + // check "max" values for comparison + FeeFrac oversized_1{4611686000000, 4000000}; + FeeFrac oversized_2{184467440000000, 100000}; + + BOOST_CHECK(oversized_1 < oversized_2); + BOOST_CHECK(oversized_1 <= oversized_2); + BOOST_CHECK(oversized_1 << oversized_2); + BOOST_CHECK(oversized_1 != oversized_2); + + // Tests paths that use double arithmetic + FeeFrac busted{(static_cast<int64_t>(INT32_MAX)) + 1, INT32_MAX}; + BOOST_CHECK(!(busted < busted)); + + FeeFrac max_fee{2100000000000000, INT32_MAX}; + BOOST_CHECK(!(max_fee < max_fee)); + BOOST_CHECK(!(max_fee > max_fee)); + BOOST_CHECK(max_fee <= max_fee); + BOOST_CHECK(max_fee >= max_fee); + + FeeFrac max_fee2{1, 1}; + BOOST_CHECK(max_fee >= max_fee2); + +} + +BOOST_AUTO_TEST_CASE(build_diagram_test) +{ + FeeFrac p1{1000, 100}; + FeeFrac empty{0, 0}; + FeeFrac zero_fee{0, 1}; + FeeFrac oversized_1{4611686000000, 4000000}; + FeeFrac oversized_2{184467440000000, 100000}; + + // Diagram-building will reorder chunks + std::vector<FeeFrac> chunks; + chunks.push_back(p1); + chunks.push_back(zero_fee); + chunks.push_back(empty); + chunks.push_back(oversized_1); + chunks.push_back(oversized_2); + + // Caller in charge of sorting chunks in case chunk size limit + // differs from cluster size limit + std::sort(chunks.begin(), chunks.end(), [](const FeeFrac& a, const FeeFrac& b) { return a > b; }); + + // Chunks are now sorted in reverse order (largest first) + BOOST_CHECK(chunks[0] == empty); // empty is considered "highest" fee + BOOST_CHECK(chunks[1] == oversized_2); + BOOST_CHECK(chunks[2] == oversized_1); + BOOST_CHECK(chunks[3] == p1); + BOOST_CHECK(chunks[4] == zero_fee); + + std::vector<FeeFrac> generated_diagram{BuildDiagramFromChunks(chunks)}; + BOOST_CHECK(generated_diagram.size() == 1 + chunks.size()); + + // Prepended with an empty, then the chunks summed in order as above + BOOST_CHECK(generated_diagram[0] == empty); + BOOST_CHECK(generated_diagram[1] == empty); + BOOST_CHECK(generated_diagram[2] == oversized_2); + BOOST_CHECK(generated_diagram[3] == oversized_2 + oversized_1); + BOOST_CHECK(generated_diagram[4] == oversized_2 + oversized_1 + p1); + BOOST_CHECK(generated_diagram[5] == oversized_2 + oversized_1 + p1 + zero_fee); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/feefrac.cpp b/src/test/fuzz/feefrac.cpp new file mode 100644 index 0000000000..2c7553360e --- /dev/null +++ b/src/test/fuzz/feefrac.cpp @@ -0,0 +1,123 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/feefrac.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <compare> +#include <cstdint> +#include <iostream> + +namespace { + +/** Compute a * b, represented in 4x32 bits, highest limb first. */ +std::array<uint32_t, 4> Mul128(uint64_t a, uint64_t b) +{ + std::array<uint32_t, 4> ret{0, 0, 0, 0}; + + /** Perform ret += v << (32 * pos), at 128-bit precision. */ + auto add_fn = [&](uint64_t v, int pos) { + uint64_t accum{0}; + for (int i = 0; i + pos < 4; ++i) { + // Add current value at limb pos in ret. + accum += ret[3 - pos - i]; + // Add low or high half of v. + if (i == 0) accum += v & 0xffffffff; + if (i == 1) accum += v >> 32; + // Store lower half of result in limb pos in ret. + ret[3 - pos - i] = accum & 0xffffffff; + // Leave carry in accum. + accum >>= 32; + } + // Make sure no overflow. + assert(accum == 0); + }; + + // Multiply the 4 individual limbs (schoolbook multiply, with base 2^32). + add_fn((a & 0xffffffff) * (b & 0xffffffff), 0); + add_fn((a >> 32) * (b & 0xffffffff), 1); + add_fn((a & 0xffffffff) * (b >> 32), 1); + add_fn((a >> 32) * (b >> 32), 2); + return ret; +} + +/* comparison helper for std::array */ +std::strong_ordering compare_arrays(const std::array<uint32_t, 4>& a, const std::array<uint32_t, 4>& b) { + for (size_t i = 0; i < a.size(); ++i) { + if (a[i] != b[i]) return a[i] <=> b[i]; + } + return std::strong_ordering::equal; +} + +std::strong_ordering MulCompare(int64_t a1, int64_t a2, int64_t b1, int64_t b2) +{ + // Compute and compare signs. + int sign_a = (a1 == 0 ? 0 : a1 < 0 ? -1 : 1) * (a2 == 0 ? 0 : a2 < 0 ? -1 : 1); + int sign_b = (b1 == 0 ? 0 : b1 < 0 ? -1 : 1) * (b2 == 0 ? 0 : b2 < 0 ? -1 : 1); + if (sign_a != sign_b) return sign_a <=> sign_b; + + // Compute absolute values. + uint64_t abs_a1 = static_cast<uint64_t>(a1), abs_a2 = static_cast<uint64_t>(a2); + uint64_t abs_b1 = static_cast<uint64_t>(b1), abs_b2 = static_cast<uint64_t>(b2); + // Use (~x + 1) instead of the equivalent (-x) to silence the linter; mod 2^64 behavior is + // intentional here. + if (a1 < 0) abs_a1 = ~abs_a1 + 1; + if (a2 < 0) abs_a2 = ~abs_a2 + 1; + if (b1 < 0) abs_b1 = ~abs_b1 + 1; + if (b2 < 0) abs_b2 = ~abs_b2 + 1; + + // Compute products of absolute values. + auto mul_abs_a = Mul128(abs_a1, abs_a2); + auto mul_abs_b = Mul128(abs_b1, abs_b2); + if (sign_a < 0) { + return compare_arrays(mul_abs_b, mul_abs_a); + } else { + return compare_arrays(mul_abs_a, mul_abs_b); + } +} + +} // namespace + +FUZZ_TARGET(feefrac) +{ + FuzzedDataProvider provider(buffer.data(), buffer.size()); + + int64_t f1 = provider.ConsumeIntegral<int64_t>(); + int32_t s1 = provider.ConsumeIntegral<int32_t>(); + if (s1 == 0) f1 = 0; + FeeFrac fr1(f1, s1); + assert(fr1.IsEmpty() == (s1 == 0)); + + int64_t f2 = provider.ConsumeIntegral<int64_t>(); + int32_t s2 = provider.ConsumeIntegral<int32_t>(); + if (s2 == 0) f2 = 0; + FeeFrac fr2(f2, s2); + assert(fr2.IsEmpty() == (s2 == 0)); + + // Feerate comparisons + auto cmp_feerate = MulCompare(f1, s2, f2, s1); + assert(FeeRateCompare(fr1, fr2) == cmp_feerate); + assert((fr1 << fr2) == std::is_lt(cmp_feerate)); + assert((fr1 >> fr2) == std::is_gt(cmp_feerate)); + + // Compare with manual invocation of FeeFrac::Mul. + auto cmp_mul = FeeFrac::Mul(f1, s2) <=> FeeFrac::Mul(f2, s1); + assert(cmp_mul == cmp_feerate); + + // Same, but using FeeFrac::MulFallback. + auto cmp_fallback = FeeFrac::MulFallback(f1, s2) <=> FeeFrac::MulFallback(f2, s1); + assert(cmp_fallback == cmp_feerate); + + // Total order comparisons + auto cmp_total = std::is_eq(cmp_feerate) ? (s2 <=> s1) : cmp_feerate; + assert((fr1 <=> fr2) == cmp_total); + assert((fr1 < fr2) == std::is_lt(cmp_total)); + assert((fr1 > fr2) == std::is_gt(cmp_total)); + assert((fr1 <= fr2) == std::is_lteq(cmp_total)); + assert((fr1 >= fr2) == std::is_gteq(cmp_total)); + assert((fr1 == fr2) == std::is_eq(cmp_total)); + assert((fr1 != fr2) == std::is_neq(cmp_total)); +} diff --git a/src/test/fuzz/feeratediagram.cpp b/src/test/fuzz/feeratediagram.cpp new file mode 100644 index 0000000000..6d710093cb --- /dev/null +++ b/src/test/fuzz/feeratediagram.cpp @@ -0,0 +1,119 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <stdint.h> + +#include <vector> + +#include <util/feefrac.h> +#include <policy/rbf.h> + +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <assert.h> + +namespace { + +/** Evaluate a diagram at a specific size, returning the fee as a fraction. + * + * Fees in diagram cannot exceed 2^32, as the returned evaluation could overflow + * the FeeFrac::fee field in the result. */ +FeeFrac EvaluateDiagram(int32_t size, Span<const FeeFrac> diagram) +{ + assert(diagram.size() > 0); + unsigned not_above = 0; + unsigned not_below = diagram.size() - 1; + // If outside the range of diagram, extend begin/end. + if (size < diagram[not_above].size) return {diagram[not_above].fee, 1}; + if (size > diagram[not_below].size) return {diagram[not_below].fee, 1}; + // Perform bisection search to locate the diagram segment that size is in. + while (not_below > not_above + 1) { + unsigned mid = (not_below + not_above) / 2; + if (diagram[mid].size <= size) not_above = mid; + if (diagram[mid].size >= size) not_below = mid; + } + // If the size matches a transition point between segments, return its fee. + if (not_below == not_above) return {diagram[not_below].fee, 1}; + // Otherwise, interpolate. + auto dir_coef = diagram[not_below] - diagram[not_above]; + assert(dir_coef.size > 0); + // Let A = diagram[not_above] and B = diagram[not_below] + const auto& point_a = diagram[not_above]; + // We want to return: + // A.fee + (B.fee - A.fee) / (B.size - A.size) * (size - A.size) + // = A.fee + dir_coef.fee / dir_coef.size * (size - A.size) + // = (A.fee * dir_coef.size + dir_coef.fee * (size - A.size)) / dir_coef.size + assert(size >= point_a.size); + return {point_a.fee * dir_coef.size + dir_coef.fee * (size - point_a.size), dir_coef.size}; +} + +std::weak_ordering CompareFeeFracWithDiagram(const FeeFrac& ff, Span<const FeeFrac> diagram) +{ + return FeeRateCompare(FeeFrac{ff.fee, 1}, EvaluateDiagram(ff.size, diagram)); +} + +std::partial_ordering CompareDiagrams(Span<const FeeFrac> dia1, Span<const FeeFrac> dia2) +{ + bool all_ge = true; + bool all_le = true; + for (const auto p1 : dia1) { + auto cmp = CompareFeeFracWithDiagram(p1, dia2); + if (std::is_lt(cmp)) all_ge = false; + if (std::is_gt(cmp)) all_le = false; + } + for (const auto p2 : dia2) { + auto cmp = CompareFeeFracWithDiagram(p2, dia1); + if (std::is_lt(cmp)) all_le = false; + if (std::is_gt(cmp)) all_ge = false; + } + if (all_ge && all_le) return std::partial_ordering::equivalent; + if (all_ge && !all_le) return std::partial_ordering::greater; + if (!all_ge && all_le) return std::partial_ordering::less; + return std::partial_ordering::unordered; +} + +void PopulateChunks(FuzzedDataProvider& fuzzed_data_provider, std::vector<FeeFrac>& chunks) +{ + chunks.clear(); + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 50) + { + chunks.emplace_back(fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(INT32_MIN>>1, INT32_MAX>>1), fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(1, 1000000)); + } + return; +} + +} // namespace + +FUZZ_TARGET(build_and_compare_feerate_diagram) +{ + // Generate a random set of chunks + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + std::vector<FeeFrac> chunks1, chunks2; + FeeFrac empty{0, 0}; + + PopulateChunks(fuzzed_data_provider, chunks1); + PopulateChunks(fuzzed_data_provider, chunks2); + + std::vector<FeeFrac> diagram1{BuildDiagramFromChunks(chunks1)}; + std::vector<FeeFrac> diagram2{BuildDiagramFromChunks(chunks2)}; + + assert(diagram1.front() == empty); + assert(diagram2.front() == empty); + + auto real = CompareFeerateDiagram(diagram1, diagram2); + auto sim = CompareDiagrams(diagram1, diagram2); + assert(real == sim); + + // Do explicit evaluation at up to 1000 points, and verify consistency with the result. + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 1000) { + int32_t size = fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(0, diagram2.back().size); + auto eval1 = EvaluateDiagram(size, diagram1); + auto eval2 = EvaluateDiagram(size, diagram2); + auto cmp = FeeRateCompare(eval1, eval2); + if (std::is_lt(cmp)) assert(!std::is_gt(real)); + if (std::is_gt(cmp)) assert(!std::is_lt(real)); + } +} diff --git a/src/test/fuzz/package_eval.cpp b/src/test/fuzz/package_eval.cpp index cf33b23cd3..c201118bce 100644 --- a/src/test/fuzz/package_eval.cpp +++ b/src/test/fuzz/package_eval.cpp @@ -276,8 +276,14 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) // (the package is a test accept and ATMP is a submission). auto single_submit = txs.size() == 1 && fuzzed_data_provider.ConsumeBool(); + // Exercise client_maxfeerate logic + std::optional<CFeeRate> client_maxfeerate{}; + if (fuzzed_data_provider.ConsumeBool()) { + client_maxfeerate = CFeeRate(fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(-1, 50 * COIN), 100); + } + const auto result_package = WITH_LOCK(::cs_main, - return ProcessNewPackage(chainstate, tx_pool, txs, /*test_accept=*/single_submit, /*max_sane_feerate=*/{})); + return ProcessNewPackage(chainstate, tx_pool, txs, /*test_accept=*/single_submit, client_maxfeerate)); // Always set bypass_limits to false because it is not supported in ProcessNewPackage and // can be a source of divergence. diff --git a/src/test/fuzz/partially_downloaded_block.cpp b/src/test/fuzz/partially_downloaded_block.cpp index 4a4b46da60..ab75afe066 100644 --- a/src/test/fuzz/partially_downloaded_block.cpp +++ b/src/test/fuzz/partially_downloaded_block.cpp @@ -60,7 +60,7 @@ FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) // The coinbase is always available available.insert(0); - std::vector<std::pair<uint256, CTransactionRef>> extra_txn; + std::vector<CTransactionRef> extra_txn; for (size_t i = 1; i < block->vtx.size(); ++i) { auto tx{block->vtx[i]}; @@ -68,7 +68,7 @@ FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) bool add_to_mempool{fuzzed_data_provider.ConsumeBool()}; if (add_to_extra_txn) { - extra_txn.emplace_back(tx->GetWitnessHash(), tx); + extra_txn.emplace_back(tx); available.insert(i); } diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index aa6385d12d..754aff4e70 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -23,12 +23,30 @@ namespace { const BasicTestingSetup* g_setup; } // namespace +const int NUM_ITERS = 10000; + +std::vector<COutPoint> g_outpoints; + void initialize_rbf() { static const auto testing_setup = MakeNoLogFileContext<>(); g_setup = testing_setup.get(); } +void initialize_package_rbf() +{ + static const auto testing_setup = MakeNoLogFileContext<>(); + g_setup = testing_setup.get(); + + // Create a fixed set of unique "UTXOs" to source parents from + // to avoid fuzzer giving circular references + for (int i = 0; i < NUM_ITERS; ++i) { + g_outpoints.emplace_back(); + g_outpoints.back().n = i; + } + +} + FUZZ_TARGET(rbf, .init = initialize_rbf) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); @@ -40,7 +58,7 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), NUM_ITERS) { const std::optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!another_mtx) { @@ -63,3 +81,113 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) (void)IsRBFOptIn(tx, pool); } } + +void CheckDiagramConcave(std::vector<FeeFrac>& diagram) +{ + // Diagrams are in monotonically-decreasing feerate order. + FeeFrac last_chunk = diagram.front(); + for (size_t i = 1; i<diagram.size(); ++i) { + FeeFrac next_chunk = diagram[i] - diagram[i-1]; + assert(next_chunk <= last_chunk); + last_chunk = next_chunk; + } +} + +FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + SetMockTime(ConsumeTime(fuzzed_data_provider)); + + std::optional<CMutableTransaction> child = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); + if (!child) return; + + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + + // Add a bunch of parent-child pairs to the mempool, and remember them. + std::vector<CTransaction> mempool_txs; + size_t iter{0}; + + LOCK2(cs_main, pool.cs); + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), NUM_ITERS) + { + // Make sure txns only have one input, and that a unique input is given to avoid circular references + std::optional<CMutableTransaction> parent = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); + if (!parent) { + continue; + } + assert(iter <= g_outpoints.size()); + parent->vin.resize(1); + parent->vin[0].prevout = g_outpoints[iter++]; + + mempool_txs.emplace_back(*parent); + pool.addUnchecked(ConsumeTxMemPoolEntry(fuzzed_data_provider, mempool_txs.back())); + if (fuzzed_data_provider.ConsumeBool() && !child->vin.empty()) { + child->vin[0].prevout = COutPoint{mempool_txs.back().GetHash(), 0}; + } + mempool_txs.emplace_back(*child); + pool.addUnchecked(ConsumeTxMemPoolEntry(fuzzed_data_provider, mempool_txs.back())); + + if (fuzzed_data_provider.ConsumeBool()) { + pool.PrioritiseTransaction(mempool_txs.back().GetHash().ToUint256(), fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(-100000, 100000)); + } + } + + // Pick some transactions at random to be the direct conflicts + CTxMemPool::setEntries direct_conflicts; + for (auto& tx : mempool_txs) { + if (fuzzed_data_provider.ConsumeBool()) { + direct_conflicts.insert(*pool.GetIter(tx.GetHash())); + } + } + + // Calculate all conflicts: + CTxMemPool::setEntries all_conflicts; + for (auto& txiter : direct_conflicts) { + pool.CalculateDescendants(txiter, all_conflicts); + } + + // Calculate the feerate diagrams for a replacement. + CAmount replacement_fees = ConsumeMoney(fuzzed_data_provider); + int64_t replacement_vsize = fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(1, 1000000); + auto calc_results{pool.CalculateFeerateDiagramsForRBF(replacement_fees, replacement_vsize, direct_conflicts, all_conflicts)}; + + if (calc_results.has_value()) { + // Sanity checks on the diagrams. + + // Diagrams start at 0. + assert(calc_results->first.front().size == 0); + assert(calc_results->first.front().fee == 0); + assert(calc_results->second.front().size == 0); + assert(calc_results->second.front().fee == 0); + + CheckDiagramConcave(calc_results->first); + CheckDiagramConcave(calc_results->second); + + CAmount replaced_fee{0}; + int64_t replaced_size{0}; + for (auto txiter : all_conflicts) { + replaced_fee += txiter->GetModifiedFee(); + replaced_size += txiter->GetTxSize(); + } + // The total fee of the new diagram should be the total fee of the old + // diagram - replaced_fee + replacement_fees + assert(calc_results->first.back().fee - replaced_fee + replacement_fees == calc_results->second.back().fee); + assert(calc_results->first.back().size - replaced_size + replacement_vsize == calc_results->second.back().size); + } + + // If internals report error, wrapper should too + auto err_tuple{ImprovesFeerateDiagram(pool, direct_conflicts, all_conflicts, replacement_fees, replacement_vsize)}; + if (!calc_results.has_value()) { + assert(err_tuple.value().first == DiagramCheckError::UNCALCULABLE); + } else { + // Diagram check succeeded + if (!err_tuple.has_value()) { + // New diagram's final fee should always match or exceed old diagram's + assert(calc_results->first.back().fee <= calc_results->second.back().fee); + } else if (calc_results->first.back().fee > calc_results->second.back().fee) { + // Or it failed, and if old diagram had higher fees, it should be a failure + assert(err_tuple.value().first == DiagramCheckError::FAILURE); + } + } +} diff --git a/src/test/fuzz/script_bitcoin_consensus.cpp b/src/test/fuzz/script_bitcoin_consensus.cpp deleted file mode 100644 index 846389863d..0000000000 --- a/src/test/fuzz/script_bitcoin_consensus.cpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <script/bitcoinconsensus.h> -#include <script/interpreter.h> -#include <test/fuzz/FuzzedDataProvider.h> -#include <test/fuzz/fuzz.h> -#include <test/fuzz/util.h> - -#include <cstdint> -#include <string> -#include <vector> - -FUZZ_TARGET(script_bitcoin_consensus) -{ - FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const std::vector<uint8_t> random_bytes_1 = ConsumeRandomLengthByteVector(fuzzed_data_provider); - const std::vector<uint8_t> random_bytes_2 = ConsumeRandomLengthByteVector(fuzzed_data_provider); - const CAmount money = ConsumeMoney(fuzzed_data_provider); - bitcoinconsensus_error err; - bitcoinconsensus_error* err_p = fuzzed_data_provider.ConsumeBool() ? &err : nullptr; - const unsigned int n_in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); - const unsigned int flags = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); - assert(bitcoinconsensus_version() == BITCOINCONSENSUS_API_VER); - if ((flags & SCRIPT_VERIFY_WITNESS) != 0 && (flags & SCRIPT_VERIFY_P2SH) == 0) { - return; - } - (void)bitcoinconsensus_verify_script(random_bytes_1.data(), random_bytes_1.size(), random_bytes_2.data(), random_bytes_2.size(), n_in, flags, err_p); - (void)bitcoinconsensus_verify_script_with_amount(random_bytes_1.data(), random_bytes_1.size(), money, random_bytes_2.data(), random_bytes_2.size(), n_in, flags, err_p); - - std::vector<UTXO> spent_outputs; - std::vector<std::vector<unsigned char>> spent_spks; - if (n_in <= 24386) { - spent_outputs.reserve(n_in); - spent_spks.reserve(n_in); - for (size_t i = 0; i < n_in; ++i) { - spent_spks.push_back(ConsumeRandomLengthByteVector(fuzzed_data_provider)); - const CAmount value{ConsumeMoney(fuzzed_data_provider)}; - const auto spk_size{static_cast<unsigned>(spent_spks.back().size())}; - spent_outputs.push_back({.scriptPubKey = spent_spks.back().data(), .scriptPubKeySize = spk_size, .value = value}); - } - } - - const auto spent_outs_size{static_cast<unsigned>(spent_outputs.size())}; - - (void)bitcoinconsensus_verify_script_with_spent_outputs( - random_bytes_1.data(), random_bytes_1.size(), money, random_bytes_2.data(), random_bytes_2.size(), - spent_outputs.data(), spent_outs_size, n_in, flags, err_p); -} diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 3611bccced..0b4019d5eb 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -291,7 +291,7 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) // Make sure ProcessNewPackage on one transaction works. // The result is not guaranteed to be the same as what is returned by ATMP. const auto result_package = WITH_LOCK(::cs_main, - return ProcessNewPackage(chainstate, tx_pool, {tx}, true, /*max_sane_feerate=*/{})); + return ProcessNewPackage(chainstate, tx_pool, {tx}, true, /*client_maxfeerate=*/{})); // If something went wrong due to a package-specific policy, it might not return a // validation result for the transaction. if (result_package.m_state.GetResult() != PackageValidationResult::PCKG_POLICY) { diff --git a/src/test/fuzz/util/descriptor.cpp b/src/test/fuzz/util/descriptor.cpp index df78bdf314..0fed2bc5e1 100644 --- a/src/test/fuzz/util/descriptor.cpp +++ b/src/test/fuzz/util/descriptor.cpp @@ -15,7 +15,7 @@ void MockedDescriptorConverter::Init() { // an extended one. if (IdIsCompPubKey(i) || IdIsUnCompPubKey(i) || IdIsXOnlyPubKey(i) || IdIsConstPrivKey(i)) { CKey privkey; - privkey.Set(UCharCast(key_data.begin()), UCharCast(key_data.end()), !IdIsUnCompPubKey(i)); + privkey.Set(key_data.begin(), key_data.end(), !IdIsUnCompPubKey(i)); if (IdIsCompPubKey(i) || IdIsUnCompPubKey(i)) { CPubKey pubkey{privkey.GetPubKey()}; keys_str[i] = HexStr(pubkey); diff --git a/src/test/miniscript_tests.cpp b/src/test/miniscript_tests.cpp index 1f28e61bc9..a3699f84b6 100644 --- a/src/test/miniscript_tests.cpp +++ b/src/test/miniscript_tests.cpp @@ -297,6 +297,7 @@ using miniscript::operator"" _mst; using Node = miniscript::Node<CPubKey>; /** Compute all challenges (pubkeys, hashes, timelocks) that occur in a given Miniscript. */ +// NOLINTNEXTLINE(misc-no-recursion) std::set<Challenge> FindChallenges(const NodeRef& ref) { std::set<Challenge> chal; for (const auto& key : ref->keys) { diff --git a/src/test/rbf_tests.cpp b/src/test/rbf_tests.cpp index e6c135eed9..ed33969710 100644 --- a/src/test/rbf_tests.cpp +++ b/src/test/rbf_tests.cpp @@ -37,7 +37,38 @@ static inline CTransactionRef make_tx(const std::vector<CTransactionRef>& inputs return MakeTransactionRef(tx); } -static void add_descendants(const CTransactionRef& tx, int32_t num_descendants, CTxMemPool& pool) +// Make two child transactions from parent (which must have at least 2 outputs). +// Each tx will have the same outputs, using the amounts specified in output_values. +static inline std::pair<CTransactionRef, CTransactionRef> make_two_siblings(const CTransactionRef parent, + const std::vector<CAmount>& output_values) +{ + assert(parent->vout.size() >= 2); + + // First tx takes first parent output + CMutableTransaction tx1 = CMutableTransaction(); + tx1.vin.resize(1); + tx1.vout.resize(output_values.size()); + + tx1.vin[0].prevout.hash = parent->GetHash(); + tx1.vin[0].prevout.n = 0; + // Add a witness so wtxid != txid + CScriptWitness witness; + witness.stack.emplace_back(10); + tx1.vin[0].scriptWitness = witness; + + for (size_t i = 0; i < output_values.size(); ++i) { + tx1.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx1.vout[i].nValue = output_values[i]; + } + + // Second tx takes second parent output + CMutableTransaction tx2 = tx1; + tx2.vin[0].prevout.n = 1; + + return std::make_pair(MakeTransactionRef(tx1), MakeTransactionRef(tx2)); +} + +static CTransactionRef add_descendants(const CTransactionRef& tx, int32_t num_descendants, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, pool.cs) { AssertLockHeld(::cs_main); @@ -50,6 +81,35 @@ static void add_descendants(const CTransactionRef& tx, int32_t num_descendants, pool.addUnchecked(entry.FromTx(next_tx)); tx_to_spend = next_tx; } + // Return last created tx + return tx_to_spend; +} + +static CTransactionRef add_descendant_to_parents(const std::vector<CTransactionRef>& parents, CTxMemPool& pool) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main, pool.cs) +{ + AssertLockHeld(::cs_main); + AssertLockHeld(pool.cs); + TestMemPoolEntryHelper entry; + // Assumes this isn't already spent in mempool + auto child_tx = make_tx(/*inputs=*/parents, /*output_values=*/{50 * CENT}); + pool.addUnchecked(entry.FromTx(child_tx)); + // Return last created tx + return child_tx; +} + +// Makes two children for a single parent +static std::pair<CTransactionRef, CTransactionRef> add_children_to_parent(const CTransactionRef parent, CTxMemPool& pool) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main, pool.cs) +{ + AssertLockHeld(::cs_main); + AssertLockHeld(pool.cs); + TestMemPoolEntryHelper entry; + // Assumes this isn't already spent in mempool + auto children_tx = make_two_siblings(/*parent=*/parent, /*output_values=*/{50 * CENT}); + pool.addUnchecked(entry.FromTx(children_tx.first)); + pool.addUnchecked(entry.FromTx(children_tx.second)); + return children_tx; } BOOST_FIXTURE_TEST_CASE(rbf_helper_functions, TestChain100Setup) @@ -89,28 +149,51 @@ BOOST_FIXTURE_TEST_CASE(rbf_helper_functions, TestChain100Setup) const auto tx8 = make_tx(/*inputs=*/ {m_coinbase_txns[4]}, /*output_values=*/ {999 * CENT}); pool.addUnchecked(entry.Fee(high_fee).FromTx(tx8)); - const auto entry1 = pool.GetIter(tx1->GetHash()).value(); - const auto entry2 = pool.GetIter(tx2->GetHash()).value(); - const auto entry3 = pool.GetIter(tx3->GetHash()).value(); - const auto entry4 = pool.GetIter(tx4->GetHash()).value(); - const auto entry5 = pool.GetIter(tx5->GetHash()).value(); - const auto entry6 = pool.GetIter(tx6->GetHash()).value(); - const auto entry7 = pool.GetIter(tx7->GetHash()).value(); - const auto entry8 = pool.GetIter(tx8->GetHash()).value(); - - BOOST_CHECK_EQUAL(entry1->GetFee(), normal_fee); - BOOST_CHECK_EQUAL(entry2->GetFee(), normal_fee); - BOOST_CHECK_EQUAL(entry3->GetFee(), low_fee); - BOOST_CHECK_EQUAL(entry4->GetFee(), high_fee); - BOOST_CHECK_EQUAL(entry5->GetFee(), low_fee); - BOOST_CHECK_EQUAL(entry6->GetFee(), low_fee); - BOOST_CHECK_EQUAL(entry7->GetFee(), high_fee); - BOOST_CHECK_EQUAL(entry8->GetFee(), high_fee); - - CTxMemPool::setEntries set_12_normal{entry1, entry2}; - CTxMemPool::setEntries set_34_cpfp{entry3, entry4}; - CTxMemPool::setEntries set_56_low{entry5, entry6}; - CTxMemPool::setEntries all_entries{entry1, entry2, entry3, entry4, entry5, entry6, entry7, entry8}; + // Normal txs, will chain txns right before CheckConflictTopology test + const auto tx9 = make_tx(/*inputs=*/ {m_coinbase_txns[5]}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx9)); + const auto tx10 = make_tx(/*inputs=*/ {m_coinbase_txns[6]}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx10)); + + // Will make these two parents of single child + const auto tx11 = make_tx(/*inputs=*/ {m_coinbase_txns[7]}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx11)); + const auto tx12 = make_tx(/*inputs=*/ {m_coinbase_txns[8]}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx12)); + + // Will make two children of this single parent + const auto tx13 = make_tx(/*inputs=*/ {m_coinbase_txns[9]}, /*output_values=*/ {995 * CENT, 995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx13)); + + const auto entry1_normal = pool.GetIter(tx1->GetHash()).value(); + const auto entry2_normal = pool.GetIter(tx2->GetHash()).value(); + const auto entry3_low = pool.GetIter(tx3->GetHash()).value(); + const auto entry4_high = pool.GetIter(tx4->GetHash()).value(); + const auto entry5_low = pool.GetIter(tx5->GetHash()).value(); + const auto entry6_low_prioritised = pool.GetIter(tx6->GetHash()).value(); + const auto entry7_high = pool.GetIter(tx7->GetHash()).value(); + const auto entry8_high = pool.GetIter(tx8->GetHash()).value(); + const auto entry9_unchained = pool.GetIter(tx9->GetHash()).value(); + const auto entry10_unchained = pool.GetIter(tx10->GetHash()).value(); + const auto entry11_unchained = pool.GetIter(tx11->GetHash()).value(); + const auto entry12_unchained = pool.GetIter(tx12->GetHash()).value(); + const auto entry13_unchained = pool.GetIter(tx13->GetHash()).value(); + + BOOST_CHECK_EQUAL(entry1_normal->GetFee(), normal_fee); + BOOST_CHECK_EQUAL(entry2_normal->GetFee(), normal_fee); + BOOST_CHECK_EQUAL(entry3_low->GetFee(), low_fee); + BOOST_CHECK_EQUAL(entry4_high->GetFee(), high_fee); + BOOST_CHECK_EQUAL(entry5_low->GetFee(), low_fee); + BOOST_CHECK_EQUAL(entry6_low_prioritised->GetFee(), low_fee); + BOOST_CHECK_EQUAL(entry7_high->GetFee(), high_fee); + BOOST_CHECK_EQUAL(entry8_high->GetFee(), high_fee); + + CTxMemPool::setEntries set_12_normal{entry1_normal, entry2_normal}; + CTxMemPool::setEntries set_34_cpfp{entry3_low, entry4_high}; + CTxMemPool::setEntries set_56_low{entry5_low, entry6_low_prioritised}; + CTxMemPool::setEntries set_78_high{entry7_high, entry8_high}; + CTxMemPool::setEntries all_entries{entry1_normal, entry2_normal, entry3_low, entry4_high, + entry5_low, entry6_low_prioritised, entry7_high, entry8_high}; CTxMemPool::setEntries empty_set; const auto unused_txid{GetRandHash()}; @@ -118,29 +201,29 @@ BOOST_FIXTURE_TEST_CASE(rbf_helper_functions, TestChain100Setup) // Tests for PaysMoreThanConflicts // These tests use feerate, not absolute fee. BOOST_CHECK(PaysMoreThanConflicts(/*iters_conflicting=*/set_12_normal, - /*replacement_feerate=*/CFeeRate(entry1->GetModifiedFee() + 1, entry1->GetTxSize() + 2), + /*replacement_feerate=*/CFeeRate(entry1_normal->GetModifiedFee() + 1, entry1_normal->GetTxSize() + 2), /*txid=*/unused_txid).has_value()); // Replacement must be strictly greater than the originals. - BOOST_CHECK(PaysMoreThanConflicts(set_12_normal, CFeeRate(entry1->GetModifiedFee(), entry1->GetTxSize()), unused_txid).has_value()); - BOOST_CHECK(PaysMoreThanConflicts(set_12_normal, CFeeRate(entry1->GetModifiedFee() + 1, entry1->GetTxSize()), unused_txid) == std::nullopt); + BOOST_CHECK(PaysMoreThanConflicts(set_12_normal, CFeeRate(entry1_normal->GetModifiedFee(), entry1_normal->GetTxSize()), unused_txid).has_value()); + BOOST_CHECK(PaysMoreThanConflicts(set_12_normal, CFeeRate(entry1_normal->GetModifiedFee() + 1, entry1_normal->GetTxSize()), unused_txid) == std::nullopt); // These tests use modified fees (including prioritisation), not base fees. - BOOST_CHECK(PaysMoreThanConflicts({entry5}, CFeeRate(entry5->GetModifiedFee() + 1, entry5->GetTxSize()), unused_txid) == std::nullopt); - BOOST_CHECK(PaysMoreThanConflicts({entry6}, CFeeRate(entry6->GetFee() + 1, entry6->GetTxSize()), unused_txid).has_value()); - BOOST_CHECK(PaysMoreThanConflicts({entry6}, CFeeRate(entry6->GetModifiedFee() + 1, entry6->GetTxSize()), unused_txid) == std::nullopt); + BOOST_CHECK(PaysMoreThanConflicts({entry5_low}, CFeeRate(entry5_low->GetModifiedFee() + 1, entry5_low->GetTxSize()), unused_txid) == std::nullopt); + BOOST_CHECK(PaysMoreThanConflicts({entry6_low_prioritised}, CFeeRate(entry6_low_prioritised->GetFee() + 1, entry6_low_prioritised->GetTxSize()), unused_txid).has_value()); + BOOST_CHECK(PaysMoreThanConflicts({entry6_low_prioritised}, CFeeRate(entry6_low_prioritised->GetModifiedFee() + 1, entry6_low_prioritised->GetTxSize()), unused_txid) == std::nullopt); // PaysMoreThanConflicts checks individual feerate, not ancestor feerate. This test compares - // replacement_feerate and entry4's feerate, which are the same. The replacement_feerate is - // considered too low even though entry4 has a low ancestor feerate. - BOOST_CHECK(PaysMoreThanConflicts(set_34_cpfp, CFeeRate(entry4->GetModifiedFee(), entry4->GetTxSize()), unused_txid).has_value()); + // replacement_feerate and entry4_high's feerate, which are the same. The replacement_feerate is + // considered too low even though entry4_high has a low ancestor feerate. + BOOST_CHECK(PaysMoreThanConflicts(set_34_cpfp, CFeeRate(entry4_high->GetModifiedFee(), entry4_high->GetTxSize()), unused_txid).has_value()); // Tests for EntriesAndTxidsDisjoint BOOST_CHECK(EntriesAndTxidsDisjoint(empty_set, {tx1->GetHash()}, unused_txid) == std::nullopt); BOOST_CHECK(EntriesAndTxidsDisjoint(set_12_normal, {tx3->GetHash()}, unused_txid) == std::nullopt); - BOOST_CHECK(EntriesAndTxidsDisjoint({entry2}, {tx2->GetHash()}, unused_txid).has_value()); + BOOST_CHECK(EntriesAndTxidsDisjoint({entry2_normal}, {tx2->GetHash()}, unused_txid).has_value()); BOOST_CHECK(EntriesAndTxidsDisjoint(set_12_normal, {tx1->GetHash()}, unused_txid).has_value()); BOOST_CHECK(EntriesAndTxidsDisjoint(set_12_normal, {tx2->GetHash()}, unused_txid).has_value()); // EntriesAndTxidsDisjoint does not calculate descendants of iters_conflicting; it uses whatever - // the caller passed in. As such, no error is returned even though entry2 is a descendant of tx1. - BOOST_CHECK(EntriesAndTxidsDisjoint({entry2}, {tx1->GetHash()}, unused_txid) == std::nullopt); + // the caller passed in. As such, no error is returned even though entry2_normal is a descendant of tx1. + BOOST_CHECK(EntriesAndTxidsDisjoint({entry2_normal}, {tx1->GetHash()}, unused_txid) == std::nullopt); // Tests for PaysForRBF const CFeeRate incremental_relay_feerate{DEFAULT_INCREMENTAL_RELAY_FEE}; @@ -163,8 +246,8 @@ BOOST_FIXTURE_TEST_CASE(rbf_helper_functions, TestChain100Setup) BOOST_CHECK(PaysForRBF(low_fee, high_fee + 99999999, 99999999, incremental_relay_feerate, unused_txid) == std::nullopt); // Tests for GetEntriesForConflicts - CTxMemPool::setEntries all_parents{entry1, entry3, entry5, entry7, entry8}; - CTxMemPool::setEntries all_children{entry2, entry4, entry6}; + CTxMemPool::setEntries all_parents{entry1_normal, entry3_low, entry5_low, entry7_high, entry8_high}; + CTxMemPool::setEntries all_children{entry2_normal, entry4_high, entry6_low_prioritised}; const std::vector<CTransactionRef> parent_inputs({m_coinbase_txns[0], m_coinbase_txns[1], m_coinbase_txns[2], m_coinbase_txns[3], m_coinbase_txns[4]}); const auto conflicts_with_parents = make_tx(parent_inputs, {50 * CENT}); @@ -215,15 +298,328 @@ BOOST_FIXTURE_TEST_CASE(rbf_helper_functions, TestChain100Setup) BOOST_CHECK(HasNoNewUnconfirmed(/*tx=*/ *spends_unconfirmed.get(), /*pool=*/ pool, /*iters_conflicting=*/ all_entries) == std::nullopt); - BOOST_CHECK(HasNoNewUnconfirmed(*spends_unconfirmed.get(), pool, {entry2}) == std::nullopt); + BOOST_CHECK(HasNoNewUnconfirmed(*spends_unconfirmed.get(), pool, {entry2_normal}) == std::nullopt); BOOST_CHECK(HasNoNewUnconfirmed(*spends_unconfirmed.get(), pool, empty_set).has_value()); const auto spends_new_unconfirmed = make_tx({tx1, tx8}, {36 * CENT}); - BOOST_CHECK(HasNoNewUnconfirmed(*spends_new_unconfirmed.get(), pool, {entry2}).has_value()); + BOOST_CHECK(HasNoNewUnconfirmed(*spends_new_unconfirmed.get(), pool, {entry2_normal}).has_value()); BOOST_CHECK(HasNoNewUnconfirmed(*spends_new_unconfirmed.get(), pool, all_entries).has_value()); const auto spends_conflicting_confirmed = make_tx({m_coinbase_txns[0], m_coinbase_txns[1]}, {45 * CENT}); - BOOST_CHECK(HasNoNewUnconfirmed(*spends_conflicting_confirmed.get(), pool, {entry1, entry3}) == std::nullopt); + BOOST_CHECK(HasNoNewUnconfirmed(*spends_conflicting_confirmed.get(), pool, {entry1_normal, entry3_low}) == std::nullopt); + + // Tests for CheckConflictTopology + + // Tx4 has 23 descendants + BOOST_CHECK_EQUAL(pool.CheckConflictTopology(set_34_cpfp).value(), strprintf("%s has 23 descendants, max 1 allowed", entry4_high->GetSharedTx()->GetHash().ToString())); + + // No descendants yet + BOOST_CHECK(pool.CheckConflictTopology({entry9_unchained}) == std::nullopt); + + // Add 1 descendant, still ok + add_descendants(tx9, 1, pool); + BOOST_CHECK(pool.CheckConflictTopology({entry9_unchained}) == std::nullopt); + + // N direct conflicts; ok + BOOST_CHECK(pool.CheckConflictTopology({entry9_unchained, entry10_unchained, entry11_unchained}) == std::nullopt); + + // Add 1 descendant, still ok, even if it's considered a direct conflict as well + const auto child_tx = add_descendants(tx10, 1, pool); + const auto entry10_child = pool.GetIter(child_tx->GetHash()).value(); + BOOST_CHECK(pool.CheckConflictTopology({entry9_unchained, entry10_unchained, entry11_unchained}) == std::nullopt); + BOOST_CHECK(pool.CheckConflictTopology({entry9_unchained, entry10_unchained, entry11_unchained, entry10_child}) == std::nullopt); + + // One more, size 3 cluster too much + const auto grand_child_tx = add_descendants(child_tx, 1, pool); + const auto entry10_grand_child = pool.GetIter(grand_child_tx->GetHash()).value(); + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry9_unchained, entry10_unchained, entry11_unchained}).value(), strprintf("%s has 2 descendants, max 1 allowed", entry10_unchained->GetSharedTx()->GetHash().ToString())); + // even if direct conflict is descendent itself + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry9_unchained, entry10_grand_child, entry11_unchained}).value(), strprintf("%s has 2 ancestors, max 1 allowed", entry10_grand_child->GetSharedTx()->GetHash().ToString())); + + // Make a single child from two singleton parents + const auto two_parent_child_tx = add_descendant_to_parents({tx11, tx12}, pool); + const auto entry_two_parent_child = pool.GetIter(two_parent_child_tx->GetHash()).value(); + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry11_unchained}).value(), strprintf("%s is not the only parent of child %s", entry11_unchained->GetSharedTx()->GetHash().ToString(), entry_two_parent_child->GetSharedTx()->GetHash().ToString())); + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry12_unchained}).value(), strprintf("%s is not the only parent of child %s", entry12_unchained->GetSharedTx()->GetHash().ToString(), entry_two_parent_child->GetSharedTx()->GetHash().ToString())); + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry_two_parent_child}).value(), strprintf("%s has 2 ancestors, max 1 allowed", entry_two_parent_child->GetSharedTx()->GetHash().ToString())); + + // Single parent with two children, we will conflict with the siblings directly only + const auto two_siblings = add_children_to_parent(tx13, pool); + const auto entry_sibling_1 = pool.GetIter(two_siblings.first->GetHash()).value(); + const auto entry_sibling_2 = pool.GetIter(two_siblings.second->GetHash()).value(); + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry_sibling_1}).value(), strprintf("%s is not the only child of parent %s", entry_sibling_1->GetSharedTx()->GetHash().ToString(), entry13_unchained->GetSharedTx()->GetHash().ToString())); + BOOST_CHECK_EQUAL(pool.CheckConflictTopology({entry_sibling_2}).value(), strprintf("%s is not the only child of parent %s", entry_sibling_2->GetSharedTx()->GetHash().ToString(), entry13_unchained->GetSharedTx()->GetHash().ToString())); + +} + +BOOST_FIXTURE_TEST_CASE(improves_feerate, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(::cs_main, pool.cs); + TestMemPoolEntryHelper entry; + + const CAmount low_fee{CENT/100}; + const CAmount normal_fee{CENT/10}; + + // low feerate parent with normal feerate child + const auto tx1 = make_tx(/*inputs=*/ {m_coinbase_txns[0]}, /*output_values=*/ {10 * COIN}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx1)); + const auto tx2 = make_tx(/*inputs=*/ {tx1}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx2)); + + const auto entry1 = pool.GetIter(tx1->GetHash()).value(); + const auto tx1_fee = entry1->GetModifiedFee(); + const auto tx1_size = entry1->GetTxSize(); + const auto entry2 = pool.GetIter(tx2->GetHash()).value(); + const auto tx2_fee = entry2->GetModifiedFee(); + const auto tx2_size = entry2->GetTxSize(); + + // Now test ImprovesFeerateDiagram with various levels of "package rbf" feerates + + // It doesn't improve itself + const auto res1 = ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee, tx1_size + tx2_size); + BOOST_CHECK(res1.has_value()); + BOOST_CHECK(res1.value().first == DiagramCheckError::FAILURE); + BOOST_CHECK(res1.value().second == "insufficient feerate: does not improve feerate diagram"); + + // With one more satoshi it does + BOOST_CHECK(ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee + 1, tx1_size + tx2_size) == std::nullopt); + + // With prioritisation of in-mempool conflicts, it affects the results of the comparison using the same args as just above + pool.PrioritiseTransaction(entry1->GetSharedTx()->GetHash(), /*nFeeDelta=*/1); + const auto res2 = ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee + 1, tx1_size + tx2_size); + BOOST_CHECK(res2.has_value()); + BOOST_CHECK(res2.value().first == DiagramCheckError::FAILURE); + BOOST_CHECK(res2.value().second == "insufficient feerate: does not improve feerate diagram"); + pool.PrioritiseTransaction(entry1->GetSharedTx()->GetHash(), /*nFeeDelta=*/-1); + + // With one less vB it does + BOOST_CHECK(ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee, tx1_size + tx2_size - 1) == std::nullopt); + + // Adding a grandchild makes the cluster size 3, which is uncalculable + const auto tx3 = make_tx(/*inputs=*/ {tx2}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx3)); + const auto res3 = ImprovesFeerateDiagram(pool, {entry1}, {entry1, entry2}, tx1_fee + tx2_fee + 1, tx1_size + tx2_size); + BOOST_CHECK(res3.has_value()); + BOOST_CHECK(res3.value().first == DiagramCheckError::UNCALCULABLE); + BOOST_CHECK(res3.value().second == strprintf("%s has 2 descendants, max 1 allowed", tx1->GetHash().GetHex())); + +} + +BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(::cs_main, pool.cs); + TestMemPoolEntryHelper entry; + + const CAmount low_fee{CENT/100}; + const CAmount normal_fee{CENT/10}; + const CAmount high_fee{CENT}; + + // low -> high -> medium fee transactions that would result in two chunks together since they + // are all same size + const auto low_tx = make_tx(/*inputs=*/ {m_coinbase_txns[0]}, /*output_values=*/ {10 * COIN}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(low_tx)); + + const auto entry_low = pool.GetIter(low_tx->GetHash()).value(); + const auto low_size = entry_low->GetTxSize(); + + // Replacement of size 1 + { + const auto replace_one{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/0, /*replacement_vsize=*/1, {entry_low}, {entry_low})}; + BOOST_CHECK(replace_one.has_value()); + std::vector<FeeFrac> expected_old_diagram{FeeFrac(0, 0), FeeFrac(low_fee, low_size)}; + BOOST_CHECK(replace_one->first == expected_old_diagram); + std::vector<FeeFrac> expected_new_diagram{FeeFrac(0, 0), FeeFrac(0, 1)}; + BOOST_CHECK(replace_one->second == expected_new_diagram); + } + + // Non-zero replacement fee/size + { + const auto replace_one_fee{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_low}, {entry_low})}; + BOOST_CHECK(replace_one_fee.has_value()); + std::vector<FeeFrac> expected_old_diagram{FeeFrac(0, 0), FeeFrac(low_fee, low_size)}; + BOOST_CHECK(replace_one_fee->first == expected_old_diagram); + std::vector<FeeFrac> expected_new_diagram{FeeFrac(0, 0), FeeFrac(high_fee, low_size)}; + BOOST_CHECK(replace_one_fee->second == expected_new_diagram); + } + + // Add a second transaction to the cluster that will make a single chunk, to be evicted in the RBF + const auto high_tx = make_tx(/*inputs=*/ {low_tx}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(high_fee).FromTx(high_tx)); + const auto entry_high = pool.GetIter(high_tx->GetHash()).value(); + const auto high_size = entry_high->GetTxSize(); + + { + const auto replace_single_chunk{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_low}, {entry_low, entry_high})}; + BOOST_CHECK(replace_single_chunk.has_value()); + std::vector<FeeFrac> expected_old_diagram{FeeFrac(0, 0), FeeFrac(low_fee + high_fee, low_size + high_size)}; + BOOST_CHECK(replace_single_chunk->first == expected_old_diagram); + std::vector<FeeFrac> expected_new_diagram{FeeFrac(0, 0), FeeFrac(high_fee, low_size)}; + BOOST_CHECK(replace_single_chunk->second == expected_new_diagram); + } + + // Conflict with the 2nd tx, resulting in new diagram with three entries + { + const auto replace_cpfp_child{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_high}, {entry_high})}; + BOOST_CHECK(replace_cpfp_child.has_value()); + std::vector<FeeFrac> expected_old_diagram{FeeFrac(0, 0), FeeFrac(low_fee + high_fee, low_size + high_size)}; + BOOST_CHECK(replace_cpfp_child->first == expected_old_diagram); + std::vector<FeeFrac> expected_new_diagram{FeeFrac(0, 0), FeeFrac(high_fee, low_size), FeeFrac(low_fee + high_fee, low_size + low_size)}; + BOOST_CHECK(replace_cpfp_child->second == expected_new_diagram); + } + + // third transaction causes the topology check to fail + const auto normal_tx = make_tx(/*inputs=*/ {high_tx}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(normal_tx)); + const auto entry_normal = pool.GetIter(normal_tx->GetHash()).value(); + const auto normal_size = entry_normal->GetTxSize(); + + { + const auto replace_too_large{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/normal_fee, /*replacement_vsize=*/normal_size, {entry_low}, {entry_low, entry_high, entry_normal})}; + BOOST_CHECK(!replace_too_large.has_value()); + BOOST_CHECK_EQUAL(util::ErrorString(replace_too_large).original, strprintf("%s has 2 descendants, max 1 allowed", low_tx->GetHash().GetHex())); + } + + // Make a size 2 cluster that is itself two chunks; evict both txns + const auto high_tx_2 = make_tx(/*inputs=*/ {m_coinbase_txns[1]}, /*output_values=*/ {10 * COIN}); + pool.addUnchecked(entry.Fee(high_fee).FromTx(high_tx_2)); + const auto entry_high_2 = pool.GetIter(high_tx_2->GetHash()).value(); + const auto high_size_2 = entry_high_2->GetTxSize(); + + const auto low_tx_2 = make_tx(/*inputs=*/ {high_tx_2}, /*output_values=*/ {9 * COIN}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(low_tx_2)); + const auto entry_low_2 = pool.GetIter(low_tx_2->GetHash()).value(); + const auto low_size_2 = entry_low_2->GetTxSize(); + + { + const auto replace_two_chunks_single_cluster{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_high_2}, {entry_high_2, entry_low_2})}; + BOOST_CHECK(replace_two_chunks_single_cluster.has_value()); + std::vector<FeeFrac> expected_old_diagram{FeeFrac(0, 0), FeeFrac(high_fee, high_size_2), FeeFrac(low_fee + high_fee, low_size_2 + high_size_2)}; + BOOST_CHECK(replace_two_chunks_single_cluster->first == expected_old_diagram); + std::vector<FeeFrac> expected_new_diagram{FeeFrac(0, 0), FeeFrac(high_fee, low_size_2)}; + BOOST_CHECK(replace_two_chunks_single_cluster->second == expected_new_diagram); + } + + // You can have more than two direct conflicts if the there are multiple affected clusters, all of size 2 or less + const auto conflict_1 = make_tx(/*inputs=*/ {m_coinbase_txns[2]}, /*output_values=*/ {10 * COIN}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(conflict_1)); + const auto conflict_1_entry = pool.GetIter(conflict_1->GetHash()).value(); + + const auto conflict_2 = make_tx(/*inputs=*/ {m_coinbase_txns[3]}, /*output_values=*/ {10 * COIN}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(conflict_2)); + const auto conflict_2_entry = pool.GetIter(conflict_2->GetHash()).value(); + + const auto conflict_3 = make_tx(/*inputs=*/ {m_coinbase_txns[4]}, /*output_values=*/ {10 * COIN}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(conflict_3)); + const auto conflict_3_entry = pool.GetIter(conflict_3->GetHash()).value(); + + { + const auto replace_multiple_clusters{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry})}; + BOOST_CHECK(replace_multiple_clusters.has_value()); + BOOST_CHECK(replace_multiple_clusters->first.size() == 4); + BOOST_CHECK(replace_multiple_clusters->second.size() == 2); + } + + // Add a child transaction to conflict_1 and make it cluster size 2, two chunks due to same feerate + const auto conflict_1_child = make_tx(/*inputs=*/{conflict_1}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(conflict_1_child)); + const auto conflict_1_child_entry = pool.GetIter(conflict_1_child->GetHash()).value(); + + { + const auto replace_multiple_clusters_2{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry, conflict_1_child_entry})}; + + BOOST_CHECK(replace_multiple_clusters_2.has_value()); + BOOST_CHECK(replace_multiple_clusters_2->first.size() == 5); + BOOST_CHECK(replace_multiple_clusters_2->second.size() == 2); + } + + // Add another descendant to conflict_1, making the cluster size > 2 should fail at this point. + const auto conflict_1_grand_child = make_tx(/*inputs=*/{conflict_1_child}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(high_fee).FromTx(conflict_1_grand_child)); + const auto conflict_1_grand_child_entry = pool.GetIter(conflict_1_child->GetHash()).value(); + + { + const auto replace_cluster_size_3{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry, conflict_1_child_entry, conflict_1_grand_child_entry})}; + + BOOST_CHECK(!replace_cluster_size_3.has_value()); + BOOST_CHECK_EQUAL(util::ErrorString(replace_cluster_size_3).original, strprintf("%s has 2 descendants, max 1 allowed", conflict_1->GetHash().GetHex())); + } +} + +BOOST_AUTO_TEST_CASE(feerate_diagram_utilities) +{ + // Sanity check the correctness of the feerate diagram comparison. + + // A strictly better case. + std::vector<FeeFrac> old_diagram{{FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}}; + std::vector<FeeFrac> new_diagram{{FeeFrac{0, 0}, FeeFrac{1000, 300}, FeeFrac{1050, 400}}}; + + BOOST_CHECK(std::is_lt(CompareFeerateDiagram(old_diagram, new_diagram))); + BOOST_CHECK(std::is_gt(CompareFeerateDiagram(new_diagram, old_diagram))); + + // Incomparable diagrams + old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; + new_diagram = {FeeFrac{0, 0}, FeeFrac{1000, 300}, FeeFrac{1000, 400}}; + + BOOST_CHECK(CompareFeerateDiagram(old_diagram, new_diagram) == std::partial_ordering::unordered); + BOOST_CHECK(CompareFeerateDiagram(new_diagram, old_diagram) == std::partial_ordering::unordered); + + // Strictly better but smaller size. + old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; + new_diagram = {FeeFrac{0, 0}, FeeFrac{1100, 300}}; + + BOOST_CHECK(std::is_lt(CompareFeerateDiagram(old_diagram, new_diagram))); + BOOST_CHECK(std::is_gt(CompareFeerateDiagram(new_diagram, old_diagram))); + + // New diagram is strictly better due to the first chunk, even though + // second chunk contributes no fees + old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; + new_diagram = {FeeFrac{0, 0}, FeeFrac{1100, 100}, FeeFrac{1100, 200}}; + + BOOST_CHECK(std::is_lt(CompareFeerateDiagram(old_diagram, new_diagram))); + BOOST_CHECK(std::is_gt(CompareFeerateDiagram(new_diagram, old_diagram))); + + // Feerate of first new chunk is better with, but second chunk is worse + old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; + new_diagram = {FeeFrac{0, 0}, FeeFrac{750, 100}, FeeFrac{999, 350}, FeeFrac{1150, 1000}}; + + BOOST_CHECK(CompareFeerateDiagram(old_diagram, new_diagram) == std::partial_ordering::unordered); + BOOST_CHECK(CompareFeerateDiagram(new_diagram, old_diagram) == std::partial_ordering::unordered); + + // If we make the second chunk slightly better, the new diagram now wins. + old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; + new_diagram = {FeeFrac{0, 0}, FeeFrac{750, 100}, FeeFrac{1000, 350}, FeeFrac{1150, 500}}; + + BOOST_CHECK(std::is_lt(CompareFeerateDiagram(old_diagram, new_diagram))); + BOOST_CHECK(std::is_gt(CompareFeerateDiagram(new_diagram, old_diagram))); + + // Identical diagrams, cannot be strictly better + old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; + new_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; + + BOOST_CHECK(std::is_eq(CompareFeerateDiagram(old_diagram, new_diagram))); + BOOST_CHECK(std::is_eq(CompareFeerateDiagram(new_diagram, old_diagram))); + + // Same aggregate fee, but different total size (trigger single tail fee check step) + old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 399}}; + new_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; + + // No change in evaluation when tail check needed. + BOOST_CHECK(std::is_gt(CompareFeerateDiagram(old_diagram, new_diagram))); + BOOST_CHECK(std::is_lt(CompareFeerateDiagram(new_diagram, old_diagram))); + + // Trigger multiple tail fee check steps + old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 399}}; + new_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}, FeeFrac{1050, 401}, FeeFrac{1050, 402}}; + + BOOST_CHECK(std::is_gt(CompareFeerateDiagram(old_diagram, new_diagram))); + BOOST_CHECK(std::is_lt(CompareFeerateDiagram(new_diagram, old_diagram))); + + // Multiple tail fee check steps, unordered result + new_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}, FeeFrac{1050, 401}, FeeFrac{1050, 402}, FeeFrac{1051, 403}}; + BOOST_CHECK(CompareFeerateDiagram(old_diagram, new_diagram) == std::partial_ordering::unordered); + BOOST_CHECK(CompareFeerateDiagram(new_diagram, old_diagram) == std::partial_ordering::unordered); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 0d2460c606..3a1cb45e8d 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -291,6 +291,7 @@ BOOST_AUTO_TEST_CASE(rpc_parse_monetary_values) BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("1e-8")), COIN/100000000); BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.1e-7")), COIN/100000000); BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.01e-6")), COIN/100000000); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.00000000000000000000000000000000000001e+30")), 1); BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.0000000000000000000000000000000000000000000000000000000000000000000000000001e+68")), COIN/100000000); BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("10000000000000000000000000000000000000000000000000000000000000000e-64")), COIN); BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000e64")), COIN); diff --git a/src/test/sanity_tests.cpp b/src/test/sanity_tests.cpp index 451bc99d44..68f82b760c 100644 --- a/src/test/sanity_tests.cpp +++ b/src/test/sanity_tests.cpp @@ -4,7 +4,6 @@ #include <key.h> #include <test/util/setup_common.h> -#include <util/time.h> #include <boost/test/unit_test.hpp> @@ -13,7 +12,6 @@ BOOST_FIXTURE_TEST_SUITE(sanity_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(basic_sanity) { BOOST_CHECK_MESSAGE(ECC_InitSanityCheck() == true, "secp256k1 sanity test"); - BOOST_CHECK_MESSAGE(ChronoSanityCheck() == true, "chrono epoch test"); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index ac457d9c77..0af2fdce08 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -2,10 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - #include <test/data/script_tests.json.h> #include <test/data/bip341_wallet_vectors.json.h> @@ -27,10 +23,6 @@ #include <util/fs.h> #include <util/strencodings.h> -#if defined(HAVE_CONSENSUS_LIB) -#include <script/bitcoinconsensus.h> -#endif - #include <cstdint> #include <fstream> #include <string> @@ -143,21 +135,6 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript if (combined_flags & SCRIPT_VERIFY_WITNESS && ~combined_flags & SCRIPT_VERIFY_P2SH) continue; BOOST_CHECK_MESSAGE(VerifyScript(scriptSig, scriptPubKey, &scriptWitness, combined_flags, MutableTransactionSignatureChecker(&tx, 0, txCredit.vout[0].nValue, MissingDataBehavior::ASSERT_FAIL), &err) == expect, message + strprintf(" (with flags %x)", combined_flags)); } - -#if defined(HAVE_CONSENSUS_LIB) - DataStream stream; - stream << TX_WITH_WITNESS(tx2); - uint32_t libconsensus_flags{flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL}; - if (libconsensus_flags == flags) { - int expectedSuccessCode = expect ? 1 : 0; - if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS) { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), txCredit.vout[0].nValue, UCharCast(stream.data()), stream.size(), 0, libconsensus_flags, nullptr) == expectedSuccessCode, message); - } else { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), 0, UCharCast(stream.data()), stream.size(), 0, libconsensus_flags, nullptr) == expectedSuccessCode, message); - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), 0, libconsensus_flags, nullptr) == expectedSuccessCode, message); - } - } -#endif } void static NegateSignatureS(std::vector<unsigned char>& vchSig) { @@ -1498,179 +1475,6 @@ static CScriptWitness ScriptWitnessFromJSON(const UniValue& univalue) return scriptwitness; } -#if defined(HAVE_CONSENSUS_LIB) - -/* Test simple (successful) usage of bitcoinconsensus_verify_script */ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_returns_true) -{ - unsigned int libconsensus_flags = 0; - int nIn = 0; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_1; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << TX_WITH_WITNESS(spendTx); - - bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 1); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_OK); -} - -/* Test bitcoinconsensus_verify_script returns invalid tx index err*/ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_index_err) -{ - unsigned int libconsensus_flags = 0; - int nIn = 3; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_EQUAL; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << TX_WITH_WITNESS(spendTx); - - bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_TX_INDEX); -} - -/* Test bitcoinconsensus_verify_script returns tx size mismatch err*/ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_size) -{ - unsigned int libconsensus_flags = 0; - int nIn = 0; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_EQUAL; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << TX_WITH_WITNESS(spendTx); - - bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size() * 2, nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_TX_SIZE_MISMATCH); -} - -/* Test bitcoinconsensus_verify_script returns invalid tx serialization error */ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_serialization) -{ - unsigned int libconsensus_flags = 0; - int nIn = 0; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_EQUAL; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << 0xffffffff; - - bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_TX_DESERIALIZE); -} - -/* Test bitcoinconsensus_verify_script returns amount required error */ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_amount_required_err) -{ - unsigned int libconsensus_flags = bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS; - int nIn = 0; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_EQUAL; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << TX_WITH_WITNESS(spendTx); - - bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_AMOUNT_REQUIRED); -} - -/* Test bitcoinconsensus_verify_script returns invalid flags err */ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_invalid_flags) -{ - unsigned int libconsensus_flags = 1 << 3; - int nIn = 0; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_EQUAL; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << TX_WITH_WITNESS(spendTx); - - bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_INVALID_FLAGS); -} - -/* Test bitcoinconsensus_verify_script returns spent outputs required err */ -BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_spent_outputs_required_err) -{ - unsigned int libconsensus_flags{bitcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT}; - const int nIn{0}; - - CScript scriptPubKey; - CScript scriptSig; - CScriptWitness wit; - - scriptPubKey << OP_EQUAL; - CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; - CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - - DataStream stream; - stream << TX_WITH_WITNESS(spendTx); - - bitcoinconsensus_error err; - int result{bitcoinconsensus_verify_script_with_spent_outputs(scriptPubKey.data(), scriptPubKey.size(), creditTx.vout[0].nValue, UCharCast(stream.data()), stream.size(), nullptr, 0, nIn, libconsensus_flags, &err)}; - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_SPENT_OUTPUTS_REQUIRED); - - result = bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), creditTx.vout[0].nValue, UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_SPENT_OUTPUTS_REQUIRED); - - result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); - BOOST_CHECK_EQUAL(result, 0); - BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_SPENT_OUTPUTS_REQUIRED); -} - -#endif // defined(HAVE_CONSENSUS_LIB) - static std::vector<unsigned int> AllConsensusFlags() { std::vector<unsigned int> ret; @@ -1718,28 +1522,12 @@ static void AssetTest(const UniValue& test) txdata.Init(tx, std::vector<CTxOut>(prevouts)); CachingTransactionSignatureChecker txcheck(&tx, idx, prevouts[idx].nValue, true, txdata); -#if defined(HAVE_CONSENSUS_LIB) - DataStream stream; - stream << TX_WITH_WITNESS(tx); - std::vector<UTXO> utxos; - utxos.resize(prevouts.size()); - for (size_t i = 0; i < prevouts.size(); i++) { - utxos[i].scriptPubKey = prevouts[i].scriptPubKey.data(); - utxos[i].scriptPubKeySize = prevouts[i].scriptPubKey.size(); - utxos[i].value = prevouts[i].nValue; - } -#endif - for (const auto flags : ALL_CONSENSUS_FLAGS) { // "final": true tests are valid for all flags. Others are only valid with flags that are // a subset of test_flags. if (fin || ((flags & test_flags) == flags)) { bool ret = VerifyScript(tx.vin[idx].scriptSig, prevouts[idx].scriptPubKey, &tx.vin[idx].scriptWitness, flags, txcheck, nullptr); BOOST_CHECK(ret); -#if defined(HAVE_CONSENSUS_LIB) - int lib_ret = bitcoinconsensus_verify_script_with_spent_outputs(prevouts[idx].scriptPubKey.data(), prevouts[idx].scriptPubKey.size(), prevouts[idx].nValue, UCharCast(stream.data()), stream.size(), utxos.data(), utxos.size(), idx, flags, nullptr); - BOOST_CHECK(lib_ret == 1); -#endif } } } @@ -1752,27 +1540,11 @@ static void AssetTest(const UniValue& test) txdata.Init(tx, std::vector<CTxOut>(prevouts)); CachingTransactionSignatureChecker txcheck(&tx, idx, prevouts[idx].nValue, true, txdata); -#if defined(HAVE_CONSENSUS_LIB) - DataStream stream; - stream << TX_WITH_WITNESS(tx); - std::vector<UTXO> utxos; - utxos.resize(prevouts.size()); - for (size_t i = 0; i < prevouts.size(); i++) { - utxos[i].scriptPubKey = prevouts[i].scriptPubKey.data(); - utxos[i].scriptPubKeySize = prevouts[i].scriptPubKey.size(); - utxos[i].value = prevouts[i].nValue; - } -#endif - for (const auto flags : ALL_CONSENSUS_FLAGS) { // If a test is supposed to fail with test_flags, it should also fail with any superset thereof. if ((flags & test_flags) == test_flags) { bool ret = VerifyScript(tx.vin[idx].scriptSig, prevouts[idx].scriptPubKey, &tx.vin[idx].scriptWitness, flags, txcheck, nullptr); BOOST_CHECK(!ret); -#if defined(HAVE_CONSENSUS_LIB) - int lib_ret = bitcoinconsensus_verify_script_with_spent_outputs(prevouts[idx].scriptPubKey.data(), prevouts[idx].scriptPubKey.size(), prevouts[idx].nValue, UCharCast(stream.data()), stream.size(), utxos.data(), utxos.size(), idx, flags, nullptr); - BOOST_CHECK(lib_ret == 0); -#endif } } } diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index 90fce9adf9..8aab2b565c 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -11,7 +11,7 @@ #include <univalue.h> #ifdef ENABLE_EXTERNAL_SIGNER -#include <boost/process.hpp> +#include <util/subprocess.hpp> #endif // ENABLE_EXTERNAL_SIGNER #include <boost/test/unit_test.hpp> @@ -34,20 +34,16 @@ BOOST_AUTO_TEST_CASE(run_command) BOOST_CHECK(result.isNull()); } { - const UniValue result = RunCommandParseJSON("echo \"{\"success\": true}\""); + const UniValue result = RunCommandParseJSON("echo {\"success\": true}"); BOOST_CHECK(result.isObject()); const UniValue& success = result.find_value("success"); BOOST_CHECK(!success.isNull()); BOOST_CHECK_EQUAL(success.get_bool(), true); } { - // An invalid command is handled by Boost - const int expected_error{2}; - BOOST_CHECK_EXCEPTION(RunCommandParseJSON("invalid_command"), boost::process::process_error, [&](const boost::process::process_error& e) { - BOOST_CHECK(std::string(e.what()).find("RunCommandParseJSON error:") == std::string::npos); - BOOST_CHECK_EQUAL(e.code().value(), expected_error); - return true; - }); + // An invalid command is handled by cpp-subprocess + const std::string expected{"execve failed: "}; + BOOST_CHECK_EXCEPTION(RunCommandParseJSON("invalid_command"), subprocess::CalledProcessError, HasReason(expected)); } { // Return non-zero exit code, no output to stderr diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index eb131dc6bb..b948ea8acb 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -132,7 +132,7 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) /*output_amount=*/CAmount(48 * COIN), /*submit=*/false); CTransactionRef tx_child = MakeTransactionRef(mtx_child); Package package_parent_child{tx_parent, tx_child}; - const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_parent_child, /*test_accept=*/true, /*max_sane_feerate=*/{}); + const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_parent_child, /*test_accept=*/true, /*client_maxfeerate=*/{}); if (auto err_parent_child{CheckPackageMempoolAcceptResult(package_parent_child, result_parent_child, /*expect_valid=*/true, nullptr)}) { BOOST_ERROR(err_parent_child.value()); } else { @@ -151,7 +151,7 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) CTransactionRef giant_ptx = create_placeholder_tx(999, 999); BOOST_CHECK(GetVirtualTransactionSize(*giant_ptx) > DEFAULT_ANCESTOR_SIZE_LIMIT_KVB * 1000); Package package_single_giant{giant_ptx}; - auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_single_giant, /*test_accept=*/true, /*max_sane_feerate=*/{}); + auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_single_giant, /*test_accept=*/true, /*client_maxfeerate=*/{}); if (auto err_single_large{CheckPackageMempoolAcceptResult(package_single_giant, result_single_large, /*expect_valid=*/false, nullptr)}) { BOOST_ERROR(err_single_large.value()); } else { @@ -275,7 +275,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) package_unrelated.emplace_back(MakeTransactionRef(mtx)); } auto result_unrelated_submit = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_unrelated, /*test_accept=*/false, /*max_sane_feerate=*/{}); + package_unrelated, /*test_accept=*/false, /*client_maxfeerate=*/{}); // We don't expect m_tx_results for each transaction when basic sanity checks haven't passed. BOOST_CHECK(result_unrelated_submit.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); @@ -315,7 +315,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) // 3 Generations is not allowed. { auto result_3gen_submit = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_3gen, /*test_accept=*/false, /*max_sane_feerate=*/{}); + package_3gen, /*test_accept=*/false, /*client_maxfeerate=*/{}); BOOST_CHECK(result_3gen_submit.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_3gen_submit.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_3gen_submit.m_state.GetRejectReason(), "package-not-child-with-parents"); @@ -332,7 +332,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) CTransactionRef tx_parent_invalid = MakeTransactionRef(mtx_parent_invalid); Package package_invalid_parent{tx_parent_invalid, tx_child}; auto result_quit_early = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_invalid_parent, /*test_accept=*/ false, /*max_sane_feerate=*/{}); + package_invalid_parent, /*test_accept=*/ false, /*client_maxfeerate=*/{}); if (auto err_parent_invalid{CheckPackageMempoolAcceptResult(package_invalid_parent, result_quit_early, /*expect_valid=*/false, m_node.mempool.get())}) { BOOST_ERROR(err_parent_invalid.value()); } else { @@ -353,7 +353,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) package_missing_parent.push_back(MakeTransactionRef(mtx_child)); { const auto result_missing_parent = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_missing_parent, /*test_accept=*/false, /*max_sane_feerate=*/{}); + package_missing_parent, /*test_accept=*/false, /*client_maxfeerate=*/{}); BOOST_CHECK(result_missing_parent.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_missing_parent.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_missing_parent.m_state.GetRejectReason(), "package-not-child-with-unconfirmed-parents"); @@ -363,7 +363,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) // Submit package with parent + child. { const auto submit_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child, /*test_accept=*/false, /*max_sane_feerate=*/{}); + package_parent_child, /*test_accept=*/false, /*client_maxfeerate=*/{}); expected_pool_size += 2; BOOST_CHECK_MESSAGE(submit_parent_child.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_parent_child.m_state.GetRejectReason()); @@ -385,7 +385,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) // Already-in-mempool transactions should be detected and de-duplicated. { const auto submit_deduped = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child, /*test_accept=*/false, /*max_sane_feerate=*/{}); + package_parent_child, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_deduped{CheckPackageMempoolAcceptResult(package_parent_child, submit_deduped, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_deduped.value()); } else { @@ -456,7 +456,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) { Package package_parent_child1{ptx_parent, ptx_child1}; const auto submit_witness1 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child1, /*test_accept=*/false, /*max_sane_feerate=*/{}); + package_parent_child1, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_witness1{CheckPackageMempoolAcceptResult(package_parent_child1, submit_witness1, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_witness1.value()); } @@ -464,7 +464,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // Child2 would have been validated individually. Package package_parent_child2{ptx_parent, ptx_child2}; const auto submit_witness2 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child2, /*test_accept=*/false, /*max_sane_feerate=*/{}); + package_parent_child2, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_witness2{CheckPackageMempoolAcceptResult(package_parent_child2, submit_witness2, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_witness2.value()); } else { @@ -478,7 +478,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // Deduplication should work when wtxid != txid. Submit package with the already-in-mempool // transactions again, which should not fail. const auto submit_segwit_dedup = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child1, /*test_accept=*/false, /*max_sane_feerate=*/{}); + package_parent_child1, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_segwit_dedup{CheckPackageMempoolAcceptResult(package_parent_child1, submit_segwit_dedup, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_segwit_dedup.value()); } else { @@ -508,7 +508,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) { Package package_child2_grandchild{ptx_child2, ptx_grandchild}; const auto submit_spend_ignored = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_child2_grandchild, /*test_accept=*/false, /*max_sane_feerate=*/{}); + package_child2_grandchild, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_spend_ignored{CheckPackageMempoolAcceptResult(package_child2_grandchild, submit_spend_ignored, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_spend_ignored.value()); } else { @@ -606,7 +606,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // parent3 should be accepted // child should be accepted { - const auto mixed_result = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_mixed, false, /*max_sane_feerate=*/{}); + const auto mixed_result = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_mixed, false, /*client_maxfeerate=*/{}); if (auto err_mixed{CheckPackageMempoolAcceptResult(package_mixed, mixed_result, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_mixed.value()); } else { @@ -670,7 +670,7 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) { BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_cpfp_deprio = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_cpfp, /*test_accept=*/ false, /*max_sane_feerate=*/{}); + package_cpfp, /*test_accept=*/ false, /*client_maxfeerate=*/{}); if (auto err_cpfp_deprio{CheckPackageMempoolAcceptResult(package_cpfp, submit_cpfp_deprio, /*expect_valid=*/false, m_node.mempool.get())}) { BOOST_ERROR(err_cpfp_deprio.value()); } else { @@ -692,7 +692,7 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) { BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_cpfp = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_cpfp, /*test_accept=*/ false, /*max_sane_feerate=*/{}); + package_cpfp, /*test_accept=*/ false, /*client_maxfeerate=*/{}); if (auto err_cpfp{CheckPackageMempoolAcceptResult(package_cpfp, submit_cpfp, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_cpfp.value()); } else { @@ -744,7 +744,7 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) // Cheap package should fail for being too low fee. { const auto submit_package_too_low = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_still_too_low, /*test_accept=*/false, /*max_sane_feerate=*/{}); + package_still_too_low, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_package_too_low{CheckPackageMempoolAcceptResult(package_still_too_low, submit_package_too_low, /*expect_valid=*/false, m_node.mempool.get())}) { BOOST_ERROR(err_package_too_low.value()); } else { @@ -770,7 +770,7 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) // Now that the child's fees have "increased" by 1 BTC, the cheap package should succeed. { const auto submit_prioritised_package = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_still_too_low, /*test_accept=*/false, /*max_sane_feerate=*/{}); + package_still_too_low, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_prioritised{CheckPackageMempoolAcceptResult(package_still_too_low, submit_prioritised_package, /*expect_valid=*/true, m_node.mempool.get())}) { BOOST_ERROR(err_prioritised.value()); } else { @@ -818,7 +818,7 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) { BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_rich_parent = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_rich_parent, /*test_accept=*/false, /*max_sane_feerate=*/{}); + package_rich_parent, /*test_accept=*/false, /*client_maxfeerate=*/{}); if (auto err_rich_parent{CheckPackageMempoolAcceptResult(package_rich_parent, submit_rich_parent, /*expect_valid=*/false, m_node.mempool.get())}) { BOOST_ERROR(err_rich_parent.value()); } else { diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 47808a2a58..9a2add748e 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -290,13 +290,18 @@ BOOST_AUTO_TEST_CASE(util_TrimString) BOOST_AUTO_TEST_CASE(util_FormatISO8601DateTime) { + BOOST_CHECK_EQUAL(FormatISO8601DateTime(971890963199), "32767-12-31T23:59:59Z"); + BOOST_CHECK_EQUAL(FormatISO8601DateTime(971890876800), "32767-12-31T00:00:00Z"); BOOST_CHECK_EQUAL(FormatISO8601DateTime(1317425777), "2011-09-30T23:36:17Z"); BOOST_CHECK_EQUAL(FormatISO8601DateTime(0), "1970-01-01T00:00:00Z"); } BOOST_AUTO_TEST_CASE(util_FormatISO8601Date) { + BOOST_CHECK_EQUAL(FormatISO8601Date(971890963199), "32767-12-31"); + BOOST_CHECK_EQUAL(FormatISO8601Date(971890876800), "32767-12-31"); BOOST_CHECK_EQUAL(FormatISO8601Date(1317425777), "2011-09-30"); + BOOST_CHECK_EQUAL(FormatISO8601Date(0), "1970-01-01"); } BOOST_AUTO_TEST_CASE(util_FormatMoney) diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 316ab86c2b..69f4e305ab 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -127,6 +127,7 @@ std::shared_ptr<const CBlock> MinerTestingSetup::BadBlock(const uint256& prev_ha return ret; } +// NOLINTNEXTLINE(misc-no-recursion) void MinerTestingSetup::BuildChain(const uint256& root, int height, const unsigned int invalid_rate, const unsigned int branch_rate, const unsigned int max_size, std::vector<std::shared_ptr<const CBlock>>& blocks) { if (height <= 0 || blocks.size() >= max_size) return; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 0bee27c2b2..82eec6241f 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -17,6 +17,7 @@ #include <random.h> #include <reverse_iterator.h> #include <util/check.h> +#include <util/feefrac.h> #include <util/moneystr.h> #include <util/overflow.h> #include <util/result.h> @@ -1238,3 +1239,132 @@ std::vector<CTxMemPool::txiter> CTxMemPool::GatherClusters(const std::vector<uin } return clustered_txs; } + +std::optional<std::string> CTxMemPool::CheckConflictTopology(const setEntries& direct_conflicts) +{ + for (const auto& direct_conflict : direct_conflicts) { + // Ancestor and descendant counts are inclusive of the tx itself. + const auto ancestor_count{direct_conflict->GetCountWithAncestors()}; + const auto descendant_count{direct_conflict->GetCountWithDescendants()}; + const bool has_ancestor{ancestor_count > 1}; + const bool has_descendant{descendant_count > 1}; + const auto& txid_string{direct_conflict->GetSharedTx()->GetHash().ToString()}; + // The only allowed configurations are: + // 1 ancestor and 0 descendant + // 0 ancestor and 1 descendant + // 0 ancestor and 0 descendant + if (ancestor_count > 2) { + return strprintf("%s has %u ancestors, max 1 allowed", txid_string, ancestor_count - 1); + } else if (descendant_count > 2) { + return strprintf("%s has %u descendants, max 1 allowed", txid_string, descendant_count - 1); + } else if (has_ancestor && has_descendant) { + return strprintf("%s has both ancestor and descendant, exceeding cluster limit of 2", txid_string); + } + // Additionally enforce that: + // If we have a child, we are its only parent. + // If we have a parent, we are its only child. + if (has_descendant) { + const auto& our_child = direct_conflict->GetMemPoolChildrenConst().begin(); + if (our_child->get().GetCountWithAncestors() > 2) { + return strprintf("%s is not the only parent of child %s", + txid_string, our_child->get().GetSharedTx()->GetHash().ToString()); + } + } else if (has_ancestor) { + const auto& our_parent = direct_conflict->GetMemPoolParentsConst().begin(); + if (our_parent->get().GetCountWithDescendants() > 2) { + return strprintf("%s is not the only child of parent %s", + txid_string, our_parent->get().GetSharedTx()->GetHash().ToString()); + } + } + } + return std::nullopt; +} + +util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CTxMemPool::CalculateFeerateDiagramsForRBF(CAmount replacement_fees, int64_t replacement_vsize, const setEntries& direct_conflicts, const setEntries& all_conflicts) +{ + Assume(replacement_vsize > 0); + + auto err_string{CheckConflictTopology(direct_conflicts)}; + if (err_string.has_value()) { + // Unsupported topology for calculating a feerate diagram + return util::Error{Untranslated(err_string.value())}; + } + + // new diagram will have chunks that consist of each ancestor of + // direct_conflicts that is at its own fee/size, along with the replacement + // tx/package at its own fee/size + + // old diagram will consist of the ancestors and descendants of each element of + // all_conflicts. every such transaction will either be at its own feerate (followed + // by any descendant at its own feerate), or as a single chunk at the descendant's + // ancestor feerate. + + std::vector<FeeFrac> old_chunks; + // Step 1: build the old diagram. + + // The above clusters are all trivially linearized; + // they have a strict topology of 1 or two connected transactions. + + // OLD: Compute existing chunks from all affected clusters + for (auto txiter : all_conflicts) { + // Does this transaction have descendants? + if (txiter->GetCountWithDescendants() > 1) { + // Consider this tx when we consider the descendant. + continue; + } + // Does this transaction have ancestors? + FeeFrac individual{txiter->GetModifiedFee(), txiter->GetTxSize()}; + if (txiter->GetCountWithAncestors() > 1) { + // We'll add chunks for either the ancestor by itself and this tx + // by itself, or for a combined package. + FeeFrac package{txiter->GetModFeesWithAncestors(), static_cast<int32_t>(txiter->GetSizeWithAncestors())}; + if (individual >> package) { + // The individual feerate is higher than the package, and + // therefore higher than the parent's fee. Chunk these + // together. + old_chunks.emplace_back(package); + } else { + // Add two points, one for the parent and one for this child. + old_chunks.emplace_back(package - individual); + old_chunks.emplace_back(individual); + } + } else { + old_chunks.emplace_back(individual); + } + } + + // No topology restrictions post-chunking; sort + std::sort(old_chunks.begin(), old_chunks.end(), std::greater()); + std::vector<FeeFrac> old_diagram = BuildDiagramFromChunks(old_chunks); + + std::vector<FeeFrac> new_chunks; + + /* Step 2: build the NEW diagram + * CON = Conflicts of proposed chunk + * CNK = Proposed chunk + * NEW = OLD - CON + CNK: New diagram includes all chunks in OLD, minus + * the conflicts, plus the proposed chunk + */ + + // OLD - CON: Add any parents of direct conflicts that are not conflicted themselves + for (auto direct_conflict : direct_conflicts) { + // If a direct conflict has an ancestor that is not in all_conflicts, + // it can be affected by the replacement of the child. + if (direct_conflict->GetMemPoolParentsConst().size() > 0) { + // Grab the parent. + const CTxMemPoolEntry& parent = direct_conflict->GetMemPoolParentsConst().begin()->get(); + if (!all_conflicts.count(mapTx.iterator_to(parent))) { + // This transaction would be left over, so add to the NEW + // diagram. + new_chunks.emplace_back(parent.GetModifiedFee(), parent.GetTxSize()); + } + } + } + // + CNK: Add the proposed chunk itself + new_chunks.emplace_back(replacement_fees, int32_t(replacement_vsize)); + + // No topology restrictions post-chunking; sort + std::sort(new_chunks.begin(), new_chunks.end(), std::greater()); + std::vector<FeeFrac> new_diagram = BuildDiagramFromChunks(new_chunks); + return std::make_pair(old_diagram, new_diagram); +} diff --git a/src/txmempool.h b/src/txmempool.h index 32f2c024f7..9dd4d56da7 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -21,6 +21,7 @@ #include <util/epochguard.h> #include <util/hasher.h> #include <util/result.h> +#include <util/feefrac.h> #include <boost/multi_index/hashed_index.hpp> #include <boost/multi_index/identity.hpp> @@ -736,6 +737,28 @@ public: return m_sequence_number; } + /** + * Calculate the old and new mempool feerate diagrams relating to the + * clusters that would be affected by a potential replacement transaction. + * (replacement_fees, replacement_vsize) values are gathered from a + * proposed set of replacement transactions that are considered as a single + * chunk, and represent their complete cluster. In other words, they have no + * in-mempool ancestors. + * + * @param[in] replacement_fees Package fees + * @param[in] replacement_vsize Package size (must be greater than 0) + * @param[in] direct_conflicts All transactions that would be removed directly by + * having a conflicting input with a proposed transaction + * @param[in] all_conflicts All transactions that would be removed + * @return old and new diagram pair respectively, or an error string if the conflicts don't match a calculable topology + */ + util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CalculateFeerateDiagramsForRBF(CAmount replacement_fees, int64_t replacement_vsize, const setEntries& direct_conflicts, const setEntries& all_conflicts) EXCLUSIVE_LOCKS_REQUIRED(cs); + + /* Check that all direct conflicts are in a cluster size of two or less. Each + * direct conflict may be in a separate cluster. + */ + std::optional<std::string> CheckConflictTopology(const setEntries& direct_conflicts); + private: /** UpdateForDescendants is used by UpdateTransactionsFromBlock to update * the descendants for a single transaction that has been added to the diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index acbce25741..da12157555 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -18,6 +18,7 @@ #include <utility> #include <vector> +// NOLINTNEXTLINE(misc-no-recursion) class UniValue { public: enum VType { VNULL, VOBJ, VARR, VSTR, VNUM, VBOOL, }; diff --git a/src/univalue/lib/univalue_write.cpp b/src/univalue/lib/univalue_write.cpp index 4a3cbba20f..4a2219061a 100644 --- a/src/univalue/lib/univalue_write.cpp +++ b/src/univalue/lib/univalue_write.cpp @@ -27,6 +27,7 @@ static std::string json_escape(const std::string& inS) return outS; } +// NOLINTNEXTLINE(misc-no-recursion) std::string UniValue::write(unsigned int prettyIndent, unsigned int indentLevel) const { @@ -66,6 +67,7 @@ static void indentStr(unsigned int prettyIndent, unsigned int indentLevel, std:: s.append(prettyIndent * indentLevel, ' '); } +// NOLINTNEXTLINE(misc-no-recursion) void UniValue::writeArray(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const { s += "["; @@ -88,6 +90,7 @@ void UniValue::writeArray(unsigned int prettyIndent, unsigned int indentLevel, s s += "]"; } +// NOLINTNEXTLINE(misc-no-recursion) void UniValue::writeObject(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const { s += "{"; diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index 8b90448b36..1c724555f3 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -421,7 +421,7 @@ void univalue_readwrite() // Valid, with leading or trailing whitespace BOOST_CHECK(v.read(" 1.0") && (v.get_real() == 1.0)); BOOST_CHECK(v.read("1.0 ") && (v.get_real() == 1.0)); - BOOST_CHECK(v.read("0.00000000000000000000000000000000000001e+30 ") && v.get_real() == 1e-8); + BOOST_CHECK(v.read("0.00000000000000000000000000000000000001e+30 ")); BOOST_CHECK(!v.read(".19e-6")); //should fail, missing leading 0, therefore invalid JSON // Invalid, initial garbage diff --git a/src/util/feefrac.cpp b/src/util/feefrac.cpp new file mode 100644 index 0000000000..68fb836936 --- /dev/null +++ b/src/util/feefrac.cpp @@ -0,0 +1,87 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/feefrac.h> +#include <algorithm> +#include <array> +#include <vector> + +std::vector<FeeFrac> BuildDiagramFromChunks(const Span<const FeeFrac> chunks) +{ + std::vector<FeeFrac> diagram; + diagram.reserve(chunks.size() + 1); + + diagram.emplace_back(0, 0); + for (auto& chunk : chunks) { + diagram.emplace_back(diagram.back() + chunk); + } + return diagram; +} + +std::partial_ordering CompareFeerateDiagram(Span<const FeeFrac> dia0, Span<const FeeFrac> dia1) +{ + /** Array to allow indexed access to input diagrams. */ + const std::array<Span<const FeeFrac>, 2> dias = {dia0, dia1}; + /** How many elements we have processed in each input. */ + size_t next_index[2] = {1, 1}; + /** Whether the corresponding input is strictly better than the other at least in one place. */ + bool better_somewhere[2] = {false, false}; + /** Get the first unprocessed point in diagram number dia. */ + const auto next_point = [&](int dia) { return dias[dia][next_index[dia]]; }; + /** Get the last processed point in diagram number dia. */ + const auto prev_point = [&](int dia) { return dias[dia][next_index[dia] - 1]; }; + + // Diagrams should be non-empty, and first elements zero in size and fee + Assert(!dia0.empty() && !dia1.empty()); + Assert(prev_point(0).IsEmpty()); + Assert(prev_point(1).IsEmpty()); + + do { + bool done_0 = next_index[0] == dias[0].size(); + bool done_1 = next_index[1] == dias[1].size(); + if (done_0 && done_1) break; + + // Determine which diagram has the first unprocessed point. If a single side is finished, use the + // other one. Only up to one can be done due to check above. + const int unproc_side = (done_0 || done_1) ? done_0 : next_point(0).size > next_point(1).size; + + // Let `P` be the next point on diagram unproc_side, and `A` and `B` the previous and next points + // on the other diagram. We want to know if P lies above or below the line AB. To determine this, we + // compute the slopes of line AB and of line AP, and compare them. These slopes are fee per size, + // and can thus be expressed as FeeFracs. + const FeeFrac& point_p = next_point(unproc_side); + const FeeFrac& point_a = prev_point(!unproc_side); + + const auto slope_ap = point_p - point_a; + Assume(slope_ap.size > 0); + std::weak_ordering cmp = std::weak_ordering::equivalent; + if (done_0 || done_1) { + // If a single side has no points left, act as if AB has slope tail_feerate(of 0). + Assume(!(done_0 && done_1)); + cmp = FeeRateCompare(slope_ap, FeeFrac(0, 1)); + } else { + // If both sides have points left, compute B, and the slope of AB explicitly. + const FeeFrac& point_b = next_point(!unproc_side); + const auto slope_ab = point_b - point_a; + Assume(slope_ab.size >= slope_ap.size); + cmp = FeeRateCompare(slope_ap, slope_ab); + + // If B and P have the same size, B can be marked as processed (in addition to P, see + // below), as we've already performed a comparison at this size. + if (point_b.size == point_p.size) ++next_index[!unproc_side]; + } + // If P lies above AB, unproc_side is better in P. If P lies below AB, then !unproc_side is + // better in P. + if (std::is_gt(cmp)) better_somewhere[unproc_side] = true; + if (std::is_lt(cmp)) better_somewhere[!unproc_side] = true; + ++next_index[unproc_side]; + + // If both diagrams are better somewhere, they are incomparable. + if (better_somewhere[0] && better_somewhere[1]) return std::partial_ordering::unordered; + + } while(true); + + // Otherwise compare the better_somewhere values. + return better_somewhere[0] <=> better_somewhere[1]; +} diff --git a/src/util/feefrac.h b/src/util/feefrac.h new file mode 100644 index 0000000000..7102f85f88 --- /dev/null +++ b/src/util/feefrac.h @@ -0,0 +1,160 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_FEEFRAC_H +#define BITCOIN_UTIL_FEEFRAC_H + +#include <stdint.h> +#include <compare> +#include <vector> +#include <span.h> +#include <util/check.h> + +/** Data structure storing a fee and size, ordered by increasing fee/size. + * + * The size of a FeeFrac cannot be zero unless the fee is also zero. + * + * FeeFracs have a total ordering, first by increasing feerate (ratio of fee over size), and then + * by decreasing size. The empty FeeFrac (fee and size both 0) sorts last. So for example, the + * following FeeFracs are in sorted order: + * + * - fee=0 size=1 (feerate 0) + * - fee=1 size=2 (feerate 0.5) + * - fee=2 size=3 (feerate 0.667...) + * - fee=2 size=2 (feerate 1) + * - fee=1 size=1 (feerate 1) + * - fee=3 size=2 (feerate 1.5) + * - fee=2 size=1 (feerate 2) + * - fee=0 size=0 (undefined feerate) + * + * A FeeFrac is considered "better" if it sorts after another, by this ordering. All standard + * comparison operators (<=>, ==, !=, >, <, >=, <=) respect this ordering. + * + * The FeeRateCompare, and >> and << operators only compare feerate and treat equal feerate but + * different size as equivalent. The empty FeeFrac is neither lower or higher in feerate than any + * other. + */ +struct FeeFrac +{ + /** Fallback version for Mul (see below). + * + * Separate to permit testing on platforms where it isn't actually needed. + */ + static inline std::pair<int64_t, uint32_t> MulFallback(int64_t a, int32_t b) noexcept + { + // Otherwise, emulate 96-bit multiplication using two 64-bit multiplies. + int64_t low = int64_t{static_cast<uint32_t>(a)} * b; + int64_t high = (a >> 32) * b; + return {high + (low >> 32), static_cast<uint32_t>(low)}; + } + + // Compute a * b, returning an unspecified but totally ordered type. +#ifdef __SIZEOF_INT128__ + static inline __int128 Mul(int64_t a, int32_t b) noexcept + { + // If __int128 is available, use 128-bit wide multiply. + return __int128{a} * b; + } +#else + static constexpr auto Mul = MulFallback; +#endif + + int64_t fee; + int32_t size; + + /** Construct an IsEmpty() FeeFrac. */ + inline FeeFrac() noexcept : fee{0}, size{0} {} + + /** Construct a FeeFrac with specified fee and size. */ + inline FeeFrac(int64_t f, int32_t s) noexcept : fee{f}, size{s} {} + + inline FeeFrac(const FeeFrac&) noexcept = default; + inline FeeFrac& operator=(const FeeFrac&) noexcept = default; + + /** Check if this is empty (size and fee are 0). */ + bool inline IsEmpty() const noexcept { + return size == 0; + } + + /** Add fee and size of another FeeFrac to this one. */ + void inline operator+=(const FeeFrac& other) noexcept + { + fee += other.fee; + size += other.size; + } + + /** Subtract fee and size of another FeeFrac from this one. */ + void inline operator-=(const FeeFrac& other) noexcept + { + fee -= other.fee; + size -= other.size; + } + + /** Sum fee and size. */ + friend inline FeeFrac operator+(const FeeFrac& a, const FeeFrac& b) noexcept + { + return {a.fee + b.fee, a.size + b.size}; + } + + /** Subtract both fee and size. */ + friend inline FeeFrac operator-(const FeeFrac& a, const FeeFrac& b) noexcept + { + return {a.fee - b.fee, a.size - b.size}; + } + + /** Check if two FeeFrac objects are equal (both same fee and same size). */ + friend inline bool operator==(const FeeFrac& a, const FeeFrac& b) noexcept + { + return a.fee == b.fee && a.size == b.size; + } + + /** Compare two FeeFracs just by feerate. */ + friend inline std::weak_ordering FeeRateCompare(const FeeFrac& a, const FeeFrac& b) noexcept + { + auto cross_a = Mul(a.fee, b.size), cross_b = Mul(b.fee, a.size); + return cross_a <=> cross_b; + } + + /** Check if a FeeFrac object has strictly lower feerate than another. */ + friend inline bool operator<<(const FeeFrac& a, const FeeFrac& b) noexcept + { + auto cross_a = Mul(a.fee, b.size), cross_b = Mul(b.fee, a.size); + return cross_a < cross_b; + } + + /** Check if a FeeFrac object has strictly higher feerate than another. */ + friend inline bool operator>>(const FeeFrac& a, const FeeFrac& b) noexcept + { + auto cross_a = Mul(a.fee, b.size), cross_b = Mul(b.fee, a.size); + return cross_a > cross_b; + } + + /** Compare two FeeFracs. <, >, <=, and >= are auto-generated from this. */ + friend inline std::strong_ordering operator<=>(const FeeFrac& a, const FeeFrac& b) noexcept + { + auto cross_a = Mul(a.fee, b.size), cross_b = Mul(b.fee, a.size); + if (cross_a == cross_b) return b.size <=> a.size; + return cross_a <=> cross_b; + } + + /** Swap two FeeFracs. */ + friend inline void swap(FeeFrac& a, FeeFrac& b) noexcept + { + std::swap(a.fee, b.fee); + std::swap(a.size, b.size); + } +}; + +/** Takes the pre-computed and topologically-valid chunks and generates a fee diagram which starts at FeeFrac of (0, 0) */ +std::vector<FeeFrac> BuildDiagramFromChunks(Span<const FeeFrac> chunks); + +/** Compares two feerate diagrams. The shorter one is implicitly + * extended with a horizontal straight line. + * + * A feerate diagram consists of a list of (fee, size) points with the property that size + * is strictly increasing and that the first entry is (0, 0). + */ +std::partial_ordering CompareFeerateDiagram(Span<const FeeFrac> dia0, Span<const FeeFrac> dia1); + +#endif // BITCOIN_UTIL_FEEFRAC_H diff --git a/src/util/string.h b/src/util/string.h index 8b69d6ccae..dab92942fb 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -65,6 +65,7 @@ void ReplaceAll(std::string& in_out, const std::string& search, const std::strin * @param unary_op Apply this operator to each item */ template <typename C, typename S, typename UnaryOp> +// NOLINTNEXTLINE(misc-no-recursion) auto Join(const C& container, const S& separator, UnaryOp unary_op) { decltype(unary_op(*container.begin())) ret; diff --git a/src/util/subprocess.hpp b/src/util/subprocess.hpp new file mode 100644 index 0000000000..e660aa143d --- /dev/null +++ b/src/util/subprocess.hpp @@ -0,0 +1,1988 @@ +// Based on the https://github.com/arun11299/cpp-subprocess project. + +/*! + +Documentation for C++ subprocessing library. + +@copyright The code is licensed under the [MIT + License](http://opensource.org/licenses/MIT): + <br> + Copyright © 2016-2018 Arun Muralidharan. + <br> + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + <br> + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + <br> + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +@author [Arun Muralidharan] +@see https://github.com/arun11299/cpp-subprocess to download the source code + +@version 1.0.0 +*/ + +#ifndef SUBPROCESS_HPP +#define SUBPROCESS_HPP + +#include <algorithm> +#include <cassert> +#include <csignal> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <exception> +#include <future> +#include <initializer_list> +#include <iostream> +#include <locale> +#include <map> +#include <memory> +#include <sstream> +#include <string> +#include <vector> + +#if (defined _MSC_VER) || (defined __MINGW32__) + #define __USING_WINDOWS__ +#endif + +#ifdef __USING_WINDOWS__ + #include <codecvt> +#endif + +extern "C" { +#ifdef __USING_WINDOWS__ + #include <Windows.h> + #include <io.h> + #include <cwchar> + + #define close _close + #define open _open + #define fileno _fileno +#else + #include <sys/wait.h> + #include <unistd.h> +#endif + #include <csignal> + #include <fcntl.h> + #include <sys/types.h> +} + +/*! + * Getting started with reading this source code. + * The source is mainly divided into four parts: + * 1. Exception Classes: + * These are very basic exception classes derived from + * runtime_error exception. + * There are two types of exception thrown from subprocess + * library: OSError and CalledProcessError + * + * 2. Popen Class + * This is the main class the users will deal with. It + * provides with all the API's to deal with processes. + * + * 3. Util namespace + * It includes some helper functions to split/join a string, + * reading from file descriptors, waiting on a process, fcntl + * options on file descriptors etc. + * + * 4. Detail namespace + * This includes some metaprogramming and helper classes. + */ + + +namespace subprocess { + +// Max buffer size allocated on stack for read error +// from pipe +static const size_t SP_MAX_ERR_BUF_SIZ = 1024; + +// Default buffer capacity for OutBuffer and ErrBuffer. +// If the data exceeds this capacity, the buffer size is grown +// by 1.5 times its previous capacity +static const size_t DEFAULT_BUF_CAP_BYTES = 8192; + + +/*----------------------------------------------- + * EXCEPTION CLASSES + *----------------------------------------------- + */ + +/*! + * class: CalledProcessError + * Thrown when there was error executing the command. + * Check Popen class API's to know when this exception + * can be thrown. + * + */ +class CalledProcessError: public std::runtime_error +{ +public: + int retcode; + CalledProcessError(const std::string& error_msg, int retcode): + std::runtime_error(error_msg), retcode(retcode) + {} +}; + + +/*! + * class: OSError + * Thrown when some system call fails to execute or give result. + * The exception message contains the name of the failed system call + * with the stringisized errno code. + * Check Popen class API's to know when this exception would be + * thrown. + * Its usual that the API exception specification would have + * this exception together with CalledProcessError. + */ +class OSError: public std::runtime_error +{ +public: + OSError(const std::string& err_msg, int err_code): + std::runtime_error( err_msg + ": " + std::strerror(err_code) ) + {} +}; + +//-------------------------------------------------------------------- + +//Environment Variable types +#ifndef _MSC_VER + using env_string_t = std::string; + using env_char_t = char; +#else + using env_string_t = std::wstring; + using env_char_t = wchar_t; +#endif +using env_map_t = std::map<env_string_t, env_string_t>; +using env_vector_t = std::vector<env_char_t>; + +//-------------------------------------------------------------------- +namespace util +{ + template <typename R> + inline bool is_ready(std::shared_future<R> const &f) + { + return f.wait_for(std::chrono::seconds(0)) == std::future_status::ready; + } + + inline void quote_argument(const std::wstring &argument, std::wstring &command_line, + bool force) + { + // + // Unless we're told otherwise, don't quote unless we actually + // need to do so --- hopefully avoid problems if programs won't + // parse quotes properly + // + + if (force == false && argument.empty() == false && + argument.find_first_of(L" \t\n\v\"") == argument.npos) { + command_line.append(argument); + } + else { + command_line.push_back(L'"'); + + for (auto it = argument.begin();; ++it) { + unsigned number_backslashes = 0; + + while (it != argument.end() && *it == L'\\') { + ++it; + ++number_backslashes; + } + + if (it == argument.end()) { + + // + // Escape all backslashes, but let the terminating + // double quotation mark we add below be interpreted + // as a metacharacter. + // + + command_line.append(number_backslashes * 2, L'\\'); + break; + } + else if (*it == L'"') { + + // + // Escape all backslashes and the following + // double quotation mark. + // + + command_line.append(number_backslashes * 2 + 1, L'\\'); + command_line.push_back(*it); + } + else { + + // + // Backslashes aren't special here. + // + + command_line.append(number_backslashes, L'\\'); + command_line.push_back(*it); + } + } + + command_line.push_back(L'"'); + } + } + +#ifdef __USING_WINDOWS__ + inline std::string get_last_error(DWORD errorMessageID) + { + if (errorMessageID == 0) + return std::string(); + + LPSTR messageBuffer = nullptr; + size_t size = FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_MAX_WIDTH_MASK, + NULL, errorMessageID, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPSTR)&messageBuffer, 0, NULL); + + std::string message(messageBuffer, size); + + LocalFree(messageBuffer); + + return message; + } + + inline FILE *file_from_handle(HANDLE h, const char *mode) + { + int md; + if (!mode) { + throw OSError("invalid_mode", 0); + } + + if (mode[0] == 'w') { + md = _O_WRONLY; + } + else if (mode[0] == 'r') { + md = _O_RDONLY; + } + else { + throw OSError("file_from_handle", 0); + } + + int os_fhandle = _open_osfhandle((intptr_t)h, md); + if (os_fhandle == -1) { + CloseHandle(h); + throw OSError("_open_osfhandle", 0); + } + + FILE *fp = _fdopen(os_fhandle, mode); + if (fp == 0) { + _close(os_fhandle); + throw OSError("_fdopen", 0); + } + + return fp; + } + + inline void configure_pipe(HANDLE* read_handle, HANDLE* write_handle, HANDLE* child_handle) + { + SECURITY_ATTRIBUTES saAttr; + + // Set the bInheritHandle flag so pipe handles are inherited. + saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); + saAttr.bInheritHandle = TRUE; + saAttr.lpSecurityDescriptor = NULL; + + // Create a pipe for the child process's STDIN. + if (!CreatePipe(read_handle, write_handle, &saAttr,0)) + throw OSError("CreatePipe", 0); + + // Ensure the write handle to the pipe for STDIN is not inherited. + if (!SetHandleInformation(*child_handle, HANDLE_FLAG_INHERIT, 0)) + throw OSError("SetHandleInformation", 0); + } + + // env_map_t MapFromWindowsEnvironment() + // * Imports current Environment in a C-string table + // * Parses the strings by splitting on the first "=" per line + // * Creates a map of the variables + // * Returns the map + inline env_map_t MapFromWindowsEnvironment(){ + wchar_t *variable_strings_ptr; + wchar_t *environment_strings_ptr; + std::wstring delimiter(L"="); + int del_len = delimiter.length(); + env_map_t mapped_environment; + + // Get a pointer to the environment block. + environment_strings_ptr = GetEnvironmentStringsW(); + // If the returned pointer is NULL, exit. + if (environment_strings_ptr == NULL) + { + throw OSError("GetEnvironmentStringsW", 0); + } + + // Variable strings are separated by NULL byte, and the block is + // terminated by a NULL byte. + + variable_strings_ptr = (wchar_t *) environment_strings_ptr; + + //Since the environment map ends with a null, we can loop until we find it. + while (*variable_strings_ptr) + { + // Create a string from Variable String + env_string_t current_line(variable_strings_ptr); + // Find the first "equals" sign. + auto pos = current_line.find(delimiter); + // Assuming it's not missing ... + if(pos!=std::wstring::npos){ + // ... parse the key and value. + env_string_t key = current_line.substr(0, pos); + env_string_t value = current_line.substr(pos + del_len); + // Map the entry. + mapped_environment[key] = value; + } + // Jump to next line in the environment map. + variable_strings_ptr += std::wcslen(variable_strings_ptr) + 1; + } + // We're done with the old environment map buffer. + FreeEnvironmentStringsW(environment_strings_ptr); + + // Return the map. + return mapped_environment; + } + + // env_vector_t WindowsEnvironmentVectorFromMap(const env_map_t &source_map) + // * Creates a vector buffer for the new environment string table + // * Copies in the mapped variables + // * Returns the vector + inline env_vector_t WindowsEnvironmentVectorFromMap(const env_map_t &source_map) + { + // Make a new environment map buffer. + env_vector_t environment_map_buffer; + // Give it some space. + environment_map_buffer.reserve(4096); + + // And fill'er up. + for(auto kv: source_map){ + // Create the line + env_string_t current_line(kv.first); current_line += L"="; current_line += kv.second; + // Add the line to the buffer. + std::copy(current_line.begin(), current_line.end(), std::back_inserter(environment_map_buffer)); + // Append a null + environment_map_buffer.push_back(0); + } + // Append one last null because of how Windows does it's environment maps. + environment_map_buffer.push_back(0); + + return environment_map_buffer; + } + + // env_vector_t CreateUpdatedWindowsEnvironmentVector(const env_map_t &changes_map) + // * Merges host environment with new mapped variables + // * Creates and returns string vector based on map + inline env_vector_t CreateUpdatedWindowsEnvironmentVector(const env_map_t &changes_map){ + // Import the environment map + env_map_t environment_map = MapFromWindowsEnvironment(); + // Merge in the changes with overwrite + for(auto& it: changes_map) + { + environment_map[it.first] = it.second; + } + // Create a Windows-usable Environment Map Buffer + env_vector_t environment_map_strings_vector = WindowsEnvironmentVectorFromMap(environment_map); + + return environment_map_strings_vector; + } + +#endif + + /*! + * Function: split + * Parameters: + * [in] str : Input string which needs to be split based upon the + * delimiters provided. + * [in] deleims : Delimiter characters based upon which the string needs + * to be split. Default constructed to ' '(space) and '\t'(tab) + * [out] vector<string> : Vector of strings split at deleimiter. + */ + static inline std::vector<std::string> + split(const std::string& str, const std::string& delims=" \t") + { + std::vector<std::string> res; + size_t init = 0; + + while (true) { + auto pos = str.find_first_of(delims, init); + if (pos == std::string::npos) { + res.emplace_back(str.substr(init, str.length())); + break; + } + res.emplace_back(str.substr(init, pos - init)); + pos++; + init = pos; + } + + return res; + } + + + /*! + * Function: join + * Parameters: + * [in] vec : Vector of strings which needs to be joined to form + * a single string with words separated by a separator char. + * [in] sep : String used to separate 2 words in the joined string. + * Default constructed to ' ' (space). + * [out] string: Joined string. + */ + static inline + std::string join(const std::vector<std::string>& vec, + const std::string& sep = " ") + { + std::string res; + for (auto& elem : vec) res.append(elem + sep); + res.erase(--res.end()); + return res; + } + + +#ifndef __USING_WINDOWS__ + /*! + * Function: set_clo_on_exec + * Sets/Resets the FD_CLOEXEC flag on the provided file descriptor + * based upon the `set` parameter. + * Parameters: + * [in] fd : The descriptor on which FD_CLOEXEC needs to be set/reset. + * [in] set : If 'true', set FD_CLOEXEC. + * If 'false' unset FD_CLOEXEC. + */ + static inline + void set_clo_on_exec(int fd, bool set = true) + { + int flags = fcntl(fd, F_GETFD, 0); + if (set) flags |= FD_CLOEXEC; + else flags &= ~FD_CLOEXEC; + //TODO: should check for errors + fcntl(fd, F_SETFD, flags); + } + + + /*! + * Function: pipe_cloexec + * Creates a pipe and sets FD_CLOEXEC flag on both + * read and write descriptors of the pipe. + * Parameters: + * [out] : A pair of file descriptors. + * First element of pair is the read descriptor of pipe. + * Second element is the write descriptor of pipe. + */ + static inline + std::pair<int, int> pipe_cloexec() noexcept(false) + { + int pipe_fds[2]; + int res = pipe(pipe_fds); + if (res) { + throw OSError("pipe failure", errno); + } + + set_clo_on_exec(pipe_fds[0]); + set_clo_on_exec(pipe_fds[1]); + + return std::make_pair(pipe_fds[0], pipe_fds[1]); + } +#endif + + + /*! + * Function: write_n + * Writes `length` bytes to the file descriptor `fd` + * from the buffer `buf`. + * Parameters: + * [in] fd : The file descriptotr to write to. + * [in] buf: Buffer from which data needs to be written to fd. + * [in] length: The number of bytes that needs to be written from + * `buf` to `fd`. + * [out] int : Number of bytes written or -1 in case of failure. + */ + static inline + int write_n(int fd, const char* buf, size_t length) + { + size_t nwritten = 0; + while (nwritten < length) { + int written = write(fd, buf + nwritten, length - nwritten); + if (written == -1) return -1; + nwritten += written; + } + return nwritten; + } + + + /*! + * Function: read_atmost_n + * Reads at the most `read_upto` bytes from the + * file object `fp` before returning. + * Parameters: + * [in] fp : The file object from which it needs to read. + * [in] buf : The buffer into which it needs to write the data. + * [in] read_upto: Max number of bytes which must be read from `fd`. + * [out] int : Number of bytes written to `buf` or read from `fd` + * OR -1 in case of error. + * NOTE: In case of EINTR while reading from socket, this API + * will retry to read from `fd`, but only till the EINTR counter + * reaches 50 after which it will return with whatever data it read. + */ + static inline + int read_atmost_n(FILE* fp, char* buf, size_t read_upto) + { +#ifdef __USING_WINDOWS__ + return (int)fread(buf, 1, read_upto, fp); +#else + int fd = fileno(fp); + int rbytes = 0; + int eintr_cnter = 0; + + while (1) { + int read_bytes = read(fd, buf + rbytes, read_upto - rbytes); + if (read_bytes == -1) { + if (errno == EINTR) { + if (eintr_cnter >= 50) return -1; + eintr_cnter++; + continue; + } + return -1; + } + if (read_bytes == 0) return rbytes; + + rbytes += read_bytes; + } + return rbytes; +#endif + } + + + /*! + * Function: read_all + * Reads all the available data from `fp` into + * `buf`. Internally calls read_atmost_n. + * Parameters: + * [in] fp : The file object from which to read from. + * [in] buf : The buffer of type `class Buffer` into which + * the read data is written to. + * [out] int: Number of bytes read OR -1 in case of failure. + * + * NOTE: `class Buffer` is a exposed public class. See below. + */ + + static inline int read_all(FILE* fp, std::vector<char>& buf) + { + auto buffer = buf.data(); + int total_bytes_read = 0; + int fill_sz = buf.size(); + + while (1) { + const int rd_bytes = read_atmost_n(fp, buffer, fill_sz); + + if (rd_bytes == -1) { // Read finished + if (total_bytes_read == 0) return -1; + break; + + } else if (rd_bytes == fill_sz) { // Buffer full + const auto orig_sz = buf.size(); + const auto new_sz = orig_sz * 2; + buf.resize(new_sz); + fill_sz = new_sz - orig_sz; + + //update the buffer pointer + buffer = buf.data(); + total_bytes_read += rd_bytes; + buffer += total_bytes_read; + + } else { // Partial data ? Continue reading + total_bytes_read += rd_bytes; + fill_sz -= rd_bytes; + break; + } + } + buf.erase(buf.begin()+total_bytes_read, buf.end()); // remove extra nulls + return total_bytes_read; + } + +#ifndef __USING_WINDOWS__ + /*! + * Function: wait_for_child_exit + * Waits for the process with pid `pid` to exit + * and returns its status. + * Parameters: + * [in] pid : The pid of the process. + * [out] pair<int, int>: + * pair.first : Return code of the waitpid call. + * pair.second : Exit status of the process. + * + * NOTE: This is a blocking call as in, it will loop + * till the child is exited. + */ + static inline + std::pair<int, int> wait_for_child_exit(int pid) + { + int status = 0; + int ret = -1; + while (1) { + ret = waitpid(pid, &status, 0); + if (ret == -1) break; + if (ret == 0) continue; + return std::make_pair(ret, status); + } + + return std::make_pair(ret, status); + } +#endif + +} // end namespace util + + + +/* ------------------------------- + * Popen Arguments + * ------------------------------- + */ + +/*! + * The buffer size of the stdin/stdout/stderr + * streams of the child process. + * Default value is 0. + */ +struct bufsize { + explicit bufsize(int sz): bufsiz(sz) {} + int bufsiz = 0; +}; + +/*! + * Option to defer spawning of the child process + * till `Popen::start_process` API is called. + * Default value is false. + */ +struct defer_spawn { + explicit defer_spawn(bool d): defer(d) {} + bool defer = false; +}; + +/*! + * Option to close all file descriptors + * when the child process is spawned. + * The close fd list does not include + * input/output/error if they are explicitly + * set as part of the Popen arguments. + * + * Default value is false. + */ +struct close_fds { + explicit close_fds(bool c): close_all(c) {} + bool close_all = false; +}; + +/*! + * Option to make the child process as the + * session leader and thus the process + * group leader. + * Default value is false. + */ +struct session_leader { + explicit session_leader(bool sl): leader_(sl) {} + bool leader_ = false; +}; + +struct shell { + explicit shell(bool s): shell_(s) {} + bool shell_ = false; +}; + +/*! + * Base class for all arguments involving string value. + */ +struct string_arg +{ + string_arg(const char* arg): arg_value(arg) {} + string_arg(std::string&& arg): arg_value(std::move(arg)) {} + string_arg(std::string arg): arg_value(std::move(arg)) {} + std::string arg_value; +}; + +/*! + * Option to specify the executable name separately + * from the args sequence. + * In this case the cmd args must only contain the + * options required for this executable. + * + * Eg: executable{"ls"} + */ +struct executable: string_arg +{ + template <typename T> + executable(T&& arg): string_arg(std::forward<T>(arg)) {} +}; + +/*! + * Option to set the current working directory + * of the spawned process. + * + * Eg: cwd{"/some/path/x"} + */ +struct cwd: string_arg +{ + template <typename T> + cwd(T&& arg): string_arg(std::forward<T>(arg)) {} +}; + +/*! + * Option to specify environment variables required by + * the spawned process. + * + * Eg: environment{{ {"K1", "V1"}, {"K2", "V2"},... }} + */ +struct environment +{ + environment(env_map_t&& env): + env_(std::move(env)) {} + explicit environment(const env_map_t& env): + env_(env) {} + env_map_t env_; +}; + + +/*! + * Used for redirecting input/output/error + */ +enum IOTYPE { + STDOUT = 1, + STDERR, + PIPE, +}; + +//TODO: A common base/interface for below stream structures ?? + +/*! + * Option to specify the input channel for the child + * process. It can be: + * 1. An already open file descriptor. + * 2. A file name. + * 3. IOTYPE. Usual a PIPE + * + * Eg: input{PIPE} + * OR in case of redirection, output of another Popen + * input{popen.output()} + */ +struct input +{ + // For an already existing file descriptor. + explicit input(int fd): rd_ch_(fd) {} + + // FILE pointer. + explicit input (FILE* fp):input(fileno(fp)) { assert(fp); } + + explicit input(const char* filename) { + int fd = open(filename, O_RDONLY); + if (fd == -1) throw OSError("File not found: ", errno); + rd_ch_ = fd; + } + explicit input(IOTYPE typ) { + assert (typ == PIPE && "STDOUT/STDERR not allowed"); +#ifndef __USING_WINDOWS__ + std::tie(rd_ch_, wr_ch_) = util::pipe_cloexec(); +#endif + } + + int rd_ch_ = -1; + int wr_ch_ = -1; +}; + + +/*! + * Option to specify the output channel for the child + * process. It can be: + * 1. An already open file descriptor. + * 2. A file name. + * 3. IOTYPE. Usually a PIPE. + * + * Eg: output{PIPE} + * OR output{"output.txt"} + */ +struct output +{ + explicit output(int fd): wr_ch_(fd) {} + + explicit output (FILE* fp):output(fileno(fp)) { assert(fp); } + + explicit output(const char* filename) { + int fd = open(filename, O_APPEND | O_CREAT | O_RDWR, 0640); + if (fd == -1) throw OSError("File not found: ", errno); + wr_ch_ = fd; + } + explicit output(IOTYPE typ) { + assert (typ == PIPE && "STDOUT/STDERR not allowed"); +#ifndef __USING_WINDOWS__ + std::tie(rd_ch_, wr_ch_) = util::pipe_cloexec(); +#endif + } + + int rd_ch_ = -1; + int wr_ch_ = -1; +}; + + +/*! + * Option to specify the error channel for the child + * process. It can be: + * 1. An already open file descriptor. + * 2. A file name. + * 3. IOTYPE. Usually a PIPE or STDOUT + * + */ +struct error +{ + explicit error(int fd): wr_ch_(fd) {} + + explicit error(FILE* fp):error(fileno(fp)) { assert(fp); } + + explicit error(const char* filename) { + int fd = open(filename, O_APPEND | O_CREAT | O_RDWR, 0640); + if (fd == -1) throw OSError("File not found: ", errno); + wr_ch_ = fd; + } + explicit error(IOTYPE typ) { + assert ((typ == PIPE || typ == STDOUT) && "STDERR not allowed"); + if (typ == PIPE) { +#ifndef __USING_WINDOWS__ + std::tie(rd_ch_, wr_ch_) = util::pipe_cloexec(); +#endif + } else { + // Need to defer it till we have checked all arguments + deferred_ = true; + } + } + + bool deferred_ = false; + int rd_ch_ = -1; + int wr_ch_ = -1; +}; + +// Impoverished, meager, needy, truly needy +// version of type erasure to store function pointers +// needed to provide the functionality of preexec_func +// ATTN: Can be used only to execute functions with no +// arguments and returning void. +// Could have used more efficient methods, ofcourse, but +// that won't yield me the consistent syntax which I am +// aiming for. If you know, then please do let me know. + +class preexec_func +{ +public: + preexec_func() {} + + template <typename Func> + explicit preexec_func(Func f): holder_(new FuncHolder<Func>(std::move(f))) + {} + + void operator()() { + (*holder_)(); + } + +private: + struct HolderBase { + virtual void operator()() const = 0; + virtual ~HolderBase(){}; + }; + template <typename T> + struct FuncHolder: HolderBase { + FuncHolder(T func): func_(std::move(func)) {} + void operator()() const override { func_(); } + // The function pointer/reference + T func_; + }; + + std::unique_ptr<HolderBase> holder_ = nullptr; +}; + +// ~~~~ End Popen Args ~~~~ + + +/*! + * class: Buffer + * This class is a very thin wrapper around std::vector<char> + * This is basically used to determine the length of the actual + * data stored inside the dynamically resized vector. + * + * This is what is returned as the output to communicate and check_output + * functions, so, users must know about this class. + * + * OutBuffer and ErrBuffer are just different typedefs to this class. + */ +class Buffer +{ +public: + Buffer() {} + explicit Buffer(size_t cap) { buf.resize(cap); } + void add_cap(size_t cap) { buf.resize(cap); } + +#if 0 + Buffer(const Buffer& other): + buf(other.buf), + length(other.length) + { + std::cout << "COPY" << std::endl; + } + + Buffer(Buffer&& other): + buf(std::move(other.buf)), + length(other.length) + { + std::cout << "MOVE" << std::endl; + } +#endif + +public: + std::vector<char> buf; + size_t length = 0; +}; + +// Buffer for storing output written to output fd +using OutBuffer = Buffer; +// Buffer for storing output written to error fd +using ErrBuffer = Buffer; + + +// Fwd Decl. +class Popen; + +/*--------------------------------------------------- + * DETAIL NAMESPACE + *--------------------------------------------------- + */ + +namespace detail { + +// Metaprogram for searching a type within +// a variadic parameter pack +// This is particularly required to do a compile time +// checking of the arguments provided to 'check_output' function +// wherein the user is not expected to provide an 'output' option. + +template <typename... T> struct param_pack{}; + +template <typename F, typename T> struct has_type; + +template <typename F> +struct has_type<F, param_pack<>> { + static constexpr bool value = false; +}; + +template <typename F, typename... T> +struct has_type<F, param_pack<F, T...>> { + static constexpr bool value = true; +}; + +template <typename F, typename H, typename... T> +struct has_type<F, param_pack<H,T...>> { + static constexpr bool value = + std::is_same<F, typename std::decay<H>::type>::value ? true : has_type<F, param_pack<T...>>::value; +}; + +//---- + +/*! + * A helper class to Popen class for setting + * options as provided in the Popen constructor + * or in check_output arguments. + * This design allows us to _not_ have any fixed position + * to any arguments and specify them in a way similar to what + * can be done in python. + */ +struct ArgumentDeducer +{ + ArgumentDeducer(Popen* p): popen_(p) {} + + void set_option(executable&& exe); + void set_option(cwd&& cwdir); + void set_option(bufsize&& bsiz); + void set_option(environment&& env); + void set_option(defer_spawn&& defer); + void set_option(shell&& sh); + void set_option(input&& inp); + void set_option(output&& out); + void set_option(error&& err); + void set_option(close_fds&& cfds); + void set_option(preexec_func&& prefunc); + void set_option(session_leader&& sleader); + +private: + Popen* popen_ = nullptr; +}; + +/*! + * A helper class to Popen. + * This takes care of all the fork-exec logic + * in the execute_child API. + */ +class Child +{ +public: + Child(Popen* p, int err_wr_pipe): + parent_(p), + err_wr_pipe_(err_wr_pipe) + {} + + void execute_child(); + +private: + // Lets call it parent even though + // technically a bit incorrect + Popen* parent_ = nullptr; + int err_wr_pipe_ = -1; +}; + +// Fwd Decl. +class Streams; + +/*! + * A helper class to Streams. + * This takes care of management of communicating + * with the child process with the means of the correct + * file descriptor. + */ +class Communication +{ +public: + Communication(Streams* stream): stream_(stream) + {} + void operator=(const Communication&) = delete; +public: + int send(const char* msg, size_t length); + int send(const std::vector<char>& msg); + + std::pair<OutBuffer, ErrBuffer> communicate(const char* msg, size_t length); + std::pair<OutBuffer, ErrBuffer> communicate(const std::vector<char>& msg) + { return communicate(msg.data(), msg.size()); } + + void set_out_buf_cap(size_t cap) { out_buf_cap_ = cap; } + void set_err_buf_cap(size_t cap) { err_buf_cap_ = cap; } + +private: + std::pair<OutBuffer, ErrBuffer> communicate_threaded( + const char* msg, size_t length); + +private: + Streams* stream_; + size_t out_buf_cap_ = DEFAULT_BUF_CAP_BYTES; + size_t err_buf_cap_ = DEFAULT_BUF_CAP_BYTES; +}; + + + +/*! + * This is a helper class to Popen. + * It takes care of management of all the file descriptors + * and file pointers. + * It dispatches of the communication aspects to the + * Communication class. + * Read through the data members to understand about the + * various file descriptors used. + */ +class Streams +{ +public: + Streams():comm_(this) {} + void operator=(const Streams&) = delete; + +public: + void setup_comm_channels(); + + void cleanup_fds() + { + if (write_to_child_ != -1 && read_from_parent_ != -1) { + close(write_to_child_); + } + if (write_to_parent_ != -1 && read_from_child_ != -1) { + close(read_from_child_); + } + if (err_write_ != -1 && err_read_ != -1) { + close(err_read_); + } + } + + void close_parent_fds() + { + if (write_to_child_ != -1) close(write_to_child_); + if (read_from_child_ != -1) close(read_from_child_); + if (err_read_ != -1) close(err_read_); + } + + void close_child_fds() + { + if (write_to_parent_ != -1) close(write_to_parent_); + if (read_from_parent_ != -1) close(read_from_parent_); + if (err_write_ != -1) close(err_write_); + } + + FILE* input() { return input_.get(); } + FILE* output() { return output_.get(); } + FILE* error() { return error_.get(); } + + void input(FILE* fp) { input_.reset(fp, fclose); } + void output(FILE* fp) { output_.reset(fp, fclose); } + void error(FILE* fp) { error_.reset(fp, fclose); } + + void set_out_buf_cap(size_t cap) { comm_.set_out_buf_cap(cap); } + void set_err_buf_cap(size_t cap) { comm_.set_err_buf_cap(cap); } + +public: /* Communication forwarding API's */ + int send(const char* msg, size_t length) + { return comm_.send(msg, length); } + + int send(const std::vector<char>& msg) + { return comm_.send(msg); } + + std::pair<OutBuffer, ErrBuffer> communicate(const char* msg, size_t length) + { return comm_.communicate(msg, length); } + + std::pair<OutBuffer, ErrBuffer> communicate(const std::vector<char>& msg) + { return comm_.communicate(msg); } + + +public:// Yes they are public + + std::shared_ptr<FILE> input_ = nullptr; + std::shared_ptr<FILE> output_ = nullptr; + std::shared_ptr<FILE> error_ = nullptr; + +#ifdef __USING_WINDOWS__ + HANDLE g_hChildStd_IN_Rd = nullptr; + HANDLE g_hChildStd_IN_Wr = nullptr; + HANDLE g_hChildStd_OUT_Rd = nullptr; + HANDLE g_hChildStd_OUT_Wr = nullptr; + HANDLE g_hChildStd_ERR_Rd = nullptr; + HANDLE g_hChildStd_ERR_Wr = nullptr; +#endif + + // Buffer size for the IO streams + int bufsiz_ = 0; + + // Pipes for communicating with child + + // Emulates stdin + int write_to_child_ = -1; // Parent owned descriptor + int read_from_parent_ = -1; // Child owned descriptor + + // Emulates stdout + int write_to_parent_ = -1; // Child owned descriptor + int read_from_child_ = -1; // Parent owned descriptor + + // Emulates stderr + int err_write_ = -1; // Write error to parent (Child owned) + int err_read_ = -1; // Read error from child (Parent owned) + +private: + Communication comm_; +}; + +} // end namespace detail + + + +/*! + * class: Popen + * This is the single most important class in the whole library + * and glues together all the helper classes to provide a common + * interface to the client. + * + * API's provided by the class: + * 1. Popen({"cmd"}, output{..}, error{..}, cwd{..}, ....) + * Command provided as a sequence. + * 2. Popen("cmd arg1"m output{..}, error{..}, cwd{..}, ....) + * Command provided in a single string. + * 3. wait() - Wait for the child to exit. + * 4. retcode() - The return code of the exited child. + * 5. pid() - PID of the spawned child. + * 6. poll() - Check the status of the running child. + * 7. kill(sig_num) - Kill the child. SIGTERM used by default. + * 8. send(...) - Send input to the input channel of the child. + * 9. communicate(...) - Get the output/error from the child and close the channels + * from the parent side. + *10. input() - Get the input channel/File pointer. Can be used for + * customizing the way of sending input to child. + *11. output() - Get the output channel/File pointer. Usually used + in case of redirection. See piping examples. + *12. error() - Get the error channel/File pointer. Usually used + in case of redirection. + *13. start_process() - Start the child process. Only to be used when + * `defer_spawn` option was provided in Popen constructor. + */ +class Popen +{ +public: + friend struct detail::ArgumentDeducer; + friend class detail::Child; + + template <typename... Args> + Popen(const std::string& cmd_args, Args&& ...args): + args_(cmd_args) + { + vargs_ = util::split(cmd_args); + init_args(std::forward<Args>(args)...); + + // Setup the communication channels of the Popen class + stream_.setup_comm_channels(); + + if (!defer_process_start_) execute_process(); + } + + template <typename... Args> + Popen(std::initializer_list<const char*> cmd_args, Args&& ...args) + { + vargs_.insert(vargs_.end(), cmd_args.begin(), cmd_args.end()); + init_args(std::forward<Args>(args)...); + + // Setup the communication channels of the Popen class + stream_.setup_comm_channels(); + + if (!defer_process_start_) execute_process(); + } + + template <typename... Args> + Popen(std::vector<std::string> vargs_, Args &&... args) : vargs_(vargs_) + { + init_args(std::forward<Args>(args)...); + + // Setup the communication channels of the Popen class + stream_.setup_comm_channels(); + + if (!defer_process_start_) execute_process(); + } + +/* + ~Popen() + { +#ifdef __USING_WINDOWS__ + CloseHandle(this->process_handle_); +#endif + } +*/ + + void start_process() noexcept(false); + + int pid() const noexcept { return child_pid_; } + + int retcode() const noexcept { return retcode_; } + + int wait() noexcept(false); + + int poll() noexcept(false); + + // Does not fail, Caller is expected to recheck the + // status with a call to poll() + void kill(int sig_num = 9); + + void set_out_buf_cap(size_t cap) { stream_.set_out_buf_cap(cap); } + + void set_err_buf_cap(size_t cap) { stream_.set_err_buf_cap(cap); } + + int send(const char* msg, size_t length) + { return stream_.send(msg, length); } + + int send(const std::string& msg) + { return send(msg.c_str(), msg.size()); } + + int send(const std::vector<char>& msg) + { return stream_.send(msg); } + + std::pair<OutBuffer, ErrBuffer> communicate(const char* msg, size_t length) + { + auto res = stream_.communicate(msg, length); + retcode_ = wait(); + return res; + } + + std::pair<OutBuffer, ErrBuffer> communicate(const std::string& msg) + { + return communicate(msg.c_str(), msg.size()); + } + + std::pair<OutBuffer, ErrBuffer> communicate(const std::vector<char>& msg) + { + auto res = stream_.communicate(msg); + retcode_ = wait(); + return res; + } + + std::pair<OutBuffer, ErrBuffer> communicate() + { + return communicate(nullptr, 0); + } + + FILE* input() { return stream_.input(); } + FILE* output() { return stream_.output();} + FILE* error() { return stream_.error(); } + + /// Stream close APIs + void close_input() { stream_.input_.reset(); } + void close_output() { stream_.output_.reset(); } + void close_error() { stream_.error_.reset(); } + +private: + template <typename F, typename... Args> + void init_args(F&& farg, Args&&... args); + void init_args(); + void populate_c_argv(); + void execute_process() noexcept(false); + +private: + detail::Streams stream_; + +#ifdef __USING_WINDOWS__ + HANDLE process_handle_; + std::future<void> cleanup_future_; +#endif + + bool defer_process_start_ = false; + bool close_fds_ = false; + bool has_preexec_fn_ = false; + bool shell_ = false; + bool session_leader_ = false; + + std::string exe_name_; + std::string cwd_; + env_map_t env_; + preexec_func preexec_fn_; + + // Command in string format + std::string args_; + // Command provided as sequence + std::vector<std::string> vargs_; + std::vector<char*> cargv_; + + bool child_created_ = false; + // Pid of the child process + int child_pid_ = -1; + + int retcode_ = -1; +}; + +inline void Popen::init_args() { + populate_c_argv(); +} + +template <typename F, typename... Args> +inline void Popen::init_args(F&& farg, Args&&... args) +{ + detail::ArgumentDeducer argd(this); + argd.set_option(std::forward<F>(farg)); + init_args(std::forward<Args>(args)...); +} + +inline void Popen::populate_c_argv() +{ + cargv_.clear(); + cargv_.reserve(vargs_.size() + 1); + for (auto& arg : vargs_) cargv_.push_back(&arg[0]); + cargv_.push_back(nullptr); +} + +inline void Popen::start_process() noexcept(false) +{ + // The process was started/tried to be started + // in the constructor itself. + // For explicitly calling this API to start the + // process, 'defer_spawn' argument must be set to + // true in the constructor. + if (!defer_process_start_) { + assert (0); + return; + } + execute_process(); +} + +inline int Popen::wait() noexcept(false) +{ +#ifdef __USING_WINDOWS__ + int ret = WaitForSingleObject(process_handle_, INFINITE); + + return 0; +#else + int ret, status; + std::tie(ret, status) = util::wait_for_child_exit(pid()); + if (ret == -1) { + if (errno != ECHILD) throw OSError("waitpid failed", errno); + return 0; + } + if (WIFEXITED(status)) return WEXITSTATUS(status); + if (WIFSIGNALED(status)) return WTERMSIG(status); + else return 255; + + return 0; +#endif +} + +inline int Popen::poll() noexcept(false) +{ +#ifdef __USING_WINDOWS__ + int ret = WaitForSingleObject(process_handle_, 0); + if (ret != WAIT_OBJECT_0) return -1; + + DWORD dretcode_; + if (FALSE == GetExitCodeProcess(process_handle_, &dretcode_)) + throw OSError("GetExitCodeProcess", 0); + + retcode_ = (int)dretcode_; + CloseHandle(process_handle_); + + return retcode_; +#else + if (!child_created_) return -1; // TODO: ?? + + int status; + + // Returns zero if child is still running + int ret = waitpid(child_pid_, &status, WNOHANG); + if (ret == 0) return -1; + + if (ret == child_pid_) { + if (WIFSIGNALED(status)) { + retcode_ = WTERMSIG(status); + } else if (WIFEXITED(status)) { + retcode_ = WEXITSTATUS(status); + } else { + retcode_ = 255; + } + return retcode_; + } + + if (ret == -1) { + // From subprocess.py + // This happens if SIGCHLD is set to be ignored + // or waiting for child process has otherwise been disabled + // for our process. This child is dead, we cannot get the + // status. + if (errno == ECHILD) retcode_ = 0; + else throw OSError("waitpid failed", errno); + } else { + retcode_ = ret; + } + + return retcode_; +#endif +} + +inline void Popen::kill(int sig_num) +{ +#ifdef __USING_WINDOWS__ + if (!TerminateProcess(this->process_handle_, (UINT)sig_num)) { + throw OSError("TerminateProcess", 0); + } +#else + if (session_leader_) killpg(child_pid_, sig_num); + else ::kill(child_pid_, sig_num); +#endif +} + + +inline void Popen::execute_process() noexcept(false) +{ +#ifdef __USING_WINDOWS__ + if (this->shell_) { + throw OSError("shell not currently supported on windows", 0); + } + + void* environment_string_table_ptr = nullptr; + env_vector_t environment_string_vector; + if(this->env_.size()){ + environment_string_vector = util::CreateUpdatedWindowsEnvironmentVector(env_); + environment_string_table_ptr = (void*)environment_string_vector.data(); + } + + if (exe_name_.length()) { + this->vargs_.insert(this->vargs_.begin(), this->exe_name_); + this->populate_c_argv(); + } + this->exe_name_ = vargs_[0]; + + std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter; + std::wstring argument; + std::wstring command_line; + + for (auto arg : this->vargs_) { + argument = converter.from_bytes(arg); + util::quote_argument(argument, command_line, false); + command_line += L" "; + } + + // CreateProcessW can modify szCmdLine so we allocate needed memory + wchar_t *szCmdline = new wchar_t[command_line.size() + 1]; + wcscpy_s(szCmdline, command_line.size() + 1, command_line.c_str()); + PROCESS_INFORMATION piProcInfo; + STARTUPINFOW siStartInfo; + BOOL bSuccess = FALSE; + DWORD creation_flags = CREATE_UNICODE_ENVIRONMENT | CREATE_NO_WINDOW; + + // Set up members of the PROCESS_INFORMATION structure. + ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION)); + + // Set up members of the STARTUPINFOW structure. + // This structure specifies the STDIN and STDOUT handles for redirection. + + ZeroMemory(&siStartInfo, sizeof(STARTUPINFOW)); + siStartInfo.cb = sizeof(STARTUPINFOW); + + siStartInfo.hStdError = this->stream_.g_hChildStd_ERR_Wr; + siStartInfo.hStdOutput = this->stream_.g_hChildStd_OUT_Wr; + siStartInfo.hStdInput = this->stream_.g_hChildStd_IN_Rd; + + siStartInfo.dwFlags |= STARTF_USESTDHANDLES; + + // Create the child process. + bSuccess = CreateProcessW(NULL, + szCmdline, // command line + NULL, // process security attributes + NULL, // primary thread security attributes + TRUE, // handles are inherited + creation_flags, // creation flags + environment_string_table_ptr, // use provided environment + NULL, // use parent's current directory + &siStartInfo, // STARTUPINFOW pointer + &piProcInfo); // receives PROCESS_INFORMATION + + // If an error occurs, exit the application. + if (!bSuccess) { + DWORD errorMessageID = ::GetLastError(); + throw CalledProcessError("CreateProcess failed: " + util::get_last_error(errorMessageID), errorMessageID); + } + + CloseHandle(piProcInfo.hThread); + + /* + TODO: use common apis to close linux handles + */ + + this->process_handle_ = piProcInfo.hProcess; + + this->cleanup_future_ = std::async(std::launch::async, [this] { + WaitForSingleObject(this->process_handle_, INFINITE); + + CloseHandle(this->stream_.g_hChildStd_ERR_Wr); + CloseHandle(this->stream_.g_hChildStd_OUT_Wr); + CloseHandle(this->stream_.g_hChildStd_IN_Rd); + }); + +/* + NOTE: In the linux version, there is a check to make sure that the process + has been started. Here, we do nothing because CreateProcess will throw + if we fail to create the process. +*/ + + +#else + + int err_rd_pipe, err_wr_pipe; + std::tie(err_rd_pipe, err_wr_pipe) = util::pipe_cloexec(); + + if (shell_) { + auto new_cmd = util::join(vargs_); + vargs_.clear(); + vargs_.insert(vargs_.begin(), {"/bin/sh", "-c"}); + vargs_.push_back(new_cmd); + populate_c_argv(); + } + + if (exe_name_.length()) { + vargs_.insert(vargs_.begin(), exe_name_); + populate_c_argv(); + } + exe_name_ = vargs_[0]; + + child_pid_ = fork(); + + if (child_pid_ < 0) { + close(err_rd_pipe); + close(err_wr_pipe); + throw OSError("fork failed", errno); + } + + child_created_ = true; + + if (child_pid_ == 0) + { + // Close descriptors belonging to parent + stream_.close_parent_fds(); + + //Close the read end of the error pipe + close(err_rd_pipe); + + detail::Child chld(this, err_wr_pipe); + chld.execute_child(); + } + else + { + close (err_wr_pipe);// close child side of pipe, else get stuck in read below + + stream_.close_child_fds(); + + try { + char err_buf[SP_MAX_ERR_BUF_SIZ] = {0,}; + + int read_bytes = util::read_atmost_n( + fdopen(err_rd_pipe, "r"), + err_buf, + SP_MAX_ERR_BUF_SIZ); + close(err_rd_pipe); + + if (read_bytes || strlen(err_buf)) { + // Call waitpid to reap the child process + // waitpid suspends the calling process until the + // child terminates. + int retcode = wait(); + + // Throw whatever information we have about child failure + throw CalledProcessError(err_buf, retcode); + } + } catch (std::exception& exp) { + stream_.cleanup_fds(); + throw; + } + + } +#endif +} + +namespace detail { + + inline void ArgumentDeducer::set_option(executable&& exe) { + popen_->exe_name_ = std::move(exe.arg_value); + } + + inline void ArgumentDeducer::set_option(cwd&& cwdir) { + popen_->cwd_ = std::move(cwdir.arg_value); + } + + inline void ArgumentDeducer::set_option(bufsize&& bsiz) { + popen_->stream_.bufsiz_ = bsiz.bufsiz; + } + + inline void ArgumentDeducer::set_option(environment&& env) { + popen_->env_ = std::move(env.env_); + } + + inline void ArgumentDeducer::set_option(defer_spawn&& defer) { + popen_->defer_process_start_ = defer.defer; + } + + inline void ArgumentDeducer::set_option(shell&& sh) { + popen_->shell_ = sh.shell_; + } + + inline void ArgumentDeducer::set_option(session_leader&& sleader) { + popen_->session_leader_ = sleader.leader_; + } + + inline void ArgumentDeducer::set_option(input&& inp) { + if (inp.rd_ch_ != -1) popen_->stream_.read_from_parent_ = inp.rd_ch_; + if (inp.wr_ch_ != -1) popen_->stream_.write_to_child_ = inp.wr_ch_; + } + + inline void ArgumentDeducer::set_option(output&& out) { + if (out.wr_ch_ != -1) popen_->stream_.write_to_parent_ = out.wr_ch_; + if (out.rd_ch_ != -1) popen_->stream_.read_from_child_ = out.rd_ch_; + } + + inline void ArgumentDeducer::set_option(error&& err) { + if (err.deferred_) { + if (popen_->stream_.write_to_parent_) { + popen_->stream_.err_write_ = popen_->stream_.write_to_parent_; + } else { + throw std::runtime_error("Set output before redirecting error to output"); + } + } + if (err.wr_ch_ != -1) popen_->stream_.err_write_ = err.wr_ch_; + if (err.rd_ch_ != -1) popen_->stream_.err_read_ = err.rd_ch_; + } + + inline void ArgumentDeducer::set_option(close_fds&& cfds) { + popen_->close_fds_ = cfds.close_all; + } + + inline void ArgumentDeducer::set_option(preexec_func&& prefunc) { + popen_->preexec_fn_ = std::move(prefunc); + popen_->has_preexec_fn_ = true; + } + + + inline void Child::execute_child() { +#ifndef __USING_WINDOWS__ + int sys_ret = -1; + auto& stream = parent_->stream_; + + try { + if (stream.write_to_parent_ == 0) + stream.write_to_parent_ = dup(stream.write_to_parent_); + + if (stream.err_write_ == 0 || stream.err_write_ == 1) + stream.err_write_ = dup(stream.err_write_); + + // Make the child owned descriptors as the + // stdin, stdout and stderr for the child process + auto _dup2_ = [](int fd, int to_fd) { + if (fd == to_fd) { + // dup2 syscall does not reset the + // CLOEXEC flag if the descriptors + // provided to it are same. + // But, we need to reset the CLOEXEC + // flag as the provided descriptors + // are now going to be the standard + // input, output and error + util::set_clo_on_exec(fd, false); + } else if(fd != -1) { + int res = dup2(fd, to_fd); + if (res == -1) throw OSError("dup2 failed", errno); + } + }; + + // Create the standard streams + _dup2_(stream.read_from_parent_, 0); // Input stream + _dup2_(stream.write_to_parent_, 1); // Output stream + _dup2_(stream.err_write_, 2); // Error stream + + // Close the duped descriptors + if (stream.read_from_parent_ != -1 && stream.read_from_parent_ > 2) + close(stream.read_from_parent_); + + if (stream.write_to_parent_ != -1 && stream.write_to_parent_ > 2) + close(stream.write_to_parent_); + + if (stream.err_write_ != -1 && stream.err_write_ > 2) + close(stream.err_write_); + + // Close all the inherited fd's except the error write pipe + if (parent_->close_fds_) { + int max_fd = sysconf(_SC_OPEN_MAX); + if (max_fd == -1) throw OSError("sysconf failed", errno); + + for (int i = 3; i < max_fd; i++) { + if (i == err_wr_pipe_) continue; + close(i); + } + } + + // Change the working directory if provided + if (parent_->cwd_.length()) { + sys_ret = chdir(parent_->cwd_.c_str()); + if (sys_ret == -1) throw OSError("chdir failed", errno); + } + + if (parent_->has_preexec_fn_) { + parent_->preexec_fn_(); + } + + if (parent_->session_leader_) { + sys_ret = setsid(); + if (sys_ret == -1) throw OSError("setsid failed", errno); + } + + // Replace the current image with the executable + if (parent_->env_.size()) { + for (auto& kv : parent_->env_) { + setenv(kv.first.c_str(), kv.second.c_str(), 1); + } + sys_ret = execvp(parent_->exe_name_.c_str(), parent_->cargv_.data()); + } else { + sys_ret = execvp(parent_->exe_name_.c_str(), parent_->cargv_.data()); + } + + if (sys_ret == -1) throw OSError("execve failed", errno); + + } catch (const OSError& exp) { + // Just write the exception message + // TODO: Give back stack trace ? + std::string err_msg(exp.what()); + //ATTN: Can we do something on error here ? + util::write_n(err_wr_pipe_, err_msg.c_str(), err_msg.length()); + } + + // Calling application would not get this + // exit failure + _exit (EXIT_FAILURE); +#endif + } + + + inline void Streams::setup_comm_channels() + { +#ifdef __USING_WINDOWS__ + util::configure_pipe(&this->g_hChildStd_IN_Rd, &this->g_hChildStd_IN_Wr, &this->g_hChildStd_IN_Wr); + this->input(util::file_from_handle(this->g_hChildStd_IN_Wr, "w")); + this->write_to_child_ = _fileno(this->input()); + + util::configure_pipe(&this->g_hChildStd_OUT_Rd, &this->g_hChildStd_OUT_Wr, &this->g_hChildStd_OUT_Rd); + this->output(util::file_from_handle(this->g_hChildStd_OUT_Rd, "r")); + this->read_from_child_ = _fileno(this->output()); + + util::configure_pipe(&this->g_hChildStd_ERR_Rd, &this->g_hChildStd_ERR_Wr, &this->g_hChildStd_ERR_Rd); + this->error(util::file_from_handle(this->g_hChildStd_ERR_Rd, "r")); + this->err_read_ = _fileno(this->error()); +#else + + if (write_to_child_ != -1) input(fdopen(write_to_child_, "wb")); + if (read_from_child_ != -1) output(fdopen(read_from_child_, "rb")); + if (err_read_ != -1) error(fdopen(err_read_, "rb")); + + auto handles = {input(), output(), error()}; + + for (auto& h : handles) { + if (h == nullptr) continue; + switch (bufsiz_) { + case 0: + setvbuf(h, nullptr, _IONBF, BUFSIZ); + break; + case 1: + setvbuf(h, nullptr, _IONBF, BUFSIZ); + break; + default: + setvbuf(h, nullptr, _IOFBF, bufsiz_); + }; + } + #endif + } + + inline int Communication::send(const char* msg, size_t length) + { + if (stream_->input() == nullptr) return -1; + return std::fwrite(msg, sizeof(char), length, stream_->input()); + } + + inline int Communication::send(const std::vector<char>& msg) + { + return send(msg.data(), msg.size()); + } + + inline std::pair<OutBuffer, ErrBuffer> + Communication::communicate(const char* msg, size_t length) + { + // Optimization from subprocess.py + // If we are using one pipe, or no pipe + // at all, using select() or threads is unnecessary. + auto hndls = {stream_->input(), stream_->output(), stream_->error()}; + int count = std::count(std::begin(hndls), std::end(hndls), nullptr); + const int len_conv = length; + + if (count >= 2) { + OutBuffer obuf; + ErrBuffer ebuf; + if (stream_->input()) { + if (msg) { + int wbytes = std::fwrite(msg, sizeof(char), length, stream_->input()); + if (wbytes < len_conv) { + if (errno != EPIPE && errno != EINVAL) { + throw OSError("fwrite error", errno); + } + } + } + // Close the input stream + stream_->input_.reset(); + } else if (stream_->output()) { + // Read till EOF + // ATTN: This could be blocking, if the process + // at the other end screws up, we get screwed as well + obuf.add_cap(out_buf_cap_); + + int rbytes = util::read_all( + stream_->output(), + obuf.buf); + + if (rbytes == -1) { + throw OSError("read to obuf failed", errno); + } + + obuf.length = rbytes; + // Close the output stream + stream_->output_.reset(); + + } else if (stream_->error()) { + // Same screwness applies here as well + ebuf.add_cap(err_buf_cap_); + + int rbytes = util::read_atmost_n( + stream_->error(), + ebuf.buf.data(), + ebuf.buf.size()); + + if (rbytes == -1) { + throw OSError("read to ebuf failed", errno); + } + + ebuf.length = rbytes; + // Close the error stream + stream_->error_.reset(); + } + return std::make_pair(std::move(obuf), std::move(ebuf)); + } + + return communicate_threaded(msg, length); + } + + + inline std::pair<OutBuffer, ErrBuffer> + Communication::communicate_threaded(const char* msg, size_t length) + { + OutBuffer obuf; + ErrBuffer ebuf; + std::future<int> out_fut, err_fut; + const int length_conv = length; + + if (stream_->output()) { + obuf.add_cap(out_buf_cap_); + + out_fut = std::async(std::launch::async, + [&obuf, this] { + return util::read_all(this->stream_->output(), obuf.buf); + }); + } + if (stream_->error()) { + ebuf.add_cap(err_buf_cap_); + + err_fut = std::async(std::launch::async, + [&ebuf, this] { + return util::read_all(this->stream_->error(), ebuf.buf); + }); + } + if (stream_->input()) { + if (msg) { + int wbytes = std::fwrite(msg, sizeof(char), length, stream_->input()); + if (wbytes < length_conv) { + if (errno != EPIPE && errno != EINVAL) { + throw OSError("fwrite error", errno); + } + } + } + stream_->input_.reset(); + } + + if (out_fut.valid()) { + int res = out_fut.get(); + if (res != -1) obuf.length = res; + else obuf.length = 0; + } + if (err_fut.valid()) { + int res = err_fut.get(); + if (res != -1) ebuf.length = res; + else ebuf.length = 0; + } + + return std::make_pair(std::move(obuf), std::move(ebuf)); + } + +} // end namespace detail + +} + +#endif // SUBPROCESS_HPP diff --git a/src/util/time.cpp b/src/util/time.cpp index 5ca9d21f8d..456662bd84 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -3,70 +3,21 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif +#include <util/time.h> #include <compat/compat.h> #include <tinyformat.h> -#include <util/time.h> #include <util/check.h> #include <atomic> #include <chrono> -#include <ctime> -#include <locale> -#include <thread> -#include <sstream> #include <string> +#include <thread> void UninterruptibleSleep(const std::chrono::microseconds& n) { std::this_thread::sleep_for(n); } static std::atomic<int64_t> nMockTime(0); //!< For testing -bool ChronoSanityCheck() -{ - // std::chrono::system_clock.time_since_epoch and time_t(0) are not guaranteed - // to use the Unix epoch timestamp, prior to C++20, but in practice they almost - // certainly will. Any differing behavior will be assumed to be an error, unless - // certain platforms prove to consistently deviate, at which point we'll cope - // with it by adding offsets. - - // Create a new clock from time_t(0) and make sure that it represents 0 - // seconds from the system_clock's time_since_epoch. Then convert that back - // to a time_t and verify that it's the same as before. - const time_t time_t_epoch{}; - auto clock = std::chrono::system_clock::from_time_t(time_t_epoch); - if (std::chrono::duration_cast<std::chrono::seconds>(clock.time_since_epoch()).count() != 0) { - return false; - } - - time_t time_val = std::chrono::system_clock::to_time_t(clock); - if (time_val != time_t_epoch) { - return false; - } - - // Check that the above zero time is actually equal to the known unix timestamp. - struct tm epoch; -#ifdef HAVE_GMTIME_R - if (gmtime_r(&time_val, &epoch) == nullptr) { -#else - if (gmtime_s(&epoch, &time_val) != 0) { -#endif - return false; - } - - if ((epoch.tm_sec != 0) || - (epoch.tm_min != 0) || - (epoch.tm_hour != 0) || - (epoch.tm_mday != 1) || - (epoch.tm_mon != 0) || - (epoch.tm_year != 70)) { - return false; - } - return true; -} - NodeClock::time_point NodeClock::now() noexcept { const std::chrono::seconds mocktime{nMockTime.load(std::memory_order_relaxed)}; @@ -96,30 +47,21 @@ std::chrono::seconds GetMockTime() int64_t GetTime() { return GetTime<std::chrono::seconds>().count(); } -std::string FormatISO8601DateTime(int64_t nTime) { - struct tm ts; - time_t time_val = nTime; -#ifdef HAVE_GMTIME_R - if (gmtime_r(&time_val, &ts) == nullptr) { -#else - if (gmtime_s(&ts, &time_val) != 0) { -#endif - return {}; - } - return strprintf("%04i-%02i-%02iT%02i:%02i:%02iZ", ts.tm_year + 1900, ts.tm_mon + 1, ts.tm_mday, ts.tm_hour, ts.tm_min, ts.tm_sec); +std::string FormatISO8601DateTime(int64_t nTime) +{ + const std::chrono::sys_seconds secs{std::chrono::seconds{nTime}}; + const auto days{std::chrono::floor<std::chrono::days>(secs)}; + const std::chrono::year_month_day ymd{days}; + const std::chrono::hh_mm_ss hms{secs - days}; + return strprintf("%04i-%02u-%02uT%02i:%02i:%02iZ", signed{ymd.year()}, unsigned{ymd.month()}, unsigned{ymd.day()}, hms.hours().count(), hms.minutes().count(), hms.seconds().count()); } -std::string FormatISO8601Date(int64_t nTime) { - struct tm ts; - time_t time_val = nTime; -#ifdef HAVE_GMTIME_R - if (gmtime_r(&time_val, &ts) == nullptr) { -#else - if (gmtime_s(&ts, &time_val) != 0) { -#endif - return {}; - } - return strprintf("%04i-%02i-%02i", ts.tm_year + 1900, ts.tm_mon + 1, ts.tm_mday); +std::string FormatISO8601Date(int64_t nTime) +{ + const std::chrono::sys_seconds secs{std::chrono::seconds{nTime}}; + const auto days{std::chrono::floor<std::chrono::days>(secs)}; + const std::chrono::year_month_day ymd{days}; + return strprintf("%04i-%02u-%02u", signed{ymd.year()}, unsigned{ymd.month()}, unsigned{ymd.day()}); } struct timeval MillisToTimeval(int64_t nTimeout) diff --git a/src/util/time.h b/src/util/time.h index 6aa776137c..108560e0e0 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -116,7 +116,4 @@ struct timeval MillisToTimeval(int64_t nTimeout); */ struct timeval MillisToTimeval(std::chrono::milliseconds ms); -/** Sanity check epoch match normal Unix epoch */ -bool ChronoSanityCheck(); - #endif // BITCOIN_UTIL_TIME_H diff --git a/src/validation.cpp b/src/validation.cpp index b6d0c38f39..903f9caf13 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -511,7 +511,7 @@ public: /** Parameters for child-with-unconfirmed-parents package validation. */ static ATMPArgs PackageChildWithParents(const CChainParams& chainparams, int64_t accept_time, - std::vector<COutPoint>& coins_to_uncache, std::optional<CFeeRate>& client_maxfeerate) { + std::vector<COutPoint>& coins_to_uncache, const std::optional<CFeeRate>& client_maxfeerate) { return ATMPArgs{/* m_chainparams */ chainparams, /* m_accept_time */ accept_time, /* m_bypass_limits */ false, @@ -1366,7 +1366,9 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: // Individual modified feerate exceeded caller-defined max; abort // N.B. this doesn't take into account CPFPs. Chunk-aware validation may be more robust. if (args.m_client_maxfeerate && CFeeRate(ws.m_modified_fees, ws.m_vsize) > args.m_client_maxfeerate.value()) { - package_state.Invalid(PackageValidationResult::PCKG_TX, "max feerate exceeded"); + // Need to set failure here both individually and at package level + ws.m_state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "max feerate exceeded", ""); + package_state.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); // Exit early to avoid doing pointless work. Update the failed tx result; the rest are unfinished. results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); return PackageMempoolAcceptResult(package_state, std::move(results)); @@ -1716,7 +1718,7 @@ MempoolAcceptResult AcceptToMemoryPool(Chainstate& active_chainstate, const CTra } PackageMempoolAcceptResult ProcessNewPackage(Chainstate& active_chainstate, CTxMemPool& pool, - const Package& package, bool test_accept, std::optional<CFeeRate> client_maxfeerate) + const Package& package, bool test_accept, const std::optional<CFeeRate>& client_maxfeerate) { AssertLockHeld(cs_main); assert(!package.empty()); @@ -2051,10 +2053,10 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, return true; } -bool FatalError(Notifications& notifications, BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage) +bool FatalError(Notifications& notifications, BlockValidationState& state, const bilingual_str& message) { - notifications.fatalError(strMessage, userMessage); - return state.Error(strMessage); + notifications.fatalError(message); + return state.Error(message.original); } /** @@ -2276,7 +2278,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // We don't write down blocks to disk if they may have been // corrupted, so this should be impossible unless we're having hardware // problems. - return FatalError(m_chainman.GetNotifications(), state, "Corrupt block found indicating potential hardware failure; shutting down"); + return FatalError(m_chainman.GetNotifications(), state, _("Corrupt block found indicating potential hardware failure.")); } LogError("%s: Consensus::CheckBlock: %s\n", __func__, state.ToString()); return false; @@ -2702,7 +2704,7 @@ bool Chainstate::FlushStateToDisk( if (fDoFullFlush || fPeriodicWrite) { // Ensure we can write block index if (!CheckDiskSpace(m_blockman.m_opts.blocks_dir)) { - return FatalError(m_chainman.GetNotifications(), state, "Disk space is too low!", _("Disk space is too low!")); + return FatalError(m_chainman.GetNotifications(), state, _("Disk space is too low!")); } { LOG_TIME_MILLIS_WITH_CATEGORY("write block and undo data to disk", BCLog::BENCH); @@ -2720,7 +2722,7 @@ bool Chainstate::FlushStateToDisk( LOG_TIME_MILLIS_WITH_CATEGORY("write block index to disk", BCLog::BENCH); if (!m_blockman.WriteBlockIndexDB()) { - return FatalError(m_chainman.GetNotifications(), state, "Failed to write to block index database"); + return FatalError(m_chainman.GetNotifications(), state, _("Failed to write to block index database.")); } } // Finally remove any pruned files @@ -2742,11 +2744,11 @@ bool Chainstate::FlushStateToDisk( // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. if (!CheckDiskSpace(m_chainman.m_options.datadir, 48 * 2 * 2 * CoinsTip().GetCacheSize())) { - return FatalError(m_chainman.GetNotifications(), state, "Disk space is too low!", _("Disk space is too low!")); + return FatalError(m_chainman.GetNotifications(), state, _("Disk space is too low!")); } // Flush the chainstate (which may refer to block index entries). if (!CoinsTip().Flush()) - return FatalError(m_chainman.GetNotifications(), state, "Failed to write to coin database"); + return FatalError(m_chainman.GetNotifications(), state, _("Failed to write to coin database.")); m_last_flush = nNow; full_flush_completed = true; TRACE5(utxocache, flush, @@ -2762,7 +2764,7 @@ bool Chainstate::FlushStateToDisk( m_chainman.m_options.signals->ChainStateFlushed(this->GetRole(), m_chain.GetLocator()); } } catch (const std::runtime_error& e) { - return FatalError(m_chainman.GetNotifications(), state, std::string("System error while flushing: ") + e.what()); + return FatalError(m_chainman.GetNotifications(), state, strprintf(_("System error while flushing: %s"), e.what())); } return true; } @@ -2998,7 +3000,7 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, if (!pblock) { std::shared_ptr<CBlock> pblockNew = std::make_shared<CBlock>(); if (!m_blockman.ReadBlockFromDisk(*pblockNew, *pindexNew)) { - return FatalError(m_chainman.GetNotifications(), state, "Failed to read block"); + return FatalError(m_chainman.GetNotifications(), state, _("Failed to read block.")); } pthisBlock = pblockNew; } else { @@ -3185,7 +3187,7 @@ bool Chainstate::ActivateBestChainStep(BlockValidationState& state, CBlockIndex* // If we're unable to disconnect a block during normal operation, // then that is a failure of our local system -- we should abort // rather than stay on a less work chain. - FatalError(m_chainman.GetNotifications(), state, "Failed to disconnect block; see debug.log for details"); + FatalError(m_chainman.GetNotifications(), state, _("Failed to disconnect block.")); return false; } fBlocksDisconnected = true; @@ -4234,9 +4236,10 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& if (NotifyHeaderTip(*this)) { if (IsInitialBlockDownload() && ppindex && *ppindex) { const CBlockIndex& last_accepted{**ppindex}; - const int64_t blocks_left{(GetTime() - last_accepted.GetBlockTime()) / GetConsensus().nPowTargetSpacing}; + int64_t blocks_left{(NodeClock::now() - last_accepted.Time()) / GetConsensus().PowTargetSpacing()}; + blocks_left = std::max<int64_t>(0, blocks_left); const double progress{100.0 * last_accepted.nHeight / (last_accepted.nHeight + blocks_left)}; - LogPrintf("Synchronizing blockheaders, height: %d (~%.2f%%)\n", last_accepted.nHeight, progress); + LogInfo("Synchronizing blockheaders, height: %d (~%.2f%%)\n", last_accepted.nHeight, progress); } } return true; @@ -4260,9 +4263,10 @@ void ChainstateManager::ReportHeadersPresync(const arith_uint256& work, int64_t bool initial_download = IsInitialBlockDownload(); GetNotifications().headerTip(GetSynchronizationState(initial_download), height, timestamp, /*presync=*/true); if (initial_download) { - const int64_t blocks_left{(GetTime() - timestamp) / GetConsensus().nPowTargetSpacing}; + int64_t blocks_left{(NodeClock::now() - NodeSeconds{std::chrono::seconds{timestamp}}) / GetConsensus().PowTargetSpacing()}; + blocks_left = std::max<int64_t>(0, blocks_left); const double progress{100.0 * height / (height + blocks_left)}; - LogPrintf("Pre-synchronizing blockheaders, height: %d (~%.2f%%)\n", height, progress); + LogInfo("Pre-synchronizing blockheaders, height: %d (~%.2f%%)\n", height, progress); } } @@ -4345,7 +4349,7 @@ bool ChainstateManager::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, } ReceivedBlockTransactions(block, pindex, blockPos); } catch (const std::runtime_error& e) { - return FatalError(GetNotifications(), state, std::string("System error: ") + e.what()); + return FatalError(GetNotifications(), state, strprintf(_("System error while saving block to disk: %s"), e.what())); } // TODO: FlushStateToDisk() handles flushing of both block and chainstate @@ -5029,7 +5033,7 @@ void ChainstateManager::LoadExternalBlockFile( } } } catch (const std::runtime_error& e) { - GetNotifications().fatalError(std::string("System error: ") + e.what()); + GetNotifications().fatalError(strprintf(_("System error while loading external block file: %s"), e.what())); } LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, Ticks<std::chrono::milliseconds>(SteadyClock::now() - start)); } @@ -5539,8 +5543,8 @@ bool ChainstateManager::ActivateSnapshot( snapshot_chainstate.reset(); bool removed = DeleteCoinsDBFromDisk(*snapshot_datadir, /*is_snapshot=*/true); if (!removed) { - GetNotifications().fatalError(strprintf("Failed to remove snapshot chainstate dir (%s). " - "Manually remove it before restarting.\n", fs::PathToString(*snapshot_datadir))); + GetNotifications().fatalError(strprintf(_("Failed to remove snapshot chainstate dir (%s). " + "Manually remove it before restarting.\n"), fs::PathToString(*snapshot_datadir))); } } return false; @@ -5782,7 +5786,7 @@ bool ChainstateManager::PopulateAndValidateSnapshot( CBlockIndex* index = nullptr; // Don't make any modifications to the genesis block since it shouldn't be - // neccessary, and since the genesis block doesn't have normal flags like + // necessary, and since the genesis block doesn't have normal flags like // BLOCK_VALID_SCRIPTS set. constexpr int AFTER_GENESIS_START{1}; @@ -5879,7 +5883,7 @@ SnapshotCompletionResult ChainstateManager::MaybeCompleteSnapshotValidation() user_error = strprintf(Untranslated("%s\n%s"), user_error, util::ErrorString(rename_result)); } - GetNotifications().fatalError(user_error.original, user_error); + GetNotifications().fatalError(user_error); }; if (index_new.GetBlockHash() != snapshot_blockhash) { @@ -6138,13 +6142,14 @@ bool ChainstateManager::DeleteSnapshotChainstate() Assert(m_snapshot_chainstate); Assert(m_ibd_chainstate); - fs::path snapshot_datadir = GetSnapshotCoinsDBPath(*m_snapshot_chainstate); + fs::path snapshot_datadir = Assert(node::FindSnapshotChainstateDir(m_options.datadir)).value(); if (!DeleteCoinsDBFromDisk(snapshot_datadir, /*is_snapshot=*/ true)) { LogPrintf("Deletion of %s failed. Please remove it manually to continue reindexing.\n", fs::PathToString(snapshot_datadir)); return false; } m_active_chainstate = m_ibd_chainstate.get(); + m_active_chainstate->m_mempool = m_snapshot_chainstate->m_mempool; m_snapshot_chainstate.reset(); return true; } @@ -6220,9 +6225,9 @@ bool ChainstateManager::ValidatedSnapshotCleanup() const fs::filesystem_error& err) { LogPrintf("Error renaming path (%s) -> (%s): %s\n", fs::PathToString(p_old), fs::PathToString(p_new), err.what()); - GetNotifications().fatalError(strprintf( + GetNotifications().fatalError(strprintf(_( "Rename of '%s' -> '%s' failed. " - "Cannot clean up the background chainstate leveldb directory.", + "Cannot clean up the background chainstate leveldb directory."), fs::PathToString(p_old), fs::PathToString(p_new))); }; diff --git a/src/validation.h b/src/validation.h index bcf153719a..e3b2a2d59b 100644 --- a/src/validation.h +++ b/src/validation.h @@ -93,7 +93,7 @@ extern const std::vector<std::string> CHECKLEVEL_DOC; CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); -bool FatalError(kernel::Notifications& notifications, BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = {}); +bool FatalError(kernel::Notifications& notifications, BlockValidationState& state, const bilingual_str& message); /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pindex); @@ -275,14 +275,14 @@ MempoolAcceptResult AcceptToMemoryPool(Chainstate& active_chainstate, const CTra * Validate (and maybe submit) a package to the mempool. See doc/policy/packages.md for full details * on package validation rules. * @param[in] test_accept When true, run validation checks but don't submit to mempool. -* @param[in] max_sane_feerate If exceeded by an individual transaction, rest of (sub)package evalution is aborted. +* @param[in] client_maxfeerate If exceeded by an individual transaction, rest of (sub)package evaluation is aborted. * Only for sanity checks against local submission of transactions. * @returns a PackageMempoolAcceptResult which includes a MempoolAcceptResult for each transaction. * If a transaction fails, validation will exit early and some results may be missing. It is also * possible for the package to be partially submitted. */ PackageMempoolAcceptResult ProcessNewPackage(Chainstate& active_chainstate, CTxMemPool& pool, - const Package& txns, bool test_accept, std::optional<CFeeRate> max_sane_feerate) + const Package& txns, bool test_accept, const std::optional<CFeeRate>& client_maxfeerate) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /* Mempool validation helper functions */ @@ -478,7 +478,7 @@ enum class CoinsCacheSizeState * current best chain. * * Eventually, the API here is targeted at being exposed externally as a - * consumable libconsensus library, so any functions added must only call + * consumable library, so any functions added must only call * other class member functions, pure functions in other parts of the consensus * library, callbacks via the validation interface, or read/write-to-disk * functions (eventually this will also be via callbacks). diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index d15273dfc9..a866666f64 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -92,7 +92,7 @@ WalletTxStatus MakeWalletTxStatus(const CWallet& wallet, const CWalletTx& wtx) WalletTxStatus result; result.block_height = wtx.state<TxStateConfirmed>() ? wtx.state<TxStateConfirmed>()->confirmed_block_height : - wtx.state<TxStateConflicted>() ? wtx.state<TxStateConflicted>()->conflicting_block_height : + wtx.state<TxStateBlockConflicted>() ? wtx.state<TxStateBlockConflicted>()->conflicting_block_height : std::numeric_limits<int>::max(); result.blocks_to_maturity = wallet.GetTxBlocksToMaturity(wtx); result.depth_in_main_chain = wallet.GetTxDepthInMainChain(wtx); @@ -101,7 +101,7 @@ WalletTxStatus MakeWalletTxStatus(const CWallet& wallet, const CWalletTx& wtx) result.is_trusted = CachedTxIsTrusted(wallet, wtx); result.is_abandoned = wtx.isAbandoned(); result.is_coinbase = wtx.IsCoinBase(); - result.is_in_main_chain = wallet.IsTxInMainChain(wtx); + result.is_in_main_chain = wtx.isConfirmed(); return result; } @@ -286,7 +286,7 @@ public: if (!res) return util::Error{util::ErrorString(res)}; const auto& txr = *res; fee = txr.fee; - change_pos = txr.change_pos ? *txr.change_pos : -1; + change_pos = txr.change_pos ? int(*txr.change_pos) : -1; return txr.tx; } diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index b9d8d9abc9..c164266f80 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -149,7 +149,7 @@ CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, c { AssertLockHeld(wallet.cs_wallet); - if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { + if (wallet.IsTxImmatureCoinBase(wtx) && wtx.isConfirmed()) { return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, filter); } @@ -253,12 +253,12 @@ bool CachedTxIsFromMe(const CWallet& wallet, const CWalletTx& wtx, const isminef return (CachedTxGetDebit(wallet, wtx, filter) > 0); } +// NOLINTNEXTLINE(misc-no-recursion) bool CachedTxIsTrusted(const CWallet& wallet, const CWalletTx& wtx, std::set<uint256>& trusted_parents) { AssertLockHeld(wallet.cs_wallet); - int nDepth = wallet.GetTxDepthInMainChain(wtx); - if (nDepth >= 1) return true; - if (nDepth < 0) return false; + if (wtx.isConfirmed()) return true; + if (wtx.isBlockConflicted()) return false; // using wtx's cached debit if (!wallet.m_spend_zero_conf_change || !CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)) return false; diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index acaa2d8b15..7f068c05ef 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -395,6 +395,7 @@ class DescribeWalletAddressVisitor public: const SigningProvider * const provider; + // NOLINTNEXTLINE(misc-no-recursion) void ProcessSubScript(const CScript& subscript, UniValue& obj) const { // Always present: script type and redeemscript @@ -445,6 +446,7 @@ public: return obj; } + // NOLINTNEXTLINE(misc-no-recursion) UniValue operator()(const ScriptHash& scripthash) const { UniValue obj(UniValue::VOBJ); @@ -465,6 +467,7 @@ public: return obj; } + // NOLINTNEXTLINE(misc-no-recursion) UniValue operator()(const WitnessV0ScriptHash& id) const { UniValue obj(UniValue::VOBJ); diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 5167e986b1..ae2dfe5795 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -854,6 +854,7 @@ enum class ScriptContext // Analyse the provided scriptPubKey, determining which keys and which redeem scripts from the ImportData struct are needed to spend it, and mark them as used. // Returns an error string, or the empty string for success. +// NOLINTNEXTLINE(misc-no-recursion) static std::string RecurseImportData(const CScript& script, ImportData& import_data, const ScriptContext script_ctx) { // Use Solver to obtain script type and parsed pubkeys or hashes: diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index e6c021d426..05b340995d 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -40,6 +40,10 @@ static void WalletTxToJSON(const CWallet& wallet, const CWalletTx& wtx, UniValue for (const uint256& conflict : wallet.GetTxConflicts(wtx)) conflicts.push_back(conflict.GetHex()); entry.pushKV("walletconflicts", conflicts); + UniValue mempool_conflicts(UniValue::VARR); + for (const Txid& mempool_conflict : wtx.mempool_conflicts) + mempool_conflicts.push_back(mempool_conflict.GetHex()); + entry.pushKV("mempoolconflicts", mempool_conflicts); entry.pushKV("time", wtx.GetTxTime()); entry.pushKV("timereceived", int64_t{wtx.nTimeReceived}); @@ -417,6 +421,10 @@ static std::vector<RPCResult> TransactionDescriptionString() }}, {RPCResult::Type::STR_HEX, "replaced_by_txid", /*optional=*/true, "Only if 'category' is 'send'. The txid if this tx was replaced."}, {RPCResult::Type::STR_HEX, "replaces_txid", /*optional=*/true, "Only if 'category' is 'send'. The txid if this tx replaces another."}, + {RPCResult::Type::ARR, "mempoolconflicts", "Transactions that directly conflict with either this transaction or an ancestor transaction", + { + {RPCResult::Type::STR_HEX, "txid", "The transaction id."}, + }}, {RPCResult::Type::STR, "to", /*optional=*/true, "If a comment to is associated with the transaction."}, {RPCResult::Type::NUM_TIME, "time", "The transaction time expressed in " + UNIX_EPOCH_TIME + "."}, {RPCResult::Type::NUM_TIME, "timereceived", "The time received expressed in " + UNIX_EPOCH_TIME + "."}, diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index 6a8ce954fb..a684d4e191 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -817,6 +817,217 @@ static RPCHelpMan migratewallet() }; } +RPCHelpMan gethdkeys() +{ + return RPCHelpMan{ + "gethdkeys", + "\nList all BIP 32 HD keys in the wallet and which descriptors use them.\n", + { + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { + {"active_only", RPCArg::Type::BOOL, RPCArg::Default{false}, "Show the keys for only active descriptors"}, + {"private", RPCArg::Type::BOOL, RPCArg::Default{false}, "Show private keys"} + }}, + }, + RPCResult{RPCResult::Type::ARR, "", "", { + { + {RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "xpub", "The extended public key"}, + {RPCResult::Type::BOOL, "has_private", "Whether the wallet has the private key for this xpub"}, + {RPCResult::Type::STR, "xprv", /*optional=*/true, "The extended private key if \"private\" is true"}, + {RPCResult::Type::ARR, "descriptors", "Array of descriptor objects that use this HD key", + { + {RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "desc", "Descriptor string representation"}, + {RPCResult::Type::BOOL, "active", "Whether this descriptor is currently used to generate new addresses"}, + }}, + }}, + }}, + } + }}, + RPCExamples{ + HelpExampleCli("gethdkeys", "") + HelpExampleRpc("gethdkeys", "") + + HelpExampleCliNamed("gethdkeys", {{"active_only", "true"}, {"private", "true"}}) + HelpExampleRpcNamed("gethdkeys", {{"active_only", "true"}, {"private", "true"}}) + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + const std::shared_ptr<const CWallet> wallet = GetWalletForJSONRPCRequest(request); + if (!wallet) return UniValue::VNULL; + + if (!wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + throw JSONRPCError(RPC_WALLET_ERROR, "gethdkeys is not available for non-descriptor wallets"); + } + + LOCK(wallet->cs_wallet); + + UniValue options{request.params[0].isNull() ? UniValue::VOBJ : request.params[0]}; + const bool active_only{options.exists("active_only") ? options["active_only"].get_bool() : false}; + const bool priv{options.exists("private") ? options["private"].get_bool() : false}; + if (priv) { + EnsureWalletIsUnlocked(*wallet); + } + + + std::set<ScriptPubKeyMan*> spkms; + if (active_only) { + spkms = wallet->GetActiveScriptPubKeyMans(); + } else { + spkms = wallet->GetAllScriptPubKeyMans(); + } + + std::map<CExtPubKey, std::set<std::tuple<std::string, bool, bool>>> wallet_xpubs; + std::map<CExtPubKey, CExtKey> wallet_xprvs; + for (auto* spkm : spkms) { + auto* desc_spkm{dynamic_cast<DescriptorScriptPubKeyMan*>(spkm)}; + CHECK_NONFATAL(desc_spkm); + LOCK(desc_spkm->cs_desc_man); + WalletDescriptor w_desc = desc_spkm->GetWalletDescriptor(); + + // Retrieve the pubkeys from the descriptor + std::set<CPubKey> desc_pubkeys; + std::set<CExtPubKey> desc_xpubs; + w_desc.descriptor->GetPubKeys(desc_pubkeys, desc_xpubs); + for (const CExtPubKey& xpub : desc_xpubs) { + std::string desc_str; + bool ok = desc_spkm->GetDescriptorString(desc_str, false); + CHECK_NONFATAL(ok); + wallet_xpubs[xpub].emplace(desc_str, wallet->IsActiveScriptPubKeyMan(*spkm), desc_spkm->HasPrivKey(xpub.pubkey.GetID())); + if (std::optional<CKey> key = priv ? desc_spkm->GetKey(xpub.pubkey.GetID()) : std::nullopt) { + wallet_xprvs[xpub] = CExtKey(xpub, *key); + } + } + } + + UniValue response(UniValue::VARR); + for (const auto& [xpub, descs] : wallet_xpubs) { + bool has_xprv = false; + UniValue descriptors(UniValue::VARR); + for (const auto& [desc, active, has_priv] : descs) { + UniValue d(UniValue::VOBJ); + d.pushKV("desc", desc); + d.pushKV("active", active); + has_xprv |= has_priv; + + descriptors.push_back(std::move(d)); + } + UniValue xpub_info(UniValue::VOBJ); + xpub_info.pushKV("xpub", EncodeExtPubKey(xpub)); + xpub_info.pushKV("has_private", has_xprv); + if (priv) { + xpub_info.pushKV("xprv", EncodeExtKey(wallet_xprvs.at(xpub))); + } + xpub_info.pushKV("descriptors", std::move(descriptors)); + + response.push_back(std::move(xpub_info)); + } + + return response; + }, + }; +} + +static RPCHelpMan createwalletdescriptor() +{ + return RPCHelpMan{"createwalletdescriptor", + "Creates the wallet's descriptor for the given address type. " + "The address type must be one that the wallet does not already have a descriptor for." + + HELP_REQUIRING_PASSPHRASE, + { + {"type", RPCArg::Type::STR, RPCArg::Optional::NO, "The address type the descriptor will produce. Options are \"legacy\", \"p2sh-segwit\", \"bech32\", and \"bech32m\"."}, + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { + {"internal", RPCArg::Type::BOOL, RPCArg::DefaultHint{"Both external and internal will be generated unless this parameter is specified"}, "Whether to only make one descriptor that is internal (if parameter is true) or external (if parameter is false)"}, + {"hdkey", RPCArg::Type::STR, RPCArg::DefaultHint{"The HD key used by all other active descriptors"}, "The HD key that the wallet knows the private key of, listed using 'gethdkeys', to use for this descriptor's key"}, + }}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::ARR, "descs", "The public descriptors that were added to the wallet", + {{RPCResult::Type::STR, "", ""}} + } + }, + }, + RPCExamples{ + HelpExampleCli("createwalletdescriptor", "bech32m") + + HelpExampleRpc("createwalletdescriptor", "bech32m") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + if (!pwallet) return UniValue::VNULL; + + // Make sure wallet is a descriptor wallet + if (!pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + throw JSONRPCError(RPC_WALLET_ERROR, "createwalletdescriptor is not available for non-descriptor wallets"); + } + + std::optional<OutputType> output_type = ParseOutputType(request.params[0].get_str()); + if (!output_type) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[0].get_str())); + } + + UniValue options{request.params[1].isNull() ? UniValue::VOBJ : request.params[1]}; + UniValue internal_only{options["internal"]}; + UniValue hdkey{options["hdkey"]}; + + std::vector<bool> internals; + if (internal_only.isNull()) { + internals.push_back(false); + internals.push_back(true); + } else { + internals.push_back(internal_only.get_bool()); + } + + LOCK(pwallet->cs_wallet); + EnsureWalletIsUnlocked(*pwallet); + + CExtPubKey xpub; + if (hdkey.isNull()) { + std::set<CExtPubKey> active_xpubs = pwallet->GetActiveHDPubKeys(); + if (active_xpubs.size() != 1) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unable to determine which HD key to use from active descriptors. Please specify with 'hdkey'"); + } + xpub = *active_xpubs.begin(); + } else { + xpub = DecodeExtPubKey(hdkey.get_str()); + if (!xpub.pubkey.IsValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unable to parse HD key. Please provide a valid xpub"); + } + } + + std::optional<CKey> key = pwallet->GetKey(xpub.pubkey.GetID()); + if (!key) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Private key for %s is not known", EncodeExtPubKey(xpub))); + } + CExtKey active_hdkey(xpub, *key); + + std::vector<std::reference_wrapper<DescriptorScriptPubKeyMan>> spkms; + WalletBatch batch{pwallet->GetDatabase()}; + for (bool internal : internals) { + WalletDescriptor w_desc = GenerateWalletDescriptor(xpub, *output_type, internal); + uint256 w_id = DescriptorID(*w_desc.descriptor); + if (!pwallet->GetScriptPubKeyMan(w_id)) { + spkms.emplace_back(pwallet->SetupDescriptorScriptPubKeyMan(batch, active_hdkey, *output_type, internal)); + } + } + if (spkms.empty()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Descriptor already exists"); + } + + // Fetch each descspkm from the wallet in order to get the descriptor strings + UniValue descs{UniValue::VARR}; + for (const auto& spkm : spkms) { + std::string desc_str; + bool ok = spkm.get().GetDescriptorString(desc_str, false); + CHECK_NONFATAL(ok); + descs.push_back(desc_str); + } + UniValue out{UniValue::VOBJ}; + out.pushKV("descs", std::move(descs)); + return out; + } + }; +} + // addresses RPCHelpMan getaddressinfo(); RPCHelpMan getnewaddress(); @@ -900,6 +1111,7 @@ Span<const CRPCCommand> GetWalletRPCCommands() {"wallet", &bumpfee}, {"wallet", &psbtbumpfee}, {"wallet", &createwallet}, + {"wallet", &createwalletdescriptor}, {"wallet", &restorewallet}, {"wallet", &dumpprivkey}, {"wallet", &dumpwallet}, @@ -907,6 +1119,7 @@ Span<const CRPCCommand> GetWalletRPCCommands() {"wallet", &getaddressesbylabel}, {"wallet", &getaddressinfo}, {"wallet", &getbalance}, + {"wallet", &gethdkeys}, {"wallet", &getnewaddress}, {"wallet", &getrawchangeaddress}, {"wallet", &getreceivedbyaddress}, diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index e10a17f003..b42275fe4b 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -11,6 +11,7 @@ #include <script/sign.h> #include <script/solver.h> #include <util/bip32.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> @@ -96,6 +97,7 @@ bool HaveKeys(const std::vector<valtype>& pubkeys, const LegacyScriptPubKeyMan& //! @param recurse_scripthash whether to recurse into nested p2sh and p2wsh //! scripts or simply treat any script that has been //! stored in the keystore as spendable +// NOLINTNEXTLINE(misc-no-recursion) IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& scriptPubKey, IsMineSigVersion sigversion, bool recurse_scripthash=true) { IsMineResult ret = IsMineResult::NO; @@ -2143,6 +2145,36 @@ std::map<CKeyID, CKey> DescriptorScriptPubKeyMan::GetKeys() const return m_map_keys; } +bool DescriptorScriptPubKeyMan::HasPrivKey(const CKeyID& keyid) const +{ + AssertLockHeld(cs_desc_man); + return m_map_keys.contains(keyid) || m_map_crypted_keys.contains(keyid); +} + +std::optional<CKey> DescriptorScriptPubKeyMan::GetKey(const CKeyID& keyid) const +{ + AssertLockHeld(cs_desc_man); + if (m_storage.HasEncryptionKeys() && !m_storage.IsLocked()) { + const auto& it = m_map_crypted_keys.find(keyid); + if (it == m_map_crypted_keys.end()) { + return std::nullopt; + } + const std::vector<unsigned char>& crypted_secret = it->second.second; + CKey key; + if (!Assume(m_storage.WithEncryptionKey([&](const CKeyingMaterial& encryption_key) { + return DecryptKey(encryption_key, crypted_secret, it->second.first, key); + }))) { + return std::nullopt; + } + return key; + } + const auto& it = m_map_keys.find(keyid); + if (it == m_map_keys.end()) { + return std::nullopt; + } + return it->second; +} + bool DescriptorScriptPubKeyMan::TopUp(unsigned int size) { WalletBatch batch(m_storage.GetDatabase()); @@ -2296,55 +2328,7 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(WalletBatch& batch, co return false; } - int64_t creation_time = GetTime(); - - std::string xpub = EncodeExtPubKey(master_key.Neuter()); - - // Build descriptor string - std::string desc_prefix; - std::string desc_suffix = "/*)"; - switch (addr_type) { - case OutputType::LEGACY: { - desc_prefix = "pkh(" + xpub + "/44h"; - break; - } - case OutputType::P2SH_SEGWIT: { - desc_prefix = "sh(wpkh(" + xpub + "/49h"; - desc_suffix += ")"; - break; - } - case OutputType::BECH32: { - desc_prefix = "wpkh(" + xpub + "/84h"; - break; - } - case OutputType::BECH32M: { - desc_prefix = "tr(" + xpub + "/86h"; - break; - } - case OutputType::UNKNOWN: { - // We should never have a DescriptorScriptPubKeyMan for an UNKNOWN OutputType, - // so if we get to this point something is wrong - assert(false); - } - } // no default case, so the compiler can warn about missing cases - assert(!desc_prefix.empty()); - - // Mainnet derives at 0', testnet and regtest derive at 1' - if (Params().IsTestChain()) { - desc_prefix += "/1h"; - } else { - desc_prefix += "/0h"; - } - - std::string internal_path = internal ? "/1" : "/0"; - std::string desc_str = desc_prefix + "/0h" + internal_path + desc_suffix; - - // Make the descriptor - FlatSigningProvider keys; - std::string error; - std::unique_ptr<Descriptor> desc = Parse(desc_str, keys, error, false); - WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0); - m_wallet_descriptor = w_desc; + m_wallet_descriptor = GenerateWalletDescriptor(master_key.Neuter(), addr_type, internal); // Store the master private key, and descriptor if (!AddDescriptorKeyWithDB(batch, master_key.key, master_key.key.GetPubKey())) { diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 2d83ae556f..4575881d96 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -633,6 +633,9 @@ public: bool SetupDescriptorGeneration(WalletBatch& batch, const CExtKey& master_key, OutputType addr_type, bool internal); bool HavePrivateKeys() const override; + bool HasPrivKey(const CKeyID& keyid) const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); + //! Retrieve the particular key if it is available. Returns nullopt if the key is not in the wallet, or if the wallet is locked. + std::optional<CKey> GetKey(const CKeyID& keyid) const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); std::optional<int64_t> GetOldestKeyPoolTime() const override; unsigned int GetKeyPoolSize() const override; @@ -669,7 +672,7 @@ public: std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys(int32_t minimum_index) const; int32_t GetEndRange() const; - bool GetDescriptorString(std::string& out, const bool priv) const; + [[nodiscard]] bool GetDescriptorString(std::string& out, const bool priv) const; void UpgradeDescriptorCache(); }; diff --git a/src/wallet/test/walletload_tests.cpp b/src/wallet/test/walletload_tests.cpp index 3dba2231f0..2e43eda582 100644 --- a/src/wallet/test/walletload_tests.cpp +++ b/src/wallet/test/walletload_tests.cpp @@ -34,6 +34,7 @@ public: std::optional<int64_t> ScriptSize() const override { return {}; } std::optional<int64_t> MaxSatisfactionWeight(bool) const override { return {}; } std::optional<int64_t> MaxSatisfactionElems() const override { return {}; } + void GetPubKeys(std::set<CPubKey>& pubkeys, std::set<CExtPubKey>& ext_pubs) const override {} }; BOOST_FIXTURE_TEST_CASE(wallet_load_descriptors, TestingSetup) diff --git a/src/wallet/transaction.cpp b/src/wallet/transaction.cpp index 6777257e53..561880482f 100644 --- a/src/wallet/transaction.cpp +++ b/src/wallet/transaction.cpp @@ -45,7 +45,7 @@ void CWalletTx::updateState(interfaces::Chain& chain) }; if (auto* conf = state<TxStateConfirmed>()) { lookup_block(conf->confirmed_block_hash, conf->confirmed_block_height, m_state); - } else if (auto* conf = state<TxStateConflicted>()) { + } else if (auto* conf = state<TxStateBlockConflicted>()) { lookup_block(conf->conflicting_block_hash, conf->conflicting_block_height, m_state); } } diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index ddeb931112..9c27574103 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -43,12 +43,12 @@ struct TxStateInMempool { }; //! State of rejected transaction that conflicts with a confirmed block. -struct TxStateConflicted { +struct TxStateBlockConflicted { uint256 conflicting_block_hash; int conflicting_block_height; - explicit TxStateConflicted(const uint256& block_hash, int height) : conflicting_block_hash(block_hash), conflicting_block_height(height) {} - std::string toString() const { return strprintf("Conflicted (block=%s, height=%i)", conflicting_block_hash.ToString(), conflicting_block_height); } + explicit TxStateBlockConflicted(const uint256& block_hash, int height) : conflicting_block_hash(block_hash), conflicting_block_height(height) {} + std::string toString() const { return strprintf("BlockConflicted (block=%s, height=%i)", conflicting_block_hash.ToString(), conflicting_block_height); } }; //! State of transaction not confirmed or conflicting with a known block and @@ -75,7 +75,7 @@ struct TxStateUnrecognized { }; //! All possible CWalletTx states -using TxState = std::variant<TxStateConfirmed, TxStateInMempool, TxStateConflicted, TxStateInactive, TxStateUnrecognized>; +using TxState = std::variant<TxStateConfirmed, TxStateInMempool, TxStateBlockConflicted, TxStateInactive, TxStateUnrecognized>; //! Subset of states transaction sync logic is implemented to handle. using SyncTxState = std::variant<TxStateConfirmed, TxStateInMempool, TxStateInactive>; @@ -90,7 +90,7 @@ static inline TxState TxStateInterpretSerialized(TxStateUnrecognized data) } else if (data.index >= 0) { return TxStateConfirmed{data.block_hash, /*height=*/-1, data.index}; } else if (data.index == -1) { - return TxStateConflicted{data.block_hash, /*height=*/-1}; + return TxStateBlockConflicted{data.block_hash, /*height=*/-1}; } return data; } @@ -102,7 +102,7 @@ static inline uint256 TxStateSerializedBlockHash(const TxState& state) [](const TxStateInactive& inactive) { return inactive.abandoned ? uint256::ONE : uint256::ZERO; }, [](const TxStateInMempool& in_mempool) { return uint256::ZERO; }, [](const TxStateConfirmed& confirmed) { return confirmed.confirmed_block_hash; }, - [](const TxStateConflicted& conflicted) { return conflicted.conflicting_block_hash; }, + [](const TxStateBlockConflicted& conflicted) { return conflicted.conflicting_block_hash; }, [](const TxStateUnrecognized& unrecognized) { return unrecognized.block_hash; } }, state); } @@ -114,7 +114,7 @@ static inline int TxStateSerializedIndex(const TxState& state) [](const TxStateInactive& inactive) { return inactive.abandoned ? -1 : 0; }, [](const TxStateInMempool& in_mempool) { return 0; }, [](const TxStateConfirmed& confirmed) { return confirmed.position_in_block; }, - [](const TxStateConflicted& conflicted) { return -1; }, + [](const TxStateBlockConflicted& conflicted) { return -1; }, [](const TxStateUnrecognized& unrecognized) { return unrecognized.index; } }, state); } @@ -258,6 +258,14 @@ public: CTransactionRef tx; TxState m_state; + // Set of mempool transactions that conflict + // directly with the transaction, or that conflict + // with an ancestor transaction. This set will be + // empty if state is InMempool or Confirmed, but + // can be nonempty if state is Inactive or + // BlockConflicted. + std::set<Txid> mempool_conflicts; + template<typename Stream> void Serialize(Stream& s) const { @@ -335,9 +343,10 @@ public: void updateState(interfaces::Chain& chain); bool isAbandoned() const { return state<TxStateInactive>() && state<TxStateInactive>()->abandoned; } - bool isConflicted() const { return state<TxStateConflicted>(); } + bool isMempoolConflicted() const { return !mempool_conflicts.empty(); } + bool isBlockConflicted() const { return state<TxStateBlockConflicted>(); } bool isInactive() const { return state<TxStateInactive>(); } - bool isUnconfirmed() const { return !isAbandoned() && !isConflicted() && !isConfirmed(); } + bool isUnconfirmed() const { return !isAbandoned() && !isBlockConflicted() && !isMempoolConflicted() && !isConfirmed(); } bool isConfirmed() const { return state<TxStateConfirmed>(); } const Txid& GetHash() const LIFETIMEBOUND { return tx->GetHash(); } const Wtxid& GetWitnessHash() const LIFETIMEBOUND { return tx->GetWitnessHash(); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 9c15c2a827..96c4397504 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -752,8 +752,8 @@ bool CWallet::IsSpent(const COutPoint& outpoint) const const uint256& wtxid = it->second; const auto mit = mapWallet.find(wtxid); if (mit != mapWallet.end()) { - int depth = GetTxDepthInMainChain(mit->second); - if (depth > 0 || (depth == 0 && !mit->second.isAbandoned())) + const auto& wtx = mit->second; + if (!wtx.isAbandoned() && !wtx.isBlockConflicted() && !wtx.isMempoolConflicted()) return true; // Spent } } @@ -1197,7 +1197,7 @@ bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx auto it = mapWallet.find(txin.prevout.hash); if (it != mapWallet.end()) { CWalletTx& prevtx = it->second; - if (auto* prev = prevtx.state<TxStateConflicted>()) { + if (auto* prev = prevtx.state<TxStateBlockConflicted>()) { MarkConflicted(prev->conflicting_block_hash, prev->conflicting_block_height, wtx.GetHash()); } } @@ -1309,7 +1309,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) assert(!wtx.isConfirmed()); assert(!wtx.InMempool()); // If already conflicted or abandoned, no need to set abandoned - if (!wtx.isConflicted() && !wtx.isAbandoned()) { + if (!wtx.isBlockConflicted() && !wtx.isAbandoned()) { wtx.m_state = TxStateInactive{/*abandoned=*/true}; return TxUpdate::NOTIFY_CHANGED; } @@ -1346,7 +1346,7 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c if (conflictconfirms < GetTxDepthInMainChain(wtx)) { // Block is 'more conflicted' than current confirm; update. // Mark transaction as conflicted with this block. - wtx.m_state = TxStateConflicted{hashBlock, conflicting_height}; + wtx.m_state = TxStateBlockConflicted{hashBlock, conflicting_height}; return TxUpdate::CHANGED; } return TxUpdate::UNCHANGED; @@ -1360,7 +1360,10 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c void CWallet::RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { // Do not flush the wallet here for performance reasons WalletBatch batch(GetDatabase(), false); + RecursiveUpdateTxState(&batch, tx_hash, try_updating_state); +} +void CWallet::RecursiveUpdateTxState(WalletBatch* batch, const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { std::set<uint256> todo; std::set<uint256> done; @@ -1377,7 +1380,7 @@ void CWallet::RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingSt TxUpdate update_state = try_updating_state(wtx); if (update_state != TxUpdate::UNCHANGED) { wtx.MarkDirty(); - batch.WriteTx(wtx); + if (batch) batch->WriteTx(wtx); // Iterate over all its outputs, and update those tx states as well (if applicable) for (unsigned int i = 0; i < wtx.tx->vout.size(); ++i) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(COutPoint(Txid::FromUint256(now), i)); @@ -1418,6 +1421,20 @@ void CWallet::transactionAddedToMempool(const CTransactionRef& tx) { if (it != mapWallet.end()) { RefreshMempoolStatus(it->second, chain()); } + + const Txid& txid = tx->GetHash(); + + for (const CTxIn& tx_in : tx->vin) { + // For each wallet transaction spending this prevout.. + for (auto range = mapTxSpends.equal_range(tx_in.prevout); range.first != range.second; range.first++) { + const uint256& spent_id = range.first->second; + // Skip the recently added tx + if (spent_id == txid) continue; + RecursiveUpdateTxState(/*batch=*/nullptr, spent_id, [&txid](CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { + return wtx.mempool_conflicts.insert(txid).second ? TxUpdate::CHANGED : TxUpdate::UNCHANGED; + }); + } + } } void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) { @@ -1455,6 +1472,21 @@ void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRe // https://github.com/bitcoin-core/bitcoin-devwiki/wiki/Wallet-Transaction-Conflict-Tracking SyncTransaction(tx, TxStateInactive{}); } + + const Txid& txid = tx->GetHash(); + + for (const CTxIn& tx_in : tx->vin) { + // Iterate over all wallet transactions spending txin.prev + // and recursively mark them as no longer conflicting with + // txid + for (auto range = mapTxSpends.equal_range(tx_in.prevout); range.first != range.second; range.first++) { + const uint256& spent_id = range.first->second; + + RecursiveUpdateTxState(/*batch=*/nullptr, spent_id, [&txid](CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { + return wtx.mempool_conflicts.erase(txid) ? TxUpdate::CHANGED : TxUpdate::UNCHANGED; + }); + } + } } void CWallet::blockConnected(ChainstateRole role, const interfaces::BlockInfo& block) @@ -1506,11 +1538,11 @@ void CWallet::blockDisconnected(const interfaces::BlockInfo& block) for (TxSpends::const_iterator _it = range.first; _it != range.second; ++_it) { CWalletTx& wtx = mapWallet.find(_it->second)->second; - if (!wtx.isConflicted()) continue; + if (!wtx.isBlockConflicted()) continue; auto try_updating_state = [&](CWalletTx& tx) { - if (!tx.isConflicted()) return TxUpdate::UNCHANGED; - if (tx.state<TxStateConflicted>()->conflicting_block_height >= disconnect_height) { + if (!tx.isBlockConflicted()) return TxUpdate::UNCHANGED; + if (tx.state<TxStateBlockConflicted>()->conflicting_block_height >= disconnect_height) { tx.m_state = TxStateInactive{}; return TxUpdate::CHANGED; } @@ -2787,7 +2819,7 @@ unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx, bool rescanning_old std::optional<uint256> block_hash; if (auto* conf = wtx.state<TxStateConfirmed>()) { block_hash = conf->confirmed_block_hash; - } else if (auto* conf = wtx.state<TxStateConflicted>()) { + } else if (auto* conf = wtx.state<TxStateBlockConflicted>()) { block_hash = conf->conflicting_block_hash; } @@ -3377,7 +3409,7 @@ int CWallet::GetTxDepthInMainChain(const CWalletTx& wtx) const if (auto* conf = wtx.state<TxStateConfirmed>()) { assert(conf->confirmed_block_height >= 0); return GetLastBlockHeight() - conf->confirmed_block_height + 1; - } else if (auto* conf = wtx.state<TxStateConflicted>()) { + } else if (auto* conf = wtx.state<TxStateBlockConflicted>()) { assert(conf->conflicting_block_height >= 0); return -1 * (GetLastBlockHeight() - conf->conflicting_block_height + 1); } else { @@ -3465,6 +3497,17 @@ std::set<ScriptPubKeyMan*> CWallet::GetActiveScriptPubKeyMans() const return spk_mans; } +bool CWallet::IsActiveScriptPubKeyMan(const ScriptPubKeyMan& spkm) const +{ + for (const auto& [_, ext_spkm] : m_external_spk_managers) { + if (ext_spkm == &spkm) return true; + } + for (const auto& [_, int_spkm] : m_internal_spk_managers) { + if (int_spkm == &spkm) return true; + } + return false; +} + std::set<ScriptPubKeyMan*> CWallet::GetAllScriptPubKeyMans() const { std::set<ScriptPubKeyMan*> spk_mans; @@ -3619,6 +3662,26 @@ DescriptorScriptPubKeyMan& CWallet::LoadDescriptorScriptPubKeyMan(uint256 id, Wa return *spk_manager; } +DescriptorScriptPubKeyMan& CWallet::SetupDescriptorScriptPubKeyMan(WalletBatch& batch, const CExtKey& master_key, const OutputType& output_type, bool internal) +{ + AssertLockHeld(cs_wallet); + auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, m_keypool_size)); + if (IsCrypted()) { + if (IsLocked()) { + throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); + } + if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, &batch)) { + throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); + } + } + spk_manager->SetupDescriptorGeneration(batch, master_key, output_type, internal); + DescriptorScriptPubKeyMan* out = spk_manager.get(); + uint256 id = spk_manager->GetID(); + AddScriptPubKeyMan(id, std::move(spk_manager)); + AddActiveScriptPubKeyManWithDb(batch, id, output_type, internal); + return *out; +} + void CWallet::SetupDescriptorScriptPubKeyMans(const CExtKey& master_key) { AssertLockHeld(cs_wallet); @@ -3629,19 +3692,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans(const CExtKey& master_key) for (bool internal : {false, true}) { for (OutputType t : OUTPUT_TYPES) { - auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, m_keypool_size)); - if (IsCrypted()) { - if (IsLocked()) { - throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); - } - if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, &batch)) { - throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); - } - } - spk_manager->SetupDescriptorGeneration(batch, master_key, t, internal); - uint256 id = spk_manager->GetID(); - AddScriptPubKeyMan(id, std::move(spk_manager)); - AddActiveScriptPubKeyManWithDb(batch, id, t, internal); + SetupDescriptorScriptPubKeyMan(batch, master_key, t, internal); } } @@ -4469,4 +4520,40 @@ void CWallet::TopUpCallback(const std::set<CScript>& spks, ScriptPubKeyMan* spkm // Update scriptPubKey cache CacheNewScriptPubKeys(spks, spkm); } + +std::set<CExtPubKey> CWallet::GetActiveHDPubKeys() const +{ + AssertLockHeld(cs_wallet); + + Assert(IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); + + std::set<CExtPubKey> active_xpubs; + for (const auto& spkm : GetActiveScriptPubKeyMans()) { + const DescriptorScriptPubKeyMan* desc_spkm = dynamic_cast<DescriptorScriptPubKeyMan*>(spkm); + assert(desc_spkm); + LOCK(desc_spkm->cs_desc_man); + WalletDescriptor w_desc = desc_spkm->GetWalletDescriptor(); + + std::set<CPubKey> desc_pubkeys; + std::set<CExtPubKey> desc_xpubs; + w_desc.descriptor->GetPubKeys(desc_pubkeys, desc_xpubs); + active_xpubs.merge(std::move(desc_xpubs)); + } + return active_xpubs; +} + +std::optional<CKey> CWallet::GetKey(const CKeyID& keyid) const +{ + Assert(IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); + + for (const auto& spkm : GetAllScriptPubKeyMans()) { + const DescriptorScriptPubKeyMan* desc_spkm = dynamic_cast<DescriptorScriptPubKeyMan*>(spkm); + assert(desc_spkm); + LOCK(desc_spkm->cs_desc_man); + if (std::optional<CKey> key = desc_spkm->GetKey(keyid)) { + return key; + } + } + return std::nullopt; +} } // namespace wallet diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 8b0ee22276..b49b5a7d0d 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -364,6 +364,7 @@ private: /** Mark a transaction (and its in-wallet descendants) as a particular tx state. */ void RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void RecursiveUpdateTxState(WalletBatch* batch, const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** Mark a transaction's inputs dirty, thus forcing the outputs to be recomputed */ void MarkInputsDirty(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -518,11 +519,6 @@ public: * referenced in transaction, and might cause assert failures. */ int GetTxDepthInMainChain(const CWalletTx& wtx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool IsTxInMainChain(const CWalletTx& wtx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) - { - AssertLockHeld(cs_wallet); - return GetTxDepthInMainChain(wtx) > 0; - } /** * @return number of blocks to maturity for this transaction: @@ -942,6 +938,7 @@ public: //! Returns all unique ScriptPubKeyMans in m_internal_spk_managers and m_external_spk_managers std::set<ScriptPubKeyMan*> GetActiveScriptPubKeyMans() const; + bool IsActiveScriptPubKeyMan(const ScriptPubKeyMan& spkm) const; //! Returns all unique ScriptPubKeyMans std::set<ScriptPubKeyMan*> GetAllScriptPubKeyMans() const; @@ -1017,6 +1014,8 @@ public: //! @param[in] internal Whether this ScriptPubKeyMan provides change addresses void DeactivateScriptPubKeyMan(uint256 id, OutputType type, bool internal); + //! Create new DescriptorScriptPubKeyMan and add it to the wallet + DescriptorScriptPubKeyMan& SetupDescriptorScriptPubKeyMan(WalletBatch& batch, const CExtKey& master_key, const OutputType& output_type, bool internal) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Create new DescriptorScriptPubKeyMans and add them to the wallet void SetupDescriptorScriptPubKeyMans(const CExtKey& master_key) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SetupDescriptorScriptPubKeyMans() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -1053,6 +1052,13 @@ public: void CacheNewScriptPubKeys(const std::set<CScript>& spks, ScriptPubKeyMan* spkm); void TopUpCallback(const std::set<CScript>& spks, ScriptPubKeyMan* spkm) override; + + //! Retrieve the xpubs in use by the active descriptors + std::set<CExtPubKey> GetActiveHDPubKeys() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + //! Find the private key for the given key id from the wallet's descriptors, if available + //! Returns nullopt when no descriptor has the key or if the wallet is locked. + std::optional<CKey> GetKey(const CKeyID& keyid) const; }; /** diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index fdd5bc36d8..0de2617d45 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -4,7 +4,9 @@ #include <wallet/walletutil.h> +#include <chainparams.h> #include <common/args.h> +#include <key_io.h> #include <logging.h> namespace wallet { @@ -43,4 +45,58 @@ WalletFeature GetClosestWalletFeature(int version) } return static_cast<WalletFeature>(0); } + +WalletDescriptor GenerateWalletDescriptor(const CExtPubKey& master_key, const OutputType& addr_type, bool internal) +{ + int64_t creation_time = GetTime(); + + std::string xpub = EncodeExtPubKey(master_key); + + // Build descriptor string + std::string desc_prefix; + std::string desc_suffix = "/*)"; + switch (addr_type) { + case OutputType::LEGACY: { + desc_prefix = "pkh(" + xpub + "/44h"; + break; + } + case OutputType::P2SH_SEGWIT: { + desc_prefix = "sh(wpkh(" + xpub + "/49h"; + desc_suffix += ")"; + break; + } + case OutputType::BECH32: { + desc_prefix = "wpkh(" + xpub + "/84h"; + break; + } + case OutputType::BECH32M: { + desc_prefix = "tr(" + xpub + "/86h"; + break; + } + case OutputType::UNKNOWN: { + // We should never have a DescriptorScriptPubKeyMan for an UNKNOWN OutputType, + // so if we get to this point something is wrong + assert(false); + } + } // no default case, so the compiler can warn about missing cases + assert(!desc_prefix.empty()); + + // Mainnet derives at 0', testnet and regtest derive at 1' + if (Params().IsTestChain()) { + desc_prefix += "/1h"; + } else { + desc_prefix += "/0h"; + } + + std::string internal_path = internal ? "/1" : "/0"; + std::string desc_str = desc_prefix + "/0h" + internal_path + desc_suffix; + + // Make the descriptor + FlatSigningProvider keys; + std::string error; + std::unique_ptr<Descriptor> desc = Parse(desc_str, keys, error, false); + WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0); + return w_desc; +} + } // namespace wallet diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index 7ad3ffe9e4..38926c1eb8 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -114,6 +114,8 @@ public: WalletDescriptor() {} WalletDescriptor(std::shared_ptr<Descriptor> descriptor, uint64_t creation_time, int32_t range_start, int32_t range_end, int32_t next_index) : descriptor(descriptor), id(DescriptorID(*descriptor)), creation_time(creation_time), range_start(range_start), range_end(range_end), next_index(next_index) { } }; + +WalletDescriptor GenerateWalletDescriptor(const CExtPubKey& master_key, const OutputType& output_type, bool internal); } // namespace wallet #endif // BITCOIN_WALLET_WALLETUTIL_H diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index 740d3b7f0e..01ba2834c4 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -36,7 +36,7 @@ class AbortNodeTest(BitcoinTestFramework): # Check that node0 aborted self.log.info("Waiting for crash") - self.nodes[0].wait_until_stopped(timeout=5, expect_error=True, expected_stderr="Error: A fatal internal error occurred, see debug.log for details") + self.nodes[0].wait_until_stopped(timeout=5, expect_error=True, expected_stderr="Error: A fatal internal error occurred, see debug.log for details: Failed to disconnect block.") self.log.info("Node crashed - now verifying restart fails") self.nodes[0].assert_start_raises_init_error() diff --git a/test/functional/feature_assumeutxo.py b/test/functional/feature_assumeutxo.py index eb9ea65c4f..19cbbcffdb 100755 --- a/test/functional/feature_assumeutxo.py +++ b/test/functional/feature_assumeutxo.py @@ -11,9 +11,6 @@ The assumeutxo value generated and used here is committed to in ## Possible test improvements -- TODO: test what happens with -reindex and -reindex-chainstate before the - snapshot is validated, and make sure it's deleted successfully. - Interesting test cases could be loading an assumeutxo snapshot file with: - TODO: Valid hash but invalid snapshot file (bad coin height or @@ -134,7 +131,7 @@ class AssumeutxoTest(BitcoinTestFramework): with self.nodes[0].assert_debug_log([log_msg]): self.nodes[0].assert_start_raises_init_error(expected_msg=error_msg) - expected_error_msg = f"Error: A fatal internal error occurred, see debug.log for details" + expected_error_msg = f"Error: A fatal internal error occurred, see debug.log for details: Assumeutxo data not found for the given blockhash '7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a'." error_details = f"Assumeutxo data not found for the given blockhash" expected_error(log_msg=error_details, error_msg=expected_error_msg) @@ -379,6 +376,17 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(loaded['coins_loaded'], SNAPSHOT_BASE_HEIGHT) assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT) + for reindex_arg in ['-reindex=1', '-reindex-chainstate=1']: + self.log.info(f"Check that restarting with {reindex_arg} will delete the snapshot chainstate") + self.restart_node(2, extra_args=[reindex_arg, *self.extra_args[2]]) + assert_equal(1, len(n2.getchainstates()["chainstates"])) + for i in range(1, 300): + block = n0.getblock(n0.getblockhash(i), 0) + n2.submitheader(block) + loaded = n2.loadtxoutset(dump_output['path']) + assert_equal(loaded['coins_loaded'], SNAPSHOT_BASE_HEIGHT) + assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT) + normal, snapshot = n2.getchainstates()['chainstates'] assert_equal(normal['blocks'], START_HEIGHT) assert_equal(normal.get('snapshot_blockhash'), None) diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index 613d2eab14..982fa79915 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -159,7 +159,7 @@ class AssumeValidTest(BitcoinTestFramework): for i in range(2202): p2p1.send_message(msg_block(self.blocks[i])) # Syncing 2200 blocks can take a while on slow systems. Give it plenty of time to sync. - p2p1.sync_with_ping(960) + p2p1.sync_with_ping(timeout=960) assert_equal(self.nodes[1].getblock(self.nodes[1].getbestblockhash())['height'], 2202) p2p2 = self.nodes[2].add_p2p_connection(BaseNode()) diff --git a/test/functional/feature_index_prune.py b/test/functional/feature_index_prune.py index d6e802b399..66c0a4f615 100755 --- a/test/functional/feature_index_prune.py +++ b/test/functional/feature_index_prune.py @@ -31,7 +31,7 @@ class FeatureIndexPruneTest(BitcoinTestFramework): expected_stats = { 'coinstatsindex': {'synced': True, 'best_block_height': height} } - self.wait_until(lambda: self.nodes[1].getindexinfo() == expected_stats) + self.wait_until(lambda: self.nodes[1].getindexinfo() == expected_stats, timeout=150) expected = {**expected_filter, **expected_stats} self.wait_until(lambda: self.nodes[2].getindexinfo() == expected) @@ -128,7 +128,7 @@ class FeatureIndexPruneTest(BitcoinTestFramework): self.log.info("make sure we get an init error when starting the nodes again with the indices") filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" - end_msg = f"{os.linesep}Error: Failed to start indexes, shutting down.." + end_msg = f"{os.linesep}Error: A fatal internal error occurred, see debug.log for details: Failed to start indexes, shutting down.." for i, msg in enumerate([filter_msg, stats_msg, filter_msg]): self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=msg+end_msg) diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index 6215610c31..e8a568f7ab 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -6,7 +6,6 @@ from decimal import Decimal -from test_framework.blocktools import COINBASE_MATURITY from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -14,8 +13,7 @@ from test_framework.util import ( assert_fee_amount, assert_greater_than, assert_raises_rpc_error, - create_lots_of_big_transactions, - gen_return_txouts, + fill_mempool, ) from test_framework.wallet import ( COIN, @@ -34,50 +32,6 @@ class MempoolLimitTest(BitcoinTestFramework): ]] self.supports_cli = False - def fill_mempool(self): - """Fill mempool until eviction.""" - self.log.info("Fill the mempool until eviction is triggered and the mempoolminfee rises") - txouts = gen_return_txouts() - node = self.nodes[0] - miniwallet = self.wallet - relayfee = node.getnetworkinfo()['relayfee'] - - tx_batch_size = 1 - num_of_batches = 75 - # Generate UTXOs to flood the mempool - # 1 to create a tx initially that will be evicted from the mempool later - # 75 transactions each with a fee rate higher than the previous one - # And 1 more to verify that this tx does not get added to the mempool with a fee rate less than the mempoolminfee - # And 2 more for the package cpfp test - self.generate(miniwallet, 1 + (num_of_batches * tx_batch_size)) - - # Mine 99 blocks so that the UTXOs are allowed to be spent - self.generate(node, COINBASE_MATURITY - 1) - - self.log.debug("Create a mempool tx that will be evicted") - tx_to_be_evicted_id = miniwallet.send_self_transfer(from_node=node, fee_rate=relayfee)["txid"] - - # Increase the tx fee rate to give the subsequent transactions a higher priority in the mempool - # The tx has an approx. vsize of 65k, i.e. multiplying the previous fee rate (in sats/kvB) - # by 130 should result in a fee that corresponds to 2x of that fee rate - base_fee = relayfee * 130 - - self.log.debug("Fill up the mempool with txs with higher fee rate") - with node.assert_debug_log(["rolling minimum fee bumped"]): - for batch_of_txid in range(num_of_batches): - fee = (batch_of_txid + 1) * base_fee - create_lots_of_big_transactions(miniwallet, node, fee, tx_batch_size, txouts) - - self.log.debug("The tx should be evicted by now") - # The number of transactions created should be greater than the ones present in the mempool - assert_greater_than(tx_batch_size * num_of_batches, len(node.getrawmempool())) - # Initial tx created should not be present in the mempool anymore as it had a lower fee rate - assert tx_to_be_evicted_id not in node.getrawmempool() - - self.log.debug("Check that mempoolminfee is larger than minrelaytxfee") - assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) - assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) - def test_rbf_carveout_disallowed(self): node = self.nodes[0] @@ -139,7 +93,7 @@ class MempoolLimitTest(BitcoinTestFramework): assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) - self.fill_mempool() + fill_mempool(self, node, self.wallet) current_info = node.getmempoolinfo() mempoolmin_feerate = current_info["mempoolminfee"] @@ -229,7 +183,7 @@ class MempoolLimitTest(BitcoinTestFramework): assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) - self.fill_mempool() + fill_mempool(self, node, self.wallet) current_info = node.getmempoolinfo() mempoolmin_feerate = current_info["mempoolminfee"] @@ -303,7 +257,7 @@ class MempoolLimitTest(BitcoinTestFramework): assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) - self.fill_mempool() + fill_mempool(self, node, self.wallet) # Deliberately try to create a tx with a fee less than the minimum mempool fee to assert that it does not get added to the mempool self.log.info('Create a mempool tx that will not pass mempoolminfee') diff --git a/test/functional/p2p_addrv2_relay.py b/test/functional/p2p_addrv2_relay.py index f9a8c44be2..ea114e7d70 100755 --- a/test/functional/p2p_addrv2_relay.py +++ b/test/functional/p2p_addrv2_relay.py @@ -11,6 +11,7 @@ import time from test_framework.messages import ( CAddress, msg_addrv2, + msg_sendaddrv2, ) from test_framework.p2p import ( P2PInterface, @@ -75,6 +76,12 @@ class AddrTest(BitcoinTestFramework): self.extra_args = [["-whitelist=addr@127.0.0.1"]] def run_test(self): + self.log.info('Check disconnection when sending sendaddrv2 after verack') + conn = self.nodes[0].add_p2p_connection(P2PInterface()) + with self.nodes[0].assert_debug_log(['sendaddrv2 received after verack from peer=0; disconnecting']): + conn.send_message(msg_sendaddrv2()) + conn.wait_for_disconnect() + self.log.info('Create connection that sends addrv2 messages') addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) msg = msg_addrv2() @@ -89,8 +96,8 @@ class AddrTest(BitcoinTestFramework): msg.addrs = ADDRS msg_size = calc_addrv2_msg_size(ADDRS) with self.nodes[0].assert_debug_log([ - f'received: addrv2 ({msg_size} bytes) peer=0', - f'sending addrv2 ({msg_size} bytes) peer=1', + f'received: addrv2 ({msg_size} bytes) peer=1', + f'sending addrv2 ({msg_size} bytes) peer=2', ]): addr_source.send_and_ping(msg) self.nodes[0].setmocktime(int(time.time()) + 30 * 60) diff --git a/test/functional/p2p_block_sync.py b/test/functional/p2p_block_sync.py index d821edc1b1..6c7f08364e 100755 --- a/test/functional/p2p_block_sync.py +++ b/test/functional/p2p_block_sync.py @@ -22,7 +22,7 @@ class BlockSyncTest(BitcoinTestFramework): # node0 -> node1 -> node2 # So node1 has both an inbound and outbound peer. # In our test, we will mine a block on node0, and ensure that it makes - # to to both node1 and node2. + # to both node1 and node2. self.connect_nodes(0, 1) self.connect_nodes(1, 2) diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index d6c06fdeed..0950579580 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -139,7 +139,7 @@ class TestP2PConn(P2PInterface): This is used when we want to send a message into the node that we expect will get us disconnected, eg an invalid block.""" self.send_message(message) - self.wait_for_disconnect(timeout) + self.wait_for_disconnect(timeout=timeout) class CompactBlocksTest(BitcoinTestFramework): def set_test_params(self): diff --git a/test/functional/p2p_compactblocks_hb.py b/test/functional/p2p_compactblocks_hb.py index c985a1f98d..023b33ff6d 100755 --- a/test/functional/p2p_compactblocks_hb.py +++ b/test/functional/p2p_compactblocks_hb.py @@ -32,10 +32,15 @@ class CompactBlocksConnectionTest(BitcoinTestFramework): self.connect_nodes(peer, 0) self.generate(self.nodes[0], 1) self.disconnect_nodes(peer, 0) - status_to = [self.peer_info(1, i)['bip152_hb_to'] for i in range(2, 6)] - status_from = [self.peer_info(i, 1)['bip152_hb_from'] for i in range(2, 6)] - assert_equal(status_to, status_from) - return status_to + + def status_to(): + return [self.peer_info(1, i)['bip152_hb_to'] for i in range(2, 6)] + + def status_from(): + return [self.peer_info(i, 1)['bip152_hb_from'] for i in range(2, 6)] + + self.wait_until(lambda: status_to() == status_from()) + return status_to() def run_test(self): self.log.info("Testing reserved high-bandwidth mode slot for outbound peer...") diff --git a/test/functional/p2p_handshake.py b/test/functional/p2p_handshake.py index 3fbb940cbd..f0b62e291d 100755 --- a/test/functional/p2p_handshake.py +++ b/test/functional/p2p_handshake.py @@ -62,6 +62,11 @@ class P2PHandshakeTest(BitcoinTestFramework): assert (services & desirable_service_flags) == desirable_service_flags self.add_outbound_connection(node, conn_type, services, wait_for_disconnect=False) + def generate_at_mocktime(self, time): + self.nodes[0].setmocktime(time) + self.generate(self.nodes[0], 1) + self.nodes[0].setmocktime(0) + def run_test(self): node = self.nodes[0] self.log.info("Check that lacking desired service flags leads to disconnect (non-pruned peers)") @@ -71,10 +76,10 @@ class P2PHandshakeTest(BitcoinTestFramework): DESIRABLE_SERVICE_FLAGS_FULL, expect_disconnect=False) self.log.info("Check that limited peers are only desired if the local chain is close to the tip (<24h)") - node.setmocktime(int(time.time()) + 25 * 3600) # tip outside the 24h window, should fail + self.generate_at_mocktime(int(time.time()) - 25 * 3600) # tip outside the 24h window, should fail self.test_desirable_service_flags(node, [NODE_NETWORK_LIMITED | NODE_WITNESS], DESIRABLE_SERVICE_FLAGS_FULL, expect_disconnect=True) - node.setmocktime(int(time.time()) + 23 * 3600) # tip inside the 24h window, should succeed + self.generate_at_mocktime(int(time.time()) - 23 * 3600) # tip inside the 24h window, should succeed self.test_desirable_service_flags(node, [NODE_NETWORK_LIMITED | NODE_WITNESS], DESIRABLE_SERVICE_FLAGS_PRUNED, expect_disconnect=False) diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index 467bbad09c..8b63d8ee26 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -92,7 +92,8 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): # Wait until the full_node is headers-wise sync best_block_hash = pruned_node.getbestblockhash() - self.wait_until(lambda: next(filter(lambda x: x['hash'] == best_block_hash, full_node.getchaintips()))['status'] == "headers-only") + default_value = {'status': ''} # No status + self.wait_until(lambda: next(filter(lambda x: x['hash'] == best_block_hash, full_node.getchaintips()), default_value)['status'] == "headers-only") # Now, since the node aims to download a window of 1024 blocks, # ensure it requests the blocks below the threshold only (with a @@ -137,7 +138,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): self.log.info("Requesting block at height 2 (tip-289) must fail (ignored).") node.send_getdata_for_block(blocks[0]) # first block outside of the 288+2 limit - node.wait_for_disconnect(5) + node.wait_for_disconnect(timeout=5) self.nodes[0].disconnect_p2ps() # connect unsynced node 2 with pruned NODE_NETWORK_LIMITED peer diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 1c0c11d74c..af47c6d9f0 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -198,15 +198,15 @@ class TestP2PConn(P2PInterface): self.send_message(msg) else: self.send_message(msg_inv(inv=[CInv(MSG_BLOCK, block.sha256)])) - self.wait_for_getheaders() + self.wait_for_getheaders(timeout=timeout) self.send_message(msg) - self.wait_for_getdata([block.sha256]) + self.wait_for_getdata([block.sha256], timeout=timeout) def request_block(self, blockhash, inv_type, timeout=60): with p2p_lock: self.last_message.pop("block", None) self.send_message(msg_getdata(inv=[CInv(inv_type, blockhash)])) - self.wait_for_block(blockhash, timeout) + self.wait_for_block(blockhash, timeout=timeout) return self.last_message["block"].block class SegWitTest(BitcoinTestFramework): @@ -2056,7 +2056,7 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[0], self.wtx_node, tx2, with_witness=True, accepted=False) # Expect a request for parent (tx) by txid despite use of WTX peer - self.wtx_node.wait_for_getdata([tx.sha256], 60) + self.wtx_node.wait_for_getdata([tx.sha256], timeout=60) with p2p_lock: lgd = self.wtx_node.lastgetdata[:] assert_equal(lgd, [CInv(MSG_WITNESS_TX, tx.sha256)]) diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 22789644f2..2701d2471d 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -45,14 +45,18 @@ def seed_addrman(node): """ Populate the addrman with addresses from different networks. Here 2 ipv4, 2 ipv6, 1 cjdns, 2 onion and 1 i2p addresses are added. """ - node.addpeeraddress(address="1.2.3.4", tried=True, port=8333) - node.addpeeraddress(address="2.0.0.0", port=8333) - node.addpeeraddress(address="1233:3432:2434:2343:3234:2345:6546:4534", tried=True, port=8333) - node.addpeeraddress(address="2803:0:1234:abcd::1", port=45324) - node.addpeeraddress(address="fc00:1:2:3:4:5:6:7", port=8333) - node.addpeeraddress(address="pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion", tried=True, port=8333) - node.addpeeraddress(address="nrfj6inpyf73gpkyool35hcmne5zwfmse3jl3aw23vk7chdemalyaqad.onion", port=45324, tried=True) - node.addpeeraddress(address="c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p", port=8333) + # These addresses currently don't collide with a deterministic addrman. + # If the addrman positioning/bucketing is changed, these might collide + # and adding them fails. + success = { "success": True } + assert_equal(node.addpeeraddress(address="1.2.3.4", tried=True, port=8333), success) + assert_equal(node.addpeeraddress(address="2.0.0.0", port=8333), success) + assert_equal(node.addpeeraddress(address="1233:3432:2434:2343:3234:2345:6546:4534", tried=True, port=8333), success) + assert_equal(node.addpeeraddress(address="2803:0:1234:abcd::1", port=45324), success) + assert_equal(node.addpeeraddress(address="fc00:1:2:3:4:5:6:7", port=8333), success) + assert_equal(node.addpeeraddress(address="pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion", tried=True, port=8333), success) + assert_equal(node.addpeeraddress(address="nrfj6inpyf73gpkyool35hcmne5zwfmse3jl3aw23vk7chdemalyaqad.onion", port=45324, tried=True), success) + assert_equal(node.addpeeraddress(address="c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p", port=8333), success) class NetTest(BitcoinTestFramework): @@ -324,10 +328,10 @@ class NetTest(BitcoinTestFramework): self.restart_node(1, ["-checkaddrman=1", "-test=addrman"], clear_addrman=True) node = self.nodes[1] - self.log.debug("Test that addpeerinfo is a hidden RPC") + self.log.debug("Test that addpeeraddress is a hidden RPC") # It is hidden from general help, but its detailed help may be called directly. - assert "addpeerinfo" not in node.help() - assert "addpeerinfo" in node.help("addpeerinfo") + assert "addpeeraddress" not in node.help() + assert "unknown command: addpeeraddress" not in node.help("addpeeraddress") self.log.debug("Test that adding an empty address fails") assert_equal(node.addpeeraddress(address="", port=8333), {"success": False}) @@ -340,26 +344,50 @@ class NetTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "JSON integer out of range", self.nodes[0].addpeeraddress, address="1.2.3.4", port=-1) assert_raises_rpc_error(-1, "JSON integer out of range", self.nodes[0].addpeeraddress, address="1.2.3.4", port=65536) + self.log.debug("Test that adding a valid address to the new table succeeds") + assert_equal(node.addpeeraddress(address="1.0.0.0", tried=False, port=8333), {"success": True}) + addrman = node.getrawaddrman() + assert_equal(len(addrman["tried"]), 0) + new_table = list(addrman["new"].values()) + assert_equal(len(new_table), 1) + assert_equal(new_table[0]["address"], "1.0.0.0") + assert_equal(new_table[0]["port"], 8333) + + self.log.debug("Test that adding an already-present new address to the new and tried tables fails") + for value in [True, False]: + assert_equal(node.addpeeraddress(address="1.0.0.0", tried=value, port=8333), {"success": False, "error": "failed-adding-to-new"}) + assert_equal(len(node.getnodeaddresses(count=0)), 1) + self.log.debug("Test that adding a valid address to the tried table succeeds") - self.addr_time = int(time.time()) - node.setmocktime(self.addr_time) assert_equal(node.addpeeraddress(address="1.2.3.4", tried=True, port=8333), {"success": True}) - with node.assert_debug_log(expected_msgs=["CheckAddrman: new 0, tried 1, total 1 started"]): - addrs = node.getnodeaddresses(count=0) # getnodeaddresses re-runs the addrman checks - assert_equal(len(addrs), 1) - assert_equal(addrs[0]["address"], "1.2.3.4") - assert_equal(addrs[0]["port"], 8333) + addrman = node.getrawaddrman() + assert_equal(len(addrman["new"]), 1) + tried_table = list(addrman["tried"].values()) + assert_equal(len(tried_table), 1) + assert_equal(tried_table[0]["address"], "1.2.3.4") + assert_equal(tried_table[0]["port"], 8333) + node.getnodeaddresses(count=0) # getnodeaddresses re-runs the addrman checks self.log.debug("Test that adding an already-present tried address to the new and tried tables fails") for value in [True, False]: - assert_equal(node.addpeeraddress(address="1.2.3.4", tried=value, port=8333), {"success": False}) - assert_equal(len(node.getnodeaddresses(count=0)), 1) - - self.log.debug("Test that adding a second address, this time to the new table, succeeds") + assert_equal(node.addpeeraddress(address="1.2.3.4", tried=value, port=8333), {"success": False, "error": "failed-adding-to-new"}) + assert_equal(len(node.getnodeaddresses(count=0)), 2) + + self.log.debug("Test that adding an address, which collides with the address in tried table, fails") + colliding_address = "1.2.5.45" # grinded address that produces a tried-table collision + assert_equal(node.addpeeraddress(address=colliding_address, tried=True, port=8333), {"success": False, "error": "failed-adding-to-tried"}) + # When adding an address to the tried table, it's first added to the new table. + # As we fail to move it to the tried table, it remains in the new table. + addrman_info = node.getaddrmaninfo() + assert_equal(addrman_info["all_networks"]["tried"], 1) + assert_equal(addrman_info["all_networks"]["new"], 2) + + self.log.debug("Test that adding an another address to the new table succeeds") assert_equal(node.addpeeraddress(address="2.0.0.0", port=8333), {"success": True}) - with node.assert_debug_log(expected_msgs=["CheckAddrman: new 1, tried 1, total 2 started"]): - addrs = node.getnodeaddresses(count=0) # getnodeaddresses re-runs the addrman checks - assert_equal(len(addrs), 2) + addrman_info = node.getaddrmaninfo() + assert_equal(addrman_info["all_networks"]["tried"], 1) + assert_equal(addrman_info["all_networks"]["new"], 3) + node.getnodeaddresses(count=0) # getnodeaddresses re-runs the addrman checks def test_sendmsgtopeer(self): node = self.nodes[0] @@ -428,7 +456,7 @@ class NetTest(BitcoinTestFramework): self.log.debug("Test that getrawaddrman is a hidden RPC") # It is hidden from general help, but its detailed help may be called directly. assert "getrawaddrman" not in node.help() - assert "getrawaddrman" in node.help("getrawaddrman") + assert "unknown command: getrawaddrman" not in node.help("getrawaddrman") def check_addr_information(result, expected): """Utility to compare a getrawaddrman result entry with an expected entry""" diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index 029e368166..e061030da3 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -18,6 +18,7 @@ from test_framework.util import ( assert_equal, assert_fee_amount, assert_raises_rpc_error, + fill_mempool, ) from test_framework.wallet import ( DEFAULT_FEE, @@ -82,7 +83,8 @@ class RPCPackagesTest(BitcoinTestFramework): self.test_conflicting() self.test_rbf() self.test_submitpackage() - self.test_maxfeerate_maxburn_submitpackage() + self.test_maxfeerate_submitpackage() + self.test_maxburn_submitpackage() def test_independent(self, coin): self.log.info("Test multiple independent transactions in a package") @@ -358,7 +360,7 @@ class RPCPackagesTest(BitcoinTestFramework): assert_equal(res["tx-results"][sec_wtxid]["error"], "version") peer.wait_for_broadcast([first_wtxid]) - def test_maxfeerate_maxburn_submitpackage(self): + def test_maxfeerate_submitpackage(self): node = self.nodes[0] # clear mempool deterministic_address = node.get_deterministic_priv_key().address @@ -369,23 +371,72 @@ class RPCPackagesTest(BitcoinTestFramework): minrate_btc_kvb = min([chained_txn["fee"] / chained_txn["tx"].get_vsize() * 1000 for chained_txn in chained_txns]) chain_hex = [t["hex"] for t in chained_txns] pkg_result = node.submitpackage(chain_hex, maxfeerate=minrate_btc_kvb - Decimal("0.00000001")) + + # First tx failed in single transaction evaluation, so package message is generic + assert_equal(pkg_result["package_msg"], "transaction failed") assert_equal(pkg_result["tx-results"][chained_txns[0]["wtxid"]]["error"], "max feerate exceeded") assert_equal(pkg_result["tx-results"][chained_txns[1]["wtxid"]]["error"], "bad-txns-inputs-missingorspent") assert_equal(node.getrawmempool(), []) + # Make chain of two transactions where parent doesn't make minfee threshold + # but child is too high fee + # Lower mempool limit to make it easier to fill_mempool + self.restart_node(0, extra_args=[ + "-datacarriersize=100000", + "-maxmempool=5", + "-persistmempool=0", + ]) + + fill_mempool(self, node, self.wallet) + + minrelay = node.getmempoolinfo()["minrelaytxfee"] + parent = self.wallet.create_self_transfer( + fee_rate=minrelay, + ) + + child = self.wallet.create_self_transfer( + fee_rate=DEFAULT_FEE, + utxo_to_spend=parent["new_utxo"], + ) + + pkg_result = node.submitpackage([parent["hex"], child["hex"]], maxfeerate=DEFAULT_FEE - Decimal("0.00000001")) + + # Child is connected even though parent is invalid and still reports fee exceeded + # this implies sub-package evaluation of both entries together. + assert_equal(pkg_result["package_msg"], "transaction failed") + assert "mempool min fee not met" in pkg_result["tx-results"][parent["wtxid"]]["error"] + assert_equal(pkg_result["tx-results"][child["wtxid"]]["error"], "max feerate exceeded") + assert parent["txid"] not in node.getrawmempool() + assert child["txid"] not in node.getrawmempool() + + # Reset maxmempool, datacarriersize, reset dynamic mempool minimum feerate, and empty mempool. + self.restart_node(0) + + assert_equal(node.getrawmempool(), []) + + def test_maxburn_submitpackage(self): + node = self.nodes[0] + + assert_equal(node.getrawmempool(), []) + self.log.info("Submitpackage maxburnamount arg testing") - tx = tx_from_hex(chain_hex[1]) + chained_txns_burn = self.wallet.create_self_transfer_chain(chain_length=2) + chained_burn_hex = [t["hex"] for t in chained_txns_burn] + + tx = tx_from_hex(chained_burn_hex[1]) tx.vout[-1].scriptPubKey = b'a' * 10001 # scriptPubKey bigger than 10k IsUnspendable - chain_hex = [chain_hex[0], tx.serialize().hex()] + chained_burn_hex = [chained_burn_hex[0], tx.serialize().hex()] # burn test is run before any package evaluation; nothing makes it in and we get broader exception - assert_raises_rpc_error(-25, "Unspendable output exceeds maximum configured by user", node.submitpackage, chain_hex, 0, chained_txns[1]["new_utxo"]["value"] - Decimal("0.00000001")) + assert_raises_rpc_error(-25, "Unspendable output exceeds maximum configured by user", node.submitpackage, chained_burn_hex, 0, chained_txns_burn[1]["new_utxo"]["value"] - Decimal("0.00000001")) assert_equal(node.getrawmempool(), []) + minrate_btc_kvb_burn = min([chained_txn_burn["fee"] / chained_txn_burn["tx"].get_vsize() * 1000 for chained_txn_burn in chained_txns_burn]) + # Relax the restrictions for both and send it; parent gets through as own subpackage - pkg_result = node.submitpackage(chain_hex, maxfeerate=minrate_btc_kvb, maxburnamount=chained_txns[1]["new_utxo"]["value"]) - assert "error" not in pkg_result["tx-results"][chained_txns[0]["wtxid"]] + pkg_result = node.submitpackage(chained_burn_hex, maxfeerate=minrate_btc_kvb_burn, maxburnamount=chained_txns_burn[1]["new_utxo"]["value"]) + assert "error" not in pkg_result["tx-results"][chained_txns_burn[0]["wtxid"]] assert_equal(pkg_result["tx-results"][tx.getwtxid()]["error"], "scriptpubkey") - assert_equal(node.getrawmempool(), [chained_txns[0]["txid"]]) + assert_equal(node.getrawmempool(), [chained_txns_burn[0]["txid"]]) if __name__ == "__main__": RPCPackagesTest().main() diff --git a/test/functional/rpc_uptime.py b/test/functional/rpc_uptime.py index cb99e483ec..f8df59d02a 100755 --- a/test/functional/rpc_uptime.py +++ b/test/functional/rpc_uptime.py @@ -23,7 +23,7 @@ class UptimeTest(BitcoinTestFramework): self._test_uptime() def _test_negative_time(self): - assert_raises_rpc_error(-8, "Mocktime cannot be negative: -1.", self.nodes[0].setmocktime, -1) + assert_raises_rpc_error(-8, "Mocktime must be in the range [0, 9223372036], not -1.", self.nodes[0].setmocktime, -1) def _test_uptime(self): wait_time = 10 diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index dc04696114..ce76008c46 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -585,22 +585,22 @@ class P2PInterface(P2PConnection): wait_until_helper_internal(test_function, timeout=timeout, lock=p2p_lock, timeout_factor=self.timeout_factor) - def wait_for_connect(self, timeout=60): + def wait_for_connect(self, *, timeout=60): test_function = lambda: self.is_connected self.wait_until(test_function, timeout=timeout, check_connected=False) - def wait_for_disconnect(self, timeout=60): + def wait_for_disconnect(self, *, timeout=60): test_function = lambda: not self.is_connected self.wait_until(test_function, timeout=timeout, check_connected=False) - def wait_for_reconnect(self, timeout=60): + def wait_for_reconnect(self, *, timeout=60): def test_function(): return self.is_connected and self.last_message.get('version') and not self.supports_v2_p2p self.wait_until(test_function, timeout=timeout, check_connected=False) # Message receiving helper methods - def wait_for_tx(self, txid, timeout=60): + def wait_for_tx(self, txid, *, timeout=60): def test_function(): if not self.last_message.get('tx'): return False @@ -608,13 +608,13 @@ class P2PInterface(P2PConnection): self.wait_until(test_function, timeout=timeout) - def wait_for_block(self, blockhash, timeout=60): + def wait_for_block(self, blockhash, *, timeout=60): def test_function(): return self.last_message.get("block") and self.last_message["block"].block.rehash() == blockhash self.wait_until(test_function, timeout=timeout) - def wait_for_header(self, blockhash, timeout=60): + def wait_for_header(self, blockhash, *, timeout=60): def test_function(): last_headers = self.last_message.get('headers') if not last_headers: @@ -623,7 +623,7 @@ class P2PInterface(P2PConnection): self.wait_until(test_function, timeout=timeout) - def wait_for_merkleblock(self, blockhash, timeout=60): + def wait_for_merkleblock(self, blockhash, *, timeout=60): def test_function(): last_filtered_block = self.last_message.get('merkleblock') if not last_filtered_block: @@ -632,7 +632,7 @@ class P2PInterface(P2PConnection): self.wait_until(test_function, timeout=timeout) - def wait_for_getdata(self, hash_list, timeout=60): + def wait_for_getdata(self, hash_list, *, timeout=60): """Waits for a getdata message. The object hashes in the inventory vector must match the provided hash_list.""" @@ -644,7 +644,7 @@ class P2PInterface(P2PConnection): self.wait_until(test_function, timeout=timeout) - def wait_for_getheaders(self, timeout=60): + def wait_for_getheaders(self, *, timeout=60): """Waits for a getheaders message. Receiving any getheaders message will satisfy the predicate. the last_message["getheaders"] @@ -656,7 +656,7 @@ class P2PInterface(P2PConnection): self.wait_until(test_function, timeout=timeout) - def wait_for_inv(self, expected_inv, timeout=60): + def wait_for_inv(self, expected_inv, *, timeout=60): """Waits for an INV message and checks that the first inv object in the message was as expected.""" if len(expected_inv) > 1: raise NotImplementedError("wait_for_inv() will only verify the first inv object") @@ -668,7 +668,7 @@ class P2PInterface(P2PConnection): self.wait_until(test_function, timeout=timeout) - def wait_for_verack(self, timeout=60): + def wait_for_verack(self, *, timeout=60): def test_function(): return "verack" in self.last_message @@ -681,11 +681,11 @@ class P2PInterface(P2PConnection): self.send_message(self.on_connection_send_msg) self.on_connection_send_msg = None # Never used again - def send_and_ping(self, message, timeout=60): + def send_and_ping(self, message, *, timeout=60): self.send_message(message) self.sync_with_ping(timeout=timeout) - def sync_with_ping(self, timeout=60): + def sync_with_ping(self, *, timeout=60): """Ensure ProcessMessages and SendMessages is called on this connection""" # Sending two pings back-to-back, requires that the node calls # `ProcessMessage` twice, and thus ensures `SendMessages` must have @@ -726,7 +726,7 @@ class NetworkThread(threading.Thread): """Start the network thread.""" self.network_event_loop.run_forever() - def close(self, timeout=10): + def close(self, *, timeout=10): """Close the connections and network event loop.""" self.network_event_loop.call_soon_threadsafe(self.network_event_loop.stop) wait_until_helper_internal(lambda: not self.network_event_loop.is_running(), timeout=timeout) @@ -933,7 +933,7 @@ class P2PTxInvStore(P2PInterface): with p2p_lock: return list(self.tx_invs_received.keys()) - def wait_for_broadcast(self, txns, timeout=60): + def wait_for_broadcast(self, txns, *, timeout=60): """Waits for the txns (list of txids) to complete initial broadcast. The mempool should mark unbroadcast=False for these transactions. """ diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index c3884270da..a2f767cc98 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -164,7 +164,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="Don't stop bitcoinds after the test execution") parser.add_argument("--cachedir", dest="cachedir", default=os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../../cache"), help="Directory for caching pregenerated datadirs (default: %(default)s)") - parser.add_argument("--tmpdir", dest="tmpdir", help="Root directory for datadirs") + parser.add_argument("--tmpdir", dest="tmpdir", help="Root directory for datadirs (must not exist)") parser.add_argument("-l", "--loglevel", dest="loglevel", default="INFO", help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.") parser.add_argument("--tracerpc", dest="trace_rpc", default=False, action="store_true", diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index c5b69a3954..dbaf42fdaa 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -496,6 +496,55 @@ def check_node_connections(*, node, num_in, num_out): assert_equal(info["connections_in"], num_in) assert_equal(info["connections_out"], num_out) +def fill_mempool(test_framework, node, miniwallet): + """Fill mempool until eviction. + + Allows for simpler testing of scenarios with floating mempoolminfee > minrelay + Requires -datacarriersize=100000 and + -maxmempool=5. + It will not ensure mempools become synced as it + is based on a single node and assumes -minrelaytxfee + is 1 sat/vbyte. + """ + test_framework.log.info("Fill the mempool until eviction is triggered and the mempoolminfee rises") + txouts = gen_return_txouts() + relayfee = node.getnetworkinfo()['relayfee'] + + assert_equal(relayfee, Decimal('0.00001000')) + + tx_batch_size = 1 + num_of_batches = 75 + # Generate UTXOs to flood the mempool + # 1 to create a tx initially that will be evicted from the mempool later + # 75 transactions each with a fee rate higher than the previous one + test_framework.generate(miniwallet, 1 + (num_of_batches * tx_batch_size)) + + # Mine COINBASE_MATURITY - 1 blocks so that the UTXOs are allowed to be spent + test_framework.generate(node, 100 - 1) + + test_framework.log.debug("Create a mempool tx that will be evicted") + tx_to_be_evicted_id = miniwallet.send_self_transfer(from_node=node, fee_rate=relayfee)["txid"] + + # Increase the tx fee rate to give the subsequent transactions a higher priority in the mempool + # The tx has an approx. vsize of 65k, i.e. multiplying the previous fee rate (in sats/kvB) + # by 130 should result in a fee that corresponds to 2x of that fee rate + base_fee = relayfee * 130 + + test_framework.log.debug("Fill up the mempool with txs with higher fee rate") + with node.assert_debug_log(["rolling minimum fee bumped"]): + for batch_of_txid in range(num_of_batches): + fee = (batch_of_txid + 1) * base_fee + create_lots_of_big_transactions(miniwallet, node, fee, tx_batch_size, txouts) + + test_framework.log.debug("The tx should be evicted by now") + # The number of transactions created should be greater than the ones present in the mempool + assert_greater_than(tx_batch_size * num_of_batches, len(node.getrawmempool())) + # Initial tx created should not be present in the mempool anymore as it had a lower fee rate + assert tx_to_be_evicted_id not in node.getrawmempool() + + test_framework.log.debug("Check that mempoolminfee is larger than minrelaytxfee") + assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) + assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) # Transaction/Block functions ############################# diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 1408854e02..3f6e47d410 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -181,6 +181,8 @@ BASE_SCRIPTS = [ 'wallet_keypool_topup.py --legacy-wallet', 'wallet_keypool_topup.py --descriptors', 'wallet_fast_rescan.py --descriptors', + 'wallet_gethdkeys.py --descriptors', + 'wallet_createwalletdescriptor.py --descriptors', 'interface_zmq.py', 'rpc_invalid_address_message.py', 'rpc_validateaddress.py', diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index e69546bb82..dda48aae1b 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -231,7 +231,11 @@ class AbandonConflictTest(BitcoinTestFramework): balance = newbalance # Invalidate the block with the double spend. B & C's 10 BTC outputs should no longer be available - self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) + blk = self.nodes[0].getbestblockhash() + # mine 10 blocks so that when the blk is invalidated, the transactions are not + # returned to the mempool + self.generate(self.nodes[1], 10) + self.nodes[0].invalidateblock(blk) assert_equal(alice.gettransaction(txAB1)["confirmations"], 0) newbalance = alice.getbalance() assert_equal(newbalance, balance - Decimal("20")) diff --git a/test/functional/wallet_backwards_compatibility.py b/test/functional/wallet_backwards_compatibility.py index 4d6e6024c5..ab008a40cd 100755 --- a/test/functional/wallet_backwards_compatibility.py +++ b/test/functional/wallet_backwards_compatibility.py @@ -355,6 +355,25 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): down_wallet_name = f"re_down_{node.version}" down_backup_path = os.path.join(self.options.tmpdir, f"{down_wallet_name}.dat") wallet.backupwallet(down_backup_path) + + # Check that taproot descriptors can be added to 0.21 wallets + # This must be done after the backup is created so that 0.21 can still load + # the backup + if self.options.descriptors and self.major_version_equals(node, 21): + assert_raises_rpc_error(-12, "No bech32m addresses available", wallet.getnewaddress, address_type="bech32m") + xpubs = wallet.gethdkeys(active_only=True) + assert_equal(len(xpubs), 1) + assert_equal(len(xpubs[0]["descriptors"]), 6) + wallet.createwalletdescriptor("bech32m") + xpubs = wallet.gethdkeys(active_only=True) + assert_equal(len(xpubs), 1) + assert_equal(len(xpubs[0]["descriptors"]), 8) + tr_descs = [desc["desc"] for desc in xpubs[0]["descriptors"] if desc["desc"].startswith("tr(")] + assert_equal(len(tr_descs), 2) + for desc in tr_descs: + assert info["hdmasterfingerprint"] in desc + wallet.getnewaddress(address_type="bech32m") + wallet.unloadwallet() # Check that no automatic upgrade broke the downgrading the wallet diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 31d3c14e55..56228d2bad 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -681,7 +681,7 @@ class WalletTest(BitcoinTestFramework): "category": baz["category"], "vout": baz["vout"]} expected_fields = frozenset({'amount', 'bip125-replaceable', 'confirmations', 'details', 'fee', - 'hex', 'lastprocessedblock', 'time', 'timereceived', 'trusted', 'txid', 'wtxid', 'walletconflicts'}) + 'hex', 'lastprocessedblock', 'time', 'timereceived', 'trusted', 'txid', 'wtxid', 'walletconflicts', 'mempoolconflicts'}) verbose_field = "decoded" expected_verbose_fields = expected_fields | {verbose_field} diff --git a/test/functional/wallet_conflicts.py b/test/functional/wallet_conflicts.py index 802b718cd5..e5739a6a59 100755 --- a/test/functional/wallet_conflicts.py +++ b/test/functional/wallet_conflicts.py @@ -9,6 +9,7 @@ Test that wallet correctly tracks transactions that have been conflicted by bloc from decimal import Decimal +from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -28,6 +29,20 @@ class TxConflicts(BitcoinTestFramework): return next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(from_tx_id)["details"] if tx_out["amount"] == Decimal(f"{search_value}")) def run_test(self): + """ + The following tests check the behavior of the wallet when + transaction conflicts are created. These conflicts are created + using raw transaction RPCs that double-spend UTXOs and have more + fees, replacing the original transaction. + """ + + self.test_block_conflicts() + self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 7, self.nodes[2].getnewaddress()) + self.test_mempool_conflict() + self.test_mempool_and_block_conflicts() + self.test_descendants_with_mempool_conflicts() + + def test_block_conflicts(self): self.log.info("Send tx from which to conflict outputs later") txid_conflict_from_1 = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) txid_conflict_from_2 = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) @@ -123,5 +138,291 @@ class TxConflicts(BitcoinTestFramework): assert_equal(former_conflicted["confirmations"], 1) assert_equal(former_conflicted["blockheight"], 217) + def test_mempool_conflict(self): + self.nodes[0].createwallet("alice") + alice = self.nodes[0].get_wallet_rpc("alice") + + bob = self.nodes[1] + + self.nodes[2].send(outputs=[{alice.getnewaddress() : 25} for _ in range(3)]) + self.generate(self.nodes[2], 1) + + self.log.info("Test a scenario where a transaction has a mempool conflict") + + unspents = alice.listunspent() + assert_equal(len(unspents), 3) + assert all([tx["amount"] == 25 for tx in unspents]) + + # tx1 spends unspent[0] and unspent[1] + raw_tx = alice.createrawtransaction(inputs=[unspents[0], unspents[1]], outputs=[{bob.getnewaddress() : 49.9999}]) + tx1 = alice.signrawtransactionwithwallet(raw_tx)['hex'] + + # tx2 spends unspent[1] and unspent[2], conflicts with tx1 + raw_tx = alice.createrawtransaction(inputs=[unspents[1], unspents[2]], outputs=[{bob.getnewaddress() : 49.99}]) + tx2 = alice.signrawtransactionwithwallet(raw_tx)['hex'] + + # tx3 spends unspent[2], conflicts with tx2 + raw_tx = alice.createrawtransaction(inputs=[unspents[2]], outputs=[{bob.getnewaddress() : 24.9899}]) + tx3 = alice.signrawtransactionwithwallet(raw_tx)['hex'] + + # broadcast tx1 + tx1_txid = alice.sendrawtransaction(tx1) + + assert_equal(alice.listunspent(), [unspents[2]]) + assert_equal(alice.getbalance(), 25) + + # broadcast tx2, replaces tx1 in mempool + tx2_txid = alice.sendrawtransaction(tx2) + + # Check that unspent[0] is now available because the transaction spending it has been replaced in the mempool + assert_equal(alice.listunspent(), [unspents[0]]) + assert_equal(alice.getbalance(), 25) + + assert_equal(alice.gettransaction(tx1_txid)["mempoolconflicts"], [tx2_txid]) + + self.log.info("Test scenario where a mempool conflict is removed") + + # broadcast tx3, replaces tx2 in mempool + # Now that tx1's conflict has been removed, tx1 is now + # not conflicted, and instead is inactive until it is + # rebroadcasted. Now unspent[0] is not available, because + # tx1 is no longer conflicted. + alice.sendrawtransaction(tx3) + + assert_equal(alice.gettransaction(tx1_txid)["mempoolconflicts"], []) + assert tx1_txid not in self.nodes[0].getrawmempool() + + # now all of alice's outputs should be considered spent + # unspent[0]: spent by inactive tx1 + # unspent[1]: spent by inactive tx1 + # unspent[2]: spent by active tx3 + assert_equal(alice.listunspent(), []) + assert_equal(alice.getbalance(), 0) + + # Clean up for next test + bob.sendall([self.nodes[2].getnewaddress()]) + self.generate(self.nodes[2], 1) + + alice.unloadwallet() + + def test_mempool_and_block_conflicts(self): + self.nodes[0].createwallet("alice_2") + alice = self.nodes[0].get_wallet_rpc("alice_2") + bob = self.nodes[1] + + self.nodes[2].send(outputs=[{alice.getnewaddress() : 25} for _ in range(3)]) + self.generate(self.nodes[2], 1) + + self.log.info("Test a scenario where a transaction has both a block conflict and a mempool conflict") + unspents = [{"txid" : element["txid"], "vout" : element["vout"]} for element in alice.listunspent()] + + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + + # alice and bob nodes are disconnected so that transactions can be + # created by alice, but broadcasted from bob so that alice's wallet + # doesn't know about them + self.disconnect_nodes(0, 1) + + # Sends funds to bob + raw_tx = alice.createrawtransaction(inputs=[unspents[0]], outputs=[{bob.getnewaddress() : 24.99999}]) + raw_tx1 = alice.signrawtransactionwithwallet(raw_tx)['hex'] + tx1_txid = bob.sendrawtransaction(raw_tx1) # broadcast original tx spending unspents[0] only to bob + + # create a conflict to previous tx (also spends unspents[0]), but don't broadcast, sends funds back to alice + raw_tx = alice.createrawtransaction(inputs=[unspents[0], unspents[2]], outputs=[{alice.getnewaddress() : 49.999}]) + tx1_conflict = alice.signrawtransactionwithwallet(raw_tx)['hex'] + + # Sends funds to bob + raw_tx = alice.createrawtransaction(inputs=[unspents[1]], outputs=[{bob.getnewaddress() : 24.9999}]) + raw_tx2 = alice.signrawtransactionwithwallet(raw_tx)['hex'] + tx2_txid = bob.sendrawtransaction(raw_tx2) # broadcast another original tx spending unspents[1] only to bob + + # create a conflict to previous tx (also spends unspents[1]), but don't broadcast, sends funds to alice + raw_tx = alice.createrawtransaction(inputs=[unspents[1]], outputs=[{alice.getnewaddress() : 24.9999}]) + tx2_conflict = alice.signrawtransactionwithwallet(raw_tx)['hex'] + + bob_unspents = [{"txid" : element, "vout" : 0} for element in [tx1_txid, tx2_txid]] + + # tx1 and tx2 are now in bob's mempool, and they are unconflicted, so bob has these funds + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], Decimal("49.99989000")) + + # spend both of bob's unspents, child tx of tx1 and tx2 + raw_tx = bob.createrawtransaction(inputs=[bob_unspents[0], bob_unspents[1]], outputs=[{bob.getnewaddress() : 49.999}]) + raw_tx3 = bob.signrawtransactionwithwallet(raw_tx)['hex'] + tx3_txid = bob.sendrawtransaction(raw_tx3) # broadcast tx only to bob + + # alice knows about 0 txs, bob knows about 3 + assert_equal(len(alice.getrawmempool()), 0) + assert_equal(len(bob.getrawmempool()), 3) + + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], Decimal("49.99900000")) + + # bob broadcasts tx_1 conflict + tx1_conflict_txid = bob.sendrawtransaction(tx1_conflict) + assert_equal(len(alice.getrawmempool()), 0) + assert_equal(len(bob.getrawmempool()), 2) # tx1_conflict kicks out both tx1, and its child tx3 + + assert tx2_txid in bob.getrawmempool() + assert tx1_conflict_txid in bob.getrawmempool() + + assert_equal(bob.gettransaction(tx1_txid)["mempoolconflicts"], [tx1_conflict_txid]) + assert_equal(bob.gettransaction(tx2_txid)["mempoolconflicts"], []) + assert_equal(bob.gettransaction(tx3_txid)["mempoolconflicts"], [tx1_conflict_txid]) + + # check that tx3 is now conflicted, so the output from tx2 can now be spent + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], Decimal("24.99990000")) + + # we will be disconnecting this block in the future + alice.sendrawtransaction(tx2_conflict) + assert_equal(len(alice.getrawmempool()), 1) # currently alice's mempool is only aware of tx2_conflict + # 11 blocks are mined so that when they are invalidated, tx_2 + # does not get put back into the mempool + blk = self.generate(self.nodes[0], 11, sync_fun=self.no_op)[0] + assert_equal(len(alice.getrawmempool()), 0) # tx2_conflict is now mined + + self.connect_nodes(0, 1) + self.sync_blocks() + assert_equal(alice.getbestblockhash(), bob.getbestblockhash()) + + # now that tx2 has a block conflict, tx1_conflict should be the only tx in bob's mempool + assert tx1_conflict_txid in bob.getrawmempool() + assert_equal(len(bob.getrawmempool()), 1) + + # tx3 should now also be block-conflicted by tx2_conflict + assert_equal(bob.gettransaction(tx3_txid)["confirmations"], -11) + # bob has no pending funds, since tx1, tx2, and tx3 are all conflicted + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + bob.invalidateblock(blk) # remove tx2_conflict + # bob should still have no pending funds because tx1 and tx3 are still conflicted, and tx2 has not been re-broadcast + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + assert_equal(len(bob.getrawmempool()), 1) + # check that tx3 is no longer block-conflicted + assert_equal(bob.gettransaction(tx3_txid)["confirmations"], 0) + + bob.sendrawtransaction(raw_tx2) + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], Decimal("24.99990000")) + + # create a conflict to previous tx (also spends unspents[2]), but don't broadcast, sends funds back to alice + raw_tx = alice.createrawtransaction(inputs=[unspents[2]], outputs=[{alice.getnewaddress() : 24.99}]) + tx1_conflict_conflict = alice.signrawtransactionwithwallet(raw_tx)['hex'] + + bob.sendrawtransaction(tx1_conflict_conflict) # kick tx1_conflict out of the mempool + bob.sendrawtransaction(raw_tx1) #re-broadcast tx1 because it is no longer conflicted + + # Now bob has no pending funds because tx1 and tx2 are spent by tx3, which hasn't been re-broadcast yet + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + + bob.sendrawtransaction(raw_tx3) + assert_equal(len(bob.getrawmempool()), 4) # The mempool contains: tx1, tx2, tx1_conflict_conflict, tx3 + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], Decimal("49.99900000")) + + # Clean up for next test + bob.reconsiderblock(blk) + assert_equal(alice.getbestblockhash(), bob.getbestblockhash()) + self.sync_mempools() + self.generate(self.nodes[2], 1) + + alice.unloadwallet() + + def test_descendants_with_mempool_conflicts(self): + self.nodes[0].createwallet("alice_3") + alice = self.nodes[0].get_wallet_rpc("alice_3") + + self.nodes[2].send(outputs=[{alice.getnewaddress() : 25} for _ in range(2)]) + self.generate(self.nodes[2], 1) + + self.nodes[1].createwallet("bob_1") + bob = self.nodes[1].get_wallet_rpc("bob_1") + + self.nodes[2].createwallet("carol") + carol = self.nodes[2].get_wallet_rpc("carol") + + self.log.info("Test a scenario where a transaction's parent has a mempool conflict") + + unspents = alice.listunspent() + assert_equal(len(unspents), 2) + assert all([tx["amount"] == 25 for tx in unspents]) + + assert_equal(alice.getrawmempool(), []) + + # Alice spends first utxo to bob in tx1 + raw_tx = alice.createrawtransaction(inputs=[unspents[0]], outputs=[{bob.getnewaddress() : 24.9999}]) + tx1 = alice.signrawtransactionwithwallet(raw_tx)['hex'] + tx1_txid = alice.sendrawtransaction(tx1) + + self.sync_mempools() + + assert_equal(alice.getbalance(), 25) + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], Decimal("24.99990000")) + + assert_equal(bob.gettransaction(tx1_txid)["mempoolconflicts"], []) + + raw_tx = bob.createrawtransaction(inputs=[bob.listunspent(minconf=0)[0]], outputs=[{carol.getnewaddress() : 24.999}]) + # Bob creates a child to tx1 + tx1_child = bob.signrawtransactionwithwallet(raw_tx)['hex'] + tx1_child_txid = bob.sendrawtransaction(tx1_child) + + self.sync_mempools() + + # Currently neither tx1 nor tx1_child should have any conflicts + assert_equal(bob.gettransaction(tx1_txid)["mempoolconflicts"], []) + assert_equal(bob.gettransaction(tx1_child_txid)["mempoolconflicts"], []) + assert tx1_txid in bob.getrawmempool() + assert tx1_child_txid in bob.getrawmempool() + assert_equal(len(bob.getrawmempool()), 2) + + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + assert_equal(carol.getbalances()["mine"]["untrusted_pending"], Decimal("24.99900000")) + + # Alice spends first unspent again, conflicting with tx1 + raw_tx = alice.createrawtransaction(inputs=[unspents[0], unspents[1]], outputs=[{carol.getnewaddress() : 49.99}]) + tx1_conflict = alice.signrawtransactionwithwallet(raw_tx)['hex'] + tx1_conflict_txid = alice.sendrawtransaction(tx1_conflict) + + self.sync_mempools() + + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + assert_equal(carol.getbalances()["mine"]["untrusted_pending"], Decimal("49.99000000")) + + assert tx1_txid not in bob.getrawmempool() + assert tx1_child_txid not in bob.getrawmempool() + assert tx1_conflict_txid in bob.getrawmempool() + assert_equal(len(bob.getrawmempool()), 1) + + # Now both tx1 and tx1_child are conflicted by tx1_conflict + assert_equal(bob.gettransaction(tx1_txid)["mempoolconflicts"], [tx1_conflict_txid]) + assert_equal(bob.gettransaction(tx1_child_txid)["mempoolconflicts"], [tx1_conflict_txid]) + + # Now create a conflict to tx1_conflict, so that it gets kicked out of the mempool + raw_tx = alice.createrawtransaction(inputs=[unspents[1]], outputs=[{carol.getnewaddress() : 24.9895}]) + tx1_conflict_conflict = alice.signrawtransactionwithwallet(raw_tx)['hex'] + tx1_conflict_conflict_txid = alice.sendrawtransaction(tx1_conflict_conflict) + + self.sync_mempools() + + # Now that tx1_conflict has been removed, both tx1 and tx1_child + assert_equal(bob.gettransaction(tx1_txid)["mempoolconflicts"], []) + assert_equal(bob.gettransaction(tx1_child_txid)["mempoolconflicts"], []) + + # Both tx1 and tx1_child are still not in the mempool because they have not be re-broadcasted + assert tx1_txid not in bob.getrawmempool() + assert tx1_child_txid not in bob.getrawmempool() + assert tx1_conflict_txid not in bob.getrawmempool() + assert tx1_conflict_conflict_txid in bob.getrawmempool() + assert_equal(len(bob.getrawmempool()), 1) + + assert_equal(alice.getbalance(), 0) + assert_equal(bob.getbalances()["mine"]["untrusted_pending"], 0) + assert_equal(carol.getbalances()["mine"]["untrusted_pending"], Decimal("24.98950000")) + + # Both tx1 and tx1_child can now be re-broadcasted + bob.sendrawtransaction(tx1) + bob.sendrawtransaction(tx1_child) + assert_equal(len(bob.getrawmempool()), 3) + + alice.unloadwallet() + bob.unloadwallet() + carol.unloadwallet() + if __name__ == '__main__': TxConflicts().main() diff --git a/test/functional/wallet_createwalletdescriptor.py b/test/functional/wallet_createwalletdescriptor.py new file mode 100755 index 0000000000..18e1703da3 --- /dev/null +++ b/test/functional/wallet_createwalletdescriptor.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test wallet createwalletdescriptor RPC.""" + +from test_framework.descriptors import descsum_create +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) +from test_framework.wallet_util import WalletUnlock + + +class WalletCreateDescriptorTest(BitcoinTestFramework): + def add_options(self, parser): + self.add_wallet_options(parser, descriptors=True, legacy=False) + + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self.test_basic() + self.test_imported_other_keys() + self.test_encrypted() + + def test_basic(self): + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet("blank", blank=True) + wallet = self.nodes[0].get_wallet_rpc("blank") + + xpub_info = def_wallet.gethdkeys(private=True) + xpub = xpub_info[0]["xpub"] + xprv = xpub_info[0]["xprv"] + expected_descs = [] + for desc in def_wallet.listdescriptors()["descriptors"]: + if desc["desc"].startswith("wpkh("): + expected_descs.append(desc["desc"]) + + assert_raises_rpc_error(-5, "Unable to determine which HD key to use from active descriptors. Please specify with 'hdkey'", wallet.createwalletdescriptor, "bech32") + assert_raises_rpc_error(-5, f"Private key for {xpub} is not known", wallet.createwalletdescriptor, type="bech32", hdkey=xpub) + + self.log.info("Test createwalletdescriptor after importing active descriptor to blank wallet") + # Import one active descriptor + assert_equal(wallet.importdescriptors([{"desc": descsum_create(f"pkh({xprv}/44h/2h/0h/0/0/*)"), "timestamp": "now", "active": True}])[0]["success"], True) + assert_equal(len(wallet.listdescriptors()["descriptors"]), 1) + assert_equal(len(wallet.gethdkeys()), 1) + + new_descs = wallet.createwalletdescriptor("bech32")["descs"] + assert_equal(len(new_descs), 2) + assert_equal(len(wallet.gethdkeys()), 1) + assert_equal(new_descs, expected_descs) + + self.log.info("Test descriptor creation options") + old_descs = set([(d["desc"], d["active"], d["internal"]) for d in wallet.listdescriptors(private=True)["descriptors"]]) + wallet.createwalletdescriptor(type="bech32m", internal=False) + curr_descs = set([(d["desc"], d["active"], d["internal"]) for d in wallet.listdescriptors(private=True)["descriptors"]]) + new_descs = list(curr_descs - old_descs) + assert_equal(len(new_descs), 1) + assert_equal(len(wallet.gethdkeys()), 1) + assert_equal(new_descs[0][0], descsum_create(f"tr({xprv}/86h/1h/0h/0/*)")) + assert_equal(new_descs[0][1], True) + assert_equal(new_descs[0][2], False) + + old_descs = curr_descs + wallet.createwalletdescriptor(type="bech32m", internal=True) + curr_descs = set([(d["desc"], d["active"], d["internal"]) for d in wallet.listdescriptors(private=True)["descriptors"]]) + new_descs = list(curr_descs - old_descs) + assert_equal(len(new_descs), 1) + assert_equal(len(wallet.gethdkeys()), 1) + assert_equal(new_descs[0][0], descsum_create(f"tr({xprv}/86h/1h/0h/1/*)")) + assert_equal(new_descs[0][1], True) + assert_equal(new_descs[0][2], True) + + def test_imported_other_keys(self): + self.log.info("Test createwalletdescriptor with multiple keys in active descriptors") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet("multiple_keys") + wallet = self.nodes[0].get_wallet_rpc("multiple_keys") + + wallet_xpub = wallet.gethdkeys()[0]["xpub"] + + xpub_info = def_wallet.gethdkeys(private=True) + xpub = xpub_info[0]["xpub"] + xprv = xpub_info[0]["xprv"] + + assert_equal(wallet.importdescriptors([{"desc": descsum_create(f"wpkh({xprv}/0/0/*)"), "timestamp": "now", "active": True}])[0]["success"], True) + assert_equal(len(wallet.gethdkeys()), 2) + + assert_raises_rpc_error(-5, "Unable to determine which HD key to use from active descriptors. Please specify with 'hdkey'", wallet.createwalletdescriptor, "bech32") + assert_raises_rpc_error(-4, "Descriptor already exists", wallet.createwalletdescriptor, type="bech32m", hdkey=wallet_xpub) + assert_raises_rpc_error(-5, "Unable to parse HD key. Please provide a valid xpub", wallet.createwalletdescriptor, type="bech32m", hdkey=xprv) + + # Able to replace tr() descriptor with other hd key + wallet.createwalletdescriptor(type="bech32m", hdkey=xpub) + + def test_encrypted(self): + self.log.info("Test createwalletdescriptor with encrypted wallets") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet("encrypted", blank=True, passphrase="pass") + wallet = self.nodes[0].get_wallet_rpc("encrypted") + + xpub_info = def_wallet.gethdkeys(private=True) + xprv = xpub_info[0]["xprv"] + + with WalletUnlock(wallet, "pass"): + assert_equal(wallet.importdescriptors([{"desc": descsum_create(f"wpkh({xprv}/0/0/*)"), "timestamp": "now", "active": True}])[0]["success"], True) + assert_equal(len(wallet.gethdkeys()), 1) + + assert_raises_rpc_error(-13, "Error: Please enter the wallet passphrase with walletpassphrase first.", wallet.createwalletdescriptor, type="bech32m") + + with WalletUnlock(wallet, "pass"): + wallet.createwalletdescriptor(type="bech32m") + + + +if __name__ == '__main__': + WalletCreateDescriptorTest().main() diff --git a/test/functional/wallet_gethdkeys.py b/test/functional/wallet_gethdkeys.py new file mode 100755 index 0000000000..f09b8c875a --- /dev/null +++ b/test/functional/wallet_gethdkeys.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test wallet gethdkeys RPC.""" + +from test_framework.descriptors import descsum_create +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) +from test_framework.wallet_util import WalletUnlock + + +class WalletGetHDKeyTest(BitcoinTestFramework): + def add_options(self, parser): + self.add_wallet_options(parser, descriptors=True, legacy=False) + + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self.test_basic_gethdkeys() + self.test_ranged_imports() + self.test_lone_key_imports() + self.test_ranged_multisig() + self.test_mixed_multisig() + + def test_basic_gethdkeys(self): + self.log.info("Test gethdkeys basics") + self.nodes[0].createwallet("basic") + wallet = self.nodes[0].get_wallet_rpc("basic") + xpub_info = wallet.gethdkeys() + assert_equal(len(xpub_info), 1) + assert_equal(xpub_info[0]["has_private"], True) + + assert "xprv" not in xpub_info[0] + xpub = xpub_info[0]["xpub"] + + xpub_info = wallet.gethdkeys(private=True) + xprv = xpub_info[0]["xprv"] + assert_equal(xpub_info[0]["xpub"], xpub) + assert_equal(xpub_info[0]["has_private"], True) + + descs = wallet.listdescriptors(True) + for desc in descs["descriptors"]: + assert xprv in desc["desc"] + + self.log.info("HD pubkey can be retrieved from encrypted wallets") + prev_xprv = xprv + wallet.encryptwallet("pass") + # HD key is rotated on encryption, there should now be 2 HD keys + assert_equal(len(wallet.gethdkeys()), 2) + # New key is active, should be able to get only that one and its descriptors + xpub_info = wallet.gethdkeys(active_only=True) + assert_equal(len(xpub_info), 1) + assert xpub_info[0]["xpub"] != xpub + assert "xprv" not in xpub_info[0] + assert_equal(xpub_info[0]["has_private"], True) + + self.log.info("HD privkey can be retrieved from encrypted wallets") + assert_raises_rpc_error(-13, "Error: Please enter the wallet passphrase with walletpassphrase first", wallet.gethdkeys, private=True) + with WalletUnlock(wallet, "pass"): + xpub_info = wallet.gethdkeys(active_only=True, private=True)[0] + assert xpub_info["xprv"] != xprv + for desc in wallet.listdescriptors(True)["descriptors"]: + if desc["active"]: + # After encrypting, HD key was rotated and should appear in all active descriptors + assert xpub_info["xprv"] in desc["desc"] + else: + # Inactive descriptors should have the previous HD key + assert prev_xprv in desc["desc"] + + def test_ranged_imports(self): + self.log.info("Keys of imported ranged descriptors appear in gethdkeys") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet("imports") + wallet = self.nodes[0].get_wallet_rpc("imports") + + xpub_info = wallet.gethdkeys() + assert_equal(len(xpub_info), 1) + active_xpub = xpub_info[0]["xpub"] + + import_xpub = def_wallet.gethdkeys(active_only=True)[0]["xpub"] + desc_import = def_wallet.listdescriptors(True)["descriptors"] + for desc in desc_import: + desc["active"] = False + wallet.importdescriptors(desc_import) + assert_equal(wallet.gethdkeys(active_only=True), xpub_info) + + xpub_info = wallet.gethdkeys() + assert_equal(len(xpub_info), 2) + for x in xpub_info: + if x["xpub"] == active_xpub: + for desc in x["descriptors"]: + assert_equal(desc["active"], True) + elif x["xpub"] == import_xpub: + for desc in x["descriptors"]: + assert_equal(desc["active"], False) + else: + assert False + + + def test_lone_key_imports(self): + self.log.info("Non-HD keys do not appear in gethdkeys") + self.nodes[0].createwallet("lonekey", blank=True) + wallet = self.nodes[0].get_wallet_rpc("lonekey") + + assert_equal(wallet.gethdkeys(), []) + wallet.importdescriptors([{"desc": descsum_create("wpkh(cTe1f5rdT8A8DFgVWTjyPwACsDPJM9ff4QngFxUixCSvvbg1x6sh)"), "timestamp": "now"}]) + assert_equal(wallet.gethdkeys(), []) + + self.log.info("HD keys of non-ranged descriptors should appear in gethdkeys") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + xpub_info = def_wallet.gethdkeys(private=True) + xpub = xpub_info[0]["xpub"] + xprv = xpub_info[0]["xprv"] + prv_desc = descsum_create(f"wpkh({xprv})") + pub_desc = descsum_create(f"wpkh({xpub})") + assert_equal(wallet.importdescriptors([{"desc": prv_desc, "timestamp": "now"}])[0]["success"], True) + xpub_info = wallet.gethdkeys() + assert_equal(len(xpub_info), 1) + assert_equal(xpub_info[0]["xpub"], xpub) + assert_equal(len(xpub_info[0]["descriptors"]), 1) + assert_equal(xpub_info[0]["descriptors"][0]["desc"], pub_desc) + assert_equal(xpub_info[0]["descriptors"][0]["active"], False) + + def test_ranged_multisig(self): + self.log.info("HD keys of a multisig appear in gethdkeys") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet("ranged_multisig") + wallet = self.nodes[0].get_wallet_rpc("ranged_multisig") + + xpub1 = wallet.gethdkeys()[0]["xpub"] + xprv1 = wallet.gethdkeys(private=True)[0]["xprv"] + xpub2 = def_wallet.gethdkeys()[0]["xpub"] + + prv_multi_desc = descsum_create(f"wsh(multi(2,{xprv1}/*,{xpub2}/*))") + pub_multi_desc = descsum_create(f"wsh(multi(2,{xpub1}/*,{xpub2}/*))") + assert_equal(wallet.importdescriptors([{"desc": prv_multi_desc, "timestamp": "now"}])[0]["success"], True) + + xpub_info = wallet.gethdkeys() + assert_equal(len(xpub_info), 2) + for x in xpub_info: + if x["xpub"] == xpub1: + found_desc = next((d for d in xpub_info[0]["descriptors"] if d["desc"] == pub_multi_desc), None) + assert found_desc is not None + assert_equal(found_desc["active"], False) + elif x["xpub"] == xpub2: + assert_equal(len(x["descriptors"]), 1) + assert_equal(x["descriptors"][0]["desc"], pub_multi_desc) + assert_equal(x["descriptors"][0]["active"], False) + else: + assert False + + def test_mixed_multisig(self): + self.log.info("Non-HD keys of a multisig do not appear in gethdkeys") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet("single_multisig") + wallet = self.nodes[0].get_wallet_rpc("single_multisig") + + xpub = wallet.gethdkeys()[0]["xpub"] + xprv = wallet.gethdkeys(private=True)[0]["xprv"] + pub = def_wallet.getaddressinfo(def_wallet.getnewaddress())["pubkey"] + + prv_multi_desc = descsum_create(f"wsh(multi(2,{xprv},{pub}))") + pub_multi_desc = descsum_create(f"wsh(multi(2,{xpub},{pub}))") + import_res = wallet.importdescriptors([{"desc": prv_multi_desc, "timestamp": "now"}]) + assert_equal(import_res[0]["success"], True) + + xpub_info = wallet.gethdkeys() + assert_equal(len(xpub_info), 1) + assert_equal(xpub_info[0]["xpub"], xpub) + found_desc = next((d for d in xpub_info[0]["descriptors"] if d["desc"] == pub_multi_desc), None) + assert found_desc is not None + assert_equal(found_desc["active"], False) + + +if __name__ == '__main__': + WalletGetHDKeyTest().main() diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index 3b407c285d..26477131cf 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -42,11 +42,6 @@ class WalletGroupTest(BitcoinTestFramework): def run_test(self): self.log.info("Setting up") - # To take full use of immediate tx relay, all nodes need to be reachable - # via inbound peers, i.e. connect first to last to close the circle - # (the default test network topology looks like this: - # node0 <-- node1 <-- node2 <-- node3 <-- node4 <-- node5) - self.connect_nodes(0, self.num_nodes - 1) # Mine some coins self.generate(self.nodes[0], COINBASE_MATURITY + 1) diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index 420bdffc49..f9d05a2fe4 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -688,7 +688,7 @@ class ImportDescriptorsTest(BitcoinTestFramework): encrypted_wallet.walletpassphrase("passphrase", 99999) with concurrent.futures.ThreadPoolExecutor(max_workers=1) as thread: - with self.nodes[0].assert_debug_log(expected_msgs=["Rescan started from block 0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206... (slow variant inspecting all blocks)"], timeout=5): + with self.nodes[0].assert_debug_log(expected_msgs=["Rescan started from block 0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206... (slow variant inspecting all blocks)"], timeout=10): importing = thread.submit(encrypted_wallet.importdescriptors, requests=[descriptor]) # Set the passphrase timeout to 1 to test that the wallet remains unlocked during the rescan diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index 48180e8294..e1bd85d8a9 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -25,8 +25,10 @@ class KeypoolRestoreTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 4 - self.extra_args = [[], ['-keypool=100'], ['-keypool=100'], ['-keypool=100']] + self.num_nodes = 5 + self.extra_args = [[]] + for _ in range(self.num_nodes - 1): + self.extra_args.append(['-keypool=100']) def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -40,12 +42,13 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.stop_node(1) shutil.copyfile(wallet_path, wallet_backup_path) self.start_node(1, self.extra_args[1]) - self.connect_nodes(0, 1) - self.connect_nodes(0, 2) - self.connect_nodes(0, 3) - - for i, output_type in enumerate(["legacy", "p2sh-segwit", "bech32"]): + for i in [1, 2, 3, 4]: + self.connect_nodes(0, i) + output_types = ["legacy", "p2sh-segwit", "bech32"] + if self.options.descriptors: + output_types.append("bech32m") + for i, output_type in enumerate(output_types): self.log.info("Generate keys for wallet with address type: {}".format(output_type)) idx = i+1 for _ in range(90): @@ -59,9 +62,10 @@ class KeypoolRestoreTest(BitcoinTestFramework): assert not address_details["isscript"] and not address_details["iswitness"] elif i == 1: assert address_details["isscript"] and not address_details["iswitness"] - else: + elif i == 2: assert not address_details["isscript"] and address_details["iswitness"] - + elif i == 3: + assert address_details["isscript"] and address_details["iswitness"] self.log.info("Send funds to wallet") self.nodes[0].sendtoaddress(addr_oldpool, 10) @@ -87,6 +91,8 @@ class KeypoolRestoreTest(BitcoinTestFramework): assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/49h/1h/0h/0/110") elif output_type == 'bech32': assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/84h/1h/0h/0/110") + elif output_type == 'bech32m': + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/86h/1h/0h/0/110") else: assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/0'/0'/110'") diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index b3edb0e253..558d63e85c 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -104,9 +104,11 @@ def main(): logging.error("Must have fuzz executable built") sys.exit(1) + fuzz_bin=os.getenv("BITCOINFUZZ", default=os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz')) + # Build list of tests test_list_all = parse_test_list( - fuzz_bin=os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'), + fuzz_bin=fuzz_bin, source_dir=config['environment']['SRCDIR'], ) @@ -151,7 +153,7 @@ def main(): try: help_output = subprocess.run( args=[ - os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'), + fuzz_bin, '-help=1', ], env=get_fuzz_env(target=test_list_selection[0], source_dir=config['environment']['SRCDIR']), @@ -173,7 +175,7 @@ def main(): return generate_corpus( fuzz_pool=fuzz_pool, src_dir=config['environment']['SRCDIR'], - build_dir=config["environment"]["BUILDDIR"], + fuzz_bin=fuzz_bin, corpus_dir=args.corpus_dir, targets=test_list_selection, ) @@ -184,7 +186,7 @@ def main(): corpus=args.corpus_dir, test_list=test_list_selection, src_dir=config['environment']['SRCDIR'], - build_dir=config["environment"]["BUILDDIR"], + fuzz_bin=fuzz_bin, merge_dirs=[Path(m_dir) for m_dir in args.m_dir], ) return @@ -194,7 +196,7 @@ def main(): corpus=args.corpus_dir, test_list=test_list_selection, src_dir=config['environment']['SRCDIR'], - build_dir=config["environment"]["BUILDDIR"], + fuzz_bin=fuzz_bin, using_libfuzzer=using_libfuzzer, use_valgrind=args.valgrind, empty_min_time=args.empty_min_time, @@ -237,7 +239,7 @@ def transform_rpc_target(targets, src_dir): return targets -def generate_corpus(*, fuzz_pool, src_dir, build_dir, corpus_dir, targets): +def generate_corpus(*, fuzz_pool, src_dir, fuzz_bin, corpus_dir, targets): """Generates new corpus. Run {targets} without input, and outputs the generated corpus to @@ -270,7 +272,7 @@ def generate_corpus(*, fuzz_pool, src_dir, build_dir, corpus_dir, targets): os.makedirs(target_corpus_dir, exist_ok=True) use_value_profile = int(random.random() < .3) command = [ - os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), + fuzz_bin, "-rss_limit_mb=8000", "-max_total_time=6000", "-reload=0", @@ -283,12 +285,12 @@ def generate_corpus(*, fuzz_pool, src_dir, build_dir, corpus_dir, targets): future.result() -def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, build_dir, merge_dirs): +def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, merge_dirs): logging.info(f"Merge the inputs from the passed dir into the corpus_dir. Passed dirs {merge_dirs}") jobs = [] for t in test_list: args = [ - os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), + fuzz_bin, '-rss_limit_mb=8000', '-set_cover_merge=1', # set_cover_merge is used instead of -merge=1 to reduce the overall @@ -325,13 +327,13 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, build_dir, merge_dirs future.result() -def run_once(*, fuzz_pool, corpus, test_list, src_dir, build_dir, using_libfuzzer, use_valgrind, empty_min_time): +def run_once(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, using_libfuzzer, use_valgrind, empty_min_time): jobs = [] for t in test_list: corpus_path = corpus / t os.makedirs(corpus_path, exist_ok=True) args = [ - os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), + fuzz_bin, ] empty_dir = not any(corpus_path.iterdir()) if using_libfuzzer: diff --git a/test/lint/README.md b/test/lint/README.md index 83264de06e..9d167bac72 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -87,3 +87,7 @@ To do so, add the upstream repository as remote: ``` git remote add --fetch secp256k1 https://github.com/bitcoin-core/secp256k1.git ``` + +lint_ignore_dirs.py +=================== +Add list of common directories to ignore when running tests diff --git a/test/lint/lint-git-commit-check.py b/test/lint/lint-git-commit-check.py index 5897a17e70..5dc30cc755 100755 --- a/test/lint/lint-git-commit-check.py +++ b/test/lint/lint-git-commit-check.py @@ -23,31 +23,18 @@ def parse_args(): """, epilog=f""" You can manually set the commit-range with the COMMIT_RANGE - environment variable (e.g. "COMMIT_RANGE='47ba2c3...ee50c9e' - {sys.argv[0]}"). Defaults to current merge base when neither - prev-commits nor the environment variable is set. + environment variable (e.g. "COMMIT_RANGE='HEAD~n..HEAD' + {sys.argv[0]}") for the last 'n' commits. """) - - parser.add_argument("--prev-commits", "-p", required=False, help="The previous n commits to check") - return parser.parse_args() def main(): - args = parse_args() + parse_args() exit_code = 0 - if not os.getenv("COMMIT_RANGE"): - if args.prev_commits: - commit_range = "HEAD~" + args.prev_commits + "...HEAD" - else: - # This assumes that the target branch of the pull request will be master. - merge_base = check_output(["git", "merge-base", "HEAD", "master"], text=True, encoding="utf8").rstrip("\n") - commit_range = merge_base + "..HEAD" - else: - commit_range = os.getenv("COMMIT_RANGE") - if commit_range == "SKIP_EMPTY_NOT_A_PR": - sys.exit(0) + assert os.getenv("COMMIT_RANGE") # E.g. COMMIT_RANGE='HEAD~n..HEAD' + commit_range = os.getenv("COMMIT_RANGE") commit_hashes = check_output(["git", "log", commit_range, "--format=%H"], text=True, encoding="utf8").splitlines() diff --git a/test/lint/lint-include-guards.py b/test/lint/lint-include-guards.py index 291e528c1d..77af05c1c2 100755 --- a/test/lint/lint-include-guards.py +++ b/test/lint/lint-include-guards.py @@ -12,19 +12,17 @@ import re import sys from subprocess import check_output +from lint_ignore_dirs import SHARED_EXCLUDED_SUBTREES + HEADER_ID_PREFIX = 'BITCOIN_' HEADER_ID_SUFFIX = '_H' EXCLUDE_FILES_WITH_PREFIX = ['contrib/devtools/bitcoin-tidy', 'src/crypto/ctaes', - 'src/leveldb', - 'src/crc32c', - 'src/secp256k1', - 'src/minisketch', 'src/tinyformat.h', 'src/bench/nanobench.h', - 'src/test/fuzz/FuzzedDataProvider.h'] + 'src/test/fuzz/FuzzedDataProvider.h'] + SHARED_EXCLUDED_SUBTREES def _get_header_file_lst() -> list[str]: diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py index 6386a92c0f..90884299d5 100755 --- a/test/lint/lint-includes.py +++ b/test/lint/lint-includes.py @@ -14,13 +14,11 @@ import sys from subprocess import check_output, CalledProcessError +from lint_ignore_dirs import SHARED_EXCLUDED_SUBTREES + EXCLUDED_DIRS = ["contrib/devtools/bitcoin-tidy/", - "src/leveldb/", - "src/crc32c/", - "src/secp256k1/", - "src/minisketch/", - ] + ] + SHARED_EXCLUDED_SUBTREES EXPECTED_BOOST_INCLUDES = ["boost/date_time/posix_time/posix_time.hpp", "boost/multi_index/detail/hash_index_iterator.hpp", @@ -32,7 +30,6 @@ EXPECTED_BOOST_INCLUDES = ["boost/date_time/posix_time/posix_time.hpp", "boost/multi_index/tag.hpp", "boost/multi_index_container.hpp", "boost/operators.hpp", - "boost/process.hpp", "boost/signals2/connection.hpp", "boost/signals2/optional_last_value.hpp", "boost/signals2/signal.hpp", diff --git a/test/lint/lint-spelling.py b/test/lint/lint-spelling.py index ac0bddeaa6..3e578b218f 100755 --- a/test/lint/lint-spelling.py +++ b/test/lint/lint-spelling.py @@ -11,8 +11,11 @@ Note: Will exit successfully regardless of spelling errors. from subprocess import check_output, STDOUT, CalledProcessError +from lint_ignore_dirs import SHARED_EXCLUDED_SUBTREES + IGNORE_WORDS_FILE = 'test/lint/spelling.ignore-words.txt' -FILES_ARGS = ['git', 'ls-files', '--', ":(exclude)build-aux/m4/", ":(exclude)contrib/seeds/*.txt", ":(exclude)depends/", ":(exclude)doc/release-notes/", ":(exclude)src/leveldb/", ":(exclude)src/crc32c/", ":(exclude)src/qt/locale/", ":(exclude)src/qt/*.qrc", ":(exclude)src/secp256k1/", ":(exclude)src/minisketch/", ":(exclude)contrib/guix/patches"] +FILES_ARGS = ['git', 'ls-files', '--', ":(exclude)build-aux/m4/", ":(exclude)contrib/seeds/*.txt", ":(exclude)depends/", ":(exclude)doc/release-notes/", ":(exclude)src/qt/locale/", ":(exclude)src/qt/*.qrc", ":(exclude)contrib/guix/patches"] +FILES_ARGS += [f":(exclude){dir}" for dir in SHARED_EXCLUDED_SUBTREES] def check_codespell_install(): diff --git a/test/lint/lint_ignore_dirs.py b/test/lint/lint_ignore_dirs.py new file mode 100644 index 0000000000..af9ee7ef6b --- /dev/null +++ b/test/lint/lint_ignore_dirs.py @@ -0,0 +1,5 @@ +SHARED_EXCLUDED_SUBTREES = ["src/leveldb/", + "src/crc32c/", + "src/secp256k1/", + "src/minisketch/", + ] |