diff options
486 files changed, 10054 insertions, 5669 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index cd522a4978..40db8321b2 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -193,17 +193,13 @@ task: task: name: 'ARM [unit tests, no functional tests] [bullseye]' << : *GLOBAL_TASK_TEMPLATE - arm_container: - cpu: 2 - memory: 8G - dockerfile: ci/test_imagefile + container: docker_arguments: CI_IMAGE_NAME_TAG: debian:bullseye FILE_ENV: "./ci/test/00_setup_env_arm.sh" << : *CREDITS_TEMPLATE env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - QEMU_USER_CMD: "" # Disable qemu and run the test natively task: name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [jammy]' @@ -248,15 +244,16 @@ task: docker_arguments: CI_IMAGE_NAME_TAG: ubuntu:lunar FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" + << : *CREDITS_TEMPLATE env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: - name: '[MSan, depends] [lunar]' + name: '[MSan, depends] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:lunar + CI_IMAGE_NAME_TAG: ubuntu:jammy FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV @@ -317,11 +314,11 @@ task: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: - name: 'macOS 10.15 [gui, no tests] [focal]' + name: 'macOS 11.0 [gui, no tests] [jammy]' << : *CONTAINER_DEPENDS_TEMPLATE container: docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:focal + CI_IMAGE_NAME_TAG: ubuntu:jammy FILE_ENV: "./ci/test/00_setup_env_mac.sh" << : *CREDITS_TEMPLATE macos_sdk_cache: @@ -336,7 +333,7 @@ task: name: 'macOS 13 native arm64 [gui, sqlite only] [no depends]' macos_instance: # Use latest image, but hardcode version to avoid silent upgrades (and breaks) - image: ghcr.io/cirruslabs/macos-ventura-xcode:14.1 # https://cirrus-ci.org/guide/macOS + image: ghcr.io/cirruslabs/macos-ventura-xcode:14.3.1 # https://cirrus-ci.org/guide/macOS << : *BASE_TEMPLATE check_clang_script: - clang --version @@ -348,21 +345,3 @@ task: CI_USE_APT_INSTALL: "no" PACKAGE_MANAGER_INSTALL: "echo" # Nothing to do FILE_ENV: "./ci/test/00_setup_env_mac_native_arm64.sh" - -task: - name: 'ARM64 Android APK [jammy]' - << : *CONTAINER_DEPENDS_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:jammy - FILE_ENV: "./ci/test/00_setup_env_android.sh" - << : *CREDITS_TEMPLATE - android_sdk_cache: - folder: "depends/SDKs/android" - fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.2.8568313" - depends_sources_cache: - folder: "depends/sources" - fingerprint_script: git rev-parse HEAD:depends/packages - << : *MAIN_TEMPLATE - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV diff --git a/.style.yapf b/.style.yapf index 69d8c6aee4..350ac63855 100644 --- a/.style.yapf +++ b/.style.yapf @@ -107,7 +107,7 @@ each_dict_entry_on_separate_line=True i18n_comment= # The i18n function call names. The presence of this function stops -# reformattting on that line, because the string it has cannot be moved +# reformatting on that line, because the string it has cannot be moved # away from the i18n comment. i18n_function_call= diff --git a/autogen.sh b/autogen.sh index d0ac7ef7ed..69c892ffa0 100755 --- a/autogen.sh +++ b/autogen.sh @@ -16,10 +16,14 @@ command -v autoreconf >/dev/null || \ autoreconf --install --force --warnings=all if expr "'$(build-aux/config.guess --timestamp)" \< "'$(depends/config.guess --timestamp)" > /dev/null; then + chmod ug+w build-aux/config.guess + chmod ug+w src/secp256k1/build-aux/config.guess cp depends/config.guess build-aux cp depends/config.guess src/secp256k1/build-aux fi if expr "'$(build-aux/config.sub --timestamp)" \< "'$(depends/config.sub --timestamp)" > /dev/null; then + chmod ug+w build-aux/config.sub + chmod ug+w src/secp256k1/build-aux/config.sub cp depends/config.sub build-aux cp depends/config.sub src/secp256k1/build-aux fi diff --git a/build_msvc/libsecp256k1/libsecp256k1.vcxproj b/build_msvc/libsecp256k1/libsecp256k1.vcxproj index 585c28e503..777515aa3a 100644 --- a/build_msvc/libsecp256k1/libsecp256k1.vcxproj +++ b/build_msvc/libsecp256k1/libsecp256k1.vcxproj @@ -14,7 +14,7 @@ </ItemGroup> <ItemDefinitionGroup> <ClCompile> - <PreprocessorDefinitions>ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;ENABLE_MODULE_ELLSWIFT;%(PreprocessorDefinitions)</PreprocessorDefinitions> <UndefinePreprocessorDefinitions>USE_ASM_X86_64;%(UndefinePreprocessorDefinitions)</UndefinePreprocessorDefinitions> <AdditionalIncludeDirectories>..\..\src\secp256k1;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <DisableSpecificWarnings>4146;4244;4267;4334</DisableSpecificWarnings> diff --git a/ci/README.md b/ci/README.md index d014565f44..b4158d0183 100644 --- a/ci/README.md +++ b/ci/README.md @@ -20,12 +20,6 @@ requires `bash`, `docker`, and `python3` to be installed. To install all require sudo apt install bash docker.io python3 ``` -To run the default test stage, - -``` -./ci/test_run_all.sh -``` - To run the test stage with a specific configuration, ``` diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index f7147582dc..b3b1c91ee0 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -33,10 +33,11 @@ if [ -z "${SKIP_PYTHON_INSTALL}" ]; then python3 --version fi -${CI_RETRY_EXE} pip3 install codespell==2.2.1 -${CI_RETRY_EXE} pip3 install flake8==5.0.4 -${CI_RETRY_EXE} pip3 install mypy==0.971 -${CI_RETRY_EXE} pip3 install pyzmq==24.0.1 +${CI_RETRY_EXE} pip3 install codespell==2.2.5 +${CI_RETRY_EXE} pip3 install flake8==6.0.0 +${CI_RETRY_EXE} pip3 install lief==0.13.2 +${CI_RETRY_EXE} pip3 install mypy==1.4.1 +${CI_RETRY_EXE} pip3 install pyzmq==25.1.0 ${CI_RETRY_EXE} pip3 install vulture==2.6 SHELLCHECK_VERSION=v0.8.0 diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index d98f05ca6b..69fd05051e 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -6,6 +6,8 @@ export LC_ALL=C.UTF-8 +set -ex + # The root dir. # The ci system copies this folder. BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) @@ -44,8 +46,6 @@ export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-40} export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} -export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} -export CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG:-ubuntu:20.04} # Randomize test order. # See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/random.html export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 9e3ea0d383..7911c1912f 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -9,9 +9,9 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_multiprocess export CI_IMAGE_NAME_TAG=ubuntu:20.04 -export PACKAGES="cmake python3 llvm clang g++-multilib" +export PACKAGES="cmake llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' LDFLAGS='--rtlib=compiler-rt -lgcc_s'" +export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' \ +LDFLAGS='--rtlib=compiler-rt -lgcc_s' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE'" export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" -export TEST_RUNNER_EXTRA="--nosandbox" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index fe42871c31..65c6119fcd 100755 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross -export CI_IMAGE_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos +export CI_IMAGE_NAME_TAG=ubuntu:22.04 export HOST=x86_64-apple-darwin export PACKAGES="cmake libz-dev libtinfo5 python3-setuptools xorriso" export XCODE_VERSION=12.2 @@ -15,4 +15,7 @@ export XCODE_BUILD_ID=12B45b export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="--with-gui --enable-reduce-exports" + +# False-positive warning is fixed with clang 17, remove this when that version +# can be used. +export BITCOIN_CONFIG="--with-gui --enable-reduce-exports LDFLAGS=-Wno-error=unused-command-line-argument" diff --git a/ci/test/00_setup_env_mac_native_arm64.sh b/ci/test/00_setup_env_mac_native_arm64.sh index a6799d7b88..09c05f3bbd 100755 --- a/ci/test/00_setup_env_mac_native_arm64.sh +++ b/ci/test/00_setup_env_mac_native_arm64.sh @@ -14,3 +14,5 @@ export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" export CCACHE_SIZE=300M +export RUN_FUZZ_TESTS=true +export FUZZ_TESTS_CONFIG="--exclude banman" # https://github.com/bitcoin/bitcoin/issues/27924 diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 8701d383dd..a5c80c2afc 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -14,8 +14,10 @@ else fi export CONTAINER_NAME=ci_native_asan -export PACKAGES="systemtap-sdt-dev clang-16 llvm-16 libclang-rt-16-dev python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" +export PACKAGES="systemtap-sdt-dev clang-16 llvm-16 libclang-rt-16-dev python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" export CI_IMAGE_NAME_TAG=ubuntu:23.04 # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang-16 CXX=clang++-16" +export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \ +CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \ +--with-sanitizers=address,float-divide-by-zero,integer,undefined CC=clang-16 CXX=clang++-16" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 05cb45c2d8..481925dbc1 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -8,11 +8,12 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). export CONTAINER_NAME=ci_native_fuzz -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev" +export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libevent-dev libboost-dev libsqlite3-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC='clang-16 -ftrivial-auto-var-init=pattern' CXX='clang++-16 -ftrivial-auto-var-init=pattern'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,float-divide-by-zero,integer \ +CC='clang-16 -ftrivial-auto-var-init=pattern' CXX='clang++-16 -ftrivial-auto-var-init=pattern'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 35a0de8034..1f9adc0682 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -6,18 +6,18 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" +LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" -LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" +LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_fuzz_msan" -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake" +export PACKAGES="cmake ninja-build" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 5fee10e37e..b2213e2f77 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="debian:bookworm" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="clang llvm libclang-rt-dev python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev valgrind" +export PACKAGES="clang llvm libclang-rt-dev libevent-dev libboost-dev libsqlite3-dev valgrind" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index bdb9bd7b5d..34d60c6c4c 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -1,23 +1,23 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2022 The Bitcoin Core developers +# Copyright (c) 2020-2023 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" +LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" -LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" +LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake" +export PACKAGES="cmake ninja-build" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" export CCACHE_SIZE=250M diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index bb10a2a2de..3a1d7808f1 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -17,5 +17,5 @@ export RUN_UNIT_TESTS="false" export GOAL="install" export NO_WERROR=1 # -Werror=maybe-uninitialized export DOWNLOAD_PREVIOUS_RELEASES="true" -export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports \ ---enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" +export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports --enable-debug \ +CFLAGS=\"-g0 -O2 -funsigned-char\" CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS=\"-g0 -O2 -funsigned-char\"" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index 5c642b0d96..920180a274 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2022 The Bitcoin Core developers +# Copyright (c) 2023 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index e3fa7ab777..8ebb1fa563 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-2023 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 97b85755ef..d6dcf73182 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -8,10 +8,10 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="debian:bookworm" export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang llvm libclang-rt-dev python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" +export PACKAGES="valgrind clang llvm libclang-rt-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--nosandbox --exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" # Temporarily pin dwarf 4, until using Valgrind 3.20 or later export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index af18703ce1..523e81c94a 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -23,4 +23,4 @@ export TEST_RUNNER_ENV="LC_ALL=C" export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests" # GUI tests disabled for now, see https://github.com/bitcoin/bitcoin/issues/23730 +export BITCOIN_CONFIG="--enable-reduce-exports" diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 3adfbf6e47..d8c36ccf85 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -10,7 +10,7 @@ export CONTAINER_NAME=ci_win64 export CI_IMAGE_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to win64 export HOST=x86_64-w64-mingw32 export DPKG_ADD_ARCH="i386" -export PACKAGES="python3 nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file" +export PACKAGES="nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file" export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" export BITCOIN_CONFIG="--enable-reduce-exports --enable-external-signer --disable-gui-tests" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index beb5aa9242..76cde42161 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -6,6 +6,8 @@ export LC_ALL=C.UTF-8 +set -ex + CFG_DONE="ci.base-install-done" # Use a global git setting to remember whether this script ran to avoid running it twice if [ "$(git config --global ${CFG_DONE})" == "true" ]; then @@ -40,11 +42,33 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - update-alternatives --install /usr/bin/clang++ clang++ "$(which clang++-16)" 100 - update-alternatives --install /usr/bin/clang clang "$(which clang-16)" 100 - git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.1 "${BASE_SCRATCH_DIR}"/msan/llvm-project - cmake -B "${BASE_SCRATCH_DIR}"/msan/build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DLIBCXX_ENABLE_DEBUG_MODE=ON -DLIBCXX_ENABLE_ASSERTIONS=ON -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes - make -C "${BASE_SCRATCH_DIR}"/msan/build/ "$MAKEJOBS" + git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.6 "${BASE_SCRATCH_DIR}"/msan/llvm-project + + cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/clang_build/ -DLLVM_ENABLE_PROJECTS="clang" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ + -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/llvm + + ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ "$MAKEJOBS" + ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ install-runtimes + + update-alternatives --install /usr/bin/clang++ clang++ "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang++ 100 + update-alternatives --install /usr/bin/clang clang "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang 100 + update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/llvm-symbolizer 100 + + cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/cxx_build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_USE_SANITIZER=MemoryWithOrigins \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ + -DLIBCXX_ENABLE_DEBUG_MODE=ON \ + -DLIBCXX_ENABLE_ASSERTIONS=ON \ + -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes + + ninja -C "${BASE_SCRATCH_DIR}"/msan/cxx_build/ "$MAKEJOBS" fi if [[ "${RUN_TIDY}" == "true" ]]; then diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index e9c54139a7..626461df03 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -28,7 +28,7 @@ export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/" if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then # Export all env vars to avoid missing some. # Though, exclude those with newlines to avoid parsing problems. - python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value]' | tee /tmp/env + python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value and "HOME" not in key]' | tee /tmp/env echo "Creating $CI_IMAGE_NAME_TAG container to run in" DOCKER_BUILDKIT=1 ${CI_RETRY_EXE} docker build \ --file "${BASE_ROOT_DIR}/ci/test_imagefile" \ @@ -42,7 +42,9 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then echo "Restart docker before run to stop and clear all containers started with --rm" - systemctl restart docker + podman container stop --all # Similar to "systemctl restart docker" + echo "Prune all dangling images" + docker image prune --force fi # shellcheck disable=SC2086 diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index ce5163caa8..9993ff68d1 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -67,7 +67,7 @@ if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" fi -BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking" +BITCOIN_CONFIG_ALL="--disable-dependency-tracking" if [ -z "$NO_DEPENDS" ]; then BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site" fi @@ -155,9 +155,7 @@ if [ "${RUN_TIDY}" = "true" ]; then # accepted in src/.bear-tidy-config # Filter out: # * qt qrc and moc generated files - # * walletutil (temporarily) - # * secp256k1 - jq 'map(select(.file | test("src/qt/qrc_.*\\.cpp$|/moc_.*\\.cpp$|src/wallet/walletutil|src/secp256k1/src/") | not))' ../compile_commands.json > tmp.json + jq 'map(select(.file | test("src/qt/qrc_.*\\.cpp$|/moc_.*\\.cpp$") | not))' ../compile_commands.json > tmp.json mv tmp.json ../compile_commands.json cd "${BASE_BUILD_DIR}/bitcoin-$HOST/" python3 "${DIR_IWYU}/include-what-you-use/iwyu_tool.py" \ @@ -171,5 +169,5 @@ if [ "${RUN_TIDY}" = "true" ]; then fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then - bash -c "LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN}" + bash -c "LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN} --empty_min_time=60" fi diff --git a/configure.ac b/configure.ac index fc692c4f6a..29ac8e3805 100644 --- a/configure.ac +++ b/configure.ac @@ -96,12 +96,6 @@ case $host in ;; esac -AC_ARG_WITH([seccomp], - [AS_HELP_STRING([--with-seccomp], - [enable experimental syscall sandbox feature (-sandbox), default is yes if seccomp-bpf is detected under Linux x86_64])], - [seccomp_found=$withval], - [seccomp_found=auto]) - AC_ARG_ENABLE([c++20], [AS_HELP_STRING([--enable-c++20], [enable compilation in c++20 mode (disabled by default)])], @@ -244,10 +238,10 @@ dnl May be useful if warnings from external headers clutter the build output dnl too much, so that it becomes difficult to spot Bitcoin Core warnings dnl or if they cause a build failure with --enable-werror. AC_ARG_ENABLE([suppress-external-warnings], - [AS_HELP_STRING([--enable-suppress-external-warnings], - [Suppress warnings from external headers (default is no)])], + [AS_HELP_STRING([--disable-suppress-external-warnings], + [Do not suppress warnings from external headers (default is to suppress)])], [suppress_external_warnings=$enableval], - [suppress_external_warnings=no]) + [suppress_external_warnings=yes]) AC_ARG_ENABLE([lcov], [AS_HELP_STRING([--enable-lcov], @@ -1008,6 +1002,7 @@ if test "$TARGET_OS" = "darwin"; then AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR]) AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR]) AX_CHECK_LINK_FLAG([-Wl,-bind_at_load], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-fixup_chains], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-fixup_chains"], [], [$LDFLAG_WERROR]) fi AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) @@ -1491,10 +1486,6 @@ if test "$use_boost" = "yes"; then AX_CHECK_PREPROC_FLAG([-DBOOST_NO_CXX98_FUNCTION_BASE], [BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_NO_CXX98_FUNCTION_BASE"], [], [$CXXFLAG_WERROR], [AC_LANG_PROGRAM([[#include <boost/config.hpp>]])]) - if test "$enable_debug" = "yes" || test "$enable_fuzz" = "yes"; then - BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE" - fi - if test "$suppress_external_warnings" != "no"; then BOOST_CPPFLAGS=SUPPRESS_WARNINGS($BOOST_CPPFLAGS) fi @@ -1543,36 +1534,6 @@ if test "$use_external_signer" != "no"; then fi AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "$use_external_signer" = "yes"]) -dnl Do not compile with syscall sandbox support when compiling under the sanitizers. -dnl The sanitizers introduce use of syscalls that are not typically used in bitcoind -dnl (such as execve when the sanitizers execute llvm-symbolizer). -if test "$use_sanitizers" != ""; then - AC_MSG_WARN([Specifying --with-sanitizers forces --without-seccomp since the sanitizers introduce use of syscalls not allowed by the bitcoind syscall sandbox (-sandbox=<mode>).]) - seccomp_found=no -fi -if test "$seccomp_found" != "no"; then - AC_MSG_CHECKING([for seccomp-bpf (Linux x86-64)]) - AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ - @%:@include <linux/seccomp.h> - ]], [[ - #if !defined(__x86_64__) - # error Syscall sandbox is an experimental feature currently available only under Linux x86-64. - #endif - ]])],[ - AC_MSG_RESULT([yes]) - seccomp_found="yes" - AC_DEFINE([USE_SYSCALL_SANDBOX], [1], [Define this symbol to build with syscall sandbox support.]) - ],[ - AC_MSG_RESULT([no]) - seccomp_found="no" - ]) -fi -dnl Currently only enable -sandbox=<mode> feature if seccomp is found. -dnl In the future, sandboxing could be also be supported with other -dnl sandboxing mechanisms besides seccomp. -use_syscall_sandbox=$seccomp_found -AM_CONDITIONAL([ENABLE_SYSCALL_SANDBOX], [test "$use_syscall_sandbox" != "no"]) - dnl Check for reduced exports if test "$use_reduce_exports" = "yes"; then AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fvisibility=hidden"], @@ -2012,7 +1973,6 @@ echo echo "Options used to compile and link:" echo " external signer = $use_external_signer" echo " multiprocess = $build_multiprocess" -echo " with experimental syscall sandbox support = $use_syscall_sandbox" echo " with libs = $build_bitcoin_libs" echo " with wallet = $enable_wallet" if test "$enable_wallet" != "no"; then diff --git a/contrib/README.md b/contrib/README.md index 3c6e978061..f375993ac4 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -35,7 +35,7 @@ Test and Verify Tools ### [TestGen](/contrib/testgen) ### Utilities to generate test vectors for the data-driven Bitcoin tests. -### [Verify Binaries](/contrib/verifybinaries) ### +### [Verify-Binaries](/contrib/verify-binaries) ### This script attempts to download and verify the signature file SHA256SUMS.asc from bitcoin.org. Command Line Tools diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 6cd022ef17..f90fa5785f 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -10,7 +10,7 @@ Otherwise the exit status will be 1 and it will log which executables failed whi import sys from typing import List -import lief #type:ignore +import lief def check_ELF_RELRO(binary) -> bool: ''' @@ -113,7 +113,7 @@ def check_ELF_control_flow(binary) -> bool: main = binary.get_function_address('main') content = binary.get_content_from_virtual_address(main, 4, lief.Binary.VA_TYPES.AUTO) - if content == [243, 15, 30, 250]: # endbr64 + if content.tolist() == [243, 15, 30, 250]: # endbr64 return True return False @@ -142,7 +142,7 @@ def check_PE_control_flow(binary) -> bool: content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) - if content == [243, 15, 30, 250]: # endbr64 + if content.tolist() == [243, 15, 30, 250]: # endbr64 return True return False @@ -158,12 +158,11 @@ def check_MACHO_NOUNDEFS(binary) -> bool: ''' return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS) -def check_MACHO_LAZY_BINDINGS(binary) -> bool: +def check_MACHO_FIXUP_CHAINS(binary) -> bool: ''' - Check for no lazy bindings. - We don't use or check for MH_BINDATLOAD. See #18295. + Check for use of chained fixups. ''' - return binary.dyld_info.lazy_bind == (0,0) + return binary.has_dyld_chained_fixups def check_MACHO_Canary(binary) -> bool: ''' @@ -190,7 +189,7 @@ def check_MACHO_control_flow(binary) -> bool: ''' content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) - if content == [243, 15, 30, 250]: # endbr64 + if content.tolist() == [243, 15, 30, 250]: # endbr64 return True return False @@ -214,8 +213,8 @@ BASE_PE = [ BASE_MACHO = [ ('NOUNDEFS', check_MACHO_NOUNDEFS), - ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS), ('Canary', check_MACHO_Canary), + ('FIXUP_CHAINS', check_MACHO_FIXUP_CHAINS), ] CHECKS = { diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 3507f954f3..d85912398d 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -13,7 +13,7 @@ Example usage: import sys from typing import List, Dict -import lief #type:ignore +import lief # Debian 10 (Buster) EOL: 2024. https://wiki.debian.org/LTS # @@ -232,7 +232,7 @@ def check_MACHO_libraries(binary) -> bool: return ok def check_MACHO_min_os(binary) -> bool: - if binary.build_version.minos == [10,15,0]: + if binary.build_version.minos == [11,0,0]: return True return False diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 54718fd7a1..802bf9fd30 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -5,7 +5,7 @@ ''' Test script for security-check.py ''' -import lief #type:ignore +import lief import os import subprocess from typing import List @@ -28,7 +28,7 @@ def clean_files(source, executable): os.remove(source) os.remove(executable) -def call_security_check(cc, source, executable, options): +def call_security_check(cc: str, source: str, executable: str, options) -> tuple: # This should behave the same as AC_TRY_LINK, so arrange well-known flags # in the same order as autoconf would. # @@ -119,29 +119,31 @@ class TestSecurityChecks(unittest.TestCase): arch = get_arch(cc, source, executable) if arch == lief.ARCHITECTURES.X86: - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary PIE NX CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE NX CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']), - (1, executable+': failed LAZY_BINDINGS PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector', '-Wl,-no_fixup_chains']), + (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector', '-Wl,-fixup_chains']), + (1, executable+': failed NOUNDEFS Canary PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all', '-Wl,-fixup_chains']), + (1, executable+': failed NOUNDEFS PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains']), + (1, executable+': failed NOUNDEFS PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-Wl,-fixup_chains']), (1, executable+': failed PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-Wl,-fixup_chains']), + (1, executable+': failed PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']), (1, executable+': failed PIE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']), (0, '')) else: # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS')) - self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all']), - (1, executable+': failed LAZY_BINDINGS')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-bind_at_load','-fstack-protector-all']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-no_fixup_chains']), + (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-fixup_chains']), + (1, executable+': failed NOUNDEFS Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains']), + (1, executable+': failed NOUNDEFS')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-bind_at_load','-fstack-protector-all', '-Wl,-fixup_chains']), (0, '')) diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index e304880140..fe8a99739f 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -121,7 +121,7 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,10.15', '-Wl,11.4']), + self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,11.0', '-Wl,11.4']), (1, f'{executable}: failed SDK')) def test_PE(self): diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 54ddd47353..a9654dec5b 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -1,44 +1,37 @@ -(use-modules (gnu) - (gnu packages) +(use-modules (gnu packages) (gnu packages autotools) - (gnu packages base) - (gnu packages bash) + ((gnu packages bash) #:select (bash-minimal)) (gnu packages bison) - (gnu packages certs) - (gnu packages cdrom) - (gnu packages check) - (gnu packages cmake) + ((gnu packages certs) #:select (nss-certs)) + ((gnu packages cdrom) #:select (xorriso)) + ((gnu packages cmake) #:select (cmake-minimal)) (gnu packages commencement) (gnu packages compression) (gnu packages cross-base) - (gnu packages curl) (gnu packages file) (gnu packages gawk) (gnu packages gcc) - (gnu packages gnome) - (gnu packages installers) - (gnu packages linux) + ((gnu packages installers) #:select (nsis-x86_64)) + ((gnu packages linux) #:select (linux-libre-headers-5.15 util-linux)) (gnu packages llvm) (gnu packages mingw) (gnu packages moreutils) (gnu packages pkg-config) - (gnu packages python) - (gnu packages python-crypto) - (gnu packages python-web) - (gnu packages shells) - (gnu packages tls) - (gnu packages version-control) + ((gnu packages python) #:select (python-minimal)) + ((gnu packages python-build) #:select (python-tomli)) + ((gnu packages python-crypto) #:select (python-asn1crypto)) + ((gnu packages python-web) #:select (python-requests)) + ((gnu packages tls) #:select (openssl)) + ((gnu packages version-control) #:select (git-minimal)) (guix build-system cmake) (guix build-system gnu) (guix build-system python) (guix build-system trivial) - (guix download) (guix gexp) (guix git-download) ((guix licenses) #:prefix license:) (guix packages) - (guix profiles) - (guix utils)) + ((guix utils) #:select (substitute-keyword-arguments))) (define-syntax-rule (search-our-patches file-name ...) "Return the list of absolute file names corresponding to each @@ -204,38 +197,44 @@ chain for " target " development.")) (search-our-patches "nsis-gcc-10-memmove.patch" "nsis-disable-installer-reloc.patch"))) -(define (fix-ppc64-nx-default lief) - (package-with-extra-patches lief - (search-our-patches "lief-fix-ppc64-nx-default.patch"))) - -;; Our python-lief package can be removed once we are using -;; guix 83bfdb409787cb2737e68b093a319b247b7858e6 or later. -;; Note we currently use cmake-minimal. +;; While LIEF is packaged in Guix, we maintain our own package, +;; to simplify building, and more easily apply updates. +;; Moreover, the Guix's package uses cmake, which caused build +;; failure; see https://github.com/bitcoin/bitcoin/pull/27296. (define-public python-lief (package (name "python-lief") - (version "0.12.3") + (version "0.13.2") (source (origin (method git-fetch) (uri (git-reference (url "https://github.com/lief-project/LIEF") (commit version))) (file-name (git-file-name name version)) + (modules '((guix build utils))) + (snippet + '(begin + ;; Configure build for Python bindings. + (substitute* "api/python/config-default.toml" + (("(ninja = )true" all m) + (string-append m "false")) + (("(parallel-jobs = )0" all m) + (string-append m (number->string (parallel-job-count))))))) (sha256 (base32 - "11i6hqmcjh56y554kqhl61698n9v66j2qk1c1g63mv2w07h2z661")))) + "0y48x358ppig5xp97ahcphfipx7cg9chldj2q5zrmn610fmi4zll")))) (build-system python-build-system) - (native-inputs (list cmake-minimal)) + (native-inputs (list cmake-minimal python-tomli)) (arguments (list #:tests? #f ;needs network #:phases #~(modify-phases %standard-phases + (add-before 'build 'change-directory + (lambda _ + (chdir "api/python"))) (replace 'build (lambda _ - (invoke - "python" "setup.py" "--sdk" "build" - (string-append - "-j" (number->string (parallel-job-count))))))))) + (invoke "python" "setup.py" "build")))))) (home-page "https://github.com/lief-project/LIEF") (synopsis "Library to instrument executable formats") (description @@ -248,18 +247,16 @@ and abstract ELF, PE and MachO formats.") (name "osslsigncode") (version "2.5") (source (origin - (method url-fetch) - (uri (string-append "https://github.com/mtrojnar/" - name "/archive/" version ".tar.gz")) + (method git-fetch) + (uri (git-reference + (url "https://github.com/mtrojnar/osslsigncode") + (commit version))) (sha256 (base32 - "03by9706gg0an6dn48pljx38vcb76ziv11bgm8ilwsf293x2k4hv")))) + "1j47vwq4caxfv0xw68kw5yh00qcpbd56d7rq6c483ma3y7s96yyz")))) (build-system cmake-build-system) (inputs `(("openssl", openssl))) - (arguments - '(#:configure-flags - (list "-DCMAKE_DISABLE_FIND_PACKAGE_CURL=TRUE"))) (home-page "https://github.com/mtrojnar/osslsigncode") (synopsis "Authenticode signing and timestamping tool") (description "osslsigncode is a small tool that implements part of the @@ -600,7 +597,7 @@ inspecting signatures in Mach-O binaries.") ;; Git git-minimal ;; Tests - (fix-ppc64-nx-default python-lief)) + python-lief) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) ;; Windows @@ -612,5 +609,5 @@ inspecting signatures in Mach-O binaries.") ((string-contains target "-linux-") (list (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list clang-toolchain-10 binutils cmake-minimal xorriso python-signapple)) + (list clang-toolchain-11 binutils cmake-minimal xorriso python-signapple)) (else '()))))) diff --git a/contrib/guix/patches/lief-fix-ppc64-nx-default.patch b/contrib/guix/patches/lief-fix-ppc64-nx-default.patch deleted file mode 100644 index 101bc1ddc0..0000000000 --- a/contrib/guix/patches/lief-fix-ppc64-nx-default.patch +++ /dev/null @@ -1,29 +0,0 @@ -Correct default for Binary::has_nx on ppc64 - -From the Linux kernel source: - - * This is the default if a program doesn't have a PT_GNU_STACK - * program header entry. The PPC64 ELF ABI has a non executable stack - * stack by default, so in the absence of a PT_GNU_STACK program header - * we turn execute permission off. - -This patch can be dropped the next time we update LIEF. - -diff --git a/src/ELF/Binary.cpp b/src/ELF/Binary.cpp -index a90be1ab..fd2d9764 100644 ---- a/src/ELF/Binary.cpp -+++ b/src/ELF/Binary.cpp -@@ -1084,7 +1084,12 @@ bool Binary::has_nx() const { - return segment->type() == SEGMENT_TYPES::PT_GNU_STACK; - }); - if (it_stack == std::end(segments_)) { -- return false; -+ if (header().machine_type() == ARCH::EM_PPC64) { -+ // The PPC64 ELF ABI has a non-executable stack by default. -+ return true; -+ } else { -+ return false; -+ } - } - - return !(*it_stack)->has(ELF_SEGMENT_FLAGS::PF_X); diff --git a/contrib/init/bitcoind.service b/contrib/init/bitcoind.service index 93de353bb4..87da17f955 100644 --- a/contrib/init/bitcoind.service +++ b/contrib/init/bitcoind.service @@ -18,10 +18,11 @@ After=network-online.target Wants=network-online.target [Service] -ExecStart=/usr/bin/bitcoind -daemonwait \ - -pid=/run/bitcoind/bitcoind.pid \ +ExecStart=/usr/bin/bitcoind -pid=/run/bitcoind/bitcoind.pid \ -conf=/etc/bitcoin/bitcoin.conf \ - -datadir=/var/lib/bitcoind + -datadir=/var/lib/bitcoind \ + -startupnotify='systemd-notify --ready' \ + -shutdownnotify='systemd-notify --stopping' # Make sure the config directory is readable by the service user PermissionsStartOnly=true @@ -30,8 +31,10 @@ ExecStartPre=/bin/chgrp bitcoin /etc/bitcoin # Process management #################### -Type=forking +Type=notify +NotifyAccess=all PIDFile=/run/bitcoind/bitcoind.pid + Restart=on-failure TimeoutStartSec=infinity TimeoutStopSec=600 diff --git a/contrib/tracing/mempool_monitor.py b/contrib/tracing/mempool_monitor.py index 9d427d4632..afb5e60372 100755 --- a/contrib/tracing/mempool_monitor.py +++ b/contrib/tracing/mempool_monitor.py @@ -27,7 +27,7 @@ PROGRAM = """ struct added_event { u8 hash[HASH_LENGTH]; - u64 vsize; + s32 vsize; s64 fee; }; @@ -35,7 +35,7 @@ struct removed_event { u8 hash[HASH_LENGTH]; char reason[MAX_REMOVAL_REASON_LENGTH]; - u64 vsize; + s32 vsize; s64 fee; u64 entry_time; }; @@ -49,11 +49,11 @@ struct rejected_event struct replaced_event { u8 replaced_hash[HASH_LENGTH]; - u64 replaced_vsize; + s32 replaced_vsize; s64 replaced_fee; u64 replaced_entry_time; u8 replacement_hash[HASH_LENGTH]; - u64 replacement_vsize; + s32 replacement_vsize; s64 replacement_fee; }; diff --git a/contrib/verify-binaries/README.md b/contrib/verify-binaries/README.md index c62d760e1a..04d683e69b 100644 --- a/contrib/verify-binaries/README.md +++ b/contrib/verify-binaries/README.md @@ -17,7 +17,7 @@ must obtain that key for your local GPG installation. You can obtain these keys by - through a browser using a key server (e.g. keyserver.ubuntu.com), - manually using the `gpg --keyserver <url> --recv-keys <key>` command, or - - you can run the packaged `verify.py ... --import-keys` script to + - you can run the packaged `verify.py --import-keys ...` script to have it automatically retrieve unrecognized keys. #### Usage diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 522a6b17ef..fa6d6d4b8b 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,4 +1,4 @@ -OSX_MIN_VERSION=10.15 +OSX_MIN_VERSION=11.0 OSX_SDK_VERSION=11.0 XCODE_VERSION=12.2 XCODE_BUILD_ID=12B45b @@ -19,7 +19,6 @@ clang_prog=$(build_prefix)/bin/clang clangxx_prog=$(clang_prog)++ llvm_config_prog=$(build_prefix)/bin/llvm-config -clang_resource_dir=$(build_prefix)/lib/clang/$(native_clang_version) else # FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's # system @@ -37,7 +36,6 @@ clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") llvm_config_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-config") -clang_resource_dir=$(shell clang -print-resource-dir) llvm_lib_dir=$(shell $(llvm_config_prog) --libdir) endif @@ -63,54 +61,33 @@ $(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$( # Explicitly point to our binaries (e.g. cctools) so that they are # ensured to be found and preferred over other possibilities. # -# -stdlib=libc++ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 +# -isysroot$(OSX_SDK) -nostdlibinc # -# Forces clang to use the libc++ headers from our SDK and completely -# forget about the libc++ headers from the standard directories +# Disable default include paths built into the compiler as well as +# those normally included for libc and libc++. The only path that +# remains implicitly is the clang resource dir. # -# -Xclang -*system<path_a> \ -# -Xclang -*system<path_b> \ -# -Xclang -*system<path_c> ... +# -iwithsysroot / -iframeworkwithsysroot # -# Adds path_a, path_b, and path_c to the bottom of clang's list of -# include search paths. This is used to explicitly specify the list of -# system include search paths and its ordering, rather than rely on -# clang's autodetection routine. This routine has been shown to: -# 1. Fail to pickup libc++ headers in $SYSROOT/usr/include/c++/v1 -# when clang was built manually (see: https://github.com/bitcoin/bitcoin/pull/17919#issuecomment-656785034) -# 2. Fail to pickup C headers in $SYSROOT/usr/include when -# C_INCLUDE_DIRS was specified at configure time (see: https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9) -# -# Talking directly to cc1 with -Xclang here grants us access to specify -# more granular categories for these system include search paths, and we -# can use the correct categories that these search paths would have been -# placed in if the autodetection routine had worked correctly. (see: -# https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9#the-treatment) -# -# Furthermore, it places these search paths after any "non-Xclang" -# specified search paths. This prevents any additional clang options or -# environment variables from coming after or in between these system -# include search paths, as that would be wrong in general but would also -# break #include_next's. +# Adds the desired paths from the SDK # + darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ - -isysroot$(OSX_SDK) \ - -Xclang -internal-externc-isystem -Xclang $(clang_resource_dir)/include \ - -Xclang -internal-externc-isystem -Xclang $(OSX_SDK)/usr/include + -isysroot$(OSX_SDK) -nostdlibinc \ + -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks + darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ - -isysroot$(OSX_SDK) \ - -stdlib=libc++ \ - -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 \ - -Xclang -internal-externc-isystem -Xclang $(clang_resource_dir)/include \ - -Xclang -internal-externc-isystem -Xclang $(OSX_SDK)/usr/include + -isysroot$(OSX_SDK) -nostdlibinc \ + -iwithsysroot/usr/include/c++/v1 \ + -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks darwin_CFLAGS=-pipe -std=$(C_STANDARD) darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 03e9002ecd..4860934a8e 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -5,6 +5,7 @@ $(package)_file_name=$($(package)_version).tar.gz $(package)_sha256_hash=6b73269efdf5c58a070e7357b66ee760501388549d6a12b423723f45888b074b $(package)_build_subdir=cctools $(package)_dependencies=native_libtapi +$(package)_patches=no_fixup_chains.patch define $(package)_set_vars $(package)_config_opts=--target=$(host) --enable-lto-support @@ -18,11 +19,13 @@ ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) define $(package)_preprocess_cmds mkdir -p $($(package)_staging_prefix_dir)/lib && \ cp $(llvm_lib_dir)/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools && \ + patch -p1 < $($(package)_patch_dir)/no_fixup_chains.patch endef else define $(package)_preprocess_cmds - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools && \ + patch -p1 < $($(package)_patch_dir)/no_fixup_chains.patch endef endif diff --git a/depends/packages/native_clang.mk b/depends/packages/native_clang.mk index b11037b83e..109796c0e6 100644 --- a/depends/packages/native_clang.mk +++ b/depends/packages/native_clang.mk @@ -1,12 +1,12 @@ package=native_clang -$(package)_version=10.0.1 +$(package)_version=11.1.0 $(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version) ifneq (,$(findstring aarch64,$(BUILD))) $(package)_file_name=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz -$(package)_sha256_hash=90dc69a4758ca15cd0ffa45d07fbf5bf4309d47d2c7745a9f0735ecffde9c31f +$(package)_sha256_hash=18df38247af3fba0e0e2991fb00d7e3cf3560b4d3509233a14af699ef0039e1c else $(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz -$(package)_sha256_hash=48b83ef827ac2c213d5b64f5ad7ed082c8bcb712b46644e0dc5045c6f462c231 +$(package)_sha256_hash=c691a558967fb7709fb81e0ed80d1f775f4502810236aa968b4406526b43bee1 endif define $(package)_stage_cmds diff --git a/depends/patches/native_cctools/no_fixup_chains.patch b/depends/patches/native_cctools/no_fixup_chains.patch new file mode 100644 index 0000000000..2516ea8200 --- /dev/null +++ b/depends/patches/native_cctools/no_fixup_chains.patch @@ -0,0 +1,23 @@ +commit 5860b35ff6c7241d1c35a1b3197b45e5c9ff86cf +Author: fanquake <fanquake@gmail.com> +Date: Thu Jun 29 11:52:43 2023 +0100 + + ld64: add support for -no_fixup_chains + + This is added in later versions, and is required if we want to be able + to disable fixup_chains, for use in security tests. + +diff --git a/cctools/ld64/src/ld/Options.cpp b/cctools/ld64/src/ld/Options.cpp +index 15e8e88..b6580af 100644 +--- a/cctools/ld64/src/ld/Options.cpp ++++ b/cctools/ld64/src/ld/Options.cpp +@@ -4128,6 +4128,9 @@ void Options::parse(int argc, const char* argv[]) + else if ( strcmp(arg, "-fixup_chains") == 0 ) { + fMakeChainedFixups = true; + } ++ else if ( strcmp(arg, "-no_fixup_chains") == 0 ) { ++ fMakeChainedFixups = false; ++ } + else if (strcmp(arg, "-debug_variant") == 0) { + fDebugVariant = true; + } diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index ab5db58cdd..6cbb6ebd72 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -5,6 +5,41 @@ The headless daemon `bitcoind` has the JSON-RPC API enabled by default, the GUI option. In the GUI it is possible to execute RPC methods in the Debug Console Dialog. +## Endpoints + +There are two JSON-RPC endpoints on the server: + +1. `/` +2. `/wallet/<walletname>/` + +### `/` endpoint + +This endpoint is always active. +It can always service non-wallet requests and can service wallet requests when +exactly one wallet is loaded. + +### `/wallet/<walletname>/` endpoint + +This endpoint is only activated when the wallet component has been compiled in. +It can service both wallet and non-wallet requests. +It MUST be used for wallet requests when two or more wallets are loaded. + +This is the endpoint used by bitcoin-cli when a `-rpcwallet=` parameter is passed in. + +Best practice would dictate using the `/wallet/<walletname>/` endpoint for ALL +requests when multiple wallets are in use. + +### Examples + +```sh +# Get block count from the / endpoint when rpcuser=alice and rpcport=38332 +$ curl --user alice --data-binary '{"jsonrpc": "1.0", "id": "0", "method": "getblockcount", "params": []}' -H 'content-type: text/plain;' localhost:38332/ + +# Get balance from the /wallet/walletname endpoint when rpcuser=alice, rpcport=38332 and rpcwallet=desc-wallet +$ curl --user alice --data-binary '{"jsonrpc": "1.0", "id": "0", "method": "getbalance", "params": []}' -H 'content-type: text/plain;' localhost:38332/wallet/desc-wallet + +``` + ## Parameter passing The JSON-RPC server supports both _by-position_ and _by-name_ [parameter diff --git a/doc/cjdns.md b/doc/cjdns.md index b69564729f..031cd1978b 100644 --- a/doc/cjdns.md +++ b/doc/cjdns.md @@ -112,5 +112,4 @@ There are several ways to see your CJDNS address in Bitcoin Core: To see which CJDNS peers your node is connected to, use `bitcoin-cli -netinfo 4` or the `getpeerinfo` RPC (i.e. `bitcoin-cli getpeerinfo`). -To see which CJDNS addresses your node knows, use the `getnodeaddresses 0 cjdns` -RPC. +You can use the `getnodeaddresses` RPC to fetch a number of CJDNS peers known to your node; run `bitcoin-cli help getnodeaddresses` for details. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 08dde2aa61..fca72914a3 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -217,13 +217,11 @@ apt install clang-tidy bear clang Then, pass clang as compiler to configure, and use bear to produce the `compile_commands.json`: ```sh -./autogen.sh && ./configure CC=clang CXX=clang++ --enable-suppress-external-warnings +./autogen.sh && ./configure CC=clang CXX=clang++ make clean && bear --config src/.bear-tidy-config -- make -j $(nproc) ``` -The output is denoised of errors from external dependencies and includes with -`--enable-suppress-external-warnings` and `--config src/.bear-tidy-config`. Both -options may be omitted to view the full list of errors. +The output is denoised of errors from external dependencies. To run clang-tidy on all source files: diff --git a/doc/i2p.md b/doc/i2p.md index 0432136554..b6c07388b7 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -9,16 +9,16 @@ started with I2P terminology. ## Run Bitcoin Core with an I2P router (proxy) -A running I2P router (proxy) with [SAM](https://geti2p.net/en/docs/api/samv3) -enabled is required. Options include: +A running I2P router (proxy) is required with the [SAM](https://geti2p.net/en/docs/api/samv3) +application bridge enabled. The following routers are recommended for use with Bitcoin Core: - [i2prouter (I2P Router)](https://geti2p.net), the official implementation in - Java + Java. The SAM bridge is not enabled by default; it must be started manually, + or configured to start automatically, in the Clients page in the + router console (`http://127.0.0.1:7657/configclients`) or in the `clients.config` file. - [i2pd (I2P Daemon)](https://github.com/PurpleI2P/i2pd) ([documentation](https://i2pd.readthedocs.io/en/latest)), a lighter - alternative in C++ -- [i2p-zero](https://github.com/i2p-zero/i2p-zero) -- [other alternatives](https://en.wikipedia.org/wiki/I2P#Routers) + alternative in C++. It enables the SAM bridge by default. Note the IP address and port the SAM proxy is listening to; usually, it is `127.0.0.1:7656`. @@ -109,8 +109,7 @@ incoming I2P connections (`-i2pacceptincoming`): To see which I2P peers your node is connected to, use `bitcoin-cli -netinfo 4` or the `getpeerinfo` RPC (e.g. `bitcoin-cli getpeerinfo`). -To see which I2P addresses your node knows, use the `getnodeaddresses 0 i2p` -RPC. +You can use the `getnodeaddresses` RPC to fetch a number of I2P peers known to your node; run `bitcoin-cli help getnodeaddresses` for details. ## Compatibility @@ -119,8 +118,7 @@ to connect to the I2P network. Any I2P router that supports it can be used. ## Ports in I2P and Bitcoin Core -Bitcoin Core uses the [SAM v3.1](https://geti2p.net/en/docs/api/samv3) -protocol. One particularity of SAM v3.1 is that it does not support ports, +One particularity of SAM v3.1 is that it does not support ports, unlike newer versions of SAM (v3.2 and up) that do support them and default the port numbers to 0. From the point of view of peers that use newer versions of SAM or other protocols that support ports, a SAM v3.1 peer is connecting to them diff --git a/doc/release-notes-24914.md b/doc/release-notes-24914.md new file mode 100644 index 0000000000..505d356fce --- /dev/null +++ b/doc/release-notes-24914.md @@ -0,0 +1,9 @@ +Wallet +------ + +- Wallet loading has changed in this release. Wallets with some corrupted records that could be + previously loaded (with warnings) may no longer load. For example, wallets with corrupted + address book entries may no longer load. If this happens, it is recommended + load the wallet in a previous version of Bitcoin Core and import the data into a new wallet. + Please also report an issue to help improve the software and make wallet loading more robust + in these cases. diff --git a/doc/release-notes-26485.md b/doc/release-notes-26485.md new file mode 100644 index 0000000000..c8df3d22fb --- /dev/null +++ b/doc/release-notes-26485.md @@ -0,0 +1,16 @@ +JSON-RPC +--- + +For RPC methods which accept `options` parameters ((`importmulti`, `listunspent`, `fundrawtransaction`, `bumpfee`, `send`, `sendall`, `walletcreatefundedpsbt`, `simulaterawtransaction`), it is now possible to pass the options as named parameters without the need for a nested object. (#26485) + +This means it is possible make calls like: + +```sh +src/bitcoin-cli -named bumpfee txid fee_rate=100 +``` + +instead of + +```sh +src/bitcoin-cli -named bumpfee txid options='{"fee_rate": 100}' +``` diff --git a/doc/release-notes-27302.md b/doc/release-notes-27302.md new file mode 100644 index 0000000000..e67a6c8b06 --- /dev/null +++ b/doc/release-notes-27302.md @@ -0,0 +1,4 @@ +Configuration +--- + +- `bitcoind` and `bitcoin-qt` will now raise an error on startup if a datadir that is being used contains a bitcoin.conf file that will be ignored, which can happen when a datadir= line is used in a bitcoin.conf file. The error message is just a diagnostic intended to prevent accidental misconfiguration, and it can be disabled to restore the previous behavior of using the datadir while ignoring the bitcoin.conf contained in it. diff --git a/doc/release-notes-27501.md b/doc/release-notes-27501.md new file mode 100644 index 0000000000..386a00fb34 --- /dev/null +++ b/doc/release-notes-27501.md @@ -0,0 +1,3 @@ +- A new `getprioritisedtransactions` RPC has been added. It returns a map of all fee deltas created by the + user with prioritisetransaction, indexed by txid. The map also indicates whether each transaction is + present in the mempool. diff --git a/doc/release-notes-27632.md b/doc/release-notes-27632.md new file mode 100644 index 0000000000..d588247c85 --- /dev/null +++ b/doc/release-notes-27632.md @@ -0,0 +1,5 @@ +Updated settings +---------------- + +- Passing an invalid `-debug`, `-debugexclude`, or `-loglevel` logging configuration + option now raises an error, rather than logging an easily missed warning. (#27632) diff --git a/doc/release-notes-27757.md b/doc/release-notes-27757.md new file mode 100644 index 0000000000..fb6aaa01a5 --- /dev/null +++ b/doc/release-notes-27757.md @@ -0,0 +1,8 @@ +Wallet +------ + +- The `deprecatedrpc=walletwarningfield` configuration option has been removed. + The `createwallet`, `loadwallet`, `restorewallet` and `unloadwallet` RPCs no + longer return the "warning" string field. The same information is provided + through the "warnings" field added in v25.0, which returns a JSON array of + strings. The "warning" string field was deprecated also in v25.0. (#27757) diff --git a/doc/release-notes-empty-template.md b/doc/release-notes-empty-template.md index 4cd2314308..887104548b 100644 --- a/doc/release-notes-empty-template.md +++ b/doc/release-notes-empty-template.md @@ -36,7 +36,7 @@ Compatibility ============== Bitcoin Core is supported and extensively tested on operating systems -using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +using the Linux kernel, macOS 11.0+, and Windows 7 and newer. Bitcoin Core should also work on most other Unix-like systems but is not as frequently tested on them. It is not recommended to use Bitcoin Core on unsupported systems. diff --git a/doc/release-notes/release-notes-25.0.md b/doc/release-notes/release-notes-25.0.md new file mode 100644 index 0000000000..919cb3b2f3 --- /dev/null +++ b/doc/release-notes/release-notes-25.0.md @@ -0,0 +1,340 @@ +25.0 Release Notes +================== + +Bitcoin Core version 25.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-25.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +P2P and network changes +----------------------- + +- Transactions of non-witness size 65 bytes and above are now allowed by mempool + and relay policy. This is to better reflect the actual afforded protections + against CVE-2017-12842 and open up additional use-cases of smaller transaction sizes. (#26265) + +New RPCs +-------- + +- The scanblocks RPC returns the relevant blockhashes from a set of descriptors by + scanning all blockfilters in the given range. It can be used in combination with + the getblockheader and rescanblockchain RPCs to achieve fast wallet rescans. Note + that this functionality can only be used if a compact block filter index + (-blockfilterindex=1) has been constructed by the node. (#23549) + +Updated RPCs +------------ + +- All JSON-RPC methods accept a new [named + parameter](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md#parameter-passing) called `args` that can + contain positional parameter values. This is a convenience to allow some + parameter values to be passed by name without having to name every value. The + python test framework and `bitcoin-cli` tool both take advantage of this, so + for example: + +```sh +bitcoin-cli -named createwallet wallet_name=mywallet load_on_startup=1 +``` + +Can now be shortened to: + +```sh +bitcoin-cli -named createwallet mywallet load_on_startup=1 +``` + +- The `verifychain` RPC will now return `false` if the checks didn't fail, + but couldn't be completed at the desired depth and level. This could be due + to missing data while pruning, due to an insufficient dbcache or due to + the node being shutdown before the call could finish. (#25574) + +- `sendrawtransaction` has a new, optional argument, `maxburnamount` with a default value of `0`. + Any transaction containing an unspendable output with a value greater than `maxburnamount` will + not be submitted. At present, the outputs deemed unspendable are those with scripts that begin + with an `OP_RETURN` code (known as 'datacarriers'), scripts that exceed the maximum script size, + and scripts that contain invalid opcodes. + +- The `testmempoolaccept` RPC now returns 2 additional results within the "fees" result: + "effective-feerate" is the feerate including fees and sizes of transactions validated together if + package validation was used, and also includes any modified fees from prioritisetransaction. The + "effective-includes" result lists the wtxids of transactions whose modified fees and sizes were used + in the effective-feerate (#26646). + +- `decodescript` may now infer a Miniscript descriptor under P2WSH context if it is not lacking + information. (#27037) + +- `finalizepsbt` is now able to finalize a transaction with inputs spending Miniscript-compatible + P2WSH scripts. (#24149) + +Changes to wallet related RPCs can be found in the Wallet section below. + +Build System +------------ + +- The `--enable-upnp-default` and `--enable-natpmp-default` options + have been removed. If you want to use port mapping, you can + configure it using a .conf file, or by passing the relevant + options at runtime. (#26896) + +Updated settings +---------------- + +- If the `-checkblocks` or `-checklevel` options are explicitly provided by the +user, but the verification checks cannot be completed due to an insufficient +dbcache, Bitcoin Core will now return an error at startup. (#25574) + +- Ports specified in `-port` and `-rpcport` options are now validated at startup. + Values that previously worked and were considered valid can now result in errors. (#22087) + +- Setting `-blocksonly` will now reduce the maximum mempool memory + to 5MB (users may still use `-maxmempool` to override). Previously, + the default 300MB would be used, leading to unexpected memory usage + for users running with `-blocksonly` expecting it to eliminate + mempool memory usage. + + As unused mempool memory is shared with dbcache, this also reduces + the dbcache size for users running with `-blocksonly`, potentially + impacting performance. +- Setting `-maxconnections=0` will now disable `-dnsseed` + and `-listen` (users may still set them to override). + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +New settings +------------ + +- The `shutdownnotify` option is used to specify a command to execute synchronously +before Bitcoin Core has begun its shutdown sequence. (#23395) + + +Wallet +------ + +- The `minconf` option, which allows a user to specify the minimum number +of confirmations a UTXO being spent has, and the `maxconf` option, +which allows specifying the maximum number of confirmations, have been +added to the following RPCs in #25375: + - `fundrawtransaction` + - `send` + - `walletcreatefundedpsbt` + - `sendall` + +- Added a new `next_index` field in the response in `listdescriptors` to + have the same format as `importdescriptors` (#26194) + +- RPC `listunspent` now has a new argument `include_immature_coinbase` + to include coinbase UTXOs that don't meet the minimum spendability + depth requirement (which before were silently skipped). (#25730) + +- Rescans for descriptor wallets are now significantly faster if compact + block filters (BIP158) are available. Since those are not constructed + by default, the configuration option "-blockfilterindex=1" has to be + provided to take advantage of the optimization. This improves the + performance of the RPC calls `rescanblockchain`, `importdescriptors` + and `restorewallet`. (#25957) + +- RPC `unloadwallet` now fails if a rescan is in progress. (#26618) + +- Wallet passphrases may now contain null characters. + Prior to this change, only characters up to the first + null character were recognized and accepted. (#27068) + +- Address Purposes strings are now restricted to the currently known values of "send", + "receive", and "refund". Wallets that have unrecognized purpose strings will have + loading warnings, and the `listlabels` RPC will raise an error if an unrecognized purpose + is requested. (#27217) + +- In the `createwallet`, `loadwallet`, `unloadwallet`, and `restorewallet` RPCs, the + "warning" string field is deprecated in favor of a "warnings" field that + returns a JSON array of strings to better handle multiple warning messages and + for consistency with other wallet RPCs. The "warning" field will be fully + removed from these RPCs in v26. It can be temporarily re-enabled during the + deprecation period by launching bitcoind with the configuration option + `-deprecatedrpc=walletwarningfield`. (#27279) + +- Descriptor wallets can now spend coins sent to P2WSH Miniscript descriptors. (#24149) + +GUI changes +----------- + +- The "Mask values" is a persistent option now. (gui#701) +- The "Mask values" option affects the "Transaction" view now, in addition to the + "Overview" one. (gui#708) + +REST +---- + +- A new `/rest/deploymentinfo` endpoint has been added for fetching various + state info regarding deployments of consensus changes. (#25412) + +Binary verification +---- + +- The binary verification script has been updated. In previous releases it + would verify that the binaries had been signed with a single "release key". + In this release and moving forward it will verify that the binaries are + signed by a _threshold of trusted keys_. For more details and + examples, see: + https://github.com/bitcoin/bitcoin/blob/master/contrib/verify-binaries/README.md + (#27358) + +Low-level changes +================= + +RPC +--- + +- The JSON-RPC server now rejects requests where a parameter is specified multiple + times with the same name, instead of silently overwriting earlier parameter values + with later ones. (#26628) +- RPC `listsinceblock` now accepts an optional `label` argument + to fetch incoming transactions having the specified label. (#25934) +- Previously `setban`, `addpeeraddress`, `walletcreatefundedpsbt`, methods + allowed non-boolean and non-null values to be passed as boolean parameters. + Any string, number, array, or object value that was passed would be treated + as false. After this change, passing any value except `true`, `false`, or + `null` now triggers a JSON value is not of expected type error. (#26213) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- 721217.xyz +- @RandyMcMillan +- amadeuszpawlik +- Amiti Uttarwar +- Andrew Chow +- Andrew Toth +- Anthony Towns +- Antoine Poinsot +- Aurèle Oulès +- Ben Woosley +- Bitcoin Hodler +- brunoerg +- Bushstar +- Carl Dong +- Chris Geihsler +- Cory Fields +- David Gumberg +- dergoegge +- Dhruv Mehta +- Dimitris Tsapakidis +- dougEfish +- Douglas Chimento +- ekzyis +- Elichai Turkel +- Ethan Heilman +- Fabian Jahr +- FractalEncrypt +- furszy +- Gleb Naumenko +- glozow +- Greg Sanders +- Hennadii Stepanov +- hernanmarino +- ishaanam +- ismaelsadeeq +- James O'Beirne +- jdjkelly@gmail.com +- Jeff Ruane +- Jeffrey Czyz +- Jeremy Rubin +- Jesse Barton +- João Barbosa +- JoaoAJMatos +- John Moffett +- Jon Atack +- Jonas Schnelli +- jonatack +- Joshua Kelly +- josibake +- Juan Pablo Civile +- kdmukai +- klementtan +- Kolby ML +- kouloumos +- Kristaps Kaupe +- laanwj +- Larry Ruane +- Leonardo Araujo +- Leonardo Lazzaro +- Luke Dashjr +- MacroFake +- MarcoFalke +- Martin Leitner-Ankerl +- Martin Zumsande +- Matt Whitlock +- Matthew Zipkin +- Michael Ford +- Miles Liu +- mruddy +- Murray Nesbitt +- muxator +- omahs +- pablomartin4btc +- Pasta +- Pieter Wuille +- Pttn +- Randall Naar +- Riahiamirreza +- roconnor-blockstream +- Russell O'Connor +- Ryan Ofsky +- S3RK +- Sebastian Falbesoner +- Seibart Nedor +- sinetek +- Sjors Provoost +- Skuli Dulfari +- SomberNight +- Stacie Waleyko +- stickies-v +- stratospher +- Suhas Daftuar +- Suriyaa Sundararuban +- TheCharlatan +- Vasil Dimov +- Vasil Stoyanov +- virtu +- w0xlt +- willcl-ark +- yancy +- Yusuf Sahin HAMZA + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/).
\ No newline at end of file diff --git a/doc/release-process.md b/doc/release-process.md index cf014c940b..930110922c 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -284,7 +284,7 @@ cat "$VERSION"/*/all.SHA256SUMS.asc > SHA256SUMS.asc - Push the flatpak to flathub, e.g. https://github.com/flathub/org.bitcoincore.bitcoin-qt/pull/2 - - Push the snap, see https://github.com/bitcoin-core/packaging/blob/master/snap/build.md + - Push the snap, see https://github.com/bitcoin-core/packaging/blob/main/snap/local/build.md - This repo diff --git a/doc/tor.md b/doc/tor.md index 581d124f7a..65aa3ece02 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -2,9 +2,7 @@ It is possible to run Bitcoin Core as a Tor onion service, and connect to such services. -The following directions assume you have a Tor proxy running on port 9050. Many distributions default to having a SOCKS proxy listening on port 9050, but others may not. In particular, the Tor Browser Bundle defaults to listening on port 9150. See [Tor Project FAQ:TBBSocksPort](https://www.torproject.org/docs/faq.html.en#TBBSocksPort) for how to properly -configure Tor. - +The following directions assume you have a Tor proxy running on port 9050. Many distributions default to having a SOCKS proxy listening on port 9050, but others may not. In particular, the Tor Browser Bundle defaults to listening on port 9150. ## Compatibility - Starting with version 22.0, Bitcoin Core only supports Tor version 3 hidden @@ -27,8 +25,7 @@ CLI `-addrinfo` returns the number of addresses known to your node per network. This can be useful to see how many onion peers your node knows, e.g. for `-onlynet=onion`. -To fetch a number of onion addresses that your node knows, for example seven -addresses, use the `getnodeaddresses 7 onion` RPC. +You can use the `getnodeaddresses` RPC to fetch a number of onion peers known to your node; run `bitcoin-cli help getnodeaddresses` for details. ## 1. Run Bitcoin Core behind a Tor proxy diff --git a/doc/tracing.md b/doc/tracing.md index d26cf52fc3..0e3414205a 100644 --- a/doc/tracing.md +++ b/doc/tracing.md @@ -220,7 +220,7 @@ about the transaction. Arguments passed: 1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) -2. Transaction virtual size as `uint64` +2. Transaction virtual size as `int32` 3. Transaction fee as `int64` #### Tracepoint `mempool:removed` @@ -231,7 +231,7 @@ about the transaction. Arguments passed: 1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) 2. Removal reason as `pointer to C-style String` (max. length 9 characters) -3. Transaction virtual size as `uint64` +3. Transaction virtual size as `int32` 4. Transaction fee as `int64` 5. Transaction mempool entry time (epoch) as `uint64` @@ -242,11 +242,11 @@ Passes information about the replaced and replacement transactions. Arguments passed: 1. Replaced transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) -2. Replaced transaction virtual size as `uint64` +2. Replaced transaction virtual size as `int32` 3. Replaced transaction fee as `int64` 4. Replaced transaction mempool entry time (epoch) as `uint64` 5. Replacement transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) -6. Replacement transaction virtual size as `uint64` +6. Replacement transaction virtual size as `int32` 7. Replacement transaction fee as `int64` Note: In cases where a single replacement transaction replaces multiple diff --git a/share/qt/Info.plist.in b/share/qt/Info.plist.in index 053359e0a8..b4e6f6a150 100644 --- a/share/qt/Info.plist.in +++ b/share/qt/Info.plist.in @@ -3,7 +3,7 @@ <plist version="0.9"> <dict> <key>LSMinimumSystemVersion</key> - <string>10.15.0</string> + <string>11</string> <key>LSArchitecturePriority</key> <array> diff --git a/src/.bear-tidy-config b/src/.bear-tidy-config index 111ef6ee44..9b6b9d2cf7 100644 --- a/src/.bear-tidy-config +++ b/src/.bear-tidy-config @@ -4,7 +4,11 @@ "include_only_existing_source": true, "paths_to_include": [], "paths_to_exclude": [ - "src/leveldb" + "src/crc32c", + "src/crypto/ctaes", + "src/leveldb", + "src/minisketch", + "src/secp256k1" ] }, "format": { diff --git a/src/Makefile.am b/src/Makefile.am index 713b9a30d1..85c3eaf08d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -143,6 +143,8 @@ BITCOIN_CORE_H = \ compat/compat.h \ compat/cpuid.h \ compat/endian.h \ + common/settings.h \ + common/system.h \ compressor.h \ consensus/consensus.h \ consensus/tx_check.h \ @@ -186,6 +188,7 @@ BITCOIN_CORE_H = \ kernel/mempool_limits.h \ kernel/mempool_options.h \ kernel/mempool_persist.h \ + kernel/notifications_interface.h \ kernel/validation_cache_sizes.h \ key.h \ key_io.h \ @@ -214,6 +217,7 @@ BITCOIN_CORE_H = \ node/database_args.h \ node/eviction.h \ node/interface_ui.h \ + node/kernel_notifications.h \ node/mempool_args.h \ node/mempool_persist_args.h \ node/miner.h \ @@ -277,7 +281,9 @@ BITCOIN_CORE_H = \ txorphanage.h \ txrequest.h \ undo.h \ + util/any.h \ util/asmap.h \ + util/batchpriority.h \ util/bip32.h \ util/bitdeque.h \ util/bytevectorhash.h \ @@ -294,6 +300,7 @@ BITCOIN_CORE_H = \ util/golombrice.h \ util/hash_type.h \ util/hasher.h \ + util/insert.h \ util/macros.h \ util/message.h \ util/moneystr.h \ @@ -303,13 +310,10 @@ BITCOIN_CORE_H = \ util/readwritefile.h \ util/result.h \ util/serfloat.h \ - util/settings.h \ util/sock.h \ util/spanparsing.h \ util/string.h \ - util/syscall_sandbox.h \ util/syserror.h \ - util/system.h \ util/thread.h \ util/threadinterrupt.h \ util/threadnames.h \ @@ -408,6 +412,7 @@ libbitcoin_node_a_SOURCES = \ node/eviction.cpp \ node/interface_ui.cpp \ node/interfaces.cpp \ + node/kernel_notifications.cpp \ node/mempool_args.cpp \ node/mempool_persist_args.cpp \ node/miner.cpp \ @@ -657,6 +662,8 @@ libbitcoin_common_a_SOURCES = \ common/init.cpp \ common/interfaces.cpp \ common/run_command.cpp \ + common/settings.cpp \ + common/system.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ @@ -708,6 +715,7 @@ libbitcoin_util_a_SOURCES = \ support/cleanse.cpp \ sync.cpp \ util/asmap.cpp \ + util/batchpriority.cpp \ util/bip32.cpp \ util/bytevectorhash.cpp \ util/chaintype.cpp \ @@ -721,12 +729,10 @@ libbitcoin_util_a_SOURCES = \ util/hasher.cpp \ util/sock.cpp \ util/syserror.cpp \ - util/system.cpp \ util/message.cpp \ util/moneystr.cpp \ util/rbf.cpp \ util/readwritefile.cpp \ - util/settings.cpp \ util/thread.cpp \ util/threadinterrupt.cpp \ util/threadnames.cpp \ @@ -734,7 +740,6 @@ libbitcoin_util_a_SOURCES = \ util/spanparsing.cpp \ util/strencodings.cpp \ util/string.cpp \ - util/syscall_sandbox.cpp \ util/time.cpp \ util/tokenpipe.cpp \ $(BITCOIN_CORE_H) @@ -905,12 +910,8 @@ libbitcoinkernel_la_SOURCES = \ kernel/bitcoinkernel.cpp \ arith_uint256.cpp \ chain.cpp \ - chainparamsbase.cpp \ - chainparams.cpp \ clientversion.cpp \ coins.cpp \ - common/args.cpp \ - common/config.cpp \ compressor.cpp \ consensus/merkle.cpp \ consensus/tx_check.cpp \ @@ -960,6 +961,7 @@ libbitcoinkernel_la_SOURCES = \ txdb.cpp \ txmempool.cpp \ uint256.cpp \ + util/batchpriority.cpp \ util/chaintype.cpp \ util/check.cpp \ util/exception.cpp \ @@ -970,12 +972,9 @@ libbitcoinkernel_la_SOURCES = \ util/moneystr.cpp \ util/rbf.cpp \ util/serfloat.cpp \ - util/settings.cpp \ util/strencodings.cpp \ util/string.cpp \ - util/syscall_sandbox.cpp \ util/syserror.cpp \ - util/system.cpp \ util/thread.cpp \ util/threadnames.cpp \ util/time.cpp \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index c8e510b482..10c8389c80 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -18,6 +18,7 @@ bench_bench_bitcoin_SOURCES = \ bench/bench.cpp \ bench/bench.h \ bench/bench_bitcoin.cpp \ + bench/bip324_ecdh.cpp \ bench/block_assemble.cpp \ bench/ccoins_caching.cpp \ bench/chacha20.cpp \ @@ -29,6 +30,7 @@ bench_bench_bitcoin_SOURCES = \ bench/data.h \ bench/descriptors.cpp \ bench/duplicate_inputs.cpp \ + bench/ellswift.cpp \ bench/examples.cpp \ bench/gcs_filter.cpp \ bench/hashpadding.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 8e45c797bf..224f1fe301 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -194,7 +194,9 @@ BITCOIN_TESTS += wallet/test/db_tests.cpp endif FUZZ_WALLET_SRC = \ + wallet/test/fuzz/coincontrol.cpp \ wallet/test/fuzz/coinselection.cpp \ + wallet/test/fuzz/fees.cpp \ wallet/test/fuzz/parse_iso8601.cpp if USE_SQLITE diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 11b93ad13e..33c299f34c 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -11,6 +11,7 @@ TEST_UTIL_H = \ test/util/blockfilter.h \ test/util/chainstate.h \ test/util/coins.h \ + test/util/index.h \ test/util/json.h \ test/util/logging.h \ test/util/mining.h \ @@ -34,6 +35,7 @@ libtest_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_util_a_SOURCES = \ test/util/blockfilter.cpp \ test/util/coins.cpp \ + test/util/index.cpp \ test/util/json.cpp \ test/util/logging.cpp \ test/util/mining.cpp \ diff --git a/src/addrdb.cpp b/src/addrdb.cpp index b679ad0602..0fcb5ed5c9 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -9,6 +9,7 @@ #include <chainparams.h> #include <clientversion.h> #include <common/args.h> +#include <common/settings.h> #include <cstdint> #include <hash.h> #include <logging.h> @@ -21,7 +22,6 @@ #include <univalue.h> #include <util/fs.h> #include <util/fs_helpers.h> -#include <util/settings.h> #include <util/translation.h> namespace { @@ -132,7 +132,7 @@ CBanDB::CBanDB(fs::path ban_list_path) bool CBanDB::Write(const banmap_t& banSet) { std::vector<std::string> errors; - if (util::WriteSettings(m_banlist_json, {{JSON_KEY, BanMapToJson(banSet)}}, errors)) { + if (common::WriteSettings(m_banlist_json, {{JSON_KEY, BanMapToJson(banSet)}}, errors)) { return true; } @@ -152,10 +152,10 @@ bool CBanDB::Read(banmap_t& banSet) return false; } - std::map<std::string, util::SettingsValue> settings; + std::map<std::string, common::SettingsValue> settings; std::vector<std::string> errors; - if (!util::ReadSettings(m_banlist_json, settings, errors)) { + if (!common::ReadSettings(m_banlist_json, settings, errors)) { for (const auto& err : errors) { LogPrintf("Cannot load banlist %s: %s\n", fs::PathToString(m_banlist_json), err); } @@ -183,10 +183,10 @@ void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) DeserializeDB(ssPeers, addr, false); } -std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman) +util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args) { auto check_addrman = std::clamp<int32_t>(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); - addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); + auto addrman{std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman)}; const auto start{SteadyClock::now()}; const auto path_addr{args.GetDataDirNet() / "peers.dat"}; @@ -200,19 +200,18 @@ std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, con DumpPeerAddresses(args, *addrman); } catch (const InvalidAddrManVersionError&) { if (!RenameOver(path_addr, (fs::path)path_addr + ".bak")) { - addrman = nullptr; - return strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again.")); + return util::Error{strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again."))}; } // Addrman can be in an inconsistent state after failure, reset it addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { - addrman = nullptr; - return strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), - e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr))); + return util::Error{strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), + e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr)))}; } - return std::nullopt; + return {std::move(addrman)}; // std::move should be unnecessary but is temporarily needed to work around clang bug + // (https://github.com/bitcoin/bitcoin/pull/25977#issuecomment-1561270092) } void DumpAnchors(const fs::path& anchors_db_path, const std::vector<CAddress>& anchors) diff --git a/src/addrdb.h b/src/addrdb.h index 08d86d0f01..0037495d18 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -6,11 +6,11 @@ #ifndef BITCOIN_ADDRDB_H #define BITCOIN_ADDRDB_H -#include <net_types.h> // For banmap_t -#include <univalue.h> +#include <net_types.h> #include <util/fs.h> +#include <util/result.h> -#include <optional> +#include <memory> #include <vector> class ArgsManager; @@ -18,7 +18,6 @@ class AddrMan; class CAddress; class CDataStream; class NetGroupManager; -struct bilingual_str; bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); /** Only used by tests. */ @@ -49,7 +48,7 @@ public: }; /** Returns an error string on failure */ -std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman); +util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args); /** * Dump the anchor IP address database (anchors.dat) diff --git a/src/banman.cpp b/src/banman.cpp index 5b2049d654..a96b7e3c53 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -5,11 +5,11 @@ #include <banman.h> +#include <common/system.h> #include <logging.h> #include <netaddress.h> #include <node/interface_ui.h> #include <sync.h> -#include <util/system.h> #include <util/time.h> #include <util/translation.h> diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index 8a5cab443f..f044feebba 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -72,20 +72,6 @@ static void FillAddrMan(AddrMan& addrman) AddAddressesToAddrMan(addrman); } -static CNetAddr ResolveIP(const std::string& ip) -{ - CNetAddr addr; - LookupHost(ip, addr, false); - return addr; -} - -static CService ResolveService(const std::string& ip, uint16_t port = 0) -{ - CService serv; - Lookup(ip, serv, port, false); - return serv; -} - /* Benchmarks */ static void AddrManAdd(benchmark::Bench& bench) @@ -118,8 +104,8 @@ static void AddrManSelectFromAlmostEmpty(benchmark::Bench& bench) AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; // Add one address to the new table - CService addr = ResolveService("250.3.1.1", 8333); - addrman.Add({CAddress(addr, NODE_NONE)}, ResolveService("250.3.1.1", 8333)); + CService addr = Lookup("250.3.1.1", 8333, false).value(); + addrman.Add({CAddress(addr, NODE_NONE)}, addr); bench.run([&] { (void)addrman.Select(); @@ -135,7 +121,7 @@ static void AddrManSelectByNetwork(benchmark::Bench& bench) i2p_service.SetSpecial("udhdrtrcetjm5sxzskjyr5ztpeszydbh4dpl3pl4utgqqw2v4jna.b32.i2p"); CAddress i2p_address(i2p_service, NODE_NONE); i2p_address.nTime = Now<NodeSeconds>(); - CNetAddr source = ResolveIP("252.2.2.2"); + const CNetAddr source{LookupHost("252.2.2.2", false).value()}; addrman.Add({i2p_address}, source); FillAddrMan(addrman); diff --git a/src/bench/bip324_ecdh.cpp b/src/bench/bip324_ecdh.cpp new file mode 100644 index 0000000000..659da0f08e --- /dev/null +++ b/src/bench/bip324_ecdh.cpp @@ -0,0 +1,51 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> + +#include <key.h> +#include <pubkey.h> +#include <random.h> +#include <span.h> + +#include <array> +#include <cstddef> + +static void BIP324_ECDH(benchmark::Bench& bench) +{ + ECC_Start(); + FastRandomContext rng; + + std::array<std::byte, 32> key_data; + std::array<std::byte, EllSwiftPubKey::size()> our_ellswift_data; + std::array<std::byte, EllSwiftPubKey::size()> their_ellswift_data; + + rng.fillrand(key_data); + rng.fillrand(our_ellswift_data); + rng.fillrand(their_ellswift_data); + + bench.batch(1).unit("ecdh").run([&] { + CKey key; + key.Set(UCharCast(key_data.data()), UCharCast(key_data.data()) + 32, true); + EllSwiftPubKey our_ellswift(our_ellswift_data); + EllSwiftPubKey their_ellswift(their_ellswift_data); + + auto ret = key.ComputeBIP324ECDHSecret(their_ellswift, our_ellswift, true); + + // To make sure that the computation is not the same on every iteration (ellswift decoding + // is variable-time), distribute bytes from the shared secret over the 3 inputs. The most + // important one is their_ellswift, because that one is actually decoded, so it's given most + // bytes. The data is copied into the middle, so that both halves are affected: + // - Copy 8 bytes from the resulting shared secret into middle of the private key. + std::copy(ret.begin(), ret.begin() + 8, key_data.begin() + 12); + // - Copy 8 bytes from the resulting shared secret into the middle of our ellswift key. + std::copy(ret.begin() + 8, ret.begin() + 16, our_ellswift_data.begin() + 28); + // - Copy 16 bytes from the resulting shared secret into the middle of their ellswift key. + std::copy(ret.begin() + 16, ret.end(), their_ellswift_data.begin() + 24); + }); + + ECC_Stop(); +} + +BENCHMARK(BIP324_ECDH, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 8ad6fde6bf..70e0b86eba 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -4,11 +4,11 @@ #include <bench/bench.h> #include <checkqueue.h> +#include <common/system.h> #include <key.h> #include <prevector.h> #include <pubkey.h> #include <random.h> -#include <util/system.h> #include <vector> diff --git a/src/bench/ellswift.cpp b/src/bench/ellswift.cpp new file mode 100644 index 0000000000..f0348421b3 --- /dev/null +++ b/src/bench/ellswift.cpp @@ -0,0 +1,31 @@ +// Copyright (c) 2022-2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> + +#include <key.h> +#include <random.h> + +static void EllSwiftCreate(benchmark::Bench& bench) +{ + ECC_Start(); + + CKey key; + key.MakeNewKey(true); + + uint256 entropy = GetRandHash(); + + bench.batch(1).unit("pubkey").run([&] { + auto ret = key.EllSwiftCreate(MakeByteSpan(entropy)); + /* Use the first 32 bytes of the ellswift encoded public key as next private key. */ + key.Set(UCharCast(ret.data()), UCharCast(ret.data()) + 32, true); + assert(key.IsValid()); + /* Use the last 32 bytes of the ellswift encoded public key as next entropy. */ + std::copy(ret.begin() + 32, ret.begin() + 64, MakeWritableByteSpan(entropy).begin()); + }); + + ECC_Stop(); +} + +BENCHMARK(EllSwiftCreate, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/load_external.cpp b/src/bench/load_external.cpp index 2ff72a3012..1378a7b20a 100644 --- a/src/bench/load_external.cpp +++ b/src/bench/load_external.cpp @@ -34,7 +34,7 @@ static void LoadExternalBlockFile(benchmark::Bench& bench) ss << static_cast<uint32_t>(benchmark::data::block413567.size()); // We can't use the streaming serialization (ss << benchmark::data::block413567) // because that first writes a compact size. - ss.write(MakeByteSpan(benchmark::data::block413567)); + ss << Span{benchmark::data::block413567}; // Create the test file. { diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index 59c4af086e..2524e215e4 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -80,6 +80,30 @@ static void PrevectorDeserialize(benchmark::Bench& bench) }); } +template <typename T> +static void PrevectorFillVectorDirect(benchmark::Bench& bench) +{ + bench.run([&] { + std::vector<prevector<28, T>> vec; + for (size_t i = 0; i < 260; ++i) { + vec.emplace_back(); + } + }); +} + + +template <typename T> +static void PrevectorFillVectorIndirect(benchmark::Bench& bench) +{ + bench.run([&] { + std::vector<prevector<28, T>> vec; + for (size_t i = 0; i < 260; ++i) { + // force allocation + vec.emplace_back(29, T{}); + } + }); +} + #define PREVECTOR_TEST(name) \ static void Prevector##name##Nontrivial(benchmark::Bench& bench) \ { \ @@ -96,3 +120,5 @@ PREVECTOR_TEST(Clear) PREVECTOR_TEST(Destructor) PREVECTOR_TEST(Resize) PREVECTOR_TEST(Deserialize) +PREVECTOR_TEST(FillVectorDirect) +PREVECTOR_TEST(FillVectorIndirect) diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index 2c77b9b22b..bf2195293e 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <interfaces/chain.h> +#include <node/chainstate.h> #include <node/context.h> #include <test/util/mining.h> #include <test/util/setup_common.h> @@ -14,26 +15,21 @@ #include <optional> -using wallet::CWallet; -using wallet::CreateMockableWalletDatabase; -using wallet::DBErrors; -using wallet::GetBalance; -using wallet::WALLET_FLAG_DESCRIPTORS; - -const std::string ADDRESS_BCRT1_UNSPENDABLE = "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj"; - +namespace wallet { static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_mine) { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; + // Set clock to genesis block, so the descriptors/keys creation time don't interfere with the blocks scanning process. + // The reason is 'generatetoaddress', which creates a chain with deterministic timestamps in the past. + SetMockTime(test_setup->m_node.chainman->GetParams().GenesisBlock().nTime); CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockableWalletDatabase()}; { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetupDescriptorScriptPubKeyMans(); - if (wallet.LoadWallet() != DBErrors::LOAD_OK) assert(false); } auto handler = test_setup->m_node.chain->handleNotifications({&wallet, [](CWallet*) {}}); @@ -63,3 +59,4 @@ BENCHMARK(WalletBalanceDirty, benchmark::PriorityLevel::HIGH); BENCHMARK(WalletBalanceClean, benchmark::PriorityLevel::HIGH); BENCHMARK(WalletBalanceMine, benchmark::PriorityLevel::HIGH); BENCHMARK(WalletBalanceWatch, benchmark::PriorityLevel::HIGH); +} // namespace wallet diff --git a/src/bench/wallet_create_tx.cpp b/src/bench/wallet_create_tx.cpp index 13b0019fd2..5e5bc76fd2 100644 --- a/src/bench/wallet_create_tx.cpp +++ b/src/bench/wallet_create_tx.cpp @@ -83,6 +83,8 @@ static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); + // Set clock to genesis block, so the descriptors/keys creation time don't interfere with the blocks scanning process. + SetMockTime(test_setup->m_node.chainman->GetParams().GenesisBlock().nTime); CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockableWalletDatabase()}; { LOCK(wallet.cs_wallet); @@ -136,6 +138,8 @@ static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type static void AvailableCoins(benchmark::Bench& bench, const std::vector<OutputType>& output_type) { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); + // Set clock to genesis block, so the descriptors/keys creation time don't interfere with the blocks scanning process. + SetMockTime(test_setup->m_node.chainman->GetParams().GenesisBlock().nTime); CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockableWalletDatabase()}; { LOCK(wallet.cs_wallet); diff --git a/src/bench/wallet_loading.cpp b/src/bench/wallet_loading.cpp index cf32fde51e..5453238728 100644 --- a/src/bench/wallet_loading.cpp +++ b/src/bench/wallet_loading.cpp @@ -16,32 +16,7 @@ #include <optional> -using wallet::CWallet; -using wallet::CreateMockableWalletDatabase; -using wallet::TxStateInactive; -using wallet::WALLET_FLAG_DESCRIPTORS; -using wallet::WalletContext; -using wallet::WalletDatabase; - -static std::shared_ptr<CWallet> BenchLoadWallet(std::unique_ptr<WalletDatabase> database, WalletContext& context, uint64_t create_flags) -{ - bilingual_str error; - std::vector<bilingual_str> warnings; - auto wallet = CWallet::Create(context, "", std::move(database), create_flags, error, warnings); - NotifyWalletLoaded(context, wallet); - if (context.chain) { - wallet->postInitProcess(); - } - return wallet; -} - -static void BenchUnloadWallet(std::shared_ptr<CWallet>&& wallet) -{ - SyncWithValidationInterfaceQueue(); - wallet->m_chain_notifications_handler.reset(); - UnloadWallet(std::move(wallet)); -} - +namespace wallet{ static void AddTx(CWallet& wallet) { CMutableTransaction mtx; @@ -66,7 +41,7 @@ static void WalletLoading(benchmark::Bench& bench, bool legacy_wallet) create_flags = WALLET_FLAG_DESCRIPTORS; } auto database = CreateMockableWalletDatabase(); - auto wallet = BenchLoadWallet(std::move(database), context, create_flags); + auto wallet = TestLoadWallet(std::move(database), context, create_flags); // Generate a bunch of transactions and addresses to put into the wallet for (int i = 0; i < 1000; ++i) { @@ -76,14 +51,14 @@ static void WalletLoading(benchmark::Bench& bench, bool legacy_wallet) database = DuplicateMockDatabase(wallet->GetDatabase()); // reload the wallet for the actual benchmark - BenchUnloadWallet(std::move(wallet)); + TestUnloadWallet(std::move(wallet)); bench.epochs(5).run([&] { - wallet = BenchLoadWallet(std::move(database), context, create_flags); + wallet = TestLoadWallet(std::move(database), context, create_flags); // Cleanup database = DuplicateMockDatabase(wallet->GetDatabase()); - BenchUnloadWallet(std::move(wallet)); + TestUnloadWallet(std::move(wallet)); }); } @@ -96,3 +71,4 @@ BENCHMARK(WalletLoadingLegacy, benchmark::PriorityLevel::HIGH); static void WalletLoadingDescriptors(benchmark::Bench& bench) { WalletLoading(bench, /*legacy_wallet=*/false); } BENCHMARK(WalletLoadingDescriptors, benchmark::PriorityLevel::HIGH); #endif +} // namespace wallet diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 16c3bfb708..432bdc8e33 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -12,12 +12,11 @@ // It is part of the libbitcoinkernel project. #include <kernel/chainparams.h> +#include <kernel/chainstatemanager_opts.h> #include <kernel/checks.h> #include <kernel/context.h> #include <kernel/validation_cache_sizes.h> -#include <chainparams.h> -#include <common/args.h> #include <consensus/validation.h> #include <core_io.h> #include <node/blockstorage.h> @@ -31,9 +30,12 @@ #include <validationinterface.h> #include <cassert> +#include <cstdint> #include <filesystem> #include <functional> #include <iosfwd> +#include <memory> +#include <string> int main(int argc, char* argv[]) { @@ -49,18 +51,14 @@ int main(int argc, char* argv[]) } std::filesystem::path abs_datadir = std::filesystem::absolute(argv[1]); std::filesystem::create_directories(abs_datadir); - gArgs.ForceSetArg("-datadir", abs_datadir.string()); - // SETUP: Misc Globals - SelectParams(ChainType::MAIN); - auto chainparams = CChainParams::Main(); - + // SETUP: Context kernel::Context kernel_context{}; // We can't use a goto here, but we can use an assert since none of the // things instantiated so far requires running the epilogue to be torn down // properly - assert(!kernel::SanityChecks(kernel_context).has_value()); + assert(kernel::SanityChecks(kernel_context)); // Necessary for CheckInputScripts (eventually called by ProcessNewBlock), // which will try the script cache first and fall back to actually @@ -80,16 +78,40 @@ int main(int argc, char* argv[]) GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); + class KernelNotifications : public kernel::Notifications + { + public: + void blockTip(SynchronizationState, CBlockIndex&) override + { + std::cout << "Block tip changed" << std::endl; + } + void headerTip(SynchronizationState, int64_t height, int64_t timestamp, bool presync) override + { + std::cout << "Header tip changed: " << height << ", " << timestamp << ", " << presync << std::endl; + } + void progress(const bilingual_str& title, int progress_percent, bool resume_possible) override + { + std::cout << "Progress: " << title.original << ", " << progress_percent << ", " << resume_possible << std::endl; + } + void warning(const bilingual_str& warning) override + { + std::cout << "Warning: " << warning.original << std::endl; + } + }; + auto notifications = std::make_unique<KernelNotifications>(); + // SETUP: Chainstate + auto chainparams = CChainParams::Main(); const ChainstateManager::Options chainman_opts{ .chainparams = *chainparams, - .datadir = gArgs.GetDataDirNet(), + .datadir = abs_datadir, .adjusted_time_callback = NodeClock::now, + .notifications = *notifications, }; const node::BlockManager::Options blockman_opts{ .chainparams = chainman_opts.chainparams, - .blocks_dir = gArgs.GetBlocksDirPath(), + .blocks_dir = abs_datadir / "blocks", }; ChainstateManager chainman{chainman_opts, blockman_opts}; @@ -122,7 +144,8 @@ int main(int argc, char* argv[]) // Main program logic starts here std::cout << "Hello! I'm going to print out some information about your datadir." << std::endl - << "\t" << "Path: " << gArgs.GetDataDirNet() << std::endl; + << "\t" + << "Path: " << abs_datadir << std::endl; { LOCK(chainman.GetMutex()); std::cout diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 48dc11b95a..45db7a9a66 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -10,6 +10,7 @@ #include <chainparamsbase.h> #include <clientversion.h> #include <common/args.h> +#include <common/system.h> #include <common/url.h> #include <compat/compat.h> #include <compat/stdin.h> @@ -23,7 +24,6 @@ #include <util/chaintype.h> #include <util/exception.h> #include <util/strencodings.h> -#include <util/system.h> #include <util/time.h> #include <util/translation.h> diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index e291f20a11..0c25ddf373 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -10,6 +10,7 @@ #include <clientversion.h> #include <coins.h> #include <common/args.h> +#include <common/system.h> #include <compat/compat.h> #include <consensus/amount.h> #include <consensus/consensus.h> @@ -27,7 +28,6 @@ #include <util/rbf.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/translation.h> #include <cstdio> diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp index f62a9f7bbf..582c18c9c6 100644 --- a/src/bitcoin-util.cpp +++ b/src/bitcoin-util.cpp @@ -12,11 +12,11 @@ #include <chainparamsbase.h> #include <clientversion.h> #include <common/args.h> +#include <common/system.h> #include <compat/compat.h> #include <core_io.h> #include <streams.h> #include <util/exception.h> -#include <util/system.h> #include <util/translation.h> #include <version.h> diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 1863173aa8..d5dfbbec27 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -10,6 +10,7 @@ #include <chainparamsbase.h> #include <clientversion.h> #include <common/args.h> +#include <common/system.h> #include <common/url.h> #include <compat/compat.h> #include <interfaces/init.h> @@ -18,7 +19,6 @@ #include <pubkey.h> #include <tinyformat.h> #include <util/exception.h> -#include <util/system.h> #include <util/translation.h> #include <wallet/wallettool.h> diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index e476b06017..e2224befef 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -11,6 +11,7 @@ #include <clientversion.h> #include <common/args.h> #include <common/init.h> +#include <common/system.h> #include <common/url.h> #include <compat/compat.h> #include <init.h> @@ -23,9 +24,7 @@ #include <util/check.h> #include <util/exception.h> #include <util/strencodings.h> -#include <util/syscall_sandbox.h> #include <util/syserror.h> -#include <util/system.h> #include <util/threadnames.h> #include <util/tokenpipe.h> #include <util/translation.h> @@ -112,20 +111,30 @@ int fork_daemon(bool nochdir, bool noclose, TokenPipeEnd& endpoint) #endif -static bool AppInit(NodeContext& node, int argc, char* argv[]) +static bool ParseArgs(ArgsManager& args, int argc, char* argv[]) { - bool fRet = false; - - util::ThreadSetInternalName("init"); - // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's main() - ArgsManager& args = *Assert(node.args); SetupServerArgs(args); std::string error; if (!args.ParseParameters(argc, argv, error)) { return InitError(Untranslated(strprintf("Error parsing command line arguments: %s", error))); } + if (auto error = common::InitConfig(args)) { + return InitError(error->message, error->details); + } + + // Error out when loose non-argument tokens are encountered on command line + for (int i = 1; i < argc; i++) { + if (!IsSwitchChar(argv[i][0])) { + return InitError(Untranslated(strprintf("Command line contains unexpected token '%s', see bitcoind -h for a list of options.", argv[i]))); + } + } + return true; +} + +static bool ProcessInitCommands(ArgsManager& args) +{ // Process help and version before taking care about datadir if (HelpRequested(args) || args.IsArgSet("-version")) { std::string strUsage = PACKAGE_NAME " version " + FormatFullVersion() + "\n"; @@ -142,6 +151,14 @@ static bool AppInit(NodeContext& node, int argc, char* argv[]) return true; } + return false; +} + +static bool AppInit(NodeContext& node) +{ + bool fRet = false; + ArgsManager& args = *Assert(node.args); + #if HAVE_DECL_FORK // Communication with parent after daemonizing. This is used for signalling in the following ways: // - a boolean token is sent when the initialization process (all the Init* functions) have finished to indicate @@ -153,23 +170,12 @@ static bool AppInit(NodeContext& node, int argc, char* argv[]) std::any context{&node}; try { - if (auto error = common::InitConfig(args)) { - return InitError(error->message, error->details); - } - - // Error out when loose non-argument tokens are encountered on command line - for (int i = 1; i < argc; i++) { - if (!IsSwitchChar(argv[i][0])) { - return InitError(Untranslated(strprintf("Command line contains unexpected token '%s', see bitcoind -h for a list of options.", argv[i]))); - } - } - // -server defaults to true for bitcoind but not for the GUI so do this here args.SoftSetBoolArg("-server", true); // Set this early so that parameter interactions go to console InitLogging(args); InitParameterInteraction(args); - if (!AppInitBasicSetup(args)) { + if (!AppInitBasicSetup(args, node.exit_status)) { // InitError will have been called with detailed error, which ends up on console return false; } @@ -235,13 +241,6 @@ static bool AppInit(NodeContext& node, int argc, char* argv[]) daemon_ep.Close(); } #endif - SetSyscallSandboxPolicy(SyscallSandboxPolicy::SHUTOFF); - if (fRet) { - WaitForShutdown(); - } - Interrupt(node); - Shutdown(node); - return fRet; } @@ -264,5 +263,22 @@ MAIN_FUNCTION // Connect bitcoind signal handlers noui_connect(); - return (AppInit(node, argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE); + util::ThreadSetInternalName("init"); + + // Interpret command line arguments + ArgsManager& args = *Assert(node.args); + if (!ParseArgs(args, argc, argv)) return EXIT_FAILURE; + // Process early info return commands such as -help or -version + if (ProcessInitCommands(args)) return EXIT_SUCCESS; + + // Start application + if (AppInit(node)) { + WaitForShutdown(); + } else { + node.exit_status = EXIT_FAILURE; + } + Interrupt(node); + Shutdown(node); + + return node.exit_status; } diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index a29e4f794e..9aa0a6ba20 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -3,16 +3,16 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <blockencodings.h> +#include <chainparams.h> +#include <common/system.h> #include <consensus/consensus.h> #include <consensus/validation.h> -#include <chainparams.h> #include <crypto/sha256.h> #include <crypto/siphash.h> #include <random.h> #include <streams.h> #include <txmempool.h> #include <validation.h> -#include <util/system.h> #include <unordered_map> diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 6f4453d1fe..539578085b 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -6,17 +6,20 @@ #include <chainparams.h> #include <chainparamsbase.h> -#include <chainparamsseeds.h> #include <common/args.h> -#include <consensus/merkle.h> +#include <consensus/params.h> #include <deploymentinfo.h> -#include <hash.h> // for signet block challenge hash #include <logging.h> -#include <script/interpreter.h> +#include <tinyformat.h> #include <util/chaintype.h> +#include <util/strencodings.h> #include <util/string.h> -#include <assert.h> +#include <cassert> +#include <cstdint> +#include <limits> +#include <stdexcept> +#include <vector> void ReadSigNetArgs(const ArgsManager& args, CChainParams::SigNetOptions& options) { @@ -26,9 +29,13 @@ void ReadSigNetArgs(const ArgsManager& args, CChainParams::SigNetOptions& option if (args.IsArgSet("-signetchallenge")) { const auto signet_challenge = args.GetArgs("-signetchallenge"); if (signet_challenge.size() != 1) { - throw std::runtime_error(strprintf("%s: -signetchallenge cannot be multiple values.", __func__)); + throw std::runtime_error("-signetchallenge cannot be multiple values."); } - options.challenge.emplace(ParseHex(signet_challenge[0])); + const auto val{TryParseHex<uint8_t>(signet_challenge[0])}; + if (!val) { + throw std::runtime_error(strprintf("-signetchallenge must be hex, not '%s'.", signet_challenge[0])); + } + options.challenge.emplace(*val); } } diff --git a/src/chainparams.h b/src/chainparams.h index 1e8366dcf5..4743e022db 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -6,20 +6,9 @@ #ifndef BITCOIN_CHAINPARAMS_H #define BITCOIN_CHAINPARAMS_H -#include <kernel/chainparams.h> +#include <kernel/chainparams.h> // IWYU pragma: export -#include <consensus/params.h> -#include <netaddress.h> -#include <primitives/block.h> -#include <protocol.h> -#include <util/chaintype.h> -#include <util/hash_type.h> - -#include <cstdint> #include <memory> -#include <string> -#include <unordered_map> -#include <vector> class ArgsManager; diff --git a/src/checkqueue.h b/src/checkqueue.h index 2c21e5f7d0..a3299fb3fe 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -7,7 +7,6 @@ #include <sync.h> #include <tinyformat.h> -#include <util/syscall_sandbox.h> #include <util/threadnames.h> #include <algorithm> @@ -149,7 +148,6 @@ public: for (int n = 0; n < threads_num; ++n) { m_worker_threads.emplace_back([this, n]() { util::ThreadRename(strprintf("scriptch.%i", n)); - SetSyscallSandboxPolicy(SyscallSandboxPolicy::VALIDATION_SCRIPT_CHECK); Loop(false /* worker thread */); }); } diff --git a/src/common/args.cpp b/src/common/args.cpp index 93f2005931..643838399f 100644 --- a/src/common/args.cpp +++ b/src/common/args.cpp @@ -6,6 +6,7 @@ #include <common/args.h> #include <chainparamsbase.h> +#include <common/settings.h> #include <logging.h> #include <sync.h> #include <tinyformat.h> @@ -14,7 +15,6 @@ #include <util/check.h> #include <util/fs.h> #include <util/fs_helpers.h> -#include <util/settings.h> #include <util/strencodings.h> #ifdef WIN32 @@ -104,7 +104,7 @@ KeyInfo InterpretKey(std::string key) * @return parsed settings value if it is valid, otherwise nullopt accompanied * by a descriptive error string */ -std::optional<util::SettingsValue> InterpretValue(const KeyInfo& key, const std::string* value, +std::optional<common::SettingsValue> InterpretValue(const KeyInfo& key, const std::string* value, unsigned int flags, std::string& error) { // Return negated settings as false values. @@ -238,15 +238,15 @@ bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::strin return false; } - std::optional<util::SettingsValue> value = InterpretValue(keyinfo, val ? &*val : nullptr, *flags, error); + std::optional<common::SettingsValue> value = InterpretValue(keyinfo, val ? &*val : nullptr, *flags, error); if (!value) return false; m_settings.command_line_options[keyinfo.name].push_back(*value); } // we do not allow -includeconf from command line, only -noincludeconf - if (auto* includes = util::FindKey(m_settings.command_line_options, "includeconf")) { - const util::SettingsSpan values{*includes}; + if (auto* includes = common::FindKey(m_settings.command_line_options, "includeconf")) { + const common::SettingsSpan values{*includes}; // Range may be empty if -noincludeconf was passed if (!values.empty()) { error = "-includeconf cannot be used from commandline; -includeconf=" + values.begin()->write(); @@ -361,7 +361,7 @@ std::optional<const ArgsManager::Command> ArgsManager::GetCommand() const std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) const { std::vector<std::string> result; - for (const util::SettingsValue& value : GetSettingsList(strArg)) { + for (const common::SettingsValue& value : GetSettingsList(strArg)) { result.push_back(value.isFalse() ? "0" : value.isTrue() ? "1" : value.get_str()); } return result; @@ -408,7 +408,7 @@ bool ArgsManager::ReadSettingsFile(std::vector<std::string>* errors) LOCK(cs_args); m_settings.rw_settings.clear(); std::vector<std::string> read_errors; - if (!util::ReadSettings(path, m_settings.rw_settings, read_errors)) { + if (!common::ReadSettings(path, m_settings.rw_settings, read_errors)) { SaveErrors(read_errors, errors); return false; } @@ -430,7 +430,7 @@ bool ArgsManager::WriteSettingsFile(std::vector<std::string>* errors, bool backu LOCK(cs_args); std::vector<std::string> write_errors; - if (!util::WriteSettings(path_tmp, m_settings.rw_settings, write_errors)) { + if (!common::WriteSettings(path_tmp, m_settings.rw_settings, write_errors)) { SaveErrors(write_errors, errors); return false; } @@ -441,10 +441,10 @@ bool ArgsManager::WriteSettingsFile(std::vector<std::string>* errors, bool backu return true; } -util::SettingsValue ArgsManager::GetPersistentSetting(const std::string& name) const +common::SettingsValue ArgsManager::GetPersistentSetting(const std::string& name) const { LOCK(cs_args); - return util::GetSetting(m_settings, m_network, name, !UseDefaultSection("-" + name), + return common::GetSetting(m_settings, m_network, name, !UseDefaultSection("-" + name), /*ignore_nonpersistent=*/true, /*get_chain_type=*/false); } @@ -460,11 +460,11 @@ std::string ArgsManager::GetArg(const std::string& strArg, const std::string& st std::optional<std::string> ArgsManager::GetArg(const std::string& strArg) const { - const util::SettingsValue value = GetSetting(strArg); + const common::SettingsValue value = GetSetting(strArg); return SettingToString(value); } -std::optional<std::string> SettingToString(const util::SettingsValue& value) +std::optional<std::string> SettingToString(const common::SettingsValue& value) { if (value.isNull()) return std::nullopt; if (value.isFalse()) return "0"; @@ -473,7 +473,7 @@ std::optional<std::string> SettingToString(const util::SettingsValue& value) return value.get_str(); } -std::string SettingToString(const util::SettingsValue& value, const std::string& strDefault) +std::string SettingToString(const common::SettingsValue& value, const std::string& strDefault) { return SettingToString(value).value_or(strDefault); } @@ -485,11 +485,11 @@ int64_t ArgsManager::GetIntArg(const std::string& strArg, int64_t nDefault) cons std::optional<int64_t> ArgsManager::GetIntArg(const std::string& strArg) const { - const util::SettingsValue value = GetSetting(strArg); + const common::SettingsValue value = GetSetting(strArg); return SettingToInt(value); } -std::optional<int64_t> SettingToInt(const util::SettingsValue& value) +std::optional<int64_t> SettingToInt(const common::SettingsValue& value) { if (value.isNull()) return std::nullopt; if (value.isFalse()) return 0; @@ -498,7 +498,7 @@ std::optional<int64_t> SettingToInt(const util::SettingsValue& value) return LocaleIndependentAtoi<int64_t>(value.get_str()); } -int64_t SettingToInt(const util::SettingsValue& value, int64_t nDefault) +int64_t SettingToInt(const common::SettingsValue& value, int64_t nDefault) { return SettingToInt(value).value_or(nDefault); } @@ -510,18 +510,18 @@ bool ArgsManager::GetBoolArg(const std::string& strArg, bool fDefault) const std::optional<bool> ArgsManager::GetBoolArg(const std::string& strArg) const { - const util::SettingsValue value = GetSetting(strArg); + const common::SettingsValue value = GetSetting(strArg); return SettingToBool(value); } -std::optional<bool> SettingToBool(const util::SettingsValue& value) +std::optional<bool> SettingToBool(const common::SettingsValue& value) { if (value.isNull()) return std::nullopt; if (value.isBool()) return value.get_bool(); return InterpretBool(value.get_str()); } -bool SettingToBool(const util::SettingsValue& value, bool fDefault) +bool SettingToBool(const common::SettingsValue& value, bool fDefault) { return SettingToBool(value).value_or(fDefault); } @@ -716,7 +716,8 @@ bool CheckDataDirOption(const ArgsManager& args) fs::path ArgsManager::GetConfigFilePath() const { - return GetConfigFile(*this, GetPathArg("-conf", BITCOIN_CONF_FILENAME)); + LOCK(cs_args); + return *Assert(m_config_path); } ChainType ArgsManager::GetChainType() const @@ -737,7 +738,7 @@ std::variant<ChainType, std::string> ArgsManager::GetChainArg() const { auto get_net = [&](const std::string& arg) { LOCK(cs_args); - util::SettingsValue value = util::GetSetting(m_settings, /* section= */ "", SettingName(arg), + common::SettingsValue value = common::GetSetting(m_settings, /* section= */ "", SettingName(arg), /* ignore_default_section_config= */ false, /*ignore_nonpersistent=*/false, /* get_chain_type= */ true); @@ -768,24 +769,24 @@ bool ArgsManager::UseDefaultSection(const std::string& arg) const return m_network == ChainTypeToString(ChainType::MAIN) || m_network_only_args.count(arg) == 0; } -util::SettingsValue ArgsManager::GetSetting(const std::string& arg) const +common::SettingsValue ArgsManager::GetSetting(const std::string& arg) const { LOCK(cs_args); - return util::GetSetting( + return common::GetSetting( m_settings, m_network, SettingName(arg), !UseDefaultSection(arg), /*ignore_nonpersistent=*/false, /*get_chain_type=*/false); } -std::vector<util::SettingsValue> ArgsManager::GetSettingsList(const std::string& arg) const +std::vector<common::SettingsValue> ArgsManager::GetSettingsList(const std::string& arg) const { LOCK(cs_args); - return util::GetSettingsList(m_settings, m_network, SettingName(arg), !UseDefaultSection(arg)); + return common::GetSettingsList(m_settings, m_network, SettingName(arg), !UseDefaultSection(arg)); } void ArgsManager::logArgsPrefix( const std::string& prefix, const std::string& section, - const std::map<std::string, std::vector<util::SettingsValue>>& args) const + const std::map<std::string, std::vector<common::SettingsValue>>& args) const { std::string section_str = section.empty() ? "" : "[" + section + "] "; for (const auto& arg : args) { diff --git a/src/common/args.h b/src/common/args.h index 537a64fcfd..ae3ed02bc7 100644 --- a/src/common/args.h +++ b/src/common/args.h @@ -5,11 +5,11 @@ #ifndef BITCOIN_COMMON_ARGS_H #define BITCOIN_COMMON_ARGS_H +#include <common/settings.h> #include <compat/compat.h> #include <sync.h> #include <util/chaintype.h> #include <util/fs.h> -#include <util/settings.h> #include <iosfwd> #include <list> @@ -28,7 +28,6 @@ extern const char * const BITCOIN_SETTINGS_FILENAME; // Return true if -datadir option points to a valid directory or is not specified. bool CheckDataDirOption(const ArgsManager& args); -fs::path GetConfigFile(const ArgsManager& args, const fs::path& configuration_file_path); /** * Most paths passed as configuration arguments are treated as relative to @@ -76,7 +75,7 @@ struct KeyInfo { KeyInfo InterpretKey(std::string key); -std::optional<util::SettingsValue> InterpretValue(const KeyInfo& key, const std::string* value, +std::optional<common::SettingsValue> InterpretValue(const KeyInfo& key, const std::string* value, unsigned int flags, std::string& error); struct SectionInfo { @@ -85,14 +84,14 @@ struct SectionInfo { int m_line; }; -std::string SettingToString(const util::SettingsValue&, const std::string&); -std::optional<std::string> SettingToString(const util::SettingsValue&); +std::string SettingToString(const common::SettingsValue&, const std::string&); +std::optional<std::string> SettingToString(const common::SettingsValue&); -int64_t SettingToInt(const util::SettingsValue&, int64_t); -std::optional<int64_t> SettingToInt(const util::SettingsValue&); +int64_t SettingToInt(const common::SettingsValue&, int64_t); +std::optional<int64_t> SettingToInt(const common::SettingsValue&); -bool SettingToBool(const util::SettingsValue&, bool); -std::optional<bool> SettingToBool(const util::SettingsValue&); +bool SettingToBool(const common::SettingsValue&, bool); +std::optional<bool> SettingToBool(const common::SettingsValue&); class ArgsManager { @@ -131,13 +130,14 @@ protected: }; mutable RecursiveMutex cs_args; - util::Settings m_settings GUARDED_BY(cs_args); + common::Settings m_settings GUARDED_BY(cs_args); std::vector<std::string> m_command GUARDED_BY(cs_args); std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); bool m_accept_any_command GUARDED_BY(cs_args){true}; std::list<SectionInfo> m_config_sections GUARDED_BY(cs_args); + std::optional<fs::path> m_config_path GUARDED_BY(cs_args); mutable fs::path m_cached_blocks_path GUARDED_BY(cs_args); mutable fs::path m_cached_datadir_path GUARDED_BY(cs_args); mutable fs::path m_cached_network_datadir_path GUARDED_BY(cs_args); @@ -159,12 +159,12 @@ protected: * false if "-nosetting" argument was passed, and a string if a "-setting=value" * argument was passed. */ - util::SettingsValue GetSetting(const std::string& arg) const; + common::SettingsValue GetSetting(const std::string& arg) const; /** * Get list of setting values. */ - std::vector<util::SettingsValue> GetSettingsList(const std::string& arg) const; + std::vector<common::SettingsValue> GetSettingsList(const std::string& arg) const; ArgsManager(); ~ArgsManager(); @@ -394,7 +394,7 @@ protected: * Get current setting from config file or read/write settings file, * ignoring nonpersistent command line or forced settings values. */ - util::SettingsValue GetPersistentSetting(const std::string& name) const; + common::SettingsValue GetPersistentSetting(const std::string& name) const; /** * Access settings with lock held. @@ -433,7 +433,7 @@ private: void logArgsPrefix( const std::string& prefix, const std::string& section, - const std::map<std::string, std::vector<util::SettingsValue>>& args) const; + const std::map<std::string, std::vector<common::SettingsValue>>& args) const; }; extern ArgsManager gArgs; diff --git a/src/common/config.cpp b/src/common/config.cpp index 5efb5efb67..1c85273f69 100644 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -4,13 +4,13 @@ #include <common/args.h> +#include <common/settings.h> #include <logging.h> #include <sync.h> #include <tinyformat.h> #include <univalue.h> #include <util/chaintype.h> #include <util/fs.h> -#include <util/settings.h> #include <util/string.h> #include <algorithm> @@ -27,11 +27,6 @@ #include <utility> #include <vector> -fs::path GetConfigFile(const ArgsManager& args, const fs::path& configuration_file_path) -{ - return AbsPathForConfigVal(args, configuration_file_path, /*net_specific=*/false); -} - static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::list<SectionInfo>& sections) { std::string str, prefix; @@ -103,7 +98,7 @@ bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& file std::optional<unsigned int> flags = GetArgFlags('-' + key.name); if (!IsConfSupported(key, error)) return false; if (flags) { - std::optional<util::SettingsValue> value = InterpretValue(key, &option.second, *flags, error); + std::optional<common::SettingsValue> value = InterpretValue(key, &option.second, *flags, error); if (!value) { return false; } @@ -126,6 +121,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) LOCK(cs_args); m_settings.ro_config.clear(); m_config_sections.clear(); + m_config_path = AbsPathForConfigVal(*this, GetPathArg("-conf", BITCOIN_CONF_FILENAME), /*net_specific=*/false); } const auto conf_path{GetConfigFilePath()}; @@ -146,9 +142,9 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) bool use_conf_file{true}; { LOCK(cs_args); - if (auto* includes = util::FindKey(m_settings.command_line_options, "includeconf")) { + if (auto* includes = common::FindKey(m_settings.command_line_options, "includeconf")) { // ParseParameters() fails if a non-negated -includeconf is passed on the command-line - assert(util::SettingsSpan(*includes).last_negated()); + assert(common::SettingsSpan(*includes).last_negated()); use_conf_file = false; } } @@ -159,9 +155,9 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) auto add_includes = [&](const std::string& network, size_t skip = 0) { size_t num_values = 0; LOCK(cs_args); - if (auto* section = util::FindKey(m_settings.ro_config, network)) { - if (auto* values = util::FindKey(*section, "includeconf")) { - for (size_t i = std::max(skip, util::SettingsSpan(*values).negated()); i < values->size(); ++i) { + if (auto* section = common::FindKey(m_settings.ro_config, network)) { + if (auto* values = common::FindKey(*section, "includeconf")) { + for (size_t i = std::max(skip, common::SettingsSpan(*values).negated()); i < values->size(); ++i) { conf_file_names.push_back((*values)[i].get_str()); } num_values = values->size(); @@ -176,7 +172,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) const size_t default_includes = add_includes({}); for (const std::string& conf_file_name : conf_file_names) { - std::ifstream conf_file_stream{GetConfigFile(*this, fs::PathFromString(conf_file_name))}; + std::ifstream conf_file_stream{AbsPathForConfigVal(*this, fs::PathFromString(conf_file_name), /*net_specific=*/false)}; if (conf_file_stream.good()) { if (!ReadConfigStream(conf_file_stream, conf_file_name, error, ignore_invalid_keys)) { return false; diff --git a/src/common/init.cpp b/src/common/init.cpp index 8933d59c27..412d73aec7 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -5,6 +5,7 @@ #include <chainparams.h> #include <common/args.h> #include <common/init.h> +#include <logging.h> #include <tinyformat.h> #include <util/fs.h> #include <util/translation.h> @@ -20,6 +21,19 @@ std::optional<ConfigError> InitConfig(ArgsManager& args, SettingsAbortFn setting if (!CheckDataDirOption(args)) { return ConfigError{ConfigStatus::FAILED, strprintf(_("Specified data directory \"%s\" does not exist."), args.GetArg("-datadir", ""))}; } + + // Record original datadir and config paths before parsing the config + // file. It is possible for the config file to contain a datadir= line + // that changes the datadir path after it is parsed. This is useful for + // CLI tools to let them use a different data storage location without + // needing to pass it every time on the command line. (It is not + // possible for the config file to cause another configuration to be + // used, though. Specifying a conf= option in the config file causes a + // parse error, and specifying a datadir= location containing another + // bitcoin.conf file just ignores the other file.) + const fs::path orig_datadir_path{args.GetDataDirBase()}; + const fs::path orig_config_path{AbsPathForConfigVal(args, args.GetPathArg("-conf", BITCOIN_CONF_FILENAME), /*net_specific=*/false)}; + std::string error; if (!args.ReadConfigFiles(error, true)) { return ConfigError{ConfigStatus::FAILED, strprintf(_("Error reading configuration file: %s"), error)}; @@ -48,6 +62,32 @@ std::optional<ConfigError> InitConfig(ArgsManager& args, SettingsAbortFn setting fs::create_directories(net_path / "wallets"); } + // Show an error or warning if there is a bitcoin.conf file in the + // datadir that is being ignored. + const fs::path base_config_path = base_path / BITCOIN_CONF_FILENAME; + if (fs::exists(base_config_path) && !fs::equivalent(orig_config_path, base_config_path)) { + const std::string cli_config_path = args.GetArg("-conf", ""); + const std::string config_source = cli_config_path.empty() + ? strprintf("data directory %s", fs::quoted(fs::PathToString(orig_datadir_path))) + : strprintf("command line argument %s", fs::quoted("-conf=" + cli_config_path)); + const std::string error = strprintf( + "Data directory %1$s contains a %2$s file which is ignored, because a different configuration file " + "%3$s from %4$s is being used instead. Possible ways to address this would be to:\n" + "- Delete or rename the %2$s file in data directory %1$s.\n" + "- Change datadir= or conf= options to specify one configuration file, not two, and use " + "includeconf= to include any other configuration files.\n" + "- Set allowignoredconf=1 option to treat this condition as a warning, not an error.", + fs::quoted(fs::PathToString(base_path)), + fs::quoted(BITCOIN_CONF_FILENAME), + fs::quoted(fs::PathToString(orig_config_path)), + config_source); + if (args.GetBoolArg("-allowignoredconf", false)) { + LogPrintf("Warning: %s\n", error); + } else { + return ConfigError{ConfigStatus::FAILED, Untranslated(error)}; + } + } + // Create settings.json if -nosettings was not specified. if (args.GetSettingsPath()) { std::vector<std::string> details; diff --git a/src/common/run_command.cpp b/src/common/run_command.cpp index 6ad9f75b5d..8bd5febd53 100644 --- a/src/common/run_command.cpp +++ b/src/common/run_command.cpp @@ -12,16 +12,7 @@ #include <univalue.h> #ifdef ENABLE_EXTERNAL_SIGNER -#if defined(__GNUC__) -// Boost 1.78 requires the following workaround. -// See: https://github.com/boostorg/process/issues/235 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wnarrowing" -#endif #include <boost/process.hpp> -#if defined(__GNUC__) -#pragma GCC diagnostic pop -#endif #endif // ENABLE_EXTERNAL_SIGNER UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in) diff --git a/src/util/settings.cpp b/src/common/settings.cpp index db3d60046e..5761e8b321 100644 --- a/src/util/settings.cpp +++ b/src/common/settings.cpp @@ -2,18 +2,21 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/fs.h> -#include <util/settings.h> +#include <common/settings.h> #include <tinyformat.h> #include <univalue.h> +#include <util/fs.h> +#include <algorithm> #include <fstream> +#include <iterator> #include <map> #include <string> +#include <utility> #include <vector> -namespace util { +namespace common { namespace { enum class Source { @@ -112,7 +115,7 @@ bool WriteSettings(const fs::path& path, { SettingsValue out(SettingsValue::VOBJ); for (const auto& value : values) { - out.__pushKV(value.first, value.second); + out.pushKVEnd(value.first, value.second); } std::ofstream file; file.open(path); @@ -255,4 +258,4 @@ size_t SettingsSpan::negated() const return 0; } -} // namespace util +} // namespace common diff --git a/src/util/settings.h b/src/common/settings.h index bb1fe585e1..0e9d376e23 100644 --- a/src/util/settings.h +++ b/src/common/settings.h @@ -2,18 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_UTIL_SETTINGS_H -#define BITCOIN_UTIL_SETTINGS_H +#ifndef BITCOIN_COMMON_SETTINGS_H +#define BITCOIN_COMMON_SETTINGS_H #include <util/fs.h> +#include <cstddef> #include <map> #include <string> #include <vector> class UniValue; -namespace util { +namespace common { //! Settings value type (string/integer/boolean/null variant). //! @@ -109,6 +110,6 @@ auto FindKey(Map&& map, Key&& key) -> decltype(&map.at(key)) return it == map.end() ? nullptr : &it->second; } -} // namespace util +} // namespace common -#endif // BITCOIN_UTIL_SETTINGS_H +#endif // BITCOIN_COMMON_SETTINGS_H diff --git a/src/util/system.cpp b/src/common/system.cpp index 598e6adb88..1d1c5fa56a 100644 --- a/src/util/system.cpp +++ b/src/common/system.cpp @@ -3,20 +3,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/system.h> +#include <common/system.h> #include <logging.h> #include <util/string.h> -#include <util/syserror.h> #include <util/time.h> -#if (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) -#include <pthread.h> -#include <pthread_np.h> -#endif - #ifndef WIN32 -#include <sched.h> #include <sys/stat.h> #else #include <codecvt> @@ -112,14 +105,3 @@ int64_t GetStartupTime() { return nStartupTime; } - -void ScheduleBatchPriority() -{ -#ifdef SCHED_BATCH - const static sched_param param{}; - const int rc = pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m); - if (rc != 0) { - LogPrintf("Failed to pthread_setschedparam: %s\n", SysErrorString(rc)); - } -#endif -} diff --git a/src/common/system.h b/src/common/system.h new file mode 100644 index 0000000000..40206aaa01 --- /dev/null +++ b/src/common/system.h @@ -0,0 +1,38 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMMON_SYSTEM_H +#define BITCOIN_COMMON_SYSTEM_H + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <compat/assumptions.h> +#include <compat/compat.h> + +#include <set> +#include <stdint.h> +#include <string> + +// Application startup time (used for uptime calculation) +int64_t GetStartupTime(); + +void SetupEnvironment(); +bool SetupNetworking(); +#ifndef WIN32 +std::string ShellEscape(const std::string& arg); +#endif +#if HAVE_SYSTEM +void runCommand(const std::string& strCommand); +#endif + +/** + * Return the number of cores available on the current system. + * @note This does count virtual cores, such as those provided by HyperThreading. + */ +int GetNumCores(); + +#endif // BITCOIN_COMMON_SYSTEM_H diff --git a/src/consensus/validation.h b/src/consensus/validation.h index ad8ee676b2..d5bf08cd61 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -145,7 +145,7 @@ class BlockValidationState : public ValidationState<BlockValidationResult> {}; // using only serialization with and without witness data. As witness_size // is equal to total_size - stripped_size, this formula is identical to: // weight = (stripped_size * 3) + total_size. -static inline int64_t GetTransactionWeight(const CTransaction& tx) +static inline int32_t GetTransactionWeight(const CTransaction& tx) { return ::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(tx, PROTOCOL_VERSION); } diff --git a/src/core_write.cpp b/src/core_write.cpp index b0e3b0b3c4..54ca306f60 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -4,6 +4,7 @@ #include <core_io.h> +#include <common/system.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/validation.h> @@ -17,7 +18,6 @@ #include <univalue.h> #include <util/check.h> #include <util/strencodings.h> -#include <util/system.h> #include <map> #include <string> diff --git a/src/external_signer.h b/src/external_signer.h index 90f07478e3..81a601811a 100644 --- a/src/external_signer.h +++ b/src/external_signer.h @@ -5,8 +5,8 @@ #ifndef BITCOIN_EXTERNAL_SIGNER_H #define BITCOIN_EXTERNAL_SIGNER_H +#include <common/system.h> #include <univalue.h> -#include <util/system.h> #include <string> #include <vector> diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 4f8a2b4d8d..a83f4421d7 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -18,7 +18,6 @@ #include <shutdown.h> #include <sync.h> #include <util/strencodings.h> -#include <util/syscall_sandbox.h> #include <util/threadnames.h> #include <util/translation.h> @@ -170,12 +169,8 @@ static bool ClientAllowed(const CNetAddr& netaddr) static bool InitHTTPAllowList() { rpc_allow_subnets.clear(); - CNetAddr localv4; - CNetAddr localv6; - LookupHost("127.0.0.1", localv4, false); - LookupHost("::1", localv6, false); - rpc_allow_subnets.push_back(CSubNet(localv4, 8)); // always allow IPv4 local subnet - rpc_allow_subnets.push_back(CSubNet(localv6)); // always allow IPv6 localhost + rpc_allow_subnets.push_back(CSubNet{LookupHost("127.0.0.1", false).value(), 8}); // always allow IPv4 local subnet + rpc_allow_subnets.push_back(CSubNet{LookupHost("::1", false).value()}); // always allow IPv6 localhost for (const std::string& strAllow : gArgs.GetArgs("-rpcallowip")) { CSubNet subnet; LookupSubNet(strAllow, subnet); @@ -226,8 +221,10 @@ static void http_request_cb(struct evhttp_request* req, void* arg) }, nullptr); } - // Disable reading to work around a libevent bug, fixed in 2.2.0. - if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) { + // Disable reading to work around a libevent bug, fixed in 2.1.9 + // See https://github.com/libevent/libevent/commit/5ff8eb26371c4dc56f384b2de35bea2d87814779 + // and https://github.com/bitcoin/bitcoin/pull/11593. + if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02010900) { evhttp_connection* conn = evhttp_request_get_connection(req); if (conn) { bufferevent* bev = evhttp_connection_get_bufferevent(conn); @@ -301,7 +298,6 @@ static void http_reject_request_cb(struct evhttp_request* req, void*) static void ThreadHTTP(struct event_base* base) { util::ThreadRename("http"); - SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_HTTP_SERVER); LogPrint(BCLog::HTTP, "Entering http event loop\n"); event_base_dispatch(base); // Event loop will be interrupted by InterruptHTTPServer() @@ -338,8 +334,8 @@ static bool HTTPBindAddresses(struct evhttp* http) LogPrintf("Binding RPC on address %s port %i\n", i->first, i->second); evhttp_bound_socket *bind_handle = evhttp_bind_socket_with_handle(http, i->first.empty() ? nullptr : i->first.c_str(), i->second); if (bind_handle) { - CNetAddr addr; - if (i->first.empty() || (LookupHost(i->first, addr, false) && addr.IsBindAny())) { + const std::optional<CNetAddr> addr{LookupHost(i->first, false)}; + if (i->first.empty() || (addr.has_value() && addr->IsBindAny())) { LogPrintf("WARNING: the RPC server is not safe to expose to untrusted networks such as the public internet\n"); } boundSockets.push_back(bind_handle); @@ -354,7 +350,6 @@ static bool HTTPBindAddresses(struct evhttp* http) static void HTTPWorkQueueRun(WorkQueue<HTTPClosure>* queue, int worker_num) { util::ThreadRename(strprintf("httpworker.%i", worker_num)); - SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_HTTP_SERVER_WORKER); queue->Run(); } @@ -608,7 +603,7 @@ void HTTPRequest::WriteReply(int nStatus, const std::string& strReply) evhttp_send_reply(req_copy, nStatus, nullptr, nullptr); // Re-enable reading from the socket. This is the second part of the libevent // workaround above. - if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) { + if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02010900) { evhttp_connection* conn = evhttp_request_get_connection(req_copy); if (conn) { bufferevent* bev = evhttp_connection_get_bufferevent(conn); diff --git a/src/i2p.cpp b/src/i2p.cpp index c67b6ed185..f03e375adf 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -336,7 +336,7 @@ void Session::GenerateAndSavePrivateKey(const Sock& sock) { DestGenerate(sock); - // umask is set to 0077 in util/system.cpp, which is ok. + // umask is set to 0077 in common/system.cpp, which is ok. if (!WriteBinaryFile(m_private_key_file, std::string(m_private_key.begin(), m_private_key.end()))) { throw std::runtime_error( diff --git a/src/index/base.cpp b/src/index/base.cpp index 7444579395..ec23cc1247 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -14,7 +14,6 @@ #include <node/interface_ui.h> #include <shutdown.h> #include <tinyformat.h> -#include <util/syscall_sandbox.h> #include <util/thread.h> #include <util/translation.h> #include <validation.h> // For g_chainman @@ -33,11 +32,7 @@ constexpr auto SYNC_LOCATOR_WRITE_INTERVAL{30s}; template <typename... Args> static void FatalError(const char* fmt, const Args&... args) { - std::string strMessage = tfm::format(fmt, args...); - SetMiscWarning(Untranslated(strMessage)); - LogPrintf("*** %s\n", strMessage); - InitError(_("A fatal internal error occurred, see debug.log for details")); - StartShutdown(); + AbortNode(tfm::format(fmt, args...)); } CBlockLocator GetLocator(interfaces::Chain& chain, const uint256& block_hash) @@ -103,15 +98,12 @@ bool BaseIndex::Init() SetBestBlockIndex(locator_index); } - // Note: this will latch to true immediately if the user starts up with an empty - // datadir and an index enabled. If this is the case, indexation will happen solely - // via `BlockConnected` signals until, possibly, the next restart. - m_synced = m_best_block_index.load() == active_chain.Tip(); - // Skip pruning check if indexes are not ready to sync (because reindex-chainstate has wiped the chain). - if (!m_synced && g_indexes_ready_to_sync) { + const CBlockIndex* start_block = m_best_block_index.load(); + bool synced = start_block == active_chain.Tip(); + if (!synced && g_indexes_ready_to_sync) { bool prune_violation = false; - if (!m_best_block_index) { + if (!start_block) { // index is not built yet // make sure we have all block data back to the genesis prune_violation = m_chainstate->m_blockman.GetFirstStoredBlock(*active_chain.Tip()) != active_chain.Genesis(); @@ -119,7 +111,7 @@ bool BaseIndex::Init() // in case the index has a best block set and is not fully synced // check if we have the required blocks to continue building the index else { - const CBlockIndex* block_to_test = m_best_block_index.load(); + const CBlockIndex* block_to_test = start_block; if (!active_chain.Contains(block_to_test)) { // if the bestblock is not part of the mainchain, find the fork // and make sure we have all data down to the fork @@ -143,6 +135,16 @@ bool BaseIndex::Init() return InitError(strprintf(Untranslated("%s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"), GetName())); } } + + // Child init + if (!CustomInit(start_block ? std::make_optional(interfaces::BlockKey{start_block->GetBlockHash(), start_block->nHeight}) : std::nullopt)) { + return false; + } + + // Note: this will latch to true immediately if the user starts up with an empty + // datadir and an index enabled. If this is the case, indexation will happen solely + // via `BlockConnected` signals until, possibly, the next restart. + m_synced = synced; return true; } @@ -164,7 +166,6 @@ static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev, CChain& void BaseIndex::ThreadSync() { - SetSyscallSandboxPolicy(SyscallSandboxPolicy::TX_INDEX); // Wait for a possible reindex-chainstate to finish until continuing // with the index sync while (!g_indexes_ready_to_sync) { @@ -408,11 +409,6 @@ bool BaseIndex::Start() RegisterValidationInterface(this); if (!Init()) return false; - const CBlockIndex* index = m_best_block_index.load(); - if (!CustomInit(index ? std::make_optional(interfaces::BlockKey{index->GetBlockHash(), index->nHeight}) : std::nullopt)) { - return false; - } - m_thread_sync = std::thread(&util::TraceThread, GetName(), [this] { ThreadSync(); }); return true; } diff --git a/src/init.cpp b/src/init.cpp index a543fd9ef2..c38352ee38 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -20,6 +20,7 @@ #include <chainparams.h> #include <chainparamsbase.h> #include <common/args.h> +#include <common/system.h> #include <consensus/amount.h> #include <deploymentstatus.h> #include <hash.h> @@ -45,6 +46,7 @@ #include <node/chainstatemanager_args.h> #include <node/context.h> #include <node/interface_ui.h> +#include <node/kernel_notifications.h> #include <node/mempool_args.h> #include <node/mempool_persist_args.h> #include <node/miner.h> @@ -75,13 +77,13 @@ #include <util/fs.h> #include <util/fs_helpers.h> #include <util/moneystr.h> +#include <util/result.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/syscall_sandbox.h> #include <util/syserror.h> -#include <util/system.h> #include <util/thread.h> #include <util/threadnames.h> +#include <util/time.h> #include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -124,6 +126,7 @@ using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINTPRIORITY; using node::fReindex; using node::g_indexes_ready_to_sync; +using node::KernelNotifications; using node::LoadChainstate; using node::MempoolPath; using node::NodeContext; @@ -444,6 +447,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE_MB), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -583,6 +587,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (%sdefault: %u)", "testnet/regtest only; ", !testnetChainParams->RequireStandard()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::NODE_RELAY); argsman.AddArg("-incrementalrelayfee=<amt>", strprintf("Fee rate (in %s/kvB) used to define cost of relay, used for mempool limiting and replacement policy. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_INCREMENTAL_RELAY_FEE)), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::NODE_RELAY); argsman.AddArg("-dustrelayfee=<amt>", strprintf("Fee rate (in %s/kvB) used to define dust, the value of an output such that it will cost more than its value in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE)), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::NODE_RELAY); + argsman.AddArg("-acceptstalefeeestimates", strprintf("Read fee estimates even if they are stale (%sdefault: %u) fee estimates are considered stale if they are %s hours old", "regtest only; ", DEFAULT_ACCEPT_STALE_FEE_ESTIMATES, Ticks<std::chrono::hours>(MAX_FILE_AGE)), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-bytespersigop", strprintf("Equivalent bytes per sigop in transactions for relay and mining (default: %u)", DEFAULT_BYTES_PER_SIGOP), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); argsman.AddArg("-datacarrier", strprintf("Relay and mine data carrier transactions (default: %u)", DEFAULT_ACCEPT_DATACARRIER), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); argsman.AddArg("-datacarriersize", strprintf("Maximum size of data in data carrier transactions we relay and mine (default: %u)", MAX_OP_RETURN_RELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); @@ -624,10 +629,6 @@ void SetupServerArgs(ArgsManager& argsman) hidden_args.emplace_back("-daemonwait"); #endif -#if defined(USE_SYSCALL_SANDBOX) - argsman.AddArg("-sandbox=<mode>", "Use the experimental syscall sandbox in the specified mode (-sandbox=log-and-abort or -sandbox=abort). Allow only expected syscalls to be used by bitcoind. Note that this is an experimental new feature that may cause bitcoind to exit or crash unexpectedly: use with caution. In the \"log-and-abort\" mode the invocation of an unexpected syscall results in a debug handler being invoked which will log the incident and terminate the program (without executing the unexpected syscall). In the \"abort\" mode the invocation of an unexpected syscall results in the entire process being killed immediately by the kernel without executing the unexpected syscall.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); -#endif // USE_SYSCALL_SANDBOX - // Add the hidden options argsman.AddHiddenArgs(hidden_args); } @@ -797,7 +798,7 @@ std::set<BlockFilterType> g_enabled_filter_types; std::terminate(); }; -bool AppInitBasicSetup(const ArgsManager& args) +bool AppInitBasicSetup(const ArgsManager& args, std::atomic<int>& exit_status) { // ********************************************************* Step 1: setup #ifdef _MSC_VER @@ -811,7 +812,7 @@ bool AppInitBasicSetup(const ArgsManager& args) // Enable heap terminate-on-corruption HeapSetInformation(nullptr, HeapEnableTerminationOnCorruption, nullptr, 0); #endif - if (!InitShutdownState()) { + if (!InitShutdownState(exit_status)) { return InitError(Untranslated("Initializing wait-for-shutdown state failed.")); } @@ -838,7 +839,7 @@ bool AppInitBasicSetup(const ArgsManager& args) return true; } -bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandbox) +bool AppInitParameterInteraction(const ArgsManager& args) { const CChainParams& chainparams = Params(); // ********************************************************* Step 2: parameter interactions @@ -946,8 +947,10 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); // ********************************************************* Step 3: parameter-to-internal-flags - init::SetLoggingCategories(args); - init::SetLoggingLevel(args); + auto result = init::SetLoggingCategories(args); + if (!result) return InitError(util::ErrorString(result)); + result = init::SetLoggingLevel(args); + if (!result) return InitError(util::ErrorString(result)); nConnectTimeout = args.GetIntArg("-timeout", DEFAULT_CONNECT_TIMEOUT); if (nConnectTimeout <= 0) { @@ -983,55 +986,25 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb if (args.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) return InitError(Untranslated("Unknown rpcserialversion requested.")); -#if defined(USE_SYSCALL_SANDBOX) - if (args.IsArgSet("-sandbox") && !args.IsArgNegated("-sandbox")) { - const std::string sandbox_arg{args.GetArg("-sandbox", "")}; - bool log_syscall_violation_before_terminating{false}; - if (sandbox_arg == "log-and-abort") { - log_syscall_violation_before_terminating = true; - } else if (sandbox_arg == "abort") { - // log_syscall_violation_before_terminating is false by default. - } else { - return InitError(Untranslated("Unknown syscall sandbox mode (-sandbox=<mode>). Available modes are \"log-and-abort\" and \"abort\".")); - } - // execve(...) is not allowed by the syscall sandbox. - const std::vector<std::string> features_using_execve{ - "-alertnotify", - "-blocknotify", - "-signer", - "-startupnotify", - "-walletnotify", - }; - for (const std::string& feature_using_execve : features_using_execve) { - if (!args.GetArg(feature_using_execve, "").empty()) { - return InitError(Untranslated(strprintf("The experimental syscall sandbox feature (-sandbox=<mode>) is incompatible with %s (which uses execve).", feature_using_execve))); - } - } - if (!SetupSyscallSandbox(log_syscall_violation_before_terminating)) { - return InitError(Untranslated("Installation of the syscall sandbox failed.")); - } - if (use_syscall_sandbox) { - SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION); - } - LogPrintf("Experimental syscall sandbox enabled (-sandbox=%s): bitcoind will terminate if an unexpected (not allowlisted) syscall is invoked.\n", sandbox_arg); - } -#endif // USE_SYSCALL_SANDBOX - // Also report errors from parsing before daemonization { + KernelNotifications notifications{}; ChainstateManager::Options chainman_opts_dummy{ .chainparams = chainparams, .datadir = args.GetDataDirNet(), + .notifications = notifications, }; - if (const auto error{ApplyArgsManOptions(args, chainman_opts_dummy)}) { - return InitError(*error); + auto chainman_result{ApplyArgsManOptions(args, chainman_opts_dummy)}; + if (!chainman_result) { + return InitError(util::ErrorString(chainman_result)); } BlockManager::Options blockman_opts_dummy{ .chainparams = chainman_opts_dummy.chainparams, .blocks_dir = args.GetBlocksDirPath(), }; - if (const auto error{ApplyArgsManOptions(args, blockman_opts_dummy)}) { - return InitError(*error); + auto blockman_result{ApplyArgsManOptions(args, blockman_opts_dummy)}; + if (!blockman_result) { + return InitError(util::ErrorString(blockman_result)); } } @@ -1054,8 +1027,9 @@ static bool LockDataDirectory(bool probeOnly) bool AppInitSanityChecks(const kernel::Context& kernel) { // ********************************************************* Step 4: sanity checks - if (auto error = kernel::SanityChecks(kernel)) { - InitError(*error); + auto result{kernel::SanityChecks(kernel)}; + if (!result) { + InitError(util::ErrorString(result)); return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down."), PACKAGE_NAME)); } @@ -1230,9 +1204,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Initialize addrman assert(!node.addrman); uiInterface.InitMessage(_("Loading P2P addresses…").translated); - if (const auto error{LoadAddrman(*node.netgroupman, args, node.addrman)}) { - return InitError(*error); - } + auto addrman{LoadAddrman(*node.netgroupman, args)}; + if (!addrman) return InitError(util::ErrorString(addrman)); + node.addrman = std::move(*addrman); } assert(!node.banman); @@ -1245,7 +1219,17 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) assert(!node.fee_estimator); // Don't initialize fee estimation with old data if we don't relay transactions, // as they would never get updated. - if (!ignores_incoming_txs) node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(FeeestPath(args)); + if (!ignores_incoming_txs) { + bool read_stale_estimates = args.GetBoolArg("-acceptstalefeeestimates", DEFAULT_ACCEPT_STALE_FEE_ESTIMATES); + if (read_stale_estimates && (chainparams.GetChainType() != ChainType::REGTEST)) { + return InitError(strprintf(_("acceptstalefeeestimates is not supported on %s chain."), chainparams.GetChainTypeString())); + } + node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(FeeestPath(args), read_stale_estimates); + + // Flush estimates to disk periodically + CBlockPolicyEstimator* fee_estimator = node.fee_estimator.get(); + node.scheduler->scheduleEvery([fee_estimator] { fee_estimator->FlushFeeEstimates(); }, FEE_FLUSH_INTERVAL); + } // Check port numbers for (const std::string port_option : { @@ -1351,12 +1335,12 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // -noproxy (or -proxy=0) as well as the empty string can be used to not set a proxy, this is the default std::string proxyArg = args.GetArg("-proxy", ""); if (proxyArg != "" && proxyArg != "0") { - CService proxyAddr; - if (!Lookup(proxyArg, proxyAddr, 9050, fNameLookup)) { + const std::optional<CService> proxyAddr{Lookup(proxyArg, 9050, fNameLookup)}; + if (!proxyAddr.has_value()) { return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); } - Proxy addrProxy = Proxy(proxyAddr, proxyRandomize); + Proxy addrProxy = Proxy(proxyAddr.value(), proxyRandomize); if (!addrProxy.IsValid()) return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); @@ -1382,11 +1366,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) "reaching the Tor network is explicitly forbidden: -onion=0")); } } else { - CService addr; - if (!Lookup(onionArg, addr, 9050, fNameLookup) || !addr.IsValid()) { + const std::optional<CService> addr{Lookup(onionArg, 9050, fNameLookup)}; + if (!addr.has_value() || !addr->IsValid()) { return InitError(strprintf(_("Invalid -onion address or hostname: '%s'"), onionArg)); } - onion_proxy = Proxy{addr, proxyRandomize}; + onion_proxy = Proxy{addr.value(), proxyRandomize}; } } @@ -1406,9 +1390,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } for (const std::string& strAddr : args.GetArgs("-externalip")) { - CService addrLocal; - if (Lookup(strAddr, addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) - AddLocal(addrLocal, LOCAL_MANUAL); + const std::optional<CService> addrLocal{Lookup(strAddr, GetListenPort(), fNameLookup)}; + if (addrLocal.has_value() && addrLocal->IsValid()) + AddLocal(addrLocal.value(), LOCAL_MANUAL); else return InitError(ResolveErrMsg("externalip", strAddr)); } @@ -1427,20 +1411,22 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 7: load block chain + node.notifications = std::make_unique<KernelNotifications>(); fReindex = args.GetBoolArg("-reindex", false); bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = args.GetDataDirNet(), .adjusted_time_callback = GetAdjustedTime, + .notifications = *node.notifications, }; - Assert(!ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction + Assert(ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction BlockManager::Options blockman_opts{ .chainparams = chainman_opts.chainparams, .blocks_dir = args.GetBlocksDirPath(), }; - Assert(!ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction + Assert(ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction // cache size calculations CacheSizes cache_sizes = CalculateCacheSizes(args, g_enabled_filter_types.size()); @@ -1463,8 +1449,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) .estimator = node.fee_estimator.get(), .check_ratio = chainparams.DefaultConsistencyChecks() ? 1 : 0, }; - if (const auto err{ApplyArgsManOptions(args, chainparams, mempool_opts)}) { - return InitError(*err); + auto result{ApplyArgsManOptions(args, chainparams, mempool_opts)}; + if (!result) { + return InitError(util::ErrorString(result)); } mempool_opts.check_ratio = std::clamp<int>(mempool_opts.check_ratio, 0, 1'000'000); @@ -1519,7 +1506,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } } - if (status == node::ChainstateLoadStatus::FAILURE_INCOMPATIBLE_DB || status == node::ChainstateLoadStatus::FAILURE_INSUFFICIENT_DBCACHE) { + if (status == node::ChainstateLoadStatus::FAILURE_FATAL || status == node::ChainstateLoadStatus::FAILURE_INCOMPATIBLE_DB || status == node::ChainstateLoadStatus::FAILURE_INSUFFICIENT_DBCACHE) { return InitError(error); } @@ -1563,8 +1550,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // If reindex-chainstate was specified, delay syncing indexes until ThreadImport has reindexed the chain if (!fReindexChainState) g_indexes_ready_to_sync = true; if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - if (const auto error{WITH_LOCK(cs_main, return CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db)))}) { - return InitError(*error); + auto result{WITH_LOCK(cs_main, return CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db)))}; + if (!result) { + return InitError(util::ErrorString(result)); } g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, fReindex); @@ -1742,13 +1730,14 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) }; for (const std::string& bind_arg : args.GetArgs("-bind")) { - CService bind_addr; + std::optional<CService> bind_addr; const size_t index = bind_arg.rfind('='); if (index == std::string::npos) { - if (Lookup(bind_arg, bind_addr, default_bind_port, /*fAllowLookup=*/false)) { - connOptions.vBinds.push_back(bind_addr); - if (IsBadPort(bind_addr.GetPort())) { - InitWarning(BadPortWarning("-bind", bind_addr.GetPort())); + bind_addr = Lookup(bind_arg, default_bind_port, /*fAllowLookup=*/false); + if (bind_addr.has_value()) { + connOptions.vBinds.push_back(bind_addr.value()); + if (IsBadPort(bind_addr.value().GetPort())) { + InitWarning(BadPortWarning("-bind", bind_addr.value().GetPort())); } continue; } @@ -1756,8 +1745,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const std::string network_type = bind_arg.substr(index + 1); if (network_type == "onion") { const std::string truncated_bind_arg = bind_arg.substr(0, index); - if (Lookup(truncated_bind_arg, bind_addr, BaseParams().OnionServiceTargetPort(), false)) { - connOptions.onion_binds.push_back(bind_addr); + bind_addr = Lookup(truncated_bind_arg, BaseParams().OnionServiceTargetPort(), false); + if (bind_addr.has_value()) { + connOptions.onion_binds.push_back(bind_addr.value()); continue; } } @@ -1835,11 +1825,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const std::string& i2psam_arg = args.GetArg("-i2psam", ""); if (!i2psam_arg.empty()) { - CService addr; - if (!Lookup(i2psam_arg, addr, 7656, fNameLookup) || !addr.IsValid()) { + const std::optional<CService> addr{Lookup(i2psam_arg, 7656, fNameLookup)}; + if (!addr.has_value() || !addr->IsValid()) { return InitError(strprintf(_("Invalid -i2psam address or hostname: '%s'"), i2psam_arg)); } - SetProxy(NET_I2P, Proxy{addr}); + SetProxy(NET_I2P, Proxy{addr.value()}); } else { if (args.IsArgSet("-onlynet") && IsReachable(NET_I2P)) { return InitError( diff --git a/src/init.h b/src/init.h index 8c5b2e77d3..4dcf3fc51e 100644 --- a/src/init.h +++ b/src/init.h @@ -38,13 +38,13 @@ void InitParameterInteraction(ArgsManager& args); * @note This can be done before daemonization. Do not call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read. */ -bool AppInitBasicSetup(const ArgsManager& args); +bool AppInitBasicSetup(const ArgsManager& args, std::atomic<int>& exit_status); /** * Initialization: parameter interaction. * @note This can be done before daemonization. Do not call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read, AppInitBasicSetup should have been called. */ -bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandbox = true); +bool AppInitParameterInteraction(const ArgsManager& args); /** * Initialization sanity checks. * @note This can be done before daemonization. Do not call Shutdown() if this function fails. diff --git a/src/init/common.cpp b/src/init/common.cpp index 9a52a09cea..f3f7c696c5 100644 --- a/src/init/common.cpp +++ b/src/init/common.cpp @@ -13,6 +13,7 @@ #include <tinyformat.h> #include <util/fs.h> #include <util/fs_helpers.h> +#include <util/result.h> #include <util/string.h> #include <util/time.h> #include <util/translation.h> @@ -58,27 +59,28 @@ void SetLoggingOptions(const ArgsManager& args) fLogIPs = args.GetBoolArg("-logips", DEFAULT_LOGIPS); } -void SetLoggingLevel(const ArgsManager& args) +util::Result<void> SetLoggingLevel(const ArgsManager& args) { if (args.IsArgSet("-loglevel")) { for (const std::string& level_str : args.GetArgs("-loglevel")) { if (level_str.find_first_of(':', 3) == std::string::npos) { // user passed a global log level, i.e. -loglevel=<level> if (!LogInstance().SetLogLevel(level_str)) { - InitWarning(strprintf(_("Unsupported global logging level -loglevel=%s. Valid values: %s."), level_str, LogInstance().LogLevelsString())); + return util::Error{strprintf(_("Unsupported global logging level %s=%s. Valid values: %s."), "-loglevel", level_str, LogInstance().LogLevelsString())}; } } else { // user passed a category-specific log level, i.e. -loglevel=<category>:<level> const auto& toks = SplitString(level_str, ':'); if (!(toks.size() == 2 && LogInstance().SetCategoryLogLevel(toks[0], toks[1]))) { - InitWarning(strprintf(_("Unsupported category-specific logging level -loglevel=%s. Expected -loglevel=<category>:<loglevel>. Valid categories: %s. Valid loglevels: %s."), level_str, LogInstance().LogCategoriesString(), LogInstance().LogLevelsString())); + return util::Error{strprintf(_("Unsupported category-specific logging level %1$s=%2$s. Expected %1$s=<category>:<loglevel>. Valid categories: %3$s. Valid loglevels: %4$s."), "-loglevel", level_str, LogInstance().LogCategoriesString(), LogInstance().LogLevelsString())}; } } } } + return {}; } -void SetLoggingCategories(const ArgsManager& args) +util::Result<void> SetLoggingCategories(const ArgsManager& args) { if (args.IsArgSet("-debug")) { // Special-case: if -debug=0/-nodebug is set, turn off debugging messages @@ -88,7 +90,7 @@ void SetLoggingCategories(const ArgsManager& args) [](std::string cat){return cat == "0" || cat == "none";})) { for (const auto& cat : categories) { if (!LogInstance().EnableCategory(cat)) { - InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debug", cat)); + return util::Error{strprintf(_("Unsupported logging category %s=%s."), "-debug", cat)}; } } } @@ -97,9 +99,10 @@ void SetLoggingCategories(const ArgsManager& args) // Now remove the logging categories which were explicitly excluded for (const std::string& cat : args.GetArgs("-debugexclude")) { if (!LogInstance().DisableCategory(cat)) { - InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); + return util::Error{strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)}; } } + return {}; } bool StartLogging(const ArgsManager& args) diff --git a/src/init/common.h b/src/init/common.h index 44c3a502ee..b61a77c6d4 100644 --- a/src/init/common.h +++ b/src/init/common.h @@ -8,13 +8,15 @@ #ifndef BITCOIN_INIT_COMMON_H #define BITCOIN_INIT_COMMON_H +#include <util/result.h> + class ArgsManager; namespace init { void AddLoggingArgs(ArgsManager& args); void SetLoggingOptions(const ArgsManager& args); -void SetLoggingCategories(const ArgsManager& args); -void SetLoggingLevel(const ArgsManager& args); +[[nodiscard]] util::Result<void> SetLoggingCategories(const ArgsManager& args); +[[nodiscard]] util::Result<void> SetLoggingLevel(const ArgsManager& args); bool StartLogging(const ArgsManager& args); void LogPackageVersion(); } // namespace init diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index a3fa753a98..dd664165d3 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -6,8 +6,8 @@ #define BITCOIN_INTERFACES_CHAIN_H #include <blockfilter.h> +#include <common/settings.h> #include <primitives/transaction.h> // For CTransactionRef -#include <util/settings.h> // For util::SettingsValue #include <functional> #include <memory> @@ -87,6 +87,9 @@ struct BlockInfo { unsigned data_pos = 0; const CBlock* data = nullptr; const CBlockUndo* undo_data = nullptr; + // The maximum time in the chain up to and including this block. + // A timestamp that can only move forward. + unsigned int chain_time_max{0}; BlockInfo(const uint256& hash LIFETIMEBOUND) : hash(hash) {} }; @@ -297,17 +300,17 @@ public: virtual int rpcSerializationFlags() = 0; //! Get settings value. - virtual util::SettingsValue getSetting(const std::string& arg) = 0; + virtual common::SettingsValue getSetting(const std::string& arg) = 0; //! Get list of settings values. - virtual std::vector<util::SettingsValue> getSettingsList(const std::string& arg) = 0; + virtual std::vector<common::SettingsValue> getSettingsList(const std::string& arg) = 0; //! Return <datadir>/settings.json setting value. - virtual util::SettingsValue getRwSetting(const std::string& name) = 0; + virtual common::SettingsValue getRwSetting(const std::string& name) = 0; //! Write a setting to <datadir>/settings.json. Optionally just update the //! setting in memory and do not write the file. - virtual bool updateRwSetting(const std::string& name, const util::SettingsValue& value, bool write=true) = 0; + virtual bool updateRwSetting(const std::string& name, const common::SettingsValue& value, bool write=true) = 0; //! Synchronously send transactionAddedToMempool notifications about all //! current mempool transactions to the specified handler and return after diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 7e87d5a523..3f8df57124 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -5,13 +5,13 @@ #ifndef BITCOIN_INTERFACES_NODE_H #define BITCOIN_INTERFACES_NODE_H +#include <common/settings.h> #include <consensus/amount.h> // For CAmount #include <net.h> // For NodeId #include <net_types.h> // For banmap_t #include <netaddress.h> // For Network #include <netbase.h> // For ConnectionDirection #include <support/allocators/secure.h> // For SecureString -#include <util/settings.h> // For util::SettingsValue #include <util/translation.h> #include <functional> @@ -80,6 +80,9 @@ public: //! Get warnings. virtual bilingual_str getWarnings() = 0; + //! Get exit status. + virtual int getExitStatus() = 0; + // Get log flags. virtual uint32_t getLogCategories() = 0; @@ -103,14 +106,14 @@ public: virtual bool isSettingIgnored(const std::string& name) = 0; //! Return setting value from <datadir>/settings.json or bitcoin.conf. - virtual util::SettingsValue getPersistentSetting(const std::string& name) = 0; + virtual common::SettingsValue getPersistentSetting(const std::string& name) = 0; //! Update a setting in <datadir>/settings.json. - virtual void updateRwSetting(const std::string& name, const util::SettingsValue& value) = 0; + virtual void updateRwSetting(const std::string& name, const common::SettingsValue& value) = 0; //! Force a setting value to be applied, overriding any other configuration //! source, but not being persisted. - virtual void forceSetting(const std::string& name, const util::SettingsValue& value) = 0; + virtual void forceSetting(const std::string& name, const common::SettingsValue& value) = 0; //! Clear all settings in <datadir>/settings.json and store a backup of //! previous settings in <datadir>/settings.json.bak. diff --git a/src/ipc/interfaces.cpp b/src/ipc/interfaces.cpp index d804d9d291..e446cc98db 100644 --- a/src/ipc/interfaces.cpp +++ b/src/ipc/interfaces.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <interfaces/init.h> #include <interfaces/ipc.h> #include <ipc/capnp/protocol.h> @@ -10,7 +11,6 @@ #include <logging.h> #include <tinyformat.h> #include <util/fs.h> -#include <util/system.h> #include <cstdio> #include <cstdlib> diff --git a/src/kernel/chain.cpp b/src/kernel/chain.cpp index 82e77125d7..1c877866d0 100644 --- a/src/kernel/chain.cpp +++ b/src/kernel/chain.cpp @@ -16,6 +16,7 @@ interfaces::BlockInfo MakeBlockInfo(const CBlockIndex* index, const CBlock* data if (index) { info.prev_hash = index->pprev ? index->pprev->phashBlock : nullptr; info.height = index->nHeight; + info.chain_time_max = index->GetBlockTimeMax(); LOCK(::cs_main); info.file_number = index->nFile; info.data_pos = index->nDataPos; diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index 2395f60164..035a913d10 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_KERNEL_CHAINSTATEMANAGER_OPTS_H #define BITCOIN_KERNEL_CHAINSTATEMANAGER_OPTS_H +#include <kernel/notifications_interface.h> + #include <arith_uint256.h> #include <dbwrapper.h> #include <txdb.h> @@ -19,6 +21,7 @@ class CChainParams; static constexpr bool DEFAULT_CHECKPOINTS_ENABLED{true}; static constexpr auto DEFAULT_MAX_TIP_AGE{24h}; +static constexpr int DEFAULT_STOPATHEIGHT{0}; namespace kernel { @@ -42,6 +45,8 @@ struct ChainstateManagerOpts { DBOptions block_tree_db{}; DBOptions coins_db{}; CoinsViewOptions coins_view{}; + Notifications& notifications; + int stop_at_height{DEFAULT_STOPATHEIGHT}; }; } // namespace kernel diff --git a/src/kernel/checks.cpp b/src/kernel/checks.cpp index 4c303c172c..bf8a2ec74c 100644 --- a/src/kernel/checks.cpp +++ b/src/kernel/checks.cpp @@ -13,21 +13,21 @@ namespace kernel { -std::optional<bilingual_str> SanityChecks(const Context&) +util::Result<void> SanityChecks(const Context&) { if (!ECC_InitSanityCheck()) { - return Untranslated("Elliptic curve cryptography sanity check failure. Aborting."); + return util::Error{Untranslated("Elliptic curve cryptography sanity check failure. Aborting.")}; } if (!Random_SanityCheck()) { - return Untranslated("OS cryptographic RNG sanity check failure. Aborting."); + return util::Error{Untranslated("OS cryptographic RNG sanity check failure. Aborting.")}; } if (!ChronoSanityCheck()) { - return Untranslated("Clock epoch mismatch. Aborting."); + return util::Error{Untranslated("Clock epoch mismatch. Aborting.")}; } - return std::nullopt; + return {}; } } diff --git a/src/kernel/checks.h b/src/kernel/checks.h index 3eb14824fb..fd8c167015 100644 --- a/src/kernel/checks.h +++ b/src/kernel/checks.h @@ -5,9 +5,7 @@ #ifndef BITCOIN_KERNEL_CHECKS_H #define BITCOIN_KERNEL_CHECKS_H -#include <optional> - -struct bilingual_str; +#include <util/result.h> namespace kernel { @@ -16,8 +14,7 @@ struct Context; /** * Ensure a usable environment with all necessary library support. */ -std::optional<bilingual_str> SanityChecks(const Context&); - -} +[[nodiscard]] util::Result<void> SanityChecks(const Context&); +} // namespace kernel #endif // BITCOIN_KERNEL_CHECKS_H diff --git a/src/kernel/mempool_entry.h b/src/kernel/mempool_entry.h index e1ba4296ef..886e1e1b3a 100644 --- a/src/kernel/mempool_entry.h +++ b/src/kernel/mempool_entry.h @@ -57,7 +57,7 @@ struct CompareIteratorByHash { * ("descendant" transactions). * * When a new entry is added to the mempool, we update the descendant state - * (nCountWithDescendants, nSizeWithDescendants, and nModFeesWithDescendants) for + * (m_count_with_descendants, nSizeWithDescendants, and nModFeesWithDescendants) for * all ancestors of the newly added transaction. * */ @@ -75,7 +75,7 @@ private: mutable Parents m_parents; mutable Children m_children; const CAmount nFee; //!< Cached to avoid expensive parent-transaction lookups - const size_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) + const int32_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) const size_t nUsageSize; //!< ... and total memory usage const int64_t nTime; //!< Local time when entering the mempool const unsigned int entryHeight; //!< Chain height when entering the mempool @@ -87,13 +87,15 @@ private: // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these // descendants as well. - uint64_t nCountWithDescendants{1}; //!< number of descendant transactions - uint64_t nSizeWithDescendants; //!< ... and size + int64_t m_count_with_descendants{1}; //!< number of descendant transactions + // Using int64_t instead of int32_t to avoid signed integer overflow issues. + int64_t nSizeWithDescendants; //!< ... and size CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) // Analogous statistics for ancestor transactions - uint64_t nCountWithAncestors{1}; - uint64_t nSizeWithAncestors; + int64_t m_count_with_ancestors{1}; + // Using int64_t instead of int32_t to avoid signed integer overflow issues. + int64_t nSizeWithAncestors; CAmount nModFeesWithAncestors; int64_t nSigOpCostWithAncestors; @@ -104,7 +106,7 @@ public: int64_t sigops_cost, LockPoints lp) : tx{tx}, nFee{fee}, - nTxWeight(GetTransactionWeight(*tx)), + nTxWeight{GetTransactionWeight(*tx)}, nUsageSize{RecursiveDynamicUsage(tx)}, nTime{time}, entryHeight{entry_height}, @@ -121,11 +123,11 @@ public: const CTransaction& GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } const CAmount& GetFee() const { return nFee; } - size_t GetTxSize() const + int32_t GetTxSize() const { return GetVirtualTransactionSize(nTxWeight, sigOpCost, ::nBytesPerSigOp); } - size_t GetTxWeight() const { return nTxWeight; } + int32_t GetTxWeight() const { return nTxWeight; } std::chrono::seconds GetTime() const { return std::chrono::seconds{nTime}; } unsigned int GetHeight() const { return entryHeight; } int64_t GetSigOpCost() const { return sigOpCost; } @@ -134,9 +136,9 @@ public: const LockPoints& GetLockPoints() const { return lockPoints; } // Adjusts the descendant state. - void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); + void UpdateDescendantState(int32_t modifySize, CAmount modifyFee, int64_t modifyCount); // Adjusts the ancestor state - void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps); + void UpdateAncestorState(int32_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps); // Updates the modified fees with descendants/ancestors. void UpdateModifiedFee(CAmount fee_diff) { @@ -151,14 +153,14 @@ public: lockPoints = lp; } - uint64_t GetCountWithDescendants() const { return nCountWithDescendants; } - uint64_t GetSizeWithDescendants() const { return nSizeWithDescendants; } + uint64_t GetCountWithDescendants() const { return m_count_with_descendants; } + int64_t GetSizeWithDescendants() const { return nSizeWithDescendants; } CAmount GetModFeesWithDescendants() const { return nModFeesWithDescendants; } bool GetSpendsCoinbase() const { return spendsCoinbase; } - uint64_t GetCountWithAncestors() const { return nCountWithAncestors; } - uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } + uint64_t GetCountWithAncestors() const { return m_count_with_ancestors; } + int64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } CAmount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } int64_t GetSigOpCostWithAncestors() const { return nSigOpCostWithAncestors; } diff --git a/src/kernel/notifications_interface.h b/src/kernel/notifications_interface.h new file mode 100644 index 0000000000..48248e9aa0 --- /dev/null +++ b/src/kernel/notifications_interface.h @@ -0,0 +1,33 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_NOTIFICATIONS_INTERFACE_H +#define BITCOIN_KERNEL_NOTIFICATIONS_INTERFACE_H + +#include <cstdint> +#include <string> + +class CBlockIndex; +enum class SynchronizationState; +struct bilingual_str; + +namespace kernel { + +/** + * A base class defining functions for notifying about certain kernel + * events. + */ +class Notifications +{ +public: + virtual ~Notifications(){}; + + virtual void blockTip(SynchronizationState state, CBlockIndex& index) {} + virtual void headerTip(SynchronizationState state, int64_t height, int64_t timestamp, bool presync) {} + virtual void progress(const bilingual_str& title, int progress_percent, bool resume_possible) {} + virtual void warning(const bilingual_str& warning) {} +}; +} // namespace kernel + +#endif // BITCOIN_KERNEL_NOTIFICATIONS_INTERFACE_H diff --git a/src/key.cpp b/src/key.cpp index 3a3f0b2bc2..efaea5b1b3 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -11,6 +11,7 @@ #include <random.h> #include <secp256k1.h> +#include <secp256k1_ellswift.h> #include <secp256k1_extrakeys.h> #include <secp256k1_recovery.h> #include <secp256k1_schnorrsig.h> @@ -331,6 +332,42 @@ bool CKey::Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const return ret; } +EllSwiftPubKey CKey::EllSwiftCreate(Span<const std::byte> ent32) const +{ + assert(fValid); + assert(ent32.size() == 32); + std::array<std::byte, EllSwiftPubKey::size()> encoded_pubkey; + + auto success = secp256k1_ellswift_create(secp256k1_context_sign, + UCharCast(encoded_pubkey.data()), + keydata.data(), + UCharCast(ent32.data())); + + // Should always succeed for valid keys (asserted above). + assert(success); + return {encoded_pubkey}; +} + +ECDHSecret CKey::ComputeBIP324ECDHSecret(const EllSwiftPubKey& their_ellswift, const EllSwiftPubKey& our_ellswift, bool initiating) const +{ + assert(fValid); + + ECDHSecret output; + // BIP324 uses the initiator as party A, and the responder as party B. Remap the inputs + // accordingly: + bool success = secp256k1_ellswift_xdh(secp256k1_context_sign, + UCharCast(output.data()), + UCharCast(initiating ? our_ellswift.data() : their_ellswift.data()), + UCharCast(initiating ? their_ellswift.data() : our_ellswift.data()), + keydata.data(), + initiating ? 0 : 1, + secp256k1_ellswift_xdh_hash_function_bip324, + nullptr); + // Should always succeed for valid keys (assert above). + assert(success); + return output; +} + bool CExtKey::Derive(CExtKey &out, unsigned int _nChild) const { if (nDepth == std::numeric_limits<unsigned char>::max()) return false; out.nDepth = nDepth + 1; @@ -22,6 +22,12 @@ */ typedef std::vector<unsigned char, secure_allocator<unsigned char> > CPrivKey; +/** Size of ECDH shared secrets. */ +constexpr static size_t ECDH_SECRET_SIZE = CSHA256::OUTPUT_SIZE; + +// Used to represent ECDH shared secret (ECDH_SECRET_SIZE bytes) +using ECDHSecret = std::array<std::byte, ECDH_SECRET_SIZE>; + /** An encapsulated private key. */ class CKey { @@ -156,6 +162,27 @@ public: //! Load private key and check that public key matches. bool Load(const CPrivKey& privkey, const CPubKey& vchPubKey, bool fSkipCheck); + + /** Create an ellswift-encoded public key for this key, with specified entropy. + * + * entropy must be a 32-byte span with additional entropy to use in the encoding. Every + * public key has ~2^256 different encodings, and this function will deterministically pick + * one of them, based on entropy. Note that even without truly random entropy, the + * resulting encoding will be indistinguishable from uniform to any adversary who does not + * know the private key (because the private key itself is always used as entropy as well). + */ + EllSwiftPubKey EllSwiftCreate(Span<const std::byte> entropy) const; + + /** Compute a BIP324-style ECDH shared secret. + * + * - their_ellswift: EllSwiftPubKey that was received from the other side. + * - our_ellswift: EllSwiftPubKey that was sent to the other side (must have been generated + * from *this using EllSwiftCreate()). + * - initiating: whether we are the initiating party (true) or responding party (false). + */ + ECDHSecret ComputeBIP324ECDHSecret(const EllSwiftPubKey& their_ellswift, + const EllSwiftPubKey& our_ellswift, + bool initiating) const; }; struct CExtKey { diff --git a/src/key_io.cpp b/src/key_io.cpp index 33499b0d23..454a96df5e 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -146,6 +146,9 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par // The rest of the symbols are converted witness program bytes. data.reserve(((dec.data.size() - 1) * 5) / 8); if (ConvertBits<5, 8, false>([&](unsigned char c) { data.push_back(c); }, dec.data.begin() + 1, dec.data.end())) { + + std::string_view byte_str{data.size() == 1 ? "byte" : "bytes"}; + if (version == 0) { { WitnessV0KeyHash keyid; @@ -162,7 +165,7 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par } } - error_str = strprintf("Invalid Bech32 v0 address program size (%s byte), per BIP141", data.size()); + error_str = strprintf("Invalid Bech32 v0 address program size (%d %s), per BIP141", data.size(), byte_str); return CNoDestination(); } @@ -179,7 +182,7 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par } if (data.size() < 2 || data.size() > BECH32_WITNESS_PROG_MAX_LEN) { - error_str = strprintf("Invalid Bech32 address program size (%s byte)", data.size()); + error_str = strprintf("Invalid Bech32 address program size (%d %s)", data.size(), byte_str); return CNoDestination(); } diff --git a/src/logging.cpp b/src/logging.cpp index 7725fefeb7..a5cfb0d28e 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -179,7 +179,7 @@ const CLogCategoryDesc LogCategories[] = {BCLog::LOCK, "lock"}, #endif {BCLog::UTIL, "util"}, - {BCLog::BLOCKSTORE, "blockstorage"}, + {BCLog::BLOCKSTORAGE, "blockstorage"}, {BCLog::TXRECONCILIATION, "txreconciliation"}, {BCLog::SCAN, "scan"}, {BCLog::ALL, "1"}, @@ -280,7 +280,7 @@ std::string LogCategoryToStr(BCLog::LogFlags category) #endif case BCLog::LogFlags::UTIL: return "util"; - case BCLog::LogFlags::BLOCKSTORE: + case BCLog::LogFlags::BLOCKSTORAGE: return "blockstorage"; case BCLog::LogFlags::TXRECONCILIATION: return "txreconciliation"; diff --git a/src/logging.h b/src/logging.h index e7c554e79f..fc03c8eac3 100644 --- a/src/logging.h +++ b/src/logging.h @@ -65,7 +65,7 @@ namespace BCLog { LOCK = (1 << 24), #endif UTIL = (1 << 25), - BLOCKSTORE = (1 << 26), + BLOCKSTORAGE = (1 << 26), TXRECONCILIATION = (1 << 27), SCAN = (1 << 28), ALL = ~(uint32_t)0, diff --git a/src/mapport.cpp b/src/mapport.cpp index 994fd12cf5..08b365db4b 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -9,12 +9,11 @@ #include <mapport.h> #include <clientversion.h> +#include <common/system.h> #include <logging.h> #include <net.h> #include <netaddress.h> #include <netbase.h> -#include <util/syscall_sandbox.h> -#include <util/system.h> #include <util/thread.h> #include <util/threadinterrupt.h> @@ -175,10 +174,10 @@ static bool ProcessUpnp() LogPrintf("UPnP: GetExternalIPAddress() returned %d\n", r); } else { if (externalIPAddress[0]) { - CNetAddr resolved; - if (LookupHost(externalIPAddress, resolved, false)) { - LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved.ToStringAddr()); - AddLocal(resolved, LOCAL_MAPPED); + std::optional<CNetAddr> resolved{LookupHost(externalIPAddress, false)}; + if (resolved.has_value()) { + LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved->ToStringAddr()); + AddLocal(resolved.value(), LOCAL_MAPPED); } } else { LogPrintf("UPnP: GetExternalIPAddress failed.\n"); @@ -219,7 +218,6 @@ static bool ProcessUpnp() static void ThreadMapPort() { - SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_MAP_PORT); bool ok; do { ok = false; diff --git a/src/net.cpp b/src/net.cpp index 337cf60680..7601a6ea84 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -30,7 +30,6 @@ #include <util/fs.h> #include <util/sock.h> #include <util/strencodings.h> -#include <util/syscall_sandbox.h> #include <util/thread.h> #include <util/threadinterrupt.h> #include <util/trace.h> @@ -38,18 +37,12 @@ #ifdef WIN32 #include <string.h> -#else -#include <fcntl.h> #endif #if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS #include <ifaddrs.h> #endif -#ifdef USE_POLL -#include <poll.h> -#endif - #include <algorithm> #include <array> #include <cstdint> @@ -130,14 +123,10 @@ uint16_t GetListenPort() { // If -bind= is provided with ":port" part, use that (first one if multiple are provided). for (const std::string& bind_arg : gArgs.GetArgs("-bind")) { - CService bind_addr; constexpr uint16_t dummy_port = 0; - if (Lookup(bind_arg, bind_addr, dummy_port, /*fAllowLookup=*/false)) { - if (bind_addr.GetPort() != dummy_port) { - return bind_addr.GetPort(); - } - } + const std::optional<CService> bind_addr{Lookup(bind_arg, dummy_port, /*fAllowLookup=*/false)}; + if (bind_addr.has_value() && bind_addr->GetPort() != dummy_port) return bind_addr->GetPort(); } // Otherwise, if -whitebind= without NetPermissionFlags::NoBan is provided, use that @@ -461,9 +450,9 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo const uint16_t default_port{pszDest != nullptr ? Params().GetDefaultPort(pszDest) : Params().GetDefaultPort()}; if (pszDest) { - std::vector<CService> resolved; - if (Lookup(pszDest, resolved, default_port, fNameLookup && !HaveNameProxy(), 256) && !resolved.empty()) { - const CService rnd{resolved[GetRand(resolved.size())]}; + const std::vector<CService> resolved{Lookup(pszDest, default_port, fNameLookup && !HaveNameProxy(), 256)}; + if (!resolved.empty()) { + const CService& rnd{resolved[GetRand(resolved.size())]}; addrConnect = CAddress{MaybeFlipIPv6toCJDNS(rnd), NODE_NONE}; if (!addrConnect.IsValid()) { LogPrint(BCLog::NET, "Resolver returned invalid address %s for %s\n", addrConnect.ToStringAddrPort(), pszDest); @@ -1385,7 +1374,6 @@ void CConnman::ThreadSocketHandler() { AssertLockNotHeld(m_total_bytes_sent_mutex); - SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET); while (!interruptNet) { DisconnectNodes(); @@ -1405,7 +1393,6 @@ void CConnman::WakeMessageHandler() void CConnman::ThreadDNSAddressSeed() { - SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_DNS_SEED); FastRandomContext rng; std::vector<std::string> seeds = Params().DNSSeeds(); Shuffle(seeds.begin(), seeds.end(), rng); @@ -1487,7 +1474,6 @@ void CConnman::ThreadDNSAddressSeed() if (HaveNameProxy()) { AddAddrFetch(seed); } else { - std::vector<CNetAddr> vIPs; std::vector<CAddress> vAdd; ServiceFlags requiredServiceBits = GetDesirableServiceFlags(NODE_NONE); std::string host = strprintf("x%x.%s", requiredServiceBits, seed); @@ -1496,8 +1482,9 @@ void CConnman::ThreadDNSAddressSeed() continue; } unsigned int nMaxIPs = 256; // Limits number of IPs learned from a DNS seed - if (LookupHost(host, vIPs, nMaxIPs, true)) { - for (const CNetAddr& ip : vIPs) { + const auto addresses{LookupHost(host, nMaxIPs, true)}; + if (!addresses.empty()) { + for (const CNetAddr& ip : addresses) { CAddress addr = CAddress(CService(ip, Params().GetDefaultPort()), requiredServiceBits); addr.nTime = rng.rand_uniform_delay(Now<NodeSeconds>() - 3 * 24h, -4 * 24h); // use a random age between 3 and 7 days old vAdd.push_back(addr); @@ -1611,7 +1598,6 @@ std::unordered_set<Network> CConnman::GetReachableEmptyNetworks() const void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) { AssertLockNotHeld(m_unused_i2p_sessions_mutex); - SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_OPEN_CONNECTION); FastRandomContext rng; // Connect to specific addresses if (!connect.empty()) @@ -1641,6 +1627,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) auto next_extra_block_relay = GetExponentialRand(start, EXTRA_BLOCK_RELAY_ONLY_PEER_INTERVAL); const bool dnsseed = gArgs.GetBoolArg("-dnsseed", DEFAULT_DNSSEED); bool add_fixed_seeds = gArgs.GetBoolArg("-fixedseeds", DEFAULT_FIXEDSEEDS); + const bool use_seednodes{gArgs.IsArgSet("-seednode")}; if (!add_fixed_seeds) { LogPrintf("Fixed seeds are disabled\n"); @@ -1670,12 +1657,12 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) LogPrintf("Adding fixed seeds as 60 seconds have passed and addrman is empty for at least one reachable network\n"); } - // Checking !dnsseed is cheaper before locking 2 mutexes. - if (!add_fixed_seeds_now && !dnsseed) { - LOCK2(m_addr_fetches_mutex, m_added_nodes_mutex); - if (m_addr_fetches.empty() && m_added_nodes.empty()) { + // Perform cheap checks before locking a mutex. + else if (!dnsseed && !use_seednodes) { + LOCK(m_added_nodes_mutex); + if (m_added_nodes.empty()) { add_fixed_seeds_now = true; - LogPrintf("Adding fixed seeds as -dnsseed=0 (or IPv4/IPv6 connections are disabled via -onlynet), -addnode is not provided and all -seednode(s) attempted\n"); + LogPrintf("Adding fixed seeds as -dnsseed=0 (or IPv4/IPv6 connections are disabled via -onlynet) and neither -addnode nor -seednode are provided\n"); } } @@ -1707,7 +1694,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) int nOutboundFullRelay = 0; int nOutboundBlockRelay = 0; int outbound_privacy_network_peers = 0; - std::set<std::vector<unsigned char>> setConnected; // netgroups of our ipv4/ipv6 outbound peers + std::set<std::vector<unsigned char>> outbound_ipv46_peer_netgroups; { LOCK(m_nodes_mutex); @@ -1729,7 +1716,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) case ConnectionType::MANUAL: case ConnectionType::OUTBOUND_FULL_RELAY: case ConnectionType::BLOCK_RELAY: - CAddress address{pnode->addr}; + const CAddress address{pnode->addr}; if (address.IsTor() || address.IsI2P() || address.IsCJDNS()) { // Since our addrman-groups for these networks are // random, without relation to the route we @@ -1740,7 +1727,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // these networks. ++outbound_privacy_network_peers; } else { - setConnected.insert(m_netgroupman.GetGroup(address)); + outbound_ipv46_peer_netgroups.insert(m_netgroupman.GetGroup(address)); } } // no default case, so the compiler can warn about missing cases } @@ -1815,7 +1802,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) m_anchors.pop_back(); if (!addr.IsValid() || IsLocal(addr) || !IsReachable(addr) || !HasAllDesirableServiceFlags(addr.nServices) || - setConnected.count(m_netgroupman.GetGroup(addr))) continue; + outbound_ipv46_peer_netgroups.count(m_netgroupman.GetGroup(addr))) continue; addrConnect = addr; LogPrint(BCLog::NET, "Trying to make an anchor connection to %s\n", addrConnect.ToStringAddrPort()); break; @@ -1855,9 +1842,9 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) std::tie(addr, addr_last_try) = addrman.Select(); } - // Require outbound connections, other than feelers, to be to distinct network groups - if (!fFeeler && setConnected.count(m_netgroupman.GetGroup(addr))) { - break; + // Require outbound IPv4/IPv6 connections, other than feelers, to be to distinct network groups + if (!fFeeler && outbound_ipv46_peer_netgroups.count(m_netgroupman.GetGroup(addr))) { + continue; } // if we selected an invalid or local address, restart @@ -1902,8 +1889,9 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // Record addrman failure attempts when node has at least 2 persistent outbound connections to peers with // different netgroups in ipv4/ipv6 networks + all peers in Tor/I2P/CJDNS networks. // Don't record addrman failure attempts when node is offline. This can be identified since all local - // network connections(if any) belong in the same netgroup and size of setConnected would only be 1. - OpenNetworkConnection(addrConnect, (int)setConnected.size() + outbound_privacy_network_peers >= std::min(nMaxConnections - 1, 2), &grant, nullptr, conn_type); + // network connections (if any) belong in the same netgroup, and the size of `outbound_ipv46_peer_netgroups` would only be 1. + const bool count_failures{((int)outbound_ipv46_peer_netgroups.size() + outbound_privacy_network_peers) >= std::min(nMaxConnections - 1, 2)}; + OpenNetworkConnection(addrConnect, count_failures, &grant, /*strDest=*/nullptr, conn_type); } } } @@ -1978,7 +1966,6 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const void CConnman::ThreadOpenAddedConnections() { AssertLockNotHeld(m_unused_i2p_sessions_mutex); - SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_ADD_CONNECTION); while (true) { CSemaphoreGrant grant(*semAddnode); @@ -2047,7 +2034,6 @@ void CConnman::ThreadMessageHandler() { LOCK(NetEventsInterface::g_msgproc_mutex); - SetSyscallSandboxPolicy(SyscallSandboxPolicy::MESSAGE_HANDLER); while (!flagInterruptMsgProc) { bool fMoreWork = false; @@ -2201,14 +2187,11 @@ void Discover() char pszHostName[256] = ""; if (gethostname(pszHostName, sizeof(pszHostName)) != SOCKET_ERROR) { - std::vector<CNetAddr> vaddr; - if (LookupHost(pszHostName, vaddr, 0, true)) + const std::vector<CNetAddr> addresses{LookupHost(pszHostName, 0, true)}; + for (const CNetAddr& addr : addresses) { - for (const CNetAddr &addr : vaddr) - { - if (AddLocal(addr, LOCAL_IF)) - LogPrintf("%s: %s - %s\n", __func__, pszHostName, addr.ToStringAddr()); - } + if (AddLocal(addr, LOCAL_IF)) + LogPrintf("%s: %s - %s\n", __func__, pszHostName, addr.ToStringAddr()); } } #elif (HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS) @@ -2944,13 +2927,13 @@ void CaptureMessageToFile(const CAddress& addr, AutoFile f{fsbridge::fopen(path, "ab")}; ser_writedata64(f, now.count()); - f.write(MakeByteSpan(msg_type)); + f << Span{msg_type}; for (auto i = msg_type.length(); i < CMessageHeader::COMMAND_SIZE; ++i) { f << uint8_t{'\0'}; } uint32_t size = data.size(); ser_writedata32(f, size); - f.write(AsBytes(data)); + f << data; } std::function<void(const CAddress& addr, @@ -1219,7 +1219,6 @@ private: std::vector<CNode*> m_nodes_copy; }; - friend struct CConnmanTest; friend struct ConnmanTestMsg; }; diff --git a/src/net_permissions.cpp b/src/net_permissions.cpp index f829e56aa2..23226bbb4f 100644 --- a/src/net_permissions.cpp +++ b/src/net_permissions.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <net_permissions.h> #include <netbase.h> #include <util/error.h> -#include <util/system.h> #include <util/translation.h> const std::vector<std::string> NET_PERMISSIONS_DOC{ @@ -88,18 +88,18 @@ bool NetWhitebindPermissions::TryParse(const std::string& str, NetWhitebindPermi if (!TryParsePermissionFlags(str, flags, offset, error)) return false; const std::string strBind = str.substr(offset); - CService addrBind; - if (!Lookup(strBind, addrBind, 0, false)) { + const std::optional<CService> addrBind{Lookup(strBind, 0, false)}; + if (!addrBind.has_value()) { error = ResolveErrMsg("whitebind", strBind); return false; } - if (addrBind.GetPort() == 0) { + if (addrBind.value().GetPort() == 0) { error = strprintf(_("Need to specify a port with -whitebind: '%s'"), strBind); return false; } output.m_flags = flags; - output.m_service = addrBind; + output.m_service = addrBind.value(); error = Untranslated(""); return true; } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index f08e771f63..8da2c701d3 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -51,9 +51,7 @@ #include <optional> #include <typeinfo> -/** How long to cache transactions in mapRelay for normal relay */ -static constexpr auto RELAY_TX_CACHE_TIME = 15min; -/** How long a transaction has to be in the mempool before it can unconditionally be relayed (even when not in mapRelay). */ +/** How long a transaction has to be in the mempool before it can unconditionally be relayed. */ static constexpr auto UNCONDITIONAL_RELAY_DELAY = 2min; /** Headers download timeout. * Timeout = base + per_header * (expected number of headers) */ @@ -570,7 +568,7 @@ private: * * @return Returns true if the peer was punished (probably disconnected) */ - bool MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message = "") + bool MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); /** Maybe disconnect a peer and discourage future connections from its address. @@ -851,6 +849,7 @@ private: std::shared_ptr<const CBlock> m_most_recent_block GUARDED_BY(m_most_recent_block_mutex); std::shared_ptr<const CBlockHeaderAndShortTxIDs> m_most_recent_compact_block GUARDED_BY(m_most_recent_block_mutex); uint256 m_most_recent_block_hash GUARDED_BY(m_most_recent_block_mutex); + std::unique_ptr<const std::map<uint256, CTransactionRef>> m_most_recent_block_txs GUARDED_BY(m_most_recent_block_mutex); // Data about the low-work headers synchronization, aggregated from all peers' HeadersSyncStates. /** Mutex guarding the other m_headers_presync_* variables. */ @@ -910,7 +909,7 @@ private: /** Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). */ CTransactionRef FindTxForGetData(const Peer::TxRelay& tx_relay, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) - EXCLUSIVE_LOCKS_REQUIRED(NetEventsInterface::g_msgproc_mutex); + EXCLUSIVE_LOCKS_REQUIRED(!m_most_recent_block_mutex, NetEventsInterface::g_msgproc_mutex); void ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic<bool>& interruptMsgProc) EXCLUSIVE_LOCKS_REQUIRED(!m_most_recent_block_mutex, peer.m_getdata_requests_mutex, NetEventsInterface::g_msgproc_mutex) @@ -919,11 +918,9 @@ private: /** Process a new block. Perform any post-processing housekeeping */ void ProcessBlock(CNode& node, const std::shared_ptr<const CBlock>& block, bool force_processing, bool min_pow_checked); - /** Relay map (txid or wtxid -> CTransactionRef) */ - typedef std::map<uint256, CTransactionRef> MapRelay; - MapRelay mapRelay GUARDED_BY(NetEventsInterface::g_msgproc_mutex); - /** Expiration-time ordered list of (expire time, relay map entry) pairs. */ - std::deque<std::pair<std::chrono::microseconds, MapRelay::iterator>> g_relay_expiration GUARDED_BY(NetEventsInterface::g_msgproc_mutex); + /** Process compact block txns */ + void ProcessCompactBlockTxns(CNode& pfrom, Peer& peer, const BlockTransactions& block_transactions) + EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex, !m_most_recent_block_mutex); /** * When a peer sends us a valid block, instruct it to announce blocks to us @@ -1738,7 +1735,7 @@ bool PeerManagerImpl::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidati return false; } -bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message) +bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state) { PeerRef peer{GetPeerRef(nodeid)}; switch (state.GetResult()) { @@ -1746,7 +1743,7 @@ bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationStat break; // The node is providing invalid data: case TxValidationResult::TX_CONSENSUS: - if (peer) Misbehaving(*peer, 100, message); + if (peer) Misbehaving(*peer, 100, ""); return true; // Conflicting (but not necessarily invalid) data or different policy: case TxValidationResult::TX_RECENT_CONSENSUS_CHANGE: @@ -1761,9 +1758,6 @@ bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationStat case TxValidationResult::TX_NO_MEMPOOL: break; } - if (message != "") { - LogPrint(BCLog::NET, "peer=%d: %s\n", nodeid, message); - } return false; } @@ -1927,10 +1921,17 @@ void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::sha std::async(std::launch::deferred, [&] { return msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock); })}; { + auto most_recent_block_txs = std::make_unique<std::map<uint256, CTransactionRef>>(); + for (const auto& tx : pblock->vtx) { + most_recent_block_txs->emplace(tx->GetHash(), tx); + most_recent_block_txs->emplace(tx->GetWitnessHash(), tx); + } + LOCK(m_most_recent_block_mutex); m_most_recent_block_hash = hashBlock; m_most_recent_block = pblock; m_most_recent_compact_block = pcmpctblock; + m_most_recent_block_txs = std::move(most_recent_block_txs); } m_connman.ForEachNode([this, pindex, &lazy_ser, &hashBlock](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { @@ -2301,13 +2302,17 @@ CTransactionRef PeerManagerImpl::FindTxForGetData(const Peer::TxRelay& tx_relay, } } - // Otherwise, the transaction must have been announced recently. - if (tx_relay.m_recently_announced_invs.contains(gtxid.GetHash())) { - // If it was, it can be relayed from either the mempool... - if (txinfo.tx) return std::move(txinfo.tx); - // ... or the relay pool. - auto mi = mapRelay.find(gtxid.GetHash()); - if (mi != mapRelay.end()) return mi->second; + // Otherwise, the transaction might have been announced recently. + bool recent = tx_relay.m_recently_announced_invs.contains(gtxid.GetHash()); + if (recent && txinfo.tx) return std::move(txinfo.tx); + + // Or it might be from the most recent block + { + LOCK(m_most_recent_block_mutex); + if (m_most_recent_block_txs != nullptr) { + auto it = m_most_recent_block_txs->find(gtxid.GetHash()); + if (it != m_most_recent_block_txs->end()) return it->second; + } } return {}; @@ -3200,6 +3205,93 @@ void PeerManagerImpl::ProcessBlock(CNode& node, const std::shared_ptr<const CBlo } } +void PeerManagerImpl::ProcessCompactBlockTxns(CNode& pfrom, Peer& peer, const BlockTransactions& block_transactions) +{ + std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); + bool fBlockRead{false}; + const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); + { + LOCK(cs_main); + + auto range_flight = mapBlocksInFlight.equal_range(block_transactions.blockhash); + size_t already_in_flight = std::distance(range_flight.first, range_flight.second); + bool requested_block_from_this_peer{false}; + + // Multimap ensures ordering of outstanding requests. It's either empty or first in line. + bool first_in_flight = already_in_flight == 0 || (range_flight.first->second.first == pfrom.GetId()); + + while (range_flight.first != range_flight.second) { + auto [node_id, block_it] = range_flight.first->second; + if (node_id == pfrom.GetId() && block_it->partialBlock) { + requested_block_from_this_peer = true; + break; + } + range_flight.first++; + } + + if (!requested_block_from_this_peer) { + LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom.GetId()); + return; + } + + PartiallyDownloadedBlock& partialBlock = *range_flight.first->second.second->partialBlock; + ReadStatus status = partialBlock.FillBlock(*pblock, block_transactions.txn); + if (status == READ_STATUS_INVALID) { + RemoveBlockRequest(block_transactions.blockhash, pfrom.GetId()); // Reset in-flight state in case Misbehaving does not result in a disconnect + Misbehaving(peer, 100, "invalid compact block/non-matching block transactions"); + return; + } else if (status == READ_STATUS_FAILED) { + if (first_in_flight) { + // Might have collided, fall back to getdata now :( + std::vector<CInv> invs; + invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(peer), block_transactions.blockhash)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + } else { + RemoveBlockRequest(block_transactions.blockhash, pfrom.GetId()); + LogPrint(BCLog::NET, "Peer %d sent us a compact block but it failed to reconstruct, waiting on first download to complete\n", pfrom.GetId()); + return; + } + } else { + // Block is either okay, or possibly we received + // READ_STATUS_CHECKBLOCK_FAILED. + // Note that CheckBlock can only fail for one of a few reasons: + // 1. bad-proof-of-work (impossible here, because we've already + // accepted the header) + // 2. merkleroot doesn't match the transactions given (already + // caught in FillBlock with READ_STATUS_FAILED, so + // impossible here) + // 3. the block is otherwise invalid (eg invalid coinbase, + // block is too big, too many legacy sigops, etc). + // So if CheckBlock failed, #3 is the only possibility. + // Under BIP 152, we don't discourage the peer unless proof of work is + // invalid (we don't require all the stateless checks to have + // been run). This is handled below, so just treat this as + // though the block was successfully read, and rely on the + // handling in ProcessNewBlock to ensure the block index is + // updated, etc. + RemoveBlockRequest(block_transactions.blockhash, pfrom.GetId()); // it is now an empty pointer + fBlockRead = true; + // mapBlockSource is used for potentially punishing peers and + // updating which peers send us compact blocks, so the race + // between here and cs_main in ProcessNewBlock is fine. + // BIP 152 permits peers to relay compact blocks after validating + // the header only; we should not punish peers if the block turns + // out to be invalid. + mapBlockSource.emplace(block_transactions.blockhash, std::make_pair(pfrom.GetId(), false)); + } + } // Don't hold cs_main when we call into ProcessNewBlock + if (fBlockRead) { + // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, + // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) + // This bypasses some anti-DoS logic in AcceptBlock (eg to prevent + // disk-space attacks), but this should be safe due to the + // protections in the compact block handler -- see related comment + // in compact block optimistic reconstruction handling. + ProcessBlock(pfrom, pblock, /*force_processing=*/true, /*min_pow_checked=*/true); + } + return; +} + void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc) @@ -3418,8 +3510,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // If the peer is old enough to have the old alert system, send it the final alert. if (greatest_common_version <= 70012) { - DataStream finalAlert{ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50")}; - m_connman.PushMessage(&pfrom, CNetMsgMaker(greatest_common_version).Make("alert", finalAlert)); + const auto finalAlert{ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50")}; + m_connman.PushMessage(&pfrom, CNetMsgMaker(greatest_common_version).Make("alert", Span{finalAlert})); } // Feeler connections exist only to verify if address is online. @@ -4272,12 +4364,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, blockhash.ToString(), pfrom.GetId()); } - // When we succeed in decoding a block's txids from a cmpctblock - // message we typically jump to the BLOCKTXN handling code, with a - // dummy (empty) BLOCKTXN message, to re-use the logic there in - // completing processing of the putative block (without cs_main). bool fProcessBLOCKTXN = false; - CDataStream blockTxnMsg(SER_NETWORK, PROTOCOL_VERSION); // If we end up treating this as a plain headers message, call that as well // without cs_main. @@ -4365,11 +4452,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), blockhash); m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); - return; } else { // Give up for this peer and wait for other peer(s) RemoveBlockRequest(pindex->GetBlockHash(), pfrom.GetId()); } + return; } BlockTransactionsRequest req; @@ -4378,10 +4465,6 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, req.indexes.push_back(i); } if (req.indexes.empty()) { - // Dirty hack to jump to BLOCKTXN code (TODO: move message handling into their own functions) - BlockTransactions txn; - txn.blockhash = blockhash; - blockTxnMsg << txn; fProcessBLOCKTXN = true; } else if (first_in_flight) { // We will try to round-trip any compact blocks we get on failure, @@ -4436,7 +4519,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } // cs_main if (fProcessBLOCKTXN) { - return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, time_received, interruptMsgProc); + BlockTransactions txn; + txn.blockhash = blockhash; + return ProcessCompactBlockTxns(pfrom, *peer, txn); } if (fRevertToHeaderProcessing) { @@ -4488,88 +4573,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, BlockTransactions resp; vRecv >> resp; - std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); - bool fBlockRead = false; - { - LOCK(cs_main); - - auto range_flight = mapBlocksInFlight.equal_range(resp.blockhash); - size_t already_in_flight = std::distance(range_flight.first, range_flight.second); - bool requested_block_from_this_peer{false}; - - // Multimap ensures ordering of outstanding requests. It's either empty or first in line. - bool first_in_flight = already_in_flight == 0 || (range_flight.first->second.first == pfrom.GetId()); - - while (range_flight.first != range_flight.second) { - auto [node_id, block_it] = range_flight.first->second; - if (node_id == pfrom.GetId() && block_it->partialBlock) { - requested_block_from_this_peer = true; - break; - } - range_flight.first++; - } - - if (!requested_block_from_this_peer) { - LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom.GetId()); - return; - } - - PartiallyDownloadedBlock& partialBlock = *range_flight.first->second.second->partialBlock; - ReadStatus status = partialBlock.FillBlock(*pblock, resp.txn); - if (status == READ_STATUS_INVALID) { - RemoveBlockRequest(resp.blockhash, pfrom.GetId()); // Reset in-flight state in case Misbehaving does not result in a disconnect - Misbehaving(*peer, 100, "invalid compact block/non-matching block transactions"); - return; - } else if (status == READ_STATUS_FAILED) { - if (first_in_flight) { - // Might have collided, fall back to getdata now :( - std::vector<CInv> invs; - invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(*peer), resp.blockhash)); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); - } else { - RemoveBlockRequest(resp.blockhash, pfrom.GetId()); - LogPrint(BCLog::NET, "Peer %d sent us a compact block but it failed to reconstruct, waiting on first download to complete\n", pfrom.GetId()); - return; - } - } else { - // Block is either okay, or possibly we received - // READ_STATUS_CHECKBLOCK_FAILED. - // Note that CheckBlock can only fail for one of a few reasons: - // 1. bad-proof-of-work (impossible here, because we've already - // accepted the header) - // 2. merkleroot doesn't match the transactions given (already - // caught in FillBlock with READ_STATUS_FAILED, so - // impossible here) - // 3. the block is otherwise invalid (eg invalid coinbase, - // block is too big, too many legacy sigops, etc). - // So if CheckBlock failed, #3 is the only possibility. - // Under BIP 152, we don't discourage the peer unless proof of work is - // invalid (we don't require all the stateless checks to have - // been run). This is handled below, so just treat this as - // though the block was successfully read, and rely on the - // handling in ProcessNewBlock to ensure the block index is - // updated, etc. - RemoveBlockRequest(resp.blockhash, pfrom.GetId()); // it is now an empty pointer - fBlockRead = true; - // mapBlockSource is used for potentially punishing peers and - // updating which peers send us compact blocks, so the race - // between here and cs_main in ProcessNewBlock is fine. - // BIP 152 permits peers to relay compact blocks after validating - // the header only; we should not punish peers if the block turns - // out to be invalid. - mapBlockSource.emplace(resp.blockhash, std::make_pair(pfrom.GetId(), false)); - } - } // Don't hold cs_main when we call into ProcessNewBlock - if (fBlockRead) { - // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, - // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) - // This bypasses some anti-DoS logic in AcceptBlock (eg to prevent - // disk-space attacks), but this should be safe due to the - // protections in the compact block handler -- see related comment - // in compact block optimistic reconstruction handling. - ProcessBlock(pfrom, pblock, /*force_processing=*/true, /*min_pow_checked=*/true); - } - return; + return ProcessCompactBlockTxns(pfrom, *peer, resp); } if (msg_type == NetMsgType::HEADERS) @@ -5778,7 +5782,6 @@ bool PeerManagerImpl::SendMessages(CNode* pto) continue; } auto txid = txinfo.tx->GetHash(); - auto wtxid = txinfo.tx->GetWitnessHash(); // Peer told you to not send transactions at that feerate? Don't bother sending it. if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { continue; @@ -5788,24 +5791,6 @@ bool PeerManagerImpl::SendMessages(CNode* pto) tx_relay->m_recently_announced_invs.insert(hash); vInv.push_back(inv); nRelayedTransactions++; - { - // Expire old relay messages - while (!g_relay_expiration.empty() && g_relay_expiration.front().first < current_time) - { - mapRelay.erase(g_relay_expiration.front().second); - g_relay_expiration.pop_front(); - } - - auto ret = mapRelay.emplace(txid, std::move(txinfo.tx)); - if (ret.second) { - g_relay_expiration.emplace_back(current_time + RELAY_TX_CACHE_TIME, ret.first); - } - // Add wtxid-based lookup into mapRelay as well, so that peers can request by wtxid - auto ret2 = mapRelay.emplace(wtxid, ret.first->second); - if (ret2.second) { - g_relay_expiration.emplace_back(current_time + RELAY_TX_CACHE_TIME, ret2.first); - } - } if (vInv.size() == MAX_INV_SZ) { m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); @@ -5831,7 +5816,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection // should only happen during initial block download. - LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->GetId()); + LogPrintf("Peer=%d%s is stalling block download, disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); pto->fDisconnect = true; // Increase timeout for the next peer so that we don't disconnect multiple peers if our own // bandwidth is insufficient. @@ -5850,7 +5835,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); int nOtherPeersWithValidatedDownloads = m_peers_downloading_from - 1; if (current_time > state.m_downloading_since + std::chrono::seconds{consensusParams.nPowTargetSpacing} * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { - LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.pindex->GetBlockHash().ToString(), pto->GetId()); + LogPrintf("Timeout downloading block %s from peer=%d%s, disconnecting\n", queuedBlock.pindex->GetBlockHash().ToString(), pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); pto->fDisconnect = true; return true; } @@ -5866,11 +5851,11 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // disconnect our sync peer for stalling; we have bigger // problems if we can't get any outbound peers. if (!pto->HasPermission(NetPermissionFlags::NoBan)) { - LogPrintf("Timeout downloading headers from peer=%d, disconnecting\n", pto->GetId()); + LogPrintf("Timeout downloading headers from peer=%d%s, disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); pto->fDisconnect = true; return true; } else { - LogPrintf("Timeout downloading headers from noban peer=%d, not disconnecting\n", pto->GetId()); + LogPrintf("Timeout downloading headers from noban peer=%d%s, not disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); // Reset the headers sync state so that we have a // chance to try downloading from a different peer. // Note: this will also result in at least one more diff --git a/src/netbase.cpp b/src/netbase.cpp index 4f78d2e31a..a8419217f4 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -21,14 +21,6 @@ #include <limits> #include <memory> -#ifndef WIN32 -#include <fcntl.h> -#endif - -#ifdef USE_POLL -#include <poll.h> -#endif - // Settings static GlobalMutex g_proxyinfo_mutex; static Proxy proxyInfo[NET_MAX] GUARDED_BY(g_proxyinfo_mutex); @@ -132,14 +124,9 @@ std::vector<std::string> GetNetworkNames(bool append_unroutable) return names; } -static bool LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) +static std::vector<CNetAddr> LookupIntern(const std::string& name, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - vIP.clear(); - - if (!ContainsNoNUL(name)) { - return false; - } - + if (!ContainsNoNUL(name)) return {}; { CNetAddr addr; // From our perspective, onion addresses are not hostnames but rather @@ -148,83 +135,65 @@ static bool LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, un // getaddrinfo to decode them and it wouldn't make sense to resolve // them, we return a network address representing it instead. See // CNetAddr::SetSpecial(const std::string&) for more details. - if (addr.SetSpecial(name)) { - vIP.push_back(addr); - return true; - } + if (addr.SetSpecial(name)) return {addr}; } + std::vector<CNetAddr> addresses; + for (const CNetAddr& resolved : dns_lookup_function(name, fAllowLookup)) { - if (nMaxSolutions > 0 && vIP.size() >= nMaxSolutions) { + if (nMaxSolutions > 0 && addresses.size() >= nMaxSolutions) { break; } /* Never allow resolving to an internal address. Consider any such result invalid */ if (!resolved.IsInternal()) { - vIP.push_back(resolved); + addresses.push_back(resolved); } } - return (vIP.size() > 0); + return addresses; } -bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) +std::vector<CNetAddr> LookupHost(const std::string& name, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ContainsNoNUL(name)) { - return false; - } + if (!ContainsNoNUL(name)) return {}; std::string strHost = name; - if (strHost.empty()) - return false; + if (strHost.empty()) return {}; if (strHost.front() == '[' && strHost.back() == ']') { strHost = strHost.substr(1, strHost.size() - 2); } - return LookupIntern(strHost, vIP, nMaxSolutions, fAllowLookup, dns_lookup_function); + return LookupIntern(strHost, nMaxSolutions, fAllowLookup, dns_lookup_function); } -bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSLookupFn dns_lookup_function) +std::optional<CNetAddr> LookupHost(const std::string& name, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ContainsNoNUL(name)) { - return false; - } - std::vector<CNetAddr> vIP; - LookupHost(name, vIP, 1, fAllowLookup, dns_lookup_function); - if(vIP.empty()) - return false; - addr = vIP.front(); - return true; + const std::vector<CNetAddr> addresses{LookupHost(name, 1, fAllowLookup, dns_lookup_function)}; + return addresses.empty() ? std::nullopt : std::make_optional(addresses.front()); } -bool Lookup(const std::string& name, std::vector<CService>& vAddr, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function) +std::vector<CService> Lookup(const std::string& name, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function) { if (name.empty() || !ContainsNoNUL(name)) { - return false; + return {}; } uint16_t port{portDefault}; std::string hostname; SplitHostPort(name, port, hostname); - std::vector<CNetAddr> vIP; - bool fRet = LookupIntern(hostname, vIP, nMaxSolutions, fAllowLookup, dns_lookup_function); - if (!fRet) - return false; - vAddr.resize(vIP.size()); - for (unsigned int i = 0; i < vIP.size(); i++) - vAddr[i] = CService(vIP[i], port); - return true; + const std::vector<CNetAddr> addresses{LookupIntern(hostname, nMaxSolutions, fAllowLookup, dns_lookup_function)}; + if (addresses.empty()) return {}; + std::vector<CService> services; + services.reserve(addresses.size()); + for (const auto& addr : addresses) + services.emplace_back(addr, port); + return services; } -bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function) +std::optional<CService> Lookup(const std::string& name, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ContainsNoNUL(name)) { - return false; - } - std::vector<CService> vService; - bool fRet = Lookup(name, vService, portDefault, fAllowLookup, 1, dns_lookup_function); - if (!fRet) - return false; - addr = vService[0]; - return true; + const std::vector<CService> services{Lookup(name, portDefault, fAllowLookup, 1, dns_lookup_function)}; + + return services.empty() ? std::nullopt : std::make_optional(services.front()); } CService LookupNumeric(const std::string& name, uint16_t portDefault, DNSLookupFn dns_lookup_function) @@ -232,12 +201,9 @@ CService LookupNumeric(const std::string& name, uint16_t portDefault, DNSLookupF if (!ContainsNoNUL(name)) { return {}; } - CService addr; // "1.2:345" will fail to resolve the ip, but will still set the port. // If the ip fails to resolve, re-init the result. - if(!Lookup(name, addr, portDefault, false, dns_lookup_function)) - addr = CService(); - return addr; + return Lookup(name, portDefault, /*fAllowLookup=*/false, dns_lookup_function).value_or(CService{}); } /** SOCKS version */ @@ -689,27 +655,27 @@ bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out) const size_t slash_pos{subnet_str.find_last_of('/')}; const std::string str_addr{subnet_str.substr(0, slash_pos)}; - CNetAddr addr; + const std::optional<CNetAddr> addr{LookupHost(str_addr, /*fAllowLookup=*/false)}; - if (LookupHost(str_addr, addr, /*fAllowLookup=*/false)) { + if (addr.has_value()) { if (slash_pos != subnet_str.npos) { const std::string netmask_str{subnet_str.substr(slash_pos + 1)}; uint8_t netmask; if (ParseUInt8(netmask_str, &netmask)) { // Valid number; assume CIDR variable-length subnet masking. - subnet_out = CSubNet{addr, netmask}; + subnet_out = CSubNet{addr.value(), netmask}; return subnet_out.IsValid(); } else { // Invalid number; try full netmask syntax. Never allow lookup for netmask. - CNetAddr full_netmask; - if (LookupHost(netmask_str, full_netmask, /*fAllowLookup=*/false)) { - subnet_out = CSubNet{addr, full_netmask}; + const std::optional<CNetAddr> full_netmask{LookupHost(netmask_str, /*fAllowLookup=*/false)}; + if (full_netmask.has_value()) { + subnet_out = CSubNet{addr.value(), full_netmask.value()}; return subnet_out.IsValid(); } } } else { // Single IP subnet (<ipv4>/32 or <ipv6>/128). - subnet_out = CSubNet{addr}; + subnet_out = CSubNet{addr.value()}; return subnet_out.IsValid(); } } diff --git a/src/netbase.h b/src/netbase.h index a43f22f240..1da4f5c51d 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -105,24 +105,25 @@ extern DNSLookupFn g_dns_lookup; * @param name The string representing a host. Could be a name or a numerical * IP address (IPv6 addresses in their bracketed form are * allowed). - * @param[out] vIP The resulting network addresses to which the specified host - * string resolved. * - * @returns Whether or not the specified host string successfully resolved to - * any resulting network addresses. + * @returns The resulting network addresses to which the specified host + * string resolved. * - * @see Lookup(const std::string&, std::vector<CService>&, uint16_t, bool, unsigned int, DNSLookupFn) + * @see Lookup(const std::string&, uint16_t, bool, unsigned int, DNSLookupFn) * for additional parameter descriptions. */ -bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); +std::vector<CNetAddr> LookupHost(const std::string& name, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); /** * Resolve a host string to its first corresponding network address. * - * @see LookupHost(const std::string&, std::vector<CNetAddr>&, uint16_t, bool, DNSLookupFn) + * @returns The resulting network address to which the specified host + * string resolved or std::nullopt if host does not resolve to an address. + * + * @see LookupHost(const std::string&, unsigned int, bool, DNSLookupFn) * for additional parameter descriptions. */ -bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); +std::optional<CNetAddr> LookupHost(const std::string& name, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); /** * Resolve a service string to its corresponding service. @@ -132,8 +133,6 @@ bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSL * disambiguated bracketed form), optionally followed by a uint16_t port * number. (e.g. example.com:8333 or * [2001:db8:85a3:8d3:1319:8a2e:370:7348]:420) - * @param[out] vAddr The resulting services to which the specified service string - * resolved. * @param portDefault The default port for resulting services if not specified * by the service string. * @param fAllowLookup Whether or not hostname lookups are permitted. If yes, @@ -141,18 +140,18 @@ bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSL * @param nMaxSolutions The maximum number of results we want, specifying 0 * means "as many solutions as we get." * - * @returns Whether or not the service string successfully resolved to any - * resulting services. + * @returns The resulting services to which the specified service string + * resolved. */ -bool Lookup(const std::string& name, std::vector<CService>& vAddr, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function = g_dns_lookup); +std::vector<CService> Lookup(const std::string& name, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function = g_dns_lookup); /** * Resolve a service string to its first corresponding service. * - * @see Lookup(const std::string&, std::vector<CService>&, uint16_t, bool, unsigned int, DNSLookupFn) + * @see Lookup(const std::string&, uint16_t, bool, unsigned int, DNSLookupFn) * for additional parameter descriptions. */ -bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); +std::optional<CService> Lookup(const std::string& name, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); /** * Resolve a service string with a numeric IP to its first corresponding @@ -160,7 +159,7 @@ bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool * * @returns The resulting CService if the resolution was successful, [::]:0 otherwise. * - * @see Lookup(const std::string&, std::vector<CService>&, uint16_t, bool, unsigned int, DNSLookupFn) + * @see Lookup(const std::string&, uint16_t, bool, unsigned int, DNSLookupFn) * for additional parameter descriptions. */ CService LookupNumeric(const std::string& name, uint16_t portDefault = 0, DNSLookupFn dns_lookup_function = g_dns_lookup); diff --git a/src/node/blockmanager_args.cpp b/src/node/blockmanager_args.cpp index 23b0bd37ab..4b296db1b0 100644 --- a/src/node/blockmanager_args.cpp +++ b/src/node/blockmanager_args.cpp @@ -7,26 +7,26 @@ #include <common/args.h> #include <node/blockstorage.h> #include <tinyformat.h> +#include <util/result.h> #include <util/translation.h> #include <validation.h> #include <cstdint> -#include <optional> namespace node { -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts) +util::Result<void> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts) { // block pruning; get the amount of disk space (in MiB) to allot for block & undo files int64_t nPruneArg{args.GetIntArg("-prune", opts.prune_target)}; if (nPruneArg < 0) { - return _("Prune cannot be configured with a negative value."); + return util::Error{_("Prune cannot be configured with a negative value.")}; } uint64_t nPruneTarget{uint64_t(nPruneArg) * 1024 * 1024}; if (nPruneArg == 1) { // manual pruning: -prune=1 nPruneTarget = BlockManager::PRUNE_TARGET_MANUAL; } else if (nPruneTarget) { if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) { - return strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024); + return util::Error{strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)}; } } opts.prune_target = nPruneTarget; @@ -34,6 +34,6 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, BlockM if (auto value{args.GetBoolArg("-fastprune")}) opts.fast_prune = *value; if (auto value{args.GetBoolArg("-stopafterblockimport")}) opts.stop_after_block_import = *value; - return std::nullopt; + return {}; } } // namespace node diff --git a/src/node/blockmanager_args.h b/src/node/blockmanager_args.h index e657c6bb45..de9fe83a5c 100644 --- a/src/node/blockmanager_args.h +++ b/src/node/blockmanager_args.h @@ -7,14 +7,12 @@ #define BITCOIN_NODE_BLOCKMANAGER_ARGS_H #include <node/blockstorage.h> - -#include <optional> +#include <util/result.h> class ArgsManager; -struct bilingual_str; namespace node { -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts); +[[nodiscard]] util::Result<void> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts); } // namespace node #endif // BITCOIN_NODE_BLOCKMANAGER_ARGS_H diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index f8d4e6c1da..223b0e6a17 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -17,9 +17,8 @@ #include <signet.h> #include <streams.h> #include <undo.h> +#include <util/batchpriority.h> #include <util/fs.h> -#include <util/syscall_sandbox.h> -#include <util/system.h> #include <validation.h> #include <map> @@ -573,7 +572,7 @@ void BlockManager::UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const const bool removed_blockfile{fs::remove(BlockFileSeq().FileName(pos), ec)}; const bool removed_undofile{fs::remove(UndoFileSeq().FileName(pos), ec)}; if (removed_blockfile || removed_undofile) { - LogPrint(BCLog::BLOCKSTORE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it); + LogPrint(BCLog::BLOCKSTORAGE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } } @@ -642,7 +641,7 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne if ((int)nFile != m_last_blockfile) { if (!fKnown) { - LogPrint(BCLog::BLOCKSTORE, "Leaving block file %i: %s\n", m_last_blockfile, m_blockfile_info[m_last_blockfile].ToString()); + LogPrint(BCLog::BLOCKSTORAGE, "Leaving block file %i: %s\n", m_last_blockfile, m_blockfile_info[m_last_blockfile].ToString()); } FlushBlockFile(!fKnown, finalize_undo); m_last_blockfile = nFile; @@ -869,7 +868,6 @@ public: void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const fs::path& mempool_path) { - SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS); ScheduleBatchPriority(); { @@ -928,8 +926,7 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile for (Chainstate* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { BlockValidationState state; if (!chainstate->ActivateBestChain(state, nullptr)) { - LogPrintf("Failed to connect best block (%s)\n", state.ToString()); - StartShutdown(); + AbortNode(strprintf("Failed to connect best block (%s)", state.ToString())); return; } } diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index 40b609d069..3900d2e620 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -207,7 +207,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize } else if (snapshot_completion == SnapshotCompletionResult::SUCCESS) { LogPrintf("[snapshot] cleaning up unneeded background chainstate, then reinitializing\n"); if (!chainman.ValidatedSnapshotCleanup()) { - AbortNode("Background chainstate cleanup failed unexpectedly."); + return {ChainstateLoadStatus::FAILURE_FATAL, Untranslated("Background chainstate cleanup failed unexpectedly.")}; } // Because ValidatedSnapshotCleanup() has torn down chainstates with @@ -253,7 +253,7 @@ ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const C "Only rebuild the block database if you are sure that your computer's date and time are correct")}; } - VerifyDBResult result = CVerifyDB().VerifyDB( + VerifyDBResult result = CVerifyDB(chainman.GetNotifications()).VerifyDB( *chainstate, chainman.GetConsensus(), chainstate->CoinsDB(), options.check_level, options.check_blocks); diff --git a/src/node/chainstate.h b/src/node/chainstate.h index 77240cafe9..2e35035c28 100644 --- a/src/node/chainstate.h +++ b/src/node/chainstate.h @@ -42,7 +42,8 @@ struct ChainstateLoadOptions { //! and exit cleanly in the interrupted case. enum class ChainstateLoadStatus { SUCCESS, - FAILURE, + FAILURE, //!< Generic failure which reindexing may fix + FAILURE_FATAL, //!< Fatal error which should not prompt to reindex FAILURE_INCOMPATIBLE_DB, FAILURE_INSUFFICIENT_DBCACHE, INTERRUPTED, diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp index b97344c9aa..a7f7303348 100644 --- a/src/node/chainstatemanager_args.cpp +++ b/src/node/chainstatemanager_args.cpp @@ -11,16 +11,16 @@ #include <node/database_args.h> #include <tinyformat.h> #include <uint256.h> +#include <util/result.h> #include <util/strencodings.h> #include <util/translation.h> #include <validation.h> #include <chrono> -#include <optional> #include <string> namespace node { -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) +util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) { if (auto value{args.GetBoolArg("-checkblockindex")}) opts.check_block_index = *value; @@ -28,7 +28,7 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, Chains if (auto value{args.GetArg("-minimumchainwork")}) { if (!IsHexNumber(*value)) { - return strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), *value); + return util::Error{strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), *value)}; } opts.minimum_chain_work = UintToArith256(uint256S(*value)); } @@ -37,10 +37,12 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, Chains if (auto value{args.GetIntArg("-maxtipage")}) opts.max_tip_age = std::chrono::seconds{*value}; + if (auto value{args.GetIntArg("-stopatheight")}) opts.stop_at_height = *value; + ReadDatabaseArgs(args, opts.block_tree_db); ReadDatabaseArgs(args, opts.coins_db); ReadCoinsViewArgs(args, opts.coins_view); - return std::nullopt; + return {}; } } // namespace node diff --git a/src/node/chainstatemanager_args.h b/src/node/chainstatemanager_args.h index 6c46b998f2..701515953e 100644 --- a/src/node/chainstatemanager_args.h +++ b/src/node/chainstatemanager_args.h @@ -5,15 +5,13 @@ #ifndef BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H #define BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H +#include <util/result.h> #include <validation.h> -#include <optional> - class ArgsManager; -struct bilingual_str; namespace node { -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts); +[[nodiscard]] util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts); } // namespace node #endif // BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H diff --git a/src/node/context.cpp b/src/node/context.cpp index af59ab932b..ca56fa0b86 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -11,6 +11,7 @@ #include <net.h> #include <net_processing.h> #include <netgroup.h> +#include <node/kernel_notifications.h> #include <policy/fees.h> #include <scheduler.h> #include <txmempool.h> diff --git a/src/node/context.h b/src/node/context.h index 84f4053c84..91b68fa5bb 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -7,7 +7,9 @@ #include <kernel/context.h> +#include <atomic> #include <cassert> +#include <cstdlib> #include <functional> #include <memory> #include <vector> @@ -30,6 +32,8 @@ class WalletLoader; } // namespace interfaces namespace node { +class KernelNotifications; + //! NodeContext struct containing references to chain state and connection //! state. //! @@ -62,6 +66,8 @@ struct NodeContext { interfaces::WalletLoader* wallet_loader{nullptr}; std::unique_ptr<CScheduler> scheduler; std::function<void()> rpc_interruption_point = [] {}; + std::unique_ptr<KernelNotifications> notifications; + std::atomic<int> exit_status{EXIT_SUCCESS}; //! Declare default constructor and destructor that are not inline, so code //! instantiating the NodeContext struct doesn't need to #include class diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 6e39ccf34e..9c98e4cf0c 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -89,11 +89,12 @@ public: void initLogging() override { InitLogging(args()); } void initParameterInteraction() override { InitParameterInteraction(args()); } bilingual_str getWarnings() override { return GetWarnings(true); } + int getExitStatus() override { return Assert(m_context)->exit_status.load(); } uint32_t getLogCategories() override { return LogInstance().GetCategoryMask(); } bool baseInitialize() override { - if (!AppInitBasicSetup(args())) return false; - if (!AppInitParameterInteraction(args(), /*use_syscall_sandbox=*/false)) return false; + if (!AppInitBasicSetup(args(), Assert(context())->exit_status)) return false; + if (!AppInitParameterInteraction(args())) return false; m_context->kernel = std::make_unique<kernel::Context>(); if (!AppInitSanityChecks(*m_context->kernel)) return false; @@ -105,7 +106,10 @@ public: } bool appInitMain(interfaces::BlockAndHeaderTipInfo* tip_info) override { - return AppInitMain(*m_context, tip_info); + if (AppInitMain(*m_context, tip_info)) return true; + // Error during initialization, set exit status before continue + m_context->exit_status.store(EXIT_FAILURE); + return false; } void appShutdown() override { @@ -125,17 +129,17 @@ public: bool isSettingIgnored(const std::string& name) override { bool ignored = false; - args().LockSettings([&](util::Settings& settings) { - if (auto* options = util::FindKey(settings.command_line_options, name)) { + args().LockSettings([&](common::Settings& settings) { + if (auto* options = common::FindKey(settings.command_line_options, name)) { ignored = !options->empty(); } }); return ignored; } - util::SettingsValue getPersistentSetting(const std::string& name) override { return args().GetPersistentSetting(name); } - void updateRwSetting(const std::string& name, const util::SettingsValue& value) override + common::SettingsValue getPersistentSetting(const std::string& name) override { return args().GetPersistentSetting(name); } + void updateRwSetting(const std::string& name, const common::SettingsValue& value) override { - args().LockSettings([&](util::Settings& settings) { + args().LockSettings([&](common::Settings& settings) { if (value.isNull()) { settings.rw_settings.erase(name); } else { @@ -144,9 +148,9 @@ public: }); args().WriteSettingsFile(); } - void forceSetting(const std::string& name, const util::SettingsValue& value) override + void forceSetting(const std::string& name, const common::SettingsValue& value) override { - args().LockSettings([&](util::Settings& settings) { + args().LockSettings([&](common::Settings& settings) { if (value.isNull()) { settings.forced_settings.erase(name); } else { @@ -157,7 +161,7 @@ public: void resetSettings() override { args().WriteSettingsFile(/*errors=*/nullptr, /*backup=*/true); - args().LockSettings([&](util::Settings& settings) { + args().LockSettings([&](common::Settings& settings) { settings.rw_settings.clear(); }); args().WriteSettingsFile(); @@ -744,27 +748,27 @@ public: RPCRunLater(name, std::move(fn), seconds); } int rpcSerializationFlags() override { return RPCSerializationFlags(); } - util::SettingsValue getSetting(const std::string& name) override + common::SettingsValue getSetting(const std::string& name) override { return args().GetSetting(name); } - std::vector<util::SettingsValue> getSettingsList(const std::string& name) override + std::vector<common::SettingsValue> getSettingsList(const std::string& name) override { return args().GetSettingsList(name); } - util::SettingsValue getRwSetting(const std::string& name) override + common::SettingsValue getRwSetting(const std::string& name) override { - util::SettingsValue result; - args().LockSettings([&](const util::Settings& settings) { - if (const util::SettingsValue* value = util::FindKey(settings.rw_settings, name)) { + common::SettingsValue result; + args().LockSettings([&](const common::Settings& settings) { + if (const common::SettingsValue* value = common::FindKey(settings.rw_settings, name)) { result = *value; } }); return result; } - bool updateRwSetting(const std::string& name, const util::SettingsValue& value, bool write) override + bool updateRwSetting(const std::string& name, const common::SettingsValue& value, bool write) override { - args().LockSettings([&](util::Settings& settings) { + args().LockSettings([&](common::Settings& settings) { if (value.isNull()) { settings.rw_settings.erase(name); } else { diff --git a/src/node/kernel_notifications.cpp b/src/node/kernel_notifications.cpp new file mode 100644 index 0000000000..926b157f3b --- /dev/null +++ b/src/node/kernel_notifications.cpp @@ -0,0 +1,75 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/kernel_notifications.h> + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <common/args.h> +#include <common/system.h> +#include <node/interface_ui.h> +#include <util/strencodings.h> +#include <util/string.h> +#include <util/translation.h> +#include <warnings.h> + +#include <cstdint> +#include <string> +#include <thread> + +static void AlertNotify(const std::string& strMessage) +{ + uiInterface.NotifyAlertChanged(); +#if HAVE_SYSTEM + std::string strCmd = gArgs.GetArg("-alertnotify", ""); + if (strCmd.empty()) return; + + // Alert text should be plain ascii coming from a trusted source, but to + // be safe we first strip anything not in safeChars, then add single quotes around + // the whole string before passing it to the shell: + std::string singleQuote("'"); + std::string safeStatus = SanitizeString(strMessage); + safeStatus = singleQuote+safeStatus+singleQuote; + ReplaceAll(strCmd, "%s", safeStatus); + + std::thread t(runCommand, strCmd); + t.detach(); // thread runs free +#endif +} + +static void DoWarning(const bilingual_str& warning) +{ + static bool fWarned = false; + SetMiscWarning(warning); + if (!fWarned) { + AlertNotify(warning.original); + fWarned = true; + } +} + +namespace node { + +void KernelNotifications::blockTip(SynchronizationState state, CBlockIndex& index) +{ + uiInterface.NotifyBlockTip(state, &index); +} + +void KernelNotifications::headerTip(SynchronizationState state, int64_t height, int64_t timestamp, bool presync) +{ + uiInterface.NotifyHeaderTip(state, height, timestamp, presync); +} + +void KernelNotifications::progress(const bilingual_str& title, int progress_percent, bool resume_possible) +{ + uiInterface.ShowProgress(title.translated, progress_percent, resume_possible); +} + +void KernelNotifications::warning(const bilingual_str& warning) +{ + DoWarning(warning); +} + +} // namespace node diff --git a/src/node/kernel_notifications.h b/src/node/kernel_notifications.h new file mode 100644 index 0000000000..3e665bbf14 --- /dev/null +++ b/src/node/kernel_notifications.h @@ -0,0 +1,31 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_KERNEL_NOTIFICATIONS_H +#define BITCOIN_NODE_KERNEL_NOTIFICATIONS_H + +#include <kernel/notifications_interface.h> + +#include <cstdint> +#include <string> + +class CBlockIndex; +enum class SynchronizationState; +struct bilingual_str; + +namespace node { +class KernelNotifications : public kernel::Notifications +{ +public: + void blockTip(SynchronizationState state, CBlockIndex& index) override; + + void headerTip(SynchronizationState state, int64_t height, int64_t timestamp, bool presync) override; + + void progress(const bilingual_str& title, int progress_percent, bool resume_possible) override; + + void warning(const bilingual_str& warning) override; +}; +} // namespace node + +#endif // BITCOIN_NODE_KERNEL_NOTIFICATIONS_H diff --git a/src/node/mempool_args.cpp b/src/node/mempool_args.cpp index 294111a58a..5381902263 100644 --- a/src/node/mempool_args.cpp +++ b/src/node/mempool_args.cpp @@ -38,7 +38,7 @@ void ApplyArgsManOptions(const ArgsManager& argsman, MemPoolLimits& mempool_limi } } -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, MemPoolOptions& mempool_opts) +util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, MemPoolOptions& mempool_opts) { mempool_opts.check_ratio = argsman.GetIntArg("-checkmempool", mempool_opts.check_ratio); @@ -52,7 +52,7 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, con if (std::optional<CAmount> inc_relay_fee = ParseMoney(argsman.GetArg("-incrementalrelayfee", ""))) { mempool_opts.incremental_relay_feerate = CFeeRate{inc_relay_fee.value()}; } else { - return AmountErrMsg("incrementalrelayfee", argsman.GetArg("-incrementalrelayfee", "")); + return util::Error{AmountErrMsg("incrementalrelayfee", argsman.GetArg("-incrementalrelayfee", ""))}; } } @@ -61,7 +61,7 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, con // High fee check is done afterward in CWallet::Create() mempool_opts.min_relay_feerate = CFeeRate{min_relay_feerate.value()}; } else { - return AmountErrMsg("minrelaytxfee", argsman.GetArg("-minrelaytxfee", "")); + return util::Error{AmountErrMsg("minrelaytxfee", argsman.GetArg("-minrelaytxfee", ""))}; } } else if (mempool_opts.incremental_relay_feerate > mempool_opts.min_relay_feerate) { // Allow only setting incremental fee to control both @@ -75,7 +75,7 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, con if (std::optional<CAmount> parsed = ParseMoney(argsman.GetArg("-dustrelayfee", ""))) { mempool_opts.dust_relay_feerate = CFeeRate{parsed.value()}; } else { - return AmountErrMsg("dustrelayfee", argsman.GetArg("-dustrelayfee", "")); + return util::Error{AmountErrMsg("dustrelayfee", argsman.GetArg("-dustrelayfee", ""))}; } } @@ -89,12 +89,12 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, con mempool_opts.require_standard = !argsman.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (!chainparams.IsTestChain() && !mempool_opts.require_standard) { - return strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.GetChainTypeString()); + return util::Error{strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.GetChainTypeString())}; } mempool_opts.full_rbf = argsman.GetBoolArg("-mempoolfullrbf", mempool_opts.full_rbf); ApplyArgsManOptions(argsman, mempool_opts.limits); - return std::nullopt; + return {}; } diff --git a/src/node/mempool_args.h b/src/node/mempool_args.h index 52d8b4f265..630fee6421 100644 --- a/src/node/mempool_args.h +++ b/src/node/mempool_args.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_NODE_MEMPOOL_ARGS_H #define BITCOIN_NODE_MEMPOOL_ARGS_H -#include <optional> +#include <util/result.h> class ArgsManager; class CChainParams; @@ -21,7 +21,7 @@ struct MemPoolOptions; * @param[in] argsman The ArgsManager in which to check set options. * @param[in,out] mempool_opts The MemPoolOptions to modify according to \p argsman. */ -[[nodiscard]] std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, kernel::MemPoolOptions& mempool_opts); +[[nodiscard]] util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, kernel::MemPoolOptions& mempool_opts); #endif // BITCOIN_NODE_MEMPOOL_ARGS_H diff --git a/src/node/miner.h b/src/node/miner.h index f1ccffff55..70de9e1db0 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -14,7 +14,10 @@ #include <optional> #include <stdint.h> +#include <boost/multi_index/identity.hpp> +#include <boost/multi_index/indexed_by.hpp> #include <boost/multi_index/ordered_index.hpp> +#include <boost/multi_index/tag.hpp> #include <boost/multi_index_container.hpp> class ArgsManager; diff --git a/src/node/mini_miner.cpp b/src/node/mini_miner.cpp index 71ae9d23c7..6f253eddfa 100644 --- a/src/node/mini_miner.cpp +++ b/src/node/mini_miner.cpp @@ -346,15 +346,20 @@ std::optional<CAmount> MiniMiner::CalculateTotalBumpFees(const CFeeRate& target_ to_process.insert(iter); ancestors.insert(iter); } + + std::set<uint256> has_been_processed; while (!to_process.empty()) { auto iter = to_process.begin(); const CTransaction& tx = (*iter)->second.GetTx(); for (const auto& input : tx.vin) { if (auto parent_it{m_entries_by_txid.find(input.prevout.hash)}; parent_it != m_entries_by_txid.end()) { - to_process.insert(parent_it); + if (!has_been_processed.count(input.prevout.hash)) { + to_process.insert(parent_it); + } ancestors.insert(parent_it); } } + has_been_processed.insert(tx.GetHash()); to_process.erase(iter); } const auto ancestor_package_size = std::accumulate(ancestors.cbegin(), ancestors.cend(), int64_t{0}, diff --git a/src/node/txreconciliation.cpp b/src/node/txreconciliation.cpp index 9938759074..d62046daaa 100644 --- a/src/node/txreconciliation.cpp +++ b/src/node/txreconciliation.cpp @@ -4,9 +4,9 @@ #include <node/txreconciliation.h> +#include <common/system.h> #include <logging.h> #include <util/check.h> -#include <util/system.h> #include <unordered_map> #include <variant> diff --git a/src/node/utxo_snapshot.cpp b/src/node/utxo_snapshot.cpp index 3dae46fb84..036a25d0a5 100644 --- a/src/node/utxo_snapshot.cpp +++ b/src/node/utxo_snapshot.cpp @@ -4,7 +4,6 @@ #include <node/utxo_snapshot.h> -#include <common/args.h> #include <logging.h> #include <streams.h> #include <sync.h> @@ -82,10 +81,10 @@ std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir) return base_blockhash; } -std::optional<fs::path> FindSnapshotChainstateDir() +std::optional<fs::path> FindSnapshotChainstateDir(const fs::path& data_dir) { fs::path possible_dir = - gArgs.GetDataDirNet() / fs::u8path(strprintf("chainstate%s", SNAPSHOT_CHAINSTATE_SUFFIX)); + data_dir / fs::u8path(strprintf("chainstate%s", SNAPSHOT_CHAINSTATE_SUFFIX)); if (fs::exists(possible_dir)) { return possible_dir; diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 44ddd77dc3..a6dd3f3f13 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -67,7 +67,7 @@ constexpr std::string_view SNAPSHOT_CHAINSTATE_SUFFIX = "_snapshot"; //! Return a path to the snapshot-based chainstate dir, if one exists. -std::optional<fs::path> FindSnapshotChainstateDir(); +std::optional<fs::path> FindSnapshotChainstateDir(const fs::path& data_dir); } // namespace node diff --git a/src/policy/feerate.h b/src/policy/feerate.h index 6f859e2d0d..41f4a4d06b 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -62,7 +62,7 @@ public: /** * Return the fee in satoshis for a vsize of 1000 vbytes */ - CAmount GetFeePerK() const { return GetFee(1000); } + CAmount GetFeePerK() const { return nSatoshisPerK; } friend bool operator<(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK < b.nSatoshisPerK; } friend bool operator>(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK > b.nSatoshisPerK; } friend bool operator==(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK == b.nSatoshisPerK; } diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 6121224979..c8f2df781b 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -6,6 +6,7 @@ #include <policy/fees.h> #include <clientversion.h> +#include <common/system.h> #include <consensus/amount.h> #include <kernel/mempool_entry.h> #include <logging.h> @@ -19,11 +20,11 @@ #include <uint256.h> #include <util/fs.h> #include <util/serfloat.h> -#include <util/system.h> #include <util/time.h> #include <algorithm> #include <cassert> +#include <chrono> #include <cmath> #include <cstddef> #include <cstdint> @@ -527,7 +528,7 @@ bool CBlockPolicyEstimator::_removeTx(const uint256& hash, bool inBlock) } } -CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath) +CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath, const bool read_stale_estimates) : m_estimation_filepath{estimation_filepath} { static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero"); @@ -545,9 +546,22 @@ CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath shortStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE)); longStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); - // If the fee estimation file is present, read recorded estimations AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "rb")}; - if (est_file.IsNull() || !Read(est_file)) { + + // Whenever the fee estimation file is not present return early + if (est_file.IsNull()) { + LogPrintf("%s is not found. Continue anyway.\n", fs::PathToString(m_estimation_filepath)); + return; + } + + std::chrono::hours file_age = GetFeeEstimatorFileAge(); + // fee estimate file must not be too old to avoid wrong fee estimates. + if (file_age > MAX_FILE_AGE && !read_stale_estimates) { + LogPrintf("Fee estimation file %s too old (age=%lld > %lld hours) and will not be used to avoid serving stale estimates.\n", fs::PathToString(m_estimation_filepath), Ticks<std::chrono::hours>(file_age), Ticks<std::chrono::hours>(MAX_FILE_AGE)); + return; + } + + if (!Read(est_file)) { LogPrintf("Failed to read fee estimates from %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath)); } } @@ -903,10 +917,16 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, FeeCalculation void CBlockPolicyEstimator::Flush() { FlushUnconfirmed(); + FlushFeeEstimates(); +} +void CBlockPolicyEstimator::FlushFeeEstimates() +{ AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "wb")}; if (est_file.IsNull() || !Write(est_file)) { LogPrintf("Failed to write fee estimates to %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath)); + } else { + LogPrintf("Flushed fee estimates to %s.\n", fs::PathToString(m_estimation_filepath.filename())); } } @@ -1011,6 +1031,13 @@ void CBlockPolicyEstimator::FlushUnconfirmed() LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n", num_entries, Ticks<SecondsDouble>(endclear - startclear)); } +std::chrono::hours CBlockPolicyEstimator::GetFeeEstimatorFileAge() +{ + auto file_time = std::filesystem::last_write_time(m_estimation_filepath); + auto now = std::filesystem::file_time_type::clock::now(); + return std::chrono::duration_cast<std::chrono::hours>(now - file_time); +} + static std::set<double> MakeFeeSet(const CFeeRate& min_incremental_fee, double max_filter_fee_rate, double fee_filter_spacing) diff --git a/src/policy/fees.h b/src/policy/fees.h index 775a72a764..52761f03ca 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -14,12 +14,25 @@ #include <util/fs.h> #include <array> +#include <chrono> #include <map> #include <memory> #include <set> #include <string> #include <vector> + +// How often to flush fee estimates to fee_estimates.dat. +static constexpr std::chrono::hours FEE_FLUSH_INTERVAL{1}; + +/** fee_estimates.dat that are more than 60 hours (2.5 days) will not be read, + * as the estimates in the file are stale. + */ +static constexpr std::chrono::hours MAX_FILE_AGE{60}; + +// Whether we allow importing a fee_estimates file older than MAX_FILE_AGE. +static constexpr bool DEFAULT_ACCEPT_STALE_FEE_ESTIMATES{false}; + class AutoFile; class CTxMemPoolEntry; class TxConfirmStats; @@ -183,7 +196,7 @@ private: const fs::path m_estimation_filepath; public: /** Create new BlockPolicyEstimator and initialize stats tracking classes with default values */ - CBlockPolicyEstimator(const fs::path& estimation_filepath); + CBlockPolicyEstimator(const fs::path& estimation_filepath, const bool read_stale_estimates); ~CBlockPolicyEstimator(); /** Process all the transactions that have been included in a block */ @@ -239,6 +252,13 @@ public: void Flush() EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); + /** Record current fee estimations. */ + void FlushFeeEstimates() + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); + + /** Calculates the age of the file, since last modified */ + std::chrono::hours GetFeeEstimatorFileAge(); + private: mutable Mutex m_cs_fee_estimator; diff --git a/src/policy/policy.h b/src/policy/policy.h index 394fb34230..9135cae91c 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -24,7 +24,7 @@ static constexpr unsigned int DEFAULT_BLOCK_MAX_WEIGHT{MAX_BLOCK_WEIGHT - 4000}; /** Default for -blockmintxfee, which sets the minimum feerate for a transaction in blocks created by mining code **/ static constexpr unsigned int DEFAULT_BLOCK_MIN_TX_FEE{1000}; /** The maximum weight for transactions we're willing to relay/mine */ -static constexpr unsigned int MAX_STANDARD_TX_WEIGHT{400000}; +static constexpr int32_t MAX_STANDARD_TX_WEIGHT{400000}; /** The minimum non-witness size for transactions we're willing to relay/mine: one larger than 64 */ static constexpr unsigned int MIN_STANDARD_TX_NONWITNESS_SIZE{65}; /** Maximum number of signature check operations in an IsStandard() P2SH script */ diff --git a/src/prevector.h b/src/prevector.h index f36cfe4ff6..bcab1ff00c 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -264,8 +264,10 @@ public: fill(item_ptr(0), other.begin(), other.end()); } - prevector(prevector<N, T, Size, Diff>&& other) { - swap(other); + prevector(prevector<N, T, Size, Diff>&& other) noexcept + : _union(std::move(other._union)), _size(other._size) + { + other._size = 0; } prevector& operator=(const prevector<N, T, Size, Diff>& other) { @@ -276,8 +278,13 @@ public: return *this; } - prevector& operator=(prevector<N, T, Size, Diff>&& other) { - swap(other); + prevector& operator=(prevector<N, T, Size, Diff>&& other) noexcept { + if (!is_direct()) { + free(_union.indirect_contents.indirect); + } + _union = std::move(other._union); + _size = other._size; + other._size = 0; return *this; } diff --git a/src/protocol.cpp b/src/protocol.cpp index 5725813b7e..5ecaabec36 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -5,7 +5,7 @@ #include <protocol.h> -#include <util/system.h> +#include <common/system.h> #include <atomic> diff --git a/src/protocol.h b/src/protocol.h index cbcd400fef..ac4545c311 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -134,7 +134,8 @@ extern const char* GETADDR; /** * The mempool message requests the TXIDs of transactions that the receiving * node has verified as valid but which have not yet appeared in a block. - * @since protocol version 60002. + * @since protocol version 60002 as described by BIP35. + * Only available with service bit NODE_BLOOM, see also BIP111. */ extern const char* MEMPOOL; /** @@ -278,8 +279,6 @@ enum ServiceFlags : uint64_t { // set by all Bitcoin Core non pruned nodes, and is unset by SPV clients or other light clients. NODE_NETWORK = (1 << 0), // NODE_BLOOM means the node is capable and willing to handle bloom-filtered connections. - // Bitcoin Core nodes used to support this by default, without advertising this bit, - // but no longer do as of protocol version 70011 (= NO_BLOOM_VERSION) NODE_BLOOM = (1 << 2), // NODE_WITNESS indicates that a node can be asked for blocks and transactions including // witness data. diff --git a/src/pubkey.cpp b/src/pubkey.cpp index ae5dccfb5a..4866feed67 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -7,6 +7,7 @@ #include <hash.h> #include <secp256k1.h> +#include <secp256k1_ellswift.h> #include <secp256k1_extrakeys.h> #include <secp256k1_recovery.h> #include <secp256k1_schnorrsig.h> @@ -335,6 +336,20 @@ bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChi return true; } +CPubKey EllSwiftPubKey::Decode() const +{ + secp256k1_pubkey pubkey; + secp256k1_ellswift_decode(secp256k1_context_static, &pubkey, UCharCast(m_pubkey.data())); + + size_t sz = CPubKey::COMPRESSED_SIZE; + std::array<uint8_t, CPubKey::COMPRESSED_SIZE> vch_bytes; + + secp256k1_ec_pubkey_serialize(secp256k1_context_static, vch_bytes.data(), &sz, &pubkey, SECP256K1_EC_COMPRESSED); + assert(sz == vch_bytes.size()); + + return CPubKey{vch_bytes.begin(), vch_bytes.end()}; +} + void CExtPubKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { code[0] = nDepth; memcpy(code+1, vchFingerprint, 4); diff --git a/src/pubkey.h b/src/pubkey.h index b3edafea7f..7d37504b01 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -142,14 +142,14 @@ public: { unsigned int len = size(); ::WriteCompactSize(s, len); - s.write(AsBytes(Span{vch, len})); + s << Span{vch, len}; } template <typename Stream> void Unserialize(Stream& s) { const unsigned int len(::ReadCompactSize(s)); if (len <= SIZE) { - s.read(AsWritableBytes(Span{vch, len})); + s >> Span{vch, len}; if (len != size()) { Invalidate(); } @@ -291,6 +291,38 @@ public: SERIALIZE_METHODS(XOnlyPubKey, obj) { READWRITE(obj.m_keydata); } }; +/** An ElligatorSwift-encoded public key. */ +struct EllSwiftPubKey +{ +private: + static constexpr size_t SIZE = 64; + std::array<std::byte, SIZE> m_pubkey; + +public: + /** Construct a new ellswift public key from a given serialization. */ + EllSwiftPubKey(const std::array<std::byte, SIZE>& ellswift) : + m_pubkey(ellswift) {} + + /** Decode to normal compressed CPubKey (for debugging purposes). */ + CPubKey Decode() const; + + // Read-only access for serialization. + const std::byte* data() const { return m_pubkey.data(); } + static constexpr size_t size() { return SIZE; } + auto begin() const { return m_pubkey.cbegin(); } + auto end() const { return m_pubkey.cend(); } + + bool friend operator==(const EllSwiftPubKey& a, const EllSwiftPubKey& b) + { + return a.m_pubkey == b.m_pubkey; + } + + bool friend operator!=(const EllSwiftPubKey& a, const EllSwiftPubKey& b) + { + return a.m_pubkey != b.m_pubkey; + } +}; + struct CExtPubKey { unsigned char version[4]; unsigned char nDepth; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index d602d2c1ac..8f45af9485 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -9,8 +9,10 @@ #include <qt/bitcoin.h> #include <chainparams.h> +#include <node/context.h> #include <common/args.h> #include <common/init.h> +#include <common/system.h> #include <init.h> #include <interfaces/handler.h> #include <interfaces/init.h> @@ -32,7 +34,6 @@ #include <uint256.h> #include <util/exception.h> #include <util/string.h> -#include <util/system.h> #include <util/threadnames.h> #include <util/translation.h> #include <validation.h> @@ -397,9 +398,7 @@ void BitcoinApplication::initializeResult(bool success, interfaces::BlockAndHead { qDebug() << __func__ << ": Initialization result: " << success; - // Set exit result. - returnValue = success ? EXIT_SUCCESS : EXIT_FAILURE; - if(success) { + if (success) { delete m_splash; m_splash = nullptr; @@ -653,7 +652,6 @@ int GuiMain(int argc, char* argv[]) app.InitPruneSetting(prune_MiB); } - int rv = EXIT_SUCCESS; try { app.createWindow(networkStyle.data()); @@ -666,10 +664,9 @@ int GuiMain(int argc, char* argv[]) WinShutdownMonitor::registerShutdownBlockReason(QObject::tr("%1 didn't yet exit safely…").arg(PACKAGE_NAME), (HWND)app.getMainWinId()); #endif app.exec(); - rv = app.getReturnValue(); } else { // A dialog with detailed error will have been shown by InitError() - rv = EXIT_FAILURE; + return EXIT_FAILURE; } } catch (const std::exception& e) { PrintExceptionContinue(&e, "Runaway exception"); @@ -678,5 +675,5 @@ int GuiMain(int argc, char* argv[]) PrintExceptionContinue(nullptr, "Runaway exception"); app.handleRunawayException(QString::fromStdString(app.node().getWarnings().translated)); } - return rv; + return app.node().getExitStatus(); } diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 9174e23de6..9622c9d57d 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -62,9 +62,6 @@ public: /// Request core initialization void requestInitialize(); - /// Get process return value - int getReturnValue() const { return returnValue; } - /// Get window identifier of QMainWindow (BitcoinGUI) WId getMainWinId() const; @@ -104,7 +101,6 @@ private: PaymentServer* paymentServer{nullptr}; WalletController* m_wallet_controller{nullptr}; #endif - int returnValue{0}; const PlatformStyle* platformStyle{nullptr}; std::unique_ptr<QWidget> shutdownWindow; SplashScreen* m_splash = nullptr; diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index d26ef52eb4..f201d8fa01 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -30,16 +30,17 @@ #include <qt/macdockiconhandler.h> #endif -#include <functional> #include <chain.h> #include <chainparams.h> +#include <common/system.h> #include <interfaces/handler.h> #include <interfaces/node.h> #include <node/interface_ui.h> -#include <util/system.h> #include <util/translation.h> #include <validation.h> +#include <functional> + #include <QAction> #include <QActionGroup> #include <QApplication> diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index b22f1bc35c..ff7405d139 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -12,11 +12,11 @@ #include <clientversion.h> #include <common/args.h> +#include <common/system.h> #include <interfaces/handler.h> #include <interfaces/node.h> #include <net.h> #include <netbase.h> -#include <util/system.h> #include <util/threadnames.h> #include <util/time.h> #include <validation.h> diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 6dec4b2e42..512fce473d 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -15,11 +15,11 @@ #include <qt/guiutil.h> #include <qt/optionsmodel.h> +#include <common/system.h> #include <interfaces/node.h> -#include <validation.h> // for DEFAULT_SCRIPTCHECK_THREADS and MAX_SCRIPTCHECK_THREADS #include <netbase.h> -#include <txdb.h> // for -dbcache defaults -#include <util/system.h> +#include <txdb.h> +#include <validation.h> #include <chrono> @@ -406,9 +406,8 @@ void OptionsDialog::updateProxyValidationState() void OptionsDialog::updateDefaultProxyNets() { - CNetAddr ui_proxy_netaddr; - LookupHost(ui->proxyIp->text().toStdString(), ui_proxy_netaddr, /*fAllowLookup=*/false); - const CService ui_proxy{ui_proxy_netaddr, ui->proxyPort->text().toUShort()}; + const std::optional<CNetAddr> ui_proxy_netaddr{LookupHost(ui->proxyIp->text().toStdString(), /*fAllowLookup=*/false)}; + const CService ui_proxy{ui_proxy_netaddr.value_or(CNetAddr{}), ui->proxyPort->text().toUShort()}; Proxy proxy; bool has_proxy; diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 87a7e703b8..c1563fe1e2 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -60,7 +60,7 @@ static const char* SettingName(OptionsModel::OptionID option) } /** Call node.updateRwSetting() with Bitcoin 22.x workaround. */ -static void UpdateRwSetting(interfaces::Node& node, OptionsModel::OptionID option, const std::string& suffix, const util::SettingsValue& value) +static void UpdateRwSetting(interfaces::Node& node, OptionsModel::OptionID option, const std::string& suffix, const common::SettingsValue& value) { if (value.isNum() && (option == OptionsModel::DatabaseCache || @@ -81,14 +81,14 @@ static void UpdateRwSetting(interfaces::Node& node, OptionsModel::OptionID optio } //! Convert enabled/size values to bitcoin -prune setting. -static util::SettingsValue PruneSetting(bool prune_enabled, int prune_size_gb) +static common::SettingsValue PruneSetting(bool prune_enabled, int prune_size_gb) { assert(!prune_enabled || prune_size_gb >= 1); // PruneSizeGB and ParsePruneSizeGB never return less return prune_enabled ? PruneGBtoMiB(prune_size_gb) : 0; } //! Get pruning enabled value to show in GUI from bitcoin -prune setting. -static bool PruneEnabled(const util::SettingsValue& prune_setting) +static bool PruneEnabled(const common::SettingsValue& prune_setting) { // -prune=1 setting is manual pruning mode, so disabled for purposes of the gui return SettingToInt(prune_setting, 0) > 1; @@ -96,7 +96,7 @@ static bool PruneEnabled(const util::SettingsValue& prune_setting) //! Get pruning size value to show in GUI from bitcoin -prune setting. If //! pruning is not enabled, just show default recommended pruning size (2GB). -static int PruneSizeGB(const util::SettingsValue& prune_setting) +static int PruneSizeGB(const common::SettingsValue& prune_setting) { int value = SettingToInt(prune_setting, 0); return value > 1 ? PruneMiBtoGB(value) : DEFAULT_PRUNE_TARGET_GB; @@ -311,8 +311,8 @@ static QString GetDefaultProxyAddress() void OptionsModel::SetPruneTargetGB(int prune_target_gb) { - const util::SettingsValue cur_value = node().getPersistentSetting("prune"); - const util::SettingsValue new_value = PruneSetting(prune_target_gb > 0, prune_target_gb); + const common::SettingsValue cur_value = node().getPersistentSetting("prune"); + const common::SettingsValue new_value = PruneSetting(prune_target_gb > 0, prune_target_gb); // Force setting to take effect. It is still safe to change the value at // this point because this function is only called after the intro screen is @@ -331,7 +331,7 @@ void OptionsModel::SetPruneTargetGB(int prune_target_gb) // Keep previous pruning size, if pruning was disabled. if (PruneEnabled(cur_value)) { - UpdateRwSetting(node(), Prune, "-prev", PruneEnabled(new_value) ? util::SettingsValue{} : cur_value); + UpdateRwSetting(node(), Prune, "-prev", PruneEnabled(new_value) ? common::SettingsValue{} : cur_value); } } @@ -457,7 +457,7 @@ QVariant OptionsModel::getOption(OptionID option, const std::string& suffix) con bool OptionsModel::setOption(OptionID option, const QVariant& value, const std::string& suffix) { auto changed = [&] { return value.isValid() && value != getOption(option, suffix); }; - auto update = [&](const util::SettingsValue& value) { return UpdateRwSetting(node(), option, suffix, value); }; + auto update = [&](const common::SettingsValue& value) { return UpdateRwSetting(node(), option, suffix, value); }; bool successful = true; /* set to false on parse error */ QSettings settings; diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index bac64e3d5f..90aae0219e 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -10,6 +10,7 @@ #include <qt/forms/ui_debugwindow.h> #include <chainparams.h> +#include <common/system.h> #include <interfaces/node.h> #include <qt/bantablemodel.h> #include <qt/clientmodel.h> @@ -21,7 +22,6 @@ #include <rpc/server.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/threadnames.h> #include <univalue.h> diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 096f8a0ded..8872f8be32 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -9,13 +9,13 @@ #include <qt/splashscreen.h> #include <clientversion.h> +#include <common/system.h> #include <interfaces/handler.h> #include <interfaces/node.h> #include <interfaces/wallet.h> #include <qt/guiutil.h> #include <qt/networkstyle.h> #include <qt/walletmodel.h> -#include <util/system.h> #include <util/translation.h> #include <functional> diff --git a/src/qt/test/optiontests.cpp b/src/qt/test/optiontests.cpp index 0838e21678..e5a5179d87 100644 --- a/src/qt/test/optiontests.cpp +++ b/src/qt/test/optiontests.cpp @@ -18,13 +18,13 @@ OptionTests::OptionTests(interfaces::Node& node) : m_node(node) { - gArgs.LockSettings([&](util::Settings& s) { m_previous_settings = s; }); + gArgs.LockSettings([&](common::Settings& s) { m_previous_settings = s; }); } void OptionTests::init() { // reset args - gArgs.LockSettings([&](util::Settings& s) { s = m_previous_settings; }); + gArgs.LockSettings([&](common::Settings& s) { s = m_previous_settings; }); gArgs.ClearPathCache(); } @@ -76,14 +76,14 @@ void OptionTests::integerGetArgBug() // Test regression https://github.com/bitcoin/bitcoin/issues/24457. Ensure // that setting integer prune value doesn't cause an exception to be thrown // in the OptionsModel constructor - gArgs.LockSettings([&](util::Settings& settings) { + gArgs.LockSettings([&](common::Settings& settings) { settings.forced_settings.erase("prune"); settings.rw_settings["prune"] = 3814; }); gArgs.WriteSettingsFile(); bilingual_str error; QVERIFY(OptionsModel{m_node}.Init(error)); - gArgs.LockSettings([&](util::Settings& settings) { + gArgs.LockSettings([&](common::Settings& settings) { settings.rw_settings.erase("prune"); }); gArgs.WriteSettingsFile(); @@ -95,7 +95,7 @@ void OptionTests::parametersInteraction() // It was fixed via https://github.com/bitcoin-core/gui/pull/568. // With fListen=false in ~/.config/Bitcoin/Bitcoin-Qt.conf and all else left as default, // bitcoin-qt should set both -listen and -listenonion to false and start successfully. - gArgs.LockSettings([&](util::Settings& s) { + gArgs.LockSettings([&](common::Settings& s) { s.forced_settings.erase("listen"); s.forced_settings.erase("listenonion"); }); diff --git a/src/qt/test/optiontests.h b/src/qt/test/optiontests.h index 0c458c97a6..2d7b94933f 100644 --- a/src/qt/test/optiontests.h +++ b/src/qt/test/optiontests.h @@ -5,9 +5,9 @@ #ifndef BITCOIN_QT_TEST_OPTIONTESTS_H #define BITCOIN_QT_TEST_OPTIONTESTS_H +#include <common/settings.h> #include <qt/optionsmodel.h> #include <univalue.h> -#include <util/settings.h> #include <QObject> @@ -26,7 +26,7 @@ private Q_SLOTS: private: interfaces::Node& m_node; - util::Settings m_previous_settings; + common::Settings m_previous_settings; }; #endif // BITCOIN_QT_TEST_OPTIONTESTS_H diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 669a05fe0f..72e8055425 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -4,12 +4,12 @@ #include <qt/test/rpcnestedtests.h> +#include <common/system.h> #include <interfaces/node.h> -#include <rpc/server.h> #include <qt/rpcconsole.h> +#include <rpc/server.h> #include <test/util/setup_common.h> #include <univalue.h> -#include <util/system.h> #include <QTest> diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index eaadbd7f7a..e45fc1ced8 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -71,6 +71,9 @@ int main(int argc, char* argv[]) gArgs.ForceSetArg("-upnp", "0"); gArgs.ForceSetArg("-natpmp", "0"); + std::string error; + if (!gArgs.ReadConfigFiles(error, true)) QWARN(error.c_str()); + // Prefer the "minimal" platform for the test instead of the normal default // platform ("xcb", "windows", or "cocoa") so tests can't unintentionally // interfere with any background GUIs and don't require extra resources. diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 2461a06930..fa110cfbc9 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -13,12 +13,12 @@ #include <qt/paymentserver.h> #include <qt/transactionrecord.h> +#include <common/system.h> #include <consensus/consensus.h> #include <interfaces/node.h> #include <interfaces/wallet.h> #include <key_io.h> #include <policy/policy.h> -#include <util/system.h> #include <validation.h> #include <wallet/types.h> diff --git a/src/random.cpp b/src/random.cpp index 54500e6cc6..39ceae4206 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -599,6 +599,12 @@ std::vector<unsigned char> FastRandomContext::randbytes(size_t len) return ret; } +void FastRandomContext::fillrand(Span<std::byte> output) +{ + if (requires_seed) RandomSeed(); + rng.Keystream(UCharCast(output.data()), output.size()); +} + FastRandomContext::FastRandomContext(const uint256& seed) noexcept : requires_seed(false), bitbuf_size(0) { rng.SetKey32(seed.begin()); diff --git a/src/random.h b/src/random.h index 49c0dff5bf..50f56ed911 100644 --- a/src/random.h +++ b/src/random.h @@ -213,6 +213,9 @@ public: /** Generate random bytes. */ std::vector<unsigned char> randbytes(size_t len); + /** Fill a byte Span with random bytes. */ + void fillrand(Span<std::byte> output); + /** Generate a random 32-bit integer. */ uint32_t rand32() noexcept { return randbits(32); } diff --git a/src/rest.cpp b/src/rest.cpp index dae064f89d..ba149c1a9e 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -24,8 +24,8 @@ #include <streams.h> #include <sync.h> #include <txmempool.h> +#include <util/any.h> #include <util/check.h> -#include <util/system.h> #include <validation.h> #include <version.h> @@ -627,7 +627,7 @@ static bool rest_deploymentinfo(const std::any& context, HTTPRequest* req, const return RESTERR(req, HTTP_BAD_REQUEST, "Block not found"); } - jsonRequest.params.pushKV("blockhash", hash_str); + jsonRequest.params.push_back(hash_str); } req->WriteHeader("Content-Type", "application/json"); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 4c25dbc345..ee3237638e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1123,7 +1123,7 @@ static RPCHelpMan verifychain() LOCK(cs_main); Chainstate& active_chainstate = chainman.ActiveChainstate(); - return CVerifyDB().VerifyDB( + return CVerifyDB(chainman.GetNotifications()).VerifyDB( active_chainstate, chainman.GetParams().GetConsensus(), active_chainstate.CoinsTip(), check_level, check_depth) == VerifyDBResult::SUCCESS; }, }; diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index d08e2d55d1..5f58eef1db 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -101,6 +101,11 @@ static const CRPCConvertParam vRPCConvertParams[] = { "listunspent", 2, "addresses" }, { "listunspent", 3, "include_unsafe" }, { "listunspent", 4, "query_options" }, + { "listunspent", 4, "minimumAmount" }, + { "listunspent", 4, "maximumAmount" }, + { "listunspent", 4, "maximumCount" }, + { "listunspent", 4, "minimumSumAmount" }, + { "listunspent", 4, "include_immature_coinbase" }, { "getblock", 1, "verbosity" }, { "getblock", 1, "verbose" }, { "getblockheader", 1, "verbose" }, @@ -124,11 +129,38 @@ static const CRPCConvertParam vRPCConvertParams[] = { "submitpackage", 0, "package" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, + { "fundrawtransaction", 1, "add_inputs"}, + { "fundrawtransaction", 1, "include_unsafe"}, + { "fundrawtransaction", 1, "minconf"}, + { "fundrawtransaction", 1, "maxconf"}, + { "fundrawtransaction", 1, "changePosition"}, + { "fundrawtransaction", 1, "includeWatching"}, + { "fundrawtransaction", 1, "lockUnspents"}, + { "fundrawtransaction", 1, "fee_rate"}, + { "fundrawtransaction", 1, "feeRate"}, + { "fundrawtransaction", 1, "subtractFeeFromOutputs"}, + { "fundrawtransaction", 1, "input_weights"}, + { "fundrawtransaction", 1, "conf_target"}, + { "fundrawtransaction", 1, "replaceable"}, + { "fundrawtransaction", 1, "solving_data"}, { "fundrawtransaction", 2, "iswitness" }, { "walletcreatefundedpsbt", 0, "inputs" }, { "walletcreatefundedpsbt", 1, "outputs" }, { "walletcreatefundedpsbt", 2, "locktime" }, { "walletcreatefundedpsbt", 3, "options" }, + { "walletcreatefundedpsbt", 3, "add_inputs"}, + { "walletcreatefundedpsbt", 3, "include_unsafe"}, + { "walletcreatefundedpsbt", 3, "minconf"}, + { "walletcreatefundedpsbt", 3, "maxconf"}, + { "walletcreatefundedpsbt", 3, "changePosition"}, + { "walletcreatefundedpsbt", 3, "includeWatching"}, + { "walletcreatefundedpsbt", 3, "lockUnspents"}, + { "walletcreatefundedpsbt", 3, "fee_rate"}, + { "walletcreatefundedpsbt", 3, "feeRate"}, + { "walletcreatefundedpsbt", 3, "subtractFeeFromOutputs"}, + { "walletcreatefundedpsbt", 3, "conf_target"}, + { "walletcreatefundedpsbt", 3, "replaceable"}, + { "walletcreatefundedpsbt", 3, "solving_data"}, { "walletcreatefundedpsbt", 4, "bip32derivs" }, { "walletprocesspsbt", 1, "sign" }, { "walletprocesspsbt", 3, "bip32derivs" }, @@ -157,18 +189,49 @@ static const CRPCConvertParam vRPCConvertParams[] = { "send", 1, "conf_target" }, { "send", 3, "fee_rate"}, { "send", 4, "options" }, + { "send", 4, "add_inputs"}, + { "send", 4, "include_unsafe"}, + { "send", 4, "minconf"}, + { "send", 4, "maxconf"}, + { "send", 4, "add_to_wallet"}, + { "send", 4, "change_position"}, + { "send", 4, "fee_rate"}, + { "send", 4, "include_watching"}, + { "send", 4, "inputs"}, + { "send", 4, "locktime"}, + { "send", 4, "lock_unspents"}, + { "send", 4, "psbt"}, + { "send", 4, "subtract_fee_from_outputs"}, + { "send", 4, "conf_target"}, + { "send", 4, "replaceable"}, + { "send", 4, "solving_data"}, { "sendall", 0, "recipients" }, { "sendall", 1, "conf_target" }, { "sendall", 3, "fee_rate"}, { "sendall", 4, "options" }, + { "sendall", 4, "add_to_wallet"}, + { "sendall", 4, "fee_rate"}, + { "sendall", 4, "include_watching"}, + { "sendall", 4, "inputs"}, + { "sendall", 4, "locktime"}, + { "sendall", 4, "lock_unspents"}, + { "sendall", 4, "psbt"}, + { "sendall", 4, "send_max"}, + { "sendall", 4, "minconf"}, + { "sendall", 4, "maxconf"}, + { "sendall", 4, "conf_target"}, + { "sendall", 4, "replaceable"}, + { "sendall", 4, "solving_data"}, { "simulaterawtransaction", 0, "rawtxs" }, { "simulaterawtransaction", 1, "options" }, + { "simulaterawtransaction", 1, "include_watchonly"}, { "importprivkey", 2, "rescan" }, { "importaddress", 2, "rescan" }, { "importaddress", 3, "p2sh" }, { "importpubkey", 2, "rescan" }, { "importmulti", 0, "requests" }, { "importmulti", 1, "options" }, + { "importmulti", 1, "rescan" }, { "importdescriptors", 0, "requests" }, { "listdescriptors", 0, "private" }, { "verifychain", 0, "checklevel" }, @@ -192,7 +255,15 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getmempooldescendants", 1, "verbose" }, { "gettxspendingprevout", 0, "outputs" }, { "bumpfee", 1, "options" }, + { "bumpfee", 1, "conf_target"}, + { "bumpfee", 1, "fee_rate"}, + { "bumpfee", 1, "replaceable"}, + { "bumpfee", 1, "outputs"}, { "psbtbumpfee", 1, "options" }, + { "psbtbumpfee", 1, "conf_target"}, + { "psbtbumpfee", 1, "fee_rate"}, + { "psbtbumpfee", 1, "replaceable"}, + { "psbtbumpfee", 1, "outputs"}, { "logging", 0, "include" }, { "logging", 1, "exclude" }, { "disconnectnode", 1, "nodeid" }, @@ -226,6 +297,14 @@ static const CRPCConvertParam vRPCConvertParams[] = }; // clang-format on +/** Parse string to UniValue or throw runtime_error if string contains invalid JSON */ +static UniValue Parse(std::string_view raw) +{ + UniValue parsed; + if (!parsed.read(raw)) throw std::runtime_error(tfm::format("Error parsing JSON: %s", raw)); + return parsed; +} + class CRPCConvertTable { private: @@ -238,13 +317,13 @@ public: /** Return arg_value as UniValue, and first parse it if it is a non-string parameter */ UniValue ArgToUniValue(std::string_view arg_value, const std::string& method, int param_idx) { - return members.count({method, param_idx}) > 0 ? ParseNonRFCJSONValue(arg_value) : arg_value; + return members.count({method, param_idx}) > 0 ? Parse(arg_value) : arg_value; } /** Return arg_value as UniValue, and first parse it if it is a non-string parameter */ UniValue ArgToUniValue(std::string_view arg_value, const std::string& method, const std::string& param_name) { - return membersByName.count({method, param_name}) > 0 ? ParseNonRFCJSONValue(arg_value) : arg_value; + return membersByName.count({method, param_name}) > 0 ? Parse(arg_value) : arg_value; } }; @@ -258,16 +337,6 @@ CRPCConvertTable::CRPCConvertTable() static CRPCConvertTable rpcCvtTable; -/** Non-RFC4627 JSON parser, accepts internal values (such as numbers, true, false, null) - * as well as objects and arrays. - */ -UniValue ParseNonRFCJSONValue(std::string_view raw) -{ - UniValue parsed; - if (!parsed.read(raw)) throw std::runtime_error(tfm::format("Error parsing JSON: %s", raw)); - return parsed; -} - UniValue RPCConvertValues(const std::string &strMethod, const std::vector<std::string> &strParams) { UniValue params(UniValue::VARR); @@ -302,10 +371,10 @@ UniValue RPCConvertNamedValues(const std::string &strMethod, const std::vector<s } if (!positional_args.empty()) { - // Use __pushKV instead of pushKV to avoid overwriting an explicit + // Use pushKVEnd instead of pushKV to avoid overwriting an explicit // "args" value with an implicit one. Let the RPC server handle the // request as given. - params.__pushKV("args", positional_args); + params.pushKVEnd("args", positional_args); } return params; diff --git a/src/rpc/client.h b/src/rpc/client.h index 3c5c4fc4d6..b67cd27fdf 100644 --- a/src/rpc/client.h +++ b/src/rpc/client.h @@ -17,9 +17,4 @@ UniValue RPCConvertValues(const std::string& strMethod, const std::vector<std::s /** Convert named arguments to command-specific RPC representation */ UniValue RPCConvertNamedValues(const std::string& strMethod, const std::vector<std::string>& strParams); -/** Non-RFC4627 JSON parser, accepts internal values (such as numbers, true, false, null) - * as well as objects and arrays. - */ -UniValue ParseNonRFCJSONValue(std::string_view raw); - #endif // BITCOIN_RPC_CLIENT_H diff --git a/src/rpc/external_signer.cpp b/src/rpc/external_signer.cpp index ac135ba216..310eec5f15 100644 --- a/src/rpc/external_signer.cpp +++ b/src/rpc/external_signer.cpp @@ -3,12 +3,12 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <common/args.h> +#include <common/system.h> #include <external_signer.h> #include <rpc/protocol.h> #include <rpc/server.h> #include <rpc/util.h> #include <util/strencodings.h> -#include <util/system.h> #include <string> #include <vector> diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 89c403b6f5..11d2874961 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -351,8 +351,8 @@ UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose, bool include_mempoo entryToJSON(pool, info, e); // Mempool has unique entries so there is no advantage in using // UniValue::pushKV, which checks if the key already exists in O(N). - // UniValue::__pushKV is used instead which currently is O(1). - o.__pushKV(hash.ToString(), info); + // UniValue::pushKVEnd is used instead which currently is O(1). + o.pushKVEnd(hash.ToString(), info); } return o; } else { diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 68017e5af2..074cecadd2 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -5,6 +5,7 @@ #include <chain.h> #include <chainparams.h> +#include <common/system.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/merkle.h> @@ -32,7 +33,6 @@ #include <univalue.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -480,6 +480,40 @@ static RPCHelpMan prioritisetransaction() }; } +static RPCHelpMan getprioritisedtransactions() +{ + return RPCHelpMan{"getprioritisedtransactions", + "Returns a map of all user-created (see prioritisetransaction) fee deltas by txid, and whether the tx is present in mempool.", + {}, + RPCResult{ + RPCResult::Type::OBJ_DYN, "prioritisation-map", "prioritisation keyed by txid", + { + {RPCResult::Type::OBJ, "txid", "", { + {RPCResult::Type::NUM, "fee_delta", "transaction fee delta in satoshis"}, + {RPCResult::Type::BOOL, "in_mempool", "whether this transaction is currently in mempool"}, + }} + }, + }, + RPCExamples{ + HelpExampleCli("getprioritisedtransactions", "") + + HelpExampleRpc("getprioritisedtransactions", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + NodeContext& node = EnsureAnyNodeContext(request.context); + CTxMemPool& mempool = EnsureMemPool(node); + UniValue rpc_result{UniValue::VOBJ}; + for (const auto& delta_info : mempool.GetPrioritisedTransactions()) { + UniValue result_inner{UniValue::VOBJ}; + result_inner.pushKV("fee_delta", delta_info.delta); + result_inner.pushKV("in_mempool", delta_info.in_mempool); + rpc_result.pushKV(delta_info.txid.GetHex(), result_inner); + } + return rpc_result; + }, + }; +} + // NOTE: Assumes a conclusive result; if result is inconclusive, it must be handled by caller static UniValue BIP22ValidationResult(const BlockValidationState& state) @@ -1048,6 +1082,7 @@ void RegisterMiningRPCCommands(CRPCTable& t) {"mining", &getnetworkhashps}, {"mining", &getmininginfo}, {"mining", &prioritisetransaction}, + {"mining", &getprioritisedtransactions}, {"mining", &getblocktemplate}, {"mining", &submitblock}, {"mining", &submitheader}, diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index f247e6ee91..a2a46ef32f 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -713,9 +713,10 @@ static RPCHelpMan setban() isSubnet = true; if (!isSubnet) { - CNetAddr resolved; - LookupHost(request.params[0].get_str(), resolved, false); - netAddr = resolved; + const std::optional<CNetAddr> addr{LookupHost(request.params[0].get_str(), false)}; + if (addr.has_value()) { + netAddr = addr.value(); + } } else LookupSubNet(request.params[0].get_str(), subNet); @@ -943,11 +944,11 @@ static RPCHelpMan addpeeraddress() const bool tried{request.params[2].isNull() ? false : request.params[2].get_bool()}; UniValue obj(UniValue::VOBJ); - CNetAddr net_addr; + std::optional<CNetAddr> net_addr{LookupHost(addr_string, false)}; bool success{false}; - if (LookupHost(addr_string, net_addr, false)) { - CService service{net_addr, port}; + if (net_addr.has_value()) { + CService service{net_addr.value(), port}; CAddress address{MaybeFlipIPv6toCJDNS(service), ServiceFlags{NODE_NETWORK | NODE_WITNESS}}; address.nTime = Now<NodeSeconds>(); // The source address is set equal to the address. This is equivalent to the peer diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index ca8db0f82a..3828401642 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -19,9 +19,8 @@ #include <rpc/util.h> #include <scheduler.h> #include <univalue.h> +#include <util/any.h> #include <util/check.h> -#include <util/syscall_sandbox.h> -#include <util/system.h> #include <stdint.h> #ifdef HAVE_MALLOC_INFO @@ -70,27 +69,6 @@ static RPCHelpMan setmocktime() }; } -#if defined(USE_SYSCALL_SANDBOX) -static RPCHelpMan invokedisallowedsyscall() -{ - return RPCHelpMan{ - "invokedisallowedsyscall", - "\nInvoke a disallowed syscall to trigger a syscall sandbox violation. Used for testing purposes.\n", - {}, - RPCResult{RPCResult::Type::NONE, "", ""}, - RPCExamples{ - HelpExampleCli("invokedisallowedsyscall", "") + HelpExampleRpc("invokedisallowedsyscall", "")}, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - if (!Params().IsTestChain()) { - throw std::runtime_error("invokedisallowedsyscall is used for testing only."); - } - TestDisallowedSandboxCall(); - return UniValue::VNULL; - }, - }; -} -#endif // USE_SYSCALL_SANDBOX - static RPCHelpMan mockscheduler() { return RPCHelpMan{"mockscheduler", @@ -428,9 +406,6 @@ void RegisterNodeRPCCommands(CRPCTable& t) {"hidden", &echo}, {"hidden", &echojson}, {"hidden", &echoipc}, -#if defined(USE_SYSCALL_SANDBOX) - {"hidden", &invokedisallowedsyscall}, -#endif // USE_SYSCALL_SANDBOX }; for (const auto& c : commands) { t.appendCommand(c.name, &c); diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index cf1b6cd92b..4c67da8b70 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -88,7 +88,7 @@ bool GenerateAuthCookie(std::string *cookie_out) std::string cookie = COOKIEAUTH_USER + ":" + HexStr(rand_pwd); /** the umask determines what permissions are used to create this file - - * these are set to 0077 in util/system.cpp. + * these are set to 0077 in common/system.cpp. */ std::ofstream file; fs::path filepath_tmp = GetAuthCookieFile(true); diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 354f949002..daf751111f 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -6,13 +6,13 @@ #include <rpc/server.h> #include <common/args.h> +#include <common/system.h> #include <logging.h> #include <rpc/util.h> #include <shutdown.h> #include <sync.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/time.h> #include <boost/signals2/signal.hpp> @@ -392,7 +392,7 @@ std::string JSONRPCExecBatch(const JSONRPCRequest& jreq, const UniValue& vReq) * Process named arguments into a vector of positional arguments, based on the * passed-in specification for the RPC call's arguments. */ -static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, const std::vector<std::string>& argNames) +static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, const std::vector<std::pair<std::string, bool>>& argNames) { JSONRPCRequest out = in; out.params = UniValue(UniValue::VARR); @@ -417,7 +417,9 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c // "args" parameter, if present. int hole = 0; int initial_hole_size = 0; - for (const std::string &argNamePattern: argNames) { + const std::string* initial_param = nullptr; + UniValue options{UniValue::VOBJ}; + for (const auto& [argNamePattern, named_only]: argNames) { std::vector<std::string> vargNames = SplitString(argNamePattern, '|'); auto fr = argsIn.end(); for (const std::string & argName : vargNames) { @@ -426,7 +428,22 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c break; } } - if (fr != argsIn.end()) { + + // Handle named-only parameters by pushing them into a temporary options + // object, and then pushing the accumulated options as the next + // positional argument. + if (named_only) { + if (fr != argsIn.end()) { + if (options.exists(fr->first)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + fr->first + " specified multiple times"); + } + options.pushKVEnd(fr->first, *fr->second); + argsIn.erase(fr); + } + continue; + } + + if (!options.empty() || fr != argsIn.end()) { for (int i = 0; i < hole; ++i) { // Fill hole between specified parameters with JSON nulls, // but not at the end (for backwards compatibility with calls @@ -434,12 +451,26 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c out.params.push_back(UniValue()); } hole = 0; - out.params.push_back(*fr->second); - argsIn.erase(fr); + if (!initial_param) initial_param = &argNamePattern; } else { hole += 1; if (out.params.empty()) initial_hole_size = hole; } + + // If named input parameter "fr" is present, push it onto out.params. If + // options are present, push them onto out.params. If both are present, + // throw an error. + if (fr != argsIn.end()) { + if (!options.empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + fr->first + " conflicts with parameter " + options.getKeys().front()); + } + out.params.push_back(*fr->second); + argsIn.erase(fr); + } + if (!options.empty()) { + out.params.push_back(std::move(options)); + options = UniValue{UniValue::VOBJ}; + } } // If leftover "args" param was found, use it as a source of positional // arguments and add named arguments after. This is a convenience for @@ -447,9 +478,8 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c // arguments as described in doc/JSON-RPC-interface.md#parameter-passing auto positional_args{argsIn.extract("args")}; if (positional_args && positional_args.mapped()->isArray()) { - const bool has_named_arguments{initial_hole_size < (int)argNames.size()}; - if (initial_hole_size < (int)positional_args.mapped()->size() && has_named_arguments) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + argNames[initial_hole_size] + " specified twice both as positional and named argument"); + if (initial_hole_size < (int)positional_args.mapped()->size() && initial_param) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + *initial_param + " specified twice both as positional and named argument"); } // Assign positional_args to out.params and append named_args after. UniValue named_args{std::move(out.params)}; diff --git a/src/rpc/server.h b/src/rpc/server.h index 01e8556050..24658ddb8b 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -95,7 +95,7 @@ public: using Actor = std::function<bool(const JSONRPCRequest& request, UniValue& result, bool last_handler)>; //! Constructor taking Actor callback supporting multiple handlers. - CRPCCommand(std::string category, std::string name, Actor actor, std::vector<std::string> args, intptr_t unique_id) + CRPCCommand(std::string category, std::string name, Actor actor, std::vector<std::pair<std::string, bool>> args, intptr_t unique_id) : category(std::move(category)), name(std::move(name)), actor(std::move(actor)), argNames(std::move(args)), unique_id(unique_id) { @@ -115,7 +115,16 @@ public: std::string category; std::string name; Actor actor; - std::vector<std::string> argNames; + //! List of method arguments and whether they are named-only. Incoming RPC + //! requests contain a "params" field that can either be an array containing + //! unnamed arguments or an object containing named arguments. The + //! "argNames" vector is used in the latter case to transform the params + //! object into an array. Each argument in "argNames" gets mapped to a + //! unique position in the array, based on the order it is listed, unless + //! the argument is a named-only argument with argNames[x].second set to + //! true. Named-only arguments are combined into a JSON object that is + //! appended after other arguments, see transformNamedArguments for details. + std::vector<std::pair<std::string, bool>> argNames; intptr_t unique_id; }; diff --git a/src/rpc/server_util.cpp b/src/rpc/server_util.cpp index 13d007b496..1d4afb3758 100644 --- a/src/rpc/server_util.cpp +++ b/src/rpc/server_util.cpp @@ -11,7 +11,7 @@ #include <rpc/protocol.h> #include <rpc/request.h> #include <txmempool.h> -#include <util/system.h> +#include <util/any.h> #include <validation.h> #include <any> diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index d1ff3f3871..19e14f88df 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -389,7 +389,8 @@ struct Sections { case RPCArg::Type::NUM: case RPCArg::Type::AMOUNT: case RPCArg::Type::RANGE: - case RPCArg::Type::BOOL: { + case RPCArg::Type::BOOL: + case RPCArg::Type::OBJ_NAMED_PARAMS: { if (is_top_level_arg) return; // Nothing more to do for non-recursive types on first recursion auto left = indent; if (arg.m_opts.type_str.size() != 0 && push_name) { @@ -485,12 +486,32 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP m_results{std::move(results)}, m_examples{std::move(examples)} { - std::set<std::string> named_args; + // Map of parameter names and types just used to check whether the names are + // unique. Parameter names always need to be unique, with the exception that + // there can be pairs of POSITIONAL and NAMED parameters with the same name. + enum ParamType { POSITIONAL = 1, NAMED = 2, NAMED_ONLY = 4 }; + std::map<std::string, int> param_names; + for (const auto& arg : m_args) { std::vector<std::string> names = SplitString(arg.m_names, '|'); // Should have unique named arguments for (const std::string& name : names) { - CHECK_NONFATAL(named_args.insert(name).second); + auto& param_type = param_names[name]; + CHECK_NONFATAL(!(param_type & POSITIONAL)); + CHECK_NONFATAL(!(param_type & NAMED_ONLY)); + param_type |= POSITIONAL; + } + if (arg.m_type == RPCArg::Type::OBJ_NAMED_PARAMS) { + for (const auto& inner : arg.m_inner) { + std::vector<std::string> inner_names = SplitString(inner.m_names, '|'); + for (const std::string& inner_name : inner_names) { + auto& param_type = param_names[inner_name]; + CHECK_NONFATAL(!(param_type & POSITIONAL) || inner.m_opts.also_positional); + CHECK_NONFATAL(!(param_type & NAMED)); + CHECK_NONFATAL(!(param_type & NAMED_ONLY)); + param_type |= inner.m_opts.also_positional ? NAMED : NAMED_ONLY; + } + } } // Default value type should match argument type only when defined if (arg.m_fallback.index() == 2) { @@ -605,12 +626,17 @@ bool RPCHelpMan::IsValidNumArgs(size_t num_args) const return num_required_args <= num_args && num_args <= m_args.size(); } -std::vector<std::string> RPCHelpMan::GetArgNames() const +std::vector<std::pair<std::string, bool>> RPCHelpMan::GetArgNames() const { - std::vector<std::string> ret; + std::vector<std::pair<std::string, bool>> ret; ret.reserve(m_args.size()); for (const auto& arg : m_args) { - ret.emplace_back(arg.m_names); + if (arg.m_type == RPCArg::Type::OBJ_NAMED_PARAMS) { + for (const auto& inner : arg.m_inner) { + ret.emplace_back(inner.m_names, /*named_only=*/true); + } + } + ret.emplace_back(arg.m_names, /*named_only=*/false); } return ret; } @@ -642,20 +668,31 @@ std::string RPCHelpMan::ToString() const // Arguments Sections sections; + Sections named_only_sections; for (size_t i{0}; i < m_args.size(); ++i) { const auto& arg = m_args.at(i); if (arg.m_opts.hidden) break; // Any arg that follows is also hidden - if (i == 0) ret += "\nArguments:\n"; - // Push named argument name and description sections.m_sections.emplace_back(::ToString(i + 1) + ". " + arg.GetFirstName(), arg.ToDescriptionString(/*is_named_arg=*/true)); sections.m_max_pad = std::max(sections.m_max_pad, sections.m_sections.back().m_left.size()); // Recursively push nested args sections.Push(arg); + + // Push named-only argument sections + if (arg.m_type == RPCArg::Type::OBJ_NAMED_PARAMS) { + for (const auto& arg_inner : arg.m_inner) { + named_only_sections.PushSection({arg_inner.GetFirstName(), arg_inner.ToDescriptionString(/*is_named_arg=*/true)}); + named_only_sections.Push(arg_inner); + } + } } + + if (!sections.m_sections.empty()) ret += "\nArguments:\n"; ret += sections.ToString(); + if (!named_only_sections.m_sections.empty()) ret += "\nNamed Arguments:\n"; + ret += named_only_sections.ToString(); // Result ret += m_results.ToDescriptionString(); @@ -669,17 +706,30 @@ std::string RPCHelpMan::ToString() const UniValue RPCHelpMan::GetArgMap() const { UniValue arr{UniValue::VARR}; + + auto push_back_arg_info = [&arr](const std::string& rpc_name, int pos, const std::string& arg_name, const RPCArg::Type& type) { + UniValue map{UniValue::VARR}; + map.push_back(rpc_name); + map.push_back(pos); + map.push_back(arg_name); + map.push_back(type == RPCArg::Type::STR || + type == RPCArg::Type::STR_HEX); + arr.push_back(map); + }; + for (int i{0}; i < int(m_args.size()); ++i) { const auto& arg = m_args.at(i); std::vector<std::string> arg_names = SplitString(arg.m_names, '|'); for (const auto& arg_name : arg_names) { - UniValue map{UniValue::VARR}; - map.push_back(m_name); - map.push_back(i); - map.push_back(arg_name); - map.push_back(arg.m_type == RPCArg::Type::STR || - arg.m_type == RPCArg::Type::STR_HEX); - arr.push_back(map); + push_back_arg_info(m_name, i, arg_name, arg.m_type); + if (arg.m_type == RPCArg::Type::OBJ_NAMED_PARAMS) { + for (const auto& inner : arg.m_inner) { + std::vector<std::string> inner_names = SplitString(inner.m_names, '|'); + for (const std::string& inner_name : inner_names) { + push_back_arg_info(m_name, i, inner_name, inner.m_type); + } + } + } } } return arr; @@ -708,6 +758,7 @@ static std::optional<UniValue::VType> ExpectedType(RPCArg::Type type) return UniValue::VBOOL; } case Type::OBJ: + case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: { return UniValue::VOBJ; } @@ -781,6 +832,7 @@ std::string RPCArg::ToDescriptionString(bool is_named_arg) const break; } case Type::OBJ: + case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: { ret += "json object"; break; @@ -809,6 +861,7 @@ std::string RPCArg::ToDescriptionString(bool is_named_arg) const } // no default case, so the compiler can warn about missing cases } ret += ")"; + if (m_type == Type::OBJ_NAMED_PARAMS) ret += " Options object that can be used to pass named arguments, listed below."; ret += m_description.empty() ? "" : " " + m_description; return ret; } @@ -1054,6 +1107,7 @@ std::string RPCArg::ToStringObj(const bool oneline) const } return res + "...]"; case Type::OBJ: + case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: // Currently unused, so avoid writing dead code NONFATAL_UNREACHABLE(); @@ -1077,6 +1131,7 @@ std::string RPCArg::ToString(const bool oneline) const return GetFirstName(); } case Type::OBJ: + case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: { const std::string res = Join(m_inner, ",", [&](const RPCArg& i) { return i.ToStringObj(oneline); }); if (m_type == Type::OBJ) { diff --git a/src/rpc/util.h b/src/rpc/util.h index 3ff02582a6..4cba5a9818 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -130,6 +130,15 @@ struct RPCArgOptions { std::string oneline_description{}; //!< Should be empty unless it is supposed to override the auto-generated summary line std::vector<std::string> type_str{}; //!< Should be empty unless it is supposed to override the auto-generated type strings. Vector length is either 0 or 2, m_opts.type_str.at(0) will override the type of the value in a key-value pair, m_opts.type_str.at(1) will override the type in the argument description. bool hidden{false}; //!< For testing only + bool also_positional{false}; //!< If set allows a named-parameter field in an OBJ_NAMED_PARAM options object + //!< to have the same name as a top-level parameter. By default the RPC + //!< framework disallows this, because if an RPC request passes the value by + //!< name, it is assigned to top-level parameter position, not to the options + //!< position, defeating the purpose of using OBJ_NAMED_PARAMS instead OBJ for + //!< that option. But sometimes it makes sense to allow less-commonly used + //!< options to be passed by name only, and more commonly used options to be + //!< passed by name or position, so the RPC framework allows this as long as + //!< methods set the also_positional flag and read values from both positions. }; struct RPCArg { @@ -139,6 +148,13 @@ struct RPCArg { STR, NUM, BOOL, + OBJ_NAMED_PARAMS, //!< Special type that behaves almost exactly like + //!< OBJ, defining an options object with a list of + //!< pre-defined keys. The only difference between OBJ + //!< and OBJ_NAMED_PARAMS is that OBJ_NAMED_PARMS + //!< also allows the keys to be passed as top-level + //!< named parameters, as a more convenient way to pass + //!< options to the RPC method without nesting them. OBJ_USER_KEYS, //!< Special type where the user must set the keys e.g. to define multiple addresses; as opposed to e.g. an options object where the keys are predefined AMOUNT, //!< Special type representing a floating point amount (can be either NUM or STR) STR_HEX, //!< Special type that is a STR with only hex chars @@ -183,7 +199,7 @@ struct RPCArg { m_description{std::move(description)}, m_opts{std::move(opts)} { - CHECK_NONFATAL(type != Type::ARR && type != Type::OBJ && type != Type::OBJ_USER_KEYS); + CHECK_NONFATAL(type != Type::ARR && type != Type::OBJ && type != Type::OBJ_NAMED_PARAMS && type != Type::OBJ_USER_KEYS); } RPCArg( @@ -200,7 +216,7 @@ struct RPCArg { m_description{std::move(description)}, m_opts{std::move(opts)} { - CHECK_NONFATAL(type == Type::ARR || type == Type::OBJ || type == Type::OBJ_USER_KEYS); + CHECK_NONFATAL(type == Type::ARR || type == Type::OBJ || type == Type::OBJ_NAMED_PARAMS || type == Type::OBJ_USER_KEYS); } bool IsOptional() const; @@ -369,7 +385,8 @@ public: UniValue GetArgMap() const; /** If the supplied number of args is neither too small nor too high */ bool IsValidNumArgs(size_t num_args) const; - std::vector<std::string> GetArgNames() const; + //! Return list of arguments and whether they are named-only. + std::vector<std::pair<std::string, bool>> GetArgNames() const; const std::string m_name; diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 1c9aedc10b..6c6f644142 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -5,7 +5,6 @@ #include <scheduler.h> #include <sync.h> -#include <util/syscall_sandbox.h> #include <util/time.h> #include <cassert> @@ -23,7 +22,6 @@ CScheduler::~CScheduler() void CScheduler::serviceQueue() { - SetSyscallSandboxPolicy(SyscallSandboxPolicy::SCHEDULER); WAIT_LOCK(newTaskMutex, lock); ++nThreadsServicingQueue; diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index e52e8cd309..7c6c282cc4 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -5,11 +5,11 @@ #include <script/sigcache.h> +#include <common/system.h> #include <logging.h> #include <pubkey.h> #include <random.h> #include <uint256.h> -#include <util/system.h> #include <cuckoocache.h> diff --git a/src/secp256k1/.cirrus.yml b/src/secp256k1/.cirrus.yml index 0b904a4e38..5a00b65a33 100644 --- a/src/secp256k1/.cirrus.yml +++ b/src/secp256k1/.cirrus.yml @@ -21,6 +21,7 @@ env: ECDH: no RECOVERY: no SCHNORRSIG: no + ELLSWIFT: no ### test options SECP256K1_TEST_ITERS: BENCH: yes @@ -74,12 +75,12 @@ task: << : *LINUX_CONTAINER matrix: &ENV_MATRIX - env: {WIDEMUL: int64, RECOVERY: yes} - - env: {WIDEMUL: int64, ECDH: yes, SCHNORRSIG: yes} + - env: {WIDEMUL: int64, ECDH: yes, SCHNORRSIG: yes, ELLSWIFT: yes} - env: {WIDEMUL: int128} - - env: {WIDEMUL: int128_struct} - - env: {WIDEMUL: int128, RECOVERY: yes, SCHNORRSIG: yes} + - env: {WIDEMUL: int128_struct, ELLSWIFT: yes} + - env: {WIDEMUL: int128, RECOVERY: yes, SCHNORRSIG: yes, ELLSWIFT: yes} - env: {WIDEMUL: int128, ECDH: yes, SCHNORRSIG: yes} - - env: {WIDEMUL: int128, ASM: x86_64} + - env: {WIDEMUL: int128, ASM: x86_64 , ELLSWIFT: yes} - env: { RECOVERY: yes, SCHNORRSIG: yes} - env: {CTIMETESTS: no, RECOVERY: yes, ECDH: yes, SCHNORRSIG: yes, CPPFLAGS: -DVERIFY} - env: {BUILD: distcheck, WITH_VALGRIND: no, CTIMETESTS: no, BENCH: no} @@ -154,6 +155,7 @@ task: ECDH: yes RECOVERY: yes SCHNORRSIG: yes + ELLSWIFT: yes CTIMETESTS: no << : *MERGE_BASE test_script: @@ -173,10 +175,11 @@ task: ECDH: yes RECOVERY: yes SCHNORRSIG: yes + ELLSWIFT: yes CTIMETESTS: no matrix: - env: {} - - env: {EXPERIMENTAL: yes, ASM: arm} + - env: {EXPERIMENTAL: yes, ASM: arm32} << : *MERGE_BASE test_script: - ./ci/cirrus.sh @@ -193,6 +196,7 @@ task: ECDH: yes RECOVERY: yes SCHNORRSIG: yes + ELLSWIFT: yes CTIMETESTS: no << : *MERGE_BASE test_script: @@ -210,6 +214,7 @@ task: ECDH: yes RECOVERY: yes SCHNORRSIG: yes + ELLSWIFT: yes CTIMETESTS: no << : *MERGE_BASE test_script: @@ -247,6 +252,7 @@ task: RECOVERY: yes EXPERIMENTAL: yes SCHNORRSIG: yes + ELLSWIFT: yes CTIMETESTS: no # Use a MinGW-w64 host to tell ./configure we're building for Windows. # This will detect some MinGW-w64 tools but then make will need only @@ -286,6 +292,7 @@ task: ECDH: yes RECOVERY: yes SCHNORRSIG: yes + ELLSWIFT: yes CTIMETESTS: no matrix: - name: "Valgrind (memcheck)" @@ -361,6 +368,7 @@ task: ECDH: yes RECOVERY: yes SCHNORRSIG: yes + ELLSWIFT: yes << : *MERGE_BASE test_script: - ./ci/cirrus.sh @@ -397,13 +405,13 @@ task: - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL pull/$env:CIRRUS_PR/merge; git reset --hard FETCH_HEAD; } configure_script: - '%x64_NATIVE_TOOLS%' - - cmake -G "Visual Studio 17 2022" -A x64 -S . -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DSECP256K1_BUILD_EXAMPLES=ON + - cmake -E env CFLAGS="/WX" cmake -G "Visual Studio 17 2022" -A x64 -S . -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DSECP256K1_BUILD_EXAMPLES=ON build_script: - '%x64_NATIVE_TOOLS%' - cmake --build build --config RelWithDebInfo -- -property:UseMultiToolTask=true;CL_MPcount=5 check_script: - '%x64_NATIVE_TOOLS%' - - ctest --test-dir build -j 5 + - ctest -C RelWithDebInfo --test-dir build -j 5 - build\src\RelWithDebInfo\bench_ecmult.exe - build\src\RelWithDebInfo\bench_internal.exe - build\src\RelWithDebInfo\bench.exe diff --git a/src/secp256k1/.gitignore b/src/secp256k1/.gitignore index bc7e499de7..574902b8b5 100644 --- a/src/secp256k1/.gitignore +++ b/src/secp256k1/.gitignore @@ -59,5 +59,7 @@ build-aux/compile build-aux/test-driver libsecp256k1.pc +### CMake +/CMakeUserPresets.json # Default CMake build directory. /build diff --git a/src/secp256k1/CHANGELOG.md b/src/secp256k1/CHANGELOG.md index 6d23662a93..8e31edc6ee 100644 --- a/src/secp256k1/CHANGELOG.md +++ b/src/secp256k1/CHANGELOG.md @@ -7,6 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.2] - 2023-05-13 +We strongly recommend updating to 0.3.2 if you use or plan to use GCC >=13 to compile libsecp256k1. When in doubt, check the GCC version using `gcc -v`. + +#### Security + - Module `ecdh`: Fix "constant-timeness" issue with GCC 13.1 (and potentially future versions of GCC) that could leave applications using libsecp256k1's ECDH module vulnerable to a timing side-channel attack. The fix avoids secret-dependent control flow during ECDH computations when libsecp256k1 is compiled with GCC 13.1. + +#### Fixed + - Fixed an old bug that permitted compilers to potentially output bad assembly code on x86_64. In theory, it could lead to a crash or a read of unrelated memory, but this has never been observed on any compilers so far. + +#### Changed + - Various improvements and changes to CMake builds. CMake builds remain experimental. + - Made API versioning consistent with GNU Autotools builds. + - Switched to `BUILD_SHARED_LIBS` variable for controlling whether to build a static or a shared library. + - Added `SECP256K1_INSTALL` variable for the controlling whether to install the build artefacts. + - Renamed asm build option `arm` to `arm32`. Use `--with-asm=arm32` instead of `--with-asm=arm` (GNU Autotools), and `-DSECP256K1_ASM=arm32` instead of `-DSECP256K1_ASM=arm` (CMake). + +#### ABI Compatibility +The ABI is compatible with versions 0.3.0 and 0.3.1. + ## [0.3.1] - 2023-04-10 We strongly recommend updating to 0.3.1 if you use or plan to use Clang >=14 to compile libsecp256k1, e.g., Xcode >=14 on macOS has Clang >=14. When in doubt, check the Clang version using `clang -v`. @@ -68,7 +87,8 @@ This version was in fact never released. The number was given by the build system since the introduction of autotools in Jan 2014 (ea0fe5a5bf0c04f9cc955b2966b614f5f378c6f6). Therefore, this version number does not uniquely identify a set of source files. -[unreleased]: https://github.com/bitcoin-core/secp256k1/compare/v0.3.1...HEAD +[unreleased]: https://github.com/bitcoin-core/secp256k1/compare/v0.3.2...HEAD +[0.3.2]: https://github.com/bitcoin-core/secp256k1/compare/v0.3.1...v0.3.2 [0.3.1]: https://github.com/bitcoin-core/secp256k1/compare/v0.3.0...v0.3.1 [0.3.0]: https://github.com/bitcoin-core/secp256k1/compare/v0.2.0...v0.3.0 [0.2.0]: https://github.com/bitcoin-core/secp256k1/compare/423b6d19d373f1224fd671a982584d7e7900bc93..v0.2.0 diff --git a/src/secp256k1/CMakeLists.txt b/src/secp256k1/CMakeLists.txt index a70165e356..3107eb3bf1 100644 --- a/src/secp256k1/CMakeLists.txt +++ b/src/secp256k1/CMakeLists.txt @@ -1,16 +1,33 @@ cmake_minimum_required(VERSION 3.13) -if(CMAKE_VERSION VERSION_GREATER 3.14) +if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.15) # MSVC runtime library flags are selected by the CMAKE_MSVC_RUNTIME_LIBRARY abstraction. cmake_policy(SET CMP0091 NEW) # MSVC warning flags are not in CMAKE_<LANG>_FLAGS by default. cmake_policy(SET CMP0092 NEW) endif() -# The package (a.k.a. release) version is based on semantic versioning 2.0.0 of -# the API. All changes in experimental modules are treated as -# backwards-compatible and therefore at most increase the minor version. -project(libsecp256k1 VERSION 0.3.2 LANGUAGES C) +project(libsecp256k1 + # The package (a.k.a. release) version is based on semantic versioning 2.0.0 of + # the API. All changes in experimental modules are treated as + # backwards-compatible and therefore at most increase the minor version. + VERSION 0.3.3 + DESCRIPTION "Optimized C library for ECDSA signatures and secret/public key operations on curve secp256k1." + HOMEPAGE_URL "https://github.com/bitcoin-core/secp256k1" + LANGUAGES C +) + +if(CMAKE_VERSION VERSION_LESS 3.21) + get_directory_property(parent_directory PARENT_DIRECTORY) + if(parent_directory) + set(PROJECT_IS_TOP_LEVEL OFF CACHE INTERNAL "Emulates CMake 3.21+ behavior.") + set(${PROJECT_NAME}_IS_TOP_LEVEL OFF CACHE INTERNAL "Emulates CMake 3.21+ behavior.") + else() + set(PROJECT_IS_TOP_LEVEL ON CACHE INTERNAL "Emulates CMake 3.21+ behavior.") + set(${PROJECT_NAME}_IS_TOP_LEVEL ON CACHE INTERNAL "Emulates CMake 3.21+ behavior.") + endif() + unset(parent_directory) +endif() # The library version is based on libtool versioning of the ABI. The set of # rules for updating the version can be found here: @@ -18,7 +35,7 @@ project(libsecp256k1 VERSION 0.3.2 LANGUAGES C) # All changes in experimental modules are treated as if they don't affect the # interface and therefore only increase the revision. set(${PROJECT_NAME}_LIB_VERSION_CURRENT 2) -set(${PROJECT_NAME}_LIB_VERSION_REVISION 2) +set(${PROJECT_NAME}_LIB_VERSION_REVISION 3) set(${PROJECT_NAME}_LIB_VERSION_AGE 0) set(CMAKE_C_STANDARD 90) @@ -26,36 +43,42 @@ set(CMAKE_C_EXTENSIONS OFF) list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) -# We do not use CMake's BUILD_SHARED_LIBS option. -option(SECP256K1_BUILD_SHARED "Build shared library." ON) -option(SECP256K1_BUILD_STATIC "Build static library." ON) -if(NOT SECP256K1_BUILD_SHARED AND NOT SECP256K1_BUILD_STATIC) - message(FATAL_ERROR "At least one of SECP256K1_BUILD_SHARED and SECP256K1_BUILD_STATIC must be enabled.") +option(BUILD_SHARED_LIBS "Build shared libraries." ON) +option(SECP256K1_DISABLE_SHARED "Disable shared library. Overrides BUILD_SHARED_LIBS." OFF) +if(SECP256K1_DISABLE_SHARED) + set(BUILD_SHARED_LIBS OFF) endif() +option(SECP256K1_INSTALL "Enable installation." ${PROJECT_IS_TOP_LEVEL}) + option(SECP256K1_ENABLE_MODULE_ECDH "Enable ECDH module." ON) if(SECP256K1_ENABLE_MODULE_ECDH) - add_definitions(-DENABLE_MODULE_ECDH=1) + add_compile_definitions(ENABLE_MODULE_ECDH=1) endif() option(SECP256K1_ENABLE_MODULE_RECOVERY "Enable ECDSA pubkey recovery module." OFF) if(SECP256K1_ENABLE_MODULE_RECOVERY) - add_definitions(-DENABLE_MODULE_RECOVERY=1) + add_compile_definitions(ENABLE_MODULE_RECOVERY=1) endif() option(SECP256K1_ENABLE_MODULE_EXTRAKEYS "Enable extrakeys module." ON) option(SECP256K1_ENABLE_MODULE_SCHNORRSIG "Enable schnorrsig module." ON) if(SECP256K1_ENABLE_MODULE_SCHNORRSIG) set(SECP256K1_ENABLE_MODULE_EXTRAKEYS ON) - add_definitions(-DENABLE_MODULE_SCHNORRSIG=1) + add_compile_definitions(ENABLE_MODULE_SCHNORRSIG=1) endif() if(SECP256K1_ENABLE_MODULE_EXTRAKEYS) - add_definitions(-DENABLE_MODULE_EXTRAKEYS=1) + add_compile_definitions(ENABLE_MODULE_EXTRAKEYS=1) +endif() + +option(SECP256K1_ENABLE_MODULE_ELLSWIFT "Enable ElligatorSwift module." ON) +if(SECP256K1_ENABLE_MODULE_ELLSWIFT) + add_compile_definitions(ENABLE_MODULE_ELLSWIFT=1) endif() option(SECP256K1_USE_EXTERNAL_DEFAULT_CALLBACKS "Enable external default callback functions." OFF) if(SECP256K1_USE_EXTERNAL_DEFAULT_CALLBACKS) - add_definitions(-DUSE_EXTERNAL_DEFAULT_CALLBACKS=1) + add_compile_definitions(USE_EXTERNAL_DEFAULT_CALLBACKS=1) endif() set(SECP256K1_ECMULT_WINDOW_SIZE "AUTO" CACHE STRING "Window size for ecmult precomputation for verification, specified as integer in range [2..24]. \"AUTO\" is a reasonable setting for desktop machines (currently 15). [default=AUTO]") @@ -65,7 +88,7 @@ check_string_option_value(SECP256K1_ECMULT_WINDOW_SIZE) if(SECP256K1_ECMULT_WINDOW_SIZE STREQUAL "AUTO") set(SECP256K1_ECMULT_WINDOW_SIZE 15) endif() -add_definitions(-DECMULT_WINDOW_SIZE=${SECP256K1_ECMULT_WINDOW_SIZE}) +add_compile_definitions(ECMULT_WINDOW_SIZE=${SECP256K1_ECMULT_WINDOW_SIZE}) set(SECP256K1_ECMULT_GEN_PREC_BITS "AUTO" CACHE STRING "Precision bits to tune the precomputed table size for signing, specified as integer 2, 4 or 8. \"AUTO\" is a reasonable setting for desktop machines (currently 4). [default=AUTO]") set_property(CACHE SECP256K1_ECMULT_GEN_PREC_BITS PROPERTY STRINGS "AUTO" 2 4 8) @@ -73,29 +96,35 @@ check_string_option_value(SECP256K1_ECMULT_GEN_PREC_BITS) if(SECP256K1_ECMULT_GEN_PREC_BITS STREQUAL "AUTO") set(SECP256K1_ECMULT_GEN_PREC_BITS 4) endif() -add_definitions(-DECMULT_GEN_PREC_BITS=${SECP256K1_ECMULT_GEN_PREC_BITS}) +add_compile_definitions(ECMULT_GEN_PREC_BITS=${SECP256K1_ECMULT_GEN_PREC_BITS}) set(SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY "OFF" CACHE STRING "Test-only override of the (autodetected by the C code) \"widemul\" setting. Legal values are: \"OFF\", \"int128_struct\", \"int128\" or \"int64\". [default=OFF]") set_property(CACHE SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY PROPERTY STRINGS "OFF" "int128_struct" "int128" "int64") check_string_option_value(SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY) if(SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY) string(TOUPPER "${SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY}" widemul_upper_value) - add_definitions(-DUSE_FORCE_WIDEMUL_${widemul_upper_value}=1) + add_compile_definitions(USE_FORCE_WIDEMUL_${widemul_upper_value}=1) endif() mark_as_advanced(FORCE SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY) -set(SECP256K1_ASM "AUTO" CACHE STRING "Assembly optimizations to use: \"AUTO\", \"OFF\", \"x86_64\" or \"arm\" (experimental). [default=AUTO]") -set_property(CACHE SECP256K1_ASM PROPERTY STRINGS "AUTO" "OFF" "x86_64" "arm") +set(SECP256K1_ASM "AUTO" CACHE STRING "Assembly optimizations to use: \"AUTO\", \"OFF\", \"x86_64\" or \"arm32\" (experimental). [default=AUTO]") +set_property(CACHE SECP256K1_ASM PROPERTY STRINGS "AUTO" "OFF" "x86_64" "arm32") check_string_option_value(SECP256K1_ASM) -if(SECP256K1_ASM STREQUAL "arm") +if(SECP256K1_ASM STREQUAL "arm32") enable_language(ASM) - add_definitions(-DUSE_EXTERNAL_ASM=1) + include(CheckArm32Assembly) + check_arm32_assembly() + if(HAVE_ARM32_ASM) + add_compile_definitions(USE_EXTERNAL_ASM=1) + else() + message(FATAL_ERROR "ARM32 assembly optimization requested but not available.") + endif() elseif(SECP256K1_ASM) - include(Check64bitAssembly) - check_64bit_assembly() - if(HAS_64BIT_ASM) + include(CheckX86_64Assembly) + check_x86_64_assembly() + if(HAVE_X86_64_ASM) set(SECP256K1_ASM "x86_64") - add_definitions(-DUSE_ASM_X86_64=1) + add_compile_definitions(USE_ASM_X86_64=1) elseif(SECP256K1_ASM STREQUAL "AUTO") set(SECP256K1_ASM "OFF") else() @@ -105,8 +134,8 @@ endif() option(SECP256K1_EXPERIMENTAL "Allow experimental configuration options." OFF) if(NOT SECP256K1_EXPERIMENTAL) - if(SECP256K1_ASM STREQUAL "arm") - message(FATAL_ERROR "ARM assembly optimization is experimental. Use -DSECP256K1_EXPERIMENTAL=ON to allow.") + if(SECP256K1_ASM STREQUAL "arm32") + message(FATAL_ERROR "ARM32 assembly optimization is experimental. Use -DSECP256K1_EXPERIMENTAL=ON to allow.") endif() endif() @@ -118,7 +147,7 @@ if(SECP256K1_VALGRIND) if(Valgrind_FOUND) set(SECP256K1_VALGRIND ON) include_directories(${Valgrind_INCLUDE_DIR}) - add_definitions(-DVALGRIND) + add_compile_definitions(VALGRIND) elseif(SECP256K1_VALGRIND STREQUAL "AUTO") set(SECP256K1_VALGRIND OFF) else() @@ -165,42 +194,51 @@ mark_as_advanced( CMAKE_SHARED_LINKER_FLAGS_COVERAGE ) -if(CMAKE_CONFIGURATION_TYPES) - set(CMAKE_CONFIGURATION_TYPES "RelWithDebInfo" "Release" "Debug" "MinSizeRel" "Coverage") -endif() - -get_property(cached_cmake_build_type CACHE CMAKE_BUILD_TYPE PROPERTY TYPE) -if(cached_cmake_build_type) +get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) +set(default_build_type "RelWithDebInfo") +if(is_multi_config) + set(CMAKE_CONFIGURATION_TYPES "${default_build_type}" "Release" "Debug" "MinSizeRel" "Coverage" CACHE STRING + "Supported configuration types." + FORCE + ) +else() set_property(CACHE CMAKE_BUILD_TYPE PROPERTY - STRINGS "RelWithDebInfo" "Release" "Debug" "MinSizeRel" "Coverage" + STRINGS "${default_build_type}" "Release" "Debug" "MinSizeRel" "Coverage" ) + if(NOT CMAKE_BUILD_TYPE) + message(STATUS "Setting build type to \"${default_build_type}\" as none was specified") + set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE STRING + "Choose the type of build." + FORCE + ) + endif() endif() -set(default_build_type "RelWithDebInfo") -if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - message(STATUS "Setting build type to \"${default_build_type}\" as none was specified") - set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE STRING "Choose the type of build." FORCE) -endif() - -include(TryAddCompileOption) +include(TryAppendCFlags) if(MSVC) - try_add_compile_option(/W2) - try_add_compile_option(/wd4146) + # Keep the following commands ordered lexicographically. + try_append_c_flags(/W3) # Production quality warning level. + try_append_c_flags(/wd4146) # Disable warning C4146 "unary minus operator applied to unsigned type, result still unsigned". + try_append_c_flags(/wd4244) # Disable warning C4244 "'conversion' conversion from 'type1' to 'type2', possible loss of data". + try_append_c_flags(/wd4267) # Disable warning C4267 "'var' : conversion from 'size_t' to 'type', possible loss of data". + # Eliminate deprecation warnings for the older, less secure functions. + add_compile_definitions(_CRT_SECURE_NO_WARNINGS) else() - try_add_compile_option(-pedantic) - try_add_compile_option(-Wall) - try_add_compile_option(-Wcast-align) - try_add_compile_option(-Wcast-align=strict) - try_add_compile_option(-Wconditional-uninitialized) - try_add_compile_option(-Wextra) - try_add_compile_option(-Wnested-externs) - try_add_compile_option(-Wno-long-long) - try_add_compile_option(-Wno-overlength-strings) - try_add_compile_option(-Wno-unused-function) - try_add_compile_option(-Wreserved-identifier) - try_add_compile_option(-Wshadow) - try_add_compile_option(-Wstrict-prototypes) - try_add_compile_option(-Wundef) + # Keep the following commands ordered lexicographically. + try_append_c_flags(-pedantic) + try_append_c_flags(-Wall) # GCC >= 2.95 and probably many other compilers. + try_append_c_flags(-Wcast-align) # GCC >= 2.95. + try_append_c_flags(-Wcast-align=strict) # GCC >= 8.0. + try_append_c_flags(-Wconditional-uninitialized) # Clang >= 3.0 only. + try_append_c_flags(-Wextra) # GCC >= 3.4, this is the newer name of -W, which we don't use because older GCCs will warn about unused functions. + try_append_c_flags(-Wnested-externs) + try_append_c_flags(-Wno-long-long) # GCC >= 3.0, -Wlong-long is implied by -pedantic. + try_append_c_flags(-Wno-overlength-strings) # GCC >= 4.2, -Woverlength-strings is implied by -pedantic. + try_append_c_flags(-Wno-unused-function) # GCC >= 3.0, -Wunused-function is implied by -Wall. + try_append_c_flags(-Wreserved-identifier) # Clang >= 13.0 only. + try_append_c_flags(-Wshadow) + try_append_c_flags(-Wstrict-prototypes) + try_append_c_flags(-Wundef) endif() set(CMAKE_C_VISIBILITY_PRESET hidden) @@ -225,13 +263,19 @@ message("\n") message("secp256k1 configure summary") message("===========================") message("Build artifacts:") -message(" shared library ...................... ${SECP256K1_BUILD_SHARED}") -message(" static library ...................... ${SECP256K1_BUILD_STATIC}") +if(BUILD_SHARED_LIBS) + set(library_type "Shared") +else() + set(library_type "Static") +endif() + +message(" library type ........................ ${library_type}") message("Optional modules:") message(" ECDH ................................ ${SECP256K1_ENABLE_MODULE_ECDH}") message(" ECDSA pubkey recovery ............... ${SECP256K1_ENABLE_MODULE_RECOVERY}") message(" extrakeys ........................... ${SECP256K1_ENABLE_MODULE_EXTRAKEYS}") message(" schnorrsig .......................... ${SECP256K1_ENABLE_MODULE_SCHNORRSIG}") +message(" ElligatorSwift ...................... ${SECP256K1_ENABLE_MODULE_ELLSWIFT}") message("Parameters:") message(" ecmult window size .................. ${SECP256K1_ECMULT_WINDOW_SIZE}") message(" ecmult gen precision bits ........... ${SECP256K1_ECMULT_GEN_PREC_BITS}") @@ -268,7 +312,7 @@ message("CFLAGS ................................ ${CMAKE_C_FLAGS}") get_directory_property(compile_options COMPILE_OPTIONS) string(REPLACE ";" " " compile_options "${compile_options}") message("Compile options ....................... " ${compile_options}) -if(DEFINED CMAKE_BUILD_TYPE) +if(NOT is_multi_config) message("Build type:") message(" - CMAKE_BUILD_TYPE ................... ${CMAKE_BUILD_TYPE}") string(TOUPPER "${CMAKE_BUILD_TYPE}" build_type) @@ -276,7 +320,7 @@ if(DEFINED CMAKE_BUILD_TYPE) message(" - LDFLAGS for executables ............ ${CMAKE_EXE_LINKER_FLAGS_${build_type}}") message(" - LDFLAGS for shared libraries ....... ${CMAKE_SHARED_LINKER_FLAGS_${build_type}}") else() - message("Available configurations .............. ${CMAKE_CONFIGURATION_TYPES}") + message("Supported configurations .............. ${CMAKE_CONFIGURATION_TYPES}") message("RelWithDebInfo configuration:") message(" - CFLAGS ............................. ${CMAKE_C_FLAGS_RELWITHDEBINFO}") message(" - LDFLAGS for executables ............ ${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO}") diff --git a/src/secp256k1/CMakePresets.json b/src/secp256k1/CMakePresets.json new file mode 100644 index 0000000000..b35cd80579 --- /dev/null +++ b/src/secp256k1/CMakePresets.json @@ -0,0 +1,19 @@ +{ + "cmakeMinimumRequired": {"major": 3, "minor": 21, "patch": 0}, + "version": 3, + "configurePresets": [ + { + "name": "dev-mode", + "displayName": "Development mode (intended only for developers of the library)", + "cacheVariables": { + "SECP256K1_EXPERIMENTAL": "ON", + "SECP256K1_ENABLE_MODULE_RECOVERY": "ON", + "SECP256K1_BUILD_EXAMPLES": "ON" + }, + "warnings": { + "dev": true, + "uninitialized": true + } + } + ] +} diff --git a/src/secp256k1/Makefile.am b/src/secp256k1/Makefile.am index 36e26e3e8a..ee14ac4509 100644 --- a/src/secp256k1/Makefile.am +++ b/src/secp256k1/Makefile.am @@ -1,5 +1,3 @@ -.PHONY: clean-precomp precomp - ACLOCAL_AMFLAGS = -I build-aux/m4 # AM_CFLAGS will be automatically prepended to CFLAGS by Automake when compiling some foo @@ -65,6 +63,7 @@ noinst_HEADERS += src/hash_impl.h noinst_HEADERS += src/field.h noinst_HEADERS += src/field_impl.h noinst_HEADERS += src/bench.h +noinst_HEADERS += src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.h noinst_HEADERS += contrib/lax_der_parsing.h noinst_HEADERS += contrib/lax_der_parsing.c noinst_HEADERS += contrib/lax_der_privatekey_parsing.h @@ -190,11 +189,11 @@ EXTRA_PROGRAMS = precompute_ecmult precompute_ecmult_gen CLEANFILES = $(EXTRA_PROGRAMS) precompute_ecmult_SOURCES = src/precompute_ecmult.c -precompute_ecmult_CPPFLAGS = $(SECP_CONFIG_DEFINES) +precompute_ecmult_CPPFLAGS = $(SECP_CONFIG_DEFINES) -DVERIFY precompute_ecmult_LDADD = $(COMMON_LIB) precompute_ecmult_gen_SOURCES = src/precompute_ecmult_gen.c -precompute_ecmult_gen_CPPFLAGS = $(SECP_CONFIG_DEFINES) +precompute_ecmult_gen_CPPFLAGS = $(SECP_CONFIG_DEFINES) -DVERIFY precompute_ecmult_gen_LDADD = $(COMMON_LIB) # See Automake manual, Section "Errors with distclean". @@ -202,7 +201,7 @@ precompute_ecmult_gen_LDADD = $(COMMON_LIB) # otherwise make's decision whether to rebuild them (even in the first # build by a normal user) depends on mtimes, and thus is very fragile. # This means that rebuilds of the prebuilt files always need to be -# forced by deleting them, e.g., by invoking `make clean-precomp`. +# forced by deleting them. src/precomputed_ecmult.c: $(MAKE) $(AM_MAKEFLAGS) precompute_ecmult$(EXEEXT) ./precompute_ecmult$(EXEEXT) @@ -217,11 +216,29 @@ precomp: $(PRECOMP) # e.g., after `make maintainer-clean`). BUILT_SOURCES = $(PRECOMP) -maintainer-clean-local: clean-precomp - +.PHONY: clean-precomp clean-precomp: rm -f $(PRECOMP) +maintainer-clean-local: clean-precomp + +### Pregenerated test vectors +### (see the comments in the previous section for detailed rationale) +TESTVECTORS = src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.h + +src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.h: + mkdir -p $(@D) + python3 $(top_srcdir)/tools/tests_wycheproof_generate.py $(top_srcdir)/src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.json > $@ +testvectors: $(TESTVECTORS) + +BUILT_SOURCES += $(TESTVECTORS) + +.PHONY: clean-testvectors +clean-testvectors: + rm -f $(TESTVECTORS) +maintainer-clean-local: clean-testvectors + +### Additional files to distribute EXTRA_DIST = autogen.sh CHANGELOG.md SECURITY.md EXTRA_DIST += doc/release-process.md doc/safegcd_implementation.md EXTRA_DIST += examples/EXAMPLES_COPYING @@ -231,6 +248,9 @@ EXTRA_DIST += sage/group_prover.sage EXTRA_DIST += sage/prove_group_implementations.sage EXTRA_DIST += sage/secp256k1_params.sage EXTRA_DIST += sage/weierstrass_prover.sage +EXTRA_DIST += src/wycheproof/WYCHEPROOF_COPYING +EXTRA_DIST += src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.json +EXTRA_DIST += tools/tests_wycheproof_generate.py if ENABLE_MODULE_ECDH include src/modules/ecdh/Makefile.am.include @@ -248,19 +268,6 @@ if ENABLE_MODULE_SCHNORRSIG include src/modules/schnorrsig/Makefile.am.include endif -EXTRA_DIST += src/wycheproof/WYCHEPROOF_COPYING -EXTRA_DIST += src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.h -EXTRA_DIST += src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.json -EXTRA_DIST += tools/tests_wycheproof_generate.py - -TESTVECTORS = src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.h - -src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.h: - python3 tools/tests_wycheproof_generate.py src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.json > $@ - -testvectors: $(TESTVECTORS) - -maintainer-clean-testvectors: clean-testvectors - -clean-testvectors: - rm -f $(TESTVECTORS) +if ENABLE_MODULE_ELLSWIFT +include src/modules/ellswift/Makefile.am.include +endif diff --git a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 index 624f5e956e..11adef4f22 100644 --- a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 +++ b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 @@ -1,12 +1,31 @@ dnl escape "$0x" below using the m4 quadrigaph @S|@, and escape it again with a \ for the shell. -AC_DEFUN([SECP_64BIT_ASM_CHECK],[ +AC_DEFUN([SECP_X86_64_ASM_CHECK],[ AC_MSG_CHECKING(for x86_64 assembly availability) AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #include <stdint.h>]],[[ uint64_t a = 11, tmp; __asm__ __volatile__("movq \@S|@0x100000000,%1; mulq %%rsi" : "+a"(a) : "S"(tmp) : "cc", "%rdx"); - ]])],[has_64bit_asm=yes],[has_64bit_asm=no]) -AC_MSG_RESULT([$has_64bit_asm]) + ]])], [has_x86_64_asm=yes], [has_x86_64_asm=no]) +AC_MSG_RESULT([$has_x86_64_asm]) +]) + +AC_DEFUN([SECP_ARM32_ASM_CHECK], [ + AC_MSG_CHECKING(for ARM32 assembly availability) + SECP_ARM32_ASM_CHECK_CFLAGS_saved_CFLAGS="$CFLAGS" + CFLAGS="-x assembler" + AC_LINK_IFELSE([AC_LANG_SOURCE([[ + .syntax unified + .eabi_attribute 24, 1 + .eabi_attribute 25, 1 + .text + .global main + main: + ldr r0, =0x002A + mov r7, #1 + swi 0 + ]])], [has_arm32_asm=yes], [has_arm32_asm=no]) + AC_MSG_RESULT([$has_arm32_asm]) + CFLAGS="$SECP_ARM32_ASM_CHECK_CFLAGS_saved_CFLAGS" ]) AC_DEFUN([SECP_VALGRIND_CHECK],[ @@ -21,6 +40,7 @@ if test x"$has_valgrind" != x"yes"; then # error "Valgrind does not support this platform." #endif ]])], [has_valgrind=yes]) + CPPFLAGS="$CPPFLAGS_TEMP" fi AC_MSG_RESULT($has_valgrind) ]) diff --git a/src/secp256k1/ci/cirrus.sh b/src/secp256k1/ci/cirrus.sh index b2af03bb5d..8d82818611 100755 --- a/src/secp256k1/ci/cirrus.sh +++ b/src/secp256k1/ci/cirrus.sh @@ -36,8 +36,7 @@ case "$WRAPPER_CMD" in *wine*) # Make sure to shutdown wineserver whenever we exit. trap "wineserver -k || true" EXIT INT HUP - # This is apparently only reliable when we run a dummy command such as "hh.exe" afterwards. - wineserver -p && wine hh.exe + wineserver -p ;; esac @@ -62,6 +61,7 @@ fi --with-ecmult-window="$ECMULTWINDOW" \ --with-ecmult-gen-precision="$ECMULTGENPRECISION" \ --enable-module-ecdh="$ECDH" --enable-module-recovery="$RECOVERY" \ + --enable-module-ellswift="$ELLSWIFT" \ --enable-module-schnorrsig="$SCHNORRSIG" \ --enable-examples="$EXAMPLES" \ --enable-ctime-tests="$CTIMETESTS" \ diff --git a/src/secp256k1/ci/linux-debian.Dockerfile b/src/secp256k1/ci/linux-debian.Dockerfile index a83a4e36db..54eafcab25 100644 --- a/src/secp256k1/ci/linux-debian.Dockerfile +++ b/src/secp256k1/ci/linux-debian.Dockerfile @@ -29,9 +29,10 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ git clone https://github.com/mstorsjo/msvc-wine && \ mkdir /opt/msvc && \ python3 msvc-wine/vsdownload.py --accept-license --dest /opt/msvc Microsoft.VisualStudio.Workload.VCTools && \ - msvc-wine/install.sh /opt/msvc - -# Initialize the wine environment. Wait until the wineserver process has -# exited before closing the session, to avoid corrupting the wine prefix. -RUN wine64 wineboot --init && \ +# Since commit 2146cbfaf037e21de56c7157ec40bb6372860f51, the +# msvc-wine effectively initializes the wine prefix when running +# the install.sh script. + msvc-wine/install.sh /opt/msvc && \ +# Wait until the wineserver process has exited before closing the session, +# to avoid corrupting the wine prefix. while (ps -A | grep wineserver) > /dev/null; do sleep 1; done diff --git a/src/secp256k1/cmake/CheckArm32Assembly.cmake b/src/secp256k1/cmake/CheckArm32Assembly.cmake new file mode 100644 index 0000000000..15c44b24b0 --- /dev/null +++ b/src/secp256k1/cmake/CheckArm32Assembly.cmake @@ -0,0 +1,6 @@ +function(check_arm32_assembly) + try_compile(HAVE_ARM32_ASM + ${CMAKE_BINARY_DIR}/check_arm32_assembly + SOURCES ${CMAKE_SOURCE_DIR}/cmake/source_arm32.s + ) +endfunction() diff --git a/src/secp256k1/cmake/CheckStringOptionValue.cmake b/src/secp256k1/cmake/CheckStringOptionValue.cmake index bc4d7b5749..5a4d939b9e 100644 --- a/src/secp256k1/cmake/CheckStringOptionValue.cmake +++ b/src/secp256k1/cmake/CheckStringOptionValue.cmake @@ -1,11 +1,9 @@ function(check_string_option_value option) get_property(expected_values CACHE ${option} PROPERTY STRINGS) if(expected_values) - foreach(value IN LISTS expected_values) - if(value STREQUAL "${${option}}") - return() - endif() - endforeach() + if(${option} IN_LIST expected_values) + return() + endif() message(FATAL_ERROR "${option} value is \"${${option}}\", but must be one of ${expected_values}.") endif() message(AUTHOR_WARNING "The STRINGS property must be set before invoking `check_string_option_value' function.") diff --git a/src/secp256k1/cmake/Check64bitAssembly.cmake b/src/secp256k1/cmake/CheckX86_64Assembly.cmake index 3f65887765..ae82cd476e 100644 --- a/src/secp256k1/cmake/Check64bitAssembly.cmake +++ b/src/secp256k1/cmake/CheckX86_64Assembly.cmake @@ -1,6 +1,6 @@ include(CheckCSourceCompiles) -function(check_64bit_assembly) +function(check_x86_64_assembly) check_c_source_compiles(" #include <stdint.h> @@ -9,6 +9,6 @@ function(check_64bit_assembly) uint64_t a = 11, tmp; __asm__ __volatile__(\"movq $0x100000000,%1; mulq %%rsi\" : \"+a\"(a) : \"S\"(tmp) : \"cc\", \"%rdx\"); } - " HAS_64BIT_ASM) - set(HAS_64BIT_ASM ${HAS_64BIT_ASM} PARENT_SCOPE) + " HAVE_X86_64_ASM) + set(HAVE_X86_64_ASM ${HAVE_X86_64_ASM} PARENT_SCOPE) endfunction() diff --git a/src/secp256k1/cmake/FindValgrind.cmake b/src/secp256k1/cmake/FindValgrind.cmake index f6c1f58649..3af5e691e4 100644 --- a/src/secp256k1/cmake/FindValgrind.cmake +++ b/src/secp256k1/cmake/FindValgrind.cmake @@ -1,4 +1,4 @@ -if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") +if(CMAKE_HOST_APPLE) find_program(BREW_COMMAND brew) execute_process( COMMAND ${BREW_COMMAND} --prefix valgrind diff --git a/src/secp256k1/cmake/TryAddCompileOption.cmake b/src/secp256k1/cmake/TryAddCompileOption.cmake deleted file mode 100644 index f53c252c2d..0000000000 --- a/src/secp256k1/cmake/TryAddCompileOption.cmake +++ /dev/null @@ -1,23 +0,0 @@ -include(CheckCCompilerFlag) - -function(try_add_compile_option option) - string(MAKE_C_IDENTIFIER ${option} result) - string(TOUPPER ${result} result) - set(result "C_SUPPORTS${result}") - set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) - if(NOT MSVC) - set(CMAKE_REQUIRED_FLAGS "-Werror") - endif() - check_c_compiler_flag(${option} ${result}) - if(${result}) - get_property(compile_options - DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - PROPERTY COMPILE_OPTIONS - ) - list(APPEND compile_options "${option}") - set_property( - DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - PROPERTY COMPILE_OPTIONS "${compile_options}" - ) - endif() -endfunction() diff --git a/src/secp256k1/cmake/TryAppendCFlags.cmake b/src/secp256k1/cmake/TryAppendCFlags.cmake new file mode 100644 index 0000000000..1d81a9317a --- /dev/null +++ b/src/secp256k1/cmake/TryAppendCFlags.cmake @@ -0,0 +1,24 @@ +include(CheckCCompilerFlag) + +function(secp256k1_check_c_flags_internal flags output) + string(MAKE_C_IDENTIFIER "${flags}" result) + string(TOUPPER "${result}" result) + set(result "C_SUPPORTS_${result}") + if(NOT MSVC) + set(CMAKE_REQUIRED_FLAGS "-Werror") + endif() + + # This avoids running a linker. + set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + check_c_compiler_flag("${flags}" ${result}) + + set(${output} ${${result}} PARENT_SCOPE) +endfunction() + +# Append flags to the COMPILE_OPTIONS directory property if CC accepts them. +macro(try_append_c_flags) + secp256k1_check_c_flags_internal("${ARGV}" result) + if(result) + add_compile_options(${ARGV}) + endif() +endmacro() diff --git a/src/secp256k1/cmake/source_arm32.s b/src/secp256k1/cmake/source_arm32.s new file mode 100644 index 0000000000..d3d9347057 --- /dev/null +++ b/src/secp256k1/cmake/source_arm32.s @@ -0,0 +1,9 @@ +.syntax unified +.eabi_attribute 24, 1 +.eabi_attribute 25, 1 +.text +.global main +main: + ldr r0, =0x002A + mov r7, #1 + swi 0 diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index 0b555eac67..82cf95132d 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -5,7 +5,7 @@ AC_PREREQ([2.60]) # backwards-compatible and therefore at most increase the minor version. define(_PKG_VERSION_MAJOR, 0) define(_PKG_VERSION_MINOR, 3) -define(_PKG_VERSION_PATCH, 2) +define(_PKG_VERSION_PATCH, 3) define(_PKG_VERSION_IS_RELEASE, false) # The library version is based on libtool versioning of the ABI. The set of @@ -14,7 +14,7 @@ define(_PKG_VERSION_IS_RELEASE, false) # All changes in experimental modules are treated as if they don't affect the # interface and therefore only increase the revision. define(_LIB_VERSION_CURRENT, 2) -define(_LIB_VERSION_REVISION, 2) +define(_LIB_VERSION_REVISION, 3) define(_LIB_VERSION_AGE, 0) AC_INIT([libsecp256k1],m4_join([.], _PKG_VERSION_MAJOR, _PKG_VERSION_MINOR, _PKG_VERSION_PATCH)m4_if(_PKG_VERSION_IS_RELEASE, [true], [], [-dev]),[https://github.com/bitcoin-core/secp256k1/issues],[libsecp256k1],[https://github.com/bitcoin-core/secp256k1]) @@ -121,8 +121,12 @@ AC_DEFUN([SECP_TRY_APPEND_DEFAULT_CFLAGS], [ # libtool makes the same assumption internally. # Note that "/opt" and "-opt" are equivalent for MSVC; we use "-opt" because "/opt" looks like a path. if test x"$GCC" != x"yes" && test x"$build_windows" = x"yes"; then - SECP_TRY_APPEND_CFLAGS([-W2 -wd4146], $1) # Moderate warning level, disable warning C4146 "unary minus operator applied to unsigned type, result still unsigned" - SECP_TRY_APPEND_CFLAGS([-external:anglebrackets -external:W0], $1) # Suppress warnings from #include <...> files + SECP_TRY_APPEND_CFLAGS([-W3], $1) # Production quality warning level. + SECP_TRY_APPEND_CFLAGS([-wd4146], $1) # Disable warning C4146 "unary minus operator applied to unsigned type, result still unsigned". + SECP_TRY_APPEND_CFLAGS([-wd4244], $1) # Disable warning C4244 "'conversion' conversion from 'type1' to 'type2', possible loss of data". + SECP_TRY_APPEND_CFLAGS([-wd4267], $1) # Disable warning C4267 "'var' : conversion from 'size_t' to 'type', possible loss of data". + # Eliminate deprecation warnings for the older, less secure functions. + CPPFLAGS="-D_CRT_SECURE_NO_WARNINGS $CPPFLAGS" # We pass -ignore:4217 to the MSVC linker to suppress warning 4217 when # importing variables from a statically linked secp256k1. # (See the libtool manual, section "Windows DLLs" for background.) @@ -186,6 +190,10 @@ AC_ARG_ENABLE(module_schnorrsig, AS_HELP_STRING([--enable-module-schnorrsig],[enable schnorrsig module [default=yes]]), [], [SECP_SET_DEFAULT([enable_module_schnorrsig], [yes], [yes])]) +AC_ARG_ENABLE(module_ellswift, + AS_HELP_STRING([--enable-module-ellswift],[enable ElligatorSwift module [default=yes]]), [], + [SECP_SET_DEFAULT([enable_module_ellswift], [yes], [yes])]) + AC_ARG_ENABLE(external_default_callbacks, AS_HELP_STRING([--enable-external-default-callbacks],[enable external default callback functions [default=no]]), [], [SECP_SET_DEFAULT([enable_external_default_callbacks], [no], [no])]) @@ -198,8 +206,8 @@ AC_ARG_ENABLE(external_default_callbacks, # * and auto (the default). AC_ARG_WITH([test-override-wide-multiply], [] ,[set_widemul=$withval], [set_widemul=auto]) -AC_ARG_WITH([asm], [AS_HELP_STRING([--with-asm=x86_64|arm|no|auto], -[assembly optimizations to use (experimental: arm) [default=auto]])],[req_asm=$withval], [req_asm=auto]) +AC_ARG_WITH([asm], [AS_HELP_STRING([--with-asm=x86_64|arm32|no|auto], +[assembly optimizations to use (experimental: arm32) [default=auto]])],[req_asm=$withval], [req_asm=auto]) AC_ARG_WITH([ecmult-window], [AS_HELP_STRING([--with-ecmult-window=SIZE|auto], [window size for ecmult precomputation for verification, specified as integer in range [2..24].] @@ -264,8 +272,8 @@ else fi if test x"$req_asm" = x"auto"; then - SECP_64BIT_ASM_CHECK - if test x"$has_64bit_asm" = x"yes"; then + SECP_X86_64_ASM_CHECK + if test x"$has_x86_64_asm" = x"yes"; then set_asm=x86_64 fi if test x"$set_asm" = x; then @@ -275,12 +283,16 @@ else set_asm=$req_asm case $set_asm in x86_64) - SECP_64BIT_ASM_CHECK - if test x"$has_64bit_asm" != x"yes"; then + SECP_X86_64_ASM_CHECK + if test x"$has_x86_64_asm" != x"yes"; then AC_MSG_ERROR([x86_64 assembly optimization requested but not available]) fi ;; - arm) + arm32) + SECP_ARM32_ASM_CHECK + if test x"$has_arm32_asm" != x"yes"; then + AC_MSG_ERROR([ARM32 assembly optimization requested but not available]) + fi ;; no) ;; @@ -297,7 +309,7 @@ case $set_asm in x86_64) SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DUSE_ASM_X86_64=1" ;; -arm) +arm32) enable_external_asm=yes ;; no) @@ -394,6 +406,10 @@ if test x"$enable_module_schnorrsig" = x"yes"; then enable_module_extrakeys=yes fi +if test x"$enable_module_ellswift" = x"yes"; then + AC_DEFINE(ENABLE_MODULE_ELLSWIFT, 1, [Define this symbol to enable the ElligatorSwift module]) +fi + # Test if extrakeys is set after the schnorrsig module to allow the schnorrsig # module to set enable_module_extrakeys=yes if test x"$enable_module_extrakeys" = x"yes"; then @@ -414,8 +430,8 @@ if test x"$enable_experimental" = x"yes"; then AC_MSG_NOTICE([Experimental features do not have stable APIs or properties, and may not be safe for production use.]) AC_MSG_NOTICE([******]) else - if test x"$set_asm" = x"arm"; then - AC_MSG_ERROR([ARM assembly optimization is experimental. Use --enable-experimental to allow.]) + if test x"$set_asm" = x"arm32"; then + AC_MSG_ERROR([ARM32 assembly optimization is experimental. Use --enable-experimental to allow.]) fi fi @@ -436,8 +452,9 @@ AM_CONDITIONAL([ENABLE_MODULE_ECDH], [test x"$enable_module_ecdh" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_RECOVERY], [test x"$enable_module_recovery" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_EXTRAKEYS], [test x"$enable_module_extrakeys" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_SCHNORRSIG], [test x"$enable_module_schnorrsig" = x"yes"]) +AM_CONDITIONAL([ENABLE_MODULE_ELLSWIFT], [test x"$enable_module_ellswift" = x"yes"]) AM_CONDITIONAL([USE_EXTERNAL_ASM], [test x"$enable_external_asm" = x"yes"]) -AM_CONDITIONAL([USE_ASM_ARM], [test x"$set_asm" = x"arm"]) +AM_CONDITIONAL([USE_ASM_ARM], [test x"$set_asm" = x"arm32"]) AM_CONDITIONAL([BUILD_WINDOWS], [test "$build_windows" = "yes"]) AC_SUBST(LIB_VERSION_CURRENT, _LIB_VERSION_CURRENT) AC_SUBST(LIB_VERSION_REVISION, _LIB_VERSION_REVISION) @@ -457,6 +474,7 @@ echo " module ecdh = $enable_module_ecdh" echo " module recovery = $enable_module_recovery" echo " module extrakeys = $enable_module_extrakeys" echo " module schnorrsig = $enable_module_schnorrsig" +echo " module ellswift = $enable_module_ellswift" echo echo " asm = $set_asm" echo " ecmult window size = $set_ecmult_window" diff --git a/src/secp256k1/doc/ellswift.md b/src/secp256k1/doc/ellswift.md new file mode 100644 index 0000000000..7fbb7c1787 --- /dev/null +++ b/src/secp256k1/doc/ellswift.md @@ -0,0 +1,483 @@ +# ElligatorSwift for secp256k1 explained + +In this document we explain how the `ellswift` module implementation is related to the +construction in the +["SwiftEC: Shallue–van de Woestijne Indifferentiable Function To Elliptic Curves"](https://eprint.iacr.org/2022/759) +paper by Jorge Chávez-Saab, Francisco RodrÃguez-HenrÃquez, and Mehdi Tibouchi. + +* [1. Introduction](#1-introduction) +* [2. The decoding function](#2-the-decoding-function) + + [2.1 Decoding for `secp256k1`](#21-decoding-for-secp256k1) +* [3. The encoding function](#3-the-encoding-function) + + [3.1 Switching to *v, w* coordinates](#31-switching-to-v-w-coordinates) + + [3.2 Avoiding computing all inverses](#32-avoiding-computing-all-inverses) + + [3.3 Finding the inverse](#33-finding-the-inverse) + + [3.4 Dealing with special cases](#34-dealing-with-special-cases) + + [3.5 Encoding for `secp256k1`](#35-encoding-for-secp256k1) +* [4. Encoding and decoding full *(x, y)* coordinates](#4-encoding-and-decoding-full-x-y-coordinates) + + [4.1 Full *(x, y)* coordinates for `secp256k1`](#41-full-x-y-coordinates-for-secp256k1) + +## 1. Introduction + +The `ellswift` module effectively introduces a new 64-byte public key format, with the property +that (uniformly random) public keys can be encoded as 64-byte arrays which are computationally +indistinguishable from uniform byte arrays. The module provides functions to convert public keys +from and to this format, as well as convenience functions for key generation and ECDH that operate +directly on ellswift-encoded keys. + +The encoding consists of the concatenation of two (32-byte big endian) encoded field elements $u$ +and $t.$ Together they encode an x-coordinate on the curve $x$, or (see further) a full point $(x, y)$ on +the curve. + +**Decoding** consists of decoding the field elements $u$ and $t$ (values above the field size $p$ +are taken modulo $p$), and then evaluating $F_u(t)$, which for every $u$ and $t$ results in a valid +x-coordinate on the curve. The functions $F_u$ will be defined in [Section 2](#2-the-decoding-function). + +**Encoding** a given $x$ coordinate is conceptually done as follows: +* Loop: + * Pick a uniformly random field element $u.$ + * Compute the set $L = F_u^{-1}(x)$ of $t$ values for which $F_u(t) = x$, which may have up to *8* elements. + * With probability $1 - \dfrac{\\#L}{8}$, restart the loop. + * Select a uniformly random $t \in L$ and return $(u, t).$ + +This is the *ElligatorSwift* algorithm, here given for just x-coordinates. An extension to full +$(x, y)$ points will be given in [Section 4](#4-encoding-and-decoding-full-x-y-coordinates). +The algorithm finds a uniformly random $(u, t)$ among (almost all) those +for which $F_u(t) = x.$ Section 3.2 in the paper proves that the number of such encodings for +almost all x-coordinates on the curve (all but at most 39) is close to two times the field size +(specifically, it lies in the range $2q \pm (22\sqrt{q} + O(1))$, where $q$ is the size of the field). + +## 2. The decoding function + +First some definitions: +* $\mathbb{F}$ is the finite field of size $q$, of characteristic 5 or more, and $q \equiv 1 \mod 3.$ + * For `secp256k1`, $q = 2^{256} - 2^{32} - 977$, which satisfies that requirement. +* Let $E$ be the elliptic curve of points $(x, y) \in \mathbb{F}^2$ for which $y^2 = x^3 + ax + b$, with $a$ and $b$ + public constants, for which $\Delta_E = -16(4a^3 + 27b^2)$ is a square, and at least one of $(-b \pm \sqrt{-3 \Delta_E} / 36)/2$ is a square. + This implies that the order of $E$ is either odd, or a multiple of *4*. + If $a=0$, this condition is always fulfilled. + * For `secp256k1`, $a=0$ and $b=7.$ +* Let the function $g(x) = x^3 + ax + b$, so the $E$ curve equation is also $y^2 = g(x).$ +* Let the function $h(x) = 3x^3 + 4a.$ +* Define $V$ as the set of solutions $(x_1, x_2, x_3, z)$ to $z^2 = g(x_1)g(x_2)g(x_3).$ +* Define $S_u$ as the set of solutions $(X, Y)$ to $X^2 + h(u)Y^2 = -g(u)$ and $Y \neq 0.$ +* $P_u$ is a function from $\mathbb{F}$ to $S_u$ that will be defined below. +* $\psi_u$ is a function from $S_u$ to $V$ that will be defined below. + +**Note**: In the paper: +* $F_u$ corresponds to $F_{0,u}$ there. +* $P_u(t)$ is called $P$ there. +* All $S_u$ sets together correspond to $S$ there. +* All $\psi_u$ functions together (operating on elements of $S$) correspond to $\psi$ there. + +Note that for $V$, the left hand side of the equation $z^2$ is square, and thus the right +hand must also be square. As multiplying non-squares results in a square in $\mathbb{F}$, +out of the three right-hand side factors an even number must be non-squares. +This implies that exactly *1* or exactly *3* out of +$\\{g(x_1), g(x_2), g(x_3)\\}$ must be square, and thus that for any $(x_1,x_2,x_3,z) \in V$, +at least one of $\\{x_1, x_2, x_3\\}$ must be a valid x-coordinate on $E.$ There is one exception +to this, namely when $z=0$, but even then one of the three values is a valid x-coordinate. + +**Define** the decoding function $F_u(t)$ as: +* Let $(x_1, x_2, x_3, z) = \psi_u(P_u(t)).$ +* Return the first element $x$ of $(x_3, x_2, x_1)$ which is a valid x-coordinate on $E$ (i.e., $g(x)$ is square). + +$P_u(t) = (X(u, t), Y(u, t))$, where: + +$$ +\begin{array}{lcl} +X(u, t) & = & \left\\{\begin{array}{ll} + \dfrac{g(u) - t^2}{2t} & a = 0 \\ + \dfrac{g(u) + h(u)(Y_0(u) + X_0(u)t)^2}{X_0(u)(1 + h(u)t^2)} & a \neq 0 +\end{array}\right. \\ +Y(u, t) & = & \left\\{\begin{array}{ll} + \dfrac{X(u, t) + t}{u \sqrt{-3}} = \dfrac{g(u) + t^2}{2tu\sqrt{-3}} & a = 0 \\ + Y_0(u) + t(X(u, t) - X_0(u)) & a \neq 0 +\end{array}\right. +\end{array} +$$ + +$P_u(t)$ is defined: +* For $a=0$, unless: + * $u = 0$ or $t = 0$ (division by zero) + * $g(u) = -t^2$ (would give $Y=0$). +* For $a \neq 0$, unless: + * $X_0(u) = 0$ or $h(u)t^2 = -1$ (division by zero) + * $Y_0(u) (1 - h(u)t^2) = 2X_0(u)t$ (would give $Y=0$). + +The functions $X_0(u)$ and $Y_0(u)$ are defined in Appendix A of the paper, and depend on various properties of $E.$ + +The function $\psi_u$ is the same for all curves: $\psi_u(X, Y) = (x_1, x_2, x_3, z)$, where: + +$$ +\begin{array}{lcl} + x_1 & = & \dfrac{X}{2Y} - \dfrac{u}{2} && \\ + x_2 & = & -\dfrac{X}{2Y} - \dfrac{u}{2} && \\ + x_3 & = & u + 4Y^2 && \\ + z & = & \dfrac{g(x_3)}{2Y}(u^2 + ux_1 + x_1^2 + a) = \dfrac{-g(u)g(x_3)}{8Y^3} +\end{array} +$$ + +### 2.1 Decoding for `secp256k1` + +Put together and specialized for $a=0$ curves, decoding $(u, t)$ to an x-coordinate is: + +**Define** $F_u(t)$ as: +* Let $X = \dfrac{u^3 + b - t^2}{2t}.$ +* Let $Y = \dfrac{X + t}{u\sqrt{-3}}.$ +* Return the first $x$ in $(u + 4Y^2, \dfrac{-X}{2Y} - \dfrac{u}{2}, \dfrac{X}{2Y} - \dfrac{u}{2})$ for which $g(x)$ is square. + +To make sure that every input decodes to a valid x-coordinate, we remap the inputs in case +$P_u$ is not defined (when $u=0$, $t=0$, or $g(u) = -t^2$): + +**Define** $F_u(t)$ as: +* Let $u'=u$ if $u \neq 0$; $1$ otherwise (guaranteeing $u' \neq 0$). +* Let $t'=t$ if $t \neq 0$; $1$ otherwise (guaranteeing $t' \neq 0$). +* Let $t''=t'$ if $g(u') \neq -t'^2$; $2t'$ otherwise (guaranteeing $t'' \neq 0$ and $g(u') \neq -t''^2$). +* Let $X = \dfrac{u'^3 + b - t''^2}{2t''}.$ +* Let $Y = \dfrac{X + t''}{u'\sqrt{-3}}.$ +* Return the first $x$ in $(u' + 4Y^2, \dfrac{-X}{2Y} - \dfrac{u'}{2}, \dfrac{X}{2Y} - \dfrac{u'}{2})$ for which $x^3 + b$ is square. + +The choices here are not strictly necessary. Just returning a fixed constant in any of the undefined cases would suffice, +but the approach here is simple enough and gives fairly uniform output even in these cases. + +**Note**: in the paper these conditions result in $\infty$ as output, due to the use of projective coordinates there. +We wish to avoid the need for callers to deal with this special case. + +This is implemented in `secp256k1_ellswift_xswiftec_frac_var` (which decodes to an x-coordinate represented as a fraction), and +in `secp256k1_ellswift_xswiftec_var` (which outputs the actual x-coordinate). + +## 3. The encoding function + +To implement $F_u^{-1}(x)$, the function to find the set of inverses $t$ for which $F_u(t) = x$, we have to reverse the process: +* Find all the $(X, Y) \in S_u$ that could have given rise to $x$, through the $x_1$, $x_2$, or $x_3$ formulas in $\psi_u.$ +* Map those $(X, Y)$ solutions to $t$ values using $P_u^{-1}(X, Y).$ +* For each of the found $t$ values, verify that $F_u(t) = x.$ +* Return the remaining $t$ values. + +The function $P_u^{-1}$, which finds $t$ given $(X, Y) \in S_u$, is significantly simpler than $P_u:$ + +$$ +P_u^{-1}(X, Y) = \left\\{\begin{array}{ll} +Yu\sqrt{-3} - X & a = 0 \\ +\dfrac{Y-Y_0(u)}{X-X_0(u)} & a \neq 0 \land X \neq X_0(u) \\ +\dfrac{-X_0(u)}{h(u)Y_0(u)} & a \neq 0 \land X = X_0(u) \land Y = Y_0(u) +\end{array}\right. +$$ + +The third step above, verifying that $F_u(t) = x$, is necessary because for the $(X, Y)$ values found through the $x_1$ and $x_2$ expressions, +it is possible that decoding through $\psi_u(X, Y)$ yields a valid $x_3$ on the curve, which would take precedence over the +$x_1$ or $x_2$ decoding. These $(X, Y)$ solutions must be rejected. + +Since we know that exactly one or exactly three out of $\\{x_1, x_2, x_3\\}$ are valid x-coordinates for any $t$, +the case where either $x_1$ or $x_2$ is valid and in addition also $x_3$ is valid must mean that all three are valid. +This means that instead of checking whether $x_3$ is on the curve, it is also possible to check whether the other one out of +$x_1$ and $x_2$ is on the curve. This is significantly simpler, as it turns out. + +Observe that $\psi_u$ guarantees that $x_1 + x_2 = -u.$ So given either $x = x_1$ or $x = x_2$, the other one of the two can be computed as +$-u - x.$ Thus, when encoding $x$ through the $x_1$ or $x_2$ expressions, one can simply check whether $g(-u-x)$ is a square, +and if so, not include the corresponding $t$ values in the returned set. As this does not need $X$, $Y$, or $t$, this condition can be determined +before those values are computed. + +It is not possible that an encoding found through the $x_1$ expression decodes to a different valid x-coordinate using $x_2$ (which would +take precedence), for the same reason: if both $x_1$ and $x_2$ decodings were valid, $x_3$ would be valid as well, and thus take +precedence over both. Because of this, the $g(-u-x)$ being square test for $x_1$ and $x_2$ is the only test necessary to guarantee the found $t$ +values round-trip back to the input $x$ correctly. This is the reason for choosing the $(x_3, x_2, x_1)$ precedence order in the decoder; +any order which does not place $x_3$ first requires more complicated round-trip checks in the encoder. + +### 3.1 Switching to *v, w* coordinates + +Before working out the formulas for all this, we switch to different variables for $S_u.$ Let $v = (X/Y - u)/2$, and +$w = 2Y.$ Or in the other direction, $X = w(u/2 + v)$ and $Y = w/2:$ +* $S_u'$ becomes the set of $(v, w)$ for which $w^2 (u^2 + uv + v^2 + a) = -g(u)$ and $w \neq 0.$ +* For $a=0$ curves, $P_u^{-1}$ can be stated for $(v,w)$ as $P_u^{'-1}(v, w) = w\left(\frac{\sqrt{-3}-1}{2}u - v\right).$ +* $\psi_u$ can be stated for $(v, w)$ as $\psi_u'(v, w) = (x_1, x_2, x_3, z)$, where + +$$ +\begin{array}{lcl} + x_1 & = & v \\ + x_2 & = & -u - v \\ + x_3 & = & u + w^2 \\ + z & = & \dfrac{g(x_3)}{w}(u^2 + uv + v^2 + a) = \dfrac{-g(u)g(x_3)}{w^3} +\end{array} +$$ + +We can now write the expressions for finding $(v, w)$ given $x$ explicitly, by solving each of the $\\{x_1, x_2, x_3\\}$ +expressions for $v$ or $w$, and using the $S_u'$ equation to find the other variable: +* Assuming $x = x_1$, we find $v = x$ and $w = \pm\sqrt{-g(u)/(u^2 + uv + v^2 + a)}$ (two solutions). +* Assuming $x = x_2$, we find $v = -u-x$ and $w = \pm\sqrt{-g(u)/(u^2 + uv + v^2 + a)}$ (two solutions). +* Assuming $x = x_3$, we find $w = \pm\sqrt{x-u}$ and $v = -u/2 \pm \sqrt{-w^2(4g(u) + w^2h(u))}/(2w^2)$ (four solutions). + +### 3.2 Avoiding computing all inverses + +The *ElligatorSwift* algorithm as stated in Section 1 requires the computation of $L = F_u^{-1}(x)$ (the +set of all $t$ such that $(u, t)$ decode to $x$) in full. This is unnecessary. + +Observe that the procedure of restarting with probability $(1 - \frac{\\#L}{8})$ and otherwise returning a +uniformly random element from $L$ is actually equivalent to always padding $L$ with $\bot$ values up to length 8, +picking a uniformly random element from that, restarting whenever $\bot$ is picked: + +**Define** *ElligatorSwift(x)* as: +* Loop: + * Pick a uniformly random field element $u.$ + * Compute the set $L = F_u^{-1}(x).$ + * Let $T$ be the 8-element vector consisting of the elements of $L$, plus $8 - \\#L$ times $\\{\bot\\}.$ + * Select a uniformly random $t \in T.$ + * If $t \neq \bot$, return $(u, t)$; restart loop otherwise. + +Now notice that the order of elements in $T$ does not matter, as all we do is pick a uniformly +random element in it, so we do not need to have all $\bot$ values at the end. +As we have 8 distinct formulas for finding $(v, w)$ (taking the variants due to $\pm$ into account), +we can associate every index in $T$ with exactly one of those formulas, making sure that: +* Formulas that yield no solutions (due to division by zero or non-existing square roots) or invalid solutions are made to return $\bot.$ +* For the $x_1$ and $x_2$ cases, if $g(-u-x)$ is a square, $\bot$ is returned instead (the round-trip check). +* In case multiple formulas would return the same non- $\bot$ result, all but one of those must be turned into $\bot$ to avoid biasing those. + +The last condition above only occurs with negligible probability for cryptographically-sized curves, but is interesting +to take into account as it allows exhaustive testing in small groups. See [Section 3.4](#34-dealing-with-special-cases) +for an analysis of all the negligible cases. + +If we define $T = (G_{0,u}(x), G_{1,u}(x), \ldots, G_{7,u}(x))$, with each $G_{i,u}$ matching one of the formulas, +the loop can be simplified to only compute one of the inverses instead of all of them: + +**Define** *ElligatorSwift(x)* as: +* Loop: + * Pick a uniformly random field element $u.$ + * Pick a uniformly random integer $c$ in $[0,8).$ + * Let $t = G_{c,u}(x).$ + * If $t \neq \bot$, return $(u, t)$; restart loop otherwise. + +This is implemented in `secp256k1_ellswift_xelligatorswift_var`. + +### 3.3 Finding the inverse + +To implement $G_{c,u}$, we map $c=0$ to the $x_1$ formula, $c=1$ to the $x_2$ formula, and $c=2$ and $c=3$ to the $x_3$ formula. +Those are then repeated as $c=4$ through $c=7$ for the other sign of $w$ (noting that in each formula, $w$ is a square root of some expression). +Ignoring the negligible cases, we get: + +**Define** $G_{c,u}(x)$ as: +* If $c \in \\{0, 1, 4, 5\\}$ (for $x_1$ and $x_2$ formulas): + * If $g(-u-x)$ is square, return $\bot$ (as $x_3$ would be valid and take precedence). + * If $c \in \\{0, 4\\}$ (the $x_1$ formula) let $v = x$, otherwise let $v = -u-x$ (the $x_2$ formula) + * Let $s = -g(u)/(u^2 + uv + v^2 + a)$ (using $s = w^2$ in what follows). +* Otherwise, when $c \in \\{2, 3, 6, 7\\}$ (for $x_3$ formulas): + * Let $s = x-u.$ + * Let $r = \sqrt{-s(4g(u) + sh(u))}.$ + * Let $v = (r/s - u)/2$ if $c \in \\{3, 7\\}$; $(-r/s - u)/2$ otherwise. +* Let $w = \sqrt{s}.$ +* Depending on $c:$ + * If $c \in \\{0, 1, 2, 3\\}:$ return $P_u^{'-1}(v, w).$ + * If $c \in \\{4, 5, 6, 7\\}:$ return $P_u^{'-1}(v, -w).$ + +Whenever a square root of a non-square is taken, $\bot$ is returned; for both square roots this happens with roughly +50% on random inputs. Similarly, when a division by 0 would occur, $\bot$ is returned as well; this will only happen +with negligible probability. A division by 0 in the first branch in fact cannot occur at all, because $u^2 + uv + v^2 + a = 0$ +implies $g(-u-x) = g(x)$ which would mean the $g(-u-x)$ is square condition has triggered +and $\bot$ would have been returned already. + +**Note**: In the paper, the $case$ variable corresponds roughly to the $c$ above, but only takes on 4 possible values (1 to 4). +The conditional negation of $w$ at the end is done randomly, which is equivalent, but makes testing harder. We choose to +have the $G_{c,u}$ be deterministic, and capture all choices in $c.$ + +Now observe that the $c \in \\{1, 5\\}$ and $c \in \\{3, 7\\}$ conditions effectively perform the same $v \rightarrow -u-v$ +transformation. Furthermore, that transformation has no effect on $s$ in the first branch +as $u^2 + ux + x^2 + a = u^2 + u(-u-x) + (-u-x)^2 + a.$ Thus we can extract it out and move it down: + +**Define** $G_{c,u}(x)$ as: +* If $c \in \\{0, 1, 4, 5\\}:$ + * If $g(-u-x)$ is square, return $\bot.$ + * Let $s = -g(u)/(u^2 + ux + x^2 + a).$ + * Let $v = x.$ +* Otherwise, when $c \in \\{2, 3, 6, 7\\}:$ + * Let $s = x-u.$ + * Let $r = \sqrt{-s(4g(u) + sh(u))}.$ + * Let $v = (r/s - u)/2.$ +* Let $w = \sqrt{s}.$ +* Depending on $c:$ + * If $c \in \\{0, 2\\}:$ return $P_u^{'-1}(v, w).$ + * If $c \in \\{1, 3\\}:$ return $P_u^{'-1}(-u-v, w).$ + * If $c \in \\{4, 6\\}:$ return $P_u^{'-1}(v, -w).$ + * If $c \in \\{5, 7\\}:$ return $P_u^{'-1}(-u-v, -w).$ + +This shows there will always be exactly 0, 4, or 8 $t$ values for a given $(u, x)$ input. +There can be 0, 1, or 2 $(v, w)$ pairs before invoking $P_u^{'-1}$, and each results in 4 distinct $t$ values. + +### 3.4 Dealing with special cases + +As mentioned before there are a few cases to deal with which only happen in a negligibly small subset of inputs. +For cryptographically sized fields, if only random inputs are going to be considered, it is unnecessary to deal with these. Still, for completeness +we analyse them here. They generally fall into two categories: cases in which the encoder would produce $t$ values that +do not decode back to $x$ (or at least cannot guarantee that they do), and cases in which the encoder might produce the same +$t$ value for multiple $c$ inputs (thereby biasing that encoding): + +* In the branch for $x_1$ and $x_2$ (where $c \in \\{0, 1, 4, 5\\}$): + * When $g(u) = 0$, we would have $s=w=Y=0$, which is not on $S_u.$ This is only possible on even-ordered curves. + Excluding this also removes the one condition under which the simplified check for $x_3$ on the curve + fails (namely when $g(x_1)=g(x_2)=0$ but $g(x_3)$ is not square). + This does exclude some valid encodings: when both $g(u)=0$ and $u^2+ux+x^2+a=0$ (also implying $g(x)=0$), + the $S_u'$ equation degenerates to $0 = 0$, and many valid $t$ values may exist. Yet, these cannot be targeted uniformly by the + encoder anyway as there will generally be more than 8. + * When $g(x) = 0$, the same $t$ would be produced as in the $x_3$ branch (where $c \in \\{2, 3, 6, 7\\}$) which we give precedence + as it can deal with $g(u)=0$. + This is again only possible on even-ordered curves. +* In the branch for $x_3$ (where $c \in \\{2, 3, 6, 7\\}$): + * When $s=0$, a division by zero would occur. + * When $v = -u-v$ and $c \in \\{3, 7\\}$, the same $t$ would be returned as in the $c \in \\{2, 6\\}$ cases. + It is equivalent to checking whether $r=0$. + This cannot occur in the $x_1$ or $x_2$ branches, as it would trigger the $g(-u-x)$ is square condition. + A similar concern for $w = -w$ does not exist, as $w=0$ is already impossible in both branches: in the first + it requires $g(u)=0$ which is already outlawed on even-ordered curves and impossible on others; in the second it would trigger division by zero. +* Curve-specific special cases also exist that need to be rejected, because they result in $(u,t)$ which is invalid to the decoder, or because of division by zero in the encoder: + * For $a=0$ curves, when $u=0$ or when $t=0$. The latter can only be reached by the encoder when $g(u)=0$, which requires an even-ordered curve. + * For $a \neq 0$ curves, when $X_0(u)=0$, when $h(u)t^2 = -1$, or when $2w(u + 2v) = 2X_0(u)$ while also either $w \neq 2Y_0(u)$ or $h(u)=0$. + +**Define** a version of $G_{c,u}(x)$ which deals with all these cases: +* If $a=0$ and $u=0$, return $\bot.$ +* If $a \neq 0$ and $X_0(u)=0$, return $\bot.$ +* If $c \in \\{0, 1, 4, 5\\}:$ + * If $g(u) = 0$ or $g(x) = 0$, return $\bot$ (even curves only). + * If $g(-u-x)$ is square, return $\bot.$ + * Let $s = -g(u)/(u^2 + ux + x^2 + a)$ (cannot cause division by zero). + * Let $v = x.$ +* Otherwise, when $c \in \\{2, 3, 6, 7\\}:$ + * Let $s = x-u.$ + * Let $r = \sqrt{-s(4g(u) + sh(u))}$; return $\bot$ if not square. + * If $c \in \\{3, 7\\}$ and $r=0$, return $\bot.$ + * If $s = 0$, return $\bot.$ + * Let $v = (r/s - u)/2.$ +* Let $w = \sqrt{s}$; return $\bot$ if not square. +* If $a \neq 0$ and $w(u+2v) = 2X_0(u)$ and either $w \neq 2Y_0(u)$ or $h(u) = 0$, return $\bot.$ +* Depending on $c:$ + * If $c \in \\{0, 2\\}$, let $t = P_u^{'-1}(v, w).$ + * If $c \in \\{1, 3\\}$, let $t = P_u^{'-1}(-u-v, w).$ + * If $c \in \\{4, 6\\}$, let $t = P_u^{'-1}(v, -w).$ + * If $c \in \\{5, 7\\}$, let $t = P_u^{'-1}(-u-v, -w).$ +* If $a=0$ and $t=0$, return $\bot$ (even curves only). +* If $a \neq 0$ and $h(u)t^2 = -1$, return $\bot.$ +* Return $t.$ + +Given any $u$, using this algorithm over all $x$ and $c$ values, every $t$ value will be reached exactly once, +for an $x$ for which $F_u(t) = x$ holds, except for these cases that will not be reached: +* All cases where $P_u(t)$ is not defined: + * For $a=0$ curves, when $u=0$, $t=0$, or $g(u) = -t^2.$ + * For $a \neq 0$ curves, when $h(u)t^2 = -1$, $X_0(u) = 0$, or $Y_0(u) (1 - h(u) t^2) = 2X_0(u)t.$ +* When $g(u)=0$, the potentially many $t$ values that decode to an $x$ satisfying $g(x)=0$ using the $x_2$ formula. These were excluded by the $g(u)=0$ condition in the $c \in \\{0, 1, 4, 5\\}$ branch. + +These cases form a negligible subset of all $(u, t)$ for cryptographically sized curves. + +### 3.5 Encoding for `secp256k1` + +Specialized for odd-ordered $a=0$ curves: + +**Define** $G_{c,u}(x)$ as: +* If $u=0$, return $\bot.$ +* If $c \in \\{0, 1, 4, 5\\}:$ + * If $(-u-x)^3 + b$ is square, return $\bot$ + * Let $s = -(u^3 + b)/(u^2 + ux + x^2)$ (cannot cause division by 0). + * Let $v = x.$ +* Otherwise, when $c \in \\{2, 3, 6, 7\\}:$ + * Let $s = x-u.$ + * Let $r = \sqrt{-s(4(u^3 + b) + 3su^2)}$; return $\bot$ if not square. + * If $c \in \\{3, 7\\}$ and $r=0$, return $\bot.$ + * If $s = 0$, return $\bot.$ + * Let $v = (r/s - u)/2.$ +* Let $w = \sqrt{s}$; return $\bot$ if not square. +* Depending on $c:$ + * If $c \in \\{0, 2\\}:$ return $w(\frac{\sqrt{-3}-1}{2}u - v).$ + * If $c \in \\{1, 3\\}:$ return $w(\frac{\sqrt{-3}+1}{2}u + v).$ + * If $c \in \\{4, 6\\}:$ return $w(\frac{-\sqrt{-3}+1}{2}u + v).$ + * If $c \in \\{5, 7\\}:$ return $w(\frac{-\sqrt{-3}-1}{2}u - v).$ + +This is implemented in `secp256k1_ellswift_xswiftec_inv_var`. + +And the x-only ElligatorSwift encoding algorithm is still: + +**Define** *ElligatorSwift(x)* as: +* Loop: + * Pick a uniformly random field element $u.$ + * Pick a uniformly random integer $c$ in $[0,8).$ + * Let $t = G_{c,u}(x).$ + * If $t \neq \bot$, return $(u, t)$; restart loop otherwise. + +Note that this logic does not take the remapped $u=0$, $t=0$, and $g(u) = -t^2$ cases into account; it just avoids them. +While it is not impossible to make the encoder target them, this would increase the maximum number of $t$ values for a given $(u, x)$ +combination beyond 8, and thereby slow down the ElligatorSwift loop proportionally, for a negligible gain in uniformity. + +## 4. Encoding and decoding full *(x, y)* coordinates + +So far we have only addressed encoding and decoding x-coordinates, but in some cases an encoding +for full points with $(x, y)$ coordinates is desirable. It is possible to encode this information +in $t$ as well. + +Note that for any $(X, Y) \in S_u$, $(\pm X, \pm Y)$ are all on $S_u.$ Moreover, all of these are +mapped to the same x-coordinate. Negating $X$ or negating $Y$ just results in $x_1$ and $x_2$ +being swapped, and does not affect $x_3.$ This will not change the outcome x-coordinate as the order +of $x_1$ and $x_2$ only matters if both were to be valid, and in that case $x_3$ would be used instead. + +Still, these four $(X, Y)$ combinations all correspond to distinct $t$ values, so we can encode +the sign of the y-coordinate in the sign of $X$ or the sign of $Y.$ They correspond to the +four distinct $P_u^{'-1}$ calls in the definition of $G_{u,c}.$ + +**Note**: In the paper, the sign of the y coordinate is encoded in a separately-coded bit. + +To encode the sign of $y$ in the sign of $Y:$ + +**Define** *Decode(u, t)* for full $(x, y)$ as: +* Let $(X, Y) = P_u(t).$ +* Let $x$ be the first value in $(u + 4Y^2, \frac{-X}{2Y} - \frac{u}{2}, \frac{X}{2Y} - \frac{u}{2})$ for which $g(x)$ is square. +* Let $y = \sqrt{g(x)}.$ +* If $sign(y) = sign(Y)$, return $(x, y)$; otherwise return $(x, -y).$ + +And encoding would be done using a $G_{c,u}(x, y)$ function defined as: + +**Define** $G_{c,u}(x, y)$ as: +* If $c \in \\{0, 1\\}:$ + * If $g(u) = 0$ or $g(x) = 0$, return $\bot$ (even curves only). + * If $g(-u-x)$ is square, return $\bot.$ + * Let $s = -g(u)/(u^2 + ux + x^2 + a)$ (cannot cause division by zero). + * Let $v = x.$ +* Otherwise, when $c \in \\{2, 3\\}:$ + * Let $s = x-u.$ + * Let $r = \sqrt{-s(4g(u) + sh(u))}$; return $\bot$ if not square. + * If $c = 3$ and $r = 0$, return $\bot.$ + * Let $v = (r/s - u)/2.$ +* Let $w = \sqrt{s}$; return $\bot$ if not square. +* Let $w' = w$ if $sign(w/2) = sign(y)$; $-w$ otherwise. +* Depending on $c:$ + * If $c \in \\{0, 2\\}:$ return $P_u^{'-1}(v, w').$ + * If $c \in \\{1, 3\\}:$ return $P_u^{'-1}(-u-v, w').$ + +Note that $c$ now only ranges $[0,4)$, as the sign of $w'$ is decided based on that of $y$, rather than on $c.$ +This change makes some valid encodings unreachable: when $y = 0$ and $sign(Y) \neq sign(0)$. + +In the above logic, $sign$ can be implemented in several ways, such as parity of the integer representation +of the input field element (for prime-sized fields) or the quadratic residuosity (for fields where +$-1$ is not square). The choice does not matter, as long as it only takes on two possible values, and for $x \neq 0$ it holds that $sign(x) \neq sign(-x)$. + +### 4.1 Full *(x, y)* coordinates for `secp256k1` + +For $a=0$ curves, there is another option. Note that for those, +the $P_u(t)$ function translates negations of $t$ to negations of (both) $X$ and $Y.$ Thus, we can use $sign(t)$ to +encode the y-coordinate directly. Combined with the earlier remapping to guarantee all inputs land on the curve, we get +as decoder: + +**Define** *Decode(u, t)* as: +* Let $u'=u$ if $u \neq 0$; $1$ otherwise. +* Let $t'=t$ if $t \neq 0$; $1$ otherwise. +* Let $t''=t'$ if $u'^3 + b + t'^2 \neq 0$; $2t'$ otherwise. +* Let $X = \dfrac{u'^3 + b - t''^2}{2t''}.$ +* Let $Y = \dfrac{X + t''}{u'\sqrt{-3}}.$ +* Let $x$ be the first element of $(u' + 4Y^2, \frac{-X}{2Y} - \frac{u'}{2}, \frac{X}{2Y} - \frac{u'}{2})$ for which $g(x)$ is square. +* Let $y = \sqrt{g(x)}.$ +* Return $(x, y)$ if $sign(y) = sign(t)$; $(x, -y)$ otherwise. + +This is implemented in `secp256k1_ellswift_swiftec_var`. The used $sign(x)$ function is the parity of $x$ when represented as in integer in $[0,q).$ + +The corresponding encoder would invoke the x-only one, but negating the output $t$ if $sign(t) \neq sign(y).$ + +This is implemented in `secp256k1_ellswift_elligatorswift_var`. + +Note that this is only intended for encoding points where both the x-coordinate and y-coordinate are unpredictable. When encoding x-only points +where the y-coordinate is implicitly even (or implicitly square, or implicitly in $[0,q/2]$), the encoder in +[Section 3.5](#35-encoding-for-secp256k1) must be used, or a bias is reintroduced that undoes all the benefit of using ElligatorSwift +in the first place. diff --git a/src/secp256k1/doc/release-process.md b/src/secp256k1/doc/release-process.md index 70a35f0910..ea6087c9ff 100644 --- a/src/secp256k1/doc/release-process.md +++ b/src/secp256k1/doc/release-process.md @@ -12,12 +12,40 @@ It is best if the maintainers are present during the release, so they can help e This process also assumes that there will be no minor releases for old major releases. +We aim to cut a regular release every 3-4 months, approximately twice as frequent as major Bitcoin Core releases. Every second release should be published one month before the feature freeze of the next major Bitcoin Core release, allowing sufficient time to update the library in Core. + +## Sanity Checks +Perform these checks before creating a release: + +1. Ensure `make distcheck` doesn't fail. +```shell +./autogen.sh && ./configure --enable-dev-mode && make distcheck +``` +2. Check installation with autotools: +```shell +dir=$(mktemp -d) +./autogen.sh && ./configure --prefix=$dir && make clean && make install && ls -l $dir/include $dir/lib +gcc -o ecdsa examples/ecdsa.c $(PKG_CONFIG_PATH=$dir/lib/pkgconfig pkg-config --cflags --libs libsecp256k1) -Wl,-rpath,"$dir/lib" && ./ecdsa +``` +3. Check installation with CMake: +```shell +dir=$(mktemp -d) +build=$(mktemp -d) +cmake -B $build -DCMAKE_INSTALL_PREFIX=$dir && cmake --build $build --target install && ls -l $dir/include $dir/lib* +gcc -o ecdsa examples/ecdsa.c -I $dir/include -L $dir/lib*/ -l secp256k1 -Wl,-rpath,"$dir/lib",-rpath,"$dir/lib64" && ./ecdsa +``` + ## Regular release 1. Open a PR to the master branch with a commit (using message `"release: prepare for $MAJOR.$MINOR.$PATCH"`, for example) that - * finalizes the release notes in [CHANGELOG.md](../CHANGELOG.md) (make sure to include an entry for `### ABI Compatibility`), - * updates `_PKG_VERSION_*` and `_LIB_VERSION_*` and sets `_PKG_VERSION_IS_RELEASE` to `true` in `configure.ac`, and - * updates `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_*` in `CMakeLists.txt`. + * finalizes the release notes in [CHANGELOG.md](../CHANGELOG.md) by + * adding a section for the release (make sure that the version number is a link to a diff between the previous and new version), + * removing the `[Unreleased]` section header, and + * including an entry for `### ABI Compatibility` if it doesn't exist that mentions the library soname of the release, + * sets `_PKG_VERSION_IS_RELEASE` to `true` in `configure.ac`, and + * if this is not a patch release + * updates `_PKG_VERSION_*` and `_LIB_VERSION_*` in `configure.ac` and + * updates `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_*` in `CMakeLists.txt`. 2. After the PR is merged, tag the commit and push it: ``` RELEASE_COMMIT=<merge commit of step 1> @@ -25,8 +53,9 @@ This process also assumes that there will be no minor releases for old major rel git push git@github.com:bitcoin-core/secp256k1.git v$MAJOR.$MINOR.$PATCH ``` 3. Open a PR to the master branch with a commit (using message `"release cleanup: bump version after $MAJOR.$MINOR.$PATCH"`, for example) that - * sets `_PKG_VERSION_IS_RELEASE` to `false` and increments `_PKG_VERSION_PATCH` and `_LIB_VERSION_REVISION` in `configure.ac`, and - * increments the `$PATCH` component of `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_REVISION` in `CMakeLists.txt`. + * sets `_PKG_VERSION_IS_RELEASE` to `false` and increments `_PKG_VERSION_PATCH` and `_LIB_VERSION_REVISION` in `configure.ac`, + * increments the `$PATCH` component of `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_REVISION` in `CMakeLists.txt`, and + * adds an `[Unreleased]` section header to the [CHANGELOG.md](../CHANGELOG.md). If other maintainers are not present to approve the PR, it can be merged without ACKs. 4. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md). @@ -35,14 +64,14 @@ This process also assumes that there will be no minor releases for old major rel Note that bugfixes only need to be backported to releases for which no compatible release without the bug exists. -1. If `$PATCH = 1`, create maintenance branch `$MAJOR.$MINOR`: +1. If there's no maintenance branch `$MAJOR.$MINOR`, create one: ``` - git checkout -b $MAJOR.$MINOR v$MAJOR.$MINOR.0 + git checkout -b $MAJOR.$MINOR v$MAJOR.$MINOR.$((PATCH - 1)) git push git@github.com:bitcoin-core/secp256k1.git $MAJOR.$MINOR ``` 2. Open a pull request to the `$MAJOR.$MINOR` branch that * includes the bugfixes, - * finalizes the release notes, + * finalizes the release notes similar to a regular release, * increments `_PKG_VERSION_PATCH` and `_LIB_VERSION_REVISION` in `configure.ac` and the `$PATCH` component of `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_REVISION` in `CMakeLists.txt` (with commit message `"release: bump versions for $MAJOR.$MINOR.$PATCH"`, for example). diff --git a/src/secp256k1/examples/CMakeLists.txt b/src/secp256k1/examples/CMakeLists.txt index 0884b645e0..e095b7f84f 100644 --- a/src/secp256k1/examples/CMakeLists.txt +++ b/src/secp256k1/examples/CMakeLists.txt @@ -2,33 +2,26 @@ add_library(example INTERFACE) target_include_directories(example INTERFACE ${PROJECT_SOURCE_DIR}/include ) -target_compile_options(example INTERFACE - $<$<C_COMPILER_ID:MSVC>:/wd4005> -) target_link_libraries(example INTERFACE + secp256k1 $<$<PLATFORM_ID:Windows>:bcrypt> ) -if(SECP256K1_BUILD_SHARED) - target_link_libraries(example INTERFACE secp256k1) -elseif(SECP256K1_BUILD_STATIC) - target_link_libraries(example INTERFACE secp256k1_static) - if(MSVC) - target_link_options(example INTERFACE /IGNORE:4217) - endif() +if(NOT BUILD_SHARED_LIBS AND MSVC) + target_link_options(example INTERFACE /IGNORE:4217) endif() add_executable(ecdsa_example ecdsa.c) target_link_libraries(ecdsa_example example) -add_test(ecdsa_example ecdsa_example) +add_test(NAME ecdsa_example COMMAND ecdsa_example) if(SECP256K1_ENABLE_MODULE_ECDH) add_executable(ecdh_example ecdh.c) target_link_libraries(ecdh_example example) - add_test(ecdh_example ecdh_example) + add_test(NAME ecdh_example COMMAND ecdh_example) endif() if(SECP256K1_ENABLE_MODULE_SCHNORRSIG) add_executable(schnorr_example schnorr.c) target_link_libraries(schnorr_example example) - add_test(schnorr_example schnorr_example) + add_test(NAME schnorr_example COMMAND schnorr_example) endif() diff --git a/src/secp256k1/examples/examples_util.h b/src/secp256k1/examples/examples_util.h index a52b1fa115..8e3a8f00cf 100644 --- a/src/secp256k1/examples/examples_util.h +++ b/src/secp256k1/examples/examples_util.h @@ -17,7 +17,13 @@ */ #if defined(_WIN32) +/* + * The defined WIN32_NO_STATUS macro disables return code definitions in + * windows.h, which avoids "macro redefinition" MSVC warnings in ntstatus.h. + */ +#define WIN32_NO_STATUS #include <windows.h> +#undef WIN32_NO_STATUS #include <ntstatus.h> #include <bcrypt.h> #elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) @@ -77,7 +83,7 @@ static void print_hex(unsigned char* data, size_t size) { #include <Windows.h> #endif /* Cleanses memory to prevent leaking sensitive info. Won't be optimized out. */ -static SECP256K1_INLINE void secure_erase(void *ptr, size_t len) { +static void secure_erase(void *ptr, size_t len) { #if defined(_MSC_VER) /* SecureZeroMemory is guaranteed not to be optimized out by MSVC. */ SecureZeroMemory(ptr, len); diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index 4ce3e500ba..a7a2be7a3a 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -122,18 +122,6 @@ typedef int (*secp256k1_nonce_function)( # endif # endif -# if (!defined(__STDC_VERSION__) || (__STDC_VERSION__ < 199901L) ) -# if SECP256K1_GNUC_PREREQ(2,7) -# define SECP256K1_INLINE __inline__ -# elif (defined(_MSC_VER)) -# define SECP256K1_INLINE __inline -# else -# define SECP256K1_INLINE -# endif -# else -# define SECP256K1_INLINE inline -# endif - /* When this header is used at build-time the SECP256K1_BUILD define needs to be set * to correctly setup export attributes and nullness checks. This is normally done * by secp256k1.c but to guard against this header being included before secp256k1.c diff --git a/src/secp256k1/include/secp256k1_ellswift.h b/src/secp256k1/include/secp256k1_ellswift.h new file mode 100644 index 0000000000..3851f93098 --- /dev/null +++ b/src/secp256k1/include/secp256k1_ellswift.h @@ -0,0 +1,198 @@ +#ifndef SECP256K1_ELLSWIFT_H +#define SECP256K1_ELLSWIFT_H + +#include "secp256k1.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* This module provides an implementation of ElligatorSwift as well as a + * version of x-only ECDH using it (including compatibility with BIP324). + * + * ElligatorSwift is described in https://eprint.iacr.org/2022/759 by + * Chavez-Saab, Rodriguez-Henriquez, and Tibouchi. It permits encoding + * uniformly chosen public keys as 64-byte arrays which are indistinguishable + * from uniformly random arrays. + * + * Let f be the function from pairs of field elements to point X coordinates, + * defined as follows (all operations modulo p = 2^256 - 2^32 - 977) + * f(u,t): + * - Let C = 0xa2d2ba93507f1df233770c2a797962cc61f6d15da14ecd47d8d27ae1cd5f852, + * a square root of -3. + * - If u=0, set u=1 instead. + * - If t=0, set t=1 instead. + * - If u^3 + t^2 + 7 = 0, multiply t by 2. + * - Let X = (u^3 + 7 - t^2) / (2 * t) + * - Let Y = (X + t) / (C * u) + * - Return the first in [u + 4 * Y^2, (-X/Y - u) / 2, (X/Y - u) / 2] that is an + * X coordinate on the curve (at least one of them is, for any u and t). + * + * Then an ElligatorSwift encoding of x consists of the 32-byte big-endian + * encodings of field elements u and t concatenated, where f(u,t) = x. + * The encoding algorithm is described in the paper, and effectively picks a + * uniformly random pair (u,t) among those which encode x. + * + * If the Y coordinate is relevant, it is given the same parity as t. + * + * Changes w.r.t. the the paper: + * - The u=0, t=0, and u^3+t^2+7=0 conditions result in decoding to the point + * at infinity in the paper. Here they are remapped to finite points. + * - The paper uses an additional encoding bit for the parity of y. Here the + * parity of t is used (negating t does not affect the decoded x coordinate, + * so this is possible). + */ + +/** A pointer to a function used by secp256k1_ellswift_xdh to hash the shared X + * coordinate along with the encoded public keys to a uniform shared secret. + * + * Returns: 1 if a shared secret was successfully computed. + * 0 will cause secp256k1_ellswift_xdh to fail and return 0. + * Other return values are not allowed, and the behaviour of + * secp256k1_ellswift_xdh is undefined for other return values. + * Out: output: pointer to an array to be filled by the function + * In: x32: pointer to the 32-byte serialized X coordinate + * of the resulting shared point (will not be NULL) + * ell_a64: pointer to the 64-byte encoded public key of party A + * (will not be NULL) + * ell_b64: pointer to the 64-byte encoded public key of party B + * (will not be NULL) + * data: arbitrary data pointer that is passed through + */ +typedef int (*secp256k1_ellswift_xdh_hash_function)( + unsigned char *output, + const unsigned char *x32, + const unsigned char *ell_a64, + const unsigned char *ell_b64, + void *data +); + +/** An implementation of an secp256k1_ellswift_xdh_hash_function which uses + * SHA256(prefix64 || ell_a64 || ell_b64 || x32), where prefix64 is the 64-byte + * array pointed to by data. */ +SECP256K1_API_VAR const secp256k1_ellswift_xdh_hash_function secp256k1_ellswift_xdh_hash_function_prefix; + +/** An implementation of an secp256k1_ellswift_xdh_hash_function compatible with + * BIP324. It returns H_tag(ell_a64 || ell_b64 || x32), where H_tag is the + * BIP340 tagged hash function with tag "bip324_ellswift_xonly_ecdh". Equivalent + * to secp256k1_ellswift_xdh_hash_function_prefix with prefix64 set to + * SHA256("bip324_ellswift_xonly_ecdh")||SHA256("bip324_ellswift_xonly_ecdh"). + * The data argument is ignored. */ +SECP256K1_API_VAR const secp256k1_ellswift_xdh_hash_function secp256k1_ellswift_xdh_hash_function_bip324; + +/** Construct a 64-byte ElligatorSwift encoding of a given pubkey. + * + * Returns: 1 always. + * Args: ctx: pointer to a context object + * Out: ell64: pointer to a 64-byte array to be filled + * In: pubkey: a pointer to a secp256k1_pubkey containing an + * initialized public key + * rnd32: pointer to 32 bytes of randomness + * + * It is recommended that rnd32 consists of 32 uniformly random bytes, not + * known to any adversary trying to detect whether public keys are being + * encoded, though 16 bytes of randomness (padded to an array of 32 bytes, + * e.g., with zeros) suffice to make the result indistinguishable from + * uniform. The randomness in rnd32 must not be a deterministic function of + * the pubkey (it can be derived from the private key, though). + * + * It is not guaranteed that the computed encoding is stable across versions + * of the library, even if all arguments to this function (including rnd32) + * are the same. + * + * This function runs in variable time. + */ +SECP256K1_API int secp256k1_ellswift_encode( + const secp256k1_context *ctx, + unsigned char *ell64, + const secp256k1_pubkey *pubkey, + const unsigned char *rnd32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Decode a 64-bytes ElligatorSwift encoded public key. + * + * Returns: always 1 + * Args: ctx: pointer to a context object + * Out: pubkey: pointer to a secp256k1_pubkey that will be filled + * In: ell64: pointer to a 64-byte array to decode + * + * This function runs in variable time. + */ +SECP256K1_API int secp256k1_ellswift_decode( + const secp256k1_context *ctx, + secp256k1_pubkey *pubkey, + const unsigned char *ell64 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Compute an ElligatorSwift public key for a secret key. + * + * Returns: 1: secret was valid, public key was stored. + * 0: secret was invalid, try again. + * Args: ctx: pointer to a context object + * Out: ell64: pointer to a 64-byte array to receive the ElligatorSwift + * public key + * In: seckey32: pointer to a 32-byte secret key + * auxrnd32: (optional) pointer to 32 bytes of randomness + * + * Constant time in seckey and auxrnd32, but not in the resulting public key. + * + * It is recommended that auxrnd32 contains 32 uniformly random bytes, though + * it is optional (and does result in encodings that are indistinguishable from + * uniform even without any auxrnd32). It differs from the (mandatory) rnd32 + * argument to secp256k1_ellswift_encode in this regard. + * + * This function can be used instead of calling secp256k1_ec_pubkey_create + * followed by secp256k1_ellswift_encode. It is safer, as it uses the secret + * key as entropy for the encoding (supplemented with auxrnd32, if provided). + * + * Like secp256k1_ellswift_encode, this function does not guarantee that the + * computed encoding is stable across versions of the library, even if all + * arguments (including auxrnd32) are the same. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ellswift_create( + const secp256k1_context *ctx, + unsigned char *ell64, + const unsigned char *seckey32, + const unsigned char *auxrnd32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Given a private key, and ElligatorSwift public keys sent in both directions, + * compute a shared secret using x-only Elliptic Curve Diffie-Hellman (ECDH). + * + * Returns: 1: shared secret was succesfully computed + * 0: secret was invalid or hashfp returned 0 + * Args: ctx: pointer to a context object. + * Out: output: pointer to an array to be filled by hashfp. + * In: ell_a64: pointer to the 64-byte encoded public key of party A + * (will not be NULL) + * ell_b64: pointer to the 64-byte encoded public key of party B + * (will not be NULL) + * seckey32: a pointer to our 32-byte secret key + * party: boolean indicating which party we are: zero if we are + * party A, non-zero if we are party B. seckey32 must be + * the private key corresponding to that party's ell_?64. + * This correspondence is not checked. + * hashfp: pointer to a hash function. + * data: arbitrary data pointer passed through to hashfp. + * + * Constant time in seckey32. + * + * This function is more efficient than decoding the public keys, and performing + * ECDH on them. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ellswift_xdh( + const secp256k1_context *ctx, + unsigned char *output, + const unsigned char *ell_a64, + const unsigned char *ell_b64, + const unsigned char *seckey32, + int party, + secp256k1_ellswift_xdh_hash_function hashfp, + void *data +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5) SECP256K1_ARG_NONNULL(7); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_ELLSWIFT_H */ diff --git a/src/secp256k1/include/secp256k1_extrakeys.h b/src/secp256k1/include/secp256k1_extrakeys.h index 52bba240b4..673fca01f9 100644 --- a/src/secp256k1/include/secp256k1_extrakeys.h +++ b/src/secp256k1/include/secp256k1_extrakeys.h @@ -185,9 +185,8 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_sec( /** Get the public key from a keypair. * * Returns: 1 always. - * Args: ctx: pointer to a context object. - * Out: pubkey: pointer to a pubkey object. If 1 is returned, it is set to - * the keypair public key. If not, it's set to an invalid value. + * Args: ctx: pointer to a context object. + * Out: pubkey: pointer to a pubkey object, set to the keypair public key. * In: keypair: pointer to a keypair. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_pub( @@ -203,9 +202,8 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_pub( * * Returns: 1 always. * Args: ctx: pointer to a context object. - * Out: pubkey: pointer to an xonly_pubkey object. If 1 is returned, it is set - * to the keypair public key after converting it to an - * xonly_pubkey. If not, it's set to an invalid value. + * Out: pubkey: pointer to an xonly_pubkey object, set to the keypair + * public key after converting it to an xonly_pubkey. * pk_parity: Ignored if NULL. Otherwise, pointer to an integer that will be set to the * pk_parity argument of secp256k1_xonly_pubkey_from_pubkey. * In: keypair: pointer to a keypair. diff --git a/src/secp256k1/include/secp256k1_schnorrsig.h b/src/secp256k1/include/secp256k1_schnorrsig.h index 733fee5282..1ee665fd19 100644 --- a/src/secp256k1/include/secp256k1_schnorrsig.h +++ b/src/secp256k1/include/secp256k1_schnorrsig.h @@ -141,12 +141,20 @@ SECP256K1_API int secp256k1_schnorrsig_sign( * variable length messages and accepts a pointer to an extraparams object that * allows customizing signing by passing additional arguments. * - * Creates the same signatures as schnorrsig_sign if msglen is 32 and the - * extraparams.ndata is the same as aux_rand32. + * Equivalent to secp256k1_schnorrsig_sign32(..., aux_rand32) if msglen is 32 + * and extraparams is initialized as follows: + * ``` + * secp256k1_schnorrsig_extraparams extraparams = SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT; + * extraparams.ndata = (unsigned char*)aux_rand32; + * ``` * + * Returns 1 on success, 0 on failure. + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * Out: sig64: pointer to a 64-byte array to store the serialized signature. * In: msg: the message being signed. Can only be NULL if msglen is 0. - * msglen: length of the message - * extraparams: pointer to a extraparams object (can be NULL) + * msglen: length of the message. + * keypair: pointer to an initialized keypair. + * extraparams: pointer to an extraparams object (can be NULL). */ SECP256K1_API int secp256k1_schnorrsig_sign_custom( const secp256k1_context *ctx, diff --git a/src/secp256k1/src/CMakeLists.txt b/src/secp256k1/src/CMakeLists.txt index 26272d0950..0bba19982a 100644 --- a/src/secp256k1/src/CMakeLists.txt +++ b/src/secp256k1/src/CMakeLists.txt @@ -1,151 +1,165 @@ # Must be included before CMAKE_INSTALL_INCLUDEDIR is used. include(GNUInstallDirs) -set(${PROJECT_NAME}_installables "") -if(SECP256K1_ASM STREQUAL "arm") - add_library(common OBJECT - asm/field_10x26_arm.s - ) - set(common_obj "$<TARGET_OBJECTS:common>") -else() - set(common_obj "") -endif() - -add_library(precomputed OBJECT +add_library(secp256k1_precomputed OBJECT EXCLUDE_FROM_ALL precomputed_ecmult.c precomputed_ecmult_gen.c ) -set(internal_obj "$<TARGET_OBJECTS:precomputed>" "${common_obj}") -add_library(secp256k1 SHARED EXCLUDE_FROM_ALL - secp256k1.c - ${internal_obj} -) -target_include_directories(secp256k1 INTERFACE - $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> -) -target_compile_definitions(secp256k1 PRIVATE - $<$<PLATFORM_ID:Windows>:DLL_EXPORT> -) -set_target_properties(secp256k1 PROPERTIES - VERSION "${${PROJECT_NAME}_LIB_VERSION_CURRENT}.${${PROJECT_NAME}_LIB_VERSION_AGE}.${${PROJECT_NAME}_LIB_VERSION_REVISION}" - SOVERSION "${${PROJECT_NAME}_LIB_VERSION_CURRENT}" -) -if(SECP256K1_BUILD_SHARED) - get_target_property(use_pic secp256k1 POSITION_INDEPENDENT_CODE) - set_target_properties(precomputed PROPERTIES POSITION_INDEPENDENT_CODE ${use_pic}) - set_target_properties(secp256k1 PROPERTIES EXCLUDE_FROM_ALL FALSE) - list(APPEND ${PROJECT_NAME}_installables secp256k1) -endif() +# Add objects explicitly rather than linking to the object libs to keep them +# from being exported. +add_library(secp256k1 secp256k1.c $<TARGET_OBJECTS:secp256k1_precomputed>) -add_library(secp256k1_static STATIC EXCLUDE_FROM_ALL - secp256k1.c - ${internal_obj} -) -target_include_directories(secp256k1_static INTERFACE - $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> -) -if(NOT MSVC) - set_target_properties(secp256k1_static PROPERTIES - OUTPUT_NAME secp256k1 +add_library(secp256k1_asm INTERFACE) +if(SECP256K1_ASM STREQUAL "arm32") + add_library(secp256k1_asm_arm OBJECT EXCLUDE_FROM_ALL) + target_sources(secp256k1_asm_arm PUBLIC + asm/field_10x26_arm.s ) + target_sources(secp256k1 PRIVATE $<TARGET_OBJECTS:secp256k1_asm_arm>) + target_link_libraries(secp256k1_asm INTERFACE secp256k1_asm_arm) endif() -if(SECP256K1_BUILD_STATIC) - set_target_properties(secp256k1_static PROPERTIES EXCLUDE_FROM_ALL FALSE) - list(APPEND ${PROJECT_NAME}_installables secp256k1_static) + +# Define our export symbol only for Win32 and only for shared libs. +# This matches libtool's usage of DLL_EXPORT +if(WIN32) + set_target_properties(secp256k1 PROPERTIES DEFINE_SYMBOL "DLL_EXPORT") endif() -add_library(binary_interface INTERFACE) -target_compile_definitions(binary_interface INTERFACE - $<$<C_COMPILER_ID:MSVC>:_CRT_SECURE_NO_WARNINGS> +# Object libs don't know if they're being built for a shared or static lib. +# Grab the PIC property from secp256k1 which knows. +get_target_property(use_pic secp256k1 POSITION_INDEPENDENT_CODE) +set_target_properties(secp256k1_precomputed PROPERTIES POSITION_INDEPENDENT_CODE ${use_pic}) + +target_include_directories(secp256k1 INTERFACE + # Add the include path for parent projects so that they don't have to manually add it. + $<BUILD_INTERFACE:$<$<NOT:$<BOOL:${PROJECT_IS_TOP_LEVEL}>>:${PROJECT_SOURCE_DIR}/include>> + $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> ) -add_library(link_library INTERFACE) -if(SECP256K1_BUILD_SHARED) - target_link_libraries(link_library INTERFACE secp256k1) -elseif(SECP256K1_BUILD_STATIC) - target_link_libraries(link_library INTERFACE secp256k1_static) +# This emulates Libtool to make sure Libtool and CMake agree on the ABI version, +# see below "Calculate the version variables" in build-aux/ltmain.sh. +math(EXPR ${PROJECT_NAME}_soversion "${${PROJECT_NAME}_LIB_VERSION_CURRENT} - ${${PROJECT_NAME}_LIB_VERSION_AGE}") +set_target_properties(secp256k1 PROPERTIES + SOVERSION ${${PROJECT_NAME}_soversion} +) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + set_target_properties(secp256k1 PROPERTIES + VERSION ${${PROJECT_NAME}_soversion}.${${PROJECT_NAME}_LIB_VERSION_AGE}.${${PROJECT_NAME}_LIB_VERSION_REVISION} + ) +elseif(APPLE) + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.17) + math(EXPR ${PROJECT_NAME}_compatibility_version "${${PROJECT_NAME}_LIB_VERSION_CURRENT} + 1") + set_target_properties(secp256k1 PROPERTIES + MACHO_COMPATIBILITY_VERSION ${${PROJECT_NAME}_compatibility_version} + MACHO_CURRENT_VERSION ${${PROJECT_NAME}_compatibility_version}.${${PROJECT_NAME}_LIB_VERSION_REVISION} + ) + unset(${PROJECT_NAME}_compatibility_version) + elseif(BUILD_SHARED_LIBS) + message(WARNING + "The 'compatibility version' and 'current version' values of the DYLIB " + "will diverge from the values set by the GNU Libtool. To ensure " + "compatibility, it is recommended to upgrade CMake to at least version 3.17." + ) + endif() +elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(${PROJECT_NAME}_windows "secp256k1") + if(MSVC) + set(${PROJECT_NAME}_windows "${PROJECT_NAME}") + endif() + set_target_properties(secp256k1 PROPERTIES + ARCHIVE_OUTPUT_NAME "${${PROJECT_NAME}_windows}" + RUNTIME_OUTPUT_NAME "${${PROJECT_NAME}_windows}-${${PROJECT_NAME}_soversion}" + ) + unset(${PROJECT_NAME}_windows) endif() +unset(${PROJECT_NAME}_soversion) if(SECP256K1_BUILD_BENCHMARK) add_executable(bench bench.c) - target_link_libraries(bench binary_interface link_library) - add_executable(bench_internal bench_internal.c ${internal_obj}) - target_link_libraries(bench_internal binary_interface) - add_executable(bench_ecmult bench_ecmult.c ${internal_obj}) - target_link_libraries(bench_ecmult binary_interface) + target_link_libraries(bench secp256k1) + add_executable(bench_internal bench_internal.c) + target_link_libraries(bench_internal secp256k1_precomputed secp256k1_asm) + add_executable(bench_ecmult bench_ecmult.c) + target_link_libraries(bench_ecmult secp256k1_precomputed secp256k1_asm) endif() if(SECP256K1_BUILD_TESTS) - add_executable(noverify_tests tests.c ${internal_obj}) - target_link_libraries(noverify_tests binary_interface) - add_test(noverify_tests noverify_tests) + add_executable(noverify_tests tests.c) + target_link_libraries(noverify_tests secp256k1_precomputed secp256k1_asm) + add_test(NAME noverify_tests COMMAND noverify_tests) if(NOT CMAKE_BUILD_TYPE STREQUAL "Coverage") - add_executable(tests tests.c ${internal_obj}) + add_executable(tests tests.c) target_compile_definitions(tests PRIVATE VERIFY) - target_link_libraries(tests binary_interface) - add_test(tests tests) + target_link_libraries(tests secp256k1_precomputed secp256k1_asm) + add_test(NAME tests COMMAND tests) endif() endif() if(SECP256K1_BUILD_EXHAUSTIVE_TESTS) - # Note: do not include $<TARGET_OBJECTS:precomputed> in exhaustive_tests (it uses runtime-generated tables). - add_executable(exhaustive_tests tests_exhaustive.c ${common_obj}) + # Note: do not include secp256k1_precomputed in exhaustive_tests (it uses runtime-generated tables). + add_executable(exhaustive_tests tests_exhaustive.c) + target_link_libraries(exhaustive_tests secp256k1_asm) target_compile_definitions(exhaustive_tests PRIVATE $<$<NOT:$<CONFIG:Coverage>>:VERIFY>) - target_link_libraries(exhaustive_tests binary_interface) - add_test(exhaustive_tests exhaustive_tests) + add_test(NAME exhaustive_tests COMMAND exhaustive_tests) endif() if(SECP256K1_BUILD_CTIME_TESTS) add_executable(ctime_tests ctime_tests.c) - target_link_libraries(ctime_tests binary_interface link_library) + target_link_libraries(ctime_tests secp256k1) endif() -install(TARGETS ${${PROJECT_NAME}_installables} - EXPORT ${PROJECT_NAME}-targets - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} -) -set(${PROJECT_NAME}_headers - "${PROJECT_SOURCE_DIR}/include/secp256k1.h" - "${PROJECT_SOURCE_DIR}/include/secp256k1_preallocated.h" -) -if(SECP256K1_ENABLE_MODULE_ECDH) - list(APPEND ${PROJECT_NAME}_headers "${PROJECT_SOURCE_DIR}/include/secp256k1_ecdh.h") -endif() -if(SECP256K1_ENABLE_MODULE_RECOVERY) - list(APPEND ${PROJECT_NAME}_headers "${PROJECT_SOURCE_DIR}/include/secp256k1_recovery.h") -endif() -if(SECP256K1_ENABLE_MODULE_EXTRAKEYS) - list(APPEND ${PROJECT_NAME}_headers "${PROJECT_SOURCE_DIR}/include/secp256k1_extrakeys.h") -endif() -if(SECP256K1_ENABLE_MODULE_SCHNORRSIG) - list(APPEND ${PROJECT_NAME}_headers "${PROJECT_SOURCE_DIR}/include/secp256k1_schnorrsig.h") -endif() -install(FILES ${${PROJECT_NAME}_headers} - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} -) +if(SECP256K1_INSTALL) + install(TARGETS secp256k1 + EXPORT ${PROJECT_NAME}-targets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) + set(${PROJECT_NAME}_headers + "${PROJECT_SOURCE_DIR}/include/secp256k1.h" + "${PROJECT_SOURCE_DIR}/include/secp256k1_preallocated.h" + ) + if(SECP256K1_ENABLE_MODULE_ECDH) + list(APPEND ${PROJECT_NAME}_headers "${PROJECT_SOURCE_DIR}/include/secp256k1_ecdh.h") + endif() + if(SECP256K1_ENABLE_MODULE_RECOVERY) + list(APPEND ${PROJECT_NAME}_headers "${PROJECT_SOURCE_DIR}/include/secp256k1_recovery.h") + endif() + if(SECP256K1_ENABLE_MODULE_EXTRAKEYS) + list(APPEND ${PROJECT_NAME}_headers "${PROJECT_SOURCE_DIR}/include/secp256k1_extrakeys.h") + endif() + if(SECP256K1_ENABLE_MODULE_SCHNORRSIG) + list(APPEND ${PROJECT_NAME}_headers "${PROJECT_SOURCE_DIR}/include/secp256k1_schnorrsig.h") + endif() + if(SECP256K1_ENABLE_MODULE_ELLSWIFT) + list(APPEND ${PROJECT_NAME}_headers "${PROJECT_SOURCE_DIR}/include/secp256k1_ellswift.h") + endif() + install(FILES ${${PROJECT_NAME}_headers} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ) -install(EXPORT ${PROJECT_NAME}-targets - FILE ${PROJECT_NAME}-targets.cmake - NAMESPACE ${PROJECT_NAME}:: - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} -) + install(EXPORT ${PROJECT_NAME}-targets + FILE ${PROJECT_NAME}-targets.cmake + NAMESPACE ${PROJECT_NAME}:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} + ) -include(CMakePackageConfigHelpers) -configure_package_config_file( - ${PROJECT_SOURCE_DIR}/cmake/config.cmake.in - ${PROJECT_NAME}-config.cmake - INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} - NO_SET_AND_CHECK_MACRO -) -write_basic_package_version_file(${PROJECT_NAME}-config-version.cmake - COMPATIBILITY SameMajorVersion -) -install( - FILES - ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} + include(CMakePackageConfigHelpers) + configure_package_config_file( + ${PROJECT_SOURCE_DIR}/cmake/config.cmake.in + ${PROJECT_NAME}-config.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} + NO_SET_AND_CHECK_MACRO + ) + write_basic_package_version_file(${PROJECT_NAME}-config-version.cmake + COMPATIBILITY SameMinorVersion + ) + + install( + FILES + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} ) +endif() diff --git a/src/secp256k1/src/bench.c b/src/secp256k1/src/bench.c index 833f70718b..1127df67ae 100644 --- a/src/secp256k1/src/bench.c +++ b/src/secp256k1/src/bench.c @@ -38,6 +38,8 @@ static void help(int default_iters) { printf(" ecdsa : all ECDSA algorithms--sign, verify, recovery (if enabled)\n"); printf(" ecdsa_sign : ECDSA siging algorithm\n"); printf(" ecdsa_verify : ECDSA verification algorithm\n"); + printf(" ec : all EC public key algorithms (keygen)\n"); + printf(" ec_keygen : EC public key generation\n"); #ifdef ENABLE_MODULE_RECOVERY printf(" ecdsa_recover : ECDSA public key recovery algorithm\n"); @@ -53,6 +55,14 @@ static void help(int default_iters) { printf(" schnorrsig_verify : Schnorr verification algorithm\n"); #endif +#ifdef ENABLE_MODULE_ELLSWIFT + printf(" ellswift : all ElligatorSwift benchmarks (encode, decode, keygen, ecdh)\n"); + printf(" ellswift_encode : ElligatorSwift encoding\n"); + printf(" ellswift_decode : ElligatorSwift decoding\n"); + printf(" ellswift_keygen : ElligatorSwift key generation\n"); + printf(" ellswift_ecdh : ECDH on ElligatorSwift keys\n"); +#endif + printf("\n"); } @@ -64,11 +74,11 @@ typedef struct { size_t siglen; unsigned char pubkey[33]; size_t pubkeylen; -} bench_verify_data; +} bench_data; static void bench_verify(void* arg, int iters) { int i; - bench_verify_data* data = (bench_verify_data*)arg; + bench_data* data = (bench_data*)arg; for (i = 0; i < iters; i++) { secp256k1_pubkey pubkey; @@ -85,15 +95,9 @@ static void bench_verify(void* arg, int iters) { } } -typedef struct { - secp256k1_context* ctx; - unsigned char msg[32]; - unsigned char key[32]; -} bench_sign_data; - static void bench_sign_setup(void* arg) { int i; - bench_sign_data *data = (bench_sign_data*)arg; + bench_data *data = (bench_data*)arg; for (i = 0; i < 32; i++) { data->msg[i] = i + 1; @@ -105,7 +109,7 @@ static void bench_sign_setup(void* arg) { static void bench_sign_run(void* arg, int iters) { int i; - bench_sign_data *data = (bench_sign_data*)arg; + bench_data *data = (bench_data*)arg; unsigned char sig[74]; for (i = 0; i < iters; i++) { @@ -121,6 +125,30 @@ static void bench_sign_run(void* arg, int iters) { } } +static void bench_keygen_setup(void* arg) { + int i; + bench_data *data = (bench_data*)arg; + + for (i = 0; i < 32; i++) { + data->key[i] = i + 65; + } +} + +static void bench_keygen_run(void *arg, int iters) { + int i; + bench_data *data = (bench_data*)arg; + + for (i = 0; i < iters; i++) { + unsigned char pub33[33]; + size_t len = 33; + secp256k1_pubkey pubkey; + CHECK(secp256k1_ec_pubkey_create(data->ctx, &pubkey, data->key)); + CHECK(secp256k1_ec_pubkey_serialize(data->ctx, pub33, &len, &pubkey, SECP256K1_EC_COMPRESSED)); + memcpy(data->key, pub33 + 1, 32); + } +} + + #ifdef ENABLE_MODULE_ECDH # include "modules/ecdh/bench_impl.h" #endif @@ -133,11 +161,15 @@ static void bench_sign_run(void* arg, int iters) { # include "modules/schnorrsig/bench_impl.h" #endif +#ifdef ENABLE_MODULE_ELLSWIFT +# include "modules/ellswift/bench_impl.h" +#endif + int main(int argc, char** argv) { int i; secp256k1_pubkey pubkey; secp256k1_ecdsa_signature sig; - bench_verify_data data; + bench_data data; int d = argc == 1; int default_iters = 20000; @@ -145,7 +177,9 @@ int main(int argc, char** argv) { /* Check for invalid user arguments */ char* valid_args[] = {"ecdsa", "verify", "ecdsa_verify", "sign", "ecdsa_sign", "ecdh", "recover", - "ecdsa_recover", "schnorrsig", "schnorrsig_verify", "schnorrsig_sign"}; + "ecdsa_recover", "schnorrsig", "schnorrsig_verify", "schnorrsig_sign", "ec", + "keygen", "ec_keygen", "ellswift", "encode", "ellswift_encode", "decode", + "ellswift_decode", "ellswift_keygen", "ellswift_ecdh"}; size_t valid_args_size = sizeof(valid_args)/sizeof(valid_args[0]); int invalid_args = have_invalid_args(argc, argv, valid_args, valid_args_size); @@ -187,6 +221,16 @@ int main(int argc, char** argv) { } #endif +#ifndef ENABLE_MODULE_ELLSWIFT + if (have_flag(argc, argv, "ellswift") || have_flag(argc, argv, "ellswift_encode") || have_flag(argc, argv, "ellswift_decode") || + have_flag(argc, argv, "encode") || have_flag(argc, argv, "decode") || have_flag(argc, argv, "ellswift_keygen") || + have_flag(argc, argv, "ellswift_ecdh")) { + fprintf(stderr, "./bench: ElligatorSwift module not enabled.\n"); + fprintf(stderr, "Use ./configure --enable-module-ellswift.\n\n"); + return 1; + } +#endif + /* ECDSA benchmark */ data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); @@ -207,6 +251,7 @@ int main(int argc, char** argv) { if (d || have_flag(argc, argv, "ecdsa") || have_flag(argc, argv, "verify") || have_flag(argc, argv, "ecdsa_verify")) run_benchmark("ecdsa_verify", bench_verify, NULL, NULL, &data, 10, iters); if (d || have_flag(argc, argv, "ecdsa") || have_flag(argc, argv, "sign") || have_flag(argc, argv, "ecdsa_sign")) run_benchmark("ecdsa_sign", bench_sign_run, bench_sign_setup, NULL, &data, 10, iters); + if (d || have_flag(argc, argv, "ec") || have_flag(argc, argv, "keygen") || have_flag(argc, argv, "ec_keygen")) run_benchmark("ec_keygen", bench_keygen_run, bench_keygen_setup, NULL, &data, 10, iters); secp256k1_context_destroy(data.ctx); @@ -225,5 +270,10 @@ int main(int argc, char** argv) { run_schnorrsig_bench(iters, argc, argv); #endif +#ifdef ENABLE_MODULE_ELLSWIFT + /* ElligatorSwift benchmarks */ + run_ellswift_bench(iters, argc, argv); +#endif + return 0; } diff --git a/src/secp256k1/src/bench.h b/src/secp256k1/src/bench.h index bf9a932ff4..1564b1a176 100644 --- a/src/secp256k1/src/bench.h +++ b/src/secp256k1/src/bench.h @@ -15,7 +15,7 @@ #if (defined(_MSC_VER) && _MSC_VER >= 1900) # include <time.h> #else -# include "sys/time.h" +# include <sys/time.h> #endif static int64_t gettime_i64(void) { diff --git a/src/secp256k1/src/bench_ecmult.c b/src/secp256k1/src/bench_ecmult.c index 98fb798d82..8818aa81b5 100644 --- a/src/secp256k1/src/bench_ecmult.c +++ b/src/secp256k1/src/bench_ecmult.c @@ -113,7 +113,7 @@ static void bench_ecmult_const(void* arg, int iters) { int i; for (i = 0; i < iters; ++i) { - secp256k1_ecmult_const(&data->output[i], &data->pubkeys[(data->offset1+i) % POINTS], &data->scalars[(data->offset2+i) % POINTS], 256); + secp256k1_ecmult_const(&data->output[i], &data->pubkeys[(data->offset1+i) % POINTS], &data->scalars[(data->offset2+i) % POINTS]); } } @@ -138,12 +138,10 @@ static void bench_ecmult_1p_teardown(void* arg, int iters) { static void bench_ecmult_0p_g(void* arg, int iters) { bench_data* data = (bench_data*)arg; - secp256k1_scalar zero; int i; - secp256k1_scalar_set_int(&zero, 0); for (i = 0; i < iters; ++i) { - secp256k1_ecmult(&data->output[i], NULL, &zero, &data->scalars[(data->offset1+i) % POINTS]); + secp256k1_ecmult(&data->output[i], NULL, &secp256k1_scalar_zero, &data->scalars[(data->offset1+i) % POINTS]); } } diff --git a/src/secp256k1/src/bench_internal.c b/src/secp256k1/src/bench_internal.c index c248ab8ebc..f3686dd289 100644 --- a/src/secp256k1/src/bench_internal.c +++ b/src/secp256k1/src/bench_internal.c @@ -65,10 +65,10 @@ static void bench_setup(void* arg) { secp256k1_scalar_set_b32(&data->scalar[0], init[0], NULL); secp256k1_scalar_set_b32(&data->scalar[1], init[1], NULL); - secp256k1_fe_set_b32(&data->fe[0], init[0]); - secp256k1_fe_set_b32(&data->fe[1], init[1]); - secp256k1_fe_set_b32(&data->fe[2], init[2]); - secp256k1_fe_set_b32(&data->fe[3], init[3]); + secp256k1_fe_set_b32_limit(&data->fe[0], init[0]); + secp256k1_fe_set_b32_limit(&data->fe[1], init[1]); + secp256k1_fe_set_b32_limit(&data->fe[2], init[2]); + secp256k1_fe_set_b32_limit(&data->fe[3], init[3]); CHECK(secp256k1_ge_set_xo_var(&data->ge[0], &data->fe[0], 0)); CHECK(secp256k1_ge_set_xo_var(&data->ge[1], &data->fe[1], 1)); secp256k1_gej_set_ge(&data->gej[0], &data->ge[0]); diff --git a/src/secp256k1/src/ctime_tests.c b/src/secp256k1/src/ctime_tests.c index 713eb427d3..af7891a91c 100644 --- a/src/secp256k1/src/ctime_tests.c +++ b/src/secp256k1/src/ctime_tests.c @@ -30,6 +30,10 @@ #include "../include/secp256k1_schnorrsig.h" #endif +#ifdef ENABLE_MODULE_ELLSWIFT +#include "../include/secp256k1_ellswift.h" +#endif + static void run_tests(secp256k1_context *ctx, unsigned char *key); int main(void) { @@ -80,6 +84,10 @@ static void run_tests(secp256k1_context *ctx, unsigned char *key) { #ifdef ENABLE_MODULE_EXTRAKEYS secp256k1_keypair keypair; #endif +#ifdef ENABLE_MODULE_ELLSWIFT + unsigned char ellswift[64]; + static const unsigned char prefix[64] = {'t', 'e', 's', 't'}; +#endif for (i = 0; i < 32; i++) { msg[i] = i + 1; @@ -171,4 +179,31 @@ static void run_tests(secp256k1_context *ctx, unsigned char *key) { SECP256K1_CHECKMEM_DEFINE(&ret, sizeof(ret)); CHECK(ret == 1); #endif + +#ifdef ENABLE_MODULE_ELLSWIFT + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + ret = secp256k1_ellswift_create(ctx, ellswift, key, NULL); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret == 1); + + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + ret = secp256k1_ellswift_create(ctx, ellswift, key, ellswift); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret == 1); + + for (i = 0; i < 2; i++) { + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + VALGRIND_MAKE_MEM_DEFINED(&ellswift, sizeof(ellswift)); + ret = secp256k1_ellswift_xdh(ctx, msg, ellswift, ellswift, key, i, secp256k1_ellswift_xdh_hash_function_bip324, NULL); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret == 1); + + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + VALGRIND_MAKE_MEM_DEFINED(&ellswift, sizeof(ellswift)); + ret = secp256k1_ellswift_xdh(ctx, msg, ellswift, ellswift, key, i, secp256k1_ellswift_xdh_hash_function_prefix, (void *)prefix); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret == 1); + } + +#endif } diff --git a/src/secp256k1/src/ecdsa_impl.h b/src/secp256k1/src/ecdsa_impl.h index 90b4b22b77..48e30851b5 100644 --- a/src/secp256k1/src/ecdsa_impl.h +++ b/src/secp256k1/src/ecdsa_impl.h @@ -239,7 +239,8 @@ static int secp256k1_ecdsa_sig_verify(const secp256k1_scalar *sigr, const secp25 } #else secp256k1_scalar_get_b32(c, sigr); - secp256k1_fe_set_b32(&xr, c); + /* we can ignore the fe_set_b32_limit return value, because we know the input is in range */ + (void)secp256k1_fe_set_b32_limit(&xr, c); /** We now have the recomputed R point in pr, and its claimed x coordinate (modulo n) * in xr. Naively, we would extract the x coordinate from pr (requiring a inversion modulo p), diff --git a/src/secp256k1/src/eckey_impl.h b/src/secp256k1/src/eckey_impl.h index e0506d3e2b..121966f8b5 100644 --- a/src/secp256k1/src/eckey_impl.h +++ b/src/secp256k1/src/eckey_impl.h @@ -17,10 +17,10 @@ static int secp256k1_eckey_pubkey_parse(secp256k1_ge *elem, const unsigned char *pub, size_t size) { if (size == 33 && (pub[0] == SECP256K1_TAG_PUBKEY_EVEN || pub[0] == SECP256K1_TAG_PUBKEY_ODD)) { secp256k1_fe x; - return secp256k1_fe_set_b32(&x, pub+1) && secp256k1_ge_set_xo_var(elem, &x, pub[0] == SECP256K1_TAG_PUBKEY_ODD); + return secp256k1_fe_set_b32_limit(&x, pub+1) && secp256k1_ge_set_xo_var(elem, &x, pub[0] == SECP256K1_TAG_PUBKEY_ODD); } else if (size == 65 && (pub[0] == SECP256K1_TAG_PUBKEY_UNCOMPRESSED || pub[0] == SECP256K1_TAG_PUBKEY_HYBRID_EVEN || pub[0] == SECP256K1_TAG_PUBKEY_HYBRID_ODD)) { secp256k1_fe x, y; - if (!secp256k1_fe_set_b32(&x, pub+1) || !secp256k1_fe_set_b32(&y, pub+33)) { + if (!secp256k1_fe_set_b32_limit(&x, pub+1) || !secp256k1_fe_set_b32_limit(&y, pub+33)) { return 0; } secp256k1_ge_set_xy(elem, &x, &y); @@ -59,10 +59,8 @@ static int secp256k1_eckey_privkey_tweak_add(secp256k1_scalar *key, const secp25 static int secp256k1_eckey_pubkey_tweak_add(secp256k1_ge *key, const secp256k1_scalar *tweak) { secp256k1_gej pt; - secp256k1_scalar one; secp256k1_gej_set_ge(&pt, key); - secp256k1_scalar_set_int(&one, 1); - secp256k1_ecmult(&pt, &pt, &one, tweak); + secp256k1_ecmult(&pt, &pt, &secp256k1_scalar_one, tweak); if (secp256k1_gej_is_infinity(&pt)) { return 0; @@ -80,15 +78,13 @@ static int secp256k1_eckey_privkey_tweak_mul(secp256k1_scalar *key, const secp25 } static int secp256k1_eckey_pubkey_tweak_mul(secp256k1_ge *key, const secp256k1_scalar *tweak) { - secp256k1_scalar zero; secp256k1_gej pt; if (secp256k1_scalar_is_zero(tweak)) { return 0; } - secp256k1_scalar_set_int(&zero, 0); secp256k1_gej_set_ge(&pt, key); - secp256k1_ecmult(&pt, &pt, tweak, &zero); + secp256k1_ecmult(&pt, &pt, tweak, &secp256k1_scalar_zero); secp256k1_ge_set_gej(key, &pt); return 1; } diff --git a/src/secp256k1/src/ecmult_const.h b/src/secp256k1/src/ecmult_const.h index 417f328535..080e04bc88 100644 --- a/src/secp256k1/src/ecmult_const.h +++ b/src/secp256k1/src/ecmult_const.h @@ -11,12 +11,9 @@ #include "group.h" /** - * Multiply: R = q*A (in constant-time) - * Here `bits` should be set to the maximum bitlength of the _absolute value_ of `q`, plus - * one because we internally sometimes add 2 to the number during the WNAF conversion. - * A must not be infinity. + * Multiply: R = q*A (in constant-time for q) */ -static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, const secp256k1_scalar *q, int bits); +static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, const secp256k1_scalar *q); /** * Same as secp256k1_ecmult_const, but takes in an x coordinate of the base point @@ -35,7 +32,6 @@ static int secp256k1_ecmult_const_xonly( const secp256k1_fe *n, const secp256k1_fe *d, const secp256k1_scalar *q, - int bits, int known_on_curve ); diff --git a/src/secp256k1/src/ecmult_const_impl.h b/src/secp256k1/src/ecmult_const_impl.h index f23e0ec89d..26b3e238d8 100644 --- a/src/secp256k1/src/ecmult_const_impl.h +++ b/src/secp256k1/src/ecmult_const_impl.h @@ -29,7 +29,7 @@ static void secp256k1_ecmult_odd_multiples_table_globalz_windowa(secp256k1_ge *p #define ECMULT_CONST_TABLE_GET_GE(r,pre,n,w) do { \ int m = 0; \ /* Extract the sign-bit for a constant time absolute-value. */ \ - int mask = (n) >> (sizeof(n) * CHAR_BIT - 1); \ + int volatile mask = (n) >> (sizeof(n) * CHAR_BIT - 1); \ int abs_n = ((n) + mask) ^ mask; \ int idx_n = abs_n >> 1; \ secp256k1_fe neg_y; \ @@ -130,7 +130,7 @@ static int secp256k1_wnaf_const(int *wnaf, const secp256k1_scalar *scalar, int w return skew; } -static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, const secp256k1_scalar *scalar, int size) { +static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, const secp256k1_scalar *scalar) { secp256k1_ge pre_a[ECMULT_TABLE_SIZE(WINDOW_A)]; secp256k1_ge tmpa; secp256k1_fe Z; @@ -144,20 +144,17 @@ static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, cons int i; - /* build wnaf representation for q. */ - int rsize = size; - if (size > 128) { - rsize = 128; - /* split q into q_1 and q_lam (where q = q_1 + q_lam*lambda, and q_1 and q_lam are ~128 bit) */ - secp256k1_scalar_split_lambda(&q_1, &q_lam, scalar); - skew_1 = secp256k1_wnaf_const(wnaf_1, &q_1, WINDOW_A - 1, 128); - skew_lam = secp256k1_wnaf_const(wnaf_lam, &q_lam, WINDOW_A - 1, 128); - } else - { - skew_1 = secp256k1_wnaf_const(wnaf_1, scalar, WINDOW_A - 1, size); - skew_lam = 0; + if (secp256k1_ge_is_infinity(a)) { + secp256k1_gej_set_infinity(r); + return; } + /* build wnaf representation for q. */ + /* split q into q_1 and q_lam (where q = q_1 + q_lam*lambda, and q_1 and q_lam are ~128 bit) */ + secp256k1_scalar_split_lambda(&q_1, &q_lam, scalar); + skew_1 = secp256k1_wnaf_const(wnaf_1, &q_1, WINDOW_A - 1, 128); + skew_lam = secp256k1_wnaf_const(wnaf_lam, &q_lam, WINDOW_A - 1, 128); + /* Calculate odd multiples of a. * All multiples are brought to the same Z 'denominator', which is stored * in Z. Due to secp256k1' isomorphism we can do all operations pretending @@ -170,28 +167,23 @@ static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, cons for (i = 0; i < ECMULT_TABLE_SIZE(WINDOW_A); i++) { secp256k1_fe_normalize_weak(&pre_a[i].y); } - if (size > 128) { - for (i = 0; i < ECMULT_TABLE_SIZE(WINDOW_A); i++) { - secp256k1_ge_mul_lambda(&pre_a_lam[i], &pre_a[i]); - } - + for (i = 0; i < ECMULT_TABLE_SIZE(WINDOW_A); i++) { + secp256k1_ge_mul_lambda(&pre_a_lam[i], &pre_a[i]); } /* first loop iteration (separated out so we can directly set r, rather * than having it start at infinity, get doubled several times, then have * its new value added to it) */ - i = wnaf_1[WNAF_SIZE_BITS(rsize, WINDOW_A - 1)]; + i = wnaf_1[WNAF_SIZE_BITS(128, WINDOW_A - 1)]; VERIFY_CHECK(i != 0); ECMULT_CONST_TABLE_GET_GE(&tmpa, pre_a, i, WINDOW_A); secp256k1_gej_set_ge(r, &tmpa); - if (size > 128) { - i = wnaf_lam[WNAF_SIZE_BITS(rsize, WINDOW_A - 1)]; - VERIFY_CHECK(i != 0); - ECMULT_CONST_TABLE_GET_GE(&tmpa, pre_a_lam, i, WINDOW_A); - secp256k1_gej_add_ge(r, r, &tmpa); - } + i = wnaf_lam[WNAF_SIZE_BITS(128, WINDOW_A - 1)]; + VERIFY_CHECK(i != 0); + ECMULT_CONST_TABLE_GET_GE(&tmpa, pre_a_lam, i, WINDOW_A); + secp256k1_gej_add_ge(r, r, &tmpa); /* remaining loop iterations */ - for (i = WNAF_SIZE_BITS(rsize, WINDOW_A - 1) - 1; i >= 0; i--) { + for (i = WNAF_SIZE_BITS(128, WINDOW_A - 1) - 1; i >= 0; i--) { int n; int j; for (j = 0; j < WINDOW_A - 1; ++j) { @@ -202,12 +194,10 @@ static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, cons ECMULT_CONST_TABLE_GET_GE(&tmpa, pre_a, n, WINDOW_A); VERIFY_CHECK(n != 0); secp256k1_gej_add_ge(r, r, &tmpa); - if (size > 128) { - n = wnaf_lam[i]; - ECMULT_CONST_TABLE_GET_GE(&tmpa, pre_a_lam, n, WINDOW_A); - VERIFY_CHECK(n != 0); - secp256k1_gej_add_ge(r, r, &tmpa); - } + n = wnaf_lam[i]; + ECMULT_CONST_TABLE_GET_GE(&tmpa, pre_a_lam, n, WINDOW_A); + VERIFY_CHECK(n != 0); + secp256k1_gej_add_ge(r, r, &tmpa); } { @@ -218,17 +208,15 @@ static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, cons secp256k1_gej_add_ge(&tmpj, r, &tmpa); secp256k1_gej_cmov(r, &tmpj, skew_1); - if (size > 128) { - secp256k1_ge_neg(&tmpa, &pre_a_lam[0]); - secp256k1_gej_add_ge(&tmpj, r, &tmpa); - secp256k1_gej_cmov(r, &tmpj, skew_lam); - } + secp256k1_ge_neg(&tmpa, &pre_a_lam[0]); + secp256k1_gej_add_ge(&tmpj, r, &tmpa); + secp256k1_gej_cmov(r, &tmpj, skew_lam); } secp256k1_fe_mul(&r->z, &r->z, &Z); } -static int secp256k1_ecmult_const_xonly(secp256k1_fe* r, const secp256k1_fe *n, const secp256k1_fe *d, const secp256k1_scalar *q, int bits, int known_on_curve) { +static int secp256k1_ecmult_const_xonly(secp256k1_fe* r, const secp256k1_fe *n, const secp256k1_fe *d, const secp256k1_scalar *q, int known_on_curve) { /* This algorithm is a generalization of Peter Dettman's technique for * avoiding the square root in a random-basepoint x-only multiplication @@ -346,7 +334,7 @@ static int secp256k1_ecmult_const_xonly(secp256k1_fe* r, const secp256k1_fe *n, #ifdef VERIFY VERIFY_CHECK(!secp256k1_scalar_is_zero(q)); #endif - secp256k1_ecmult_const(&rj, &p, q, bits); + secp256k1_ecmult_const(&rj, &p, q); #ifdef VERIFY VERIFY_CHECK(!secp256k1_gej_is_infinity(&rj)); #endif diff --git a/src/secp256k1/src/ecmult_gen_compute_table_impl.h b/src/secp256k1/src/ecmult_gen_compute_table_impl.h index ff6a2992dc..7d672b9950 100644 --- a/src/secp256k1/src/ecmult_gen_compute_table_impl.h +++ b/src/secp256k1/src/ecmult_gen_compute_table_impl.h @@ -31,7 +31,7 @@ static void secp256k1_ecmult_gen_compute_table(secp256k1_ge_storage* table, cons secp256k1_fe nums_x; secp256k1_ge nums_ge; int r; - r = secp256k1_fe_set_b32(&nums_x, nums_b32); + r = secp256k1_fe_set_b32_limit(&nums_x, nums_b32); (void)r; VERIFY_CHECK(r); r = secp256k1_ge_set_xo_var(&nums_ge, &nums_x, 0); diff --git a/src/secp256k1/src/ecmult_gen_impl.h b/src/secp256k1/src/ecmult_gen_impl.h index 4f5ea9f3c0..af412173e9 100644 --- a/src/secp256k1/src/ecmult_gen_impl.h +++ b/src/secp256k1/src/ecmult_gen_impl.h @@ -87,7 +87,6 @@ static void secp256k1_ecmult_gen_blind(secp256k1_ecmult_gen_context *ctx, const secp256k1_fe s; unsigned char nonce32[32]; secp256k1_rfc6979_hmac_sha256 rng; - int overflow; unsigned char keydata[64]; if (seed32 == NULL) { /* When seed is NULL, reset the initial point and blinding value. */ @@ -106,11 +105,9 @@ static void secp256k1_ecmult_gen_blind(secp256k1_ecmult_gen_context *ctx, const memcpy(keydata + 32, seed32, 32); secp256k1_rfc6979_hmac_sha256_initialize(&rng, keydata, 64); memset(keydata, 0, sizeof(keydata)); - /* Accept unobservably small non-uniformity. */ secp256k1_rfc6979_hmac_sha256_generate(&rng, nonce32, 32); - overflow = !secp256k1_fe_set_b32(&s, nonce32); - overflow |= secp256k1_fe_is_zero(&s); - secp256k1_fe_cmov(&s, &secp256k1_fe_one, overflow); + secp256k1_fe_set_b32_mod(&s, nonce32); + secp256k1_fe_cmov(&s, &secp256k1_fe_one, secp256k1_fe_normalizes_to_zero(&s)); /* Randomize the projection to defend against multiplier sidechannels. Do this before our own call to secp256k1_ecmult_gen below. */ secp256k1_gej_rescale(&ctx->initial, &s); diff --git a/src/secp256k1/src/ecmult_impl.h b/src/secp256k1/src/ecmult_impl.h index a9a63850ef..f4624677d7 100644 --- a/src/secp256k1/src/ecmult_impl.h +++ b/src/secp256k1/src/ecmult_impl.h @@ -279,9 +279,6 @@ static void secp256k1_ecmult_strauss_wnaf(const struct secp256k1_strauss_state * */ tmp = a[np]; if (no) { -#ifdef VERIFY - secp256k1_fe_normalize_var(&Z); -#endif secp256k1_gej_rescale(&tmp, &Z); } secp256k1_ecmult_odd_multiples_table(ECMULT_TABLE_SIZE(WINDOW_A), state->pre_a + no * ECMULT_TABLE_SIZE(WINDOW_A), state->aux + no * ECMULT_TABLE_SIZE(WINDOW_A), &Z, &tmp); @@ -683,7 +680,7 @@ static int secp256k1_ecmult_pippenger_batch(const secp256k1_callback* error_call } state_space->ps = (struct secp256k1_pippenger_point_state *) secp256k1_scratch_alloc(error_callback, scratch, entries * sizeof(*state_space->ps)); state_space->wnaf_na = (int *) secp256k1_scratch_alloc(error_callback, scratch, entries*(WNAF_SIZE(bucket_window+1)) * sizeof(int)); - buckets = (secp256k1_gej *) secp256k1_scratch_alloc(error_callback, scratch, (1<<bucket_window) * sizeof(*buckets)); + buckets = (secp256k1_gej *) secp256k1_scratch_alloc(error_callback, scratch, ((size_t)1 << bucket_window) * sizeof(*buckets)); if (state_space->ps == NULL || state_space->wnaf_na == NULL || buckets == NULL) { secp256k1_scratch_apply_checkpoint(error_callback, scratch, scratch_checkpoint); return 0; @@ -773,14 +770,12 @@ static size_t secp256k1_pippenger_max_points(const secp256k1_callback* error_cal * require a scratch space */ static int secp256k1_ecmult_multi_simple_var(secp256k1_gej *r, const secp256k1_scalar *inp_g_sc, secp256k1_ecmult_multi_callback cb, void *cbdata, size_t n_points) { size_t point_idx; - secp256k1_scalar szero; secp256k1_gej tmpj; - secp256k1_scalar_set_int(&szero, 0); secp256k1_gej_set_infinity(r); secp256k1_gej_set_infinity(&tmpj); /* r = inp_g_sc*G */ - secp256k1_ecmult(r, &tmpj, &szero, inp_g_sc); + secp256k1_ecmult(r, &tmpj, &secp256k1_scalar_zero, inp_g_sc); for (point_idx = 0; point_idx < n_points; point_idx++) { secp256k1_ge point; secp256k1_gej pointj; @@ -828,9 +823,7 @@ static int secp256k1_ecmult_multi_var(const secp256k1_callback* error_callback, if (inp_g_sc == NULL && n == 0) { return 1; } else if (n == 0) { - secp256k1_scalar szero; - secp256k1_scalar_set_int(&szero, 0); - secp256k1_ecmult(r, r, &szero, inp_g_sc); + secp256k1_ecmult(r, r, &secp256k1_scalar_zero, inp_g_sc); return 1; } if (scratch == NULL) { diff --git a/src/secp256k1/src/field.h b/src/secp256k1/src/field.h index 64ceead4d2..e632f9e3e2 100644 --- a/src/secp256k1/src/field.h +++ b/src/secp256k1/src/field.h @@ -7,19 +7,36 @@ #ifndef SECP256K1_FIELD_H #define SECP256K1_FIELD_H -/** Field element module. - * - * Field elements can be represented in several ways, but code accessing - * it (and implementations) need to take certain properties into account: - * - Each field element can be normalized or not. - * - Each field element has a magnitude, which represents how far away - * its representation is away from normalization. Normalized elements - * always have a magnitude of 0 or 1, but a magnitude of 1 doesn't - * imply normality. - */ - #include "util.h" +/* This file defines the generic interface for working with secp256k1_fe + * objects, which represent field elements (integers modulo 2^256 - 2^32 - 977). + * + * The actual definition of the secp256k1_fe type depends on the chosen field + * implementation; see the field_5x52.h and field_10x26.h files for details. + * + * All secp256k1_fe objects have implicit properties that determine what + * operations are permitted on it. These are purely a function of what + * secp256k1_fe_ operations are applied on it, generally (implicitly) fixed at + * compile time, and do not depend on the chosen field implementation. Despite + * that, what these properties actually entail for the field representation + * values depends on the chosen field implementation. These properties are: + * - magnitude: an integer in [0,32] + * - normalized: 0 or 1; normalized=1 implies magnitude <= 1. + * + * In VERIFY mode, they are materialized explicitly as fields in the struct, + * allowing run-time verification of these properties. In that case, the field + * implementation also provides a secp256k1_fe_verify routine to verify that + * these fields match the run-time value and perform internal consistency + * checks. */ +#ifdef VERIFY +# define SECP256K1_FE_VERIFY_FIELDS \ + int magnitude; \ + int normalized; +#else +# define SECP256K1_FE_VERIFY_FIELDS +#endif + #if defined(SECP256K1_WIDEMUL_INT128) #include "field_5x52.h" #elif defined(SECP256K1_WIDEMUL_INT64) @@ -28,117 +45,297 @@ #error "Please select wide multiplication implementation" #endif +#ifdef VERIFY +/* Magnitude and normalized value for constants. */ +#define SECP256K1_FE_VERIFY_CONST(d7, d6, d5, d4, d3, d2, d1, d0) \ + /* Magnitude is 0 for constant 0; 1 otherwise. */ \ + , (((d7) | (d6) | (d5) | (d4) | (d3) | (d2) | (d1) | (d0)) != 0) \ + /* Normalized is 1 unless sum(d_i<<(32*i) for i=0..7) exceeds field modulus. */ \ + , (!(((d7) & (d6) & (d5) & (d4) & (d3) & (d2)) == 0xfffffffful && ((d1) == 0xfffffffful || ((d1) == 0xfffffffe && (d0 >= 0xfffffc2f))))) +#else +#define SECP256K1_FE_VERIFY_CONST(d7, d6, d5, d4, d3, d2, d1, d0) +#endif + +/** This expands to an initializer for a secp256k1_fe valued sum((i*32) * d_i, i=0..7) mod p. + * + * It has magnitude 1, unless d_i are all 0, in which case the magnitude is 0. + * It is normalized, unless sum(2^(i*32) * d_i, i=0..7) >= p. + * + * SECP256K1_FE_CONST_INNER is provided by the implementation. + */ +#define SECP256K1_FE_CONST(d7, d6, d5, d4, d3, d2, d1, d0) {SECP256K1_FE_CONST_INNER((d7), (d6), (d5), (d4), (d3), (d2), (d1), (d0)) SECP256K1_FE_VERIFY_CONST((d7), (d6), (d5), (d4), (d3), (d2), (d1), (d0)) } + static const secp256k1_fe secp256k1_fe_one = SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 1); static const secp256k1_fe secp256k1_const_beta = SECP256K1_FE_CONST( 0x7ae96a2bul, 0x657c0710ul, 0x6e64479eul, 0xac3434e9ul, 0x9cf04975ul, 0x12f58995ul, 0xc1396c28ul, 0x719501eeul ); -/** Normalize a field element. This brings the field element to a canonical representation, reduces - * its magnitude to 1, and reduces it modulo field size `p`. +#ifndef VERIFY +/* In non-VERIFY mode, we #define the fe operations to be identical to their + * internal field implementation, to avoid the potential overhead of a + * function call (even though presumably inlinable). */ +# define secp256k1_fe_normalize secp256k1_fe_impl_normalize +# define secp256k1_fe_normalize_weak secp256k1_fe_impl_normalize_weak +# define secp256k1_fe_normalize_var secp256k1_fe_impl_normalize_var +# define secp256k1_fe_normalizes_to_zero secp256k1_fe_impl_normalizes_to_zero +# define secp256k1_fe_normalizes_to_zero_var secp256k1_fe_impl_normalizes_to_zero_var +# define secp256k1_fe_set_int secp256k1_fe_impl_set_int +# define secp256k1_fe_clear secp256k1_fe_impl_clear +# define secp256k1_fe_is_zero secp256k1_fe_impl_is_zero +# define secp256k1_fe_is_odd secp256k1_fe_impl_is_odd +# define secp256k1_fe_cmp_var secp256k1_fe_impl_cmp_var +# define secp256k1_fe_set_b32_mod secp256k1_fe_impl_set_b32_mod +# define secp256k1_fe_set_b32_limit secp256k1_fe_impl_set_b32_limit +# define secp256k1_fe_get_b32 secp256k1_fe_impl_get_b32 +# define secp256k1_fe_negate secp256k1_fe_impl_negate +# define secp256k1_fe_mul_int secp256k1_fe_impl_mul_int +# define secp256k1_fe_add secp256k1_fe_impl_add +# define secp256k1_fe_mul secp256k1_fe_impl_mul +# define secp256k1_fe_sqr secp256k1_fe_impl_sqr +# define secp256k1_fe_cmov secp256k1_fe_impl_cmov +# define secp256k1_fe_to_storage secp256k1_fe_impl_to_storage +# define secp256k1_fe_from_storage secp256k1_fe_impl_from_storage +# define secp256k1_fe_inv secp256k1_fe_impl_inv +# define secp256k1_fe_inv_var secp256k1_fe_impl_inv_var +# define secp256k1_fe_get_bounds secp256k1_fe_impl_get_bounds +# define secp256k1_fe_half secp256k1_fe_impl_half +# define secp256k1_fe_add_int secp256k1_fe_impl_add_int +# define secp256k1_fe_is_square_var secp256k1_fe_impl_is_square_var +#endif /* !defined(VERIFY) */ + +/** Normalize a field element. + * + * On input, r must be a valid field element. + * On output, r represents the same value but has normalized=1 and magnitude=1. */ static void secp256k1_fe_normalize(secp256k1_fe *r); -/** Weakly normalize a field element: reduce its magnitude to 1, but don't fully normalize. */ +/** Give a field element magnitude 1. + * + * On input, r must be a valid field element. + * On output, r represents the same value but has magnitude=1. Normalized is unchanged. + */ static void secp256k1_fe_normalize_weak(secp256k1_fe *r); -/** Normalize a field element, without constant-time guarantee. */ +/** Normalize a field element, without constant-time guarantee. + * + * Identical in behavior to secp256k1_fe_normalize, but not constant time in r. + */ static void secp256k1_fe_normalize_var(secp256k1_fe *r); -/** Verify whether a field element represents zero i.e. would normalize to a zero value. */ +/** Determine whether r represents field element 0. + * + * On input, r must be a valid field element. + * Returns whether r = 0 (mod p). + */ static int secp256k1_fe_normalizes_to_zero(const secp256k1_fe *r); -/** Verify whether a field element represents zero i.e. would normalize to a zero value, - * without constant-time guarantee. */ +/** Determine whether r represents field element 0, without constant-time guarantee. + * + * Identical in behavior to secp256k1_normalizes_to_zero, but not constant time in r. + */ static int secp256k1_fe_normalizes_to_zero_var(const secp256k1_fe *r); -/** Set a field element equal to a small (not greater than 0x7FFF), non-negative integer. - * Resulting field element is normalized; it has magnitude 0 if a == 0, and magnitude 1 otherwise. +/** Set a field element to an integer in range [0,0x7FFF]. + * + * On input, r does not need to be initialized, a must be in [0,0x7FFF]. + * On output, r represents value a, is normalized and has magnitude (a!=0). */ static void secp256k1_fe_set_int(secp256k1_fe *r, int a); -/** Sets a field element equal to zero, initializing all fields. */ +/** Set a field element to 0. + * + * On input, a does not need to be initialized. + * On output, a represents 0, is normalized and has magnitude 0. + */ static void secp256k1_fe_clear(secp256k1_fe *a); -/** Verify whether a field element is zero. Requires the input to be normalized. */ +/** Determine whether a represents field element 0. + * + * On input, a must be a valid normalized field element. + * Returns whether a = 0 (mod p). + * + * This behaves identical to secp256k1_normalizes_to_zero{,_var}, but requires + * normalized input (and is much faster). + */ static int secp256k1_fe_is_zero(const secp256k1_fe *a); -/** Check the "oddness" of a field element. Requires the input to be normalized. */ +/** Determine whether a (mod p) is odd. + * + * On input, a must be a valid normalized field element. + * Returns (int(a) mod p) & 1. + */ static int secp256k1_fe_is_odd(const secp256k1_fe *a); -/** Compare two field elements. Requires magnitude-1 inputs. */ +/** Determine whether two field elements are equal. + * + * On input, a and b must be valid field elements with magnitudes not exceeding + * 1 and 31, respectively. + * Returns a = b (mod p). + */ static int secp256k1_fe_equal(const secp256k1_fe *a, const secp256k1_fe *b); -/** Same as secp256k1_fe_equal, but may be variable time. */ +/** Determine whether two field elements are equal, without constant-time guarantee. + * + * Identical in behavior to secp256k1_fe_equal, but not constant time in either a or b. + */ static int secp256k1_fe_equal_var(const secp256k1_fe *a, const secp256k1_fe *b); -/** Compare two field elements. Requires both inputs to be normalized */ +/** Compare the values represented by 2 field elements, without constant-time guarantee. + * + * On input, a and b must be valid normalized field elements. + * Returns 1 if a > b, -1 if a < b, and 0 if a = b (comparisons are done as integers + * in range 0..p-1). + */ static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b); -/** Set a field element equal to 32-byte big endian value. If successful, the resulting field element is normalized. */ -static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a); +/** Set a field element equal to a provided 32-byte big endian value, reducing it. + * + * On input, r does not need to be initalized. a must be a pointer to an initialized 32-byte array. + * On output, r = a (mod p). It will have magnitude 1, and not be normalized. + */ +static void secp256k1_fe_set_b32_mod(secp256k1_fe *r, const unsigned char *a); + +/** Set a field element equal to a provided 32-byte big endian value, checking for overflow. + * + * On input, r does not need to be initalized. a must be a pointer to an initialized 32-byte array. + * On output, r = a if (a < p), it will be normalized with magnitude 1, and 1 is returned. + * If a >= p, 0 is returned, and r will be made invalid (and must not be used without overwriting). + */ +static int secp256k1_fe_set_b32_limit(secp256k1_fe *r, const unsigned char *a); -/** Convert a field element to a 32-byte big endian value. Requires the input to be normalized */ +/** Convert a field element to 32-byte big endian byte array. + * On input, a must be a valid normalized field element, and r a pointer to a 32-byte array. + * On output, r = a (mod p). + */ static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a); -/** Set a field element equal to the additive inverse of another. Takes a maximum magnitude of the input - * as an argument. The magnitude of the output is one higher. */ +/** Negate a field element. + * + * On input, r does not need to be initialized. a must be a valid field element with + * magnitude not exceeding m. m must be an integer in [0,31]. + * Performs {r = -a}. + * On output, r will not be normalized, and will have magnitude m+1. + */ static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k1_fe *a, int m); -/** Adds a small integer (up to 0x7FFF) to r. The resulting magnitude increases by one. */ +/** Add a small integer to a field element. + * + * Performs {r += a}. The magnitude of r increases by 1, and normalized is cleared. + * a must be in range [0,0x7FFF]. + */ static void secp256k1_fe_add_int(secp256k1_fe *r, int a); -/** Multiplies the passed field element with a small integer constant. Multiplies the magnitude by that - * small integer. */ +/** Multiply a field element with a small integer. + * + * On input, r must be a valid field element. a must be an integer in [0,32]. + * The magnitude of r times a must not exceed 32. + * Performs {r *= a}. + * On output, r's magnitude is multiplied by a, and r will not be normalized. + */ static void secp256k1_fe_mul_int(secp256k1_fe *r, int a); -/** Adds a field element to another. The result has the sum of the inputs' magnitudes as magnitude. */ +/** Increment a field element by another. + * + * On input, r and a must be valid field elements, not necessarily normalized. + * The sum of their magnitudes must not exceed 32. + * Performs {r += a}. + * On output, r will not be normalized, and will have magnitude incremented by a's. + */ static void secp256k1_fe_add(secp256k1_fe *r, const secp256k1_fe *a); -/** Sets a field element to be the product of two others. Requires the inputs' magnitudes to be at most 8. - * The output magnitude is 1 (but not guaranteed to be normalized). */ +/** Multiply two field elements. + * + * On input, a and b must be valid field elements; r does not need to be initialized. + * r and a may point to the same object, but neither can be equal to b. The magnitudes + * of a and b must not exceed 8. + * Performs {r = a * b} + * On output, r will have magnitude 1, but won't be normalized. + */ static void secp256k1_fe_mul(secp256k1_fe *r, const secp256k1_fe *a, const secp256k1_fe * SECP256K1_RESTRICT b); -/** Sets a field element to be the square of another. Requires the input's magnitude to be at most 8. - * The output magnitude is 1 (but not guaranteed to be normalized). */ +/** Square a field element. + * + * On input, a must be a valid field element; r does not need to be initialized. The magnitude + * of a must not exceed 8. + * Performs {r = a**2} + * On output, r will have magnitude 1, but won't be normalized. + */ static void secp256k1_fe_sqr(secp256k1_fe *r, const secp256k1_fe *a); -/** If a has a square root, it is computed in r and 1 is returned. If a does not - * have a square root, the root of its negation is computed and 0 is returned. - * The input's magnitude can be at most 8. The output magnitude is 1 (but not - * guaranteed to be normalized). The result in r will always be a square - * itself. */ -static int secp256k1_fe_sqrt(secp256k1_fe *r, const secp256k1_fe *a); +/** Compute a square root of a field element. + * + * On input, a must be a valid field element with magnitude<=8; r need not be initialized. + * Performs {r = sqrt(a)} or {r = sqrt(-a)}, whichever exists. The resulting value + * represented by r will be a square itself. Variables r and a must not point to the same object. + * On output, r will have magnitude 1 but will not be normalized. + */ +static int secp256k1_fe_sqrt(secp256k1_fe * SECP256K1_RESTRICT r, const secp256k1_fe * SECP256K1_RESTRICT a); -/** Sets a field element to be the (modular) inverse of another. Requires the input's magnitude to be - * at most 8. The output magnitude is 1 (but not guaranteed to be normalized). */ +/** Compute the modular inverse of a field element. + * + * On input, a must be a valid field element; r need not be initialized. + * Performs {r = a**(p-2)} (which maps 0 to 0, and every other element to its + * inverse). + * On output, r will have magnitude (a.magnitude != 0) and be normalized. + */ static void secp256k1_fe_inv(secp256k1_fe *r, const secp256k1_fe *a); -/** Potentially faster version of secp256k1_fe_inv, without constant-time guarantee. */ +/** Compute the modular inverse of a field element, without constant-time guarantee. + * + * Behaves identically to secp256k1_fe_inv, but is not constant-time in a. + */ static void secp256k1_fe_inv_var(secp256k1_fe *r, const secp256k1_fe *a); -/** Convert a field element to the storage type. */ +/** Convert a field element to secp256k1_fe_storage. + * + * On input, a must be a valid normalized field element. + * Performs {r = a}. + */ static void secp256k1_fe_to_storage(secp256k1_fe_storage *r, const secp256k1_fe *a); -/** Convert a field element back from the storage type. */ +/** Convert a field element back from secp256k1_fe_storage. + * + * On input, r need not be initialized. + * Performs {r = a}. + * On output, r will be normalized and will have magnitude 1. + */ static void secp256k1_fe_from_storage(secp256k1_fe *r, const secp256k1_fe_storage *a); /** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. Both *r and *a must be initialized.*/ static void secp256k1_fe_storage_cmov(secp256k1_fe_storage *r, const secp256k1_fe_storage *a, int flag); -/** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. Both *r and *a must be initialized.*/ +/** Conditionally move a field element in constant time. + * + * On input, both r and a must be valid field elements. Flag must be 0 or 1. + * Performs {r = flag ? a : r}. + * + * On output, r's magnitude will be the maximum of both input magnitudes. + * It will be normalized if and only if both inputs were normalized. + */ static void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag); -/** Halves the value of a field element modulo the field prime. Constant-time. - * For an input magnitude 'm', the output magnitude is set to 'floor(m/2) + 1'. - * The output is not guaranteed to be normalized, regardless of the input. */ +/** Halve the value of a field element modulo the field prime in constant-time. + * + * On input, r must be a valid field element. + * On output, r will be normalized and have magnitude floor(m/2) + 1 where m is + * the magnitude of r on input. + */ static void secp256k1_fe_half(secp256k1_fe *r); -/** Sets each limb of 'r' to its upper bound at magnitude 'm'. The output will also have its - * magnitude set to 'm' and is normalized if (and only if) 'm' is zero. */ +/** Sets r to a field element with magnitude m, normalized if (and only if) m==0. + * The value is chosen so that it is likely to trigger edge cases related to + * internal overflows. */ static void secp256k1_fe_get_bounds(secp256k1_fe *r, int m); -/** Determine whether a is a square (modulo p). */ +/** Determine whether a is a square (modulo p). + * + * On input, a must be a valid field element. + */ static int secp256k1_fe_is_square_var(const secp256k1_fe *a); +/** Check invariants on a field element (no-op unless VERIFY is enabled). */ +static void secp256k1_fe_verify(const secp256k1_fe *a); + #endif /* SECP256K1_FIELD_H */ diff --git a/src/secp256k1/src/field_10x26.h b/src/secp256k1/src/field_10x26.h index 9eb65607f1..203c10167c 100644 --- a/src/secp256k1/src/field_10x26.h +++ b/src/secp256k1/src/field_10x26.h @@ -9,15 +9,28 @@ #include <stdint.h> +/** This field implementation represents the value as 10 uint32_t limbs in base + * 2^26. */ typedef struct { - /* X = sum(i=0..9, n[i]*2^(i*26)) mod p - * where p = 2^256 - 0x1000003D1 - */ + /* A field element f represents the sum(i=0..9, f.n[i] << (i*26)) mod p, + * where p is the field modulus, 2^256 - 2^32 - 977. + * + * The individual limbs f.n[i] can exceed 2^26; the field's magnitude roughly + * corresponds to how much excess is allowed. The value + * sum(i=0..9, f.n[i] << (i*26)) may exceed p, unless the field element is + * normalized. */ uint32_t n[10]; -#ifdef VERIFY - int magnitude; - int normalized; -#endif + /* + * Magnitude m requires: + * n[i] <= 2 * m * (2^26 - 1) for i=0..8 + * n[9] <= 2 * m * (2^22 - 1) + * + * Normalized requires: + * n[i] <= (2^26 - 1) for i=0..8 + * sum(i=0..9, n[i] << (i*26)) < p + * (together these imply n[9] <= 2^22 - 1) + */ + SECP256K1_FE_VERIFY_FIELDS } secp256k1_fe; /* Unpacks a constant into a overlapping multi-limbed FE element. */ @@ -34,12 +47,6 @@ typedef struct { (((uint32_t)d7) >> 10) \ } -#ifdef VERIFY -#define SECP256K1_FE_CONST(d7, d6, d5, d4, d3, d2, d1, d0) {SECP256K1_FE_CONST_INNER((d7), (d6), (d5), (d4), (d3), (d2), (d1), (d0)), 1, 1} -#else -#define SECP256K1_FE_CONST(d7, d6, d5, d4, d3, d2, d1, d0) {SECP256K1_FE_CONST_INNER((d7), (d6), (d5), (d4), (d3), (d2), (d1), (d0))} -#endif - typedef struct { uint32_t n[8]; } secp256k1_fe_storage; diff --git a/src/secp256k1/src/field_10x26_impl.h b/src/secp256k1/src/field_10x26_impl.h index 3b7f4d2480..c1b32b80a8 100644 --- a/src/secp256k1/src/field_10x26_impl.h +++ b/src/secp256k1/src/field_10x26_impl.h @@ -12,47 +12,32 @@ #include "field.h" #include "modinv32_impl.h" -/** See the comment at the top of field_5x52_impl.h for more details. - * - * Here, we represent field elements as 10 uint32_t's in base 2^26, least significant first, - * where limbs can contain >26 bits. - * A magnitude M means: - * - 2*M*(2^22-1) is the max (inclusive) of the most significant limb - * - 2*M*(2^26-1) is the max (inclusive) of the remaining limbs - */ - #ifdef VERIFY -static void secp256k1_fe_verify(const secp256k1_fe *a) { +static void secp256k1_fe_impl_verify(const secp256k1_fe *a) { const uint32_t *d = a->n; - int m = a->normalized ? 1 : 2 * a->magnitude, r = 1; - r &= (d[0] <= 0x3FFFFFFUL * m); - r &= (d[1] <= 0x3FFFFFFUL * m); - r &= (d[2] <= 0x3FFFFFFUL * m); - r &= (d[3] <= 0x3FFFFFFUL * m); - r &= (d[4] <= 0x3FFFFFFUL * m); - r &= (d[5] <= 0x3FFFFFFUL * m); - r &= (d[6] <= 0x3FFFFFFUL * m); - r &= (d[7] <= 0x3FFFFFFUL * m); - r &= (d[8] <= 0x3FFFFFFUL * m); - r &= (d[9] <= 0x03FFFFFUL * m); - r &= (a->magnitude >= 0); - r &= (a->magnitude <= 32); + int m = a->normalized ? 1 : 2 * a->magnitude; + VERIFY_CHECK(d[0] <= 0x3FFFFFFUL * m); + VERIFY_CHECK(d[1] <= 0x3FFFFFFUL * m); + VERIFY_CHECK(d[2] <= 0x3FFFFFFUL * m); + VERIFY_CHECK(d[3] <= 0x3FFFFFFUL * m); + VERIFY_CHECK(d[4] <= 0x3FFFFFFUL * m); + VERIFY_CHECK(d[5] <= 0x3FFFFFFUL * m); + VERIFY_CHECK(d[6] <= 0x3FFFFFFUL * m); + VERIFY_CHECK(d[7] <= 0x3FFFFFFUL * m); + VERIFY_CHECK(d[8] <= 0x3FFFFFFUL * m); + VERIFY_CHECK(d[9] <= 0x03FFFFFUL * m); if (a->normalized) { - r &= (a->magnitude <= 1); - if (r && (d[9] == 0x03FFFFFUL)) { + if (d[9] == 0x03FFFFFUL) { uint32_t mid = d[8] & d[7] & d[6] & d[5] & d[4] & d[3] & d[2]; if (mid == 0x3FFFFFFUL) { - r &= ((d[1] + 0x40UL + ((d[0] + 0x3D1UL) >> 26)) <= 0x3FFFFFFUL); + VERIFY_CHECK((d[1] + 0x40UL + ((d[0] + 0x3D1UL) >> 26)) <= 0x3FFFFFFUL); } } } - VERIFY_CHECK(r == 1); } #endif -static void secp256k1_fe_get_bounds(secp256k1_fe *r, int m) { - VERIFY_CHECK(m >= 0); - VERIFY_CHECK(m <= 2048); +static void secp256k1_fe_impl_get_bounds(secp256k1_fe *r, int m) { r->n[0] = 0x3FFFFFFUL * 2 * m; r->n[1] = 0x3FFFFFFUL * 2 * m; r->n[2] = 0x3FFFFFFUL * 2 * m; @@ -63,14 +48,9 @@ static void secp256k1_fe_get_bounds(secp256k1_fe *r, int m) { r->n[7] = 0x3FFFFFFUL * 2 * m; r->n[8] = 0x3FFFFFFUL * 2 * m; r->n[9] = 0x03FFFFFUL * 2 * m; -#ifdef VERIFY - r->magnitude = m; - r->normalized = (m == 0); - secp256k1_fe_verify(r); -#endif } -static void secp256k1_fe_normalize(secp256k1_fe *r) { +static void secp256k1_fe_impl_normalize(secp256k1_fe *r) { uint32_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4], t5 = r->n[5], t6 = r->n[6], t7 = r->n[7], t8 = r->n[8], t9 = r->n[9]; @@ -117,15 +97,9 @@ static void secp256k1_fe_normalize(secp256k1_fe *r) { r->n[0] = t0; r->n[1] = t1; r->n[2] = t2; r->n[3] = t3; r->n[4] = t4; r->n[5] = t5; r->n[6] = t6; r->n[7] = t7; r->n[8] = t8; r->n[9] = t9; - -#ifdef VERIFY - r->magnitude = 1; - r->normalized = 1; - secp256k1_fe_verify(r); -#endif } -static void secp256k1_fe_normalize_weak(secp256k1_fe *r) { +static void secp256k1_fe_impl_normalize_weak(secp256k1_fe *r) { uint32_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4], t5 = r->n[5], t6 = r->n[6], t7 = r->n[7], t8 = r->n[8], t9 = r->n[9]; @@ -149,14 +123,9 @@ static void secp256k1_fe_normalize_weak(secp256k1_fe *r) { r->n[0] = t0; r->n[1] = t1; r->n[2] = t2; r->n[3] = t3; r->n[4] = t4; r->n[5] = t5; r->n[6] = t6; r->n[7] = t7; r->n[8] = t8; r->n[9] = t9; - -#ifdef VERIFY - r->magnitude = 1; - secp256k1_fe_verify(r); -#endif } -static void secp256k1_fe_normalize_var(secp256k1_fe *r) { +static void secp256k1_fe_impl_normalize_var(secp256k1_fe *r) { uint32_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4], t5 = r->n[5], t6 = r->n[6], t7 = r->n[7], t8 = r->n[8], t9 = r->n[9]; @@ -204,15 +173,9 @@ static void secp256k1_fe_normalize_var(secp256k1_fe *r) { r->n[0] = t0; r->n[1] = t1; r->n[2] = t2; r->n[3] = t3; r->n[4] = t4; r->n[5] = t5; r->n[6] = t6; r->n[7] = t7; r->n[8] = t8; r->n[9] = t9; - -#ifdef VERIFY - r->magnitude = 1; - r->normalized = 1; - secp256k1_fe_verify(r); -#endif } -static int secp256k1_fe_normalizes_to_zero(const secp256k1_fe *r) { +static int secp256k1_fe_impl_normalizes_to_zero(const secp256k1_fe *r) { uint32_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4], t5 = r->n[5], t6 = r->n[6], t7 = r->n[7], t8 = r->n[8], t9 = r->n[9]; @@ -241,7 +204,7 @@ static int secp256k1_fe_normalizes_to_zero(const secp256k1_fe *r) { return (z0 == 0) | (z1 == 0x3FFFFFFUL); } -static int secp256k1_fe_normalizes_to_zero_var(const secp256k1_fe *r) { +static int secp256k1_fe_impl_normalizes_to_zero_var(const secp256k1_fe *r) { uint32_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9; uint32_t z0, z1; uint32_t x; @@ -293,53 +256,29 @@ static int secp256k1_fe_normalizes_to_zero_var(const secp256k1_fe *r) { return (z0 == 0) | (z1 == 0x3FFFFFFUL); } -SECP256K1_INLINE static void secp256k1_fe_set_int(secp256k1_fe *r, int a) { - VERIFY_CHECK(0 <= a && a <= 0x7FFF); +SECP256K1_INLINE static void secp256k1_fe_impl_set_int(secp256k1_fe *r, int a) { r->n[0] = a; r->n[1] = r->n[2] = r->n[3] = r->n[4] = r->n[5] = r->n[6] = r->n[7] = r->n[8] = r->n[9] = 0; -#ifdef VERIFY - r->magnitude = (a != 0); - r->normalized = 1; - secp256k1_fe_verify(r); -#endif } -SECP256K1_INLINE static int secp256k1_fe_is_zero(const secp256k1_fe *a) { +SECP256K1_INLINE static int secp256k1_fe_impl_is_zero(const secp256k1_fe *a) { const uint32_t *t = a->n; -#ifdef VERIFY - VERIFY_CHECK(a->normalized); - secp256k1_fe_verify(a); -#endif return (t[0] | t[1] | t[2] | t[3] | t[4] | t[5] | t[6] | t[7] | t[8] | t[9]) == 0; } -SECP256K1_INLINE static int secp256k1_fe_is_odd(const secp256k1_fe *a) { -#ifdef VERIFY - VERIFY_CHECK(a->normalized); - secp256k1_fe_verify(a); -#endif +SECP256K1_INLINE static int secp256k1_fe_impl_is_odd(const secp256k1_fe *a) { return a->n[0] & 1; } -SECP256K1_INLINE static void secp256k1_fe_clear(secp256k1_fe *a) { +SECP256K1_INLINE static void secp256k1_fe_impl_clear(secp256k1_fe *a) { int i; -#ifdef VERIFY - a->magnitude = 0; - a->normalized = 1; -#endif for (i=0; i<10; i++) { a->n[i] = 0; } } -static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { +static int secp256k1_fe_impl_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { int i; -#ifdef VERIFY - VERIFY_CHECK(a->normalized); - VERIFY_CHECK(b->normalized); - secp256k1_fe_verify(a); - secp256k1_fe_verify(b); -#endif for (i = 9; i >= 0; i--) { if (a->n[i] > b->n[i]) { return 1; @@ -351,8 +290,7 @@ static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { return 0; } -static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { - int ret; +static void secp256k1_fe_impl_set_b32_mod(secp256k1_fe *r, const unsigned char *a) { r->n[0] = (uint32_t)a[31] | ((uint32_t)a[30] << 8) | ((uint32_t)a[29] << 16) | ((uint32_t)(a[28] & 0x3) << 24); r->n[1] = (uint32_t)((a[28] >> 2) & 0x3f) | ((uint32_t)a[27] << 6) | ((uint32_t)a[26] << 14) | ((uint32_t)(a[25] & 0xf) << 22); r->n[2] = (uint32_t)((a[25] >> 4) & 0xf) | ((uint32_t)a[24] << 4) | ((uint32_t)a[23] << 12) | ((uint32_t)(a[22] & 0x3f) << 20); @@ -363,26 +301,15 @@ static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { r->n[7] = (uint32_t)((a[9] >> 6) & 0x3) | ((uint32_t)a[8] << 2) | ((uint32_t)a[7] << 10) | ((uint32_t)a[6] << 18); r->n[8] = (uint32_t)a[5] | ((uint32_t)a[4] << 8) | ((uint32_t)a[3] << 16) | ((uint32_t)(a[2] & 0x3) << 24); r->n[9] = (uint32_t)((a[2] >> 2) & 0x3f) | ((uint32_t)a[1] << 6) | ((uint32_t)a[0] << 14); +} - ret = !((r->n[9] == 0x3FFFFFUL) & ((r->n[8] & r->n[7] & r->n[6] & r->n[5] & r->n[4] & r->n[3] & r->n[2]) == 0x3FFFFFFUL) & ((r->n[1] + 0x40UL + ((r->n[0] + 0x3D1UL) >> 26)) > 0x3FFFFFFUL)); -#ifdef VERIFY - r->magnitude = 1; - if (ret) { - r->normalized = 1; - secp256k1_fe_verify(r); - } else { - r->normalized = 0; - } -#endif - return ret; +static int secp256k1_fe_impl_set_b32_limit(secp256k1_fe *r, const unsigned char *a) { + secp256k1_fe_impl_set_b32_mod(r, a); + return !((r->n[9] == 0x3FFFFFUL) & ((r->n[8] & r->n[7] & r->n[6] & r->n[5] & r->n[4] & r->n[3] & r->n[2]) == 0x3FFFFFFUL) & ((r->n[1] + 0x40UL + ((r->n[0] + 0x3D1UL) >> 26)) > 0x3FFFFFFUL)); } /** Convert a field element to a 32-byte big endian value. Requires the input to be normalized */ -static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a) { -#ifdef VERIFY - VERIFY_CHECK(a->normalized); - secp256k1_fe_verify(a); -#endif +static void secp256k1_fe_impl_get_b32(unsigned char *r, const secp256k1_fe *a) { r[0] = (a->n[9] >> 14) & 0xff; r[1] = (a->n[9] >> 6) & 0xff; r[2] = ((a->n[9] & 0x3F) << 2) | ((a->n[8] >> 24) & 0x3); @@ -417,15 +344,15 @@ static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a) { r[31] = a->n[0] & 0xff; } -SECP256K1_INLINE static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k1_fe *a, int m) { -#ifdef VERIFY - VERIFY_CHECK(a->magnitude <= m); - secp256k1_fe_verify(a); +SECP256K1_INLINE static void secp256k1_fe_impl_negate(secp256k1_fe *r, const secp256k1_fe *a, int m) { + /* For all legal values of m (0..31), the following properties hold: */ VERIFY_CHECK(0x3FFFC2FUL * 2 * (m + 1) >= 0x3FFFFFFUL * 2 * m); VERIFY_CHECK(0x3FFFFBFUL * 2 * (m + 1) >= 0x3FFFFFFUL * 2 * m); VERIFY_CHECK(0x3FFFFFFUL * 2 * (m + 1) >= 0x3FFFFFFUL * 2 * m); VERIFY_CHECK(0x03FFFFFUL * 2 * (m + 1) >= 0x03FFFFFUL * 2 * m); -#endif + + /* Due to the properties above, the left hand in the subtractions below is never less than + * the right hand. */ r->n[0] = 0x3FFFC2FUL * 2 * (m + 1) - a->n[0]; r->n[1] = 0x3FFFFBFUL * 2 * (m + 1) - a->n[1]; r->n[2] = 0x3FFFFFFUL * 2 * (m + 1) - a->n[2]; @@ -436,14 +363,9 @@ SECP256K1_INLINE static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k r->n[7] = 0x3FFFFFFUL * 2 * (m + 1) - a->n[7]; r->n[8] = 0x3FFFFFFUL * 2 * (m + 1) - a->n[8]; r->n[9] = 0x03FFFFFUL * 2 * (m + 1) - a->n[9]; -#ifdef VERIFY - r->magnitude = m + 1; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } -SECP256K1_INLINE static void secp256k1_fe_mul_int(secp256k1_fe *r, int a) { +SECP256K1_INLINE static void secp256k1_fe_impl_mul_int(secp256k1_fe *r, int a) { r->n[0] *= a; r->n[1] *= a; r->n[2] *= a; @@ -454,17 +376,9 @@ SECP256K1_INLINE static void secp256k1_fe_mul_int(secp256k1_fe *r, int a) { r->n[7] *= a; r->n[8] *= a; r->n[9] *= a; -#ifdef VERIFY - r->magnitude *= a; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } -SECP256K1_INLINE static void secp256k1_fe_add(secp256k1_fe *r, const secp256k1_fe *a) { -#ifdef VERIFY - secp256k1_fe_verify(a); -#endif +SECP256K1_INLINE static void secp256k1_fe_impl_add(secp256k1_fe *r, const secp256k1_fe *a) { r->n[0] += a->n[0]; r->n[1] += a->n[1]; r->n[2] += a->n[2]; @@ -475,25 +389,10 @@ SECP256K1_INLINE static void secp256k1_fe_add(secp256k1_fe *r, const secp256k1_f r->n[7] += a->n[7]; r->n[8] += a->n[8]; r->n[9] += a->n[9]; -#ifdef VERIFY - r->magnitude += a->magnitude; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } -SECP256K1_INLINE static void secp256k1_fe_add_int(secp256k1_fe *r, int a) { -#ifdef VERIFY - secp256k1_fe_verify(r); - VERIFY_CHECK(a >= 0); - VERIFY_CHECK(a <= 0x7FFF); -#endif +SECP256K1_INLINE static void secp256k1_fe_impl_add_int(secp256k1_fe *r, int a) { r->n[0] += a; -#ifdef VERIFY - r->magnitude += 1; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } #if defined(USE_EXTERNAL_ASM) @@ -1115,37 +1014,15 @@ SECP256K1_INLINE static void secp256k1_fe_sqr_inner(uint32_t *r, const uint32_t } #endif -static void secp256k1_fe_mul(secp256k1_fe *r, const secp256k1_fe *a, const secp256k1_fe * SECP256K1_RESTRICT b) { -#ifdef VERIFY - VERIFY_CHECK(a->magnitude <= 8); - VERIFY_CHECK(b->magnitude <= 8); - secp256k1_fe_verify(a); - secp256k1_fe_verify(b); - VERIFY_CHECK(r != b); - VERIFY_CHECK(a != b); -#endif +SECP256K1_INLINE static void secp256k1_fe_impl_mul(secp256k1_fe *r, const secp256k1_fe *a, const secp256k1_fe * SECP256K1_RESTRICT b) { secp256k1_fe_mul_inner(r->n, a->n, b->n); -#ifdef VERIFY - r->magnitude = 1; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } -static void secp256k1_fe_sqr(secp256k1_fe *r, const secp256k1_fe *a) { -#ifdef VERIFY - VERIFY_CHECK(a->magnitude <= 8); - secp256k1_fe_verify(a); -#endif +SECP256K1_INLINE static void secp256k1_fe_impl_sqr(secp256k1_fe *r, const secp256k1_fe *a) { secp256k1_fe_sqr_inner(r->n, a->n); -#ifdef VERIFY - r->magnitude = 1; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } -static SECP256K1_INLINE void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag) { +SECP256K1_INLINE static void secp256k1_fe_impl_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag) { uint32_t mask0, mask1; volatile int vflag = flag; SECP256K1_CHECKMEM_CHECK_VERIFY(r->n, sizeof(r->n)); @@ -1161,25 +1038,14 @@ static SECP256K1_INLINE void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_ r->n[7] = (r->n[7] & mask0) | (a->n[7] & mask1); r->n[8] = (r->n[8] & mask0) | (a->n[8] & mask1); r->n[9] = (r->n[9] & mask0) | (a->n[9] & mask1); -#ifdef VERIFY - if (flag) { - r->magnitude = a->magnitude; - r->normalized = a->normalized; - } -#endif } -static SECP256K1_INLINE void secp256k1_fe_half(secp256k1_fe *r) { +static SECP256K1_INLINE void secp256k1_fe_impl_half(secp256k1_fe *r) { uint32_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4], t5 = r->n[5], t6 = r->n[6], t7 = r->n[7], t8 = r->n[8], t9 = r->n[9]; uint32_t one = (uint32_t)1; uint32_t mask = -(t0 & one) >> 6; -#ifdef VERIFY - secp256k1_fe_verify(r); - VERIFY_CHECK(r->magnitude < 32); -#endif - /* Bounds analysis (over the rationals). * * Let m = r->magnitude @@ -1226,10 +1092,8 @@ static SECP256K1_INLINE void secp256k1_fe_half(secp256k1_fe *r) { * * Current bounds: t0..t8 <= C * (m/2 + 1/2) * t9 <= D * (m/2 + 1/4) - */ - -#ifdef VERIFY - /* Therefore the output magnitude (M) has to be set such that: + * + * Therefore the output magnitude (M) has to be set such that: * t0..t8: C * M >= C * (m/2 + 1/2) * t9: D * M >= D * (m/2 + 1/4) * @@ -1239,10 +1103,6 @@ static SECP256K1_INLINE void secp256k1_fe_half(secp256k1_fe *r) { * and since we want the smallest such integer value for M: * M == floor(m/2) + 1 */ - r->magnitude = (r->magnitude >> 1) + 1; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } static SECP256K1_INLINE void secp256k1_fe_storage_cmov(secp256k1_fe_storage *r, const secp256k1_fe_storage *a, int flag) { @@ -1261,10 +1121,7 @@ static SECP256K1_INLINE void secp256k1_fe_storage_cmov(secp256k1_fe_storage *r, r->n[7] = (r->n[7] & mask0) | (a->n[7] & mask1); } -static void secp256k1_fe_to_storage(secp256k1_fe_storage *r, const secp256k1_fe *a) { -#ifdef VERIFY - VERIFY_CHECK(a->normalized); -#endif +static void secp256k1_fe_impl_to_storage(secp256k1_fe_storage *r, const secp256k1_fe *a) { r->n[0] = a->n[0] | a->n[1] << 26; r->n[1] = a->n[1] >> 6 | a->n[2] << 20; r->n[2] = a->n[2] >> 12 | a->n[3] << 14; @@ -1275,7 +1132,7 @@ static void secp256k1_fe_to_storage(secp256k1_fe_storage *r, const secp256k1_fe r->n[7] = a->n[8] >> 16 | a->n[9] << 10; } -static SECP256K1_INLINE void secp256k1_fe_from_storage(secp256k1_fe *r, const secp256k1_fe_storage *a) { +static SECP256K1_INLINE void secp256k1_fe_impl_from_storage(secp256k1_fe *r, const secp256k1_fe_storage *a) { r->n[0] = a->n[0] & 0x3FFFFFFUL; r->n[1] = a->n[0] >> 26 | ((a->n[1] << 6) & 0x3FFFFFFUL); r->n[2] = a->n[1] >> 20 | ((a->n[2] << 12) & 0x3FFFFFFUL); @@ -1286,11 +1143,6 @@ static SECP256K1_INLINE void secp256k1_fe_from_storage(secp256k1_fe *r, const se r->n[7] = a->n[5] >> 22 | ((a->n[6] << 10) & 0x3FFFFFFUL); r->n[8] = a->n[6] >> 16 | ((a->n[7] << 16) & 0x3FFFFFFUL); r->n[9] = a->n[7] >> 10; -#ifdef VERIFY - r->magnitude = 1; - r->normalized = 1; - secp256k1_fe_verify(r); -#endif } static void secp256k1_fe_from_signed30(secp256k1_fe *r, const secp256k1_modinv32_signed30 *a) { @@ -1321,12 +1173,6 @@ static void secp256k1_fe_from_signed30(secp256k1_fe *r, const secp256k1_modinv32 r->n[7] = (a6 >> 2 ) & M26; r->n[8] = (a6 >> 28 | a7 << 2) & M26; r->n[9] = (a7 >> 24 | a8 << 6); - -#ifdef VERIFY - r->magnitude = 1; - r->normalized = 1; - secp256k1_fe_verify(r); -#endif } static void secp256k1_fe_to_signed30(secp256k1_modinv32_signed30 *r, const secp256k1_fe *a) { @@ -1334,10 +1180,6 @@ static void secp256k1_fe_to_signed30(secp256k1_modinv32_signed30 *r, const secp2 const uint64_t a0 = a->n[0], a1 = a->n[1], a2 = a->n[2], a3 = a->n[3], a4 = a->n[4], a5 = a->n[5], a6 = a->n[6], a7 = a->n[7], a8 = a->n[8], a9 = a->n[9]; -#ifdef VERIFY - VERIFY_CHECK(a->normalized); -#endif - r->v[0] = (a0 | a1 << 26) & M30; r->v[1] = (a1 >> 4 | a2 << 22) & M30; r->v[2] = (a2 >> 8 | a3 << 18) & M30; @@ -1355,37 +1197,27 @@ static const secp256k1_modinv32_modinfo secp256k1_const_modinfo_fe = { 0x2DDACACFL }; -static void secp256k1_fe_inv(secp256k1_fe *r, const secp256k1_fe *x) { - secp256k1_fe tmp; +static void secp256k1_fe_impl_inv(secp256k1_fe *r, const secp256k1_fe *x) { + secp256k1_fe tmp = *x; secp256k1_modinv32_signed30 s; - tmp = *x; secp256k1_fe_normalize(&tmp); secp256k1_fe_to_signed30(&s, &tmp); secp256k1_modinv32(&s, &secp256k1_const_modinfo_fe); secp256k1_fe_from_signed30(r, &s); - -#ifdef VERIFY - VERIFY_CHECK(secp256k1_fe_normalizes_to_zero(r) == secp256k1_fe_normalizes_to_zero(&tmp)); -#endif } -static void secp256k1_fe_inv_var(secp256k1_fe *r, const secp256k1_fe *x) { - secp256k1_fe tmp; +static void secp256k1_fe_impl_inv_var(secp256k1_fe *r, const secp256k1_fe *x) { + secp256k1_fe tmp = *x; secp256k1_modinv32_signed30 s; - tmp = *x; secp256k1_fe_normalize_var(&tmp); secp256k1_fe_to_signed30(&s, &tmp); secp256k1_modinv32_var(&s, &secp256k1_const_modinfo_fe); secp256k1_fe_from_signed30(r, &s); - -#ifdef VERIFY - VERIFY_CHECK(secp256k1_fe_normalizes_to_zero(r) == secp256k1_fe_normalizes_to_zero(&tmp)); -#endif } -static int secp256k1_fe_is_square_var(const secp256k1_fe *x) { +static int secp256k1_fe_impl_is_square_var(const secp256k1_fe *x) { secp256k1_fe tmp; secp256k1_modinv32_signed30 s; int jac, ret; @@ -1403,10 +1235,6 @@ static int secp256k1_fe_is_square_var(const secp256k1_fe *x) { secp256k1_fe dummy; ret = secp256k1_fe_sqrt(&dummy, &tmp); } else { -#ifdef VERIFY - secp256k1_fe dummy; - VERIFY_CHECK(jac == 2*secp256k1_fe_sqrt(&dummy, &tmp) - 1); -#endif ret = jac >= 0; } return ret; diff --git a/src/secp256k1/src/field_5x52.h b/src/secp256k1/src/field_5x52.h index 50ee3f9ec9..f20c246fdd 100644 --- a/src/secp256k1/src/field_5x52.h +++ b/src/secp256k1/src/field_5x52.h @@ -9,15 +9,28 @@ #include <stdint.h> +/** This field implementation represents the value as 5 uint64_t limbs in base + * 2^52. */ typedef struct { - /* X = sum(i=0..4, n[i]*2^(i*52)) mod p - * where p = 2^256 - 0x1000003D1 - */ + /* A field element f represents the sum(i=0..4, f.n[i] << (i*52)) mod p, + * where p is the field modulus, 2^256 - 2^32 - 977. + * + * The individual limbs f.n[i] can exceed 2^52; the field's magnitude roughly + * corresponds to how much excess is allowed. The value + * sum(i=0..4, f.n[i] << (i*52)) may exceed p, unless the field element is + * normalized. */ uint64_t n[5]; -#ifdef VERIFY - int magnitude; - int normalized; -#endif + /* + * Magnitude m requires: + * n[i] <= 2 * m * (2^52 - 1) for i=0..3 + * n[4] <= 2 * m * (2^48 - 1) + * + * Normalized requires: + * n[i] <= (2^52 - 1) for i=0..3 + * sum(i=0..4, n[i] << (i*52)) < p + * (together these imply n[4] <= 2^48 - 1) + */ + SECP256K1_FE_VERIFY_FIELDS } secp256k1_fe; /* Unpacks a constant into a overlapping multi-limbed FE element. */ @@ -29,12 +42,6 @@ typedef struct { ((uint64_t)(d6) >> 16) | (((uint64_t)(d7)) << 16) \ } -#ifdef VERIFY -#define SECP256K1_FE_CONST(d7, d6, d5, d4, d3, d2, d1, d0) {SECP256K1_FE_CONST_INNER((d7), (d6), (d5), (d4), (d3), (d2), (d1), (d0)), 1, 1} -#else -#define SECP256K1_FE_CONST(d7, d6, d5, d4, d3, d2, d1, d0) {SECP256K1_FE_CONST_INNER((d7), (d6), (d5), (d4), (d3), (d2), (d1), (d0))} -#endif - typedef struct { uint64_t n[4]; } secp256k1_fe_storage; diff --git a/src/secp256k1/src/field_5x52_asm_impl.h b/src/secp256k1/src/field_5x52_asm_impl.h index a2118044ab..04a9af2105 100644 --- a/src/secp256k1/src/field_5x52_asm_impl.h +++ b/src/secp256k1/src/field_5x52_asm_impl.h @@ -14,6 +14,8 @@ #ifndef SECP256K1_FIELD_INNER5X52_IMPL_H #define SECP256K1_FIELD_INNER5X52_IMPL_H +#include "util.h" + SECP256K1_INLINE static void secp256k1_fe_mul_inner(uint64_t *r, const uint64_t *a, const uint64_t * SECP256K1_RESTRICT b) { /** * Registers: rdx:rax = multiplication accumulator @@ -278,7 +280,7 @@ __asm__ __volatile__( "addq %%rsi,%%r8\n" /* r[4] = c */ "movq %%r8,32(%%rdi)\n" -: "+S"(a), "=m"(tmp1), "=m"(tmp2), "=m"(tmp3) +: "+S"(a), "=&m"(tmp1), "=&m"(tmp2), "=&m"(tmp3) : "b"(b), "D"(r) : "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory" ); @@ -493,7 +495,7 @@ __asm__ __volatile__( "addq %%rsi,%%r8\n" /* r[4] = c */ "movq %%r8,32(%%rdi)\n" -: "+S"(a), "=m"(tmp1), "=m"(tmp2), "=m"(tmp3) +: "+S"(a), "=&m"(tmp1), "=&m"(tmp2), "=&m"(tmp3) : "D"(r) : "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory" ); diff --git a/src/secp256k1/src/field_5x52_impl.h b/src/secp256k1/src/field_5x52_impl.h index 6b97157d0f..0a4cc1a630 100644 --- a/src/secp256k1/src/field_5x52_impl.h +++ b/src/secp256k1/src/field_5x52_impl.h @@ -18,59 +18,33 @@ #include "field_5x52_int128_impl.h" #endif -/** Implements arithmetic modulo FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F, - * represented as 5 uint64_t's in base 2^52, least significant first. Note that the limbs are allowed to - * contain >52 bits each. - * - * Each field element has a 'magnitude' associated with it. Internally, a magnitude M means: - * - 2*M*(2^48-1) is the max (inclusive) of the most significant limb - * - 2*M*(2^52-1) is the max (inclusive) of the remaining limbs - * - * Operations have different rules for propagating magnitude to their outputs. If an operation takes a - * magnitude M as a parameter, that means the magnitude of input field elements can be at most M (inclusive). - * - * Each field element also has a 'normalized' flag. A field element is normalized if its magnitude is either - * 0 or 1, and its value is already reduced modulo the order of the field. - */ - #ifdef VERIFY -static void secp256k1_fe_verify(const secp256k1_fe *a) { +static void secp256k1_fe_impl_verify(const secp256k1_fe *a) { const uint64_t *d = a->n; - int m = a->normalized ? 1 : 2 * a->magnitude, r = 1; + int m = a->normalized ? 1 : 2 * a->magnitude; /* secp256k1 'p' value defined in "Standards for Efficient Cryptography" (SEC2) 2.7.1. */ - r &= (d[0] <= 0xFFFFFFFFFFFFFULL * m); - r &= (d[1] <= 0xFFFFFFFFFFFFFULL * m); - r &= (d[2] <= 0xFFFFFFFFFFFFFULL * m); - r &= (d[3] <= 0xFFFFFFFFFFFFFULL * m); - r &= (d[4] <= 0x0FFFFFFFFFFFFULL * m); - r &= (a->magnitude >= 0); - r &= (a->magnitude <= 2048); + VERIFY_CHECK(d[0] <= 0xFFFFFFFFFFFFFULL * m); + VERIFY_CHECK(d[1] <= 0xFFFFFFFFFFFFFULL * m); + VERIFY_CHECK(d[2] <= 0xFFFFFFFFFFFFFULL * m); + VERIFY_CHECK(d[3] <= 0xFFFFFFFFFFFFFULL * m); + VERIFY_CHECK(d[4] <= 0x0FFFFFFFFFFFFULL * m); if (a->normalized) { - r &= (a->magnitude <= 1); - if (r && (d[4] == 0x0FFFFFFFFFFFFULL) && ((d[3] & d[2] & d[1]) == 0xFFFFFFFFFFFFFULL)) { - r &= (d[0] < 0xFFFFEFFFFFC2FULL); + if ((d[4] == 0x0FFFFFFFFFFFFULL) && ((d[3] & d[2] & d[1]) == 0xFFFFFFFFFFFFFULL)) { + VERIFY_CHECK(d[0] < 0xFFFFEFFFFFC2FULL); } } - VERIFY_CHECK(r == 1); } #endif -static void secp256k1_fe_get_bounds(secp256k1_fe *r, int m) { - VERIFY_CHECK(m >= 0); - VERIFY_CHECK(m <= 2048); +static void secp256k1_fe_impl_get_bounds(secp256k1_fe *r, int m) { r->n[0] = 0xFFFFFFFFFFFFFULL * 2 * m; r->n[1] = 0xFFFFFFFFFFFFFULL * 2 * m; r->n[2] = 0xFFFFFFFFFFFFFULL * 2 * m; r->n[3] = 0xFFFFFFFFFFFFFULL * 2 * m; r->n[4] = 0x0FFFFFFFFFFFFULL * 2 * m; -#ifdef VERIFY - r->magnitude = m; - r->normalized = (m == 0); - secp256k1_fe_verify(r); -#endif } -static void secp256k1_fe_normalize(secp256k1_fe *r) { +static void secp256k1_fe_impl_normalize(secp256k1_fe *r) { uint64_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4]; /* Reduce t4 at the start so there will be at most a single carry from the first pass */ @@ -105,15 +79,9 @@ static void secp256k1_fe_normalize(secp256k1_fe *r) { t4 &= 0x0FFFFFFFFFFFFULL; r->n[0] = t0; r->n[1] = t1; r->n[2] = t2; r->n[3] = t3; r->n[4] = t4; - -#ifdef VERIFY - r->magnitude = 1; - r->normalized = 1; - secp256k1_fe_verify(r); -#endif } -static void secp256k1_fe_normalize_weak(secp256k1_fe *r) { +static void secp256k1_fe_impl_normalize_weak(secp256k1_fe *r) { uint64_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4]; /* Reduce t4 at the start so there will be at most a single carry from the first pass */ @@ -130,14 +98,9 @@ static void secp256k1_fe_normalize_weak(secp256k1_fe *r) { VERIFY_CHECK(t4 >> 49 == 0); r->n[0] = t0; r->n[1] = t1; r->n[2] = t2; r->n[3] = t3; r->n[4] = t4; - -#ifdef VERIFY - r->magnitude = 1; - secp256k1_fe_verify(r); -#endif } -static void secp256k1_fe_normalize_var(secp256k1_fe *r) { +static void secp256k1_fe_impl_normalize_var(secp256k1_fe *r) { uint64_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4]; /* Reduce t4 at the start so there will be at most a single carry from the first pass */ @@ -173,15 +136,9 @@ static void secp256k1_fe_normalize_var(secp256k1_fe *r) { } r->n[0] = t0; r->n[1] = t1; r->n[2] = t2; r->n[3] = t3; r->n[4] = t4; - -#ifdef VERIFY - r->magnitude = 1; - r->normalized = 1; - secp256k1_fe_verify(r); -#endif } -static int secp256k1_fe_normalizes_to_zero(const secp256k1_fe *r) { +static int secp256k1_fe_impl_normalizes_to_zero(const secp256k1_fe *r) { uint64_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4]; /* z0 tracks a possible raw value of 0, z1 tracks a possible raw value of P */ @@ -204,7 +161,7 @@ static int secp256k1_fe_normalizes_to_zero(const secp256k1_fe *r) { return (z0 == 0) | (z1 == 0xFFFFFFFFFFFFFULL); } -static int secp256k1_fe_normalizes_to_zero_var(const secp256k1_fe *r) { +static int secp256k1_fe_impl_normalizes_to_zero_var(const secp256k1_fe *r) { uint64_t t0, t1, t2, t3, t4; uint64_t z0, z1; uint64_t x; @@ -245,53 +202,29 @@ static int secp256k1_fe_normalizes_to_zero_var(const secp256k1_fe *r) { return (z0 == 0) | (z1 == 0xFFFFFFFFFFFFFULL); } -SECP256K1_INLINE static void secp256k1_fe_set_int(secp256k1_fe *r, int a) { - VERIFY_CHECK(0 <= a && a <= 0x7FFF); +SECP256K1_INLINE static void secp256k1_fe_impl_set_int(secp256k1_fe *r, int a) { r->n[0] = a; r->n[1] = r->n[2] = r->n[3] = r->n[4] = 0; -#ifdef VERIFY - r->magnitude = (a != 0); - r->normalized = 1; - secp256k1_fe_verify(r); -#endif } -SECP256K1_INLINE static int secp256k1_fe_is_zero(const secp256k1_fe *a) { +SECP256K1_INLINE static int secp256k1_fe_impl_is_zero(const secp256k1_fe *a) { const uint64_t *t = a->n; -#ifdef VERIFY - VERIFY_CHECK(a->normalized); - secp256k1_fe_verify(a); -#endif return (t[0] | t[1] | t[2] | t[3] | t[4]) == 0; } -SECP256K1_INLINE static int secp256k1_fe_is_odd(const secp256k1_fe *a) { -#ifdef VERIFY - VERIFY_CHECK(a->normalized); - secp256k1_fe_verify(a); -#endif +SECP256K1_INLINE static int secp256k1_fe_impl_is_odd(const secp256k1_fe *a) { return a->n[0] & 1; } -SECP256K1_INLINE static void secp256k1_fe_clear(secp256k1_fe *a) { +SECP256K1_INLINE static void secp256k1_fe_impl_clear(secp256k1_fe *a) { int i; -#ifdef VERIFY - a->magnitude = 0; - a->normalized = 1; -#endif for (i=0; i<5; i++) { a->n[i] = 0; } } -static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { +static int secp256k1_fe_impl_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { int i; -#ifdef VERIFY - VERIFY_CHECK(a->normalized); - VERIFY_CHECK(b->normalized); - secp256k1_fe_verify(a); - secp256k1_fe_verify(b); -#endif for (i = 4; i >= 0; i--) { if (a->n[i] > b->n[i]) { return 1; @@ -303,8 +236,7 @@ static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { return 0; } -static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { - int ret; +static void secp256k1_fe_impl_set_b32_mod(secp256k1_fe *r, const unsigned char *a) { r->n[0] = (uint64_t)a[31] | ((uint64_t)a[30] << 8) | ((uint64_t)a[29] << 16) @@ -339,25 +271,15 @@ static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { | ((uint64_t)a[2] << 24) | ((uint64_t)a[1] << 32) | ((uint64_t)a[0] << 40); - ret = !((r->n[4] == 0x0FFFFFFFFFFFFULL) & ((r->n[3] & r->n[2] & r->n[1]) == 0xFFFFFFFFFFFFFULL) & (r->n[0] >= 0xFFFFEFFFFFC2FULL)); -#ifdef VERIFY - r->magnitude = 1; - if (ret) { - r->normalized = 1; - secp256k1_fe_verify(r); - } else { - r->normalized = 0; - } -#endif - return ret; +} + +static int secp256k1_fe_impl_set_b32_limit(secp256k1_fe *r, const unsigned char *a) { + secp256k1_fe_impl_set_b32_mod(r, a); + return !((r->n[4] == 0x0FFFFFFFFFFFFULL) & ((r->n[3] & r->n[2] & r->n[1]) == 0xFFFFFFFFFFFFFULL) & (r->n[0] >= 0xFFFFEFFFFFC2FULL)); } /** Convert a field element to a 32-byte big endian value. Requires the input to be normalized */ -static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a) { -#ifdef VERIFY - VERIFY_CHECK(a->normalized); - secp256k1_fe_verify(a); -#endif +static void secp256k1_fe_impl_get_b32(unsigned char *r, const secp256k1_fe *a) { r[0] = (a->n[4] >> 40) & 0xFF; r[1] = (a->n[4] >> 32) & 0xFF; r[2] = (a->n[4] >> 24) & 0xFF; @@ -392,100 +314,50 @@ static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a) { r[31] = a->n[0] & 0xFF; } -SECP256K1_INLINE static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k1_fe *a, int m) { -#ifdef VERIFY - VERIFY_CHECK(a->magnitude <= m); - secp256k1_fe_verify(a); +SECP256K1_INLINE static void secp256k1_fe_impl_negate(secp256k1_fe *r, const secp256k1_fe *a, int m) { + /* For all legal values of m (0..31), the following properties hold: */ VERIFY_CHECK(0xFFFFEFFFFFC2FULL * 2 * (m + 1) >= 0xFFFFFFFFFFFFFULL * 2 * m); VERIFY_CHECK(0xFFFFFFFFFFFFFULL * 2 * (m + 1) >= 0xFFFFFFFFFFFFFULL * 2 * m); VERIFY_CHECK(0x0FFFFFFFFFFFFULL * 2 * (m + 1) >= 0x0FFFFFFFFFFFFULL * 2 * m); -#endif + + /* Due to the properties above, the left hand in the subtractions below is never less than + * the right hand. */ r->n[0] = 0xFFFFEFFFFFC2FULL * 2 * (m + 1) - a->n[0]; r->n[1] = 0xFFFFFFFFFFFFFULL * 2 * (m + 1) - a->n[1]; r->n[2] = 0xFFFFFFFFFFFFFULL * 2 * (m + 1) - a->n[2]; r->n[3] = 0xFFFFFFFFFFFFFULL * 2 * (m + 1) - a->n[3]; r->n[4] = 0x0FFFFFFFFFFFFULL * 2 * (m + 1) - a->n[4]; -#ifdef VERIFY - r->magnitude = m + 1; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } -SECP256K1_INLINE static void secp256k1_fe_mul_int(secp256k1_fe *r, int a) { +SECP256K1_INLINE static void secp256k1_fe_impl_mul_int(secp256k1_fe *r, int a) { r->n[0] *= a; r->n[1] *= a; r->n[2] *= a; r->n[3] *= a; r->n[4] *= a; -#ifdef VERIFY - r->magnitude *= a; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } -SECP256K1_INLINE static void secp256k1_fe_add_int(secp256k1_fe *r, int a) { -#ifdef VERIFY - secp256k1_fe_verify(r); - VERIFY_CHECK(a >= 0); - VERIFY_CHECK(a <= 0x7FFF); -#endif +SECP256K1_INLINE static void secp256k1_fe_impl_add_int(secp256k1_fe *r, int a) { r->n[0] += a; -#ifdef VERIFY - r->magnitude += 1; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } -SECP256K1_INLINE static void secp256k1_fe_add(secp256k1_fe *r, const secp256k1_fe *a) { -#ifdef VERIFY - secp256k1_fe_verify(a); -#endif +SECP256K1_INLINE static void secp256k1_fe_impl_add(secp256k1_fe *r, const secp256k1_fe *a) { r->n[0] += a->n[0]; r->n[1] += a->n[1]; r->n[2] += a->n[2]; r->n[3] += a->n[3]; r->n[4] += a->n[4]; -#ifdef VERIFY - r->magnitude += a->magnitude; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } -static void secp256k1_fe_mul(secp256k1_fe *r, const secp256k1_fe *a, const secp256k1_fe * SECP256K1_RESTRICT b) { -#ifdef VERIFY - VERIFY_CHECK(a->magnitude <= 8); - VERIFY_CHECK(b->magnitude <= 8); - secp256k1_fe_verify(a); - secp256k1_fe_verify(b); - VERIFY_CHECK(r != b); - VERIFY_CHECK(a != b); -#endif +SECP256K1_INLINE static void secp256k1_fe_impl_mul(secp256k1_fe *r, const secp256k1_fe *a, const secp256k1_fe * SECP256K1_RESTRICT b) { secp256k1_fe_mul_inner(r->n, a->n, b->n); -#ifdef VERIFY - r->magnitude = 1; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } -static void secp256k1_fe_sqr(secp256k1_fe *r, const secp256k1_fe *a) { -#ifdef VERIFY - VERIFY_CHECK(a->magnitude <= 8); - secp256k1_fe_verify(a); -#endif +SECP256K1_INLINE static void secp256k1_fe_impl_sqr(secp256k1_fe *r, const secp256k1_fe *a) { secp256k1_fe_sqr_inner(r->n, a->n); -#ifdef VERIFY - r->magnitude = 1; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } -static SECP256K1_INLINE void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag) { +SECP256K1_INLINE static void secp256k1_fe_impl_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag) { uint64_t mask0, mask1; volatile int vflag = flag; SECP256K1_CHECKMEM_CHECK_VERIFY(r->n, sizeof(r->n)); @@ -496,24 +368,13 @@ static SECP256K1_INLINE void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_ r->n[2] = (r->n[2] & mask0) | (a->n[2] & mask1); r->n[3] = (r->n[3] & mask0) | (a->n[3] & mask1); r->n[4] = (r->n[4] & mask0) | (a->n[4] & mask1); -#ifdef VERIFY - if (flag) { - r->magnitude = a->magnitude; - r->normalized = a->normalized; - } -#endif } -static SECP256K1_INLINE void secp256k1_fe_half(secp256k1_fe *r) { +static SECP256K1_INLINE void secp256k1_fe_impl_half(secp256k1_fe *r) { uint64_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4]; uint64_t one = (uint64_t)1; uint64_t mask = -(t0 & one) >> 12; -#ifdef VERIFY - secp256k1_fe_verify(r); - VERIFY_CHECK(r->magnitude < 32); -#endif - /* Bounds analysis (over the rationals). * * Let m = r->magnitude @@ -550,10 +411,8 @@ static SECP256K1_INLINE void secp256k1_fe_half(secp256k1_fe *r) { * * Current bounds: t0..t3 <= C * (m/2 + 1/2) * t4 <= D * (m/2 + 1/4) - */ - -#ifdef VERIFY - /* Therefore the output magnitude (M) has to be set such that: + * + * Therefore the output magnitude (M) has to be set such that: * t0..t3: C * M >= C * (m/2 + 1/2) * t4: D * M >= D * (m/2 + 1/4) * @@ -563,10 +422,6 @@ static SECP256K1_INLINE void secp256k1_fe_half(secp256k1_fe *r) { * and since we want the smallest such integer value for M: * M == floor(m/2) + 1 */ - r->magnitude = (r->magnitude >> 1) + 1; - r->normalized = 0; - secp256k1_fe_verify(r); -#endif } static SECP256K1_INLINE void secp256k1_fe_storage_cmov(secp256k1_fe_storage *r, const secp256k1_fe_storage *a, int flag) { @@ -581,27 +436,19 @@ static SECP256K1_INLINE void secp256k1_fe_storage_cmov(secp256k1_fe_storage *r, r->n[3] = (r->n[3] & mask0) | (a->n[3] & mask1); } -static void secp256k1_fe_to_storage(secp256k1_fe_storage *r, const secp256k1_fe *a) { -#ifdef VERIFY - VERIFY_CHECK(a->normalized); -#endif +static void secp256k1_fe_impl_to_storage(secp256k1_fe_storage *r, const secp256k1_fe *a) { r->n[0] = a->n[0] | a->n[1] << 52; r->n[1] = a->n[1] >> 12 | a->n[2] << 40; r->n[2] = a->n[2] >> 24 | a->n[3] << 28; r->n[3] = a->n[3] >> 36 | a->n[4] << 16; } -static SECP256K1_INLINE void secp256k1_fe_from_storage(secp256k1_fe *r, const secp256k1_fe_storage *a) { +static SECP256K1_INLINE void secp256k1_fe_impl_from_storage(secp256k1_fe *r, const secp256k1_fe_storage *a) { r->n[0] = a->n[0] & 0xFFFFFFFFFFFFFULL; r->n[1] = a->n[0] >> 52 | ((a->n[1] << 12) & 0xFFFFFFFFFFFFFULL); r->n[2] = a->n[1] >> 40 | ((a->n[2] << 24) & 0xFFFFFFFFFFFFFULL); r->n[3] = a->n[2] >> 28 | ((a->n[3] << 36) & 0xFFFFFFFFFFFFFULL); r->n[4] = a->n[3] >> 16; -#ifdef VERIFY - r->magnitude = 1; - r->normalized = 1; - secp256k1_fe_verify(r); -#endif } static void secp256k1_fe_from_signed62(secp256k1_fe *r, const secp256k1_modinv64_signed62 *a) { @@ -622,22 +469,12 @@ static void secp256k1_fe_from_signed62(secp256k1_fe *r, const secp256k1_modinv64 r->n[2] = (a1 >> 42 | a2 << 20) & M52; r->n[3] = (a2 >> 32 | a3 << 30) & M52; r->n[4] = (a3 >> 22 | a4 << 40); - -#ifdef VERIFY - r->magnitude = 1; - r->normalized = 1; - secp256k1_fe_verify(r); -#endif } static void secp256k1_fe_to_signed62(secp256k1_modinv64_signed62 *r, const secp256k1_fe *a) { const uint64_t M62 = UINT64_MAX >> 2; const uint64_t a0 = a->n[0], a1 = a->n[1], a2 = a->n[2], a3 = a->n[3], a4 = a->n[4]; -#ifdef VERIFY - VERIFY_CHECK(a->normalized); -#endif - r->v[0] = (a0 | a1 << 52) & M62; r->v[1] = (a1 >> 10 | a2 << 42) & M62; r->v[2] = (a2 >> 20 | a3 << 32) & M62; @@ -650,37 +487,27 @@ static const secp256k1_modinv64_modinfo secp256k1_const_modinfo_fe = { 0x27C7F6E22DDACACFLL }; -static void secp256k1_fe_inv(secp256k1_fe *r, const secp256k1_fe *x) { - secp256k1_fe tmp; +static void secp256k1_fe_impl_inv(secp256k1_fe *r, const secp256k1_fe *x) { + secp256k1_fe tmp = *x; secp256k1_modinv64_signed62 s; - tmp = *x; secp256k1_fe_normalize(&tmp); secp256k1_fe_to_signed62(&s, &tmp); secp256k1_modinv64(&s, &secp256k1_const_modinfo_fe); secp256k1_fe_from_signed62(r, &s); - -#ifdef VERIFY - VERIFY_CHECK(secp256k1_fe_normalizes_to_zero(r) == secp256k1_fe_normalizes_to_zero(&tmp)); -#endif } -static void secp256k1_fe_inv_var(secp256k1_fe *r, const secp256k1_fe *x) { - secp256k1_fe tmp; +static void secp256k1_fe_impl_inv_var(secp256k1_fe *r, const secp256k1_fe *x) { + secp256k1_fe tmp = *x; secp256k1_modinv64_signed62 s; - tmp = *x; secp256k1_fe_normalize_var(&tmp); secp256k1_fe_to_signed62(&s, &tmp); secp256k1_modinv64_var(&s, &secp256k1_const_modinfo_fe); secp256k1_fe_from_signed62(r, &s); - -#ifdef VERIFY - VERIFY_CHECK(secp256k1_fe_normalizes_to_zero(r) == secp256k1_fe_normalizes_to_zero(&tmp)); -#endif } -static int secp256k1_fe_is_square_var(const secp256k1_fe *x) { +static int secp256k1_fe_impl_is_square_var(const secp256k1_fe *x) { secp256k1_fe tmp; secp256k1_modinv64_signed62 s; int jac, ret; @@ -698,10 +525,6 @@ static int secp256k1_fe_is_square_var(const secp256k1_fe *x) { secp256k1_fe dummy; ret = secp256k1_fe_sqrt(&dummy, &tmp); } else { -#ifdef VERIFY - secp256k1_fe dummy; - VERIFY_CHECK(jac == 2*secp256k1_fe_sqrt(&dummy, &tmp) - 1); -#endif ret = jac >= 0; } return ret; diff --git a/src/secp256k1/src/field_5x52_int128_impl.h b/src/secp256k1/src/field_5x52_int128_impl.h index 18567b95f3..b2a391dec9 100644 --- a/src/secp256k1/src/field_5x52_int128_impl.h +++ b/src/secp256k1/src/field_5x52_int128_impl.h @@ -10,6 +10,7 @@ #include <stdint.h> #include "int128.h" +#include "util.h" #ifdef VERIFY #define VERIFY_BITS(x, n) VERIFY_CHECK(((x) >> (n)) == 0) diff --git a/src/secp256k1/src/field_impl.h b/src/secp256k1/src/field_impl.h index 0a03076bbc..f9769a4a39 100644 --- a/src/secp256k1/src/field_impl.h +++ b/src/secp256k1/src/field_impl.h @@ -7,6 +7,7 @@ #ifndef SECP256K1_FIELD_IMPL_H #define SECP256K1_FIELD_IMPL_H +#include "field.h" #include "util.h" #if defined(SECP256K1_WIDEMUL_INT128) @@ -19,6 +20,12 @@ SECP256K1_INLINE static int secp256k1_fe_equal(const secp256k1_fe *a, const secp256k1_fe *b) { secp256k1_fe na; +#ifdef VERIFY + secp256k1_fe_verify(a); + secp256k1_fe_verify(b); + VERIFY_CHECK(a->magnitude <= 1); + VERIFY_CHECK(b->magnitude <= 31); +#endif secp256k1_fe_negate(&na, a, 1); secp256k1_fe_add(&na, b); return secp256k1_fe_normalizes_to_zero(&na); @@ -26,6 +33,12 @@ SECP256K1_INLINE static int secp256k1_fe_equal(const secp256k1_fe *a, const secp SECP256K1_INLINE static int secp256k1_fe_equal_var(const secp256k1_fe *a, const secp256k1_fe *b) { secp256k1_fe na; +#ifdef VERIFY + secp256k1_fe_verify(a); + secp256k1_fe_verify(b); + VERIFY_CHECK(a->magnitude <= 1); + VERIFY_CHECK(b->magnitude <= 31); +#endif secp256k1_fe_negate(&na, a, 1); secp256k1_fe_add(&na, b); return secp256k1_fe_normalizes_to_zero_var(&na); @@ -42,9 +55,13 @@ static int secp256k1_fe_sqrt(secp256k1_fe *r, const secp256k1_fe *a) { * itself always a square (a ** ((p+1)/4) is the square of a ** ((p+1)/8)). */ secp256k1_fe x2, x3, x6, x9, x11, x22, x44, x88, x176, x220, x223, t1; - int j; + int j, ret; +#ifdef VERIFY VERIFY_CHECK(r != a); + secp256k1_fe_verify(a); + VERIFY_CHECK(a->magnitude <= 8); +#endif /** The binary representation of (p + 1)/4 has 3 blocks of 1s, with lengths in * { 2, 22, 223 }. Use an addition chain to calculate 2^n - 1 for each block: @@ -128,7 +145,286 @@ static int secp256k1_fe_sqrt(secp256k1_fe *r, const secp256k1_fe *a) { /* Check that a square root was actually calculated */ secp256k1_fe_sqr(&t1, r); - return secp256k1_fe_equal(&t1, a); + ret = secp256k1_fe_equal(&t1, a); + +#ifdef VERIFY + if (!ret) { + secp256k1_fe_negate(&t1, &t1, 1); + secp256k1_fe_normalize_var(&t1); + VERIFY_CHECK(secp256k1_fe_equal_var(&t1, a)); + } +#endif + return ret; +} + +#ifndef VERIFY +static void secp256k1_fe_verify(const secp256k1_fe *a) { (void)a; } +#else +static void secp256k1_fe_impl_verify(const secp256k1_fe *a); +static void secp256k1_fe_verify(const secp256k1_fe *a) { + /* Magnitude between 0 and 32. */ + VERIFY_CHECK((a->magnitude >= 0) && (a->magnitude <= 32)); + /* Normalized is 0 or 1. */ + VERIFY_CHECK((a->normalized == 0) || (a->normalized == 1)); + /* If normalized, magnitude must be 0 or 1. */ + if (a->normalized) VERIFY_CHECK(a->magnitude <= 1); + /* Invoke implementation-specific checks. */ + secp256k1_fe_impl_verify(a); +} + +static void secp256k1_fe_impl_normalize(secp256k1_fe *r); +SECP256K1_INLINE static void secp256k1_fe_normalize(secp256k1_fe *r) { + secp256k1_fe_verify(r); + secp256k1_fe_impl_normalize(r); + r->magnitude = 1; + r->normalized = 1; + secp256k1_fe_verify(r); +} + +static void secp256k1_fe_impl_normalize_weak(secp256k1_fe *r); +SECP256K1_INLINE static void secp256k1_fe_normalize_weak(secp256k1_fe *r) { + secp256k1_fe_verify(r); + secp256k1_fe_impl_normalize_weak(r); + r->magnitude = 1; + secp256k1_fe_verify(r); +} + +static void secp256k1_fe_impl_normalize_var(secp256k1_fe *r); +SECP256K1_INLINE static void secp256k1_fe_normalize_var(secp256k1_fe *r) { + secp256k1_fe_verify(r); + secp256k1_fe_impl_normalize_var(r); + r->magnitude = 1; + r->normalized = 1; + secp256k1_fe_verify(r); +} + +static int secp256k1_fe_impl_normalizes_to_zero(const secp256k1_fe *r); +SECP256K1_INLINE static int secp256k1_fe_normalizes_to_zero(const secp256k1_fe *r) { + secp256k1_fe_verify(r); + return secp256k1_fe_impl_normalizes_to_zero(r); +} + +static int secp256k1_fe_impl_normalizes_to_zero_var(const secp256k1_fe *r); +SECP256K1_INLINE static int secp256k1_fe_normalizes_to_zero_var(const secp256k1_fe *r) { + secp256k1_fe_verify(r); + return secp256k1_fe_impl_normalizes_to_zero_var(r); +} + +static void secp256k1_fe_impl_set_int(secp256k1_fe *r, int a); +SECP256K1_INLINE static void secp256k1_fe_set_int(secp256k1_fe *r, int a) { + VERIFY_CHECK(0 <= a && a <= 0x7FFF); + secp256k1_fe_impl_set_int(r, a); + r->magnitude = (a != 0); + r->normalized = 1; + secp256k1_fe_verify(r); +} + +static void secp256k1_fe_impl_add_int(secp256k1_fe *r, int a); +SECP256K1_INLINE static void secp256k1_fe_add_int(secp256k1_fe *r, int a) { + VERIFY_CHECK(0 <= a && a <= 0x7FFF); + secp256k1_fe_verify(r); + secp256k1_fe_impl_add_int(r, a); + r->magnitude += 1; + r->normalized = 0; + secp256k1_fe_verify(r); +} + +static void secp256k1_fe_impl_clear(secp256k1_fe *a); +SECP256K1_INLINE static void secp256k1_fe_clear(secp256k1_fe *a) { + a->magnitude = 0; + a->normalized = 1; + secp256k1_fe_impl_clear(a); + secp256k1_fe_verify(a); +} + +static int secp256k1_fe_impl_is_zero(const secp256k1_fe *a); +SECP256K1_INLINE static int secp256k1_fe_is_zero(const secp256k1_fe *a) { + secp256k1_fe_verify(a); + VERIFY_CHECK(a->normalized); + return secp256k1_fe_impl_is_zero(a); +} + +static int secp256k1_fe_impl_is_odd(const secp256k1_fe *a); +SECP256K1_INLINE static int secp256k1_fe_is_odd(const secp256k1_fe *a) { + secp256k1_fe_verify(a); + VERIFY_CHECK(a->normalized); + return secp256k1_fe_impl_is_odd(a); +} + +static int secp256k1_fe_impl_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b); +SECP256K1_INLINE static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { + secp256k1_fe_verify(a); + secp256k1_fe_verify(b); + VERIFY_CHECK(a->normalized); + VERIFY_CHECK(b->normalized); + return secp256k1_fe_impl_cmp_var(a, b); +} + +static void secp256k1_fe_impl_set_b32_mod(secp256k1_fe *r, const unsigned char *a); +SECP256K1_INLINE static void secp256k1_fe_set_b32_mod(secp256k1_fe *r, const unsigned char *a) { + secp256k1_fe_impl_set_b32_mod(r, a); + r->magnitude = 1; + r->normalized = 0; + secp256k1_fe_verify(r); } +static int secp256k1_fe_impl_set_b32_limit(secp256k1_fe *r, const unsigned char *a); +SECP256K1_INLINE static int secp256k1_fe_set_b32_limit(secp256k1_fe *r, const unsigned char *a) { + if (secp256k1_fe_impl_set_b32_limit(r, a)) { + r->magnitude = 1; + r->normalized = 1; + secp256k1_fe_verify(r); + return 1; + } else { + /* Mark the output field element as invalid. */ + r->magnitude = -1; + return 0; + } +} + +static void secp256k1_fe_impl_get_b32(unsigned char *r, const secp256k1_fe *a); +SECP256K1_INLINE static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a) { + secp256k1_fe_verify(a); + VERIFY_CHECK(a->normalized); + secp256k1_fe_impl_get_b32(r, a); +} + +static void secp256k1_fe_impl_negate(secp256k1_fe *r, const secp256k1_fe *a, int m); +SECP256K1_INLINE static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k1_fe *a, int m) { + secp256k1_fe_verify(a); + VERIFY_CHECK(m >= 0 && m <= 31); + VERIFY_CHECK(a->magnitude <= m); + secp256k1_fe_impl_negate(r, a, m); + r->magnitude = m + 1; + r->normalized = 0; + secp256k1_fe_verify(r); +} + +static void secp256k1_fe_impl_mul_int(secp256k1_fe *r, int a); +SECP256K1_INLINE static void secp256k1_fe_mul_int(secp256k1_fe *r, int a) { + secp256k1_fe_verify(r); + VERIFY_CHECK(a >= 0 && a <= 32); + VERIFY_CHECK(a*r->magnitude <= 32); + secp256k1_fe_impl_mul_int(r, a); + r->magnitude *= a; + r->normalized = 0; + secp256k1_fe_verify(r); +} + +static void secp256k1_fe_impl_add(secp256k1_fe *r, const secp256k1_fe *a); +SECP256K1_INLINE static void secp256k1_fe_add(secp256k1_fe *r, const secp256k1_fe *a) { + secp256k1_fe_verify(r); + secp256k1_fe_verify(a); + VERIFY_CHECK(r->magnitude + a->magnitude <= 32); + secp256k1_fe_impl_add(r, a); + r->magnitude += a->magnitude; + r->normalized = 0; + secp256k1_fe_verify(r); +} + +static void secp256k1_fe_impl_mul(secp256k1_fe *r, const secp256k1_fe *a, const secp256k1_fe * SECP256K1_RESTRICT b); +SECP256K1_INLINE static void secp256k1_fe_mul(secp256k1_fe *r, const secp256k1_fe *a, const secp256k1_fe * SECP256K1_RESTRICT b) { + secp256k1_fe_verify(a); + secp256k1_fe_verify(b); + VERIFY_CHECK(a->magnitude <= 8); + VERIFY_CHECK(b->magnitude <= 8); + VERIFY_CHECK(r != b); + VERIFY_CHECK(a != b); + secp256k1_fe_impl_mul(r, a, b); + r->magnitude = 1; + r->normalized = 0; + secp256k1_fe_verify(r); +} + +static void secp256k1_fe_impl_sqr(secp256k1_fe *r, const secp256k1_fe *a); +SECP256K1_INLINE static void secp256k1_fe_sqr(secp256k1_fe *r, const secp256k1_fe *a) { + secp256k1_fe_verify(a); + VERIFY_CHECK(a->magnitude <= 8); + secp256k1_fe_impl_sqr(r, a); + r->magnitude = 1; + r->normalized = 0; + secp256k1_fe_verify(r); +} + +static void secp256k1_fe_impl_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag); +SECP256K1_INLINE static void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag) { + VERIFY_CHECK(flag == 0 || flag == 1); + secp256k1_fe_verify(a); + secp256k1_fe_verify(r); + secp256k1_fe_impl_cmov(r, a, flag); + if (a->magnitude > r->magnitude) r->magnitude = a->magnitude; + if (!a->normalized) r->normalized = 0; + secp256k1_fe_verify(r); +} + +static void secp256k1_fe_impl_to_storage(secp256k1_fe_storage *r, const secp256k1_fe *a); +SECP256K1_INLINE static void secp256k1_fe_to_storage(secp256k1_fe_storage *r, const secp256k1_fe *a) { + secp256k1_fe_verify(a); + VERIFY_CHECK(a->normalized); + secp256k1_fe_impl_to_storage(r, a); +} + +static void secp256k1_fe_impl_from_storage(secp256k1_fe *r, const secp256k1_fe_storage *a); +SECP256K1_INLINE static void secp256k1_fe_from_storage(secp256k1_fe *r, const secp256k1_fe_storage *a) { + secp256k1_fe_impl_from_storage(r, a); + r->magnitude = 1; + r->normalized = 1; + secp256k1_fe_verify(r); +} + +static void secp256k1_fe_impl_inv(secp256k1_fe *r, const secp256k1_fe *x); +SECP256K1_INLINE static void secp256k1_fe_inv(secp256k1_fe *r, const secp256k1_fe *x) { + int input_is_zero = secp256k1_fe_normalizes_to_zero(x); + secp256k1_fe_verify(x); + secp256k1_fe_impl_inv(r, x); + r->magnitude = x->magnitude > 0; + r->normalized = 1; + VERIFY_CHECK(secp256k1_fe_normalizes_to_zero(r) == input_is_zero); + secp256k1_fe_verify(r); +} + +static void secp256k1_fe_impl_inv_var(secp256k1_fe *r, const secp256k1_fe *x); +SECP256K1_INLINE static void secp256k1_fe_inv_var(secp256k1_fe *r, const secp256k1_fe *x) { + int input_is_zero = secp256k1_fe_normalizes_to_zero(x); + secp256k1_fe_verify(x); + secp256k1_fe_impl_inv_var(r, x); + r->magnitude = x->magnitude > 0; + r->normalized = 1; + VERIFY_CHECK(secp256k1_fe_normalizes_to_zero(r) == input_is_zero); + secp256k1_fe_verify(r); +} + +static int secp256k1_fe_impl_is_square_var(const secp256k1_fe *x); +SECP256K1_INLINE static int secp256k1_fe_is_square_var(const secp256k1_fe *x) { + int ret; + secp256k1_fe tmp = *x, sqrt; + secp256k1_fe_verify(x); + ret = secp256k1_fe_impl_is_square_var(x); + secp256k1_fe_normalize_weak(&tmp); + VERIFY_CHECK(ret == secp256k1_fe_sqrt(&sqrt, &tmp)); + return ret; +} + +static void secp256k1_fe_impl_get_bounds(secp256k1_fe* r, int m); +SECP256K1_INLINE static void secp256k1_fe_get_bounds(secp256k1_fe* r, int m) { + VERIFY_CHECK(m >= 0); + VERIFY_CHECK(m <= 32); + secp256k1_fe_impl_get_bounds(r, m); + r->magnitude = m; + r->normalized = (m == 0); + secp256k1_fe_verify(r); +} + +static void secp256k1_fe_impl_half(secp256k1_fe *r); +SECP256K1_INLINE static void secp256k1_fe_half(secp256k1_fe *r) { + secp256k1_fe_verify(r); + VERIFY_CHECK(r->magnitude < 32); + secp256k1_fe_impl_half(r); + r->magnitude = (r->magnitude >> 1) + 1; + r->normalized = 0; + secp256k1_fe_verify(r); +} + +#endif /* defined(VERIFY) */ + #endif /* SECP256K1_FIELD_IMPL_H */ diff --git a/src/secp256k1/src/group.h b/src/secp256k1/src/group.h index b79ba597db..877c3eaeed 100644 --- a/src/secp256k1/src/group.h +++ b/src/secp256k1/src/group.h @@ -51,6 +51,12 @@ static void secp256k1_ge_set_xy(secp256k1_ge *r, const secp256k1_fe *x, const se * for Y. Return value indicates whether the result is valid. */ static int secp256k1_ge_set_xo_var(secp256k1_ge *r, const secp256k1_fe *x, int odd); +/** Determine whether x is a valid X coordinate on the curve. */ +static int secp256k1_ge_x_on_curve_var(const secp256k1_fe *x); + +/** Determine whether fraction xn/xd is a valid X coordinate on the curve (xd != 0). */ +static int secp256k1_ge_x_frac_on_curve_var(const secp256k1_fe *xn, const secp256k1_fe *xd); + /** Check whether a group element is the point at infinity. */ static int secp256k1_ge_is_infinity(const secp256k1_ge *a); @@ -164,4 +170,10 @@ static void secp256k1_gej_rescale(secp256k1_gej *r, const secp256k1_fe *b); */ static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge); +/** Check invariants on an affine group element (no-op unless VERIFY is enabled). */ +static void secp256k1_ge_verify(const secp256k1_ge *a); + +/** Check invariants on a Jacobian group element (no-op unless VERIFY is enabled). */ +static void secp256k1_gej_verify(const secp256k1_gej *a); + #endif /* SECP256K1_GROUP_H */ diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index 82ce3f8d8b..dcd171f574 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -9,6 +9,7 @@ #include "field.h" #include "group.h" +#include "util.h" /* Begin of section generated by sage/gen_exhaustive_groups.sage. */ #define SECP256K1_G_ORDER_7 SECP256K1_GE_CONST(\ @@ -72,37 +73,80 @@ static const secp256k1_ge secp256k1_ge_const_g = SECP256K1_G; #endif /* End of section generated by sage/gen_exhaustive_groups.sage. */ -static const secp256k1_fe secp256k1_fe_const_b = SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, SECP256K1_B); +static void secp256k1_ge_verify(const secp256k1_ge *a) { +#ifdef VERIFY + secp256k1_fe_verify(&a->x); + secp256k1_fe_verify(&a->y); + VERIFY_CHECK(a->infinity == 0 || a->infinity == 1); +#endif + (void)a; +} + +static void secp256k1_gej_verify(const secp256k1_gej *a) { +#ifdef VERIFY + secp256k1_fe_verify(&a->x); + secp256k1_fe_verify(&a->y); + secp256k1_fe_verify(&a->z); + VERIFY_CHECK(a->infinity == 0 || a->infinity == 1); +#endif + (void)a; +} +/* Set r to the affine coordinates of Jacobian point (a.x, a.y, 1/zi). */ static void secp256k1_ge_set_gej_zinv(secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zi) { secp256k1_fe zi2; secp256k1_fe zi3; + secp256k1_gej_verify(a); + secp256k1_fe_verify(zi); VERIFY_CHECK(!a->infinity); secp256k1_fe_sqr(&zi2, zi); secp256k1_fe_mul(&zi3, &zi2, zi); secp256k1_fe_mul(&r->x, &a->x, &zi2); secp256k1_fe_mul(&r->y, &a->y, &zi3); r->infinity = a->infinity; + secp256k1_ge_verify(r); +} + +/* Set r to the affine coordinates of Jacobian point (a.x, a.y, 1/zi). */ +static void secp256k1_ge_set_ge_zinv(secp256k1_ge *r, const secp256k1_ge *a, const secp256k1_fe *zi) { + secp256k1_fe zi2; + secp256k1_fe zi3; + secp256k1_ge_verify(a); + secp256k1_fe_verify(zi); + VERIFY_CHECK(!a->infinity); + secp256k1_fe_sqr(&zi2, zi); + secp256k1_fe_mul(&zi3, &zi2, zi); + secp256k1_fe_mul(&r->x, &a->x, &zi2); + secp256k1_fe_mul(&r->y, &a->y, &zi3); + r->infinity = a->infinity; + secp256k1_ge_verify(r); } static void secp256k1_ge_set_xy(secp256k1_ge *r, const secp256k1_fe *x, const secp256k1_fe *y) { + secp256k1_fe_verify(x); + secp256k1_fe_verify(y); r->infinity = 0; r->x = *x; r->y = *y; + secp256k1_ge_verify(r); } static int secp256k1_ge_is_infinity(const secp256k1_ge *a) { + secp256k1_ge_verify(a); return a->infinity; } static void secp256k1_ge_neg(secp256k1_ge *r, const secp256k1_ge *a) { + secp256k1_ge_verify(a); *r = *a; secp256k1_fe_normalize_weak(&r->y); secp256k1_fe_negate(&r->y, &r->y, 1); + secp256k1_ge_verify(r); } static void secp256k1_ge_set_gej(secp256k1_ge *r, secp256k1_gej *a) { secp256k1_fe z2, z3; + secp256k1_gej_verify(a); r->infinity = a->infinity; secp256k1_fe_inv(&a->z, &a->z); secp256k1_fe_sqr(&z2, &a->z); @@ -112,14 +156,17 @@ static void secp256k1_ge_set_gej(secp256k1_ge *r, secp256k1_gej *a) { secp256k1_fe_set_int(&a->z, 1); r->x = a->x; r->y = a->y; + secp256k1_ge_verify(r); } static void secp256k1_ge_set_gej_var(secp256k1_ge *r, secp256k1_gej *a) { secp256k1_fe z2, z3; - if (a->infinity) { + secp256k1_gej_verify(a); + if (secp256k1_gej_is_infinity(a)) { secp256k1_ge_set_infinity(r); return; } + r->infinity = 0; secp256k1_fe_inv_var(&a->z, &a->z); secp256k1_fe_sqr(&z2, &a->z); secp256k1_fe_mul(&z3, &a->z, &z2); @@ -127,6 +174,7 @@ static void secp256k1_ge_set_gej_var(secp256k1_ge *r, secp256k1_gej *a) { secp256k1_fe_mul(&a->y, &a->y, &z3); secp256k1_fe_set_int(&a->z, 1); secp256k1_ge_set_xy(r, &a->x, &a->y); + secp256k1_ge_verify(r); } static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a, size_t len) { @@ -135,6 +183,7 @@ static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a size_t last_i = SIZE_MAX; for (i = 0; i < len; i++) { + secp256k1_gej_verify(&a[i]); if (a[i].infinity) { secp256k1_ge_set_infinity(&r[i]); } else { @@ -168,6 +217,7 @@ static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a if (!a[i].infinity) { secp256k1_ge_set_gej_zinv(&r[i], &a[i], &r[i].x); } + secp256k1_ge_verify(&r[i]); } } @@ -176,21 +226,25 @@ static void secp256k1_ge_table_set_globalz(size_t len, secp256k1_ge *a, const se secp256k1_fe zs; if (len > 0) { + /* Verify inputs a[len-1] and zr[len-1]. */ + secp256k1_ge_verify(&a[i]); + secp256k1_fe_verify(&zr[i]); /* Ensure all y values are in weak normal form for fast negation of points */ secp256k1_fe_normalize_weak(&a[i].y); zs = zr[i]; /* Work our way backwards, using the z-ratios to scale the x/y values. */ while (i > 0) { - secp256k1_gej tmpa; + /* Verify all inputs a[i] and zr[i]. */ + secp256k1_fe_verify(&zr[i]); + secp256k1_ge_verify(&a[i]); if (i != len - 1) { secp256k1_fe_mul(&zs, &zs, &zr[i]); } i--; - tmpa.x = a[i].x; - tmpa.y = a[i].y; - tmpa.infinity = 0; - secp256k1_ge_set_gej_zinv(&a[i], &tmpa, &zs); + secp256k1_ge_set_ge_zinv(&a[i], &a[i], &zs); + /* Verify the output a[i]. */ + secp256k1_ge_verify(&a[i]); } } } @@ -200,12 +254,14 @@ static void secp256k1_gej_set_infinity(secp256k1_gej *r) { secp256k1_fe_clear(&r->x); secp256k1_fe_clear(&r->y); secp256k1_fe_clear(&r->z); + secp256k1_gej_verify(r); } static void secp256k1_ge_set_infinity(secp256k1_ge *r) { r->infinity = 1; secp256k1_fe_clear(&r->x); secp256k1_fe_clear(&r->y); + secp256k1_ge_verify(r); } static void secp256k1_gej_clear(secp256k1_gej *r) { @@ -223,31 +279,35 @@ static void secp256k1_ge_clear(secp256k1_ge *r) { static int secp256k1_ge_set_xo_var(secp256k1_ge *r, const secp256k1_fe *x, int odd) { secp256k1_fe x2, x3; + int ret; + secp256k1_fe_verify(x); r->x = *x; secp256k1_fe_sqr(&x2, x); secp256k1_fe_mul(&x3, x, &x2); r->infinity = 0; secp256k1_fe_add_int(&x3, SECP256K1_B); - if (!secp256k1_fe_sqrt(&r->y, &x3)) { - return 0; - } + ret = secp256k1_fe_sqrt(&r->y, &x3); secp256k1_fe_normalize_var(&r->y); if (secp256k1_fe_is_odd(&r->y) != odd) { secp256k1_fe_negate(&r->y, &r->y, 1); } - return 1; - + secp256k1_ge_verify(r); + return ret; } static void secp256k1_gej_set_ge(secp256k1_gej *r, const secp256k1_ge *a) { + secp256k1_ge_verify(a); r->infinity = a->infinity; r->x = a->x; r->y = a->y; secp256k1_fe_set_int(&r->z, 1); + secp256k1_gej_verify(r); } static int secp256k1_gej_eq_var(const secp256k1_gej *a, const secp256k1_gej *b) { secp256k1_gej tmp; + secp256k1_gej_verify(b); + secp256k1_gej_verify(a); secp256k1_gej_neg(&tmp, a); secp256k1_gej_add_var(&tmp, &tmp, b, NULL); return secp256k1_gej_is_infinity(&tmp); @@ -255,6 +315,8 @@ static int secp256k1_gej_eq_var(const secp256k1_gej *a, const secp256k1_gej *b) static int secp256k1_gej_eq_x_var(const secp256k1_fe *x, const secp256k1_gej *a) { secp256k1_fe r, r2; + secp256k1_fe_verify(x); + secp256k1_gej_verify(a); VERIFY_CHECK(!a->infinity); secp256k1_fe_sqr(&r, &a->z); secp256k1_fe_mul(&r, &r, x); r2 = a->x; secp256k1_fe_normalize_weak(&r2); @@ -262,20 +324,24 @@ static int secp256k1_gej_eq_x_var(const secp256k1_fe *x, const secp256k1_gej *a) } static void secp256k1_gej_neg(secp256k1_gej *r, const secp256k1_gej *a) { + secp256k1_gej_verify(a); r->infinity = a->infinity; r->x = a->x; r->y = a->y; r->z = a->z; secp256k1_fe_normalize_weak(&r->y); secp256k1_fe_negate(&r->y, &r->y, 1); + secp256k1_gej_verify(r); } static int secp256k1_gej_is_infinity(const secp256k1_gej *a) { + secp256k1_gej_verify(a); return a->infinity; } static int secp256k1_ge_is_valid_var(const secp256k1_ge *a) { secp256k1_fe y2, x3; + secp256k1_ge_verify(a); if (a->infinity) { return 0; } @@ -291,6 +357,7 @@ static SECP256K1_INLINE void secp256k1_gej_double(secp256k1_gej *r, const secp25 /* Operations: 3 mul, 4 sqr, 8 add/half/mul_int/negate */ secp256k1_fe l, s, t; + secp256k1_gej_verify(a); r->infinity = a->infinity; /* Formula used: @@ -317,6 +384,7 @@ static SECP256K1_INLINE void secp256k1_gej_double(secp256k1_gej *r, const secp25 secp256k1_fe_mul(&r->y, &t, &l); /* Y3 = L*(X3 + T) (1) */ secp256k1_fe_add(&r->y, &s); /* Y3 = L*(X3 + T) + S^2 (2) */ secp256k1_fe_negate(&r->y, &r->y, 2); /* Y3 = -(L*(X3 + T) + S^2) (3) */ + secp256k1_gej_verify(r); } static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, secp256k1_fe *rzr) { @@ -330,6 +398,7 @@ static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, s * the infinity flag even though the point doubles to infinity, and the result * point will be gibberish (z = 0 but infinity = 0). */ + secp256k1_gej_verify(a); if (a->infinity) { secp256k1_gej_set_infinity(r); if (rzr != NULL) { @@ -344,12 +413,15 @@ static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, s } secp256k1_gej_double(r, a); + secp256k1_gej_verify(r); } static void secp256k1_gej_add_var(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_gej *b, secp256k1_fe *rzr) { /* 12 mul, 4 sqr, 11 add/negate/normalizes_to_zero (ignoring special cases) */ secp256k1_fe z22, z12, u1, u2, s1, s2, h, i, h2, h3, t; + secp256k1_gej_verify(a); + secp256k1_gej_verify(b); if (a->infinity) { VERIFY_CHECK(rzr == NULL); *r = *b; @@ -404,11 +476,14 @@ static void secp256k1_gej_add_var(secp256k1_gej *r, const secp256k1_gej *a, cons secp256k1_fe_mul(&r->y, &t, &i); secp256k1_fe_mul(&h3, &h3, &s1); secp256k1_fe_add(&r->y, &h3); + secp256k1_gej_verify(r); } static void secp256k1_gej_add_ge_var(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_ge *b, secp256k1_fe *rzr) { /* 8 mul, 3 sqr, 13 add/negate/normalize_weak/normalizes_to_zero (ignoring special cases) */ secp256k1_fe z12, u1, u2, s1, s2, h, i, h2, h3, t; + secp256k1_gej_verify(a); + secp256k1_ge_verify(b); if (a->infinity) { VERIFY_CHECK(rzr == NULL); secp256k1_gej_set_ge(r, b); @@ -461,12 +536,16 @@ static void secp256k1_gej_add_ge_var(secp256k1_gej *r, const secp256k1_gej *a, c secp256k1_fe_mul(&r->y, &t, &i); secp256k1_fe_mul(&h3, &h3, &s1); secp256k1_fe_add(&r->y, &h3); + secp256k1_gej_verify(r); + if (rzr != NULL) secp256k1_fe_verify(rzr); } static void secp256k1_gej_add_zinv_var(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_ge *b, const secp256k1_fe *bzinv) { /* 9 mul, 3 sqr, 13 add/negate/normalize_weak/normalizes_to_zero (ignoring special cases) */ secp256k1_fe az, z12, u1, u2, s1, s2, h, i, h2, h3, t; + secp256k1_ge_verify(b); + secp256k1_fe_verify(bzinv); if (a->infinity) { secp256k1_fe bzinv2, bzinv3; r->infinity = b->infinity; @@ -525,6 +604,7 @@ static void secp256k1_gej_add_zinv_var(secp256k1_gej *r, const secp256k1_gej *a, secp256k1_fe_mul(&r->y, &t, &i); secp256k1_fe_mul(&h3, &h3, &s1); secp256k1_fe_add(&r->y, &h3); + secp256k1_gej_verify(r); } @@ -533,6 +613,8 @@ static void secp256k1_gej_add_ge(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_fe zz, u1, u2, s1, s2, t, tt, m, n, q, rr; secp256k1_fe m_alt, rr_alt; int degenerate; + secp256k1_gej_verify(a); + secp256k1_ge_verify(b); VERIFY_CHECK(!b->infinity); VERIFY_CHECK(a->infinity == 0 || a->infinity == 1); @@ -658,21 +740,28 @@ static void secp256k1_gej_add_ge(secp256k1_gej *r, const secp256k1_gej *a, const * We have degenerate = false, r->z = (y1 + y2) * Z. * Then r->infinity = ((y1 + y2)Z == 0) = (y1 == -y2) = false. */ r->infinity = secp256k1_fe_normalizes_to_zero(&r->z); + secp256k1_gej_verify(r); } static void secp256k1_gej_rescale(secp256k1_gej *r, const secp256k1_fe *s) { /* Operations: 4 mul, 1 sqr */ secp256k1_fe zz; - VERIFY_CHECK(!secp256k1_fe_is_zero(s)); + secp256k1_gej_verify(r); + secp256k1_fe_verify(s); +#ifdef VERIFY + VERIFY_CHECK(!secp256k1_fe_normalizes_to_zero_var(s)); +#endif secp256k1_fe_sqr(&zz, s); secp256k1_fe_mul(&r->x, &r->x, &zz); /* r->x *= s^2 */ secp256k1_fe_mul(&r->y, &r->y, &zz); secp256k1_fe_mul(&r->y, &r->y, s); /* r->y *= s^3 */ secp256k1_fe_mul(&r->z, &r->z, s); /* r->z *= s */ + secp256k1_gej_verify(r); } static void secp256k1_ge_to_storage(secp256k1_ge_storage *r, const secp256k1_ge *a) { secp256k1_fe x, y; + secp256k1_ge_verify(a); VERIFY_CHECK(!a->infinity); x = a->x; secp256k1_fe_normalize(&x); @@ -686,14 +775,18 @@ static void secp256k1_ge_from_storage(secp256k1_ge *r, const secp256k1_ge_storag secp256k1_fe_from_storage(&r->x, &a->x); secp256k1_fe_from_storage(&r->y, &a->y); r->infinity = 0; + secp256k1_ge_verify(r); } static SECP256K1_INLINE void secp256k1_gej_cmov(secp256k1_gej *r, const secp256k1_gej *a, int flag) { + secp256k1_gej_verify(r); + secp256k1_gej_verify(a); secp256k1_fe_cmov(&r->x, &a->x, flag); secp256k1_fe_cmov(&r->y, &a->y, flag); secp256k1_fe_cmov(&r->z, &a->z, flag); r->infinity ^= (r->infinity ^ a->infinity) & flag; + secp256k1_gej_verify(r); } static SECP256K1_INLINE void secp256k1_ge_storage_cmov(secp256k1_ge_storage *r, const secp256k1_ge_storage *a, int flag) { @@ -703,7 +796,9 @@ static SECP256K1_INLINE void secp256k1_ge_storage_cmov(secp256k1_ge_storage *r, static void secp256k1_ge_mul_lambda(secp256k1_ge *r, const secp256k1_ge *a) { *r = *a; + secp256k1_ge_verify(a); secp256k1_fe_mul(&r->x, &r->x, &secp256k1_const_beta); + secp256k1_ge_verify(r); } static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge) { @@ -711,6 +806,7 @@ static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge) { secp256k1_gej out; int i; + secp256k1_ge_verify(ge); /* A very simple EC multiplication ladder that avoids a dependency on ecmult. */ secp256k1_gej_set_infinity(&out); for (i = 0; i < 32; ++i) { @@ -727,4 +823,32 @@ static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge) { #endif } +static int secp256k1_ge_x_on_curve_var(const secp256k1_fe *x) { + secp256k1_fe c; + secp256k1_fe_sqr(&c, x); + secp256k1_fe_mul(&c, &c, x); + secp256k1_fe_add_int(&c, SECP256K1_B); + return secp256k1_fe_is_square_var(&c); +} + +static int secp256k1_ge_x_frac_on_curve_var(const secp256k1_fe *xn, const secp256k1_fe *xd) { + /* We want to determine whether (xn/xd) is on the curve. + * + * (xn/xd)^3 + 7 is square <=> xd*xn^3 + 7*xd^4 is square (multiplying by xd^4, a square). + */ + secp256k1_fe r, t; +#ifdef VERIFY + VERIFY_CHECK(!secp256k1_fe_normalizes_to_zero_var(xd)); +#endif + secp256k1_fe_mul(&r, xd, xn); /* r = xd*xn */ + secp256k1_fe_sqr(&t, xn); /* t = xn^2 */ + secp256k1_fe_mul(&r, &r, &t); /* r = xd*xn^3 */ + secp256k1_fe_sqr(&t, xd); /* t = xd^2 */ + secp256k1_fe_sqr(&t, &t); /* t = xd^4 */ + VERIFY_CHECK(SECP256K1_B <= 31); + secp256k1_fe_mul_int(&t, SECP256K1_B); /* t = 7*xd^4 */ + secp256k1_fe_add(&r, &t); /* r = xd*xn^3 + 7*xd^4 */ + return secp256k1_fe_is_square_var(&r); +} + #endif /* SECP256K1_GROUP_IMPL_H */ diff --git a/src/secp256k1/src/int128_native_impl.h b/src/secp256k1/src/int128_native_impl.h index 996e542cf9..7f02e1590b 100644 --- a/src/secp256k1/src/int128_native_impl.h +++ b/src/secp256k1/src/int128_native_impl.h @@ -2,6 +2,7 @@ #define SECP256K1_INT128_NATIVE_IMPL_H #include "int128.h" +#include "util.h" static SECP256K1_INLINE void secp256k1_u128_load(secp256k1_uint128 *r, uint64_t hi, uint64_t lo) { *r = (((uint128_t)hi) << 64) + lo; diff --git a/src/secp256k1/src/int128_struct_impl.h b/src/secp256k1/src/int128_struct_impl.h index cc17bad167..990982da84 100644 --- a/src/secp256k1/src/int128_struct_impl.h +++ b/src/secp256k1/src/int128_struct_impl.h @@ -2,6 +2,7 @@ #define SECP256K1_INT128_STRUCT_IMPL_H #include "int128.h" +#include "util.h" #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) /* MSVC */ # include <intrin.h> diff --git a/src/secp256k1/src/modinv32_impl.h b/src/secp256k1/src/modinv32_impl.h index 8e400b697b..0ea2699863 100644 --- a/src/secp256k1/src/modinv32_impl.h +++ b/src/secp256k1/src/modinv32_impl.h @@ -64,7 +64,7 @@ static void secp256k1_modinv32_normalize_30(secp256k1_modinv32_signed30 *r, int3 const int32_t M30 = (int32_t)(UINT32_MAX >> 2); int32_t r0 = r->v[0], r1 = r->v[1], r2 = r->v[2], r3 = r->v[3], r4 = r->v[4], r5 = r->v[5], r6 = r->v[6], r7 = r->v[7], r8 = r->v[8]; - int32_t cond_add, cond_negate; + volatile int32_t cond_add, cond_negate; #ifdef VERIFY /* Verify that all limbs are in range (-2^30,2^30). */ @@ -186,7 +186,8 @@ static int32_t secp256k1_modinv32_divsteps_30(int32_t zeta, uint32_t f0, uint32_ * being inside [-2^31,2^31) means that casting to signed works correctly. */ uint32_t u = 1, v = 0, q = 0, r = 1; - uint32_t c1, c2, f = f0, g = g0, x, y, z; + volatile uint32_t c1, c2; + uint32_t mask1, mask2, f = f0, g = g0, x, y, z; int i; for (i = 0; i < 30; ++i) { @@ -195,23 +196,25 @@ static int32_t secp256k1_modinv32_divsteps_30(int32_t zeta, uint32_t f0, uint32_ VERIFY_CHECK((q * f0 + r * g0) == g << i); /* Compute conditional masks for (zeta < 0) and for (g & 1). */ c1 = zeta >> 31; - c2 = -(g & 1); + mask1 = c1; + c2 = g & 1; + mask2 = -c2; /* Compute x,y,z, conditionally negated versions of f,u,v. */ - x = (f ^ c1) - c1; - y = (u ^ c1) - c1; - z = (v ^ c1) - c1; + x = (f ^ mask1) - mask1; + y = (u ^ mask1) - mask1; + z = (v ^ mask1) - mask1; /* Conditionally add x,y,z to g,q,r. */ - g += x & c2; - q += y & c2; - r += z & c2; - /* In what follows, c1 is a condition mask for (zeta < 0) and (g & 1). */ - c1 &= c2; + g += x & mask2; + q += y & mask2; + r += z & mask2; + /* In what follows, mask1 is a condition mask for (zeta < 0) and (g & 1). */ + mask1 &= mask2; /* Conditionally change zeta into -zeta-2 or zeta-1. */ - zeta = (zeta ^ c1) - 1; + zeta = (zeta ^ mask1) - 1; /* Conditionally add g,q,r to f,u,v. */ - f += g & c1; - u += q & c1; - v += r & c1; + f += g & mask1; + u += q & mask1; + v += r & mask1; /* Shifts */ g >>= 1; u <<= 1; diff --git a/src/secp256k1/src/modinv64_impl.h b/src/secp256k1/src/modinv64_impl.h index e33727d385..c7cef872a4 100644 --- a/src/secp256k1/src/modinv64_impl.h +++ b/src/secp256k1/src/modinv64_impl.h @@ -88,7 +88,7 @@ static int secp256k1_modinv64_det_check_pow2(const secp256k1_modinv64_trans2x2 * static void secp256k1_modinv64_normalize_62(secp256k1_modinv64_signed62 *r, int64_t sign, const secp256k1_modinv64_modinfo *modinfo) { const int64_t M62 = (int64_t)(UINT64_MAX >> 2); int64_t r0 = r->v[0], r1 = r->v[1], r2 = r->v[2], r3 = r->v[3], r4 = r->v[4]; - int64_t cond_add, cond_negate; + volatile int64_t cond_add, cond_negate; #ifdef VERIFY /* Verify that all limbs are in range (-2^62,2^62). */ @@ -175,7 +175,8 @@ static int64_t secp256k1_modinv64_divsteps_59(int64_t zeta, uint64_t f0, uint64_ * being inside [-2^63,2^63) means that casting to signed works correctly. */ uint64_t u = 8, v = 0, q = 0, r = 8; - uint64_t c1, c2, f = f0, g = g0, x, y, z; + volatile uint64_t c1, c2; + uint64_t mask1, mask2, f = f0, g = g0, x, y, z; int i; for (i = 3; i < 62; ++i) { @@ -184,23 +185,25 @@ static int64_t secp256k1_modinv64_divsteps_59(int64_t zeta, uint64_t f0, uint64_ VERIFY_CHECK((q * f0 + r * g0) == g << i); /* Compute conditional masks for (zeta < 0) and for (g & 1). */ c1 = zeta >> 63; - c2 = -(g & 1); + mask1 = c1; + c2 = g & 1; + mask2 = -c2; /* Compute x,y,z, conditionally negated versions of f,u,v. */ - x = (f ^ c1) - c1; - y = (u ^ c1) - c1; - z = (v ^ c1) - c1; + x = (f ^ mask1) - mask1; + y = (u ^ mask1) - mask1; + z = (v ^ mask1) - mask1; /* Conditionally add x,y,z to g,q,r. */ - g += x & c2; - q += y & c2; - r += z & c2; + g += x & mask2; + q += y & mask2; + r += z & mask2; /* In what follows, c1 is a condition mask for (zeta < 0) and (g & 1). */ - c1 &= c2; + mask1 &= mask2; /* Conditionally change zeta into -zeta-2 or zeta-1. */ - zeta = (zeta ^ c1) - 1; + zeta = (zeta ^ mask1) - 1; /* Conditionally add g,q,r to f,u,v. */ - f += g & c1; - u += q & c1; - v += r & c1; + f += g & mask1; + u += q & mask1; + v += r & mask1; /* Shifts */ g >>= 1; u <<= 1; diff --git a/src/secp256k1/src/modules/ecdh/main_impl.h b/src/secp256k1/src/modules/ecdh/main_impl.h index 5408c9de70..82b082a9f0 100644 --- a/src/secp256k1/src/modules/ecdh/main_impl.h +++ b/src/secp256k1/src/modules/ecdh/main_impl.h @@ -50,7 +50,7 @@ int secp256k1_ecdh(const secp256k1_context* ctx, unsigned char *output, const se overflow |= secp256k1_scalar_is_zero(&s); secp256k1_scalar_cmov(&s, &secp256k1_scalar_one, overflow); - secp256k1_ecmult_const(&res, &pt, &s, 256); + secp256k1_ecmult_const(&res, &pt, &s); secp256k1_ge_set_gej(&pt, &res); /* Compute a hash of the point */ diff --git a/src/secp256k1/src/modules/ellswift/Makefile.am.include b/src/secp256k1/src/modules/ellswift/Makefile.am.include new file mode 100644 index 0000000000..e7efea2981 --- /dev/null +++ b/src/secp256k1/src/modules/ellswift/Makefile.am.include @@ -0,0 +1,4 @@ +include_HEADERS += include/secp256k1_ellswift.h +noinst_HEADERS += src/modules/ellswift/bench_impl.h +noinst_HEADERS += src/modules/ellswift/main_impl.h +noinst_HEADERS += src/modules/ellswift/tests_impl.h diff --git a/src/secp256k1/src/modules/ellswift/bench_impl.h b/src/secp256k1/src/modules/ellswift/bench_impl.h new file mode 100644 index 0000000000..b16a3a3687 --- /dev/null +++ b/src/secp256k1/src/modules/ellswift/bench_impl.h @@ -0,0 +1,106 @@ +/*********************************************************************** + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php.* + ***********************************************************************/ + +#ifndef SECP256K1_MODULE_ELLSWIFT_BENCH_H +#define SECP256K1_MODULE_ELLSWIFT_BENCH_H + +#include "../../../include/secp256k1_ellswift.h" + +typedef struct { + secp256k1_context *ctx; + secp256k1_pubkey point[256]; + unsigned char rnd64[64]; +} bench_ellswift_data; + +static void bench_ellswift_setup(void *arg) { + int i; + bench_ellswift_data *data = (bench_ellswift_data*)arg; + static const unsigned char init[64] = { + 0x78, 0x1f, 0xb7, 0xd4, 0x67, 0x7f, 0x08, 0x68, + 0xdb, 0xe3, 0x1d, 0x7f, 0x1b, 0xb0, 0xf6, 0x9e, + 0x0a, 0x64, 0xca, 0x32, 0x9e, 0xc6, 0x20, 0x79, + 0x03, 0xf3, 0xd0, 0x46, 0x7a, 0x0f, 0xd2, 0x21, + 0xb0, 0x2c, 0x46, 0xd8, 0xba, 0xca, 0x26, 0x4f, + 0x8f, 0x8c, 0xd4, 0xdd, 0x2d, 0x04, 0xbe, 0x30, + 0x48, 0x51, 0x1e, 0xd4, 0x16, 0xfd, 0x42, 0x85, + 0x62, 0xc9, 0x02, 0xf9, 0x89, 0x84, 0xff, 0xdc + }; + memcpy(data->rnd64, init, 64); + for (i = 0; i < 256; ++i) { + int j; + CHECK(secp256k1_ellswift_decode(data->ctx, &data->point[i], data->rnd64)); + for (j = 0; j < 64; ++j) { + data->rnd64[j] += 1; + } + } + CHECK(secp256k1_ellswift_encode(data->ctx, data->rnd64, &data->point[255], init + 16)); +} + +static void bench_ellswift_encode(void *arg, int iters) { + int i; + bench_ellswift_data *data = (bench_ellswift_data*)arg; + + for (i = 0; i < iters; i++) { + CHECK(secp256k1_ellswift_encode(data->ctx, data->rnd64, &data->point[i & 255], data->rnd64 + 16)); + } +} + +static void bench_ellswift_create(void *arg, int iters) { + int i; + bench_ellswift_data *data = (bench_ellswift_data*)arg; + + for (i = 0; i < iters; i++) { + unsigned char buf[64]; + CHECK(secp256k1_ellswift_create(data->ctx, buf, data->rnd64, data->rnd64 + 32)); + memcpy(data->rnd64, buf, 64); + } +} + +static void bench_ellswift_decode(void *arg, int iters) { + int i; + secp256k1_pubkey out; + size_t len; + bench_ellswift_data *data = (bench_ellswift_data*)arg; + + for (i = 0; i < iters; i++) { + CHECK(secp256k1_ellswift_decode(data->ctx, &out, data->rnd64) == 1); + len = 33; + CHECK(secp256k1_ec_pubkey_serialize(data->ctx, data->rnd64 + (i % 32), &len, &out, SECP256K1_EC_COMPRESSED)); + } +} + +static void bench_ellswift_xdh(void *arg, int iters) { + int i; + bench_ellswift_data *data = (bench_ellswift_data*)arg; + + for (i = 0; i < iters; i++) { + int party = i & 1; + CHECK(secp256k1_ellswift_xdh(data->ctx, + data->rnd64 + (i % 33), + data->rnd64, + data->rnd64, + data->rnd64 + ((i + 16) % 33), + party, + secp256k1_ellswift_xdh_hash_function_bip324, + NULL) == 1); + } +} + +void run_ellswift_bench(int iters, int argc, char **argv) { + bench_ellswift_data data; + int d = argc == 1; + + /* create a context with signing capabilities */ + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); + + if (d || have_flag(argc, argv, "ellswift") || have_flag(argc, argv, "encode") || have_flag(argc, argv, "ellswift_encode")) run_benchmark("ellswift_encode", bench_ellswift_encode, bench_ellswift_setup, NULL, &data, 10, iters); + if (d || have_flag(argc, argv, "ellswift") || have_flag(argc, argv, "decode") || have_flag(argc, argv, "ellswift_decode")) run_benchmark("ellswift_decode", bench_ellswift_decode, bench_ellswift_setup, NULL, &data, 10, iters); + if (d || have_flag(argc, argv, "ellswift") || have_flag(argc, argv, "keygen") || have_flag(argc, argv, "ellswift_keygen")) run_benchmark("ellswift_keygen", bench_ellswift_create, bench_ellswift_setup, NULL, &data, 10, iters); + if (d || have_flag(argc, argv, "ellswift") || have_flag(argc, argv, "ecdh") || have_flag(argc, argv, "ellswift_ecdh")) run_benchmark("ellswift_ecdh", bench_ellswift_xdh, bench_ellswift_setup, NULL, &data, 10, iters); + + secp256k1_context_destroy(data.ctx); +} + +#endif diff --git a/src/secp256k1/src/modules/ellswift/main_impl.h b/src/secp256k1/src/modules/ellswift/main_impl.h new file mode 100644 index 0000000000..00bb8a3da5 --- /dev/null +++ b/src/secp256k1/src/modules/ellswift/main_impl.h @@ -0,0 +1,589 @@ +/*********************************************************************** + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php.* + ***********************************************************************/ + +#ifndef SECP256K1_MODULE_ELLSWIFT_MAIN_H +#define SECP256K1_MODULE_ELLSWIFT_MAIN_H + +#include "../../../include/secp256k1.h" +#include "../../../include/secp256k1_ellswift.h" +#include "../../eckey.h" +#include "../../hash.h" + +/** c1 = (sqrt(-3)-1)/2 */ +static const secp256k1_fe secp256k1_ellswift_c1 = SECP256K1_FE_CONST(0x851695d4, 0x9a83f8ef, 0x919bb861, 0x53cbcb16, 0x630fb68a, 0xed0a766a, 0x3ec693d6, 0x8e6afa40); +/** c2 = (-sqrt(-3)-1)/2 = -(c1+1) */ +static const secp256k1_fe secp256k1_ellswift_c2 = SECP256K1_FE_CONST(0x7ae96a2b, 0x657c0710, 0x6e64479e, 0xac3434e9, 0x9cf04975, 0x12f58995, 0xc1396c28, 0x719501ee); +/** c3 = (-sqrt(-3)+1)/2 = -c1 = c2+1 */ +static const secp256k1_fe secp256k1_ellswift_c3 = SECP256K1_FE_CONST(0x7ae96a2b, 0x657c0710, 0x6e64479e, 0xac3434e9, 0x9cf04975, 0x12f58995, 0xc1396c28, 0x719501ef); +/** c4 = (sqrt(-3)+1)/2 = -c2 = c1+1 */ +static const secp256k1_fe secp256k1_ellswift_c4 = SECP256K1_FE_CONST(0x851695d4, 0x9a83f8ef, 0x919bb861, 0x53cbcb16, 0x630fb68a, 0xed0a766a, 0x3ec693d6, 0x8e6afa41); + +/** Decode ElligatorSwift encoding (u, t) to a fraction xn/xd representing a curve X coordinate. */ +static void secp256k1_ellswift_xswiftec_frac_var(secp256k1_fe *xn, secp256k1_fe *xd, const secp256k1_fe *u, const secp256k1_fe *t) { + /* The implemented algorithm is the following (all operations in GF(p)): + * + * - Let c0 = sqrt(-3) = 0xa2d2ba93507f1df233770c2a797962cc61f6d15da14ecd47d8d27ae1cd5f852. + * - If u = 0, set u = 1. + * - If t = 0, set t = 1. + * - If u^3+7+t^2 = 0, set t = 2*t. + * - Let X = (u^3+7-t^2)/(2*t). + * - Let Y = (X+t)/(c0*u). + * - If x3 = u+4*Y^2 is a valid x coordinate, return it. + * - If x2 = (-X/Y-u)/2 is a valid x coordinate, return it. + * - Return x1 = (X/Y-u)/2 (which is now guaranteed to be a valid x coordinate). + * + * Introducing s=t^2, g=u^3+7, and simplifying x1=-(x2+u) we get: + * + * - Let c0 = ... + * - If u = 0, set u = 1. + * - If t = 0, set t = 1. + * - Let s = t^2 + * - Let g = u^3+7 + * - If g+s = 0, set t = 2*t, s = 4*s + * - Let X = (g-s)/(2*t). + * - Let Y = (X+t)/(c0*u) = (g+s)/(2*c0*t*u). + * - If x3 = u+4*Y^2 is a valid x coordinate, return it. + * - If x2 = (-X/Y-u)/2 is a valid x coordinate, return it. + * - Return x1 = -(x2+u). + * + * Now substitute Y^2 = -(g+s)^2/(12*s*u^2) and X/Y = c0*u*(g-s)/(g+s). This + * means X and Y do not need to be evaluated explicitly anymore. + * + * - ... + * - If g+s = 0, set s = 4*s. + * - If x3 = u-(g+s)^2/(3*s*u^2) is a valid x coordinate, return it. + * - If x2 = (-c0*u*(g-s)/(g+s)-u)/2 is a valid x coordinate, return it. + * - Return x1 = -(x2+u). + * + * Simplifying x2 using 2 additional constants: + * + * - Let c1 = (c0-1)/2 = 0x851695d49a83f8ef919bb86153cbcb16630fb68aed0a766a3ec693d68e6afa40. + * - Let c2 = (-c0-1)/2 = 0x7ae96a2b657c07106e64479eac3434e99cf0497512f58995c1396c28719501ee. + * - ... + * - If x2 = u*(c1*s+c2*g)/(g+s) is a valid x coordinate, return it. + * - ... + * + * Writing x3 as a fraction: + * + * - ... + * - If x3 = (3*s*u^3-(g+s)^2)/(3*s*u^2) ... + * - ... + + * Overall, we get: + * + * - Let c1 = 0x851695d49a83f8ef919bb86153cbcb16630fb68aed0a766a3ec693d68e6afa40. + * - Let c2 = 0x7ae96a2b657c07106e64479eac3434e99cf0497512f58995c1396c28719501ee. + * - If u = 0, set u = 1. + * - If t = 0, set s = 1, else set s = t^2. + * - Let g = u^3+7. + * - If g+s = 0, set s = 4*s. + * - If x3 = (3*s*u^3-(g+s)^2)/(3*s*u^2) is a valid x coordinate, return it. + * - If x2 = u*(c1*s+c2*g)/(g+s) is a valid x coordinate, return it. + * - Return x1 = -(x2+u). + */ + secp256k1_fe u1, s, g, p, d, n, l; + u1 = *u; + if (EXPECT(secp256k1_fe_normalizes_to_zero_var(&u1), 0)) u1 = secp256k1_fe_one; + secp256k1_fe_sqr(&s, t); + if (EXPECT(secp256k1_fe_normalizes_to_zero_var(t), 0)) s = secp256k1_fe_one; + secp256k1_fe_sqr(&l, &u1); /* l = u^2 */ + secp256k1_fe_mul(&g, &l, &u1); /* g = u^3 */ + secp256k1_fe_add_int(&g, SECP256K1_B); /* g = u^3 + 7 */ + p = g; /* p = g */ + secp256k1_fe_add(&p, &s); /* p = g+s */ + if (EXPECT(secp256k1_fe_normalizes_to_zero_var(&p), 0)) { + secp256k1_fe_mul_int(&s, 4); + /* Recompute p = g+s */ + p = g; /* p = g */ + secp256k1_fe_add(&p, &s); /* p = g+s */ + } + secp256k1_fe_mul(&d, &s, &l); /* d = s*u^2 */ + secp256k1_fe_mul_int(&d, 3); /* d = 3*s*u^2 */ + secp256k1_fe_sqr(&l, &p); /* l = (g+s)^2 */ + secp256k1_fe_negate(&l, &l, 1); /* l = -(g+s)^2 */ + secp256k1_fe_mul(&n, &d, &u1); /* n = 3*s*u^3 */ + secp256k1_fe_add(&n, &l); /* n = 3*s*u^3-(g+s)^2 */ + if (secp256k1_ge_x_frac_on_curve_var(&n, &d)) { + /* Return x3 = n/d = (3*s*u^3-(g+s)^2)/(3*s*u^2) */ + *xn = n; + *xd = d; + return; + } + *xd = p; + secp256k1_fe_mul(&l, &secp256k1_ellswift_c1, &s); /* l = c1*s */ + secp256k1_fe_mul(&n, &secp256k1_ellswift_c2, &g); /* n = c2*g */ + secp256k1_fe_add(&n, &l); /* n = c1*s+c2*g */ + secp256k1_fe_mul(&n, &n, &u1); /* n = u*(c1*s+c2*g) */ + /* Possible optimization: in the invocation below, p^2 = (g+s)^2 is computed, + * which we already have computed above. This could be deduplicated. */ + if (secp256k1_ge_x_frac_on_curve_var(&n, &p)) { + /* Return x2 = n/p = u*(c1*s+c2*g)/(g+s) */ + *xn = n; + return; + } + secp256k1_fe_mul(&l, &p, &u1); /* l = u*(g+s) */ + secp256k1_fe_add(&n, &l); /* n = u*(c1*s+c2*g)+u*(g+s) */ + secp256k1_fe_negate(xn, &n, 2); /* n = -u*(c1*s+c2*g)-u*(g+s) */ +#ifdef VERIFY + VERIFY_CHECK(secp256k1_ge_x_frac_on_curve_var(xn, &p)); +#endif + /* Return x3 = n/p = -(u*(c1*s+c2*g)/(g+s)+u) */ +} + +/** Decode ElligatorSwift encoding (u, t) to X coordinate. */ +static void secp256k1_ellswift_xswiftec_var(secp256k1_fe *x, const secp256k1_fe *u, const secp256k1_fe *t) { + secp256k1_fe xn, xd; + secp256k1_ellswift_xswiftec_frac_var(&xn, &xd, u, t); + secp256k1_fe_inv_var(&xd, &xd); + secp256k1_fe_mul(x, &xn, &xd); +} + +/** Decode ElligatorSwift encoding (u, t) to point P. */ +static void secp256k1_ellswift_swiftec_var(secp256k1_ge *p, const secp256k1_fe *u, const secp256k1_fe *t) { + secp256k1_fe x; + secp256k1_ellswift_xswiftec_var(&x, u, t); + secp256k1_ge_set_xo_var(p, &x, secp256k1_fe_is_odd(t)); +} + +/* Try to complete an ElligatorSwift encoding (u, t) for X coordinate x, given u and x. + * + * There may be up to 8 distinct t values such that (u, t) decodes back to x, but also + * fewer, or none at all. Each such partial inverse can be accessed individually using a + * distinct input argument c (in range 0-7), and some or all of these may return failure. + * The following guarantees exist: + * - Given (x, u), no two distinct c values give the same successful result t. + * - Every successful result maps back to x through secp256k1_ellswift_xswiftec_var. + * - Given (x, u), all t values that map back to x can be reached by combining the + * successful results from this function over all c values, with the exception of: + * - this function cannot be called with u=0 + * - no result with t=0 will be returned + * - no result for which u^3 + t^2 + 7 = 0 will be returned. + * + * The rather unusual encoding of bits in c (a large "if" based on the middle bit, and then + * using the low and high bits to pick signs of square roots) is to match the paper's + * encoding more closely: c=0 through c=3 match branches 1..4 in the paper, while c=4 through + * c=7 are copies of those with an additional negation of sqrt(w). + */ +static int secp256k1_ellswift_xswiftec_inv_var(secp256k1_fe *t, const secp256k1_fe *x_in, const secp256k1_fe *u_in, int c) { + /* The implemented algorithm is this (all arithmetic, except involving c, is mod p): + * + * - If (c & 2) = 0: + * - If (-x-u) is a valid X coordinate, fail. + * - Let s=-(u^3+7)/(u^2+u*x+x^2). + * - If s is not square, fail. + * - Let v=x. + * - If (c & 2) = 2: + * - Let s=x-u. + * - If s is not square, fail. + * - Let r=sqrt(-s*(4*(u^3+7)+3*u^2*s)); fail if it doesn't exist. + * - If (c & 1) = 1 and r = 0, fail. + * - If s=0, fail. + * - Let v=(r/s-u)/2. + * - Let w=sqrt(s). + * - If (c & 5) = 0: return -w*(c3*u + v). + * - If (c & 5) = 1: return w*(c4*u + v). + * - If (c & 5) = 4: return w*(c3*u + v). + * - If (c & 5) = 5: return -w*(c4*u + v). + */ + secp256k1_fe x = *x_in, u = *u_in, g, v, s, m, r, q; + int ret; + + secp256k1_fe_normalize_weak(&x); + secp256k1_fe_normalize_weak(&u); + +#ifdef VERIFY + VERIFY_CHECK(c >= 0 && c < 8); + VERIFY_CHECK(secp256k1_ge_x_on_curve_var(&x)); +#endif + + if (!(c & 2)) { + /* c is in {0, 1, 4, 5}. In this case we look for an inverse under the x1 (if c=0 or + * c=4) formula, or x2 (if c=1 or c=5) formula. */ + + /* If -u-x is a valid X coordinate, fail. This would yield an encoding that roundtrips + * back under the x3 formula instead (which has priority over x1 and x2, so the decoding + * would not match x). */ + m = x; /* m = x */ + secp256k1_fe_add(&m, &u); /* m = u+x */ + secp256k1_fe_negate(&m, &m, 2); /* m = -u-x */ + /* Test if (-u-x) is a valid X coordinate. If so, fail. */ + if (secp256k1_ge_x_on_curve_var(&m)) return 0; + + /* Let s = -(u^3 + 7)/(u^2 + u*x + x^2) [first part] */ + secp256k1_fe_sqr(&s, &m); /* s = (u+x)^2 */ + secp256k1_fe_negate(&s, &s, 1); /* s = -(u+x)^2 */ + secp256k1_fe_mul(&m, &u, &x); /* m = u*x */ + secp256k1_fe_add(&s, &m); /* s = -(u^2 + u*x + x^2) */ + + /* Note that at this point, s = 0 is impossible. If it were the case: + * s = -(u^2 + u*x + x^2) = 0 + * => u^2 + u*x + x^2 = 0 + * => (u + 2*x) * (u^2 + u*x + x^2) = 0 + * => 2*x^3 + 3*x^2*u + 3*x*u^2 + u^3 = 0 + * => (x + u)^3 + x^3 = 0 + * => x^3 = -(x + u)^3 + * => x^3 + B = (-u - x)^3 + B + * + * However, we know x^3 + B is square (because x is on the curve) and + * that (-u-x)^3 + B is not square (the secp256k1_ge_x_on_curve_var(&m) + * test above would have failed). This is a contradiction, and thus the + * assumption s=0 is false. */ +#ifdef VERIFY + VERIFY_CHECK(!secp256k1_fe_normalizes_to_zero_var(&s)); +#endif + + /* If s is not square, fail. We have not fully computed s yet, but s is square iff + * -(u^3+7)*(u^2+u*x+x^2) is square (because a/b is square iff a*b is square and b is + * nonzero). */ + secp256k1_fe_sqr(&g, &u); /* g = u^2 */ + secp256k1_fe_mul(&g, &g, &u); /* g = u^3 */ + secp256k1_fe_add_int(&g, SECP256K1_B); /* g = u^3+7 */ + secp256k1_fe_mul(&m, &s, &g); /* m = -(u^3 + 7)*(u^2 + u*x + x^2) */ + if (!secp256k1_fe_is_square_var(&m)) return 0; + + /* Let s = -(u^3 + 7)/(u^2 + u*x + x^2) [second part] */ + secp256k1_fe_inv_var(&s, &s); /* s = -1/(u^2 + u*x + x^2) [no div by 0] */ + secp256k1_fe_mul(&s, &s, &g); /* s = -(u^3 + 7)/(u^2 + u*x + x^2) */ + + /* Let v = x. */ + v = x; + } else { + /* c is in {2, 3, 6, 7}. In this case we look for an inverse under the x3 formula. */ + + /* Let s = x-u. */ + secp256k1_fe_negate(&m, &u, 1); /* m = -u */ + s = m; /* s = -u */ + secp256k1_fe_add(&s, &x); /* s = x-u */ + + /* If s is not square, fail. */ + if (!secp256k1_fe_is_square_var(&s)) return 0; + + /* Let r = sqrt(-s*(4*(u^3+7)+3*u^2*s)); fail if it doesn't exist. */ + secp256k1_fe_sqr(&g, &u); /* g = u^2 */ + secp256k1_fe_mul(&q, &s, &g); /* q = s*u^2 */ + secp256k1_fe_mul_int(&q, 3); /* q = 3*s*u^2 */ + secp256k1_fe_mul(&g, &g, &u); /* g = u^3 */ + secp256k1_fe_mul_int(&g, 4); /* g = 4*u^3 */ + secp256k1_fe_add_int(&g, 4 * SECP256K1_B); /* g = 4*(u^3+7) */ + secp256k1_fe_add(&q, &g); /* q = 4*(u^3+7)+3*s*u^2 */ + secp256k1_fe_mul(&q, &q, &s); /* q = s*(4*(u^3+7)+3*u^2*s) */ + secp256k1_fe_negate(&q, &q, 1); /* q = -s*(4*(u^3+7)+3*u^2*s) */ + if (!secp256k1_fe_is_square_var(&q)) return 0; + ret = secp256k1_fe_sqrt(&r, &q); /* r = sqrt(-s*(4*(u^3+7)+3*u^2*s)) */ + VERIFY_CHECK(ret); + + /* If (c & 1) = 1 and r = 0, fail. */ + if (EXPECT((c & 1) && secp256k1_fe_normalizes_to_zero_var(&r), 0)) return 0; + + /* If s = 0, fail. */ + if (EXPECT(secp256k1_fe_normalizes_to_zero_var(&s), 0)) return 0; + + /* Let v = (r/s-u)/2. */ + secp256k1_fe_inv_var(&v, &s); /* v = 1/s [no div by 0] */ + secp256k1_fe_mul(&v, &v, &r); /* v = r/s */ + secp256k1_fe_add(&v, &m); /* v = r/s-u */ + secp256k1_fe_half(&v); /* v = (r/s-u)/2 */ + } + + /* Let w = sqrt(s). */ + ret = secp256k1_fe_sqrt(&m, &s); /* m = sqrt(s) = w */ + VERIFY_CHECK(ret); + + /* Return logic. */ + if ((c & 5) == 0 || (c & 5) == 5) { + secp256k1_fe_negate(&m, &m, 1); /* m = -w */ + } + /* Now m = {-w if c&5=0 or c&5=5; w otherwise}. */ + secp256k1_fe_mul(&u, &u, c&1 ? &secp256k1_ellswift_c4 : &secp256k1_ellswift_c3); + /* u = {c4 if c&1=1; c3 otherwise}*u */ + secp256k1_fe_add(&u, &v); /* u = {c4 if c&1=1; c3 otherwise}*u + v */ + secp256k1_fe_mul(t, &m, &u); + return 1; +} + +/** Use SHA256 as a PRNG, returning SHA256(hasher || cnt). + * + * hasher is a SHA256 object to which an incrementing 4-byte counter is written to generate randomness. + * Writing 13 bytes (4 bytes for counter, plus 9 bytes for the SHA256 padding) cannot cross a + * 64-byte block size boundary (to make sure it only triggers a single SHA256 compression). */ +static void secp256k1_ellswift_prng(unsigned char* out32, const secp256k1_sha256 *hasher, uint32_t cnt) { + secp256k1_sha256 hash = *hasher; + unsigned char buf4[4]; +#ifdef VERIFY + size_t blocks = hash.bytes >> 6; +#endif + buf4[0] = cnt; + buf4[1] = cnt >> 8; + buf4[2] = cnt >> 16; + buf4[3] = cnt >> 24; + secp256k1_sha256_write(&hash, buf4, 4); + secp256k1_sha256_finalize(&hash, out32); +#ifdef VERIFY + /* Writing and finalizing together should trigger exactly one SHA256 compression. */ + VERIFY_CHECK(((hash.bytes) >> 6) == (blocks + 1)); +#endif +} + +/** Find an ElligatorSwift encoding (u, t) for X coordinate x, and random Y coordinate. + * + * u32 is the 32-byte big endian encoding of u; t is the output field element t that still + * needs encoding. + * + * hasher is a hasher in the secp256k1_ellswift_prng sense, with the same restrictions. */ +static void secp256k1_ellswift_xelligatorswift_var(unsigned char *u32, secp256k1_fe *t, const secp256k1_fe *x, const secp256k1_sha256 *hasher) { + /* Pool of 3-bit branch values. */ + unsigned char branch_hash[32]; + /* Number of 3-bit values in branch_hash left. */ + int branches_left = 0; + /* Field elements u and branch values are extracted from RNG based on hasher for consecutive + * values of cnt. cnt==0 is first used to populate a pool of 64 4-bit branch values. The 64 + * cnt values that follow are used to generate field elements u. cnt==65 (and multiples + * thereof) are used to repopulate the pool and start over, if that were ever necessary. + * On average, 4 iterations are needed. */ + uint32_t cnt = 0; + while (1) { + int branch; + secp256k1_fe u; + /* If the pool of branch values is empty, populate it. */ + if (branches_left == 0) { + secp256k1_ellswift_prng(branch_hash, hasher, cnt++); + branches_left = 64; + } + /* Take a 3-bit branch value from the branch pool (top bit is discarded). */ + --branches_left; + branch = (branch_hash[branches_left >> 1] >> ((branches_left & 1) << 2)) & 7; + /* Compute a new u value by hashing. */ + secp256k1_ellswift_prng(u32, hasher, cnt++); + /* overflow is not a problem (we prefer uniform u32 over uniform u). */ + secp256k1_fe_set_b32_mod(&u, u32); + /* Since u is the output of a hash, it should practically never be 0. We could apply the + * u=0 to u=1 correction here too to deal with that case still, but it's such a low + * probability event that we do not bother. */ +#ifdef VERIFY + VERIFY_CHECK(!secp256k1_fe_normalizes_to_zero_var(&u)); +#endif + /* Find a remainder t, and return it if found. */ + if (EXPECT(secp256k1_ellswift_xswiftec_inv_var(t, x, &u, branch), 0)) break; + } +} + +/** Find an ElligatorSwift encoding (u, t) for point P. + * + * This is similar secp256k1_ellswift_xelligatorswift_var, except it takes a full group element p + * as input, and returns an encoding that matches the provided Y coordinate rather than a random + * one. + */ +static void secp256k1_ellswift_elligatorswift_var(unsigned char *u32, secp256k1_fe *t, const secp256k1_ge *p, const secp256k1_sha256 *hasher) { + secp256k1_ellswift_xelligatorswift_var(u32, t, &p->x, hasher); + secp256k1_fe_normalize_var(t); + if (secp256k1_fe_is_odd(t) != secp256k1_fe_is_odd(&p->y)) { + secp256k1_fe_negate(t, t, 1); + secp256k1_fe_normalize_var(t); + } +} + +/** Set hash state to the BIP340 tagged hash midstate for "secp256k1_ellswift_encode". */ +static void secp256k1_ellswift_sha256_init_encode(secp256k1_sha256* hash) { + secp256k1_sha256_initialize(hash); + hash->s[0] = 0xd1a6524bul; + hash->s[1] = 0x028594b3ul; + hash->s[2] = 0x96e42f4eul; + hash->s[3] = 0x1037a177ul; + hash->s[4] = 0x1b8fcb8bul; + hash->s[5] = 0x56023885ul; + hash->s[6] = 0x2560ede1ul; + hash->s[7] = 0xd626b715ul; + + hash->bytes = 64; +} + +int secp256k1_ellswift_encode(const secp256k1_context *ctx, unsigned char *ell64, const secp256k1_pubkey *pubkey, const unsigned char *rnd32) { + secp256k1_ge p; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(ell64 != NULL); + ARG_CHECK(pubkey != NULL); + ARG_CHECK(rnd32 != NULL); + + if (secp256k1_pubkey_load(ctx, &p, pubkey)) { + secp256k1_fe t; + unsigned char p64[64] = {0}; + size_t ser_size; + int ser_ret; + secp256k1_sha256 hash; + + /* Set up hasher state; the used RNG is H(pubkey || "\x00"*31 || rnd32 || cnt++), using + * BIP340 tagged hash with tag "secp256k1_ellswift_encode". */ + secp256k1_ellswift_sha256_init_encode(&hash); + ser_ret = secp256k1_eckey_pubkey_serialize(&p, p64, &ser_size, 1); + VERIFY_CHECK(ser_ret && ser_size == 33); + secp256k1_sha256_write(&hash, p64, sizeof(p64)); + secp256k1_sha256_write(&hash, rnd32, 32); + + /* Compute ElligatorSwift encoding and construct output. */ + secp256k1_ellswift_elligatorswift_var(ell64, &t, &p, &hash); /* puts u in ell64[0..32] */ + secp256k1_fe_get_b32(ell64 + 32, &t); /* puts t in ell64[32..64] */ + return 1; + } + /* Only reached in case the provided pubkey is invalid. */ + memset(ell64, 0, 64); + return 0; +} + +/** Set hash state to the BIP340 tagged hash midstate for "secp256k1_ellswift_create". */ +static void secp256k1_ellswift_sha256_init_create(secp256k1_sha256* hash) { + secp256k1_sha256_initialize(hash); + hash->s[0] = 0xd29e1bf5ul; + hash->s[1] = 0xf7025f42ul; + hash->s[2] = 0x9b024773ul; + hash->s[3] = 0x094cb7d5ul; + hash->s[4] = 0xe59ed789ul; + hash->s[5] = 0x03bc9786ul; + hash->s[6] = 0x68335b35ul; + hash->s[7] = 0x4e363b53ul; + + hash->bytes = 64; +} + +int secp256k1_ellswift_create(const secp256k1_context *ctx, unsigned char *ell64, const unsigned char *seckey32, const unsigned char *auxrnd32) { + secp256k1_ge p; + secp256k1_fe t; + secp256k1_sha256 hash; + secp256k1_scalar seckey_scalar; + int ret; + static const unsigned char zero32[32] = {0}; + + /* Sanity check inputs. */ + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(ell64 != NULL); + memset(ell64, 0, 64); + ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); + ARG_CHECK(seckey32 != NULL); + + /* Compute (affine) public key */ + ret = secp256k1_ec_pubkey_create_helper(&ctx->ecmult_gen_ctx, &seckey_scalar, &p, seckey32); + secp256k1_declassify(ctx, &p, sizeof(p)); /* not constant time in produced pubkey */ + secp256k1_fe_normalize_var(&p.x); + secp256k1_fe_normalize_var(&p.y); + + /* Set up hasher state. The used RNG is H(privkey || "\x00"*32 [|| auxrnd32] || cnt++), + * using BIP340 tagged hash with tag "secp256k1_ellswift_create". */ + secp256k1_ellswift_sha256_init_create(&hash); + secp256k1_sha256_write(&hash, seckey32, 32); + secp256k1_sha256_write(&hash, zero32, sizeof(zero32)); + secp256k1_declassify(ctx, &hash, sizeof(hash)); /* private key is hashed now */ + if (auxrnd32) secp256k1_sha256_write(&hash, auxrnd32, 32); + + /* Compute ElligatorSwift encoding and construct output. */ + secp256k1_ellswift_elligatorswift_var(ell64, &t, &p, &hash); /* puts u in ell64[0..32] */ + secp256k1_fe_get_b32(ell64 + 32, &t); /* puts t in ell64[32..64] */ + + secp256k1_memczero(ell64, 64, !ret); + secp256k1_scalar_clear(&seckey_scalar); + + return ret; +} + +int secp256k1_ellswift_decode(const secp256k1_context *ctx, secp256k1_pubkey *pubkey, const unsigned char *ell64) { + secp256k1_fe u, t; + secp256k1_ge p; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(pubkey != NULL); + ARG_CHECK(ell64 != NULL); + + secp256k1_fe_set_b32_mod(&u, ell64); + secp256k1_fe_set_b32_mod(&t, ell64 + 32); + secp256k1_fe_normalize_var(&t); + secp256k1_ellswift_swiftec_var(&p, &u, &t); + secp256k1_pubkey_save(pubkey, &p); + return 1; +} + +static int ellswift_xdh_hash_function_prefix(unsigned char *output, const unsigned char *x32, const unsigned char *ell_a64, const unsigned char *ell_b64, void *data) { + secp256k1_sha256 sha; + + secp256k1_sha256_initialize(&sha); + secp256k1_sha256_write(&sha, data, 64); + secp256k1_sha256_write(&sha, ell_a64, 64); + secp256k1_sha256_write(&sha, ell_b64, 64); + secp256k1_sha256_write(&sha, x32, 32); + secp256k1_sha256_finalize(&sha, output); + + return 1; +} + +/** Set hash state to the BIP340 tagged hash midstate for "bip324_ellswift_xonly_ecdh". */ +static void secp256k1_ellswift_sha256_init_bip324(secp256k1_sha256* hash) { + secp256k1_sha256_initialize(hash); + hash->s[0] = 0x8c12d730ul; + hash->s[1] = 0x827bd392ul; + hash->s[2] = 0x9e4fb2eeul; + hash->s[3] = 0x207b373eul; + hash->s[4] = 0x2292bd7aul; + hash->s[5] = 0xaa5441bcul; + hash->s[6] = 0x15c3779ful; + hash->s[7] = 0xcfb52549ul; + + hash->bytes = 64; +} + +static int ellswift_xdh_hash_function_bip324(unsigned char* output, const unsigned char *x32, const unsigned char *ell_a64, const unsigned char *ell_b64, void *data) { + secp256k1_sha256 sha; + + (void)data; + + secp256k1_ellswift_sha256_init_bip324(&sha); + secp256k1_sha256_write(&sha, ell_a64, 64); + secp256k1_sha256_write(&sha, ell_b64, 64); + secp256k1_sha256_write(&sha, x32, 32); + secp256k1_sha256_finalize(&sha, output); + + return 1; +} + +const secp256k1_ellswift_xdh_hash_function secp256k1_ellswift_xdh_hash_function_prefix = ellswift_xdh_hash_function_prefix; +const secp256k1_ellswift_xdh_hash_function secp256k1_ellswift_xdh_hash_function_bip324 = ellswift_xdh_hash_function_bip324; + +int secp256k1_ellswift_xdh(const secp256k1_context *ctx, unsigned char *output, const unsigned char *ell_a64, const unsigned char *ell_b64, const unsigned char *seckey32, int party, secp256k1_ellswift_xdh_hash_function hashfp, void *data) { + int ret = 0; + int overflow; + secp256k1_scalar s; + secp256k1_fe xn, xd, px, u, t; + unsigned char sx[32]; + const unsigned char* theirs64; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(output != NULL); + ARG_CHECK(ell_a64 != NULL); + ARG_CHECK(ell_b64 != NULL); + ARG_CHECK(seckey32 != NULL); + ARG_CHECK(hashfp != NULL); + + /* Load remote public key (as fraction). */ + theirs64 = party ? ell_a64 : ell_b64; + secp256k1_fe_set_b32_mod(&u, theirs64); + secp256k1_fe_set_b32_mod(&t, theirs64 + 32); + secp256k1_ellswift_xswiftec_frac_var(&xn, &xd, &u, &t); + + /* Load private key (using one if invalid). */ + secp256k1_scalar_set_b32(&s, seckey32, &overflow); + overflow = secp256k1_scalar_is_zero(&s); + secp256k1_scalar_cmov(&s, &secp256k1_scalar_one, overflow); + + /* Compute shared X coordinate. */ + secp256k1_ecmult_const_xonly(&px, &xn, &xd, &s, 1); + secp256k1_fe_normalize(&px); + secp256k1_fe_get_b32(sx, &px); + + /* Invoke hasher */ + ret = hashfp(output, sx, ell_a64, ell_b64, data); + + memset(sx, 0, 32); + secp256k1_fe_clear(&px); + secp256k1_scalar_clear(&s); + + return !!ret & !overflow; +} + +#endif diff --git a/src/secp256k1/src/modules/ellswift/tests_impl.h b/src/secp256k1/src/modules/ellswift/tests_impl.h new file mode 100644 index 0000000000..86ca09862b --- /dev/null +++ b/src/secp256k1/src/modules/ellswift/tests_impl.h @@ -0,0 +1,434 @@ +/*********************************************************************** + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php.* + ***********************************************************************/ + +#ifndef SECP256K1_MODULE_ELLSWIFT_TESTS_H +#define SECP256K1_MODULE_ELLSWIFT_TESTS_H + +#include "../../../include/secp256k1_ellswift.h" + +struct ellswift_xswiftec_inv_test { + int enc_bitmap; + secp256k1_fe u; + secp256k1_fe x; + secp256k1_fe encs[8]; +}; + +struct ellswift_decode_test { + unsigned char enc[64]; + secp256k1_fe x; + int odd_y; +}; + +struct ellswift_xdh_test { + unsigned char priv_ours[32]; + unsigned char ellswift_ours[64]; + unsigned char ellswift_theirs[64]; + int initiating; + unsigned char shared_secret[32]; +}; + +/* Set of (point, encodings) test vectors, selected to maximize branch coverage, part of the BIP324 + * test vectors. Created using an independent implementation, and tested decoding against paper + * authors' code. */ +static const struct ellswift_xswiftec_inv_test ellswift_xswiftec_inv_tests[] = { + {0xcc, SECP256K1_FE_CONST(0x05ff6bda, 0xd900fc32, 0x61bc7fe3, 0x4e2fb0f5, 0x69f06e09, 0x1ae437d3, 0xa52e9da0, 0xcbfb9590), SECP256K1_FE_CONST(0x80cdf637, 0x74ec7022, 0xc89a5a85, 0x58e373a2, 0x79170285, 0xe0ab2741, 0x2dbce510, 0xbdfe23fc), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x45654798, 0xece071ba, 0x79286d04, 0xf7f3eb1c, 0x3f1d17dd, 0x883610f2, 0xad2efd82, 0xa287466b), SECP256K1_FE_CONST(0x0aeaa886, 0xf6b76c71, 0x58452418, 0xcbf5033a, 0xdc5747e9, 0xe9b5d3b2, 0x303db969, 0x36528557), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0xba9ab867, 0x131f8e45, 0x86d792fb, 0x080c14e3, 0xc0e2e822, 0x77c9ef0d, 0x52d1027c, 0x5d78b5c4), SECP256K1_FE_CONST(0xf5155779, 0x0948938e, 0xa7badbe7, 0x340afcc5, 0x23a8b816, 0x164a2c4d, 0xcfc24695, 0xc9ad76d8)}}, + {0x33, SECP256K1_FE_CONST(0x1737a85f, 0x4c8d146c, 0xec96e3ff, 0xdca76d99, 0x03dcf3bd, 0x53061868, 0xd478c78c, 0x63c2aa9e), SECP256K1_FE_CONST(0x39e48dd1, 0x50d2f429, 0xbe088dfd, 0x5b61882e, 0x7e840748, 0x3702ae9a, 0x5ab35927, 0xb15f85ea), {SECP256K1_FE_CONST(0x1be8cc0b, 0x04be0c68, 0x1d0c6a68, 0xf733f82c, 0x6c896e0c, 0x8a262fcd, 0x392918e3, 0x03a7abf4), SECP256K1_FE_CONST(0x605b5814, 0xbf9b8cb0, 0x66667c9e, 0x5480d22d, 0xc5b6c92f, 0x14b4af3e, 0xe0a9eb83, 0xb03685e3), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0xe41733f4, 0xfb41f397, 0xe2f39597, 0x08cc07d3, 0x937691f3, 0x75d9d032, 0xc6d6e71b, 0xfc58503b), SECP256K1_FE_CONST(0x9fa4a7eb, 0x4064734f, 0x99998361, 0xab7f2dd2, 0x3a4936d0, 0xeb4b50c1, 0x1f56147b, 0x4fc9764c), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0x00, SECP256K1_FE_CONST(0x1aaa1cce, 0xbf9c7241, 0x91033df3, 0x66b36f69, 0x1c4d902c, 0x228033ff, 0x4516d122, 0xb2564f68), SECP256K1_FE_CONST(0xc7554125, 0x9d3ba98f, 0x207eaa30, 0xc69634d1, 0x87d0b6da, 0x594e719e, 0x420f4898, 0x638fc5b0), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0x33, SECP256K1_FE_CONST(0x2323a1d0, 0x79b0fd72, 0xfc8bb62e, 0xc34230a8, 0x15cb0596, 0xc2bfac99, 0x8bd6b842, 0x60f5dc26), SECP256K1_FE_CONST(0x239342df, 0xb675500a, 0x34a19631, 0x0b8d87d5, 0x4f49dcac, 0x9da50c17, 0x43ceab41, 0xa7b249ff), {SECP256K1_FE_CONST(0xf63580b8, 0xaa49c484, 0x6de56e39, 0xe1b3e73f, 0x171e881e, 0xba8c66f6, 0x14e67e5c, 0x975dfc07), SECP256K1_FE_CONST(0xb6307b33, 0x2e699f1c, 0xf77841d9, 0x0af25365, 0x404deb7f, 0xed5edb30, 0x90db49e6, 0x42a156b6), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x09ca7f47, 0x55b63b7b, 0x921a91c6, 0x1e4c18c0, 0xe8e177e1, 0x45739909, 0xeb1981a2, 0x68a20028), SECP256K1_FE_CONST(0x49cf84cc, 0xd19660e3, 0x0887be26, 0xf50dac9a, 0xbfb21480, 0x12a124cf, 0x6f24b618, 0xbd5ea579), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0x33, SECP256K1_FE_CONST(0x2dc90e64, 0x0cb646ae, 0x9164c0b5, 0xa9ef0169, 0xfebe34dc, 0x4437d6e4, 0x6acb0e27, 0xe219d1e8), SECP256K1_FE_CONST(0xd236f19b, 0xf349b951, 0x6e9b3f4a, 0x5610fe96, 0x0141cb23, 0xbbc8291b, 0x9534f1d7, 0x1de62a47), {SECP256K1_FE_CONST(0xe69df7d9, 0xc026c366, 0x00ebdf58, 0x80726758, 0x47c0c431, 0xc8eb7306, 0x82533e96, 0x4b6252c9), SECP256K1_FE_CONST(0x4f18bbdf, 0x7c2d6c5f, 0x818c1880, 0x2fa35cd0, 0x69eaa79f, 0xff74e4fc, 0x837c80d9, 0x3fece2f8), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x19620826, 0x3fd93c99, 0xff1420a7, 0x7f8d98a7, 0xb83f3bce, 0x37148cf9, 0x7dacc168, 0xb49da966), SECP256K1_FE_CONST(0xb0e74420, 0x83d293a0, 0x7e73e77f, 0xd05ca32f, 0x96155860, 0x008b1b03, 0x7c837f25, 0xc0131937), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0xcc, SECP256K1_FE_CONST(0x3edd7b39, 0x80e2f2f3, 0x4d1409a2, 0x07069f88, 0x1fda5f96, 0xf08027ac, 0x4465b63d, 0xc278d672), SECP256K1_FE_CONST(0x053a98de, 0x4a27b196, 0x1155822b, 0x3a3121f0, 0x3b2a1445, 0x8bd80eb4, 0xa560c4c7, 0xa85c149c), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0xb3dae4b7, 0xdcf858e4, 0xc6968057, 0xcef2b156, 0x46543152, 0x6538199c, 0xf52dc1b2, 0xd62fda30), SECP256K1_FE_CONST(0x4aa77dd5, 0x5d6b6d3c, 0xfa10cc9d, 0x0fe42f79, 0x232e4575, 0x661049ae, 0x36779c1d, 0x0c666d88), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x4c251b48, 0x2307a71b, 0x39697fa8, 0x310d4ea9, 0xb9abcead, 0x9ac7e663, 0x0ad23e4c, 0x29d021ff), SECP256K1_FE_CONST(0xb558822a, 0xa29492c3, 0x05ef3362, 0xf01bd086, 0xdcd1ba8a, 0x99efb651, 0xc98863e1, 0xf3998ea7)}}, + {0x00, SECP256K1_FE_CONST(0x4295737e, 0xfcb1da6f, 0xb1d96b9c, 0xa7dcd1e3, 0x20024b37, 0xa736c494, 0x8b625981, 0x73069f70), SECP256K1_FE_CONST(0xfa7ffe4f, 0x25f88362, 0x831c087a, 0xfe2e8a9b, 0x0713e2ca, 0xc1ddca6a, 0x383205a2, 0x66f14307), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0xff, SECP256K1_FE_CONST(0x587c1a0c, 0xee91939e, 0x7f784d23, 0xb963004a, 0x3bf44f5d, 0x4e32a008, 0x1995ba20, 0xb0fca59e), SECP256K1_FE_CONST(0x2ea98853, 0x0715e8d1, 0x0363907f, 0xf2512452, 0x4d471ba2, 0x454d5ce3, 0xbe3f0419, 0x4dfd3a3c), {SECP256K1_FE_CONST(0xcfd5a094, 0xaa0b9b88, 0x91b76c6a, 0xb9438f66, 0xaa1c095a, 0x65f9f701, 0x35e81712, 0x92245e74), SECP256K1_FE_CONST(0xa89057d7, 0xc6563f0d, 0x6efa19ae, 0x84412b8a, 0x7b47e791, 0xa191ecdf, 0xdf2af84f, 0xd97bc339), SECP256K1_FE_CONST(0x475d0ae9, 0xef46920d, 0xf07b3411, 0x7be5a081, 0x7de1023e, 0x3cc32689, 0xe9be145b, 0x406b0aef), SECP256K1_FE_CONST(0xa0759178, 0xad802324, 0x54f827ef, 0x05ea3e72, 0xad8d7541, 0x8e6d4cc1, 0xcd4f5306, 0xc5e7c453), SECP256K1_FE_CONST(0x302a5f6b, 0x55f46477, 0x6e489395, 0x46bc7099, 0x55e3f6a5, 0x9a0608fe, 0xca17e8ec, 0x6ddb9dbb), SECP256K1_FE_CONST(0x576fa828, 0x39a9c0f2, 0x9105e651, 0x7bbed475, 0x84b8186e, 0x5e6e1320, 0x20d507af, 0x268438f6), SECP256K1_FE_CONST(0xb8a2f516, 0x10b96df2, 0x0f84cbee, 0x841a5f7e, 0x821efdc1, 0xc33cd976, 0x1641eba3, 0xbf94f140), SECP256K1_FE_CONST(0x5f8a6e87, 0x527fdcdb, 0xab07d810, 0xfa15c18d, 0x52728abe, 0x7192b33e, 0x32b0acf8, 0x3a1837dc)}}, + {0xcc, SECP256K1_FE_CONST(0x5fa88b33, 0x65a635cb, 0xbcee003c, 0xce9ef51d, 0xd1a310de, 0x277e441a, 0xbccdb7be, 0x1e4ba249), SECP256K1_FE_CONST(0x79461ff6, 0x2bfcbcac, 0x4249ba84, 0xdd040f2c, 0xec3c63f7, 0x25204dc7, 0xf464c16b, 0xf0ff3170), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x6bb700e1, 0xf4d7e236, 0xe8d193ff, 0x4a76c1b3, 0xbcd4e2b2, 0x5acac3d5, 0x1c8dac65, 0x3fe909a0), SECP256K1_FE_CONST(0xf4c73410, 0x633da7f6, 0x3a4f1d55, 0xaec6dd32, 0xc4c6d89e, 0xe74075ed, 0xb5515ed9, 0x0da9e683), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x9448ff1e, 0x0b281dc9, 0x172e6c00, 0xb5893e4c, 0x432b1d4d, 0xa5353c2a, 0xe3725399, 0xc016f28f), SECP256K1_FE_CONST(0x0b38cbef, 0x9cc25809, 0xc5b0e2aa, 0x513922cd, 0x3b392761, 0x18bf8a12, 0x4aaea125, 0xf25615ac)}}, + {0xcc, SECP256K1_FE_CONST(0x6fb31c75, 0x31f03130, 0xb42b155b, 0x952779ef, 0xbb46087d, 0xd9807d24, 0x1a48eac6, 0x3c3d96d6), SECP256K1_FE_CONST(0x56f81be7, 0x53e8d4ae, 0x4940ea6f, 0x46f6ec9f, 0xda66a6f9, 0x6cc95f50, 0x6cb2b574, 0x90e94260), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x59059774, 0x795bdb7a, 0x837fbe11, 0x40a5fa59, 0x984f48af, 0x8df95d57, 0xdd6d1c05, 0x437dcec1), SECP256K1_FE_CONST(0x22a644db, 0x79376ad4, 0xe7b3a009, 0xe58b3f13, 0x137c54fd, 0xf911122c, 0xc93667c4, 0x7077d784), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0xa6fa688b, 0x86a42485, 0x7c8041ee, 0xbf5a05a6, 0x67b0b750, 0x7206a2a8, 0x2292e3f9, 0xbc822d6e), SECP256K1_FE_CONST(0xdd59bb24, 0x86c8952b, 0x184c5ff6, 0x1a74c0ec, 0xec83ab02, 0x06eeedd3, 0x36c9983a, 0x8f8824ab)}}, + {0x00, SECP256K1_FE_CONST(0x704cd226, 0xe71cb682, 0x6a590e80, 0xdac90f2d, 0x2f5830f0, 0xfdf135a3, 0xeae3965b, 0xff25ff12), SECP256K1_FE_CONST(0x138e0afa, 0x68936ee6, 0x70bd2b8d, 0xb53aedbb, 0x7bea2a85, 0x97388b24, 0xd0518edd, 0x22ad66ec), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0x33, SECP256K1_FE_CONST(0x725e9147, 0x92cb8c89, 0x49e7e116, 0x8b7cdd8a, 0x8094c91c, 0x6ec2202c, 0xcd53a6a1, 0x8771edeb), SECP256K1_FE_CONST(0x8da16eb8, 0x6d347376, 0xb6181ee9, 0x74832275, 0x7f6b36e3, 0x913ddfd3, 0x32ac595d, 0x788e0e44), {SECP256K1_FE_CONST(0xdd357786, 0xb9f68733, 0x30391aa5, 0x62580965, 0x4e43116e, 0x82a5a5d8, 0x2ffd1d66, 0x24101fc4), SECP256K1_FE_CONST(0xa0b7efca, 0x01814594, 0xc59c9aae, 0x8e497001, 0x86ca5d95, 0xe88bcc80, 0x399044d9, 0xc2d8613d), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x22ca8879, 0x460978cc, 0xcfc6e55a, 0x9da7f69a, 0xb1bcee91, 0x7d5a5a27, 0xd002e298, 0xdbefdc6b), SECP256K1_FE_CONST(0x5f481035, 0xfe7eba6b, 0x3a636551, 0x71b68ffe, 0x7935a26a, 0x1774337f, 0xc66fbb25, 0x3d279af2), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0x00, SECP256K1_FE_CONST(0x78fe6b71, 0x7f2ea4a3, 0x2708d79c, 0x151bf503, 0xa5312a18, 0xc0963437, 0xe865cc6e, 0xd3f6ae97), SECP256K1_FE_CONST(0x8701948e, 0x80d15b5c, 0xd8f72863, 0xeae40afc, 0x5aced5e7, 0x3f69cbc8, 0x179a3390, 0x2c094d98), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0x44, SECP256K1_FE_CONST(0x7c37bb9c, 0x5061dc07, 0x413f11ac, 0xd5a34006, 0xe64c5c45, 0x7fdb9a43, 0x8f217255, 0xa961f50d), SECP256K1_FE_CONST(0x5c1a76b4, 0x4568eb59, 0xd6789a74, 0x42d9ed7c, 0xdc6226b7, 0x752b4ff8, 0xeaf8e1a9, 0x5736e507), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0xb94d30cd, 0x7dbff60b, 0x64620c17, 0xca0fafaa, 0x40b3d1f5, 0x2d077a60, 0xa2e0cafd, 0x145086c2), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x46b2cf32, 0x824009f4, 0x9b9df3e8, 0x35f05055, 0xbf4c2e0a, 0xd2f8859f, 0x5d1f3501, 0xebaf756d), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0x00, SECP256K1_FE_CONST(0x82388888, 0x967f82a6, 0xb444438a, 0x7d44838e, 0x13c0d478, 0xb9ca060d, 0xa95a41fb, 0x94303de6), SECP256K1_FE_CONST(0x29e96541, 0x70628fec, 0x8b497289, 0x8b113cf9, 0x8807f460, 0x9274f4f3, 0x140d0674, 0x157c90a0), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0x33, SECP256K1_FE_CONST(0x91298f57, 0x70af7a27, 0xf0a47188, 0xd24c3b7b, 0xf98ab299, 0x0d84b0b8, 0x98507e3c, 0x561d6472), SECP256K1_FE_CONST(0x144f4ccb, 0xd9a74698, 0xa88cbf6f, 0xd00ad886, 0xd339d29e, 0xa19448f2, 0xc572cac0, 0xa07d5562), {SECP256K1_FE_CONST(0xe6a0ffa3, 0x807f09da, 0xdbe71e0f, 0x4be4725f, 0x2832e76c, 0xad8dc1d9, 0x43ce8393, 0x75eff248), SECP256K1_FE_CONST(0x837b8e68, 0xd4917544, 0x764ad090, 0x3cb11f86, 0x15d2823c, 0xefbb06d8, 0x9049dbab, 0xc69befda), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x195f005c, 0x7f80f625, 0x2418e1f0, 0xb41b8da0, 0xd7cd1893, 0x52723e26, 0xbc317c6b, 0x8a1009e7), SECP256K1_FE_CONST(0x7c847197, 0x2b6e8abb, 0x89b52f6f, 0xc34ee079, 0xea2d7dc3, 0x1044f927, 0x6fb62453, 0x39640c55), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0x00, SECP256K1_FE_CONST(0xb682f3d0, 0x3bbb5dee, 0x4f54b5eb, 0xfba931b4, 0xf52f6a19, 0x1e5c2f48, 0x3c73c66e, 0x9ace97e1), SECP256K1_FE_CONST(0x904717bf, 0x0bc0cb78, 0x73fcdc38, 0xaa97f19e, 0x3a626309, 0x72acff92, 0xb24cc6dd, 0xa197cb96), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0x77, SECP256K1_FE_CONST(0xc17ec69e, 0x665f0fb0, 0xdbab48d9, 0xc2f94d12, 0xec8a9d7e, 0xacb58084, 0x83309180, 0x1eb0b80b), SECP256K1_FE_CONST(0x147756e6, 0x6d96e31c, 0x426d3cc8, 0x5ed0c4cf, 0xbef6341d, 0xd8b28558, 0x5aa574ea, 0x0204b55e), {SECP256K1_FE_CONST(0x6f4aea43, 0x1a0043bd, 0xd03134d6, 0xd9159119, 0xce034b88, 0xc32e50e8, 0xe36c4ee4, 0x5eac7ae9), SECP256K1_FE_CONST(0xfd5be16d, 0x4ffa2690, 0x126c67c3, 0xef7cb9d2, 0x9b74d397, 0xc78b06b3, 0x605fda34, 0xdc9696a6), SECP256K1_FE_CONST(0x5e9c6079, 0x2a2f000e, 0x45c6250f, 0x296f875e, 0x174efc0e, 0x9703e628, 0x706103a9, 0xdd2d82c7), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x90b515bc, 0xe5ffbc42, 0x2fcecb29, 0x26ea6ee6, 0x31fcb477, 0x3cd1af17, 0x1c93b11a, 0xa1538146), SECP256K1_FE_CONST(0x02a41e92, 0xb005d96f, 0xed93983c, 0x1083462d, 0x648b2c68, 0x3874f94c, 0x9fa025ca, 0x23696589), SECP256K1_FE_CONST(0xa1639f86, 0xd5d0fff1, 0xba39daf0, 0xd69078a1, 0xe8b103f1, 0x68fc19d7, 0x8f9efc55, 0x22d27968), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0xcc, SECP256K1_FE_CONST(0xc25172fc, 0x3f29b6fc, 0x4a1155b8, 0x57523315, 0x5486b274, 0x64b74b8b, 0x260b499a, 0x3f53cb14), SECP256K1_FE_CONST(0x1ea9cbdb, 0x35cf6e03, 0x29aa31b0, 0xbb0a702a, 0x65123ed0, 0x08655a93, 0xb7dcd528, 0x0e52e1ab), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x7422edc7, 0x843136af, 0x0053bb88, 0x54448a82, 0x99994f9d, 0xdcefd3a9, 0xa92d4546, 0x2c59298a), SECP256K1_FE_CONST(0x78c7774a, 0x266f8b97, 0xea23d05d, 0x064f033c, 0x77319f92, 0x3f6b78bc, 0xe4e20bf0, 0x5fa5398d), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x8bdd1238, 0x7bcec950, 0xffac4477, 0xabbb757d, 0x6666b062, 0x23102c56, 0x56d2bab8, 0xd3a6d2a5), SECP256K1_FE_CONST(0x873888b5, 0xd9907468, 0x15dc2fa2, 0xf9b0fcc3, 0x88ce606d, 0xc0948743, 0x1b1df40e, 0xa05ac2a2)}}, + {0x00, SECP256K1_FE_CONST(0xcab6626f, 0x832a4b12, 0x80ba7add, 0x2fc5322f, 0xf011caed, 0xedf7ff4d, 0xb6735d50, 0x26dc0367), SECP256K1_FE_CONST(0x2b2bef08, 0x52c6f7c9, 0x5d72ac99, 0xa23802b8, 0x75029cd5, 0x73b248d1, 0xf1b3fc80, 0x33788eb6), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0x33, SECP256K1_FE_CONST(0xd8621b4f, 0xfc85b9ed, 0x56e99d8d, 0xd1dd24ae, 0xdcecb147, 0x63b861a1, 0x7112dc77, 0x1a104fd2), SECP256K1_FE_CONST(0x812cabe9, 0x72a22aa6, 0x7c7da0c9, 0x4d8a9362, 0x96eb9949, 0xd70c37cb, 0x2b248757, 0x4cb3ce58), {SECP256K1_FE_CONST(0xfbc5febc, 0x6fdbc9ae, 0x3eb88a93, 0xb982196e, 0x8b6275a6, 0xd5a73c17, 0x387e000c, 0x711bd0e3), SECP256K1_FE_CONST(0x8724c96b, 0xd4e5527f, 0x2dd195a5, 0x1c468d2d, 0x211ba2fa, 0xc7cbe0b4, 0xb3434253, 0x409fb42d), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x043a0143, 0x90243651, 0xc147756c, 0x467de691, 0x749d8a59, 0x2a58c3e8, 0xc781fff2, 0x8ee42b4c), SECP256K1_FE_CONST(0x78db3694, 0x2b1aad80, 0xd22e6a5a, 0xe3b972d2, 0xdee45d05, 0x38341f4b, 0x4cbcbdab, 0xbf604802), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0x00, SECP256K1_FE_CONST(0xda463164, 0xc6f4bf71, 0x29ee5f0e, 0xc00f65a6, 0x75a8adf1, 0xbd931b39, 0xb64806af, 0xdcda9a22), SECP256K1_FE_CONST(0x25b9ce9b, 0x390b408e, 0xd611a0f1, 0x3ff09a59, 0x8a57520e, 0x426ce4c6, 0x49b7f94f, 0x2325620d), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0xcc, SECP256K1_FE_CONST(0xdafc971e, 0x4a3a7b6d, 0xcfb42a08, 0xd9692d82, 0xad9e7838, 0x523fcbda, 0x1d4827e1, 0x4481ae2d), SECP256K1_FE_CONST(0x250368e1, 0xb5c58492, 0x304bd5f7, 0x2696d27d, 0x526187c7, 0xadc03425, 0xe2b7d81d, 0xbb7e4e02), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x370c28f1, 0xbe665efa, 0xcde6aa43, 0x6bf86fe2, 0x1e6e314c, 0x1e53dd04, 0x0e6c73a4, 0x6b4c8c49), SECP256K1_FE_CONST(0xcd8acee9, 0x8ffe5653, 0x1a84d7eb, 0x3e48fa40, 0x34206ce8, 0x25ace907, 0xd0edf0ea, 0xeb5e9ca2), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0xc8f3d70e, 0x4199a105, 0x321955bc, 0x9407901d, 0xe191ceb3, 0xe1ac22fb, 0xf1938c5a, 0x94b36fe6), SECP256K1_FE_CONST(0x32753116, 0x7001a9ac, 0xe57b2814, 0xc1b705bf, 0xcbdf9317, 0xda5316f8, 0x2f120f14, 0x14a15f8d)}}, + {0x44, SECP256K1_FE_CONST(0xe0294c8b, 0xc1a36b41, 0x66ee92bf, 0xa70a5c34, 0x976fa982, 0x9405efea, 0x8f9cd54d, 0xcb29b99e), SECP256K1_FE_CONST(0xae9690d1, 0x3b8d20a0, 0xfbbf37be, 0xd8474f67, 0xa04e142f, 0x56efd787, 0x70a76b35, 0x9165d8a1), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0xdcd45d93, 0x5613916a, 0xf167b029, 0x058ba3a7, 0x00d37150, 0xb9df3472, 0x8cb05412, 0xc16d4182), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x232ba26c, 0xa9ec6e95, 0x0e984fd6, 0xfa745c58, 0xff2c8eaf, 0x4620cb8d, 0x734fabec, 0x3e92baad), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0x00, SECP256K1_FE_CONST(0xe148441c, 0xd7b92b8b, 0x0e4fa3bd, 0x68712cfd, 0x0d709ad1, 0x98cace61, 0x1493c10e, 0x97f5394e), SECP256K1_FE_CONST(0x164a6397, 0x94d74c53, 0xafc4d329, 0x4e79cdb3, 0xcd25f99f, 0x6df45c00, 0x0f758aba, 0x54d699c0), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0xff, SECP256K1_FE_CONST(0xe4b00ec9, 0x7aadcca9, 0x7644d3b0, 0xc8a931b1, 0x4ce7bcf7, 0xbc877954, 0x6d6e35aa, 0x5937381c), SECP256K1_FE_CONST(0x94e9588d, 0x41647b3f, 0xcc772dc8, 0xd83c67ce, 0x3be00353, 0x8517c834, 0x103d2cd4, 0x9d62ef4d), {SECP256K1_FE_CONST(0xc88d25f4, 0x1407376b, 0xb2c03a7f, 0xffeb3ec7, 0x811cc434, 0x91a0c3aa, 0xc0378cdc, 0x78357bee), SECP256K1_FE_CONST(0x51c02636, 0xce00c234, 0x5ecd89ad, 0xb6089fe4, 0xd5e18ac9, 0x24e3145e, 0x6669501c, 0xd37a00d4), SECP256K1_FE_CONST(0x205b3512, 0xdb40521c, 0xb200952e, 0x67b46f67, 0xe09e7839, 0xe0de4400, 0x4138329e, 0xbd9138c5), SECP256K1_FE_CONST(0x58aab390, 0xab6fb55c, 0x1d1b8089, 0x7a207ce9, 0x4a78fa5b, 0x4aa61a33, 0x398bcae9, 0xadb20d3e), SECP256K1_FE_CONST(0x3772da0b, 0xebf8c894, 0x4d3fc580, 0x0014c138, 0x7ee33bcb, 0x6e5f3c55, 0x3fc87322, 0x87ca8041), SECP256K1_FE_CONST(0xae3fd9c9, 0x31ff3dcb, 0xa1327652, 0x49f7601b, 0x2a1e7536, 0xdb1ceba1, 0x9996afe2, 0x2c85fb5b), SECP256K1_FE_CONST(0xdfa4caed, 0x24bfade3, 0x4dff6ad1, 0x984b9098, 0x1f6187c6, 0x1f21bbff, 0xbec7cd60, 0x426ec36a), SECP256K1_FE_CONST(0xa7554c6f, 0x54904aa3, 0xe2e47f76, 0x85df8316, 0xb58705a4, 0xb559e5cc, 0xc6743515, 0x524deef1)}}, + {0x00, SECP256K1_FE_CONST(0xe5bbb9ef, 0x360d0a50, 0x1618f006, 0x7d36dceb, 0x75f5be9a, 0x620232aa, 0x9fd5139d, 0x0863fde5), SECP256K1_FE_CONST(0xe5bbb9ef, 0x360d0a50, 0x1618f006, 0x7d36dceb, 0x75f5be9a, 0x620232aa, 0x9fd5139d, 0x0863fde5), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0xff, SECP256K1_FE_CONST(0xe6bcb5c3, 0xd63467d4, 0x90bfa54f, 0xbbc6092a, 0x7248c25e, 0x11b248dc, 0x2964a6e1, 0x5edb1457), SECP256K1_FE_CONST(0x19434a3c, 0x29cb982b, 0x6f405ab0, 0x4439f6d5, 0x8db73da1, 0xee4db723, 0xd69b591d, 0xa124e7d8), {SECP256K1_FE_CONST(0x67119877, 0x832ab8f4, 0x59a82165, 0x6d8261f5, 0x44a553b8, 0x9ae4f25c, 0x52a97134, 0xb70f3426), SECP256K1_FE_CONST(0xffee02f5, 0xe649c07f, 0x0560eff1, 0x867ec7b3, 0x2d0e595e, 0x9b1c0ea6, 0xe2a4fc70, 0xc97cd71f), SECP256K1_FE_CONST(0xb5e0c189, 0xeb5b4bac, 0xd025b744, 0x4d74178b, 0xe8d5246c, 0xfa4a9a20, 0x7964a057, 0xee969992), SECP256K1_FE_CONST(0x5746e459, 0x1bf7f4c3, 0x044609ea, 0x372e9086, 0x03975d27, 0x9fdef834, 0x9f0b08d3, 0x2f07619d), SECP256K1_FE_CONST(0x98ee6788, 0x7cd5470b, 0xa657de9a, 0x927d9e0a, 0xbb5aac47, 0x651b0da3, 0xad568eca, 0x48f0c809), SECP256K1_FE_CONST(0x0011fd0a, 0x19b63f80, 0xfa9f100e, 0x7981384c, 0xd2f1a6a1, 0x64e3f159, 0x1d5b038e, 0x36832510), SECP256K1_FE_CONST(0x4a1f3e76, 0x14a4b453, 0x2fda48bb, 0xb28be874, 0x172adb93, 0x05b565df, 0x869b5fa7, 0x1169629d), SECP256K1_FE_CONST(0xa8b91ba6, 0xe4080b3c, 0xfbb9f615, 0xc8d16f79, 0xfc68a2d8, 0x602107cb, 0x60f4f72b, 0xd0f89a92)}}, + {0x33, SECP256K1_FE_CONST(0xf28fba64, 0xaf766845, 0xeb2f4302, 0x456e2b9f, 0x8d80affe, 0x57e7aae4, 0x2738d7cd, 0xdb1c2ce6), SECP256K1_FE_CONST(0xf28fba64, 0xaf766845, 0xeb2f4302, 0x456e2b9f, 0x8d80affe, 0x57e7aae4, 0x2738d7cd, 0xdb1c2ce6), {SECP256K1_FE_CONST(0x4f867ad8, 0xbb3d8404, 0x09d26b67, 0x307e6210, 0x0153273f, 0x72fa4b74, 0x84becfa1, 0x4ebe7408), SECP256K1_FE_CONST(0x5bbc4f59, 0xe452cc5f, 0x22a99144, 0xb10ce898, 0x9a89a995, 0xec3cea1c, 0x91ae10e8, 0xf721bb5d), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0xb0798527, 0x44c27bfb, 0xf62d9498, 0xcf819def, 0xfeacd8c0, 0x8d05b48b, 0x7b41305d, 0xb1418827), SECP256K1_FE_CONST(0xa443b0a6, 0x1bad33a0, 0xdd566ebb, 0x4ef31767, 0x6576566a, 0x13c315e3, 0x6e51ef16, 0x08de40d2), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, + {0xcc, SECP256K1_FE_CONST(0xf455605b, 0xc85bf48e, 0x3a908c31, 0x023faf98, 0x381504c6, 0xc6d3aeb9, 0xede55f8d, 0xd528924d), SECP256K1_FE_CONST(0xd31fbcd5, 0xcdb798f6, 0xc00db669, 0x2f8fe896, 0x7fa9c79d, 0xd10958f4, 0xa194f013, 0x74905e99), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0x0c00c571, 0x5b56fe63, 0x2d814ad8, 0xa77f8e66, 0x628ea47a, 0x6116834f, 0x8c1218f3, 0xa03cbd50), SECP256K1_FE_CONST(0xdf88e44f, 0xac84fa52, 0xdf4d59f4, 0x8819f18f, 0x6a8cd415, 0x1d162afa, 0xf773166f, 0x57c7ff46), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0xf3ff3a8e, 0xa4a9019c, 0xd27eb527, 0x58807199, 0x9d715b85, 0x9ee97cb0, 0x73ede70b, 0x5fc33edf), SECP256K1_FE_CONST(0x20771bb0, 0x537b05ad, 0x20b2a60b, 0x77e60e70, 0x95732bea, 0xe2e9d505, 0x088ce98f, 0xa837fce9)}}, + {0xff, SECP256K1_FE_CONST(0xf58cd4d9, 0x830bad32, 0x2699035e, 0x8246007d, 0x4be27e19, 0xb6f53621, 0x317b4f30, 0x9b3daa9d), SECP256K1_FE_CONST(0x78ec2b3d, 0xc0948de5, 0x60148bbc, 0x7c6dc963, 0x3ad5df70, 0xa5a5750c, 0xbed72180, 0x4f082a3b), {SECP256K1_FE_CONST(0x6c4c580b, 0x76c75940, 0x43569f9d, 0xae16dc28, 0x01c16a1f, 0xbe128608, 0x81b75f8e, 0xf929bce5), SECP256K1_FE_CONST(0x94231355, 0xe7385c5f, 0x25ca436a, 0xa6419147, 0x1aea4393, 0xd6e86ab7, 0xa35fe2af, 0xacaefd0d), SECP256K1_FE_CONST(0xdff2a195, 0x1ada6db5, 0x74df8340, 0x48149da3, 0x397a75b8, 0x29abf58c, 0x7e69db1b, 0x41ac0989), SECP256K1_FE_CONST(0xa52b66d3, 0xc9070355, 0x48028bf8, 0x04711bf4, 0x22aba95f, 0x1a666fc8, 0x6f4648e0, 0x5f29caae), SECP256K1_FE_CONST(0x93b3a7f4, 0x8938a6bf, 0xbca96062, 0x51e923d7, 0xfe3e95e0, 0x41ed79f7, 0x7e48a070, 0x06d63f4a), SECP256K1_FE_CONST(0x6bdcecaa, 0x18c7a3a0, 0xda35bc95, 0x59be6eb8, 0xe515bc6c, 0x29179548, 0x5ca01d4f, 0x5350ff22), SECP256K1_FE_CONST(0x200d5e6a, 0xe525924a, 0x8b207cbf, 0xb7eb625c, 0xc6858a47, 0xd6540a73, 0x819624e3, 0xbe53f2a6), SECP256K1_FE_CONST(0x5ad4992c, 0x36f8fcaa, 0xb7fd7407, 0xfb8ee40b, 0xdd5456a0, 0xe5999037, 0x90b9b71e, 0xa0d63181)}}, + {0x00, SECP256K1_FE_CONST(0xfd7d912a, 0x40f182a3, 0x588800d6, 0x9ebfb504, 0x8766da20, 0x6fd7ebc8, 0xd2436c81, 0xcbef6421), SECP256K1_FE_CONST(0x8d37c862, 0x054debe7, 0x31694536, 0xff46b273, 0xec122b35, 0xa9bf1445, 0xac3c4ff9, 0xf262c952), {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0)}}, +}; + +/* Set of (encoding, xcoord) test vectors, selected to maximize branch coverage, part of the BIP324 + * test vectors. Created using an independent implementation, and tested decoding against the paper + * authors' code. */ +static const struct ellswift_decode_test ellswift_decode_tests[] = { + {{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, SECP256K1_FE_CONST(0xedd1fd3e, 0x327ce90c, 0xc7a35426, 0x14289aee, 0x9682003e, 0x9cf7dcc9, 0xcf2ca974, 0x3be5aa0c), 0}, + {{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xd3, 0x47, 0x5b, 0xf7, 0x65, 0x5b, 0x0f, 0xb2, 0xd8, 0x52, 0x92, 0x10, 0x35, 0xb2, 0xef, 0x60, 0x7f, 0x49, 0x06, 0x9b, 0x97, 0x45, 0x4e, 0x67, 0x95, 0x25, 0x10, 0x62, 0x74, 0x17, 0x71}, SECP256K1_FE_CONST(0xb5da00b7, 0x3cd65605, 0x20e7c364, 0x086e7cd2, 0x3a34bf60, 0xd0e707be, 0x9fc34d4c, 0xd5fdfa2c), 1}, + {{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x82, 0x27, 0x7c, 0x4a, 0x71, 0xf9, 0xd2, 0x2e, 0x66, 0xec, 0xe5, 0x23, 0xf8, 0xfa, 0x08, 0x74, 0x1a, 0x7c, 0x09, 0x12, 0xc6, 0x6a, 0x69, 0xce, 0x68, 0x51, 0x4b, 0xfd, 0x35, 0x15, 0xb4, 0x9f}, SECP256K1_FE_CONST(0xf482f2e2, 0x41753ad0, 0xfb89150d, 0x8491dc1e, 0x34ff0b8a, 0xcfbb442c, 0xfe999e2e, 0x5e6fd1d2), 1}, + {{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x84, 0x21, 0xcc, 0x93, 0x0e, 0x77, 0xc9, 0xf5, 0x14, 0xb6, 0x91, 0x5c, 0x3d, 0xbe, 0x2a, 0x94, 0xc6, 0xd8, 0xf6, 0x90, 0xb5, 0xb7, 0x39, 0x86, 0x4b, 0xa6, 0x78, 0x9f, 0xb8, 0xa5, 0x5d, 0xd0}, SECP256K1_FE_CONST(0x9f59c402, 0x75f5085a, 0x006f05da, 0xe77eb98c, 0x6fd0db1a, 0xb4a72ac4, 0x7eae90a4, 0xfc9e57e0), 0}, + {{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbd, 0xe7, 0x0d, 0xf5, 0x19, 0x39, 0xb9, 0x4c, 0x9c, 0x24, 0x97, 0x9f, 0xa7, 0xdd, 0x04, 0xeb, 0xd9, 0xb3, 0x57, 0x2d, 0xa7, 0x80, 0x22, 0x90, 0x43, 0x8a, 0xf2, 0xa6, 0x81, 0x89, 0x54, 0x41}, SECP256K1_FE_CONST(0xaaaaaaaa, 0xaaaaaaaa, 0xaaaaaaaa, 0xaaaaaaaa, 0xaaaaaaaa, 0xaaaaaaaa, 0xaaaaaaa9, 0xfffffd6b), 1}, + {{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd1, 0x9c, 0x18, 0x2d, 0x27, 0x59, 0xcd, 0x99, 0x82, 0x42, 0x28, 0xd9, 0x47, 0x99, 0xf8, 0xc6, 0x55, 0x7c, 0x38, 0xa1, 0xc0, 0xd6, 0x77, 0x9b, 0x9d, 0x4b, 0x72, 0x9c, 0x6f, 0x1c, 0xcc, 0x42}, SECP256K1_FE_CONST(0x70720db7, 0xe238d041, 0x21f5b1af, 0xd8cc5ad9, 0xd18944c6, 0xbdc94881, 0xf502b7a3, 0xaf3aecff), 0}, + {{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f}, SECP256K1_FE_CONST(0xedd1fd3e, 0x327ce90c, 0xc7a35426, 0x14289aee, 0x9682003e, 0x9cf7dcc9, 0xcf2ca974, 0x3be5aa0c), 0}, + {{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0x64, 0xbb, 0xd5}, SECP256K1_FE_CONST(0x50873db3, 0x1badcc71, 0x890e4f67, 0x753a6575, 0x7f97aaa7, 0xdd5f1e82, 0xb753ace3, 0x2219064b), 0}, + {{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x70, 0x28, 0xde, 0x7d}, SECP256K1_FE_CONST(0x1eea9cc5, 0x9cfcf2fa, 0x151ac6c2, 0x74eea411, 0x0feb4f7b, 0x68c59657, 0x32e9992e, 0x976ef68e), 0}, + {{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcb, 0xcf, 0xb7, 0xe7}, SECP256K1_FE_CONST(0x12303941, 0xaedc2088, 0x80735b1f, 0x1795c8e5, 0x5be520ea, 0x93e10335, 0x7b5d2adb, 0x7ed59b8e), 0}, + {{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf3, 0x11, 0x3a, 0xd9}, SECP256K1_FE_CONST(0x7eed6b70, 0xe7b0767c, 0x7d7feac0, 0x4e57aa2a, 0x12fef5e0, 0xf48f878f, 0xcbb88b3b, 0x6b5e0783), 0}, + {{0x0a, 0x2d, 0x2b, 0xa9, 0x35, 0x07, 0xf1, 0xdf, 0x23, 0x37, 0x70, 0xc2, 0xa7, 0x97, 0x96, 0x2c, 0xc6, 0x1f, 0x6d, 0x15, 0xda, 0x14, 0xec, 0xd4, 0x7d, 0x8d, 0x27, 0xae, 0x1c, 0xd5, 0xf8, 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, SECP256K1_FE_CONST(0x532167c1, 0x1200b08c, 0x0e84a354, 0xe74dcc40, 0xf8b25f4f, 0xe686e308, 0x69526366, 0x278a0688), 0}, + {{0x0a, 0x2d, 0x2b, 0xa9, 0x35, 0x07, 0xf1, 0xdf, 0x23, 0x37, 0x70, 0xc2, 0xa7, 0x97, 0x96, 0x2c, 0xc6, 0x1f, 0x6d, 0x15, 0xda, 0x14, 0xec, 0xd4, 0x7d, 0x8d, 0x27, 0xae, 0x1c, 0xd5, 0xf8, 0x53, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f}, SECP256K1_FE_CONST(0x532167c1, 0x1200b08c, 0x0e84a354, 0xe74dcc40, 0xf8b25f4f, 0xe686e308, 0x69526366, 0x278a0688), 0}, + {{0x0f, 0xfd, 0xe9, 0xca, 0x81, 0xd7, 0x51, 0xe9, 0xcd, 0xaf, 0xfc, 0x1a, 0x50, 0x77, 0x92, 0x45, 0x32, 0x0b, 0x28, 0x99, 0x6d, 0xba, 0xf3, 0x2f, 0x82, 0x2f, 0x20, 0x11, 0x7c, 0x22, 0xfb, 0xd6, 0xc7, 0x4d, 0x99, 0xef, 0xce, 0xaa, 0x55, 0x0f, 0x1a, 0xd1, 0xc0, 0xf4, 0x3f, 0x46, 0xe7, 0xff, 0x1e, 0xe3, 0xbd, 0x01, 0x62, 0xb7, 0xbf, 0x55, 0xf2, 0x96, 0x5d, 0xa9, 0xc3, 0x45, 0x06, 0x46}, SECP256K1_FE_CONST(0x74e880b3, 0xffd18fe3, 0xcddf7902, 0x522551dd, 0xf97fa4a3, 0x5a3cfda8, 0x197f9470, 0x81a57b8f), 0}, + {{0x0f, 0xfd, 0xe9, 0xca, 0x81, 0xd7, 0x51, 0xe9, 0xcd, 0xaf, 0xfc, 0x1a, 0x50, 0x77, 0x92, 0x45, 0x32, 0x0b, 0x28, 0x99, 0x6d, 0xba, 0xf3, 0x2f, 0x82, 0x2f, 0x20, 0x11, 0x7c, 0x22, 0xfb, 0xd6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x15, 0x6c, 0xa8, 0x96}, SECP256K1_FE_CONST(0x377b643f, 0xce2271f6, 0x4e5c8101, 0x566107c1, 0xbe498074, 0x50917838, 0x04f65478, 0x1ac9217c), 1}, + {{0x12, 0x36, 0x58, 0x44, 0x4f, 0x32, 0xbe, 0x8f, 0x02, 0xea, 0x20, 0x34, 0xaf, 0xa7, 0xef, 0x4b, 0xbe, 0x8a, 0xdc, 0x91, 0x8c, 0xeb, 0x49, 0xb1, 0x27, 0x73, 0xb6, 0x25, 0xf4, 0x90, 0xb3, 0x68, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x8d, 0xc5, 0xfe, 0x11}, SECP256K1_FE_CONST(0xed16d65c, 0xf3a9538f, 0xcb2c139f, 0x1ecbc143, 0xee148271, 0x20cbc265, 0x9e667256, 0x800b8142), 0}, + {{0x14, 0x6f, 0x92, 0x46, 0x4d, 0x15, 0xd3, 0x6e, 0x35, 0x38, 0x2b, 0xd3, 0xca, 0x5b, 0x0f, 0x97, 0x6c, 0x95, 0xcb, 0x08, 0xac, 0xdc, 0xf2, 0xd5, 0xb3, 0x57, 0x06, 0x17, 0x99, 0x08, 0x39, 0xd7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x31, 0x45, 0xe9, 0x3b}, SECP256K1_FE_CONST(0x0d5cd840, 0x427f941f, 0x65193079, 0xab8e2e83, 0x024ef2ee, 0x7ca558d8, 0x8879ffd8, 0x79fb6657), 0}, + {{0x15, 0xfd, 0xf5, 0xcf, 0x09, 0xc9, 0x07, 0x59, 0xad, 0xd2, 0x27, 0x2d, 0x57, 0x4d, 0x2b, 0xb5, 0xfe, 0x14, 0x29, 0xf9, 0xf3, 0xc1, 0x4c, 0x65, 0xe3, 0x19, 0x4b, 0xf6, 0x1b, 0x82, 0xaa, 0x73, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x04, 0xcf, 0xd9, 0x06}, SECP256K1_FE_CONST(0x16d0e439, 0x46aec93f, 0x62d57eb8, 0xcde68951, 0xaf136cf4, 0xb307938d, 0xd1447411, 0xe07bffe1), 1}, + {{0x1f, 0x67, 0xed, 0xf7, 0x79, 0xa8, 0xa6, 0x49, 0xd6, 0xde, 0xf6, 0x00, 0x35, 0xf2, 0xfa, 0x22, 0xd0, 0x22, 0xdd, 0x35, 0x90, 0x79, 0xa1, 0xa1, 0x44, 0x07, 0x3d, 0x84, 0xf1, 0x9b, 0x92, 0xd5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, SECP256K1_FE_CONST(0x025661f9, 0xaba9d15c, 0x3118456b, 0xbe980e3e, 0x1b8ba2e0, 0x47c737a4, 0xeb48a040, 0xbb566f6c), 0}, + {{0x1f, 0x67, 0xed, 0xf7, 0x79, 0xa8, 0xa6, 0x49, 0xd6, 0xde, 0xf6, 0x00, 0x35, 0xf2, 0xfa, 0x22, 0xd0, 0x22, 0xdd, 0x35, 0x90, 0x79, 0xa1, 0xa1, 0x44, 0x07, 0x3d, 0x84, 0xf1, 0x9b, 0x92, 0xd5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f}, SECP256K1_FE_CONST(0x025661f9, 0xaba9d15c, 0x3118456b, 0xbe980e3e, 0x1b8ba2e0, 0x47c737a4, 0xeb48a040, 0xbb566f6c), 0}, + {{0x1f, 0xe1, 0xe5, 0xef, 0x3f, 0xce, 0xb5, 0xc1, 0x35, 0xab, 0x77, 0x41, 0x33, 0x3c, 0xe5, 0xa6, 0xe8, 0x0d, 0x68, 0x16, 0x76, 0x53, 0xf6, 0xb2, 0xb2, 0x4b, 0xcb, 0xcf, 0xaa, 0xaf, 0xf5, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f}, SECP256K1_FE_CONST(0x98bec3b2, 0xa351fa96, 0xcfd191c1, 0x77835193, 0x1b9e9ba9, 0xad1149f6, 0xd9eadca8, 0x0981b801), 0}, + {{0x40, 0x56, 0xa3, 0x4a, 0x21, 0x0e, 0xec, 0x78, 0x92, 0xe8, 0x82, 0x06, 0x75, 0xc8, 0x60, 0x09, 0x9f, 0x85, 0x7b, 0x26, 0xaa, 0xd8, 0x54, 0x70, 0xee, 0x6d, 0x3c, 0xf1, 0x30, 0x4a, 0x9d, 0xcf, 0x37, 0x5e, 0x70, 0x37, 0x42, 0x71, 0xf2, 0x0b, 0x13, 0xc9, 0x98, 0x6e, 0xd7, 0xd3, 0xc1, 0x77, 0x99, 0x69, 0x8c, 0xfc, 0x43, 0x5d, 0xbe, 0xd3, 0xa9, 0xf3, 0x4b, 0x38, 0xc8, 0x23, 0xc2, 0xb4}, SECP256K1_FE_CONST(0x868aac20, 0x03b29dbc, 0xad1a3e80, 0x3855e078, 0xa89d1654, 0x3ac64392, 0xd1224172, 0x98cec76e), 0}, + {{0x41, 0x97, 0xec, 0x37, 0x23, 0xc6, 0x54, 0xcf, 0xdd, 0x32, 0xab, 0x07, 0x55, 0x06, 0x64, 0x8b, 0x2f, 0xf5, 0x07, 0x03, 0x62, 0xd0, 0x1a, 0x4f, 0xff, 0x14, 0xb3, 0x36, 0xb7, 0x8f, 0x96, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xb3, 0xab, 0x1e, 0x95}, SECP256K1_FE_CONST(0xba5a6314, 0x502a8952, 0xb8f456e0, 0x85928105, 0xf665377a, 0x8ce27726, 0xa5b0eb7e, 0xc1ac0286), 0}, + {{0x47, 0xeb, 0x3e, 0x20, 0x8f, 0xed, 0xcd, 0xf8, 0x23, 0x4c, 0x94, 0x21, 0xe9, 0xcd, 0x9a, 0x7a, 0xe8, 0x73, 0xbf, 0xbd, 0xbc, 0x39, 0x37, 0x23, 0xd1, 0xba, 0x1e, 0x1e, 0x6a, 0x8e, 0x6b, 0x24, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7c, 0xd1, 0x2c, 0xb1}, SECP256K1_FE_CONST(0xd192d520, 0x07e541c9, 0x807006ed, 0x0468df77, 0xfd214af0, 0xa795fe11, 0x9359666f, 0xdcf08f7c), 0}, + {{0x5e, 0xb9, 0x69, 0x6a, 0x23, 0x36, 0xfe, 0x2c, 0x3c, 0x66, 0x6b, 0x02, 0xc7, 0x55, 0xdb, 0x4c, 0x0c, 0xfd, 0x62, 0x82, 0x5c, 0x7b, 0x58, 0x9a, 0x7b, 0x7b, 0xb4, 0x42, 0xe1, 0x41, 0xc1, 0xd6, 0x93, 0x41, 0x3f, 0x00, 0x52, 0xd4, 0x9e, 0x64, 0xab, 0xec, 0x6d, 0x58, 0x31, 0xd6, 0x6c, 0x43, 0x61, 0x28, 0x30, 0xa1, 0x7d, 0xf1, 0xfe, 0x43, 0x83, 0xdb, 0x89, 0x64, 0x68, 0x10, 0x02, 0x21}, SECP256K1_FE_CONST(0xef6e1da6, 0xd6c7627e, 0x80f7a723, 0x4cb08a02, 0x2c1ee1cf, 0x29e4d0f9, 0x642ae924, 0xcef9eb38), 1}, + {{0x7b, 0xf9, 0x6b, 0x7b, 0x6d, 0xa1, 0x5d, 0x34, 0x76, 0xa2, 0xb1, 0x95, 0x93, 0x4b, 0x69, 0x0a, 0x3a, 0x3d, 0xe3, 0xe8, 0xab, 0x84, 0x74, 0x85, 0x68, 0x63, 0xb0, 0xde, 0x3a, 0xf9, 0x0b, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, SECP256K1_FE_CONST(0x50851dfc, 0x9f418c31, 0x4a437295, 0xb24feeea, 0x27af3d0c, 0xd2308348, 0xfda6e21c, 0x463e46ff), 0}, + {{0x7b, 0xf9, 0x6b, 0x7b, 0x6d, 0xa1, 0x5d, 0x34, 0x76, 0xa2, 0xb1, 0x95, 0x93, 0x4b, 0x69, 0x0a, 0x3a, 0x3d, 0xe3, 0xe8, 0xab, 0x84, 0x74, 0x85, 0x68, 0x63, 0xb0, 0xde, 0x3a, 0xf9, 0x0b, 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f}, SECP256K1_FE_CONST(0x50851dfc, 0x9f418c31, 0x4a437295, 0xb24feeea, 0x27af3d0c, 0xd2308348, 0xfda6e21c, 0x463e46ff), 0}, + {{0x85, 0x1b, 0x1c, 0xa9, 0x45, 0x49, 0x37, 0x1c, 0x4f, 0x1f, 0x71, 0x87, 0x32, 0x1d, 0x39, 0xbf, 0x51, 0xc6, 0xb7, 0xfb, 0x61, 0xf7, 0xcb, 0xf0, 0x27, 0xc9, 0xda, 0x62, 0x02, 0x1b, 0x7a, 0x65, 0xfc, 0x54, 0xc9, 0x68, 0x37, 0xfb, 0x22, 0xb3, 0x62, 0xed, 0xa6, 0x3e, 0xc5, 0x2e, 0xc8, 0x3d, 0x81, 0xbe, 0xdd, 0x16, 0x0c, 0x11, 0xb2, 0x2d, 0x96, 0x5d, 0x9f, 0x4a, 0x6d, 0x64, 0xd2, 0x51}, SECP256K1_FE_CONST(0x3e731051, 0xe12d3323, 0x7eb324f2, 0xaa5b16bb, 0x868eb49a, 0x1aa1fadc, 0x19b6e876, 0x1b5a5f7b), 1}, + {{0x94, 0x3c, 0x2f, 0x77, 0x51, 0x08, 0xb7, 0x37, 0xfe, 0x65, 0xa9, 0x53, 0x1e, 0x19, 0xf2, 0xfc, 0x2a, 0x19, 0x7f, 0x56, 0x03, 0xe3, 0xa2, 0x88, 0x1d, 0x1d, 0x83, 0xe4, 0x00, 0x8f, 0x91, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, SECP256K1_FE_CONST(0x311c61f0, 0xab2f32b7, 0xb1f0223f, 0xa72f0a78, 0x752b8146, 0xe46107f8, 0x876dd9c4, 0xf92b2942), 0}, + {{0x94, 0x3c, 0x2f, 0x77, 0x51, 0x08, 0xb7, 0x37, 0xfe, 0x65, 0xa9, 0x53, 0x1e, 0x19, 0xf2, 0xfc, 0x2a, 0x19, 0x7f, 0x56, 0x03, 0xe3, 0xa2, 0x88, 0x1d, 0x1d, 0x83, 0xe4, 0x00, 0x8f, 0x91, 0x25, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f}, SECP256K1_FE_CONST(0x311c61f0, 0xab2f32b7, 0xb1f0223f, 0xa72f0a78, 0x752b8146, 0xe46107f8, 0x876dd9c4, 0xf92b2942), 0}, + {{0xa0, 0xf1, 0x84, 0x92, 0x18, 0x3e, 0x61, 0xe8, 0x06, 0x3e, 0x57, 0x36, 0x06, 0x59, 0x14, 0x21, 0xb0, 0x6b, 0xc3, 0x51, 0x36, 0x31, 0x57, 0x8a, 0x73, 0xa3, 0x9c, 0x1c, 0x33, 0x06, 0x23, 0x9f, 0x2f, 0x32, 0x90, 0x4f, 0x0d, 0x2a, 0x33, 0xec, 0xca, 0x8a, 0x54, 0x51, 0x70, 0x5b, 0xb5, 0x37, 0xd3, 0xbf, 0x44, 0xe0, 0x71, 0x22, 0x60, 0x25, 0xcd, 0xbf, 0xd2, 0x49, 0xfe, 0x0f, 0x7a, 0xd6}, SECP256K1_FE_CONST(0x97a09cf1, 0xa2eae7c4, 0x94df3c6f, 0x8a9445bf, 0xb8c09d60, 0x832f9b0b, 0x9d5eabe2, 0x5fbd14b9), 0}, + {{0xa1, 0xed, 0x0a, 0x0b, 0xd7, 0x9d, 0x8a, 0x23, 0xcf, 0xe4, 0xec, 0x5f, 0xef, 0x5b, 0xa5, 0xcc, 0xcf, 0xd8, 0x44, 0xe4, 0xff, 0x5c, 0xb4, 0xb0, 0xf2, 0xe7, 0x16, 0x27, 0x34, 0x1f, 0x1c, 0x5b, 0x17, 0xc4, 0x99, 0x24, 0x9e, 0x0a, 0xc0, 0x8d, 0x5d, 0x11, 0xea, 0x1c, 0x2c, 0x8c, 0xa7, 0x00, 0x16, 0x16, 0x55, 0x9a, 0x79, 0x94, 0xea, 0xde, 0xc9, 0xca, 0x10, 0xfb, 0x4b, 0x85, 0x16, 0xdc}, SECP256K1_FE_CONST(0x65a89640, 0x744192cd, 0xac64b2d2, 0x1ddf989c, 0xdac75007, 0x25b645be, 0xf8e2200a, 0xe39691f2), 0}, + {{0xba, 0x94, 0x59, 0x4a, 0x43, 0x27, 0x21, 0xaa, 0x35, 0x80, 0xb8, 0x4c, 0x16, 0x1d, 0x0d, 0x13, 0x4b, 0xc3, 0x54, 0xb6, 0x90, 0x40, 0x4d, 0x7c, 0xd4, 0xec, 0x57, 0xc1, 0x6d, 0x3f, 0xbe, 0x98, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xea, 0x50, 0x7d, 0xd7}, SECP256K1_FE_CONST(0x5e0d7656, 0x4aae92cb, 0x347e01a6, 0x2afd389a, 0x9aa401c7, 0x6c8dd227, 0x543dc9cd, 0x0efe685a), 0}, + {{0xbc, 0xaf, 0x72, 0x19, 0xf2, 0xf6, 0xfb, 0xf5, 0x5f, 0xe5, 0xe0, 0x62, 0xdc, 0xe0, 0xe4, 0x8c, 0x18, 0xf6, 0x81, 0x03, 0xf1, 0x0b, 0x81, 0x98, 0xe9, 0x74, 0xc1, 0x84, 0x75, 0x0e, 0x1b, 0xe3, 0x93, 0x20, 0x16, 0xcb, 0xf6, 0x9c, 0x44, 0x71, 0xbd, 0x1f, 0x65, 0x6c, 0x6a, 0x10, 0x7f, 0x19, 0x73, 0xde, 0x4a, 0xf7, 0x08, 0x6d, 0xb8, 0x97, 0x27, 0x70, 0x60, 0xe2, 0x56, 0x77, 0xf1, 0x9a}, SECP256K1_FE_CONST(0x2d97f96c, 0xac882dfe, 0x73dc44db, 0x6ce0f1d3, 0x1d624135, 0x8dd5d74e, 0xb3d3b500, 0x03d24c2b), 0}, + {{0xbc, 0xaf, 0x72, 0x19, 0xf2, 0xf6, 0xfb, 0xf5, 0x5f, 0xe5, 0xe0, 0x62, 0xdc, 0xe0, 0xe4, 0x8c, 0x18, 0xf6, 0x81, 0x03, 0xf1, 0x0b, 0x81, 0x98, 0xe9, 0x74, 0xc1, 0x84, 0x75, 0x0e, 0x1b, 0xe3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x65, 0x07, 0xd0, 0x9a}, SECP256K1_FE_CONST(0xe7008afe, 0x6e8cbd50, 0x55df120b, 0xd748757c, 0x686dadb4, 0x1cce75e4, 0xaddcc5e0, 0x2ec02b44), 1}, + {{0xc5, 0x98, 0x1b, 0xae, 0x27, 0xfd, 0x84, 0x40, 0x1c, 0x72, 0xa1, 0x55, 0xe5, 0x70, 0x7f, 0xbb, 0x81, 0x1b, 0x2b, 0x62, 0x06, 0x45, 0xd1, 0x02, 0x8e, 0xa2, 0x70, 0xcb, 0xe0, 0xee, 0x22, 0x5d, 0x4b, 0x62, 0xaa, 0x4d, 0xca, 0x65, 0x06, 0xc1, 0xac, 0xdb, 0xec, 0xc0, 0x55, 0x25, 0x69, 0xb4, 0xb2, 0x14, 0x36, 0xa5, 0x69, 0x2e, 0x25, 0xd9, 0x0d, 0x3b, 0xc2, 0xeb, 0x7c, 0xe2, 0x40, 0x78}, SECP256K1_FE_CONST(0x948b40e7, 0x181713bc, 0x018ec170, 0x2d3d054d, 0x15746c59, 0xa7020730, 0xdd13ecf9, 0x85a010d7), 0}, + {{0xc8, 0x94, 0xce, 0x48, 0xbf, 0xec, 0x43, 0x30, 0x14, 0xb9, 0x31, 0xa6, 0xad, 0x42, 0x26, 0xd7, 0xdb, 0xd8, 0xea, 0xa7, 0xb6, 0xe3, 0xfa, 0xa8, 0xd0, 0xef, 0x94, 0x05, 0x2b, 0xcf, 0x8c, 0xff, 0x33, 0x6e, 0xeb, 0x39, 0x19, 0xe2, 0xb4, 0xef, 0xb7, 0x46, 0xc7, 0xf7, 0x1b, 0xbc, 0xa7, 0xe9, 0x38, 0x32, 0x30, 0xfb, 0xbc, 0x48, 0xff, 0xaf, 0xe7, 0x7e, 0x8b, 0xcc, 0x69, 0x54, 0x24, 0x71}, SECP256K1_FE_CONST(0xf1c91acd, 0xc2525330, 0xf9b53158, 0x434a4d43, 0xa1c547cf, 0xf29f1550, 0x6f5da4eb, 0x4fe8fa5a), 1}, + {{0xcb, 0xb0, 0xde, 0xab, 0x12, 0x57, 0x54, 0xf1, 0xfd, 0xb2, 0x03, 0x8b, 0x04, 0x34, 0xed, 0x9c, 0xb3, 0xfb, 0x53, 0xab, 0x73, 0x53, 0x91, 0x12, 0x99, 0x94, 0xa5, 0x35, 0xd9, 0x25, 0xf6, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, SECP256K1_FE_CONST(0x872d81ed, 0x8831d999, 0x8b67cb71, 0x05243edb, 0xf86c10ed, 0xfebb786c, 0x110b02d0, 0x7b2e67cd), 0}, + {{0xd9, 0x17, 0xb7, 0x86, 0xda, 0xc3, 0x56, 0x70, 0xc3, 0x30, 0xc9, 0xc5, 0xae, 0x59, 0x71, 0xdf, 0xb4, 0x95, 0xc8, 0xae, 0x52, 0x3e, 0xd9, 0x7e, 0xe2, 0x42, 0x01, 0x17, 0xb1, 0x71, 0xf4, 0x1e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x20, 0x01, 0xf6, 0xf6}, SECP256K1_FE_CONST(0xe45b71e1, 0x10b831f2, 0xbdad8651, 0x994526e5, 0x8393fde4, 0x328b1ec0, 0x4d598971, 0x42584691), 1}, + {{0xe2, 0x8b, 0xd8, 0xf5, 0x92, 0x9b, 0x46, 0x7e, 0xb7, 0x0e, 0x04, 0x33, 0x23, 0x74, 0xff, 0xb7, 0xe7, 0x18, 0x02, 0x18, 0xad, 0x16, 0xea, 0xa4, 0x6b, 0x71, 0x61, 0xaa, 0x67, 0x9e, 0xb4, 0x26, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, SECP256K1_FE_CONST(0x66b8c980, 0xa75c72e5, 0x98d383a3, 0x5a62879f, 0x844242ad, 0x1e73ff12, 0xedaa59f4, 0xe58632b5), 0}, + {{0xe2, 0x8b, 0xd8, 0xf5, 0x92, 0x9b, 0x46, 0x7e, 0xb7, 0x0e, 0x04, 0x33, 0x23, 0x74, 0xff, 0xb7, 0xe7, 0x18, 0x02, 0x18, 0xad, 0x16, 0xea, 0xa4, 0x6b, 0x71, 0x61, 0xaa, 0x67, 0x9e, 0xb4, 0x26, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f}, SECP256K1_FE_CONST(0x66b8c980, 0xa75c72e5, 0x98d383a3, 0x5a62879f, 0x844242ad, 0x1e73ff12, 0xedaa59f4, 0xe58632b5), 0}, + {{0xe7, 0xee, 0x58, 0x14, 0xc1, 0x70, 0x6b, 0xf8, 0xa8, 0x93, 0x96, 0xa9, 0xb0, 0x32, 0xbc, 0x01, 0x4c, 0x2c, 0xac, 0x9c, 0x12, 0x11, 0x27, 0xdb, 0xf6, 0xc9, 0x92, 0x78, 0xf8, 0xbb, 0x53, 0xd1, 0xdf, 0xd0, 0x4d, 0xbc, 0xda, 0x8e, 0x35, 0x24, 0x66, 0xb6, 0xfc, 0xd5, 0xf2, 0xde, 0xa3, 0xe1, 0x7d, 0x5e, 0x13, 0x31, 0x15, 0x88, 0x6e, 0xda, 0x20, 0xdb, 0x8a, 0x12, 0xb5, 0x4d, 0xe7, 0x1b}, SECP256K1_FE_CONST(0xe842c6e3, 0x529b2342, 0x70a5e977, 0x44edc34a, 0x04d7ba94, 0xe44b6d25, 0x23c9cf01, 0x95730a50), 1}, + {{0xf2, 0x92, 0xe4, 0x68, 0x25, 0xf9, 0x22, 0x5a, 0xd2, 0x3d, 0xc0, 0x57, 0xc1, 0xd9, 0x1c, 0x4f, 0x57, 0xfc, 0xb1, 0x38, 0x6f, 0x29, 0xef, 0x10, 0x48, 0x1c, 0xb1, 0xd2, 0x25, 0x18, 0x59, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x70, 0x11, 0xc9, 0x89}, SECP256K1_FE_CONST(0x3cea2c53, 0xb8b01701, 0x66ac7da6, 0x7194694a, 0xdacc84d5, 0x6389225e, 0x330134da, 0xb85a4d55), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, SECP256K1_FE_CONST(0xedd1fd3e, 0x327ce90c, 0xc7a35426, 0x14289aee, 0x9682003e, 0x9cf7dcc9, 0xcf2ca974, 0x3be5aa0c), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f, 0x01, 0xd3, 0x47, 0x5b, 0xf7, 0x65, 0x5b, 0x0f, 0xb2, 0xd8, 0x52, 0x92, 0x10, 0x35, 0xb2, 0xef, 0x60, 0x7f, 0x49, 0x06, 0x9b, 0x97, 0x45, 0x4e, 0x67, 0x95, 0x25, 0x10, 0x62, 0x74, 0x17, 0x71}, SECP256K1_FE_CONST(0xb5da00b7, 0x3cd65605, 0x20e7c364, 0x086e7cd2, 0x3a34bf60, 0xd0e707be, 0x9fc34d4c, 0xd5fdfa2c), 1}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f, 0x42, 0x18, 0xf2, 0x0a, 0xe6, 0xc6, 0x46, 0xb3, 0x63, 0xdb, 0x68, 0x60, 0x58, 0x22, 0xfb, 0x14, 0x26, 0x4c, 0xa8, 0xd2, 0x58, 0x7f, 0xdd, 0x6f, 0xbc, 0x75, 0x0d, 0x58, 0x7e, 0x76, 0xa7, 0xee}, SECP256K1_FE_CONST(0xaaaaaaaa, 0xaaaaaaaa, 0xaaaaaaaa, 0xaaaaaaaa, 0xaaaaaaaa, 0xaaaaaaaa, 0xaaaaaaa9, 0xfffffd6b), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f, 0x82, 0x27, 0x7c, 0x4a, 0x71, 0xf9, 0xd2, 0x2e, 0x66, 0xec, 0xe5, 0x23, 0xf8, 0xfa, 0x08, 0x74, 0x1a, 0x7c, 0x09, 0x12, 0xc6, 0x6a, 0x69, 0xce, 0x68, 0x51, 0x4b, 0xfd, 0x35, 0x15, 0xb4, 0x9f}, SECP256K1_FE_CONST(0xf482f2e2, 0x41753ad0, 0xfb89150d, 0x8491dc1e, 0x34ff0b8a, 0xcfbb442c, 0xfe999e2e, 0x5e6fd1d2), 1}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f, 0x84, 0x21, 0xcc, 0x93, 0x0e, 0x77, 0xc9, 0xf5, 0x14, 0xb6, 0x91, 0x5c, 0x3d, 0xbe, 0x2a, 0x94, 0xc6, 0xd8, 0xf6, 0x90, 0xb5, 0xb7, 0x39, 0x86, 0x4b, 0xa6, 0x78, 0x9f, 0xb8, 0xa5, 0x5d, 0xd0}, SECP256K1_FE_CONST(0x9f59c402, 0x75f5085a, 0x006f05da, 0xe77eb98c, 0x6fd0db1a, 0xb4a72ac4, 0x7eae90a4, 0xfc9e57e0), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f, 0xd1, 0x9c, 0x18, 0x2d, 0x27, 0x59, 0xcd, 0x99, 0x82, 0x42, 0x28, 0xd9, 0x47, 0x99, 0xf8, 0xc6, 0x55, 0x7c, 0x38, 0xa1, 0xc0, 0xd6, 0x77, 0x9b, 0x9d, 0x4b, 0x72, 0x9c, 0x6f, 0x1c, 0xcc, 0x42}, SECP256K1_FE_CONST(0x70720db7, 0xe238d041, 0x21f5b1af, 0xd8cc5ad9, 0xd18944c6, 0xbdc94881, 0xf502b7a3, 0xaf3aecff), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f}, SECP256K1_FE_CONST(0xedd1fd3e, 0x327ce90c, 0xc7a35426, 0x14289aee, 0x9682003e, 0x9cf7dcc9, 0xcf2ca974, 0x3be5aa0c), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0x64, 0xbb, 0xd5}, SECP256K1_FE_CONST(0x50873db3, 0x1badcc71, 0x890e4f67, 0x753a6575, 0x7f97aaa7, 0xdd5f1e82, 0xb753ace3, 0x2219064b), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x70, 0x28, 0xde, 0x7d}, SECP256K1_FE_CONST(0x1eea9cc5, 0x9cfcf2fa, 0x151ac6c2, 0x74eea411, 0x0feb4f7b, 0x68c59657, 0x32e9992e, 0x976ef68e), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcb, 0xcf, 0xb7, 0xe7}, SECP256K1_FE_CONST(0x12303941, 0xaedc2088, 0x80735b1f, 0x1795c8e5, 0x5be520ea, 0x93e10335, 0x7b5d2adb, 0x7ed59b8e), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf3, 0x11, 0x3a, 0xd9}, SECP256K1_FE_CONST(0x7eed6b70, 0xe7b0767c, 0x7d7feac0, 0x4e57aa2a, 0x12fef5e0, 0xf48f878f, 0xcbb88b3b, 0x6b5e0783), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x13, 0xce, 0xa4, 0xa7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, SECP256K1_FE_CONST(0x64998443, 0x5b62b4a2, 0x5d40c613, 0x3e8d9ab8, 0xc53d4b05, 0x9ee8a154, 0xa3be0fcf, 0x4e892edb), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x13, 0xce, 0xa4, 0xa7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f}, SECP256K1_FE_CONST(0x64998443, 0x5b62b4a2, 0x5d40c613, 0x3e8d9ab8, 0xc53d4b05, 0x9ee8a154, 0xa3be0fcf, 0x4e892edb), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x15, 0x02, 0x8c, 0x59, 0x00, 0x63, 0xf6, 0x4d, 0x5a, 0x7f, 0x1c, 0x14, 0x91, 0x5c, 0xd6, 0x1e, 0xac, 0x88, 0x6a, 0xb2, 0x95, 0xbe, 0xbd, 0x91, 0x99, 0x25, 0x04, 0xcf, 0x77, 0xed, 0xb0, 0x28, 0xbd, 0xd6, 0x26, 0x7f}, SECP256K1_FE_CONST(0x3fde5713, 0xf8282eea, 0xd7d39d42, 0x01f44a7c, 0x85a5ac8a, 0x0681f35e, 0x54085c6b, 0x69543374), 1}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x27, 0x15, 0xde, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, SECP256K1_FE_CONST(0x3524f77f, 0xa3a6eb43, 0x89c3cb5d, 0x27f1f914, 0x62086429, 0xcd6c0cb0, 0xdf43ea8f, 0x1e7b3fb4), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x27, 0x15, 0xde, 0x86, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f}, SECP256K1_FE_CONST(0x3524f77f, 0xa3a6eb43, 0x89c3cb5d, 0x27f1f914, 0x62086429, 0xcd6c0cb0, 0xdf43ea8f, 0x1e7b3fb4), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x2c, 0x2c, 0x57, 0x09, 0xe7, 0x15, 0x6c, 0x41, 0x77, 0x17, 0xf2, 0xfe, 0xab, 0x14, 0x71, 0x41, 0xec, 0x3d, 0xa1, 0x9f, 0xb7, 0x59, 0x57, 0x5c, 0xc6, 0xe3, 0x7b, 0x2e, 0xa5, 0xac, 0x93, 0x09, 0xf2, 0x6f, 0x0f, 0x66}, SECP256K1_FE_CONST(0xd2469ab3, 0xe04acbb2, 0x1c65a180, 0x9f39caaf, 0xe7a77c13, 0xd10f9dd3, 0x8f391c01, 0xdc499c52), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3a, 0x08, 0xcc, 0x1e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf7, 0x60, 0xe9, 0xf0}, SECP256K1_FE_CONST(0x38e2a5ce, 0x6a93e795, 0xe16d2c39, 0x8bc99f03, 0x69202ce2, 0x1e8f09d5, 0x6777b40f, 0xc512bccc), 1}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3e, 0x91, 0x25, 0x7d, 0x93, 0x20, 0x16, 0xcb, 0xf6, 0x9c, 0x44, 0x71, 0xbd, 0x1f, 0x65, 0x6c, 0x6a, 0x10, 0x7f, 0x19, 0x73, 0xde, 0x4a, 0xf7, 0x08, 0x6d, 0xb8, 0x97, 0x27, 0x70, 0x60, 0xe2, 0x56, 0x77, 0xf1, 0x9a}, SECP256K1_FE_CONST(0x864b3dc9, 0x02c37670, 0x9c10a93a, 0xd4bbe29f, 0xce0012f3, 0xdc8672c6, 0x286bba28, 0xd7d6d6fc), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x79, 0x5d, 0x6c, 0x1c, 0x32, 0x2c, 0xad, 0xf5, 0x99, 0xdb, 0xb8, 0x64, 0x81, 0x52, 0x2b, 0x3c, 0xc5, 0x5f, 0x15, 0xa6, 0x79, 0x32, 0xdb, 0x2a, 0xfa, 0x01, 0x11, 0xd9, 0xed, 0x69, 0x81, 0xbc, 0xd1, 0x24, 0xbf, 0x44}, SECP256K1_FE_CONST(0x766dfe4a, 0x700d9bee, 0x288b903a, 0xd58870e3, 0xd4fe2f0e, 0xf780bcac, 0x5c823f32, 0x0d9a9bef), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x8e, 0x42, 0x6f, 0x03, 0x92, 0x38, 0x90, 0x78, 0xc1, 0x2b, 0x1a, 0x89, 0xe9, 0x54, 0x2f, 0x05, 0x93, 0xbc, 0x96, 0xb6, 0xbf, 0xde, 0x82, 0x24, 0xf8, 0x65, 0x4e, 0xf5, 0xd5, 0xcd, 0xa9, 0x35, 0xa3, 0x58, 0x21, 0x94}, SECP256K1_FE_CONST(0xfaec7bc1, 0x987b6323, 0x3fbc5f95, 0x6edbf37d, 0x54404e74, 0x61c58ab8, 0x631bc68e, 0x451a0478), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x91, 0x19, 0x21, 0x39, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x45, 0xf0, 0xf1, 0xeb}, SECP256K1_FE_CONST(0xec29a50b, 0xae138dbf, 0x7d8e2482, 0x5006bb5f, 0xc1a2cc12, 0x43ba335b, 0xc6116fb9, 0xe498ec1f), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x98, 0xeb, 0x9a, 0xb7, 0x6e, 0x84, 0x49, 0x9c, 0x48, 0x3b, 0x3b, 0xf0, 0x62, 0x14, 0xab, 0xfe, 0x06, 0x5d, 0xdd, 0xf4, 0x3b, 0x86, 0x01, 0xde, 0x59, 0x6d, 0x63, 0xb9, 0xe4, 0x5a, 0x16, 0x6a, 0x58, 0x05, 0x41, 0xfe}, SECP256K1_FE_CONST(0x1e0ff2de, 0xe9b09b13, 0x6292a9e9, 0x10f0d6ac, 0x3e552a64, 0x4bba39e6, 0x4e9dd3e3, 0xbbd3d4d4), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x9b, 0x77, 0xb7, 0xf2, 0xc7, 0x4d, 0x99, 0xef, 0xce, 0xaa, 0x55, 0x0f, 0x1a, 0xd1, 0xc0, 0xf4, 0x3f, 0x46, 0xe7, 0xff, 0x1e, 0xe3, 0xbd, 0x01, 0x62, 0xb7, 0xbf, 0x55, 0xf2, 0x96, 0x5d, 0xa9, 0xc3, 0x45, 0x06, 0x46}, SECP256K1_FE_CONST(0x8b7dd5c3, 0xedba9ee9, 0x7b70eff4, 0x38f22dca, 0x9849c825, 0x4a2f3345, 0xa0a572ff, 0xeaae0928), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x9b, 0x77, 0xb7, 0xf2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x15, 0x6c, 0xa8, 0x96}, SECP256K1_FE_CONST(0x0881950c, 0x8f51d6b9, 0xa6387465, 0xd5f12609, 0xef1bb254, 0x12a08a74, 0xcb2dfb20, 0x0c74bfbf), 1}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa2, 0xf5, 0xcd, 0x83, 0x88, 0x16, 0xc1, 0x6c, 0x4f, 0xe8, 0xa1, 0x66, 0x1d, 0x60, 0x6f, 0xdb, 0x13, 0xcf, 0x9a, 0xf0, 0x4b, 0x97, 0x9a, 0x2e, 0x15, 0x9a, 0x09, 0x40, 0x9e, 0xbc, 0x86, 0x45, 0xd5, 0x8f, 0xde, 0x02}, SECP256K1_FE_CONST(0x2f083207, 0xb9fd9b55, 0x0063c31c, 0xd62b8746, 0xbd543bdc, 0x5bbf10e3, 0xa35563e9, 0x27f440c8), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xb1, 0x3f, 0x75, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, SECP256K1_FE_CONST(0x4f51e0be, 0x078e0cdd, 0xab274215, 0x6adba7e7, 0xa148e731, 0x57072fd6, 0x18cd6094, 0x2b146bd0), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xb1, 0x3f, 0x75, 0xc0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f}, SECP256K1_FE_CONST(0x4f51e0be, 0x078e0cdd, 0xab274215, 0x6adba7e7, 0xa148e731, 0x57072fd6, 0x18cd6094, 0x2b146bd0), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe7, 0xbc, 0x1f, 0x8d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, SECP256K1_FE_CONST(0x16c2ccb5, 0x4352ff4b, 0xd794f6ef, 0xd613c721, 0x97ab7082, 0xda5b563b, 0xdf9cb3ed, 0xaafe74c2), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe7, 0xbc, 0x1f, 0x8d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f}, SECP256K1_FE_CONST(0x16c2ccb5, 0x4352ff4b, 0xd794f6ef, 0xd613c721, 0x97ab7082, 0xda5b563b, 0xdf9cb3ed, 0xaafe74c2), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xef, 0x64, 0xd1, 0x62, 0x75, 0x05, 0x46, 0xce, 0x42, 0xb0, 0x43, 0x13, 0x61, 0xe5, 0x2d, 0x4f, 0x52, 0x42, 0xd8, 0xf2, 0x4f, 0x33, 0xe6, 0xb1, 0xf9, 0x9b, 0x59, 0x16, 0x47, 0xcb, 0xc8, 0x08, 0xf4, 0x62, 0xaf, 0x51}, SECP256K1_FE_CONST(0xd41244d1, 0x1ca4f652, 0x40687759, 0xf95ca9ef, 0xbab767ed, 0xedb38fd1, 0x8c36e18c, 0xd3b6f6a9), 1}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0xe5, 0xbe, 0x52, 0x37, 0x2d, 0xd6, 0xe8, 0x94, 0xb2, 0xa3, 0x26, 0xfc, 0x36, 0x05, 0xa6, 0xe8, 0xf3, 0xc6, 0x9c, 0x71, 0x0b, 0xf2, 0x7d, 0x63, 0x0d, 0xfe, 0x20, 0x04, 0x98, 0x8b, 0x78, 0xeb, 0x6e, 0xab, 0x36}, SECP256K1_FE_CONST(0x64bf84dd, 0x5e03670f, 0xdb24c0f5, 0xd3c2c365, 0x736f51db, 0x6c92d950, 0x10716ad2, 0xd36134c8), 0}, + {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xfb, 0xb9, 0x82, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf6, 0xd6, 0xdb, 0x1f}, SECP256K1_FE_CONST(0x1c92ccdf, 0xcf4ac550, 0xc28db57c, 0xff0c8515, 0xcb26936c, 0x786584a7, 0x0114008d, 0x6c33a34b), 0}, +}; + +/* Set of expected ellswift_xdh BIP324 shared secrets, given private key, encodings, initiating, + * taken from the BIP324 test vectors. Created using an independent implementation, and tested + * against the paper authors' decoding code. */ +static const struct ellswift_xdh_test ellswift_xdh_tests_bip324[] = { + {{0x61, 0x06, 0x2e, 0xa5, 0x07, 0x1d, 0x80, 0x0b, 0xbf, 0xd5, 0x9e, 0x2e, 0x8b, 0x53, 0xd4, 0x7d, 0x19, 0x4b, 0x09, 0x5a, 0xe5, 0xa4, 0xdf, 0x04, 0x93, 0x6b, 0x49, 0x77, 0x2e, 0xf0, 0xd4, 0xd7}, {0xec, 0x0a, 0xdf, 0xf2, 0x57, 0xbb, 0xfe, 0x50, 0x0c, 0x18, 0x8c, 0x80, 0xb4, 0xfd, 0xd6, 0x40, 0xf6, 0xb4, 0x5a, 0x48, 0x2b, 0xbc, 0x15, 0xfc, 0x7c, 0xef, 0x59, 0x31, 0xde, 0xff, 0x0a, 0xa1, 0x86, 0xf6, 0xeb, 0x9b, 0xba, 0x7b, 0x85, 0xdc, 0x4d, 0xcc, 0x28, 0xb2, 0x87, 0x22, 0xde, 0x1e, 0x3d, 0x91, 0x08, 0xb9, 0x85, 0xe2, 0x96, 0x70, 0x45, 0x66, 0x8f, 0x66, 0x09, 0x8e, 0x47, 0x5b}, {0xa4, 0xa9, 0x4d, 0xfc, 0xe6, 0x9b, 0x4a, 0x2a, 0x0a, 0x09, 0x93, 0x13, 0xd1, 0x0f, 0x9f, 0x7e, 0x7d, 0x64, 0x9d, 0x60, 0x50, 0x1c, 0x9e, 0x1d, 0x27, 0x4c, 0x30, 0x0e, 0x0d, 0x89, 0xaa, 0xfa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x8f, 0xaf, 0x88, 0xd5}, 1, {0xc6, 0x99, 0x2a, 0x11, 0x7f, 0x5e, 0xdb, 0xea, 0x70, 0xc3, 0xf5, 0x11, 0xd3, 0x2d, 0x26, 0xb9, 0x79, 0x8b, 0xe4, 0xb8, 0x1a, 0x62, 0xea, 0xee, 0x1a, 0x5a, 0xca, 0xa8, 0x45, 0x9a, 0x35, 0x92}}, + {{0x1f, 0x9c, 0x58, 0x1b, 0x35, 0x23, 0x18, 0x38, 0xf0, 0xf1, 0x7c, 0xf0, 0xc9, 0x79, 0x83, 0x5b, 0xac, 0xcb, 0x7f, 0x3a, 0xbb, 0xbb, 0x96, 0xff, 0xcc, 0x31, 0x8a, 0xb7, 0x1e, 0x6e, 0x12, 0x6f}, {0xa1, 0x85, 0x5e, 0x10, 0xe9, 0x4e, 0x00, 0xba, 0xa2, 0x30, 0x41, 0xd9, 0x16, 0xe2, 0x59, 0xf7, 0x04, 0x4e, 0x49, 0x1d, 0xa6, 0x17, 0x12, 0x69, 0x69, 0x47, 0x63, 0xf0, 0x18, 0xc7, 0xe6, 0x36, 0x93, 0xd2, 0x95, 0x75, 0xdc, 0xb4, 0x64, 0xac, 0x81, 0x6b, 0xaa, 0x1b, 0xe3, 0x53, 0xba, 0x12, 0xe3, 0x87, 0x6c, 0xba, 0x76, 0x28, 0xbd, 0x0b, 0xd8, 0xe7, 0x55, 0xe7, 0x21, 0xeb, 0x01, 0x40}, {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xfc, 0x2f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, 0, {0xa0, 0x13, 0x8f, 0x56, 0x4f, 0x74, 0xd0, 0xad, 0x70, 0xbc, 0x33, 0x7d, 0xac, 0xc9, 0xd0, 0xbf, 0x1d, 0x23, 0x49, 0x36, 0x4c, 0xaf, 0x11, 0x88, 0xa1, 0xe6, 0xe8, 0xdd, 0xb3, 0xb7, 0xb1, 0x84}}, + {{0x02, 0x86, 0xc4, 0x1c, 0xd3, 0x09, 0x13, 0xdb, 0x0f, 0xdf, 0xf7, 0xa6, 0x4e, 0xbd, 0xa5, 0xc8, 0xe3, 0xe7, 0xce, 0xf1, 0x0f, 0x2a, 0xeb, 0xc0, 0x0a, 0x76, 0x50, 0x44, 0x3c, 0xf4, 0xc6, 0x0d}, {0xd1, 0xee, 0x8a, 0x93, 0xa0, 0x11, 0x30, 0xcb, 0xf2, 0x99, 0x24, 0x9a, 0x25, 0x8f, 0x94, 0xfe, 0xb5, 0xf4, 0x69, 0xe7, 0xd0, 0xf2, 0xf2, 0x8f, 0x69, 0xee, 0x5e, 0x9a, 0xa8, 0xf9, 0xb5, 0x4a, 0x60, 0xf2, 0xc3, 0xff, 0x2d, 0x02, 0x36, 0x34, 0xec, 0x7f, 0x41, 0x27, 0xa9, 0x6c, 0xc1, 0x16, 0x62, 0xe4, 0x02, 0x89, 0x4c, 0xf1, 0xf6, 0x94, 0xfb, 0x9a, 0x7e, 0xaa, 0x5f, 0x1d, 0x92, 0x44}, {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x22, 0xd5, 0xe4, 0x41, 0x52, 0x4d, 0x57, 0x1a, 0x52, 0xb3, 0xde, 0xf1, 0x26, 0x18, 0x9d, 0x3f, 0x41, 0x68, 0x90, 0xa9, 0x9d, 0x4d, 0xa6, 0xed, 0xe2, 0xb0, 0xcd, 0xe1, 0x76, 0x0c, 0xe2, 0xc3, 0xf9, 0x84, 0x57, 0xae}, 1, {0x25, 0x0b, 0x93, 0x57, 0x0d, 0x41, 0x11, 0x49, 0x10, 0x5a, 0xb8, 0xcb, 0x0b, 0xc5, 0x07, 0x99, 0x14, 0x90, 0x63, 0x06, 0x36, 0x8c, 0x23, 0xe9, 0xd7, 0x7c, 0x2a, 0x33, 0x26, 0x5b, 0x99, 0x4c}}, + {{0x6c, 0x77, 0x43, 0x2d, 0x1f, 0xda, 0x31, 0xe9, 0xf9, 0x42, 0xf8, 0xaf, 0x44, 0x60, 0x7e, 0x10, 0xf3, 0xad, 0x38, 0xa6, 0x5f, 0x8a, 0x4b, 0xdd, 0xae, 0x82, 0x3e, 0x5e, 0xff, 0x90, 0xdc, 0x38}, {0xd2, 0x68, 0x50, 0x70, 0xc1, 0xe6, 0x37, 0x6e, 0x63, 0x3e, 0x82, 0x52, 0x96, 0x63, 0x4f, 0xd4, 0x61, 0xfa, 0x9e, 0x5b, 0xdf, 0x21, 0x09, 0xbc, 0xeb, 0xd7, 0x35, 0xe5, 0xa9, 0x1f, 0x3e, 0x58, 0x7c, 0x5c, 0xb7, 0x82, 0xab, 0xb7, 0x97, 0xfb, 0xf6, 0xbb, 0x50, 0x74, 0xfd, 0x15, 0x42, 0xa4, 0x74, 0xf2, 0xa4, 0x5b, 0x67, 0x37, 0x63, 0xec, 0x2d, 0xb7, 0xfb, 0x99, 0xb7, 0x37, 0xbb, 0xb9}, {0x56, 0xbd, 0x0c, 0x06, 0xf1, 0x03, 0x52, 0xc3, 0xa1, 0xa9, 0xf4, 0xb4, 0xc9, 0x2f, 0x6f, 0xa2, 0xb2, 0x6d, 0xf1, 0x24, 0xb5, 0x78, 0x78, 0x35, 0x3c, 0x1f, 0xc6, 0x91, 0xc5, 0x1a, 0xbe, 0xa7, 0x7c, 0x88, 0x17, 0xda, 0xee, 0xb9, 0xfa, 0x54, 0x6b, 0x77, 0xc8, 0xda, 0xf7, 0x9d, 0x89, 0xb2, 0x2b, 0x0e, 0x1b, 0x87, 0x57, 0x4e, 0xce, 0x42, 0x37, 0x1f, 0x00, 0x23, 0x7a, 0xa9, 0xd8, 0x3a}, 0, {0x19, 0x18, 0xb7, 0x41, 0xef, 0x5f, 0x9d, 0x1d, 0x76, 0x70, 0xb0, 0x50, 0xc1, 0x52, 0xb4, 0xa4, 0xea, 0xd2, 0xc3, 0x1b, 0xe9, 0xae, 0xcb, 0x06, 0x81, 0xc0, 0xcd, 0x43, 0x24, 0x15, 0x08, 0x53}}, + {{0xa6, 0xec, 0x25, 0x12, 0x7c, 0xa1, 0xaa, 0x4c, 0xf1, 0x6b, 0x20, 0x08, 0x4b, 0xa1, 0xe6, 0x51, 0x6b, 0xaa, 0xe4, 0xd3, 0x24, 0x22, 0x28, 0x8e, 0x9b, 0x36, 0xd8, 0xbd, 0xdd, 0x2d, 0xe3, 0x5a}, {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x05, 0x3d, 0x7e, 0xcc, 0xa5, 0x3e, 0x33, 0xe1, 0x85, 0xa8, 0xb9, 0xbe, 0x4e, 0x76, 0x99, 0xa9, 0x7c, 0x6f, 0xf4, 0xc7, 0x95, 0x52, 0x2e, 0x59, 0x18, 0xab, 0x7c, 0xd6, 0xb6, 0x88, 0x4f, 0x67, 0xe6, 0x83, 0xf3, 0xdc}, {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa7, 0x73, 0x0b, 0xe3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, 1, {0xdd, 0x21, 0x0a, 0xa6, 0x62, 0x9f, 0x20, 0xbb, 0x32, 0x8e, 0x5d, 0x89, 0xda, 0xa6, 0xeb, 0x2a, 0xc3, 0xd1, 0xc6, 0x58, 0xa7, 0x25, 0x53, 0x6f, 0xf1, 0x54, 0xf3, 0x1b, 0x53, 0x6c, 0x23, 0xb2}}, + {{0x0a, 0xf9, 0x52, 0x65, 0x9e, 0xd7, 0x6f, 0x80, 0xf5, 0x85, 0x96, 0x6b, 0x95, 0xab, 0x6e, 0x6f, 0xd6, 0x86, 0x54, 0x67, 0x28, 0x27, 0x87, 0x86, 0x84, 0xc8, 0xb5, 0x47, 0xb1, 0xb9, 0x4f, 0x5a}, {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc8, 0x10, 0x17, 0xfd, 0x92, 0xfd, 0x31, 0x63, 0x7c, 0x26, 0xc9, 0x06, 0xb4, 0x20, 0x92, 0xe1, 0x1c, 0xc0, 0xd3, 0xaf, 0xae, 0x8d, 0x90, 0x19, 0xd2, 0x57, 0x8a, 0xf2, 0x27, 0x35, 0xce, 0x7b, 0xc4, 0x69, 0xc7, 0x2d}, {0x96, 0x52, 0xd7, 0x8b, 0xae, 0xfc, 0x02, 0x8c, 0xd3, 0x7a, 0x6a, 0x92, 0x62, 0x5b, 0x8b, 0x8f, 0x85, 0xfd, 0xe1, 0xe4, 0xc9, 0x44, 0xad, 0x3f, 0x20, 0xe1, 0x98, 0xbe, 0xf8, 0xc0, 0x2f, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf2, 0xe9, 0x18, 0x70}, 0, {0x35, 0x68, 0xf2, 0xae, 0xa2, 0xe1, 0x4e, 0xf4, 0xee, 0x4a, 0x3c, 0x2a, 0x8b, 0x8d, 0x31, 0xbc, 0x5e, 0x31, 0x87, 0xba, 0x86, 0xdb, 0x10, 0x73, 0x9b, 0x4f, 0xf8, 0xec, 0x92, 0xff, 0x66, 0x55}}, + {{0xf9, 0x0e, 0x08, 0x0c, 0x64, 0xb0, 0x58, 0x24, 0xc5, 0xa2, 0x4b, 0x25, 0x01, 0xd5, 0xae, 0xaf, 0x08, 0xaf, 0x38, 0x72, 0xee, 0x86, 0x0a, 0xa8, 0x0b, 0xdc, 0xd4, 0x30, 0xf7, 0xb6, 0x34, 0x94}, {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x11, 0x51, 0x73, 0x76, 0x5d, 0xc2, 0x02, 0xcf, 0x02, 0x9a, 0xd3, 0xf1, 0x54, 0x79, 0x73, 0x5d, 0x57, 0x69, 0x7a, 0xf1, 0x2b, 0x01, 0x31, 0xdd, 0x21, 0x43, 0x0d, 0x57, 0x72, 0xe4, 0xef, 0x11, 0x47, 0x4d, 0x58, 0xb9}, {0x12, 0xa5, 0x0f, 0x3f, 0xaf, 0xea, 0x7c, 0x1e, 0xea, 0xda, 0x4c, 0xf8, 0xd3, 0x37, 0x77, 0x70, 0x4b, 0x77, 0x36, 0x14, 0x53, 0xaf, 0xc8, 0x3b, 0xda, 0x91, 0xee, 0xf3, 0x49, 0xae, 0x04, 0x4d, 0x20, 0x12, 0x6c, 0x62, 0x00, 0x54, 0x7e, 0xa5, 0xa6, 0x91, 0x17, 0x76, 0xc0, 0x5d, 0xee, 0x2a, 0x7f, 0x1a, 0x9b, 0xa7, 0xdf, 0xba, 0xbb, 0xbd, 0x27, 0x3c, 0x3e, 0xf2, 0x9e, 0xf4, 0x6e, 0x46}, 1, {0xe2, 0x54, 0x61, 0xfb, 0x0e, 0x4c, 0x16, 0x2e, 0x18, 0x12, 0x3e, 0xcd, 0xe8, 0x83, 0x42, 0xd5, 0x4d, 0x44, 0x96, 0x31, 0xe9, 0xb7, 0x5a, 0x26, 0x6f, 0xd9, 0x26, 0x0c, 0x2b, 0xb2, 0xf4, 0x1d}}, +}; + +/** This is a hasher for ellswift_xdh which just returns the shared X coordinate. + * + * This is generally a bad idea as it means changes to the encoding of the + * exchanged public keys do not affect the shared secret. However, it's used here + * in tests to be able to verify the X coordinate through other means. + */ +static int ellswift_xdh_hash_x32(unsigned char *output, const unsigned char *x32, const unsigned char *ell_a64, const unsigned char *ell_b64, void *data) { + (void)ell_a64; + (void)ell_b64; + (void)data; + memcpy(output, x32, 32); + return 1; +} + +void run_ellswift_tests(void) { + int i = 0; + /* Test vectors. */ + for (i = 0; (unsigned)i < sizeof(ellswift_xswiftec_inv_tests) / sizeof(ellswift_xswiftec_inv_tests[0]); ++i) { + const struct ellswift_xswiftec_inv_test *testcase = &ellswift_xswiftec_inv_tests[i]; + int c; + for (c = 0; c < 8; ++c) { + secp256k1_fe t; + int ret = secp256k1_ellswift_xswiftec_inv_var(&t, &testcase->x, &testcase->u, c); + CHECK(ret == ((testcase->enc_bitmap >> c) & 1)); + if (ret) { + secp256k1_fe x2; + CHECK(check_fe_equal(&t, &testcase->encs[c])); + secp256k1_ellswift_xswiftec_var(&x2, &testcase->u, &testcase->encs[c]); + CHECK(check_fe_equal(&testcase->x, &x2)); + } + } + } + for (i = 0; (unsigned)i < sizeof(ellswift_decode_tests) / sizeof(ellswift_decode_tests[0]); ++i) { + const struct ellswift_decode_test *testcase = &ellswift_decode_tests[i]; + secp256k1_pubkey pubkey; + secp256k1_ge ge; + int ret; + ret = secp256k1_ellswift_decode(CTX, &pubkey, testcase->enc); + CHECK(ret); + ret = secp256k1_pubkey_load(CTX, &ge, &pubkey); + CHECK(ret); + CHECK(check_fe_equal(&testcase->x, &ge.x)); + CHECK(secp256k1_fe_is_odd(&ge.y) == testcase->odd_y); + } + for (i = 0; (unsigned)i < sizeof(ellswift_xdh_tests_bip324) / sizeof(ellswift_xdh_tests_bip324[0]); ++i) { + const struct ellswift_xdh_test *test = &ellswift_xdh_tests_bip324[i]; + unsigned char shared_secret[32]; + int ret; + int party = !test->initiating; + const unsigned char* ell_a64 = party ? test->ellswift_theirs : test->ellswift_ours; + const unsigned char* ell_b64 = party ? test->ellswift_ours : test->ellswift_theirs; + ret = secp256k1_ellswift_xdh(CTX, shared_secret, + ell_a64, ell_b64, + test->priv_ours, + party, + secp256k1_ellswift_xdh_hash_function_bip324, + NULL); + CHECK(ret); + CHECK(secp256k1_memcmp_var(shared_secret, test->shared_secret, 32) == 0); + } + /* Verify that secp256k1_ellswift_encode + decode roundtrips. */ + for (i = 0; i < 1000 * COUNT; i++) { + unsigned char rnd32[32]; + unsigned char ell64[64]; + secp256k1_ge g, g2; + secp256k1_pubkey pubkey, pubkey2; + /* Generate random public key and random randomizer. */ + random_group_element_test(&g); + secp256k1_pubkey_save(&pubkey, &g); + secp256k1_testrand256(rnd32); + /* Convert the public key to ElligatorSwift and back. */ + secp256k1_ellswift_encode(CTX, ell64, &pubkey, rnd32); + secp256k1_ellswift_decode(CTX, &pubkey2, ell64); + secp256k1_pubkey_load(CTX, &g2, &pubkey2); + /* Compare with original. */ + ge_equals_ge(&g, &g2); + } + /* Verify the behavior of secp256k1_ellswift_create */ + for (i = 0; i < 400 * COUNT; i++) { + unsigned char auxrnd32[32], sec32[32]; + secp256k1_scalar sec; + secp256k1_gej res; + secp256k1_ge dec; + secp256k1_pubkey pub; + unsigned char ell64[64]; + int ret; + /* Generate random secret key and random randomizer. */ + if (i & 1) secp256k1_testrand256_test(auxrnd32); + random_scalar_order_test(&sec); + secp256k1_scalar_get_b32(sec32, &sec); + /* Construct ElligatorSwift-encoded public keys for that key. */ + ret = secp256k1_ellswift_create(CTX, ell64, sec32, (i & 1) ? auxrnd32 : NULL); + CHECK(ret); + /* Decode it, and compare with traditionally-computed public key. */ + secp256k1_ellswift_decode(CTX, &pub, ell64); + secp256k1_pubkey_load(CTX, &dec, &pub); + secp256k1_ecmult(&res, NULL, &secp256k1_scalar_zero, &sec); + ge_equals_gej(&dec, &res); + } + /* Verify that secp256k1_ellswift_xdh computes the right shared X coordinate. */ + for (i = 0; i < 800 * COUNT; i++) { + unsigned char ell64[64], sec32[32], share32[32]; + secp256k1_scalar sec; + secp256k1_ge dec, res; + secp256k1_fe share_x; + secp256k1_gej decj, resj; + secp256k1_pubkey pub; + int ret; + /* Generate random secret key. */ + random_scalar_order_test(&sec); + secp256k1_scalar_get_b32(sec32, &sec); + /* Generate random ElligatorSwift encoding for the remote key and decode it. */ + secp256k1_testrand256_test(ell64); + secp256k1_testrand256_test(ell64 + 32); + secp256k1_ellswift_decode(CTX, &pub, ell64); + secp256k1_pubkey_load(CTX, &dec, &pub); + secp256k1_gej_set_ge(&decj, &dec); + /* Compute the X coordinate of seckey*pubkey using ellswift_xdh. Note that we + * pass ell64 as claimed (but incorrect) encoding for sec32 here; this works + * because the "hasher" function we use here ignores the ell64 arguments. */ + ret = secp256k1_ellswift_xdh(CTX, share32, ell64, ell64, sec32, i & 1, &ellswift_xdh_hash_x32, NULL); + CHECK(ret); + (void)secp256k1_fe_set_b32_limit(&share_x, share32); /* no overflow is possible */ + secp256k1_fe_verify(&share_x); + /* Compute seckey*pubkey directly. */ + secp256k1_ecmult(&resj, &decj, &sec, NULL); + secp256k1_ge_set_gej(&res, &resj); + /* Compare. */ + CHECK(check_fe_equal(&res.x, &share_x)); + } + /* Verify the joint behavior of secp256k1_ellswift_xdh */ + for (i = 0; i < 200 * COUNT; i++) { + unsigned char auxrnd32a[32], auxrnd32b[32], auxrnd32a_bad[32], auxrnd32b_bad[32]; + unsigned char sec32a[32], sec32b[32], sec32a_bad[32], sec32b_bad[32]; + secp256k1_scalar seca, secb; + unsigned char ell64a[64], ell64b[64], ell64a_bad[64], ell64b_bad[64]; + unsigned char share32a[32], share32b[32], share32_bad[32]; + unsigned char prefix64[64]; + secp256k1_ellswift_xdh_hash_function hash_function; + void* data; + int ret; + + /* Pick hasher to use. */ + if ((i % 3) == 0) { + hash_function = ellswift_xdh_hash_x32; + data = NULL; + } else if ((i % 3) == 1) { + hash_function = secp256k1_ellswift_xdh_hash_function_bip324; + data = NULL; + } else { + hash_function = secp256k1_ellswift_xdh_hash_function_prefix; + secp256k1_testrand256_test(prefix64); + secp256k1_testrand256_test(prefix64 + 32); + data = prefix64; + } + + /* Generate random secret keys and random randomizers. */ + secp256k1_testrand256_test(auxrnd32a); + secp256k1_testrand256_test(auxrnd32b); + random_scalar_order_test(&seca); + random_scalar_order_test(&secb); + secp256k1_scalar_get_b32(sec32a, &seca); + secp256k1_scalar_get_b32(sec32b, &secb); + + /* Construct ElligatorSwift-encoded public keys for those keys. */ + /* For A: */ + ret = secp256k1_ellswift_create(CTX, ell64a, sec32a, auxrnd32a); + CHECK(ret); + /* For B: */ + ret = secp256k1_ellswift_create(CTX, ell64b, sec32b, auxrnd32b); + CHECK(ret); + + /* Compute the shared secret both ways and compare with each other. */ + /* For A: */ + ret = secp256k1_ellswift_xdh(CTX, share32a, ell64a, ell64b, sec32a, 0, hash_function, data); + CHECK(ret); + /* For B: */ + ret = secp256k1_ellswift_xdh(CTX, share32b, ell64a, ell64b, sec32b, 1, hash_function, data); + CHECK(ret); + /* And compare: */ + CHECK(secp256k1_memcmp_var(share32a, share32b, 32) == 0); + + /* Verify that the shared secret doesn't match if other side's public key is incorrect. */ + /* For A (using a bad public key for B): */ + memcpy(ell64b_bad, ell64b, sizeof(ell64a_bad)); + secp256k1_testrand_flip(ell64b_bad, sizeof(ell64b_bad)); + ret = secp256k1_ellswift_xdh(CTX, share32_bad, ell64a, ell64b_bad, sec32a, 0, hash_function, data); + CHECK(ret); /* Mismatching encodings don't get detected by secp256k1_ellswift_xdh. */ + CHECK(secp256k1_memcmp_var(share32_bad, share32a, 32) != 0); + /* For B (using a bad public key for A): */ + memcpy(ell64a_bad, ell64a, sizeof(ell64a_bad)); + secp256k1_testrand_flip(ell64a_bad, sizeof(ell64a_bad)); + ret = secp256k1_ellswift_xdh(CTX, share32_bad, ell64a_bad, ell64b, sec32b, 1, hash_function, data); + CHECK(ret); + CHECK(secp256k1_memcmp_var(share32_bad, share32b, 32) != 0); + + /* Verify that the shared secret doesn't match if the private key is incorrect. */ + /* For A: */ + memcpy(sec32a_bad, sec32a, sizeof(sec32a_bad)); + secp256k1_testrand_flip(sec32a_bad, sizeof(sec32a_bad)); + ret = secp256k1_ellswift_xdh(CTX, share32_bad, ell64a, ell64b, sec32a_bad, 0, hash_function, data); + CHECK(!ret || secp256k1_memcmp_var(share32_bad, share32a, 32) != 0); + /* For B: */ + memcpy(sec32b_bad, sec32b, sizeof(sec32b_bad)); + secp256k1_testrand_flip(sec32b_bad, sizeof(sec32b_bad)); + ret = secp256k1_ellswift_xdh(CTX, share32_bad, ell64a, ell64b, sec32b_bad, 1, hash_function, data); + CHECK(!ret || secp256k1_memcmp_var(share32_bad, share32b, 32) != 0); + + if (hash_function != ellswift_xdh_hash_x32) { + /* Verify that the shared secret doesn't match when a different encoding of the same public key is used. */ + /* For A (changing B's public key): */ + memcpy(auxrnd32b_bad, auxrnd32b, sizeof(auxrnd32b_bad)); + secp256k1_testrand_flip(auxrnd32b_bad, sizeof(auxrnd32b_bad)); + ret = secp256k1_ellswift_create(CTX, ell64b_bad, sec32b, auxrnd32b_bad); + CHECK(ret); + ret = secp256k1_ellswift_xdh(CTX, share32_bad, ell64a, ell64b_bad, sec32a, 0, hash_function, data); + CHECK(ret); + CHECK(secp256k1_memcmp_var(share32_bad, share32a, 32) != 0); + /* For B (changing A's public key): */ + memcpy(auxrnd32a_bad, auxrnd32a, sizeof(auxrnd32a_bad)); + secp256k1_testrand_flip(auxrnd32a_bad, sizeof(auxrnd32a_bad)); + ret = secp256k1_ellswift_create(CTX, ell64a_bad, sec32a, auxrnd32a_bad); + CHECK(ret); + ret = secp256k1_ellswift_xdh(CTX, share32_bad, ell64a_bad, ell64b, sec32b, 1, hash_function, data); + CHECK(ret); + CHECK(secp256k1_memcmp_var(share32_bad, share32b, 32) != 0); + + /* Verify that swapping sides changes the shared secret. */ + /* For A (claiming to be B): */ + ret = secp256k1_ellswift_xdh(CTX, share32_bad, ell64a, ell64b, sec32a, 1, hash_function, data); + CHECK(ret); + CHECK(secp256k1_memcmp_var(share32_bad, share32a, 32) != 0); + /* For B (claiming to be A): */ + ret = secp256k1_ellswift_xdh(CTX, share32_bad, ell64a, ell64b, sec32b, 0, hash_function, data); + CHECK(ret); + CHECK(secp256k1_memcmp_var(share32_bad, share32b, 32) != 0); + } + } + + /* Test hash initializers. */ + { + secp256k1_sha256 sha, sha_optimized; + static const unsigned char encode_tag[25] = "secp256k1_ellswift_encode"; + static const unsigned char create_tag[25] = "secp256k1_ellswift_create"; + static const unsigned char bip324_tag[26] = "bip324_ellswift_xonly_ecdh"; + + /* Check that hash initialized by + * secp256k1_ellswift_sha256_init_encode has the expected + * state. */ + secp256k1_sha256_initialize_tagged(&sha, encode_tag, sizeof(encode_tag)); + secp256k1_ellswift_sha256_init_encode(&sha_optimized); + test_sha256_eq(&sha, &sha_optimized); + + /* Check that hash initialized by + * secp256k1_ellswift_sha256_init_create has the expected + * state. */ + secp256k1_sha256_initialize_tagged(&sha, create_tag, sizeof(create_tag)); + secp256k1_ellswift_sha256_init_create(&sha_optimized); + test_sha256_eq(&sha, &sha_optimized); + + /* Check that hash initialized by + * secp256k1_ellswift_sha256_init_bip324 has the expected + * state. */ + secp256k1_sha256_initialize_tagged(&sha, bip324_tag, sizeof(bip324_tag)); + secp256k1_ellswift_sha256_init_bip324(&sha_optimized); + test_sha256_eq(&sha, &sha_optimized); + } +} + +#endif diff --git a/src/secp256k1/src/modules/extrakeys/main_impl.h b/src/secp256k1/src/modules/extrakeys/main_impl.h index e1003052f4..0c7e266777 100644 --- a/src/secp256k1/src/modules/extrakeys/main_impl.h +++ b/src/secp256k1/src/modules/extrakeys/main_impl.h @@ -9,6 +9,7 @@ #include "../../../include/secp256k1.h" #include "../../../include/secp256k1_extrakeys.h" +#include "../../util.h" static SECP256K1_INLINE int secp256k1_xonly_pubkey_load(const secp256k1_context* ctx, secp256k1_ge *ge, const secp256k1_xonly_pubkey *pubkey) { return secp256k1_pubkey_load(ctx, ge, (const secp256k1_pubkey *) pubkey); @@ -27,7 +28,7 @@ int secp256k1_xonly_pubkey_parse(const secp256k1_context* ctx, secp256k1_xonly_p memset(pubkey, 0, sizeof(*pubkey)); ARG_CHECK(input32 != NULL); - if (!secp256k1_fe_set_b32(&x, input32)) { + if (!secp256k1_fe_set_b32_limit(&x, input32)) { return 0; } if (!secp256k1_ge_set_xo_var(&pk, &x, 0)) { diff --git a/src/secp256k1/src/modules/extrakeys/tests_exhaustive_impl.h b/src/secp256k1/src/modules/extrakeys/tests_exhaustive_impl.h index 5ecc90d50f..d3d817a131 100644 --- a/src/secp256k1/src/modules/extrakeys/tests_exhaustive_impl.h +++ b/src/secp256k1/src/modules/extrakeys/tests_exhaustive_impl.h @@ -47,7 +47,7 @@ static void test_exhaustive_extrakeys(const secp256k1_context *ctx, const secp25 CHECK(secp256k1_memcmp_var(xonly_pubkey_bytes[i - 1], buf, 32) == 0); /* Compare the xonly_pubkey bytes against the precomputed group. */ - secp256k1_fe_set_b32(&fe, xonly_pubkey_bytes[i - 1]); + secp256k1_fe_set_b32_mod(&fe, xonly_pubkey_bytes[i - 1]); CHECK(secp256k1_fe_equal_var(&fe, &group[i].x)); /* Check the parity against the precomputed group. */ diff --git a/src/secp256k1/src/modules/recovery/main_impl.h b/src/secp256k1/src/modules/recovery/main_impl.h index e7906eb62e..76a005e017 100644 --- a/src/secp256k1/src/modules/recovery/main_impl.h +++ b/src/secp256k1/src/modules/recovery/main_impl.h @@ -98,7 +98,7 @@ static int secp256k1_ecdsa_sig_recover(const secp256k1_scalar *sigr, const secp2 } secp256k1_scalar_get_b32(brx, sigr); - r = secp256k1_fe_set_b32(&fx, brx); + r = secp256k1_fe_set_b32_limit(&fx, brx); (void)r; VERIFY_CHECK(r); /* brx comes from a scalar, so is less than the order; certainly less than p */ if (recid & 2) { diff --git a/src/secp256k1/src/modules/schnorrsig/main_impl.h b/src/secp256k1/src/modules/schnorrsig/main_impl.h index cd651591c4..4e7b45a045 100644 --- a/src/secp256k1/src/modules/schnorrsig/main_impl.h +++ b/src/secp256k1/src/modules/schnorrsig/main_impl.h @@ -232,7 +232,7 @@ int secp256k1_schnorrsig_verify(const secp256k1_context* ctx, const unsigned cha ARG_CHECK(msg != NULL || msglen == 0); ARG_CHECK(pubkey != NULL); - if (!secp256k1_fe_set_b32(&rx, &sig64[0])) { + if (!secp256k1_fe_set_b32_limit(&rx, &sig64[0])) { return 0; } diff --git a/src/secp256k1/src/modules/schnorrsig/tests_impl.h b/src/secp256k1/src/modules/schnorrsig/tests_impl.h index 062005ee63..90337ff03e 100644 --- a/src/secp256k1/src/modules/schnorrsig/tests_impl.h +++ b/src/secp256k1/src/modules/schnorrsig/tests_impl.h @@ -20,17 +20,6 @@ static void nonce_function_bip340_bitflip(unsigned char **args, size_t n_flip, s CHECK(secp256k1_memcmp_var(nonces[0], nonces[1], 32) != 0); } -/* Tests for the equality of two sha256 structs. This function only produces a - * correct result if an integer multiple of 64 many bytes have been written - * into the hash functions. */ -static void test_sha256_eq(const secp256k1_sha256 *sha1, const secp256k1_sha256 *sha2) { - /* Is buffer fully consumed? */ - CHECK((sha1->bytes & 0x3F) == 0); - - CHECK(sha1->bytes == sha2->bytes); - CHECK(secp256k1_memcmp_var(sha1->s, sha2->s, sizeof(sha1->s)) == 0); -} - static void run_nonce_function_bip340_tests(void) { unsigned char tag[13] = "BIP0340/nonce"; unsigned char aux_tag[11] = "BIP0340/aux"; @@ -215,28 +204,36 @@ static void test_schnorrsig_sha256_tagged(void) { /* Helper function for schnorrsig_bip_vectors * Signs the message and checks that it's the same as expected_sig. */ -static void test_schnorrsig_bip_vectors_check_signing(const unsigned char *sk, const unsigned char *pk_serialized, const unsigned char *aux_rand, const unsigned char *msg32, const unsigned char *expected_sig) { +static void test_schnorrsig_bip_vectors_check_signing(const unsigned char *sk, const unsigned char *pk_serialized, const unsigned char *aux_rand, const unsigned char *msg, size_t msglen, const unsigned char *expected_sig) { unsigned char sig[64]; secp256k1_keypair keypair; secp256k1_xonly_pubkey pk, pk_expected; + secp256k1_schnorrsig_extraparams extraparams = SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT; + extraparams.ndata = (unsigned char*)aux_rand; + CHECK(secp256k1_keypair_create(CTX, &keypair, sk)); - CHECK(secp256k1_schnorrsig_sign32(CTX, sig, msg32, &keypair, aux_rand)); + CHECK(secp256k1_schnorrsig_sign_custom(CTX, sig, msg, msglen, &keypair, &extraparams)); CHECK(secp256k1_memcmp_var(sig, expected_sig, 64) == 0); + if (msglen == 32) { + memset(sig, 0, 64); + CHECK(secp256k1_schnorrsig_sign32(CTX, sig, msg, &keypair, aux_rand)); + CHECK(secp256k1_memcmp_var(sig, expected_sig, 64) == 0); + } CHECK(secp256k1_xonly_pubkey_parse(CTX, &pk_expected, pk_serialized)); CHECK(secp256k1_keypair_xonly_pub(CTX, &pk, NULL, &keypair)); CHECK(secp256k1_memcmp_var(&pk, &pk_expected, sizeof(pk)) == 0); - CHECK(secp256k1_schnorrsig_verify(CTX, sig, msg32, 32, &pk)); + CHECK(secp256k1_schnorrsig_verify(CTX, sig, msg, msglen, &pk)); } /* Helper function for schnorrsig_bip_vectors * Checks that both verify and verify_batch (TODO) return the same value as expected. */ -static void test_schnorrsig_bip_vectors_check_verify(const unsigned char *pk_serialized, const unsigned char *msg32, const unsigned char *sig, int expected) { +static void test_schnorrsig_bip_vectors_check_verify(const unsigned char *pk_serialized, const unsigned char *msg, size_t msglen, const unsigned char *sig, int expected) { secp256k1_xonly_pubkey pk; CHECK(secp256k1_xonly_pubkey_parse(CTX, &pk, pk_serialized)); - CHECK(expected == secp256k1_schnorrsig_verify(CTX, sig, msg32, 32, &pk)); + CHECK(expected == secp256k1_schnorrsig_verify(CTX, sig, msg, msglen, &pk)); } /* Test vectors according to BIP-340 ("Schnorr Signatures for secp256k1"). See @@ -256,7 +253,7 @@ static void test_schnorrsig_bip_vectors(void) { 0xB5, 0x31, 0xC8, 0x45, 0x83, 0x6F, 0x99, 0xB0, 0x86, 0x01, 0xF1, 0x13, 0xBC, 0xE0, 0x36, 0xF9 }; - unsigned char aux_rand[32] = { + const unsigned char aux_rand[32] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -278,8 +275,8 @@ static void test_schnorrsig_bip_vectors(void) { 0xEB, 0xEE, 0xE8, 0xFD, 0xB2, 0x17, 0x2F, 0x47, 0x7D, 0xF4, 0x90, 0x0D, 0x31, 0x05, 0x36, 0xC0 }; - test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sig); - test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 1); + test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sizeof(msg), sig); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 1); } { /* Test vector 1 */ @@ -295,7 +292,7 @@ static void test_schnorrsig_bip_vectors(void) { 0x58, 0xFE, 0xAE, 0x1D, 0xA2, 0xDE, 0xCE, 0xD8, 0x43, 0x24, 0x0F, 0x7B, 0x50, 0x2B, 0xA6, 0x59 }; - unsigned char aux_rand[32] = { + const unsigned char aux_rand[32] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -317,8 +314,8 @@ static void test_schnorrsig_bip_vectors(void) { 0x89, 0x7E, 0xFC, 0xB6, 0x39, 0xEA, 0x87, 0x1C, 0xFA, 0x95, 0xF6, 0xDE, 0x33, 0x9E, 0x4B, 0x0A }; - test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sig); - test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 1); + test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sizeof(msg), sig); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 1); } { /* Test vector 2 */ @@ -334,7 +331,7 @@ static void test_schnorrsig_bip_vectors(void) { 0x01, 0x39, 0x71, 0x53, 0x09, 0xB0, 0x86, 0xC9, 0x60, 0xE1, 0x8F, 0xD9, 0x69, 0x77, 0x4E, 0xB8 }; - unsigned char aux_rand[32] = { + const unsigned char aux_rand[32] = { 0xC8, 0x7A, 0xA5, 0x38, 0x24, 0xB4, 0xD7, 0xAE, 0x2E, 0xB0, 0x35, 0xA2, 0xB5, 0xBB, 0xBC, 0xCC, 0x08, 0x0E, 0x76, 0xCD, 0xC6, 0xD1, 0x69, 0x2C, @@ -356,8 +353,8 @@ static void test_schnorrsig_bip_vectors(void) { 0x7A, 0xDE, 0xA9, 0x8D, 0x82, 0xF8, 0x48, 0x1E, 0x0E, 0x1E, 0x03, 0x67, 0x4A, 0x6F, 0x3F, 0xB7 }; - test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sig); - test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 1); + test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sizeof(msg), sig); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 1); } { /* Test vector 3 */ @@ -373,7 +370,7 @@ static void test_schnorrsig_bip_vectors(void) { 0x3A, 0x0D, 0x95, 0xFB, 0xF2, 0x1D, 0x46, 0x8A, 0x1B, 0x33, 0xF8, 0xC1, 0x60, 0xD8, 0xF5, 0x17 }; - unsigned char aux_rand[32] = { + const unsigned char aux_rand[32] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, @@ -395,8 +392,8 @@ static void test_schnorrsig_bip_vectors(void) { 0xF2, 0x5F, 0xD7, 0x88, 0x81, 0xEB, 0xB3, 0x27, 0x71, 0xFC, 0x59, 0x22, 0xEF, 0xC6, 0x6E, 0xA3 }; - test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sig); - test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 1); + test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sizeof(msg), sig); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 1); } { /* Test vector 4 */ @@ -422,7 +419,7 @@ static void test_schnorrsig_bip_vectors(void) { 0x60, 0xCB, 0x71, 0xC0, 0x4E, 0x80, 0xF5, 0x93, 0x06, 0x0B, 0x07, 0xD2, 0x83, 0x08, 0xD7, 0xF4 }; - test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 1); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 1); } { /* Test vector 5 */ @@ -460,7 +457,7 @@ static void test_schnorrsig_bip_vectors(void) { 0x7A, 0x73, 0xC6, 0x43, 0xE1, 0x66, 0xBE, 0x5E, 0xBE, 0xAF, 0xA3, 0x4B, 0x1A, 0xC5, 0x53, 0xE2 }; - test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 0); } { /* Test vector 7 */ @@ -486,7 +483,7 @@ static void test_schnorrsig_bip_vectors(void) { 0x62, 0x2A, 0x95, 0x4C, 0xFE, 0x54, 0x57, 0x35, 0xAA, 0xEA, 0x51, 0x34, 0xFC, 0xCD, 0xB2, 0xBD }; - test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 0); } { /* Test vector 8 */ @@ -512,7 +509,7 @@ static void test_schnorrsig_bip_vectors(void) { 0xE8, 0xD7, 0xC9, 0x3E, 0x00, 0xC5, 0xED, 0x0C, 0x18, 0x34, 0xFF, 0x0D, 0x0C, 0x2E, 0x6D, 0xA6 }; - test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 0); } { /* Test vector 9 */ @@ -538,7 +535,7 @@ static void test_schnorrsig_bip_vectors(void) { 0x4F, 0xB7, 0x34, 0x76, 0xF0, 0xD5, 0x94, 0xDC, 0xB6, 0x5C, 0x64, 0x25, 0xBD, 0x18, 0x60, 0x51 }; - test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 0); } { /* Test vector 10 */ @@ -564,7 +561,7 @@ static void test_schnorrsig_bip_vectors(void) { 0xDB, 0xA8, 0x7F, 0x11, 0xAC, 0x67, 0x54, 0xF9, 0x37, 0x80, 0xD5, 0xA1, 0x83, 0x7C, 0xF1, 0x97 }; - test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 0); } { /* Test vector 11 */ @@ -590,7 +587,7 @@ static void test_schnorrsig_bip_vectors(void) { 0xD1, 0xD7, 0x13, 0xA8, 0xAE, 0x82, 0xB3, 0x2F, 0xA7, 0x9D, 0x5F, 0x7F, 0xC4, 0x07, 0xD3, 0x9B }; - test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 0); } { /* Test vector 12 */ @@ -616,7 +613,7 @@ static void test_schnorrsig_bip_vectors(void) { 0xD1, 0xD7, 0x13, 0xA8, 0xAE, 0x82, 0xB3, 0x2F, 0xA7, 0x9D, 0x5F, 0x7F, 0xC4, 0x07, 0xD3, 0x9B }; - test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 0); } { /* Test vector 13 */ @@ -642,7 +639,7 @@ static void test_schnorrsig_bip_vectors(void) { 0xBA, 0xAE, 0xDC, 0xE6, 0xAF, 0x48, 0xA0, 0x3B, 0xBF, 0xD2, 0x5E, 0x8C, 0xD0, 0x36, 0x41, 0x41 }; - test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 0); } { /* Test vector 14 */ @@ -656,6 +653,147 @@ static void test_schnorrsig_bip_vectors(void) { /* No need to check the signature of the test vector as parsing the pubkey already fails */ CHECK(!secp256k1_xonly_pubkey_parse(CTX, &pk_parsed, pk)); } + { + /* Test vector 15 */ + const unsigned char sk[32] = { + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + }; + const unsigned char pk[32] = { + 0x77, 0x8C, 0xAA, 0x53, 0xB4, 0x39, 0x3A, 0xC4, + 0x67, 0x77, 0x4D, 0x09, 0x49, 0x7A, 0x87, 0x22, + 0x4B, 0xF9, 0xFA, 0xB6, 0xF6, 0xE6, 0x8B, 0x23, + 0x08, 0x64, 0x97, 0x32, 0x4D, 0x6F, 0xD1, 0x17, + }; + const unsigned char aux_rand[32] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + /* const unsigned char msg[0] = {}; */ + const unsigned char sig[64] = { + 0x71, 0x53, 0x5D, 0xB1, 0x65, 0xEC, 0xD9, 0xFB, + 0xBC, 0x04, 0x6E, 0x5F, 0xFA, 0xEA, 0x61, 0x18, + 0x6B, 0xB6, 0xAD, 0x43, 0x67, 0x32, 0xFC, 0xCC, + 0x25, 0x29, 0x1A, 0x55, 0x89, 0x54, 0x64, 0xCF, + 0x60, 0x69, 0xCE, 0x26, 0xBF, 0x03, 0x46, 0x62, + 0x28, 0xF1, 0x9A, 0x3A, 0x62, 0xDB, 0x8A, 0x64, + 0x9F, 0x2D, 0x56, 0x0F, 0xAC, 0x65, 0x28, 0x27, + 0xD1, 0xAF, 0x05, 0x74, 0xE4, 0x27, 0xAB, 0x63, + }; + test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, NULL, 0, sig); + test_schnorrsig_bip_vectors_check_verify(pk, NULL, 0, sig, 1); + } + { + /* Test vector 16 */ + const unsigned char sk[32] = { + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + }; + const unsigned char pk[32] = { + 0x77, 0x8C, 0xAA, 0x53, 0xB4, 0x39, 0x3A, 0xC4, + 0x67, 0x77, 0x4D, 0x09, 0x49, 0x7A, 0x87, 0x22, + 0x4B, 0xF9, 0xFA, 0xB6, 0xF6, 0xE6, 0x8B, 0x23, + 0x08, 0x64, 0x97, 0x32, 0x4D, 0x6F, 0xD1, 0x17, + }; + const unsigned char aux_rand[32] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + const unsigned char msg[] = { 0x11 }; + const unsigned char sig[64] = { + 0x08, 0xA2, 0x0A, 0x0A, 0xFE, 0xF6, 0x41, 0x24, + 0x64, 0x92, 0x32, 0xE0, 0x69, 0x3C, 0x58, 0x3A, + 0xB1, 0xB9, 0x93, 0x4A, 0xE6, 0x3B, 0x4C, 0x35, + 0x11, 0xF3, 0xAE, 0x11, 0x34, 0xC6, 0xA3, 0x03, + 0xEA, 0x31, 0x73, 0xBF, 0xEA, 0x66, 0x83, 0xBD, + 0x10, 0x1F, 0xA5, 0xAA, 0x5D, 0xBC, 0x19, 0x96, + 0xFE, 0x7C, 0xAC, 0xFC, 0x5A, 0x57, 0x7D, 0x33, + 0xEC, 0x14, 0x56, 0x4C, 0xEC, 0x2B, 0xAC, 0xBF, + }; + test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sizeof(msg), sig); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 1); + } + { + /* Test vector 17 */ + const unsigned char sk[32] = { + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + }; + const unsigned char pk[32] = { + 0x77, 0x8C, 0xAA, 0x53, 0xB4, 0x39, 0x3A, 0xC4, + 0x67, 0x77, 0x4D, 0x09, 0x49, 0x7A, 0x87, 0x22, + 0x4B, 0xF9, 0xFA, 0xB6, 0xF6, 0xE6, 0x8B, 0x23, + 0x08, 0x64, 0x97, 0x32, 0x4D, 0x6F, 0xD1, 0x17, + }; + const unsigned char aux_rand[32] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + const unsigned char msg[] = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x11, + }; + const unsigned char sig[64] = { + 0x51, 0x30, 0xF3, 0x9A, 0x40, 0x59, 0xB4, 0x3B, + 0xC7, 0xCA, 0xC0, 0x9A, 0x19, 0xEC, 0xE5, 0x2B, + 0x5D, 0x86, 0x99, 0xD1, 0xA7, 0x1E, 0x3C, 0x52, + 0xDA, 0x9A, 0xFD, 0xB6, 0xB5, 0x0A, 0xC3, 0x70, + 0xC4, 0xA4, 0x82, 0xB7, 0x7B, 0xF9, 0x60, 0xF8, + 0x68, 0x15, 0x40, 0xE2, 0x5B, 0x67, 0x71, 0xEC, + 0xE1, 0xE5, 0xA3, 0x7F, 0xD8, 0x0E, 0x5A, 0x51, + 0x89, 0x7C, 0x55, 0x66, 0xA9, 0x7E, 0xA5, 0xA5, + }; + test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sizeof(msg), sig); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 1); + } + { + /* Test vector 18 */ + const unsigned char sk[32] = { + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, 0x03, 0x40, + }; + const unsigned char pk[32] = { + 0x77, 0x8C, 0xAA, 0x53, 0xB4, 0x39, 0x3A, 0xC4, + 0x67, 0x77, 0x4D, 0x09, 0x49, 0x7A, 0x87, 0x22, + 0x4B, 0xF9, 0xFA, 0xB6, 0xF6, 0xE6, 0x8B, 0x23, + 0x08, 0x64, 0x97, 0x32, 0x4D, 0x6F, 0xD1, 0x17, + }; + const unsigned char aux_rand[32] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + const unsigned char sig[64] = { + 0x40, 0x3B, 0x12, 0xB0, 0xD8, 0x55, 0x5A, 0x34, + 0x41, 0x75, 0xEA, 0x7E, 0xC7, 0x46, 0x56, 0x63, + 0x03, 0x32, 0x1E, 0x5D, 0xBF, 0xA8, 0xBE, 0x6F, + 0x09, 0x16, 0x35, 0x16, 0x3E, 0xCA, 0x79, 0xA8, + 0x58, 0x5E, 0xD3, 0xE3, 0x17, 0x08, 0x07, 0xE7, + 0xC0, 0x3B, 0x72, 0x0F, 0xC5, 0x4C, 0x7B, 0x23, + 0x89, 0x7F, 0xCB, 0xA0, 0xE9, 0xD0, 0xB4, 0xA0, + 0x68, 0x94, 0xCF, 0xD2, 0x49, 0xF2, 0x23, 0x67, + }; + unsigned char msg[100]; + memset(msg, 0x99, sizeof(msg)); + test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sizeof(msg), sig); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sizeof(msg), sig, 1); + } } /* Nonce function that returns constant 0 */ diff --git a/src/secp256k1/src/precompute_ecmult.c b/src/secp256k1/src/precompute_ecmult.c index 10aba5b97d..742142cf58 100644 --- a/src/secp256k1/src/precompute_ecmult.c +++ b/src/secp256k1/src/precompute_ecmult.c @@ -68,7 +68,6 @@ int main(void) { fprintf(fp, "/* This file contains an array secp256k1_pre_g with odd multiples of the base point G and\n"); fprintf(fp, " * an array secp256k1_pre_g_128 with odd multiples of 2^128*G for accelerating the computation of a*P + b*G.\n"); fprintf(fp, " */\n"); - fprintf(fp, "#include \"../include/secp256k1.h\"\n"); fprintf(fp, "#include \"group.h\"\n"); fprintf(fp, "#include \"ecmult.h\"\n"); fprintf(fp, "#include \"precomputed_ecmult.h\"\n"); diff --git a/src/secp256k1/src/precompute_ecmult_gen.c b/src/secp256k1/src/precompute_ecmult_gen.c index bfe212fdd2..ce648cb9b0 100644 --- a/src/secp256k1/src/precompute_ecmult_gen.c +++ b/src/secp256k1/src/precompute_ecmult_gen.c @@ -33,7 +33,6 @@ int main(int argc, char **argv) { fprintf(fp, "/* This file was automatically generated by precompute_ecmult_gen. */\n"); fprintf(fp, "/* See ecmult_gen_impl.h for details about the contents of this file. */\n"); - fprintf(fp, "#include \"../include/secp256k1.h\"\n"); fprintf(fp, "#include \"group.h\"\n"); fprintf(fp, "#include \"ecmult_gen.h\"\n"); fprintf(fp, "#include \"precomputed_ecmult_gen.h\"\n"); diff --git a/src/secp256k1/src/precomputed_ecmult.c b/src/secp256k1/src/precomputed_ecmult.c index fbc634ef1b..cbd030ce50 100644 --- a/src/secp256k1/src/precomputed_ecmult.c +++ b/src/secp256k1/src/precomputed_ecmult.c @@ -2,7 +2,6 @@ /* This file contains an array secp256k1_pre_g with odd multiples of the base point G and * an array secp256k1_pre_g_128 with odd multiples of 2^128*G for accelerating the computation of a*P + b*G. */ -#include "../include/secp256k1.h" #include "group.h" #include "ecmult.h" #include "precomputed_ecmult.h" diff --git a/src/secp256k1/src/precomputed_ecmult.h b/src/secp256k1/src/precomputed_ecmult.h index a4aa83e4ca..17df102967 100644 --- a/src/secp256k1/src/precomputed_ecmult.h +++ b/src/secp256k1/src/precomputed_ecmult.h @@ -11,6 +11,7 @@ extern "C" { #endif +#include "ecmult.h" #include "group.h" #if defined(EXHAUSTIVE_TEST_ORDER) # if EXHAUSTIVE_TEST_ORDER == 7 diff --git a/src/secp256k1/src/precomputed_ecmult_gen.c b/src/secp256k1/src/precomputed_ecmult_gen.c index e9d62a1c1b..75ec59c27a 100644 --- a/src/secp256k1/src/precomputed_ecmult_gen.c +++ b/src/secp256k1/src/precomputed_ecmult_gen.c @@ -1,6 +1,5 @@ /* This file was automatically generated by precompute_ecmult_gen. */ /* See ecmult_gen_impl.h for details about the contents of this file. */ -#include "../include/secp256k1.h" #include "group.h" #include "ecmult_gen.h" #include "precomputed_ecmult_gen.h" diff --git a/src/secp256k1/src/scalar_4x64_impl.h b/src/secp256k1/src/scalar_4x64_impl.h index 1959dae986..1d14740577 100644 --- a/src/secp256k1/src/scalar_4x64_impl.h +++ b/src/secp256k1/src/scalar_4x64_impl.h @@ -10,6 +10,7 @@ #include "checkmem.h" #include "int128.h" #include "modinv64_impl.h" +#include "util.h" /* Limbs of the secp256k1 order. */ #define SECP256K1_N_0 ((uint64_t)0xBFD25E8CD0364141ULL) @@ -110,8 +111,9 @@ static int secp256k1_scalar_add(secp256k1_scalar *r, const secp256k1_scalar *a, static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int flag) { secp256k1_uint128 t; + volatile int vflag = flag; VERIFY_CHECK(bit < 256); - bit += ((uint32_t) flag - 1) & 0x100; /* forcing (bit >> 6) > 3 makes this a noop */ + bit += ((uint32_t) vflag - 1) & 0x100; /* forcing (bit >> 6) > 3 makes this a noop */ secp256k1_u128_from_u64(&t, r->d[0]); secp256k1_u128_accum_u64(&t, ((uint64_t)((bit >> 6) == 0)) << (bit & 0x3F)); r->d[0] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); @@ -131,10 +133,10 @@ static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int static void secp256k1_scalar_set_b32(secp256k1_scalar *r, const unsigned char *b32, int *overflow) { int over; - r->d[0] = (uint64_t)b32[31] | (uint64_t)b32[30] << 8 | (uint64_t)b32[29] << 16 | (uint64_t)b32[28] << 24 | (uint64_t)b32[27] << 32 | (uint64_t)b32[26] << 40 | (uint64_t)b32[25] << 48 | (uint64_t)b32[24] << 56; - r->d[1] = (uint64_t)b32[23] | (uint64_t)b32[22] << 8 | (uint64_t)b32[21] << 16 | (uint64_t)b32[20] << 24 | (uint64_t)b32[19] << 32 | (uint64_t)b32[18] << 40 | (uint64_t)b32[17] << 48 | (uint64_t)b32[16] << 56; - r->d[2] = (uint64_t)b32[15] | (uint64_t)b32[14] << 8 | (uint64_t)b32[13] << 16 | (uint64_t)b32[12] << 24 | (uint64_t)b32[11] << 32 | (uint64_t)b32[10] << 40 | (uint64_t)b32[9] << 48 | (uint64_t)b32[8] << 56; - r->d[3] = (uint64_t)b32[7] | (uint64_t)b32[6] << 8 | (uint64_t)b32[5] << 16 | (uint64_t)b32[4] << 24 | (uint64_t)b32[3] << 32 | (uint64_t)b32[2] << 40 | (uint64_t)b32[1] << 48 | (uint64_t)b32[0] << 56; + r->d[0] = secp256k1_read_be64(&b32[24]); + r->d[1] = secp256k1_read_be64(&b32[16]); + r->d[2] = secp256k1_read_be64(&b32[8]); + r->d[3] = secp256k1_read_be64(&b32[0]); over = secp256k1_scalar_reduce(r, secp256k1_scalar_check_overflow(r)); if (overflow) { *overflow = over; @@ -142,10 +144,10 @@ static void secp256k1_scalar_set_b32(secp256k1_scalar *r, const unsigned char *b } static void secp256k1_scalar_get_b32(unsigned char *bin, const secp256k1_scalar* a) { - bin[0] = a->d[3] >> 56; bin[1] = a->d[3] >> 48; bin[2] = a->d[3] >> 40; bin[3] = a->d[3] >> 32; bin[4] = a->d[3] >> 24; bin[5] = a->d[3] >> 16; bin[6] = a->d[3] >> 8; bin[7] = a->d[3]; - bin[8] = a->d[2] >> 56; bin[9] = a->d[2] >> 48; bin[10] = a->d[2] >> 40; bin[11] = a->d[2] >> 32; bin[12] = a->d[2] >> 24; bin[13] = a->d[2] >> 16; bin[14] = a->d[2] >> 8; bin[15] = a->d[2]; - bin[16] = a->d[1] >> 56; bin[17] = a->d[1] >> 48; bin[18] = a->d[1] >> 40; bin[19] = a->d[1] >> 32; bin[20] = a->d[1] >> 24; bin[21] = a->d[1] >> 16; bin[22] = a->d[1] >> 8; bin[23] = a->d[1]; - bin[24] = a->d[0] >> 56; bin[25] = a->d[0] >> 48; bin[26] = a->d[0] >> 40; bin[27] = a->d[0] >> 32; bin[28] = a->d[0] >> 24; bin[29] = a->d[0] >> 16; bin[30] = a->d[0] >> 8; bin[31] = a->d[0]; + secp256k1_write_be64(&bin[0], a->d[3]); + secp256k1_write_be64(&bin[8], a->d[2]); + secp256k1_write_be64(&bin[16], a->d[1]); + secp256k1_write_be64(&bin[24], a->d[0]); } SECP256K1_INLINE static int secp256k1_scalar_is_zero(const secp256k1_scalar *a) { @@ -188,7 +190,8 @@ static int secp256k1_scalar_is_high(const secp256k1_scalar *a) { static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { /* If we are flag = 0, mask = 00...00 and this is a no-op; * if we are flag = 1, mask = 11...11 and this is identical to secp256k1_scalar_negate */ - uint64_t mask = !flag - 1; + volatile int vflag = flag; + uint64_t mask = -vflag; uint64_t nonzero = (secp256k1_scalar_is_zero(r) != 0) - 1; secp256k1_uint128 t; secp256k1_u128_from_u64(&t, r->d[0] ^ mask); @@ -380,7 +383,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "movq %%r10, %q5\n" /* extract m6 */ "movq %%r8, %q6\n" - : "=g"(m0), "=g"(m1), "=g"(m2), "=g"(m3), "=g"(m4), "=g"(m5), "=g"(m6) + : "=&g"(m0), "=&g"(m1), "=&g"(m2), "=g"(m3), "=g"(m4), "=g"(m5), "=g"(m6) : "S"(l), "i"(SECP256K1_N_C_0), "i"(SECP256K1_N_C_1) : "rax", "rdx", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "cc"); diff --git a/src/secp256k1/src/scalar_8x32_impl.h b/src/secp256k1/src/scalar_8x32_impl.h index a2555cbbcd..80ef3ef248 100644 --- a/src/secp256k1/src/scalar_8x32_impl.h +++ b/src/secp256k1/src/scalar_8x32_impl.h @@ -9,6 +9,7 @@ #include "checkmem.h" #include "modinv32_impl.h" +#include "util.h" /* Limbs of the secp256k1 order. */ #define SECP256K1_N_0 ((uint32_t)0xD0364141UL) @@ -141,8 +142,9 @@ static int secp256k1_scalar_add(secp256k1_scalar *r, const secp256k1_scalar *a, static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int flag) { uint64_t t; + volatile int vflag = flag; VERIFY_CHECK(bit < 256); - bit += ((uint32_t) flag - 1) & 0x100; /* forcing (bit >> 5) > 7 makes this a noop */ + bit += ((uint32_t) vflag - 1) & 0x100; /* forcing (bit >> 5) > 7 makes this a noop */ t = (uint64_t)r->d[0] + (((uint32_t)((bit >> 5) == 0)) << (bit & 0x1F)); r->d[0] = t & 0xFFFFFFFFULL; t >>= 32; t += (uint64_t)r->d[1] + (((uint32_t)((bit >> 5) == 1)) << (bit & 0x1F)); @@ -167,14 +169,14 @@ static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int static void secp256k1_scalar_set_b32(secp256k1_scalar *r, const unsigned char *b32, int *overflow) { int over; - r->d[0] = (uint32_t)b32[31] | (uint32_t)b32[30] << 8 | (uint32_t)b32[29] << 16 | (uint32_t)b32[28] << 24; - r->d[1] = (uint32_t)b32[27] | (uint32_t)b32[26] << 8 | (uint32_t)b32[25] << 16 | (uint32_t)b32[24] << 24; - r->d[2] = (uint32_t)b32[23] | (uint32_t)b32[22] << 8 | (uint32_t)b32[21] << 16 | (uint32_t)b32[20] << 24; - r->d[3] = (uint32_t)b32[19] | (uint32_t)b32[18] << 8 | (uint32_t)b32[17] << 16 | (uint32_t)b32[16] << 24; - r->d[4] = (uint32_t)b32[15] | (uint32_t)b32[14] << 8 | (uint32_t)b32[13] << 16 | (uint32_t)b32[12] << 24; - r->d[5] = (uint32_t)b32[11] | (uint32_t)b32[10] << 8 | (uint32_t)b32[9] << 16 | (uint32_t)b32[8] << 24; - r->d[6] = (uint32_t)b32[7] | (uint32_t)b32[6] << 8 | (uint32_t)b32[5] << 16 | (uint32_t)b32[4] << 24; - r->d[7] = (uint32_t)b32[3] | (uint32_t)b32[2] << 8 | (uint32_t)b32[1] << 16 | (uint32_t)b32[0] << 24; + r->d[0] = secp256k1_read_be32(&b32[28]); + r->d[1] = secp256k1_read_be32(&b32[24]); + r->d[2] = secp256k1_read_be32(&b32[20]); + r->d[3] = secp256k1_read_be32(&b32[16]); + r->d[4] = secp256k1_read_be32(&b32[12]); + r->d[5] = secp256k1_read_be32(&b32[8]); + r->d[6] = secp256k1_read_be32(&b32[4]); + r->d[7] = secp256k1_read_be32(&b32[0]); over = secp256k1_scalar_reduce(r, secp256k1_scalar_check_overflow(r)); if (overflow) { *overflow = over; @@ -182,14 +184,14 @@ static void secp256k1_scalar_set_b32(secp256k1_scalar *r, const unsigned char *b } static void secp256k1_scalar_get_b32(unsigned char *bin, const secp256k1_scalar* a) { - bin[0] = a->d[7] >> 24; bin[1] = a->d[7] >> 16; bin[2] = a->d[7] >> 8; bin[3] = a->d[7]; - bin[4] = a->d[6] >> 24; bin[5] = a->d[6] >> 16; bin[6] = a->d[6] >> 8; bin[7] = a->d[6]; - bin[8] = a->d[5] >> 24; bin[9] = a->d[5] >> 16; bin[10] = a->d[5] >> 8; bin[11] = a->d[5]; - bin[12] = a->d[4] >> 24; bin[13] = a->d[4] >> 16; bin[14] = a->d[4] >> 8; bin[15] = a->d[4]; - bin[16] = a->d[3] >> 24; bin[17] = a->d[3] >> 16; bin[18] = a->d[3] >> 8; bin[19] = a->d[3]; - bin[20] = a->d[2] >> 24; bin[21] = a->d[2] >> 16; bin[22] = a->d[2] >> 8; bin[23] = a->d[2]; - bin[24] = a->d[1] >> 24; bin[25] = a->d[1] >> 16; bin[26] = a->d[1] >> 8; bin[27] = a->d[1]; - bin[28] = a->d[0] >> 24; bin[29] = a->d[0] >> 16; bin[30] = a->d[0] >> 8; bin[31] = a->d[0]; + secp256k1_write_be32(&bin[0], a->d[7]); + secp256k1_write_be32(&bin[4], a->d[6]); + secp256k1_write_be32(&bin[8], a->d[5]); + secp256k1_write_be32(&bin[12], a->d[4]); + secp256k1_write_be32(&bin[16], a->d[3]); + secp256k1_write_be32(&bin[20], a->d[2]); + secp256k1_write_be32(&bin[24], a->d[1]); + secp256k1_write_be32(&bin[28], a->d[0]); } SECP256K1_INLINE static int secp256k1_scalar_is_zero(const secp256k1_scalar *a) { @@ -241,7 +243,8 @@ static int secp256k1_scalar_is_high(const secp256k1_scalar *a) { static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { /* If we are flag = 0, mask = 00...00 and this is a no-op; * if we are flag = 1, mask = 11...11 and this is identical to secp256k1_scalar_negate */ - uint32_t mask = !flag - 1; + volatile int vflag = flag; + uint32_t mask = -vflag; uint32_t nonzero = 0xFFFFFFFFUL * (secp256k1_scalar_is_zero(r) == 0); uint64_t t = (uint64_t)(r->d[0] ^ mask) + ((SECP256K1_N_0 + 1) & mask); r->d[0] = t & nonzero; t >>= 32; diff --git a/src/secp256k1/src/scalar_low_impl.h b/src/secp256k1/src/scalar_low_impl.h index bfd1139110..428a5deb33 100644 --- a/src/secp256k1/src/scalar_low_impl.h +++ b/src/secp256k1/src/scalar_low_impl.h @@ -9,6 +9,7 @@ #include "checkmem.h" #include "scalar.h" +#include "util.h" #include <string.h> diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index 7af333ca90..4c11e7f0b8 100644 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -247,8 +247,8 @@ static int secp256k1_pubkey_load(const secp256k1_context* ctx, secp256k1_ge* ge, } else { /* Otherwise, fall back to 32-byte big endian for X and Y. */ secp256k1_fe x, y; - secp256k1_fe_set_b32(&x, pubkey->data); - secp256k1_fe_set_b32(&y, pubkey->data + 32); + ARG_CHECK(secp256k1_fe_set_b32_limit(&x, pubkey->data)); + ARG_CHECK(secp256k1_fe_set_b32_limit(&y, pubkey->data + 32)); secp256k1_ge_set_xy(ge, &x, &y); } ARG_CHECK(!secp256k1_fe_is_zero(&ge->x)); @@ -811,3 +811,7 @@ int secp256k1_tagged_sha256(const secp256k1_context* ctx, unsigned char *hash32, #ifdef ENABLE_MODULE_SCHNORRSIG # include "modules/schnorrsig/main_impl.h" #endif + +#ifdef ENABLE_MODULE_ELLSWIFT +# include "modules/ellswift/main_impl.h" +#endif diff --git a/src/secp256k1/src/testrand.h b/src/secp256k1/src/testrand.h index d109bb9f8b..721099d039 100644 --- a/src/secp256k1/src/testrand.h +++ b/src/secp256k1/src/testrand.h @@ -7,6 +7,8 @@ #ifndef SECP256K1_TESTRAND_H #define SECP256K1_TESTRAND_H +#include "util.h" + /* A non-cryptographic RNG used only for test infrastructure. */ /** Seed the pseudorandom number generator for testing. */ diff --git a/src/secp256k1/src/testrand_impl.h b/src/secp256k1/src/testrand_impl.h index e9b9d7ded4..1b7481a53b 100644 --- a/src/secp256k1/src/testrand_impl.h +++ b/src/secp256k1/src/testrand_impl.h @@ -13,6 +13,7 @@ #include "testrand.h" #include "hash.h" +#include "util.h" static uint64_t secp256k1_test_state[4]; static uint64_t secp256k1_test_rng_integer; diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index 7f61f737c6..8ada3f869b 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -10,7 +10,15 @@ #include <time.h> +#ifdef USE_EXTERNAL_DEFAULT_CALLBACKS + #pragma message("Ignoring USE_EXTERNAL_CALLBACKS in tests.") + #undef USE_EXTERNAL_DEFAULT_CALLBACKS +#endif +#if defined(VERIFY) && defined(COVERAGE) + #pragma message("Defining VERIFY for tests being built for coverage analysis support is meaningless.") +#endif #include "secp256k1.c" + #include "../include/secp256k1.h" #include "../include/secp256k1_preallocated.h" #include "testrand_impl.h" @@ -85,7 +93,7 @@ static void random_field_element_test(secp256k1_fe *fe) { do { unsigned char b32[32]; secp256k1_testrand256_test(b32); - if (secp256k1_fe_set_b32(fe, b32)) { + if (secp256k1_fe_set_b32_limit(fe, b32)) { break; } } while(1); @@ -689,6 +697,17 @@ static void run_sha256_counter_tests(void) { } } +/* Tests for the equality of two sha256 structs. This function only produces a + * correct result if an integer multiple of 64 many bytes have been written + * into the hash functions. This function is used by some module tests. */ +static void test_sha256_eq(const secp256k1_sha256 *sha1, const secp256k1_sha256 *sha2) { + /* Is buffer fully consumed? */ + CHECK((sha1->bytes & 0x3F) == 0); + + CHECK(sha1->bytes == sha2->bytes); + CHECK(secp256k1_memcmp_var(sha1->s, sha2->s, sizeof(sha1->s)) == 0); +} + static void run_hmac_sha256_tests(void) { static const char *keys[6] = { "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", @@ -2221,7 +2240,7 @@ static void scalar_test(void) { for (i = 0; i < 100; ++i) { int low; int shift = 1 + secp256k1_testrand_int(15); - int expected = r.d[0] % (1 << shift); + int expected = r.d[0] % (1ULL << shift); low = secp256k1_scalar_shr_int(&r, shift); CHECK(expected == low); } @@ -2299,26 +2318,23 @@ static void scalar_test(void) { { /* Test multiplicative identity. */ - secp256k1_scalar r1, v1; - secp256k1_scalar_set_int(&v1,1); - secp256k1_scalar_mul(&r1, &s1, &v1); + secp256k1_scalar r1; + secp256k1_scalar_mul(&r1, &s1, &secp256k1_scalar_one); CHECK(secp256k1_scalar_eq(&r1, &s1)); } { /* Test additive identity. */ - secp256k1_scalar r1, v0; - secp256k1_scalar_set_int(&v0,0); - secp256k1_scalar_add(&r1, &s1, &v0); + secp256k1_scalar r1; + secp256k1_scalar_add(&r1, &s1, &secp256k1_scalar_zero); CHECK(secp256k1_scalar_eq(&r1, &s1)); } { /* Test zero product property. */ - secp256k1_scalar r1, v0; - secp256k1_scalar_set_int(&v0,0); - secp256k1_scalar_mul(&r1, &s1, &v0); - CHECK(secp256k1_scalar_eq(&r1, &v0)); + secp256k1_scalar r1; + secp256k1_scalar_mul(&r1, &s1, &secp256k1_scalar_zero); + CHECK(secp256k1_scalar_eq(&r1, &secp256k1_scalar_zero)); } } @@ -2350,12 +2366,24 @@ static void run_scalar_tests(void) { } { + /* Check that the scalar constants secp256k1_scalar_zero and + secp256k1_scalar_one contain the expected values. */ + secp256k1_scalar zero, one; + + CHECK(secp256k1_scalar_is_zero(&secp256k1_scalar_zero)); + secp256k1_scalar_set_int(&zero, 0); + CHECK(secp256k1_scalar_eq(&zero, &secp256k1_scalar_zero)); + + CHECK(secp256k1_scalar_is_one(&secp256k1_scalar_one)); + secp256k1_scalar_set_int(&one, 1); + CHECK(secp256k1_scalar_eq(&one, &secp256k1_scalar_one)); + } + + { /* (-1)+1 should be zero. */ - secp256k1_scalar s, o; - secp256k1_scalar_set_int(&s, 1); - CHECK(secp256k1_scalar_is_one(&s)); - secp256k1_scalar_negate(&o, &s); - secp256k1_scalar_add(&o, &o, &s); + secp256k1_scalar o; + secp256k1_scalar_negate(&o, &secp256k1_scalar_one); + secp256k1_scalar_add(&o, &o, &secp256k1_scalar_one); CHECK(secp256k1_scalar_is_zero(&o)); secp256k1_scalar_negate(&o, &o); CHECK(secp256k1_scalar_is_zero(&o)); @@ -2380,7 +2408,6 @@ static void run_scalar_tests(void) { secp256k1_scalar y; secp256k1_scalar z; secp256k1_scalar zz; - secp256k1_scalar one; secp256k1_scalar r1; secp256k1_scalar r2; secp256k1_scalar zzv; @@ -2917,7 +2944,6 @@ static void run_scalar_tests(void) { 0x1e, 0x86, 0x5d, 0x89, 0x63, 0xe6, 0x0a, 0x46, 0x5c, 0x02, 0x97, 0x1b, 0x62, 0x43, 0x86, 0xf5}} }; - secp256k1_scalar_set_int(&one, 1); for (i = 0; i < 33; i++) { secp256k1_scalar_set_b32(&x, chal[i][0], &overflow); CHECK(!overflow); @@ -2940,7 +2966,7 @@ static void run_scalar_tests(void) { CHECK(secp256k1_scalar_eq(&x, &z)); secp256k1_scalar_mul(&zz, &zz, &y); CHECK(!secp256k1_scalar_check_overflow(&zz)); - CHECK(secp256k1_scalar_eq(&one, &zz)); + CHECK(secp256k1_scalar_eq(&secp256k1_scalar_one, &zz)); } } } @@ -2952,7 +2978,7 @@ static void random_fe(secp256k1_fe *x) { unsigned char bin[32]; do { secp256k1_testrand256(bin); - if (secp256k1_fe_set_b32(x, bin)) { + if (secp256k1_fe_set_b32_limit(x, bin)) { return; } } while(1); @@ -2962,7 +2988,7 @@ static void random_fe_test(secp256k1_fe *x) { unsigned char bin[32]; do { secp256k1_testrand256_test(bin); - if (secp256k1_fe_set_b32(x, bin)) { + if (secp256k1_fe_set_b32_limit(x, bin)) { return; } } while(1); @@ -3016,7 +3042,7 @@ static void run_field_convert(void) { unsigned char b322[32]; secp256k1_fe_storage fes2; /* Check conversions to fe. */ - CHECK(secp256k1_fe_set_b32(&fe2, b32)); + CHECK(secp256k1_fe_set_b32_limit(&fe2, b32)); CHECK(secp256k1_fe_equal_var(&fe, &fe2)); secp256k1_fe_from_storage(&fe2, &fes); CHECK(secp256k1_fe_equal_var(&fe, &fe2)); @@ -3027,13 +3053,75 @@ static void run_field_convert(void) { CHECK(secp256k1_memcmp_var(&fes2, &fes, sizeof(fes)) == 0); } +static void run_field_be32_overflow(void) { + { + static const unsigned char zero_overflow[32] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFC, 0x2F, + }; + static const unsigned char zero[32] = { 0x00 }; + unsigned char out[32]; + secp256k1_fe fe; + CHECK(secp256k1_fe_set_b32_limit(&fe, zero_overflow) == 0); + secp256k1_fe_set_b32_mod(&fe, zero_overflow); + CHECK(secp256k1_fe_normalizes_to_zero(&fe) == 1); + secp256k1_fe_normalize(&fe); + CHECK(secp256k1_fe_is_zero(&fe) == 1); + secp256k1_fe_get_b32(out, &fe); + CHECK(secp256k1_memcmp_var(out, zero, 32) == 0); + } + { + static const unsigned char one_overflow[32] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFC, 0x30, + }; + static const unsigned char one[32] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + }; + unsigned char out[32]; + secp256k1_fe fe; + CHECK(secp256k1_fe_set_b32_limit(&fe, one_overflow) == 0); + secp256k1_fe_set_b32_mod(&fe, one_overflow); + secp256k1_fe_normalize(&fe); + CHECK(secp256k1_fe_cmp_var(&fe, &secp256k1_fe_one) == 0); + secp256k1_fe_get_b32(out, &fe); + CHECK(secp256k1_memcmp_var(out, one, 32) == 0); + } + { + static const unsigned char ff_overflow[32] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + }; + static const unsigned char ff[32] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x03, 0xD0, + }; + unsigned char out[32]; + secp256k1_fe fe; + const secp256k1_fe fe_ff = SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0x01, 0x000003d0); + CHECK(secp256k1_fe_set_b32_limit(&fe, ff_overflow) == 0); + secp256k1_fe_set_b32_mod(&fe, ff_overflow); + secp256k1_fe_normalize(&fe); + CHECK(secp256k1_fe_cmp_var(&fe, &fe_ff) == 0); + secp256k1_fe_get_b32(out, &fe); + CHECK(secp256k1_memcmp_var(out, ff, 32) == 0); + } +} + /* Returns true if two field elements have the same representation. */ static int fe_identical(const secp256k1_fe *a, const secp256k1_fe *b) { int ret = 1; -#ifdef VERIFY - ret &= (a->magnitude == b->magnitude); - ret &= (a->normalized == b->normalized); -#endif /* Compare the struct member that holds the limbs. */ ret &= (secp256k1_memcmp_var(a->n, b->n, sizeof(a->n)) == 0); return ret; @@ -3121,16 +3209,22 @@ static void run_field_misc(void) { q = x; secp256k1_fe_cmov(&x, &z, 0); #ifdef VERIFY - CHECK(x.normalized && x.magnitude == 1); + CHECK(!x.normalized); + CHECK((x.magnitude == q.magnitude) || (x.magnitude == z.magnitude)); + CHECK((x.magnitude >= q.magnitude) && (x.magnitude >= z.magnitude)); #endif + x = q; secp256k1_fe_cmov(&x, &x, 1); CHECK(!fe_identical(&x, &z)); CHECK(fe_identical(&x, &q)); secp256k1_fe_cmov(&q, &z, 1); #ifdef VERIFY - CHECK(!q.normalized && q.magnitude == z.magnitude); + CHECK(!q.normalized); + CHECK((q.magnitude == x.magnitude) || (q.magnitude == z.magnitude)); + CHECK((q.magnitude >= x.magnitude) && (q.magnitude >= z.magnitude)); #endif CHECK(fe_identical(&q, &z)); + q = z; secp256k1_fe_normalize_var(&x); secp256k1_fe_normalize_var(&z); CHECK(!secp256k1_fe_equal_var(&x, &z)); @@ -3144,7 +3238,7 @@ static void run_field_misc(void) { secp256k1_fe_normalize_var(&q); secp256k1_fe_cmov(&q, &z, (j&1)); #ifdef VERIFY - CHECK((q.normalized != (j&1)) && q.magnitude == ((j&1) ? z.magnitude : 1)); + CHECK(!q.normalized && q.magnitude == z.magnitude); #endif } secp256k1_fe_normalize_var(&z); @@ -3605,7 +3699,7 @@ static void run_inverse_tests(void) b32[31] = i & 0xff; b32[30] = (i >> 8) & 0xff; secp256k1_scalar_set_b32(&x_scalar, b32, NULL); - secp256k1_fe_set_b32(&x_fe, b32); + secp256k1_fe_set_b32_mod(&x_fe, b32); for (var = 0; var <= 1; ++var) { test_inverse_scalar(NULL, &x_scalar, var); test_inverse_field(NULL, &x_fe, var); @@ -3622,7 +3716,7 @@ static void run_inverse_tests(void) for (i = 0; i < 64 * COUNT; ++i) { (testrand ? secp256k1_testrand256_test : secp256k1_testrand256)(b32); secp256k1_scalar_set_b32(&x_scalar, b32, NULL); - secp256k1_fe_set_b32(&x_fe, b32); + secp256k1_fe_set_b32_mod(&x_fe, b32); for (var = 0; var <= 1; ++var) { test_inverse_scalar(NULL, &x_scalar, var); test_inverse_field(NULL, &x_fe, var); @@ -3692,7 +3786,7 @@ static void test_ge(void) { */ secp256k1_ge *ge = (secp256k1_ge *)checked_malloc(&CTX->error_callback, sizeof(secp256k1_ge) * (1 + 4 * runs)); secp256k1_gej *gej = (secp256k1_gej *)checked_malloc(&CTX->error_callback, sizeof(secp256k1_gej) * (1 + 4 * runs)); - secp256k1_fe zf; + secp256k1_fe zf, r; secp256k1_fe zfi2, zfi3; secp256k1_gej_set_infinity(&gej[0]); @@ -3734,6 +3828,11 @@ static void test_ge(void) { secp256k1_fe_sqr(&zfi2, &zfi3); secp256k1_fe_mul(&zfi3, &zfi3, &zfi2); + /* Generate random r */ + do { + random_field_element_test(&r); + } while(secp256k1_fe_is_zero(&r)); + for (i1 = 0; i1 < 1 + 4 * runs; i1++) { int i2; for (i2 = 0; i2 < 1 + 4 * runs; i2++) { @@ -3846,6 +3945,29 @@ static void test_ge(void) { free(ge_set_all); } + /* Test that all elements have X coordinates on the curve. */ + for (i = 1; i < 4 * runs + 1; i++) { + secp256k1_fe n; + CHECK(secp256k1_ge_x_on_curve_var(&ge[i].x)); + /* And the same holds after random rescaling. */ + secp256k1_fe_mul(&n, &zf, &ge[i].x); + CHECK(secp256k1_ge_x_frac_on_curve_var(&n, &zf)); + } + + /* Test correspondence of secp256k1_ge_x{,_frac}_on_curve_var with ge_set_xo. */ + { + secp256k1_fe n; + secp256k1_ge q; + int ret_on_curve, ret_frac_on_curve, ret_set_xo; + secp256k1_fe_mul(&n, &zf, &r); + ret_on_curve = secp256k1_ge_x_on_curve_var(&r); + ret_frac_on_curve = secp256k1_ge_x_frac_on_curve_var(&n, &zf); + ret_set_xo = secp256k1_ge_set_xo_var(&q, &r, 0); + CHECK(ret_on_curve == ret_frac_on_curve); + CHECK(ret_on_curve == ret_set_xo); + if (ret_set_xo) CHECK(secp256k1_fe_equal_var(&r, &q.x)); + } + /* Test batch gej -> ge conversion with many infinities. */ for (i = 0; i < 4 * runs + 1; i++) { int odd; @@ -4338,9 +4460,9 @@ static void test_ecmult_target(const secp256k1_scalar* target, int mode) { secp256k1_ecmult(&p2j, &pj, &n2, &zero); secp256k1_ecmult(&ptj, &pj, target, &zero); } else { - secp256k1_ecmult_const(&p1j, &p, &n1, 256); - secp256k1_ecmult_const(&p2j, &p, &n2, 256); - secp256k1_ecmult_const(&ptj, &p, target, 256); + secp256k1_ecmult_const(&p1j, &p, &n1); + secp256k1_ecmult_const(&p2j, &p, &n2); + secp256k1_ecmult_const(&ptj, &p, target); } /* Add them all up: n1*P + n2*P + target*P = (n1+n2+target)*P = (n1+n1-n1-n2)*P = 0. */ @@ -4403,7 +4525,7 @@ static void ecmult_const_random_mult(void) { 0xb84e4e1b, 0xfb77e21f, 0x96baae2a, 0x63dec956 ); secp256k1_gej b; - secp256k1_ecmult_const(&b, &a, &xn, 256); + secp256k1_ecmult_const(&b, &a, &xn); CHECK(secp256k1_ge_is_valid_var(&a)); ge_equals_gej(&expected_b, &b); @@ -4419,12 +4541,12 @@ static void ecmult_const_commutativity(void) { random_scalar_order_test(&a); random_scalar_order_test(&b); - secp256k1_ecmult_const(&res1, &secp256k1_ge_const_g, &a, 256); - secp256k1_ecmult_const(&res2, &secp256k1_ge_const_g, &b, 256); + secp256k1_ecmult_const(&res1, &secp256k1_ge_const_g, &a); + secp256k1_ecmult_const(&res2, &secp256k1_ge_const_g, &b); secp256k1_ge_set_gej(&mid1, &res1); secp256k1_ge_set_gej(&mid2, &res2); - secp256k1_ecmult_const(&res1, &mid1, &b, 256); - secp256k1_ecmult_const(&res2, &mid2, &a, 256); + secp256k1_ecmult_const(&res1, &mid1, &b); + secp256k1_ecmult_const(&res2, &mid2, &a); secp256k1_ge_set_gej(&mid1, &res1); secp256k1_ge_set_gej(&mid2, &res2); ge_equals_ge(&mid1, &mid2); @@ -4440,13 +4562,13 @@ static void ecmult_const_mult_zero_one(void) { secp256k1_scalar_negate(&negone, &one); random_group_element_test(&point); - secp256k1_ecmult_const(&res1, &point, &zero, 3); + secp256k1_ecmult_const(&res1, &point, &zero); secp256k1_ge_set_gej(&res2, &res1); CHECK(secp256k1_ge_is_infinity(&res2)); - secp256k1_ecmult_const(&res1, &point, &one, 2); + secp256k1_ecmult_const(&res1, &point, &one); secp256k1_ge_set_gej(&res2, &res1); ge_equals_ge(&res2, &point); - secp256k1_ecmult_const(&res1, &point, &negone, 256); + secp256k1_ecmult_const(&res1, &point, &negone); secp256k1_gej_neg(&res1, &res1); secp256k1_ge_set_gej(&res2, &res1); ge_equals_ge(&res2, &point); @@ -4476,7 +4598,7 @@ static void ecmult_const_mult_xonly(void) { n = base.x; } /* Perform x-only multiplication. */ - res = secp256k1_ecmult_const_xonly(&resx, &n, (i & 1) ? &d : NULL, &q, 256, i & 2); + res = secp256k1_ecmult_const_xonly(&resx, &n, (i & 1) ? &d : NULL, &q, i & 2); CHECK(res); /* Perform normal multiplication. */ secp256k1_gej_set_ge(&basej, &base); @@ -4498,7 +4620,7 @@ static void ecmult_const_mult_xonly(void) { random_field_element_test(&x); secp256k1_fe_sqr(&c, &x); secp256k1_fe_mul(&c, &c, &x); - secp256k1_fe_add(&c, &secp256k1_fe_const_b); + secp256k1_fe_add_int(&c, SECP256K1_B); } while (secp256k1_fe_is_square_var(&c)); /* If i is odd, n=d*x for random non-zero d. */ if (i & 1) { @@ -4509,7 +4631,7 @@ static void ecmult_const_mult_xonly(void) { } else { n = x; } - res = secp256k1_ecmult_const_xonly(&r, &n, (i & 1) ? &d : NULL, &q, 256, 0); + res = secp256k1_ecmult_const_xonly(&r, &n, (i & 1) ? &d : NULL, &q, 0); CHECK(res == 0); } } @@ -4534,7 +4656,7 @@ static void ecmult_const_chain_multiply(void) { for (i = 0; i < 100; ++i) { secp256k1_ge tmp; secp256k1_ge_set_gej(&tmp, &point); - secp256k1_ecmult_const(&point, &tmp, &scalar, 256); + secp256k1_ecmult_const(&point, &tmp, &scalar); } secp256k1_ge_set_gej(&res, &point); ge_equals_gej(&res, &expected_point); @@ -4570,7 +4692,6 @@ static int ecmult_multi_false_callback(secp256k1_scalar *sc, secp256k1_ge *pt, s static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func ecmult_multi) { int ncount; - secp256k1_scalar szero; secp256k1_scalar sc[32]; secp256k1_ge pt[32]; secp256k1_gej r; @@ -4579,7 +4700,6 @@ static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi data.sc = sc; data.pt = pt; - secp256k1_scalar_set_int(&szero, 0); /* No points to multiply */ CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, NULL, ecmult_multi_callback, &data, 0)); @@ -4597,21 +4717,21 @@ static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi pt[1] = secp256k1_ge_const_g; /* only G scalar */ - secp256k1_ecmult(&r2, &ptgj, &szero, &sc[0]); + secp256k1_ecmult(&r2, &ptgj, &secp256k1_scalar_zero, &sc[0]); CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &sc[0], ecmult_multi_callback, &data, 0)); CHECK(secp256k1_gej_eq_var(&r, &r2)); /* 1-point */ - secp256k1_ecmult(&r2, &ptgj, &sc[0], &szero); - CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, 1)); + secp256k1_ecmult(&r2, &ptgj, &sc[0], &secp256k1_scalar_zero); + CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, 1)); CHECK(secp256k1_gej_eq_var(&r, &r2)); /* Try to multiply 1 point, but callback returns false */ - CHECK(!ecmult_multi(&CTX->error_callback, scratch, &r, &szero, ecmult_multi_false_callback, &data, 1)); + CHECK(!ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_false_callback, &data, 1)); /* 2-point */ secp256k1_ecmult(&r2, &ptgj, &sc[0], &sc[1]); - CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, 2)); + CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, 2)); CHECK(secp256k1_gej_eq_var(&r, &r2)); /* 2-point with G scalar */ @@ -4631,7 +4751,7 @@ static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi random_scalar_order(&sc[i]); secp256k1_ge_set_infinity(&pt[i]); } - CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, sizes[j])); + CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, sizes[j])); CHECK(secp256k1_gej_is_infinity(&r)); } @@ -4641,7 +4761,7 @@ static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi pt[i] = ptg; secp256k1_scalar_set_int(&sc[i], 0); } - CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, sizes[j])); + CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, sizes[j])); CHECK(secp256k1_gej_is_infinity(&r)); } @@ -4654,7 +4774,7 @@ static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi pt[2 * i + 1] = ptg; } - CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, sizes[j])); + CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, sizes[j])); CHECK(secp256k1_gej_is_infinity(&r)); random_scalar_order(&sc[0]); @@ -4667,7 +4787,7 @@ static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi secp256k1_ge_neg(&pt[2*i+1], &pt[2*i]); } - CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, sizes[j])); + CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, sizes[j])); CHECK(secp256k1_gej_is_infinity(&r)); } @@ -4682,7 +4802,7 @@ static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi secp256k1_scalar_negate(&sc[i], &sc[i]); } - CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, 32)); + CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, 32)); CHECK(secp256k1_gej_is_infinity(&r)); } @@ -4700,8 +4820,8 @@ static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi secp256k1_gej_add_ge_var(&r, &r, &pt[i], NULL); } - secp256k1_ecmult(&r2, &r, &sc[0], &szero); - CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, 20)); + secp256k1_ecmult(&r2, &r, &sc[0], &secp256k1_scalar_zero); + CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, 20)); CHECK(secp256k1_gej_eq_var(&r, &r2)); } @@ -4721,8 +4841,8 @@ static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi } secp256k1_gej_set_ge(&p0j, &pt[0]); - secp256k1_ecmult(&r2, &p0j, &rs, &szero); - CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, 20)); + secp256k1_ecmult(&r2, &p0j, &rs, &secp256k1_scalar_zero); + CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, 20)); CHECK(secp256k1_gej_eq_var(&r, &r2)); } @@ -4733,13 +4853,13 @@ static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi } secp256k1_scalar_clear(&sc[0]); - CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, 20)); + CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, 20)); secp256k1_scalar_clear(&sc[1]); secp256k1_scalar_clear(&sc[2]); secp256k1_scalar_clear(&sc[3]); secp256k1_scalar_clear(&sc[4]); - CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, 6)); - CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, 5)); + CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, 6)); + CHECK(ecmult_multi(&CTX->error_callback, scratch, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, 5)); CHECK(secp256k1_gej_is_infinity(&r)); /* Run through s0*(t0*P) + s1*(t1*P) exhaustively for many small values of s0, s1, t0, t1 */ @@ -4763,8 +4883,8 @@ static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi secp256k1_scalar_set_int(&t1, (t1i + 1) / 2); secp256k1_scalar_cond_negate(&t1, t1i & 1); - secp256k1_ecmult(&t0p, &ptgj, &t0, &szero); - secp256k1_ecmult(&t1p, &ptgj, &t1, &szero); + secp256k1_ecmult(&t0p, &ptgj, &t0, &secp256k1_scalar_zero); + secp256k1_ecmult(&t1p, &ptgj, &t1, &secp256k1_scalar_zero); for(s0i = 0; s0i < TOP; s0i++) { for(s1i = 0; s1i < TOP; s1i++) { @@ -4783,8 +4903,8 @@ static void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi secp256k1_scalar_mul(&tmp2, &t1, &sc[1]); secp256k1_scalar_add(&tmp1, &tmp1, &tmp2); - secp256k1_ecmult(&expected, &ptgj, &tmp1, &szero); - CHECK(ecmult_multi(&CTX->error_callback, scratch, &actual, &szero, ecmult_multi_callback, &data, 2)); + secp256k1_ecmult(&expected, &ptgj, &tmp1, &secp256k1_scalar_zero); + CHECK(ecmult_multi(&CTX->error_callback, scratch, &actual, &secp256k1_scalar_zero, ecmult_multi_callback, &data, 2)); CHECK(secp256k1_gej_eq_var(&actual, &expected)); } } @@ -4960,7 +5080,6 @@ static int test_ecmult_multi_random(secp256k1_scratch *scratch) { } static void test_ecmult_multi_batch_single(secp256k1_ecmult_multi_func ecmult_multi) { - secp256k1_scalar szero; secp256k1_scalar sc; secp256k1_ge pt; secp256k1_gej r; @@ -4971,11 +5090,10 @@ static void test_ecmult_multi_batch_single(secp256k1_ecmult_multi_func ecmult_mu random_scalar_order(&sc); data.sc = ≻ data.pt = &pt; - secp256k1_scalar_set_int(&szero, 0); /* Try to multiply 1 point, but scratch space is empty.*/ scratch_empty = secp256k1_scratch_create(&CTX->error_callback, 0); - CHECK(!ecmult_multi(&CTX->error_callback, scratch_empty, &r, &szero, ecmult_multi_callback, &data, 1)); + CHECK(!ecmult_multi(&CTX->error_callback, scratch_empty, &r, &secp256k1_scalar_zero, ecmult_multi_callback, &data, 1)); secp256k1_scratch_destroy(&CTX->error_callback, scratch_empty); } @@ -5083,7 +5201,6 @@ static void test_ecmult_multi_batch_size_helper(void) { static void test_ecmult_multi_batching(void) { static const int n_points = 2*ECMULT_PIPPENGER_THRESHOLD; secp256k1_scalar scG; - secp256k1_scalar szero; secp256k1_scalar *sc = (secp256k1_scalar *)checked_malloc(&CTX->error_callback, sizeof(secp256k1_scalar) * n_points); secp256k1_ge *pt = (secp256k1_ge *)checked_malloc(&CTX->error_callback, sizeof(secp256k1_ge) * n_points); secp256k1_gej r; @@ -5093,11 +5210,10 @@ static void test_ecmult_multi_batching(void) { secp256k1_scratch *scratch; secp256k1_gej_set_infinity(&r2); - secp256k1_scalar_set_int(&szero, 0); /* Get random scalars and group elements and compute result */ random_scalar_order(&scG); - secp256k1_ecmult(&r2, &r2, &szero, &scG); + secp256k1_ecmult(&r2, &r2, &secp256k1_scalar_zero, &scG); for(i = 0; i < n_points; i++) { secp256k1_ge ptg; secp256k1_gej ptgj; @@ -5432,7 +5548,7 @@ static void test_ecmult_accumulate(secp256k1_sha256* acc, const secp256k1_scalar secp256k1_ecmult(&rj3, &infj, &zero, x); secp256k1_ecmult_multi_var(NULL, scratch, &rj4, x, NULL, NULL, 0); secp256k1_ecmult_multi_var(NULL, scratch, &rj5, &zero, test_ecmult_accumulate_cb, (void*)x, 1); - secp256k1_ecmult_const(&rj6, &secp256k1_ge_const_g, x, 256); + secp256k1_ecmult_const(&rj6, &secp256k1_ge_const_g, x); secp256k1_ge_set_gej_var(&r, &rj1); ge_equals_gej(&r, &rj2); ge_equals_gej(&r, &rj3); @@ -7423,6 +7539,10 @@ static void run_ecdsa_wycheproof(void) { # include "modules/schnorrsig/tests_impl.h" #endif +#ifdef ENABLE_MODULE_ELLSWIFT +# include "modules/ellswift/tests_impl.h" +#endif + static void run_secp256k1_memczero_test(void) { unsigned char buf1[6] = {1, 2, 3, 4, 5, 6}; unsigned char buf2[sizeof(buf1)]; @@ -7439,16 +7559,31 @@ static void run_secp256k1_memczero_test(void) { } static void run_secp256k1_byteorder_tests(void) { - const uint32_t x = 0xFF03AB45; - const unsigned char x_be[4] = {0xFF, 0x03, 0xAB, 0x45}; - unsigned char buf[4]; - uint32_t x_; + { + const uint32_t x = 0xFF03AB45; + const unsigned char x_be[4] = {0xFF, 0x03, 0xAB, 0x45}; + unsigned char buf[4]; + uint32_t x_; + + secp256k1_write_be32(buf, x); + CHECK(secp256k1_memcmp_var(buf, x_be, sizeof(buf)) == 0); + + x_ = secp256k1_read_be32(buf); + CHECK(x == x_); + } + + { + const uint64_t x = 0xCAFE0123BEEF4567; + const unsigned char x_be[8] = {0xCA, 0xFE, 0x01, 0x23, 0xBE, 0xEF, 0x45, 0x67}; + unsigned char buf[8]; + uint64_t x_; - secp256k1_write_be32(buf, x); - CHECK(secp256k1_memcmp_var(buf, x_be, sizeof(buf)) == 0); + secp256k1_write_be64(buf, x); + CHECK(secp256k1_memcmp_var(buf, x_be, sizeof(buf)) == 0); - x_ = secp256k1_read_be32(buf); - CHECK(x == x_); + x_ = secp256k1_read_be64(buf); + CHECK(x == x_); + } } static void int_cmov_test(void) { @@ -7487,23 +7622,23 @@ static void fe_cmov_test(void) { secp256k1_fe a = zero; secp256k1_fe_cmov(&r, &a, 0); - CHECK(secp256k1_memcmp_var(&r, &max, sizeof(r)) == 0); + CHECK(fe_identical(&r, &max)); r = zero; a = max; secp256k1_fe_cmov(&r, &a, 1); - CHECK(secp256k1_memcmp_var(&r, &max, sizeof(r)) == 0); + CHECK(fe_identical(&r, &max)); a = zero; secp256k1_fe_cmov(&r, &a, 1); - CHECK(secp256k1_memcmp_var(&r, &zero, sizeof(r)) == 0); + CHECK(fe_identical(&r, &zero)); a = one; secp256k1_fe_cmov(&r, &a, 1); - CHECK(secp256k1_memcmp_var(&r, &one, sizeof(r)) == 0); + CHECK(fe_identical(&r, &one)); r = one; a = zero; secp256k1_fe_cmov(&r, &a, 0); - CHECK(secp256k1_memcmp_var(&r, &one, sizeof(r)) == 0); + CHECK(fe_identical(&r, &one)); } static void fe_storage_cmov_test(void) { @@ -7693,6 +7828,7 @@ int main(int argc, char **argv) { run_field_half(); run_field_misc(); run_field_convert(); + run_field_be32_overflow(); run_fe_mul(); run_sqr(); run_sqrt(); @@ -7754,6 +7890,10 @@ int main(int argc, char **argv) { run_schnorrsig_tests(); #endif +#ifdef ENABLE_MODULE_ELLSWIFT + run_ellswift_tests(); +#endif + /* util tests */ run_secp256k1_memczero_test(); run_secp256k1_byteorder_tests(); diff --git a/src/secp256k1/src/tests_exhaustive.c b/src/secp256k1/src/tests_exhaustive.c index 63b6ef03af..d35acdd58e 100644 --- a/src/secp256k1/src/tests_exhaustive.c +++ b/src/secp256k1/src/tests_exhaustive.c @@ -13,13 +13,19 @@ #define EXHAUSTIVE_TEST_ORDER 13 #endif +#ifdef USE_EXTERNAL_DEFAULT_CALLBACKS + #pragma message("Ignoring USE_EXTERNAL_CALLBACKS in exhaustive_tests.") + #undef USE_EXTERNAL_DEFAULT_CALLBACKS +#endif #include "secp256k1.c" + #include "../include/secp256k1.h" #include "assumptions.h" #include "group.h" #include "testrand_impl.h" #include "ecmult_compute_table_impl.h" #include "ecmult_gen_compute_table_impl.h" +#include "util.h" static int count = 2; @@ -54,7 +60,7 @@ static void random_fe(secp256k1_fe *x) { unsigned char bin[32]; do { secp256k1_testrand256(bin); - if (secp256k1_fe_set_b32(x, bin)) { + if (secp256k1_fe_set_b32_limit(x, bin)) { return; } } while(1); @@ -192,7 +198,7 @@ static void test_exhaustive_ecmult(const secp256k1_ge *group, const secp256k1_ge } for (j = 0; j < EXHAUSTIVE_TEST_ORDER; j++) { - for (i = 1; i < EXHAUSTIVE_TEST_ORDER; i++) { + for (i = 0; i < EXHAUSTIVE_TEST_ORDER; i++) { int ret; secp256k1_gej tmp; secp256k1_fe xn, xd, tmpf; @@ -203,19 +209,19 @@ static void test_exhaustive_ecmult(const secp256k1_ge *group, const secp256k1_ge secp256k1_scalar_set_int(&ng, j); /* Test secp256k1_ecmult_const. */ - secp256k1_ecmult_const(&tmp, &group[i], &ng, 256); + secp256k1_ecmult_const(&tmp, &group[i], &ng); ge_equals_gej(&group[(i * j) % EXHAUSTIVE_TEST_ORDER], &tmp); - if (j != 0) { + if (i != 0 && j != 0) { /* Test secp256k1_ecmult_const_xonly with all curve X coordinates, and xd=NULL. */ - ret = secp256k1_ecmult_const_xonly(&tmpf, &group[i].x, NULL, &ng, 256, 0); + ret = secp256k1_ecmult_const_xonly(&tmpf, &group[i].x, NULL, &ng, 0); CHECK(ret); CHECK(secp256k1_fe_equal_var(&tmpf, &group[(i * j) % EXHAUSTIVE_TEST_ORDER].x)); /* Test secp256k1_ecmult_const_xonly with all curve X coordinates, with random xd. */ random_fe_non_zero(&xd); secp256k1_fe_mul(&xn, &xd, &group[i].x); - ret = secp256k1_ecmult_const_xonly(&tmpf, &xn, &xd, &ng, 256, 0); + ret = secp256k1_ecmult_const_xonly(&tmpf, &xn, &xd, &ng, 0); CHECK(ret); CHECK(secp256k1_fe_equal_var(&tmpf, &group[(i * j) % EXHAUSTIVE_TEST_ORDER].x)); } diff --git a/src/secp256k1/src/util.h b/src/secp256k1/src/util.h index e75c5ad552..e2ee8a8f19 100644 --- a/src/secp256k1/src/util.h +++ b/src/secp256k1/src/util.h @@ -7,6 +7,8 @@ #ifndef SECP256K1_UTIL_H #define SECP256K1_UTIL_H +#include "../include/secp256k1.h" + #include <stdlib.h> #include <stdint.h> #include <stdio.h> @@ -17,6 +19,38 @@ #define DEBUG_CONFIG_MSG(x) "DEBUG_CONFIG: " x #define DEBUG_CONFIG_DEF(x) DEBUG_CONFIG_MSG(#x "=" STR(x)) +/* Debug helper for printing arrays of unsigned char. */ +#define PRINT_BUF(buf, len) do { \ + printf("%s[%lu] = ", #buf, (unsigned long)len); \ + print_buf_plain(buf, len); \ +} while(0) + +static void print_buf_plain(const unsigned char *buf, size_t len) { + size_t i; + printf("{"); + for (i = 0; i < len; i++) { + if (i % 8 == 0) { + printf("\n "); + } else { + printf(" "); + } + printf("0x%02X,", buf[i]); + } + printf("\n}\n"); +} + +# if (!defined(__STDC_VERSION__) || (__STDC_VERSION__ < 199901L) ) +# if SECP256K1_GNUC_PREREQ(2,7) +# define SECP256K1_INLINE __inline__ +# elif (defined(_MSC_VER)) +# define SECP256K1_INLINE __inline +# else +# define SECP256K1_INLINE +# endif +# else +# define SECP256K1_INLINE inline +# endif + typedef struct { void (*fn)(const char *text, void* data); const void* data; @@ -319,4 +353,28 @@ SECP256K1_INLINE static void secp256k1_write_be32(unsigned char* p, uint32_t x) p[0] = x >> 24; } +/* Read a uint64_t in big endian */ +SECP256K1_INLINE static uint64_t secp256k1_read_be64(const unsigned char* p) { + return (uint64_t)p[0] << 56 | + (uint64_t)p[1] << 48 | + (uint64_t)p[2] << 40 | + (uint64_t)p[3] << 32 | + (uint64_t)p[4] << 24 | + (uint64_t)p[5] << 16 | + (uint64_t)p[6] << 8 | + (uint64_t)p[7]; +} + +/* Write a uint64_t in big endian */ +SECP256K1_INLINE static void secp256k1_write_be64(unsigned char* p, uint64_t x) { + p[7] = x; + p[6] = x >> 8; + p[5] = x >> 16; + p[4] = x >> 24; + p[3] = x >> 32; + p[2] = x >> 40; + p[1] = x >> 48; + p[0] = x >> 56; +} + #endif /* SECP256K1_UTIL_H */ diff --git a/src/secp256k1/tools/tests_wycheproof_generate.py b/src/secp256k1/tools/tests_wycheproof_generate.py index 333f6fbef0..b26dfa89d6 100755 --- a/src/secp256k1/tools/tests_wycheproof_generate.py +++ b/src/secp256k1/tools/tests_wycheproof_generate.py @@ -7,8 +7,6 @@ Generate a C file with ECDSA testvectors from the Wycheproof project. ''' import json -import hashlib -import urllib.request import sys filename_input = sys.argv[1] @@ -19,7 +17,8 @@ with open(filename_input) as f: num_groups = len(doc['testGroups']) def to_c_array(x): - if x == "": return "" + if x == "": + return "" s = ',0x'.join(a+b for a,b in zip(x[::2], x[1::2])) return "0x" + s @@ -43,18 +42,23 @@ for i in range(num_groups): sig_size = len(test_vector['sig']) // 2 msg_size = len(test_vector['msg']) // 2 - if test_vector['result'] == "invalid": expected_verify = 0 - elif test_vector['result'] == "valid": expected_verify = 1 - else: raise ValueError("invalid result field") + if test_vector['result'] == "invalid": + expected_verify = 0 + elif test_vector['result'] == "valid": + expected_verify = 1 + else: + raise ValueError("invalid result field") - if num_vectors != 0 and sig_size != 0: signatures += ",\n " + if num_vectors != 0 and sig_size != 0: + signatures += ",\n " new_msg = False msg = to_c_array(test_vector['msg']) msg_offset = offset_msg_running # check for repeated msg - if msg not in cache_msgs.keys(): - if num_vectors != 0 and msg_size != 0: messages += ",\n " + if msg not in cache_msgs: + if num_vectors != 0 and msg_size != 0: + messages += ",\n " cache_msgs[msg] = offset_msg_running messages += msg new_msg = True @@ -65,8 +69,9 @@ for i in range(num_groups): pk = to_c_array(public_key['uncompressed']) pk_offset = offset_pk_running # check for repeated pk - if pk not in cache_public_keys.keys(): - if num_vectors != 0: public_keys += ",\n " + if pk not in cache_public_keys: + if num_vectors != 0: + public_keys += ",\n " cache_public_keys[pk] = offset_pk_running public_keys += pk new_pk = True @@ -76,15 +81,11 @@ for i in range(num_groups): signatures += to_c_array(test_vector['sig']) out += " /" + "* tcId: " + str(test_vector['tcId']) + ". " + test_vector['comment'] + " *" + "/\n" - out += " {" + "{0}, {1}, {2}, {3}, {4}, {5}".format( - pk_offset, - msg_offset, - msg_size, - offset_sig, - sig_size, - expected_verify) + " },\n" - if new_msg: offset_msg_running += msg_size - if new_pk: offset_pk_running += 65 + out += f" {{{pk_offset}, {msg_offset}, {msg_size}, {offset_sig}, {sig_size}, {expected_verify} }},\n" + if new_msg: + offset_msg_running += msg_size + if new_pk: + offset_pk_running += 65 offset_sig += sig_size num_vectors += 1 @@ -101,7 +102,7 @@ typedef struct { print("/* Note: this file was autogenerated using tests_wycheproof_generate.py. Do not edit. */") -print("#define SECP256K1_ECDSA_WYCHEPROOF_NUMBER_TESTVECTORS ({})".format(num_vectors)) +print(f"#define SECP256K1_ECDSA_WYCHEPROOF_NUMBER_TESTVECTORS ({num_vectors})") print(struct_definition) diff --git a/src/serialize.h b/src/serialize.h index 7bc7b10779..cf865eb3f4 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -188,6 +188,7 @@ template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; } } \ FORMATTER_METHODS(cls, obj) +// clang-format off #ifndef CHAR_EQUALS_INT8 template <typename Stream> void Serialize(Stream&, char) = delete; // char serialization forbidden. Use uint8_t or int8_t #endif @@ -201,8 +202,7 @@ template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_wri template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); } template<typename Stream, int N> inline void Serialize(Stream& s, const char (&a)[N]) { s.write(MakeByteSpan(a)); } template<typename Stream, int N> inline void Serialize(Stream& s, const unsigned char (&a)[N]) { s.write(MakeByteSpan(a)); } -template<typename Stream> inline void Serialize(Stream& s, const Span<const unsigned char>& span) { s.write(AsBytes(span)); } -template<typename Stream> inline void Serialize(Stream& s, const Span<unsigned char>& span) { s.write(AsBytes(span)); } +template <typename Stream, typename B> void Serialize(Stream& s, Span<B> span) { (void)/* force byte-type */UCharCast(span.data()); s.write(AsBytes(span)); } #ifndef CHAR_EQUALS_INT8 template <typename Stream> void Unserialize(Stream&, char) = delete; // char serialization forbidden. Use uint8_t or int8_t @@ -217,10 +217,11 @@ template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); } template<typename Stream, int N> inline void Unserialize(Stream& s, char (&a)[N]) { s.read(MakeWritableByteSpan(a)); } template<typename Stream, int N> inline void Unserialize(Stream& s, unsigned char (&a)[N]) { s.read(MakeWritableByteSpan(a)); } -template<typename Stream> inline void Unserialize(Stream& s, Span<unsigned char>& span) { s.read(AsWritableBytes(span)); } +template <typename Stream, typename B> void Unserialize(Stream& s, Span<B> span) { (void)/* force byte-type */UCharCast(span.data()); s.read(AsWritableBytes(span)); } template <typename Stream> inline void Serialize(Stream& s, bool a) { uint8_t f = a; ser_writedata8(s, f); } template <typename Stream> inline void Unserialize(Stream& s, bool& a) { uint8_t f = ser_readdata8(s); a = f; } +// clang-format on /** @@ -472,10 +473,10 @@ struct CustomUintFormatter if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range"); if (BigEndian) { uint64_t raw = htobe64(v); - s.write({AsBytePtr(&raw) + 8 - Bytes, Bytes}); + s.write(AsBytes(Span{&raw, 1}).last(Bytes)); } else { uint64_t raw = htole64(v); - s.write({AsBytePtr(&raw), Bytes}); + s.write(AsBytes(Span{&raw, 1}).first(Bytes)); } } @@ -485,10 +486,10 @@ struct CustomUintFormatter static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small"); uint64_t raw = 0; if (BigEndian) { - s.read({AsBytePtr(&raw) + 8 - Bytes, Bytes}); + s.read(AsWritableBytes(Span{&raw, 1}).last(Bytes)); v = static_cast<I>(be64toh(raw)); } else { - s.read({AsBytePtr(&raw), Bytes}); + s.read(AsWritableBytes(Span{&raw, 1}).first(Bytes)); v = static_cast<I>(le64toh(raw)); } } diff --git a/src/shutdown.cpp b/src/shutdown.cpp index 2fffc0663c..d70017d734 100644 --- a/src/shutdown.cpp +++ b/src/shutdown.cpp @@ -11,6 +11,7 @@ #include <logging.h> #include <node/interface_ui.h> +#include <util/check.h> #include <util/tokenpipe.h> #include <warnings.h> @@ -20,6 +21,8 @@ #include <condition_variable> #endif +static std::atomic<int>* g_exit_status{nullptr}; + bool AbortNode(const std::string& strMessage, bilingual_str user_message) { SetMiscWarning(Untranslated(strMessage)); @@ -28,6 +31,7 @@ bool AbortNode(const std::string& strMessage, bilingual_str user_message) user_message = _("A fatal internal error occurred, see debug.log for details"); } InitError(user_message); + Assert(g_exit_status)->store(EXIT_FAILURE); StartShutdown(); return false; } @@ -44,8 +48,9 @@ static TokenPipeEnd g_shutdown_r; static TokenPipeEnd g_shutdown_w; #endif -bool InitShutdownState() +bool InitShutdownState(std::atomic<int>& exit_status) { + g_exit_status = &exit_status; #ifndef WIN32 std::optional<TokenPipe> pipe = TokenPipe::Make(); if (!pipe) return false; diff --git a/src/shutdown.h b/src/shutdown.h index 07a8315788..c119bee96f 100644 --- a/src/shutdown.h +++ b/src/shutdown.h @@ -8,13 +8,15 @@ #include <util/translation.h> // For bilingual_str +#include <atomic> + /** Abort with a message */ bool AbortNode(const std::string& strMessage, bilingual_str user_message = bilingual_str{}); /** Initialize shutdown state. This must be called before using either StartShutdown(), * AbortShutdown() or WaitForShutdown(). Calling ShutdownRequested() is always safe. */ -bool InitShutdownState(); +bool InitShutdownState(std::atomic<int>& exit_status); /** Request shutdown of the application. */ void StartShutdown(); diff --git a/src/signet.cpp b/src/signet.cpp index b76b1e342f..b73d82bb2e 100644 --- a/src/signet.cpp +++ b/src/signet.cpp @@ -8,6 +8,7 @@ #include <cstdint> #include <vector> +#include <common/system.h> #include <consensus/merkle.h> #include <consensus/params.h> #include <consensus/validation.h> @@ -22,7 +23,6 @@ #include <streams.h> #include <uint256.h> #include <util/strencodings.h> -#include <util/system.h> static constexpr uint8_t SIGNET_HEADER[4] = {0xec, 0xc7, 0xda, 0xa2}; diff --git a/src/span.h b/src/span.h index 4692eca7fb..0563556dc2 100644 --- a/src/span.h +++ b/src/span.h @@ -243,21 +243,16 @@ T& SpanPopBack(Span<T>& span) return back; } -//! Convert a data pointer to a std::byte data pointer. -//! Where possible, please use the safer AsBytes helpers. -inline const std::byte* AsBytePtr(const void* data) { return reinterpret_cast<const std::byte*>(data); } -inline std::byte* AsBytePtr(void* data) { return reinterpret_cast<std::byte*>(data); } - // From C++20 as_bytes and as_writeable_bytes template <typename T> Span<const std::byte> AsBytes(Span<T> s) noexcept { - return {AsBytePtr(s.data()), s.size_bytes()}; + return {reinterpret_cast<const std::byte*>(s.data()), s.size_bytes()}; } template <typename T> Span<std::byte> AsWritableBytes(Span<T> s) noexcept { - return {AsBytePtr(s.data()), s.size_bytes()}; + return {reinterpret_cast<std::byte*>(s.data()), s.size_bytes()}; } template <typename V> @@ -274,6 +269,7 @@ Span<std::byte> MakeWritableByteSpan(V&& v) noexcept // Helper functions to safely cast to unsigned char pointers. inline unsigned char* UCharCast(char* c) { return (unsigned char*)c; } inline unsigned char* UCharCast(unsigned char* c) { return c; } +inline unsigned char* UCharCast(std::byte* c) { return (unsigned char*)c; } inline const unsigned char* UCharCast(const char* c) { return (unsigned char*)c; } inline const unsigned char* UCharCast(const unsigned char* c) { return c; } inline const unsigned char* UCharCast(const std::byte* c) { return reinterpret_cast<const unsigned char*>(c); } diff --git a/src/streams.h b/src/streams.h index e346aa0a3f..03df20b5db 100644 --- a/src/streams.h +++ b/src/streams.h @@ -293,14 +293,6 @@ public: vch.insert(vch.end(), src.begin(), src.end()); } - template<typename Stream> - void Serialize(Stream& s) const - { - // Special case: stream << stream concatenates like stream += stream - if (!vch.empty()) - s.write(MakeByteSpan(vch)); - } - template<typename T> DataStream& operator<<(const T& obj) { diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index a0918bf463..b2076bea07 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -32,9 +32,9 @@ struct secure_allocator : public std::allocator<T> { { } ~secure_allocator() noexcept {} - template <typename _Other> + template <typename Other> struct rebind { - typedef secure_allocator<_Other> other; + typedef secure_allocator<Other> other; }; T* allocate(std::size_t n, const void* hint = nullptr) diff --git a/src/support/allocators/zeroafterfree.h b/src/support/allocators/zeroafterfree.h index 795eea3bc0..2dc644c242 100644 --- a/src/support/allocators/zeroafterfree.h +++ b/src/support/allocators/zeroafterfree.h @@ -27,9 +27,9 @@ struct zero_after_free_allocator : public std::allocator<T> { { } ~zero_after_free_allocator() noexcept {} - template <typename _Other> + template <typename Other> struct rebind { - typedef zero_after_free_allocator<_Other> other; + typedef zero_after_free_allocator<Other> other; }; void deallocate(T* p, std::size_t n) diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 2242c7a75a..c02322ec7d 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -33,16 +33,16 @@ static int32_t GetCheckRatio(const NodeContext& node_ctx) static CNetAddr ResolveIP(const std::string& ip) { - CNetAddr addr; - BOOST_CHECK_MESSAGE(LookupHost(ip, addr, false), strprintf("failed to resolve: %s", ip)); - return addr; + const std::optional<CNetAddr> addr{LookupHost(ip, false)}; + BOOST_CHECK_MESSAGE(addr.has_value(), strprintf("failed to resolve: %s", ip)); + return addr.value_or(CNetAddr{}); } static CService ResolveService(const std::string& ip, uint16_t port = 0) { - CService serv; - BOOST_CHECK_MESSAGE(Lookup(ip, serv, port, false), strprintf("failed to resolve: %s:%i", ip, port)); - return serv; + const std::optional<CService> serv{Lookup(ip, port, false)}; + BOOST_CHECK_MESSAGE(serv.has_value(), strprintf("failed to resolve: %s:%i", ip, port)); + return serv.value_or(CService{}); } @@ -952,18 +952,23 @@ BOOST_AUTO_TEST_CASE(load_addrman) { AddrMan addrman{EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)}; - CService addr1, addr2, addr3; - BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); - BOOST_CHECK(Lookup("250.7.2.2", addr2, 9999, false)); - BOOST_CHECK(Lookup("250.7.3.3", addr3, 9999, false)); - BOOST_CHECK(Lookup("250.7.3.3"s, addr3, 9999, false)); - BOOST_CHECK(!Lookup("250.7.3.3\0example.com"s, addr3, 9999, false)); + std::optional<CService> addr1, addr2, addr3, addr4; + addr1 = Lookup("250.7.1.1", 8333, false); + BOOST_CHECK(addr1.has_value()); + addr2 = Lookup("250.7.2.2", 9999, false); + BOOST_CHECK(addr2.has_value()); + addr3 = Lookup("250.7.3.3", 9999, false); + BOOST_CHECK(addr3.has_value()); + addr3 = Lookup("250.7.3.3"s, 9999, false); + BOOST_CHECK(addr3.has_value()); + addr4 = Lookup("250.7.3.3\0example.com"s, 9999, false); + BOOST_CHECK(!addr4.has_value()); // Add three addresses to new table. - CService source; - BOOST_CHECK(Lookup("252.5.1.1", source, 8333, false)); - std::vector<CAddress> addresses{CAddress(addr1, NODE_NONE), CAddress(addr2, NODE_NONE), CAddress(addr3, NODE_NONE)}; - BOOST_CHECK(addrman.Add(addresses, source)); + const std::optional<CService> source{Lookup("252.5.1.1", 8333, false)}; + BOOST_CHECK(source.has_value()); + std::vector<CAddress> addresses{CAddress(addr1.value(), NODE_NONE), CAddress(addr2.value(), NODE_NONE), CAddress(addr3.value(), NODE_NONE)}; + BOOST_CHECK(addrman.Add(addresses, source.value())); BOOST_CHECK(addrman.Size() == 3); // Test that the de-serialization does not throw an exception. @@ -1008,12 +1013,12 @@ static CDataStream MakeCorruptPeersDat() int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); s << nUBuckets; - CService serv; - BOOST_CHECK(Lookup("252.1.1.1", serv, 7777, false)); - CAddress addr = CAddress(serv, NODE_NONE); - CNetAddr resolved; - BOOST_CHECK(LookupHost("252.2.2.2", resolved, false)); - AddrInfo info = AddrInfo(addr, resolved); + const std::optional<CService> serv{Lookup("252.1.1.1", 7777, false)}; + BOOST_REQUIRE(serv.has_value()); + CAddress addr = CAddress(serv.value(), NODE_NONE); + std::optional<CNetAddr> resolved{LookupHost("252.2.2.2", false)}; + BOOST_REQUIRE(resolved.has_value()); + AddrInfo info = AddrInfo(addr, resolved.value()); s << info; return s; diff --git a/src/test/allocator_tests.cpp b/src/test/allocator_tests.cpp index f74e50a890..8c0af6f26f 100644 --- a/src/test/allocator_tests.cpp +++ b/src/test/allocator_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <support/lockedpool.h> -#include <util/system.h> #include <limits> #include <memory> diff --git a/src/test/argsman_tests.cpp b/src/test/argsman_tests.cpp index 48bffc4ac9..0b789e7f5c 100644 --- a/src/test/argsman_tests.cpp +++ b/src/test/argsman_tests.cpp @@ -85,7 +85,7 @@ class CheckValueTest : public TestChain100Setup { public: struct Expect { - util::SettingsValue setting; + common::SettingsValue setting; bool default_string = false; bool default_int = false; bool default_bool = false; @@ -95,7 +95,7 @@ public: std::optional<std::vector<std::string>> list_value; const char* error = nullptr; - explicit Expect(util::SettingsValue s) : setting(std::move(s)) {} + explicit Expect(common::SettingsValue s) : setting(std::move(s)) {} Expect& DefaultString() { default_string = true; return *this; } Expect& DefaultInt() { default_int = true; return *this; } Expect& DefaultBool() { default_bool = true; return *this; } @@ -1022,14 +1022,14 @@ BOOST_AUTO_TEST_CASE(util_ReadWriteSettings) // Test writing setting. TestArgsManager args1; args1.ForceSetArg("-datadir", fs::PathToString(m_path_root)); - args1.LockSettings([&](util::Settings& settings) { settings.rw_settings["name"] = "value"; }); + args1.LockSettings([&](common::Settings& settings) { settings.rw_settings["name"] = "value"; }); args1.WriteSettingsFile(); // Test reading setting. TestArgsManager args2; args2.ForceSetArg("-datadir", fs::PathToString(m_path_root)); args2.ReadSettingsFile(); - args2.LockSettings([&](util::Settings& settings) { BOOST_CHECK_EQUAL(settings.rw_settings["name"].get_str(), "value"); }); + args2.LockSettings([&](common::Settings& settings) { BOOST_CHECK_EQUAL(settings.rw_settings["name"].get_str(), "value"); }); // Test error logging, and remove previously written setting. { diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 1ff5d6cf59..24af51cce1 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -12,8 +12,8 @@ #include <pow.h> #include <script/standard.h> #include <test/util/blockfilter.h> +#include <test/util/index.h> #include <test/util/setup_common.h> -#include <util/time.h> #include <validation.h> #include <boost/test/unit_test.hpp> @@ -142,12 +142,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) BOOST_REQUIRE(filter_index.Start()); // Allow filter index to catch up with the block index. - constexpr auto timeout{10s}; - const auto time_start{SteadyClock::now()}; - while (!filter_index.BlockUntilSyncedToCurrentChain()) { - BOOST_REQUIRE(time_start + timeout > SteadyClock::now()); - UninterruptibleSleep(std::chrono::milliseconds{100}); - } + IndexWaitSynced(filter_index); // Check that filter index has all blocks that were in the chain before it started. { diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 5d4c5eea0e..93c0412593 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -5,6 +5,7 @@ #include <common/bloom.h> #include <clientversion.h> +#include <common/system.h> #include <key.h> #include <key_io.h> #include <merkleblock.h> @@ -16,7 +17,6 @@ #include <test/util/setup_common.h> #include <uint256.h> #include <util/strencodings.h> -#include <util/system.h> #include <vector> diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 503a58076b..9dc88ea671 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -6,28 +6,15 @@ #include <index/coinstatsindex.h> #include <interfaces/chain.h> #include <kernel/coinstats.h> +#include <test/util/index.h> #include <test/util/setup_common.h> #include <test/util/validation.h> -#include <util/time.h> #include <validation.h> #include <boost/test/unit_test.hpp> -#include <chrono> - BOOST_AUTO_TEST_SUITE(coinstatsindex_tests) -static void IndexWaitSynced(BaseIndex& index) -{ - // Allow the CoinStatsIndex to catch up with the block index that is syncing - // in a background thread. - const auto timeout = GetTime<std::chrono::seconds>() + 120s; - while (!index.BlockUntilSyncedToCurrentChain()) { - BOOST_REQUIRE(timeout > GetTime<std::chrono::milliseconds>()); - UninterruptibleSleep(100ms); - } -} - BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) { CoinStatsIndex coin_stats_index{interfaces::MakeChain(m_node), 1 << 20, true}; diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index 392f3591f1..edb1dca457 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -4,6 +4,7 @@ #include <arith_uint256.h> #include <common/args.h> +#include <common/system.h> #include <compressor.h> #include <consensus/amount.h> #include <consensus/merkle.h> @@ -32,7 +33,6 @@ #include <util/overflow.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <version.h> #include <cassert> diff --git a/src/test/fuzz/key.cpp b/src/test/fuzz/key.cpp index 3eab2e20c0..25ea547435 100644 --- a/src/test/fuzz/key.cpp +++ b/src/test/fuzz/key.cpp @@ -15,13 +15,17 @@ #include <script/signingprovider.h> #include <script/standard.h> #include <streams.h> +#include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <util/chaintype.h> #include <util/strencodings.h> +#include <array> #include <cassert> +#include <cstddef> #include <cstdint> #include <numeric> +#include <optional> #include <string> #include <vector> @@ -303,3 +307,79 @@ FUZZ_TARGET_INIT(key, initialize_key) } } } + +FUZZ_TARGET_INIT(ellswift_roundtrip, initialize_key) +{ + FuzzedDataProvider fdp{buffer.data(), buffer.size()}; + + auto key_bytes = fdp.ConsumeBytes<uint8_t>(32); + key_bytes.resize(32); + CKey key; + key.Set(key_bytes.begin(), key_bytes.end(), true); + if (!key.IsValid()) return; + + auto ent32 = fdp.ConsumeBytes<std::byte>(32); + ent32.resize(32); + + auto encoded_ellswift = key.EllSwiftCreate(ent32); + auto decoded_pubkey = encoded_ellswift.Decode(); + + assert(key.VerifyPubKey(decoded_pubkey)); +} + +FUZZ_TARGET_INIT(bip324_ecdh, initialize_key) +{ + FuzzedDataProvider fdp{buffer.data(), buffer.size()}; + + // We generate private key, k1. + auto rnd32 = fdp.ConsumeBytes<uint8_t>(32); + rnd32.resize(32); + CKey k1; + k1.Set(rnd32.begin(), rnd32.end(), true); + if (!k1.IsValid()) return; + + // They generate private key, k2. + rnd32 = fdp.ConsumeBytes<uint8_t>(32); + rnd32.resize(32); + CKey k2; + k2.Set(rnd32.begin(), rnd32.end(), true); + if (!k2.IsValid()) return; + + // We construct an ellswift encoding for our key, k1_ellswift. + auto ent32_1 = fdp.ConsumeBytes<std::byte>(32); + ent32_1.resize(32); + auto k1_ellswift = k1.EllSwiftCreate(ent32_1); + + // They construct an ellswift encoding for their key, k2_ellswift. + auto ent32_2 = fdp.ConsumeBytes<std::byte>(32); + ent32_2.resize(32); + auto k2_ellswift = k2.EllSwiftCreate(ent32_2); + + // They construct another (possibly distinct) ellswift encoding for their key, k2_ellswift_bad. + auto ent32_2_bad = fdp.ConsumeBytes<std::byte>(32); + ent32_2_bad.resize(32); + auto k2_ellswift_bad = k2.EllSwiftCreate(ent32_2_bad); + assert((ent32_2_bad == ent32_2) == (k2_ellswift_bad == k2_ellswift)); + + // Determine who is who. + bool initiating = fdp.ConsumeBool(); + + // We compute our shared secret using our key and their public key. + auto ecdh_secret_1 = k1.ComputeBIP324ECDHSecret(k2_ellswift, k1_ellswift, initiating); + // They compute their shared secret using their key and our public key. + auto ecdh_secret_2 = k2.ComputeBIP324ECDHSecret(k1_ellswift, k2_ellswift, !initiating); + // Those must match, as everyone is behaving correctly. + assert(ecdh_secret_1 == ecdh_secret_2); + + if (k1_ellswift != k2_ellswift) { + // Unless the two keys are exactly identical, acting as the wrong party breaks things. + auto ecdh_secret_bad = k1.ComputeBIP324ECDHSecret(k2_ellswift, k1_ellswift, !initiating); + assert(ecdh_secret_bad != ecdh_secret_1); + } + + if (k2_ellswift_bad != k2_ellswift) { + // Unless both encodings created by them are identical, using the second one breaks things. + auto ecdh_secret_bad = k1.ComputeBIP324ECDHSecret(k2_ellswift_bad, k1_ellswift, initiating); + assert(ecdh_secret_bad != ecdh_secret_1); + } +} diff --git a/src/test/fuzz/mini_miner.cpp b/src/test/fuzz/mini_miner.cpp index f49d940393..2b371f6d5f 100644 --- a/src/test/fuzz/mini_miner.cpp +++ b/src/test/fuzz/mini_miner.cpp @@ -118,10 +118,11 @@ FUZZ_TARGET_INIT(mini_miner_selection, initialize_miner) LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { CMutableTransaction mtx = CMutableTransaction(); - const size_t num_inputs = 2; + assert(!available_coins.empty()); + const size_t num_inputs = std::min(size_t{2}, available_coins.size()); const size_t num_outputs = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(2, 5); for (size_t n{0}; n < num_inputs; ++n) { - auto prevout = available_coins.front(); + auto prevout = available_coins.at(0); mtx.vin.push_back(CTxIn(prevout, CScript())); available_coins.pop_front(); } diff --git a/src/test/fuzz/netbase_dns_lookup.cpp b/src/test/fuzz/netbase_dns_lookup.cpp index 81e216b358..dcf500acc3 100644 --- a/src/test/fuzz/netbase_dns_lookup.cpp +++ b/src/test/fuzz/netbase_dns_lookup.cpp @@ -29,33 +29,29 @@ FUZZ_TARGET(netbase_dns_lookup) }; { - std::vector<CNetAddr> resolved_addresses; - if (LookupHost(name, resolved_addresses, max_results, allow_lookup, fuzzed_dns_lookup_function)) { - for (const CNetAddr& resolved_address : resolved_addresses) { - assert(!resolved_address.IsInternal()); - } + const std::vector<CNetAddr> resolved_addresses{LookupHost(name, max_results, allow_lookup, fuzzed_dns_lookup_function)}; + for (const CNetAddr& resolved_address : resolved_addresses) { + assert(!resolved_address.IsInternal()); } assert(resolved_addresses.size() <= max_results || max_results == 0); } { - CNetAddr resolved_address; - if (LookupHost(name, resolved_address, allow_lookup, fuzzed_dns_lookup_function)) { - assert(!resolved_address.IsInternal()); + const std::optional<CNetAddr> resolved_address{LookupHost(name, allow_lookup, fuzzed_dns_lookup_function)}; + if (resolved_address.has_value()) { + assert(!resolved_address.value().IsInternal()); } } { - std::vector<CService> resolved_services; - if (Lookup(name, resolved_services, default_port, allow_lookup, max_results, fuzzed_dns_lookup_function)) { - for (const CNetAddr& resolved_service : resolved_services) { - assert(!resolved_service.IsInternal()); - } + const std::vector<CService> resolved_services{Lookup(name, default_port, allow_lookup, max_results, fuzzed_dns_lookup_function)}; + for (const CNetAddr& resolved_service : resolved_services) { + assert(!resolved_service.IsInternal()); } assert(resolved_services.size() <= max_results || max_results == 0); } { - CService resolved_service; - if (Lookup(name, resolved_service, default_port, allow_lookup, fuzzed_dns_lookup_function)) { - assert(!resolved_service.IsInternal()); + const std::optional<CService> resolved_service{Lookup(name, default_port, allow_lookup, fuzzed_dns_lookup_function)}; + if (resolved_service.has_value()) { + assert(!resolved_service.value().IsInternal()); } } { diff --git a/src/test/fuzz/p2p_transport_serialization.cpp b/src/test/fuzz/p2p_transport_serialization.cpp index ec3cdbff5a..a6fe3037e6 100644 --- a/src/test/fuzz/p2p_transport_serialization.cpp +++ b/src/test/fuzz/p2p_transport_serialization.cpp @@ -77,7 +77,7 @@ FUZZ_TARGET_INIT(p2p_transport_serialization, initialize_p2p_transport_serializa assert(msg.m_time == m_time); std::vector<unsigned char> header; - auto msg2 = CNetMsgMaker{msg.m_recv.GetVersion()}.Make(msg.m_type, MakeUCharSpan(msg.m_recv)); + auto msg2 = CNetMsgMaker{msg.m_recv.GetVersion()}.Make(msg.m_type, Span{msg.m_recv}); serializer.prepareForTransport(msg2, header); } } diff --git a/src/test/fuzz/parse_univalue.cpp b/src/test/fuzz/parse_univalue.cpp index be15a38e92..6d33c1a8cc 100644 --- a/src/test/fuzz/parse_univalue.cpp +++ b/src/test/fuzz/parse_univalue.cpp @@ -22,12 +22,9 @@ FUZZ_TARGET_INIT(parse_univalue, initialize_parse_univalue) const std::string random_string(buffer.begin(), buffer.end()); bool valid = true; const UniValue univalue = [&] { - try { - return ParseNonRFCJSONValue(random_string); - } catch (const std::runtime_error&) { - valid = false; - return UniValue{}; - } + UniValue uv; + if (!uv.read(random_string)) valid = false; + return valid ? uv : UniValue{}; }(); if (!valid) { return; diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 116fbd9015..aa3cfe81df 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -31,7 +31,7 @@ void initialize_policy_estimator() FUZZ_TARGET_INIT(policy_estimator, initialize_policy_estimator) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - CBlockPolicyEstimator block_policy_estimator{FeeestPath(*g_setup->m_node.args)}; + CBlockPolicyEstimator block_policy_estimator{FeeestPath(*g_setup->m_node.args), DEFAULT_ACCEPT_STALE_FEE_ESTIMATES}; LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, diff --git a/src/test/fuzz/policy_estimator_io.cpp b/src/test/fuzz/policy_estimator_io.cpp index 7c3289cd26..3df40197d8 100644 --- a/src/test/fuzz/policy_estimator_io.cpp +++ b/src/test/fuzz/policy_estimator_io.cpp @@ -28,7 +28,7 @@ FUZZ_TARGET_INIT(policy_estimator_io, initialize_policy_estimator_io) FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); AutoFile fuzzed_auto_file{fuzzed_auto_file_provider.open()}; // Re-using block_policy_estimator across runs to avoid costly creation of CBlockPolicyEstimator object. - static CBlockPolicyEstimator block_policy_estimator{FeeestPath(*g_setup->m_node.args)}; + static CBlockPolicyEstimator block_policy_estimator{FeeestPath(*g_setup->m_node.args), DEFAULT_ACCEPT_STALE_FEE_ESTIMATES}; if (block_policy_estimator.Read(fuzzed_auto_file)) { block_policy_estimator.Write(fuzzed_auto_file); } diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 70dd1e17c5..744ff4701d 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -2,15 +2,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <banman.h> -#include <chainparams.h> #include <consensus/consensus.h> #include <net.h> #include <net_processing.h> +#include <primitives/transaction.h> #include <protocol.h> -#include <scheduler.h> #include <script/script.h> +#include <serialize.h> +#include <span.h> #include <streams.h> +#include <sync.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> @@ -20,42 +21,32 @@ #include <test/util/setup_common.h> #include <test/util/validation.h> #include <util/chaintype.h> +#include <util/check.h> +#include <util/time.h> +#include <validation.h> #include <validationinterface.h> #include <version.h> + #include <atomic> -#include <cassert> -#include <chrono> -#include <cstdint> -#include <iosfwd> +#include <cstdlib> #include <iostream> #include <memory> #include <string> +#include <string_view> +#include <vector> namespace { const TestingSetup* g_setup; +std::string_view LIMIT_TO_MESSAGE_TYPE{}; } // namespace -size_t& GetNumMsgTypes() -{ - static size_t g_num_msg_types{0}; - return g_num_msg_types; -} -#define FUZZ_TARGET_MSG(msg_type) \ - struct msg_type##_Count_Before_Main { \ - msg_type##_Count_Before_Main() \ - { \ - ++GetNumMsgTypes(); \ - } \ - } const static g_##msg_type##_count_before_main; \ - FUZZ_TARGET_INIT(process_message_##msg_type, initialize_process_message) \ - { \ - fuzz_target(buffer, #msg_type); \ - } - void initialize_process_message() { - Assert(GetNumMsgTypes() == getAllNetMessageTypes().size()); // If this fails, add or remove the message type below + if (const auto val{std::getenv("LIMIT_TO_MESSAGE_TYPE")}) { + LIMIT_TO_MESSAGE_TYPE = val; + Assert(std::count(getAllNetMessageTypes().begin(), getAllNetMessageTypes().end(), LIMIT_TO_MESSAGE_TYPE)); // Unknown message type passed + } static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>( /*chain_type=*/ChainType::REGTEST, @@ -67,7 +58,7 @@ void initialize_process_message() SyncWithValidationInterfaceQueue(); } -void fuzz_target(FuzzBufferType buffer, const std::string& LIMIT_TO_MESSAGE_TYPE) +FUZZ_TARGET_INIT(process_message, initialize_process_message) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); @@ -101,40 +92,3 @@ void fuzz_target(FuzzBufferType buffer, const std::string& LIMIT_TO_MESSAGE_TYPE SyncWithValidationInterfaceQueue(); g_setup->m_node.connman->StopNodes(); } - -FUZZ_TARGET_INIT(process_message, initialize_process_message) { fuzz_target(buffer, ""); } -FUZZ_TARGET_MSG(addr); -FUZZ_TARGET_MSG(addrv2); -FUZZ_TARGET_MSG(block); -FUZZ_TARGET_MSG(blocktxn); -FUZZ_TARGET_MSG(cfcheckpt); -FUZZ_TARGET_MSG(cfheaders); -FUZZ_TARGET_MSG(cfilter); -FUZZ_TARGET_MSG(cmpctblock); -FUZZ_TARGET_MSG(feefilter); -FUZZ_TARGET_MSG(filteradd); -FUZZ_TARGET_MSG(filterclear); -FUZZ_TARGET_MSG(filterload); -FUZZ_TARGET_MSG(getaddr); -FUZZ_TARGET_MSG(getblocks); -FUZZ_TARGET_MSG(getblocktxn); -FUZZ_TARGET_MSG(getcfcheckpt); -FUZZ_TARGET_MSG(getcfheaders); -FUZZ_TARGET_MSG(getcfilters); -FUZZ_TARGET_MSG(getdata); -FUZZ_TARGET_MSG(getheaders); -FUZZ_TARGET_MSG(headers); -FUZZ_TARGET_MSG(inv); -FUZZ_TARGET_MSG(mempool); -FUZZ_TARGET_MSG(merkleblock); -FUZZ_TARGET_MSG(notfound); -FUZZ_TARGET_MSG(ping); -FUZZ_TARGET_MSG(pong); -FUZZ_TARGET_MSG(sendaddrv2); -FUZZ_TARGET_MSG(sendcmpct); -FUZZ_TARGET_MSG(sendheaders); -FUZZ_TARGET_MSG(sendtxrcncl); -FUZZ_TARGET_MSG(tx); -FUZZ_TARGET_MSG(verack); -FUZZ_TARGET_MSG(version); -FUZZ_TARGET_MSG(wtxidrelay); diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 6424f756a0..45a2294081 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -73,6 +73,7 @@ const std::vector<std::string> RPC_COMMANDS_NOT_SAFE_FOR_FUZZING{ "addpeeraddress", // avoid DNS lookups "dumptxoutset", // avoid writing to disk "dumpwallet", // avoid writing to disk + "enumeratesigners", "echoipc", // avoid assertion failure (Assertion `"EnsureAnyNodeContext(request.context).init" && check' failed.) "generatetoaddress", // avoid prohibitively slow execution (when `num_blocks` is large) "generatetodescriptor", // avoid prohibitively slow execution (when `nblocks` is large) @@ -136,6 +137,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "getnetworkinfo", "getnodeaddresses", "getpeerinfo", + "getprioritisedtransactions", "getrawmempool", "getrawtransaction", "getrpcinfo", diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 5de24a939d..e81efac6e0 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -5,6 +5,8 @@ #include <blockfilter.h> #include <clientversion.h> #include <common/args.h> +#include <common/settings.h> +#include <common/system.h> #include <common/url.h> #include <netbase.h> #include <outputtype.h> @@ -21,10 +23,8 @@ #include <test/fuzz/util.h> #include <util/error.h> #include <util/fees.h> -#include <util/settings.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/translation.h> #include <cassert> @@ -63,13 +63,9 @@ FUZZ_TARGET(string) (void)IsDeprecatedRPCEnabled(random_string_1); (void)Join(random_string_vector, random_string_1); (void)JSONRPCError(fuzzed_data_provider.ConsumeIntegral<int>(), random_string_1); - const util::Settings settings; + const common::Settings settings; (void)OnlyHasDefaultSectionSetting(settings, random_string_1, random_string_2); (void)ParseNetwork(random_string_1); - try { - (void)ParseNonRFCJSONValue(random_string_1); - } catch (const std::runtime_error&) { - } (void)ParseOutputType(random_string_1); (void)RemovePrefix(random_string_1, random_string_2); (void)ResolveErrMsg(random_string_1, random_string_2); diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp index 04cbbe52cb..73c01d9297 100644 --- a/src/test/fuzz/system.cpp +++ b/src/test/fuzz/system.cpp @@ -55,7 +55,7 @@ FUZZ_TARGET_INIT(system, initialize_system) [&] { const OptionsCategory options_category = fuzzed_data_provider.PickValueInArray<OptionsCategory>({OptionsCategory::OPTIONS, OptionsCategory::CONNECTION, OptionsCategory::WALLET, OptionsCategory::WALLET_DEBUG_TEST, OptionsCategory::ZMQ, OptionsCategory::DEBUG_TEST, OptionsCategory::CHAINPARAMS, OptionsCategory::NODE_RELAY, OptionsCategory::BLOCK_CREATION, OptionsCategory::RPC, OptionsCategory::GUI, OptionsCategory::COMMANDS, OptionsCategory::REGISTER_COMMANDS, OptionsCategory::HIDDEN}); // Avoid hitting: - // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + // common/args.cpp:563: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. const std::string argument_name = GetArgumentName(fuzzed_data_provider.ConsumeRandomLengthString(16)); if (args_manager.GetArgFlags(argument_name) != std::nullopt) { return; @@ -64,7 +64,7 @@ FUZZ_TARGET_INIT(system, initialize_system) }, [&] { // Avoid hitting: - // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + // common/args.cpp:563: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. const std::vector<std::string> names = ConsumeRandomLengthStringVector(fuzzed_data_provider); std::vector<std::string> hidden_arguments; for (const std::string& name : names) { diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index 7035c53d13..c561675d1a 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -101,7 +101,14 @@ FUZZ_TARGET_INIT(transaction, initialize_transaction) (void)AreInputsStandard(tx, coins_view_cache); (void)IsWitnessStandard(tx, coins_view_cache); - UniValue u(UniValue::VOBJ); - TxToUniv(tx, /*block_hash=*/uint256::ZERO, /*entry=*/u); - TxToUniv(tx, /*block_hash=*/uint256::ONE, /*entry=*/u); + if (tx.GetTotalSize() < 250'000) { // Avoid high memory usage (with msan) due to json encoding + { + UniValue u{UniValue::VOBJ}; + TxToUniv(tx, /*block_hash=*/uint256::ZERO, /*entry=*/u); + } + { + UniValue u{UniValue::VOBJ}; + TxToUniv(tx, /*block_hash=*/uint256::ONE, /*entry=*/u); + } + } } diff --git a/src/test/fuzz/utxo_total_supply.cpp b/src/test/fuzz/utxo_total_supply.cpp index 19f41880f4..318797faf2 100644 --- a/src/test/fuzz/utxo_total_supply.cpp +++ b/src/test/fuzz/utxo_total_supply.cpp @@ -119,7 +119,9 @@ FUZZ_TARGET(utxo_total_supply) current_block = PrepareNextBlock(); StoreLastTxo(); - LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 100'000) + // Limit to avoid timeout, but enough to cover duplicate_coinbase_height + // and CVE-2018-17144. + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 2'000) { CallOneOf( fuzzed_data_provider, @@ -144,12 +146,12 @@ FUZZ_TARGET(utxo_total_supply) const auto prev_utxo_stats = utxo_stats; if (was_valid) { - circulation += GetBlockSubsidy(ActiveHeight(), Params().GetConsensus()); - if (duplicate_coinbase_height == ActiveHeight()) { // we mined the duplicate coinbase assert(current_block->vtx.at(0)->vin.at(0).scriptSig == duplicate_coinbase_script); } + + circulation += GetBlockSubsidy(ActiveHeight(), Params().GetConsensus()); } UpdateUtxoStats(); diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 715b6885f5..c73b675388 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -3,10 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <common/args.h> +#include <common/settings.h> #include <logging.h> #include <test/util/setup_common.h> #include <univalue.h> -#include <util/settings.h> #include <util/strencodings.h> #include <limits> @@ -57,8 +57,8 @@ BOOST_AUTO_TEST_CASE(setting_args) ArgsManager args; SetupArgs(args, {{"-foo", ArgsManager::ALLOW_ANY}}); - auto set_foo = [&](const util::SettingsValue& value) { - args.LockSettings([&](util::Settings& settings) { + auto set_foo = [&](const common::SettingsValue& value) { + args.LockSettings([&](common::Settings& settings) { settings.rw_settings["foo"] = value; }); }; diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index ea5b94f3a5..86a8d17a76 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -4,6 +4,7 @@ #include <key.h> +#include <common/system.h> #include <key_io.h> #include <streams.h> #include <test/util/random.h> @@ -11,7 +12,6 @@ #include <uint256.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <string> #include <vector> @@ -344,4 +344,24 @@ BOOST_AUTO_TEST_CASE(bip340_test_vectors) } } +BOOST_AUTO_TEST_CASE(key_ellswift) +{ + for (const auto& secret : {strSecret1, strSecret2, strSecret1C, strSecret2C}) { + CKey key = DecodeSecret(secret); + BOOST_CHECK(key.IsValid()); + + uint256 ent32 = InsecureRand256(); + auto ellswift = key.EllSwiftCreate(AsBytes(Span{ent32})); + + CPubKey decoded_pubkey = ellswift.Decode(); + if (!key.IsCompressed()) { + // The decoding constructor returns a compressed pubkey. If the + // original was uncompressed, we must decompress the decoded one + // to compare. + decoded_pubkey.Decompress(); + } + BOOST_CHECK(key.GetPubKey() == decoded_pubkey); + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/logging_tests.cpp b/src/test/logging_tests.cpp index beb9398c74..2699d316da 100644 --- a/src/test/logging_tests.cpp +++ b/src/test/logging_tests.cpp @@ -200,7 +200,9 @@ BOOST_FIXTURE_TEST_CASE(logging_Conf, LogSetup) const char* argv_test[] = {"bitcoind", "-loglevel=debug"}; std::string err; BOOST_REQUIRE(args.ParseParameters(2, argv_test, err)); - init::SetLoggingLevel(args); + + auto result = init::SetLoggingLevel(args); + BOOST_REQUIRE(result); BOOST_CHECK_EQUAL(LogInstance().LogLevel(), BCLog::Level::Debug); } @@ -212,7 +214,9 @@ BOOST_FIXTURE_TEST_CASE(logging_Conf, LogSetup) const char* argv_test[] = {"bitcoind", "-loglevel=net:trace"}; std::string err; BOOST_REQUIRE(args.ParseParameters(2, argv_test, err)); - init::SetLoggingLevel(args); + + auto result = init::SetLoggingLevel(args); + BOOST_REQUIRE(result); BOOST_CHECK_EQUAL(LogInstance().LogLevel(), BCLog::DEFAULT_LOG_LEVEL); const auto& category_levels{LogInstance().CategoryLevels()}; @@ -229,7 +233,9 @@ BOOST_FIXTURE_TEST_CASE(logging_Conf, LogSetup) const char* argv_test[] = {"bitcoind", "-loglevel=debug", "-loglevel=net:trace", "-loglevel=http:info"}; std::string err; BOOST_REQUIRE(args.ParseParameters(4, argv_test, err)); - init::SetLoggingLevel(args); + + auto result = init::SetLoggingLevel(args); + BOOST_REQUIRE(result); BOOST_CHECK_EQUAL(LogInstance().LogLevel(), BCLog::Level::Debug); const auto& category_levels{LogInstance().CategoryLevels()}; diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 94e553a304..db58a0baec 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <policy/policy.h> #include <test/util/txmempool.h> #include <txmempool.h> -#include <util/system.h> #include <util/time.h> #include <test/util/setup_common.h> diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index cfab762307..94e3f27930 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <coins.h> +#include <common/system.h> #include <consensus/consensus.h> #include <consensus/merkle.h> #include <consensus/tx_verify.h> @@ -15,7 +16,6 @@ #include <txmempool.h> #include <uint256.h> #include <util/strencodings.h> -#include <util/system.h> #include <util/time.h> #include <validation.h> #include <versionbits.h> diff --git a/src/test/miniminer_tests.cpp b/src/test/miniminer_tests.cpp index 3f4a5fbe74..1ee9e0c066 100644 --- a/src/test/miniminer_tests.cpp +++ b/src/test/miniminer_tests.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <node/mini_miner.h> #include <txmempool.h> -#include <util/system.h> #include <util/time.h> #include <test/util/setup_common.h> @@ -138,7 +137,7 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) std::vector<CTransactionRef> all_transactions{tx1, tx2, tx3, tx4, tx5, tx6, tx7, tx8}; struct TxDimensions { - size_t vsize; CAmount mod_fee; CFeeRate feerate; + int32_t vsize; CAmount mod_fee; CFeeRate feerate; }; std::map<uint256, TxDimensions> tx_dims; for (const auto& tx : all_transactions) { @@ -464,8 +463,8 @@ BOOST_FIXTURE_TEST_CASE(calculate_cluster, TestChain100Setup) } const auto vec_iters_zigzag = pool.GetIterVec(zigzag_txids); // It doesn't matter which tx we calculate cluster for, everybody is in it. - const std::vector<size_t> indeces{0, 22, 72, zigzag_txids.size() - 1}; - for (const auto index : indeces) { + const std::vector<size_t> indices{0, 22, 72, zigzag_txids.size() - 1}; + for (const auto index : indices) { const auto cluster = pool.GatherClusters({zigzag_txids[index]}); BOOST_CHECK_EQUAL(cluster.size(), zigzag_txids.size()); CTxMemPool::setEntries clusterset{cluster.begin(), cluster.end()}; diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 631c213627..aa577f7b27 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -135,7 +135,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) CNetAddr addr; // IPv4, INADDR_ANY - BOOST_REQUIRE(LookupHost("0.0.0.0", addr, false)); + addr = LookupHost("0.0.0.0", false).value(); BOOST_REQUIRE(!addr.IsValid()); BOOST_REQUIRE(addr.IsIPv4()); @@ -144,7 +144,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_CHECK_EQUAL(addr.ToStringAddr(), "0.0.0.0"); // IPv4, INADDR_NONE - BOOST_REQUIRE(LookupHost("255.255.255.255", addr, false)); + addr = LookupHost("255.255.255.255", false).value(); BOOST_REQUIRE(!addr.IsValid()); BOOST_REQUIRE(addr.IsIPv4()); @@ -153,7 +153,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_CHECK_EQUAL(addr.ToStringAddr(), "255.255.255.255"); // IPv4, casual - BOOST_REQUIRE(LookupHost("12.34.56.78", addr, false)); + addr = LookupHost("12.34.56.78", false).value(); BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsIPv4()); @@ -162,7 +162,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_CHECK_EQUAL(addr.ToStringAddr(), "12.34.56.78"); // IPv6, in6addr_any - BOOST_REQUIRE(LookupHost("::", addr, false)); + addr = LookupHost("::", false).value(); BOOST_REQUIRE(!addr.IsValid()); BOOST_REQUIRE(addr.IsIPv6()); @@ -171,7 +171,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_CHECK_EQUAL(addr.ToStringAddr(), "::"); // IPv6, casual - BOOST_REQUIRE(LookupHost("1122:3344:5566:7788:9900:aabb:ccdd:eeff", addr, false)); + addr = LookupHost("1122:3344:5566:7788:9900:aabb:ccdd:eeff", false).value(); BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsIPv6()); @@ -186,14 +186,14 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) // id of "32", return the address as "fe80::1%32". const std::string link_local{"fe80::1"}; const std::string scoped_addr{link_local + "%32"}; - BOOST_REQUIRE(LookupHost(scoped_addr, addr, false)); + addr = LookupHost(scoped_addr, false).value(); BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsIPv6()); BOOST_CHECK(!addr.IsBindAny()); BOOST_CHECK_EQUAL(addr.ToStringAddr(), scoped_addr); // Test that the delimiter "%" and default zone id of 0 can be omitted for the default scope. - BOOST_REQUIRE(LookupHost(link_local + "%0", addr, false)); + addr = LookupHost(link_local + "%0", false).value(); BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsIPv6()); BOOST_CHECK(!addr.IsBindAny()); @@ -318,10 +318,9 @@ BOOST_AUTO_TEST_CASE(cnetaddr_tostring_canonical_ipv6) {"2001:db8:aaaa:bbbb:cccc:dddd:eeee:AaAa", "2001:db8:aaaa:bbbb:cccc:dddd:eeee:aaaa"}, }; for (const auto& [input_address, expected_canonical_representation_output] : canonical_representations_ipv6) { - CNetAddr net_addr; - BOOST_REQUIRE(LookupHost(input_address, net_addr, false)); - BOOST_REQUIRE(net_addr.IsIPv6()); - BOOST_CHECK_EQUAL(net_addr.ToStringAddr(), expected_canonical_representation_output); + const std::optional<CNetAddr> net_addr{LookupHost(input_address, false)}; + BOOST_REQUIRE(net_addr.value().IsIPv6()); + BOOST_CHECK_EQUAL(net_addr.value().ToStringAddr(), expected_canonical_representation_output); } } @@ -334,12 +333,12 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v1) BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000000000000000"); s.clear(); - BOOST_REQUIRE(LookupHost("1.2.3.4", addr, false)); + addr = LookupHost("1.2.3.4", false).value(); s << addr; BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000ffff01020304"); s.clear(); - BOOST_REQUIRE(LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", addr, false)); + addr = LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", false).value(); s << addr; BOOST_CHECK_EQUAL(HexStr(s), "1a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b"); s.clear(); @@ -370,12 +369,12 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v2) BOOST_CHECK_EQUAL(HexStr(s), "021000000000000000000000000000000000"); s.clear(); - BOOST_REQUIRE(LookupHost("1.2.3.4", addr, false)); + addr = LookupHost("1.2.3.4", false).value(); s << addr; BOOST_CHECK_EQUAL(HexStr(s), "010401020304"); s.clear(); - BOOST_REQUIRE(LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", addr, false)); + addr = LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", false).value(); s << addr; BOOST_CHECK_EQUAL(HexStr(s), "02101a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b"); s.clear(); diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 7e91819ddc..05953bfd10 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -24,9 +24,7 @@ BOOST_FIXTURE_TEST_SUITE(netbase_tests, BasicTestingSetup) static CNetAddr ResolveIP(const std::string& ip) { - CNetAddr addr; - LookupHost(ip, addr, false); - return addr; + return LookupHost(ip, false).value_or(CNetAddr{}); } static CSubNet ResolveSubNet(const std::string& subnet) @@ -477,11 +475,10 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) { - CNetAddr addr; - BOOST_CHECK(LookupHost("127.0.0.1"s, addr, false)); - BOOST_CHECK(!LookupHost("127.0.0.1\0"s, addr, false)); - BOOST_CHECK(!LookupHost("127.0.0.1\0example.com"s, addr, false)); - BOOST_CHECK(!LookupHost("127.0.0.1\0example.com\0"s, addr, false)); + BOOST_CHECK(LookupHost("127.0.0.1"s, false).has_value()); + BOOST_CHECK(!LookupHost("127.0.0.1\0"s, false).has_value()); + BOOST_CHECK(!LookupHost("127.0.0.1\0example.com"s, false).has_value()); + BOOST_CHECK(!LookupHost("127.0.0.1\0example.com\0"s, false).has_value()); CSubNet ret; BOOST_CHECK(LookupSubNet("1.2.3.0/24"s, ret)); BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0"s, ret)); diff --git a/src/test/rbf_tests.cpp b/src/test/rbf_tests.cpp index 0ec253747b..10205cd641 100644 --- a/src/test/rbf_tests.cpp +++ b/src/test/rbf_tests.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2021-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <policy/rbf.h> #include <random.h> #include <test/util/txmempool.h> #include <txmempool.h> -#include <util/system.h> #include <util/time.h> #include <test/util/setup_common.h> diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index da0029c737..2f783a4b95 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -42,11 +42,11 @@ private: class RPCTestingSetup : public TestingSetup { public: - UniValue TransformParams(const UniValue& params, std::vector<std::string> arg_names) const; + UniValue TransformParams(const UniValue& params, std::vector<std::pair<std::string, bool>> arg_names) const; UniValue CallRPC(std::string args); }; -UniValue RPCTestingSetup::TransformParams(const UniValue& params, std::vector<std::string> arg_names) const +UniValue RPCTestingSetup::TransformParams(const UniValue& params, std::vector<std::pair<std::string, bool>> arg_names) const { UniValue transformed_params; CRPCTable table; @@ -84,7 +84,7 @@ BOOST_FIXTURE_TEST_SUITE(rpc_tests, RPCTestingSetup) BOOST_AUTO_TEST_CASE(rpc_namedparams) { - const std::vector<std::string> arg_names{"arg1", "arg2", "arg3", "arg4", "arg5"}; + const std::vector<std::pair<std::string, bool>> arg_names{{"arg1", false}, {"arg2", false}, {"arg3", false}, {"arg4", false}, {"arg5", false}}; // Make sure named arguments are transformed into positional arguments in correct places separated by nulls BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg2": 2, "arg4": 4})"), arg_names).write(), "[null,2,null,4]"); @@ -109,6 +109,28 @@ BOOST_AUTO_TEST_CASE(rpc_namedparams) BOOST_CHECK_EQUAL(TransformParams(JSON(R"([1,2,3,4,5,6,7,8,9,10])"), arg_names).write(), "[1,2,3,4,5,6,7,8,9,10]"); } +BOOST_AUTO_TEST_CASE(rpc_namedonlyparams) +{ + const std::vector<std::pair<std::string, bool>> arg_names{{"arg1", false}, {"arg2", false}, {"opt1", true}, {"opt2", true}, {"options", false}}; + + // Make sure optional parameters are really optional. + BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg1": 1, "arg2": 2})"), arg_names).write(), "[1,2]"); + + // Make sure named-only parameters are passed as options. + BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg1": 1, "arg2": 2, "opt1": 10, "opt2": 20})"), arg_names).write(), R"([1,2,{"opt1":10,"opt2":20}])"); + + // Make sure options can be passed directly. + BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg1": 1, "arg2": 2, "options":{"opt1": 10, "opt2": 20}})"), arg_names).write(), R"([1,2,{"opt1":10,"opt2":20}])"); + + // Make sure options and named parameters conflict. + BOOST_CHECK_EXCEPTION(TransformParams(JSON(R"({"arg1": 1, "arg2": 2, "opt1": 10, "options":{"opt1": 10}})"), arg_names), UniValue, + HasJSON(R"({"code":-8,"message":"Parameter options conflicts with parameter opt1"})")); + + // Make sure options object specified through args array conflicts. + BOOST_CHECK_EXCEPTION(TransformParams(JSON(R"({"args": [1, 2, {"opt1": 10}], "opt2": 20})"), arg_names), UniValue, + HasJSON(R"({"code":-8,"message":"Parameter options specified twice both as positional and named argument"})")); +} + BOOST_AUTO_TEST_CASE(rpc_rawparams) { // Test raw transaction API argument handling @@ -278,6 +300,7 @@ BOOST_AUTO_TEST_CASE(rpc_parse_monetary_values) BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.00000001000000")), 1LL); //should pass, cut trailing 0 BOOST_CHECK_THROW(AmountFromValue(ValueFromString("19e-9")), UniValue); //should fail BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.19e-6")), 19); //should pass, leading 0 is present + BOOST_CHECK_EXCEPTION(AmountFromValue(".19e-6"), UniValue, HasJSON(R"({"code":-3,"message":"Invalid amount"})")); //should fail, no leading 0 BOOST_CHECK_THROW(AmountFromValue(ValueFromString("92233720368.54775808")), UniValue); //overflow error BOOST_CHECK_THROW(AmountFromValue(ValueFromString("1e+11")), UniValue); //overflow error @@ -285,36 +308,6 @@ BOOST_AUTO_TEST_CASE(rpc_parse_monetary_values) BOOST_CHECK_THROW(AmountFromValue(ValueFromString("93e+9")), UniValue); //overflow error } -BOOST_AUTO_TEST_CASE(json_parse_errors) -{ - // Valid - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("1.0").get_real(), 1.0); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("true").get_bool(), true); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("[false]")[0].get_bool(), false); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("{\"a\": true}")["a"].get_bool(), true); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("{\"1\": \"true\"}")["1"].get_str(), "true"); - // Valid, with leading or trailing whitespace - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue(" 1.0").get_real(), 1.0); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("1.0 ").get_real(), 1.0); - - BOOST_CHECK_THROW(AmountFromValue(ParseNonRFCJSONValue(".19e-6")), std::runtime_error); //should fail, missing leading 0, therefore invalid JSON - BOOST_CHECK_EQUAL(AmountFromValue(ParseNonRFCJSONValue("0.00000000000000000000000000000000000001e+30 ")), 1); - // Invalid, initial garbage - BOOST_CHECK_THROW(ParseNonRFCJSONValue("[1.0"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("a1.0"), std::runtime_error); - // Invalid, trailing garbage - BOOST_CHECK_THROW(ParseNonRFCJSONValue("1.0sds"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("1.0]"), std::runtime_error); - // Invalid, keys have to be names - BOOST_CHECK_THROW(ParseNonRFCJSONValue("{1: \"true\"}"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("{true: 1}"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("{[1]: 1}"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("{{\"a\": \"a\"}: 1}"), std::runtime_error); - // BTC addresses should fail parsing - BOOST_CHECK_THROW(ParseNonRFCJSONValue("175tWpb8K1S7NmH4Zx6rewF9WQrcZv245W"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("3J98t1WpEZ73CNmQviecrnyiWrnqRhWNL"), std::runtime_error); -} - BOOST_AUTO_TEST_CASE(rpc_ban) { BOOST_CHECK_NO_THROW(CallRPC(std::string("clearbanned"))); @@ -506,6 +499,53 @@ BOOST_AUTO_TEST_CASE(rpc_getblockstats_calculate_percentiles_by_weight) } } +// Make sure errors are triggered appropriately if parameters have the same names. +BOOST_AUTO_TEST_CASE(check_dup_param_names) +{ + enum ParamType { POSITIONAL, NAMED, NAMED_ONLY }; + auto make_rpc = [](std::vector<std::tuple<std::string, ParamType>> param_names) { + std::vector<RPCArg> params; + std::vector<RPCArg> options; + auto push_options = [&] { if (!options.empty()) params.emplace_back(RPCArg{strprintf("options%i", params.size()), RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", std::move(options)}); }; + for (auto& [param_name, param_type] : param_names) { + if (param_type == POSITIONAL) { + push_options(); + params.emplace_back(std::move(param_name), RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "description"); + } else { + options.emplace_back(std::move(param_name), RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "description", RPCArgOptions{.also_positional = param_type == NAMED}); + } + } + push_options(); + return RPCHelpMan{"method_name", "description", params, RPCResults{}, RPCExamples{""}}; + }; + + // No errors if parameter names are unique. + make_rpc({{"p1", POSITIONAL}, {"p2", POSITIONAL}}); + make_rpc({{"p1", POSITIONAL}, {"p2", NAMED}}); + make_rpc({{"p1", POSITIONAL}, {"p2", NAMED_ONLY}}); + make_rpc({{"p1", NAMED}, {"p2", POSITIONAL}}); + make_rpc({{"p1", NAMED}, {"p2", NAMED}}); + make_rpc({{"p1", NAMED}, {"p2", NAMED_ONLY}}); + make_rpc({{"p1", NAMED_ONLY}, {"p2", POSITIONAL}}); + make_rpc({{"p1", NAMED_ONLY}, {"p2", NAMED}}); + make_rpc({{"p1", NAMED_ONLY}, {"p2", NAMED_ONLY}}); + + // Error if parameters names are duplicates, unless one parameter is + // positional and the other is named and .also_positional is true. + BOOST_CHECK_THROW(make_rpc({{"p1", POSITIONAL}, {"p1", POSITIONAL}}), NonFatalCheckError); + make_rpc({{"p1", POSITIONAL}, {"p1", NAMED}}); + BOOST_CHECK_THROW(make_rpc({{"p1", POSITIONAL}, {"p1", NAMED_ONLY}}), NonFatalCheckError); + make_rpc({{"p1", NAMED}, {"p1", POSITIONAL}}); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED}, {"p1", NAMED}}), NonFatalCheckError); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED}, {"p1", NAMED_ONLY}}), NonFatalCheckError); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED_ONLY}, {"p1", POSITIONAL}}), NonFatalCheckError); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED_ONLY}, {"p1", NAMED}}), NonFatalCheckError); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED_ONLY}, {"p1", NAMED_ONLY}}), NonFatalCheckError); + + // Make sure duplicate aliases are detected too. + BOOST_CHECK_THROW(make_rpc({{"p1", POSITIONAL}, {"p2|p1", NAMED_ONLY}}), NonFatalCheckError); +} + BOOST_AUTO_TEST_CASE(help_example) { // test different argument types diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index d8898743b0..c89f2c1f5b 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -5,6 +5,7 @@ #include <test/data/script_tests.json.h> #include <test/data/bip341_wallet_vectors.json.h> +#include <common/system.h> #include <core_io.h> #include <key.h> #include <rpc/util.h> @@ -20,7 +21,6 @@ #include <test/util/transaction_utils.h> #include <util/fs.h> #include <util/strencodings.h> -#include <util/system.h> #if defined(HAVE_CONSENSUS_LIB) #include <script/bitcoinconsensus.h> diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 09f77d2b61..b445ff8ffc 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -186,32 +186,32 @@ BOOST_AUTO_TEST_CASE(noncanonical) std::vector<char>::size_type n; // zero encoded with three bytes: - ss.write(MakeByteSpan("\xfd\x00\x00").first(3)); + ss << Span{"\xfd\x00\x00"}.first(3); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); // 0xfc encoded with three bytes: - ss.write(MakeByteSpan("\xfd\xfc\x00").first(3)); + ss << Span{"\xfd\xfc\x00"}.first(3); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); // 0xfd encoded with three bytes is OK: - ss.write(MakeByteSpan("\xfd\xfd\x00").first(3)); + ss << Span{"\xfd\xfd\x00"}.first(3); n = ReadCompactSize(ss); BOOST_CHECK(n == 0xfd); // zero encoded with five bytes: - ss.write(MakeByteSpan("\xfe\x00\x00\x00\x00").first(5)); + ss << Span{"\xfe\x00\x00\x00\x00"}.first(5); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); // 0xffff encoded with five bytes: - ss.write(MakeByteSpan("\xfe\xff\xff\x00\x00").first(5)); + ss << Span{"\xfe\xff\xff\x00\x00"}.first(5); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); // zero encoded with nine bytes: - ss.write(MakeByteSpan("\xff\x00\x00\x00\x00\x00\x00\x00\x00").first(9)); + ss << Span{"\xff\x00\x00\x00\x00\x00\x00\x00\x00"}.first(9); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); // 0x01ffffff encoded with nine bytes: - ss.write(MakeByteSpan("\xff\xff\xff\xff\x01\x00\x00\x00\x00").first(9)); + ss << Span{"\xff\xff\xff\xff\x01\x00\x00\x00\x00"}.first(9); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); } @@ -241,6 +241,15 @@ BOOST_AUTO_TEST_CASE(class_methods) ss2 << intval << boolval << stringval << charstrval << txval; ss2 >> methodtest3; BOOST_CHECK(methodtest3 == methodtest4); + { + DataStream ds; + const std::string in{"ab"}; + ds << Span{in}; + std::array<std::byte, 2> out; + ds >> Span{out}; + BOOST_CHECK_EQUAL(out.at(0), std::byte{'a'}); + BOOST_CHECK_EQUAL(out.at(1), std::byte{'b'}); + } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index fff84d24f0..eb11df0497 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/settings.h> +#include <common/settings.h> #include <test/util/setup_common.h> #include <test/util/str.h> @@ -21,21 +21,21 @@ #include <system_error> #include <vector> -inline bool operator==(const util::SettingsValue& a, const util::SettingsValue& b) +inline bool operator==(const common::SettingsValue& a, const common::SettingsValue& b) { return a.write() == b.write(); } -inline std::ostream& operator<<(std::ostream& os, const util::SettingsValue& value) +inline std::ostream& operator<<(std::ostream& os, const common::SettingsValue& value) { os << value.write(); return os; } -inline std::ostream& operator<<(std::ostream& os, const std::pair<std::string, util::SettingsValue>& kv) +inline std::ostream& operator<<(std::ostream& os, const std::pair<std::string, common::SettingsValue>& kv) { - util::SettingsValue out(util::SettingsValue::VOBJ); - out.__pushKV(kv.first, kv.second); + common::SettingsValue out(common::SettingsValue::VOBJ); + out.pushKVEnd(kv.first, kv.second); os << out.write(); return os; } @@ -60,7 +60,7 @@ BOOST_AUTO_TEST_CASE(ReadWrite) "null": null })"); - std::map<std::string, util::SettingsValue> expected{ + std::map<std::string, common::SettingsValue> expected{ {"string", "string"}, {"num", 5}, {"bool", true}, @@ -68,15 +68,15 @@ BOOST_AUTO_TEST_CASE(ReadWrite) }; // Check file read. - std::map<std::string, util::SettingsValue> values; + std::map<std::string, common::SettingsValue> values; std::vector<std::string> errors; - BOOST_CHECK(util::ReadSettings(path, values, errors)); + BOOST_CHECK(common::ReadSettings(path, values, errors)); BOOST_CHECK_EQUAL_COLLECTIONS(values.begin(), values.end(), expected.begin(), expected.end()); BOOST_CHECK(errors.empty()); // Check no errors if file doesn't exist. fs::remove(path); - BOOST_CHECK(util::ReadSettings(path, values, errors)); + BOOST_CHECK(common::ReadSettings(path, values, errors)); BOOST_CHECK(values.empty()); BOOST_CHECK(errors.empty()); @@ -85,29 +85,29 @@ BOOST_AUTO_TEST_CASE(ReadWrite) "dupe": "string", "dupe": "dupe" })"); - BOOST_CHECK(!util::ReadSettings(path, values, errors)); + BOOST_CHECK(!common::ReadSettings(path, values, errors)); std::vector<std::string> dup_keys = {strprintf("Found duplicate key dupe in settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), dup_keys.begin(), dup_keys.end()); BOOST_CHECK(values.empty()); // Check non-kv json files not allowed WriteText(path, R"("non-kv")"); - BOOST_CHECK(!util::ReadSettings(path, values, errors)); + BOOST_CHECK(!common::ReadSettings(path, values, errors)); std::vector<std::string> non_kv = {strprintf("Found non-object value \"non-kv\" in settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), non_kv.begin(), non_kv.end()); // Check invalid json not allowed WriteText(path, R"(invalid json)"); - BOOST_CHECK(!util::ReadSettings(path, values, errors)); + BOOST_CHECK(!common::ReadSettings(path, values, errors)); std::vector<std::string> fail_parse = {strprintf("Unable to parse settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), fail_parse.begin(), fail_parse.end()); } //! Check settings struct contents against expected json strings. -static void CheckValues(const util::Settings& settings, const std::string& single_val, const std::string& list_val) +static void CheckValues(const common::Settings& settings, const std::string& single_val, const std::string& list_val) { - util::SettingsValue single_value = GetSetting(settings, "section", "name", false, false, false); - util::SettingsValue list_value(util::SettingsValue::VARR); + common::SettingsValue single_value = GetSetting(settings, "section", "name", false, false, false); + common::SettingsValue list_value(common::SettingsValue::VARR); for (const auto& item : GetSettingsList(settings, "section", "name", false)) { list_value.push_back(item); } @@ -118,7 +118,7 @@ static void CheckValues(const util::Settings& settings, const std::string& singl // Simple settings merge test case. BOOST_AUTO_TEST_CASE(Simple) { - util::Settings settings; + common::Settings settings; settings.command_line_options["name"].push_back("val1"); settings.command_line_options["name"].push_back("val2"); settings.ro_config["section"]["name"].push_back(2); @@ -126,7 +126,7 @@ BOOST_AUTO_TEST_CASE(Simple) // The last given arg takes precedence when specified via commandline. CheckValues(settings, R"("val2")", R"(["val1","val2",2])"); - util::Settings settings2; + common::Settings settings2; settings2.ro_config["section"]["name"].push_back("val2"); settings2.ro_config["section"]["name"].push_back("val3"); @@ -140,7 +140,7 @@ BOOST_AUTO_TEST_CASE(Simple) // its default value. BOOST_AUTO_TEST_CASE(NullOverride) { - util::Settings settings; + common::Settings settings; settings.command_line_options["name"].push_back("value"); BOOST_CHECK_EQUAL(R"("value")", GetSetting(settings, "section", "name", false, false, false).write().c_str()); settings.forced_settings["name"] = {}; @@ -195,11 +195,11 @@ BOOST_FIXTURE_TEST_CASE(Merge, MergeTestingSetup) bool ignore_default_section_config) { std::string desc; int value_suffix = 0; - util::Settings settings; + common::Settings settings; const std::string& name = ignore_default_section_config ? "wallet" : "server"; auto push_values = [&](Action action, const char* value_prefix, const std::string& name_prefix, - std::vector<util::SettingsValue>& dest) { + std::vector<common::SettingsValue>& dest) { if (action == SET || action == SECTION_SET) { for (int i = 0; i < 2; ++i) { dest.push_back(value_prefix + ToString(++value_suffix)); diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index e2d11afa6a..68ef719c71 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <consensus/tx_check.h> #include <consensus/validation.h> #include <hash.h> @@ -14,7 +15,6 @@ #include <test/util/random.h> #include <test/util/setup_common.h> #include <util/strencodings.h> -#include <util/system.h> #include <version.h> #include <iostream> diff --git a/src/test/sock_tests.cpp b/src/test/sock_tests.cpp index 9e6f73745e..26ee724bf8 100644 --- a/src/test/sock_tests.cpp +++ b/src/test/sock_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <compat/compat.h> #include <test/util/setup_common.h> #include <util/sock.h> -#include <util/system.h> #include <util/threadinterrupt.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index 7ce350b84b..740f461548 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -7,16 +7,7 @@ #include <univalue.h> #ifdef ENABLE_EXTERNAL_SIGNER -#if defined(__GNUC__) -// Boost 1.78 requires the following workaround. -// See: https://github.com/boostorg/process/issues/235 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wnarrowing" -#endif #include <boost/process.hpp> -#if defined(__GNUC__) -#pragma GCC diagnostic pop -#endif #endif // ENABLE_EXTERNAL_SIGNER #include <boost/test/unit_test.hpp> diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index b666517ae2..2677502ef0 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -6,8 +6,8 @@ #include <index/txindex.h> #include <interfaces/chain.h> #include <script/standard.h> +#include <test/util/index.h> #include <test/util/setup_common.h> -#include <util/time.h> #include <validation.h> #include <boost/test/unit_test.hpp> @@ -32,12 +32,7 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) BOOST_REQUIRE(txindex.Start()); // Allow tx index to catch up with the block index. - constexpr auto timeout{10s}; - const auto time_start{SteadyClock::now()}; - while (!txindex.BlockUntilSyncedToCurrentChain()) { - BOOST_REQUIRE(time_start + timeout > SteadyClock::now()); - UninterruptibleSleep(std::chrono::milliseconds{100}); - } + IndexWaitSynced(txindex); // Check that txindex excludes genesis block transactions. const CBlock& genesis_block = Params().GenesisBlock(); diff --git a/src/test/util/index.cpp b/src/test/util/index.cpp new file mode 100644 index 0000000000..2bda7b1773 --- /dev/null +++ b/src/test/util/index.cpp @@ -0,0 +1,18 @@ +// Copyright (c) 2020-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/index.h> + +#include <index/base.h> +#include <util/check.h> +#include <util/time.h> + +void IndexWaitSynced(BaseIndex& index) +{ + const auto timeout{SteadyClock::now() + 120s}; + while (!index.BlockUntilSyncedToCurrentChain()) { + Assert(timeout > SteadyClock::now()); + UninterruptibleSleep(100ms); + } +} diff --git a/src/test/util/index.h b/src/test/util/index.h new file mode 100644 index 0000000000..02030fd562 --- /dev/null +++ b/src/test/util/index.h @@ -0,0 +1,13 @@ +// Copyright (c) 2020-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_INDEX_H +#define BITCOIN_TEST_UTIL_INDEX_H + +class BaseIndex; + +/** Block until the index is synced to the current chain */ +void IndexWaitSynced(BaseIndex& index); + +#endif // BITCOIN_TEST_UTIL_INDEX_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index eedb406cbd..93a60db832 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -9,6 +9,7 @@ #include <addrman.h> #include <banman.h> #include <chainparams.h> +#include <common/system.h> #include <common/url.h> #include <consensus/consensus.h> #include <consensus/params.h> @@ -23,6 +24,7 @@ #include <node/blockstorage.h> #include <node/chainstate.h> #include <node/context.h> +#include <node/kernel_notifications.h> #include <node/mempool_args.h> #include <node/miner.h> #include <node/validation_cache_args.h> @@ -45,7 +47,6 @@ #include <util/chaintype.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/thread.h> #include <util/threadnames.h> #include <util/time.h> @@ -64,6 +65,7 @@ using node::ApplyArgsManOptions; using node::BlockAssembler; using node::BlockManager; using node::CalculateCacheSizes; +using node::KernelNotifications; using node::LoadChainstate; using node::RegenerateCommitments; using node::VerifyLoadedChainstate; @@ -177,16 +179,19 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, const std::vecto m_node.scheduler->m_service_thread = std::thread(util::TraceThread, "scheduler", [&] { m_node.scheduler->serviceQueue(); }); GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler); - m_node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(FeeestPath(*m_node.args)); + m_node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(FeeestPath(*m_node.args), DEFAULT_ACCEPT_STALE_FEE_ESTIMATES); m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node)); m_cache_sizes = CalculateCacheSizes(m_args); + m_node.notifications = std::make_unique<KernelNotifications>(); + const ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = m_args.GetDataDirNet(), .adjusted_time_callback = GetAdjustedTime, .check_block_index = true, + .notifications = *m_node.notifications, }; const BlockManager::Options blockman_opts{ .chainparams = chainman_opts.chainparams, diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index bd5a81be45..b7429df02c 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -8,7 +8,7 @@ #include <common/args.h> #include <key.h> #include <node/caches.h> -#include <node/context.h> +#include <node/context.h> // IWYU pragma: export #include <primitives/transaction.h> #include <pubkey.h> #include <random.h> diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp index 1873cf5ec8..4797d9c310 100644 --- a/src/test/util/txmempool.cpp +++ b/src/test/util/txmempool.cpp @@ -22,8 +22,8 @@ CTxMemPool::Options MemPoolOptionsForTest(const NodeContext& node) // chainparams.DefaultConsistencyChecks for tests .check_ratio = 1, }; - const auto err{ApplyArgsManOptions(*node.args, ::Params(), mempool_opts)}; - Assert(!err); + const auto result{ApplyArgsManOptions(*node.args, ::Params(), mempool_opts)}; + Assert(result); return mempool_opts; } diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 812737429d..26677bfa55 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -1687,7 +1687,7 @@ BOOST_AUTO_TEST_CASE(message_hash) BOOST_AUTO_TEST_CASE(remove_prefix) { - BOOST_CHECK_EQUAL(RemovePrefix("./util/system.h", "./"), "util/system.h"); + BOOST_CHECK_EQUAL(RemovePrefix("./common/system.h", "./"), "common/system.h"); BOOST_CHECK_EQUAL(RemovePrefixView("foo", "foo"), ""); BOOST_CHECK_EQUAL(RemovePrefix("foo", "fo"), "o"); BOOST_CHECK_EQUAL(RemovePrefixView("foo", "f"), "oo"); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 05e2787075..b797de46af 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -4,6 +4,7 @@ // #include <chainparams.h> #include <consensus/validation.h> +#include <node/kernel_notifications.h> #include <node/utxo_snapshot.h> #include <random.h> #include <rpc/blockchain.h> @@ -23,6 +24,7 @@ #include <boost/test/unit_test.hpp> using node::BlockManager; +using node::KernelNotifications; using node::SnapshotMetadata; BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, ChainTestingSetup) @@ -182,7 +184,7 @@ struct SnapshotTestSetup : TestChain100Setup { { LOCK(::cs_main); BOOST_CHECK(!chainman.IsSnapshotValidated()); - BOOST_CHECK(!node::FindSnapshotChainstateDir()); + BOOST_CHECK(!node::FindSnapshotChainstateDir(chainman.m_options.datadir)); } size_t initial_size; @@ -232,7 +234,7 @@ struct SnapshotTestSetup : TestChain100Setup { auto_infile >> coin; })); - BOOST_CHECK(!node::FindSnapshotChainstateDir()); + BOOST_CHECK(!node::FindSnapshotChainstateDir(chainman.m_options.datadir)); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { @@ -256,7 +258,7 @@ struct SnapshotTestSetup : TestChain100Setup { })); BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(this)); - BOOST_CHECK(fs::exists(*node::FindSnapshotChainstateDir())); + BOOST_CHECK(fs::exists(*node::FindSnapshotChainstateDir(chainman.m_options.datadir))); // Ensure our active chain is the snapshot chainstate. BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull()); @@ -269,7 +271,7 @@ struct SnapshotTestSetup : TestChain100Setup { { LOCK(::cs_main); - fs::path found = *node::FindSnapshotChainstateDir(); + fs::path found = *node::FindSnapshotChainstateDir(chainman.m_options.datadir); // Note: WriteSnapshotBaseBlockhash() is implicitly tested above. BOOST_CHECK_EQUAL( @@ -377,10 +379,12 @@ struct SnapshotTestSetup : TestChain100Setup { LOCK(::cs_main); chainman.ResetChainstates(); BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0); + m_node.notifications = std::make_unique<KernelNotifications>(); const ChainstateManager::Options chainman_opts{ .chainparams = ::Params(), - .datadir = m_args.GetDataDirNet(), + .datadir = chainman.m_options.datadir, .adjusted_time_callback = GetAdjustedTime, + .notifications = *m_node.notifications, }; const BlockManager::Options blockman_opts{ .chainparams = chainman_opts.chainparams, @@ -487,7 +491,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_init, SnapshotTestSetup) this->SetupSnapshot(); - fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(); + fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(chainman.m_options.datadir); BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot"); @@ -561,7 +565,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_completion, SnapshotTestSetup SnapshotCompletionResult res; auto mock_shutdown = [](bilingual_str msg) {}; - fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(); + fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(chainman.m_options.datadir); BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot"); diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 92b55f9fc4..a8d6fb4b3f 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -16,7 +16,6 @@ #include <netbase.h> #include <util/readwritefile.h> #include <util/strencodings.h> -#include <util/syscall_sandbox.h> #include <util/thread.h> #include <util/time.h> @@ -133,15 +132,15 @@ bool TorControlConnection::Connect(const std::string& tor_control_center, const Disconnect(); } - CService control_service; - if (!Lookup(tor_control_center, control_service, 9051, fNameLookup)) { + const std::optional<CService> control_service{Lookup(tor_control_center, 9051, fNameLookup)}; + if (!control_service.has_value()) { LogPrintf("tor: Failed to look up control center %s\n", tor_control_center); return false; } struct sockaddr_storage control_address; socklen_t control_address_len = sizeof(control_address); - if (!control_service.GetSockAddr(reinterpret_cast<struct sockaddr*>(&control_address), &control_address_len)) { + if (!control_service.value().GetSockAddr(reinterpret_cast<struct sockaddr*>(&control_address), &control_address_len)) { LogPrintf("tor: Error parsing socket address %s\n", tor_control_center); return false; } @@ -653,7 +652,6 @@ static std::thread torControlThread; static void TorControlThread(CService onion_service_target) { - SetSyscallSandboxPolicy(SyscallSandboxPolicy::TOR_CONTROL); TorController ctrl(gBase, gArgs.GetArg("-torcontrol", DEFAULT_TOR_CONTROL), onion_service_target); event_base_dispatch(gBase); diff --git a/src/txdb.cpp b/src/txdb.cpp index 15351a4355..b2095bd4a3 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -31,22 +31,22 @@ static constexpr uint8_t DB_COINS{'c'}; static constexpr uint8_t DB_TXINDEX_BLOCK{'T'}; // uint8_t DB_TXINDEX{'t'} -std::optional<bilingual_str> CheckLegacyTxindex(CBlockTreeDB& block_tree_db) +util::Result<void> CheckLegacyTxindex(CBlockTreeDB& block_tree_db) { CBlockLocator ignored{}; if (block_tree_db.Read(DB_TXINDEX_BLOCK, ignored)) { - return _("The -txindex upgrade started by a previous version cannot be completed. Restart with the previous version or run a full -reindex."); + return util::Error{_("The -txindex upgrade started by a previous version cannot be completed. Restart with the previous version or run a full -reindex.")}; } bool txindex_legacy_flag{false}; block_tree_db.ReadFlag("txindex", txindex_legacy_flag); if (txindex_legacy_flag) { // Disable legacy txindex and warn once about occupied disk space if (!block_tree_db.WriteFlag("txindex", false)) { - return Untranslated("Failed to write block index db flag 'txindex'='0'"); + return util::Error{Untranslated("Failed to write block index db flag 'txindex'='0'")}; } - return _("The block index db contains a legacy 'txindex'. To clear the occupied disk space, run a full -reindex, otherwise ignore this error. This error message will not be displayed again."); + return util::Error{_("The block index db contains a legacy 'txindex'. To clear the occupied disk space, run a full -reindex, otherwise ignore this error. This error message will not be displayed again.")}; } - return std::nullopt; + return {}; } bool CCoinsViewDB::NeedsUpgrade() diff --git a/src/txdb.h b/src/txdb.h index 63c7152097..04d0ecb39f 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -11,7 +11,11 @@ #include <kernel/cs_main.h> #include <sync.h> #include <util/fs.h> +#include <util/result.h> +#include <cstddef> +#include <cstdint> +#include <functional> #include <memory> #include <optional> #include <string> @@ -20,11 +24,11 @@ class CBlockFileInfo; class CBlockIndex; +class COutPoint; class uint256; namespace Consensus { struct Params; }; -struct bilingual_str; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; @@ -98,6 +102,6 @@ public: EXCLUSIVE_LOCKS_REQUIRED(::cs_main); }; -std::optional<bilingual_str> CheckLegacyTxindex(CBlockTreeDB& block_tree_db); +[[nodiscard]] util::Result<void> CheckLegacyTxindex(CBlockTreeDB& block_tree_db); #endif // BITCOIN_TXDB_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 6019346eed..845fbdb66e 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -7,6 +7,7 @@ #include <chain.h> #include <coins.h> +#include <common/system.h> #include <consensus/consensus.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> @@ -19,7 +20,6 @@ #include <util/moneystr.h> #include <util/overflow.h> #include <util/result.h> -#include <util/system.h> #include <util/time.h> #include <util/trace.h> #include <util/translation.h> @@ -75,7 +75,7 @@ void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap& cachedDescendan } // descendants now contains all in-mempool descendants of updateIt. // Update and add to cached descendant map - int64_t modifySize = 0; + int32_t modifySize = 0; CAmount modifyFee = 0; int64_t modifyCount = 0; for (const CTxMemPoolEntry& descendant : descendants) { @@ -91,7 +91,7 @@ void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap& cachedDescendan // Don't directly remove the transaction here -- doing so would // invalidate iterators in cachedDescendants. Mark it for removal // by inserting into descendants_to_remove. - if (descendant.GetCountWithAncestors() > uint64_t(m_limits.ancestor_count) || descendant.GetSizeWithAncestors() > uint64_t(m_limits.ancestor_size_vbytes)) { + if (descendant.GetCountWithAncestors() > uint64_t(m_limits.ancestor_count) || descendant.GetSizeWithAncestors() > m_limits.ancestor_size_vbytes) { descendants_to_remove.insert(descendant.GetTx().GetHash()); } } @@ -278,8 +278,8 @@ void CTxMemPool::UpdateAncestorsOf(bool add, txiter it, setEntries &setAncestors for (const CTxMemPoolEntry& parent : parents) { UpdateChild(mapTx.iterator_to(parent), it, add); } - const int64_t updateCount = (add ? 1 : -1); - const int64_t updateSize = updateCount * it->GetTxSize(); + const int32_t updateCount = (add ? 1 : -1); + const int32_t updateSize{updateCount * it->GetTxSize()}; const CAmount updateFee = updateCount * it->GetModifiedFee(); for (txiter ancestorIt : setAncestors) { mapTx.modify(ancestorIt, [=](CTxMemPoolEntry& e) { e.UpdateDescendantState(updateSize, updateFee, updateCount); }); @@ -323,7 +323,7 @@ void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, b setEntries setDescendants; CalculateDescendants(removeIt, setDescendants); setDescendants.erase(removeIt); // don't update state for self - int64_t modifySize = -((int64_t)removeIt->GetTxSize()); + int32_t modifySize = -removeIt->GetTxSize(); CAmount modifyFee = -removeIt->GetModifiedFee(); int modifySigOps = -removeIt->GetSigOpCost(); for (txiter dit : setDescendants) { @@ -365,22 +365,22 @@ void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, b } } -void CTxMemPoolEntry::UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount) +void CTxMemPoolEntry::UpdateDescendantState(int32_t modifySize, CAmount modifyFee, int64_t modifyCount) { nSizeWithDescendants += modifySize; - assert(int64_t(nSizeWithDescendants) > 0); + assert(nSizeWithDescendants > 0); nModFeesWithDescendants = SaturatingAdd(nModFeesWithDescendants, modifyFee); - nCountWithDescendants += modifyCount; - assert(int64_t(nCountWithDescendants) > 0); + m_count_with_descendants += modifyCount; + assert(m_count_with_descendants > 0); } -void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps) +void CTxMemPoolEntry::UpdateAncestorState(int32_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps) { nSizeWithAncestors += modifySize; - assert(int64_t(nSizeWithAncestors) > 0); + assert(nSizeWithAncestors > 0); nModFeesWithAncestors = SaturatingAdd(nModFeesWithAncestors, modifyFee); - nCountWithAncestors += modifyCount; - assert(int64_t(nCountWithAncestors) > 0); + m_count_with_ancestors += modifyCount; + assert(m_count_with_ancestors > 0); nSigOpCostWithAncestors += modifySigOps; assert(int(nSigOpCostWithAncestors) >= 0); } @@ -699,7 +699,7 @@ void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendhei // Verify ancestor state is correct. auto ancestors{AssumeCalculateMemPoolAncestors(__func__, *it, Limits::NoLimits())}; uint64_t nCountCheck = ancestors.size() + 1; - uint64_t nSizeCheck = it->GetTxSize(); + int32_t nSizeCheck = it->GetTxSize(); CAmount nFeesCheck = it->GetModifiedFee(); int64_t nSigOpCheck = it->GetSigOpCost(); @@ -720,7 +720,7 @@ void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendhei // Check children against mapNextTx CTxMemPoolEntry::Children setChildrenCheck; auto iter = mapNextTx.lower_bound(COutPoint(it->GetTx().GetHash(), 0)); - uint64_t child_sizes = 0; + int32_t child_sizes{0}; for (; iter != mapNextTx.end() && iter->first->hash == it->GetTx().GetHash(); ++iter) { txiter childit = mapTx.find(iter->second->GetHash()); assert(childit != mapTx.end()); // mapNextTx points to in-mempool transactions @@ -876,8 +876,17 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD } ++nTransactionsUpdated; } + if (delta == 0) { + mapDeltas.erase(hash); + LogPrintf("PrioritiseTransaction: %s (%sin mempool) delta cleared\n", hash.ToString(), it == mapTx.end() ? "not " : ""); + } else { + LogPrintf("PrioritiseTransaction: %s (%sin mempool) fee += %s, new delta=%s\n", + hash.ToString(), + it == mapTx.end() ? "not " : "", + FormatMoney(nFeeDelta), + FormatMoney(delta)); + } } - LogPrintf("PrioritiseTransaction: %s fee += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); } void CTxMemPool::ApplyDelta(const uint256& hash, CAmount &nFeeDelta) const @@ -896,6 +905,22 @@ void CTxMemPool::ClearPrioritisation(const uint256& hash) mapDeltas.erase(hash); } +std::vector<CTxMemPool::delta_info> CTxMemPool::GetPrioritisedTransactions() const +{ + AssertLockNotHeld(cs); + LOCK(cs); + std::vector<delta_info> result; + result.reserve(mapDeltas.size()); + for (const auto& [txid, delta] : mapDeltas) { + const auto iter{mapTx.find(txid)}; + const bool in_mempool{iter != mapTx.end()}; + std::optional<CAmount> modified_fee; + if (in_mempool) modified_fee = iter->GetModifiedFee(); + result.emplace_back(delta_info{in_mempool, delta, modified_fee, txid}); + } + return result; +} + const CTransaction* CTxMemPool::GetConflictTx(const COutPoint& prevout) const { const auto it = mapNextTx.find(prevout); diff --git a/src/txmempool.h b/src/txmempool.h index 769b7f69ea..846def02cd 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -33,8 +33,11 @@ #include <util/result.h> #include <boost/multi_index/hashed_index.hpp> +#include <boost/multi_index/identity.hpp> +#include <boost/multi_index/indexed_by.hpp> #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/sequenced_index.hpp> +#include <boost/multi_index/tag.hpp> #include <boost/multi_index_container.hpp> class CBlockIndex; @@ -219,7 +222,7 @@ struct TxMempoolInfo CAmount fee; /** Virtual size of the transaction. */ - size_t vsize; + int32_t vsize; /** The fee delta. */ int64_t nFeeDelta; @@ -516,6 +519,19 @@ public: void ApplyDelta(const uint256& hash, CAmount &nFeeDelta) const EXCLUSIVE_LOCKS_REQUIRED(cs); void ClearPrioritisation(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs); + struct delta_info { + /** Whether this transaction is in the mempool. */ + const bool in_mempool; + /** The fee delta added using PrioritiseTransaction(). */ + const CAmount delta; + /** The modified fee (base fee + delta) of this entry. Only present if in_mempool=true. */ + std::optional<CAmount> modified_fee; + /** The prioritised transaction's txid. */ + const uint256 txid; + }; + /** Return a vector of all entries in mapDeltas with their corresponding delta_info. */ + std::vector<delta_info> GetPrioritisedTransactions() const EXCLUSIVE_LOCKS_REQUIRED(!cs); + /** Get the transaction in the pool that spends the same prevout */ const CTransaction* GetConflictTx(const COutPoint& prevout) const EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/txorphanage.cpp b/src/txorphanage.cpp index 19f9fae998..af86baa8ac 100644 --- a/src/txorphanage.cpp +++ b/src/txorphanage.cpp @@ -55,10 +55,10 @@ bool TxOrphanage::AddTx(const CTransactionRef& tx, NodeId peer) int TxOrphanage::EraseTx(const uint256& txid) { LOCK(m_mutex); - return _EraseTx(txid); + return EraseTxNoLock(txid); } -int TxOrphanage::_EraseTx(const uint256& txid) +int TxOrphanage::EraseTxNoLock(const uint256& txid) { AssertLockHeld(m_mutex); std::map<uint256, OrphanTx>::iterator it = m_orphans.find(txid); @@ -103,7 +103,7 @@ void TxOrphanage::EraseForPeer(NodeId peer) std::map<uint256, OrphanTx>::iterator maybeErase = iter++; // increment to avoid iterator becoming invalid if (maybeErase->second.fromPeer == peer) { - nErased += _EraseTx(maybeErase->second.tx->GetHash()); + nErased += EraseTxNoLock(maybeErase->second.tx->GetHash()); } } if (nErased > 0) LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx from peer=%d\n", nErased, peer); @@ -125,7 +125,7 @@ void TxOrphanage::LimitOrphans(unsigned int max_orphans) { std::map<uint256, OrphanTx>::iterator maybeErase = iter++; if (maybeErase->second.nTimeExpire <= nNow) { - nErased += _EraseTx(maybeErase->second.tx->GetHash()); + nErased += EraseTxNoLock(maybeErase->second.tx->GetHash()); } else { nMinExpTime = std::min(maybeErase->second.nTimeExpire, nMinExpTime); } @@ -139,7 +139,7 @@ void TxOrphanage::LimitOrphans(unsigned int max_orphans) { // Evict a random orphan: size_t randompos = rng.randrange(m_orphan_list.size()); - _EraseTx(m_orphan_list[randompos]->first); + EraseTxNoLock(m_orphan_list[randompos]->first); ++nEvicted; } if (nEvicted > 0) LogPrint(BCLog::MEMPOOL, "orphanage overflow, removed %u tx\n", nEvicted); @@ -231,7 +231,7 @@ void TxOrphanage::EraseForBlock(const CBlock& block) if (vOrphanErase.size()) { int nErased = 0; for (const uint256& orphanHash : vOrphanErase) { - nErased += _EraseTx(orphanHash); + nErased += EraseTxNoLock(orphanHash); } LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); } diff --git a/src/txorphanage.h b/src/txorphanage.h index 45276c6c98..a4705bf382 100644 --- a/src/txorphanage.h +++ b/src/txorphanage.h @@ -99,7 +99,7 @@ protected: std::map<uint256, OrphanMap::iterator> m_wtxid_to_orphan_it GUARDED_BY(m_mutex); /** Erase an orphan by txid */ - int _EraseTx(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + int EraseTxNoLock(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(m_mutex); }; #endif // BITCOIN_TXORPHANAGE_H diff --git a/src/txrequest.cpp b/src/txrequest.cpp index 40d36132de..dd042103bd 100644 --- a/src/txrequest.cpp +++ b/src/txrequest.cpp @@ -10,8 +10,12 @@ #include <random.h> #include <uint256.h> -#include <boost/multi_index_container.hpp> +#include <boost/multi_index/indexed_by.hpp> #include <boost/multi_index/ordered_index.hpp> +#include <boost/multi_index/sequenced_index.hpp> +#include <boost/multi_index/tag.hpp> +#include <boost/multi_index_container.hpp> +#include <boost/tuple/tuple.hpp> #include <chrono> #include <unordered_map> diff --git a/src/uint256.h b/src/uint256.h index 1cc3721487..d35b3a66fa 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -22,6 +22,7 @@ class base_blob { protected: static constexpr int WIDTH = BITS / 8; + static_assert(BITS % 8 == 0, "base_blob currently only supports whole bytes."); std::array<uint8_t, WIDTH> m_data; static_assert(WIDTH == sizeof(m_data), "Sanity check"); @@ -77,7 +78,7 @@ public: template<typename Stream> void Serialize(Stream& s) const { - s.write(MakeByteSpan(m_data)); + s << Span(m_data); } template<typename Stream> diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 004135ef97..94f80f9c27 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -87,7 +87,7 @@ public: template <class It> void push_backV(It first, It last); - void __pushKV(std::string key, UniValue val); + void pushKVEnd(std::string key, UniValue val); void pushKV(std::string key, UniValue val); void pushKVs(UniValue obj); diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp index c3d19caae0..656d2e8203 100644 --- a/src/univalue/lib/univalue.cpp +++ b/src/univalue/lib/univalue.cpp @@ -115,7 +115,7 @@ void UniValue::push_backV(const std::vector<UniValue>& vec) values.insert(values.end(), vec.begin(), vec.end()); } -void UniValue::__pushKV(std::string key, UniValue val) +void UniValue::pushKVEnd(std::string key, UniValue val) { checkType(VOBJ); @@ -131,7 +131,7 @@ void UniValue::pushKV(std::string key, UniValue val) if (findKey(key, idx)) values[idx] = std::move(val); else - __pushKV(std::move(key), std::move(val)); + pushKVEnd(std::move(key), std::move(val)); } void UniValue::pushKVs(UniValue obj) @@ -140,7 +140,7 @@ void UniValue::pushKVs(UniValue obj) obj.checkType(VOBJ); for (size_t i = 0; i < obj.keys.size(); i++) - __pushKV(std::move(obj.keys.at(i)), std::move(obj.values.at(i))); + pushKVEnd(std::move(obj.keys.at(i)), std::move(obj.values.at(i))); } void UniValue::getObjMap(std::map<std::string,UniValue>& kv) const diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index 5ddf300393..8b90448b36 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -86,7 +86,7 @@ void univalue_push_throw() UniValue j; BOOST_CHECK_THROW(j.push_back(1), std::runtime_error); BOOST_CHECK_THROW(j.push_backV({1}), std::runtime_error); - BOOST_CHECK_THROW(j.__pushKV("k", 1), std::runtime_error); + BOOST_CHECK_THROW(j.pushKVEnd("k", 1), std::runtime_error); BOOST_CHECK_THROW(j.pushKV("k", 1), std::runtime_error); BOOST_CHECK_THROW(j.pushKVs({}), std::runtime_error); } @@ -364,7 +364,7 @@ void univalue_object() obj.setObject(); UniValue uv; uv.setInt(42); - obj.__pushKV("age", uv); + obj.pushKVEnd("age", uv); BOOST_CHECK_EQUAL(obj.size(), 1); BOOST_CHECK_EQUAL(obj["age"].getValStr(), "42"); @@ -412,6 +412,33 @@ void univalue_readwrite() BOOST_CHECK_EQUAL(strJson1, v.write()); + // Valid + BOOST_CHECK(v.read("1.0") && (v.get_real() == 1.0)); + BOOST_CHECK(v.read("true") && v.get_bool()); + BOOST_CHECK(v.read("[false]") && !v[0].get_bool()); + BOOST_CHECK(v.read("{\"a\": true}") && v["a"].get_bool()); + BOOST_CHECK(v.read("{\"1\": \"true\"}") && (v["1"].get_str() == "true")); + // Valid, with leading or trailing whitespace + BOOST_CHECK(v.read(" 1.0") && (v.get_real() == 1.0)); + BOOST_CHECK(v.read("1.0 ") && (v.get_real() == 1.0)); + BOOST_CHECK(v.read("0.00000000000000000000000000000000000001e+30 ") && v.get_real() == 1e-8); + + BOOST_CHECK(!v.read(".19e-6")); //should fail, missing leading 0, therefore invalid JSON + // Invalid, initial garbage + BOOST_CHECK(!v.read("[1.0")); + BOOST_CHECK(!v.read("a1.0")); + // Invalid, trailing garbage + BOOST_CHECK(!v.read("1.0sds")); + BOOST_CHECK(!v.read("1.0]")); + // Invalid, keys have to be names + BOOST_CHECK(!v.read("{1: \"true\"}")); + BOOST_CHECK(!v.read("{true: 1}")); + BOOST_CHECK(!v.read("{[1]: 1}")); + BOOST_CHECK(!v.read("{{\"a\": \"a\"}: 1}")); + // BTC addresses should fail parsing + BOOST_CHECK(!v.read("175tWpb8K1S7NmH4Zx6rewF9WQrcZv245W")); + BOOST_CHECK(!v.read("3J98t1WpEZ73CNmQviecrnyiWrnqRhWNL")); + /* Check for (correctly reporting) a parsing error if the initial JSON construct is followed by more stuff. Note that whitespace is, of course, exempt. */ diff --git a/src/util/any.h b/src/util/any.h new file mode 100644 index 0000000000..4562c5bd8a --- /dev/null +++ b/src/util/any.h @@ -0,0 +1,26 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_ANY_H +#define BITCOIN_UTIL_ANY_H + +#include <any> + +namespace util { + +/** + * Helper function to access the contained object of a std::any instance. + * Returns a pointer to the object if passed instance has a value and the type + * matches, nullptr otherwise. + */ +template<typename T> +T* AnyPtr(const std::any& any) noexcept +{ + T* const* ptr = std::any_cast<T*>(&any); + return ptr ? *ptr : nullptr; +} + +} // namespace util + +#endif // BITCOIN_UTIL_ANY_H diff --git a/src/util/batchpriority.cpp b/src/util/batchpriority.cpp new file mode 100644 index 0000000000..c73aef1eb4 --- /dev/null +++ b/src/util/batchpriority.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <logging.h> +#include <util/syserror.h> + +#if (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) +#include <pthread.h> +#include <pthread_np.h> +#endif + +#ifndef WIN32 +#include <sched.h> +#endif + +void ScheduleBatchPriority() +{ +#ifdef SCHED_BATCH + const static sched_param param{}; + const int rc = pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m); + if (rc != 0) { + LogPrintf("Failed to pthread_setschedparam: %s\n", SysErrorString(rc)); + } +#endif +} diff --git a/src/util/batchpriority.h b/src/util/batchpriority.h new file mode 100644 index 0000000000..5ffc8dd684 --- /dev/null +++ b/src/util/batchpriority.h @@ -0,0 +1,15 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_BATCHPRIORITY_H +#define BITCOIN_UTIL_BATCHPRIORITY_H + +/** + * On platforms that support it, tell the kernel the calling thread is + * CPU-intensive and non-interactive. See SCHED_BATCH in sched(7) for details. + * + */ +void ScheduleBatchPriority(); + +#endif // BITCOIN_UTIL_BATCHPRIORITY_H diff --git a/src/util/insert.h b/src/util/insert.h new file mode 100644 index 0000000000..5332eca60a --- /dev/null +++ b/src/util/insert.h @@ -0,0 +1,24 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_INSERT_H +#define BITCOIN_UTIL_INSERT_H + +#include <set> + +namespace util { + +//! Simplification of std insertion +template <typename Tdst, typename Tsrc> +inline void insert(Tdst& dst, const Tsrc& src) { + dst.insert(dst.begin(), src.begin(), src.end()); +} +template <typename TsetT, typename Tsrc> +inline void insert(std::set<TsetT>& dst, const Tsrc& src) { + dst.insert(src.begin(), src.end()); +} + +} // namespace util + +#endif // BITCOIN_UTIL_INSERT_H diff --git a/src/util/result.h b/src/util/result.h index 972b1aada0..b99995c7e5 100644 --- a/src/util/result.h +++ b/src/util/result.h @@ -31,16 +31,19 @@ struct Error { //! `std::optional<T>` can be updated to return `util::Result<T>` and return //! error strings usually just replacing `return std::nullopt;` with `return //! util::Error{error_string};`. -template <class T> +template <class M> class Result { private: + using T = std::conditional_t<std::is_same_v<M, void>, std::monostate, M>; + std::variant<bilingual_str, T> m_variant; template <typename FT> friend bilingual_str ErrorString(const Result<FT>& result); public: + Result() : m_variant{std::in_place_index_t<1>{}, std::monostate{}} {} // constructor for void Result(T obj) : m_variant{std::in_place_index_t<1>{}, std::move(obj)} {} Result(Error error) : m_variant{std::in_place_index_t<0>{}, std::move(error.message)} {} diff --git a/src/util/sock.cpp b/src/util/sock.cpp index 53d20bdf19..c83869bc77 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -2,12 +2,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <compat/compat.h> #include <logging.h> #include <tinyformat.h> #include <util/sock.h> #include <util/syserror.h> -#include <util/system.h> #include <util/threadinterrupt.h> #include <util/time.h> diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 05e7b957c4..d792562735 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -17,8 +17,8 @@ #include <cstdint> #include <limits> #include <optional> -#include <string> -#include <string_view> +#include <string> // IWYU pragma: export +#include <string_view> // IWYU pragma: export #include <system_error> #include <type_traits> #include <vector> diff --git a/src/util/string.h b/src/util/string.h index fb93d2a80e..8b69d6ccae 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -12,8 +12,8 @@ #include <cstring> #include <locale> #include <sstream> -#include <string> -#include <string_view> +#include <string> // IWYU pragma: export +#include <string_view> // IWYU pragma: export #include <vector> void ReplaceAll(std::string& in_out, const std::string& search, const std::string& substitute); diff --git a/src/util/syscall_sandbox.cpp b/src/util/syscall_sandbox.cpp deleted file mode 100644 index b1579bdb9c..0000000000 --- a/src/util/syscall_sandbox.cpp +++ /dev/null @@ -1,927 +0,0 @@ -// Copyright (c) 2020-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif // defined(HAVE_CONFIG_H) - -#include <util/syscall_sandbox.h> - -#if defined(USE_SYSCALL_SANDBOX) -#include <array> -#include <cassert> -#include <cstdint> -#include <exception> -#include <map> -#include <new> -#include <set> -#include <string> -#include <vector> - -#include <logging.h> -#include <tinyformat.h> -#include <util/threadnames.h> - -#include <linux/audit.h> -#include <linux/filter.h> -#include <linux/seccomp.h> -#include <linux/unistd.h> -#include <signal.h> -#include <sys/prctl.h> -#include <sys/types.h> -#include <unistd.h> - -namespace { -bool g_syscall_sandbox_enabled{false}; -bool g_syscall_sandbox_log_violation_before_terminating{false}; - -#if !defined(__x86_64__) -#error Syscall sandbox is an experimental feature currently available only under Linux x86-64. -#endif // defined(__x86_64__) - -#ifndef SECCOMP_RET_KILL_PROCESS -#define SECCOMP_RET_KILL_PROCESS 0x80000000U -#endif - -// Define system call numbers for x86_64 that are referenced in the system call profile -// but not provided by the kernel headers used in the GUIX build. -// Usually, they can be found via "grep name /usr/include/x86_64-linux-gnu/asm/unistd_64.h" - -#ifndef __NR_clone3 -#define __NR_clone3 435 -#endif - -#ifndef __NR_statx -#define __NR_statx 332 -#endif - -#ifndef __NR_getrandom -#define __NR_getrandom 318 -#endif - -#ifndef __NR_membarrier -#define __NR_membarrier 324 -#endif - -#ifndef __NR_copy_file_range -#define __NR_copy_file_range 326 -#endif - -#ifndef __NR_rseq -#define __NR_rseq 334 -#endif - -// This list of syscalls in LINUX_SYSCALLS is only used to map syscall numbers to syscall names in -// order to be able to print user friendly error messages which include the syscall name in addition -// to the syscall number. -// -// Example output in case of a syscall violation where the syscall is present in LINUX_SYSCALLS: -// -// ``` -// 2021-06-09T12:34:56Z ERROR: The syscall "execve" (syscall number 59) is not allowed by the syscall sandbox in thread "msghand". Please report. -// ``` -// -// Example output in case of a syscall violation where the syscall is not present in LINUX_SYSCALLS: -// -// ``` -// 2021-06-09T12:34:56Z ERROR: The syscall "*unknown*" (syscall number 314) is not allowed by the syscall sandbox in thread "msghand". Please report. -// `` -// -// LINUX_SYSCALLS contains two types of syscalls: -// 1.) Syscalls that are present under all architectures or relevant Linux kernel versions for which -// we support the syscall sandbox feature (currently only Linux x86-64). Examples include read, -// write, open, close, etc. -// 2.) Syscalls that are present under a subset of architectures or relevant Linux kernel versions -// for which we support the syscall sandbox feature. This type of syscalls should be added to -// LINUX_SYSCALLS conditional on availability like in the following example: -// ... -// #if defined(__NR_arch_dependent_syscall) -// {__NR_arch_dependent_syscall, "arch_dependent_syscall"}, -// #endif // defined(__NR_arch_dependent_syscall) -// ... -const std::map<uint32_t, std::string> LINUX_SYSCALLS{ - {__NR_accept, "accept"}, - {__NR_accept4, "accept4"}, - {__NR_access, "access"}, - {__NR_acct, "acct"}, - {__NR_add_key, "add_key"}, - {__NR_adjtimex, "adjtimex"}, - {__NR_afs_syscall, "afs_syscall"}, - {__NR_alarm, "alarm"}, - {__NR_arch_prctl, "arch_prctl"}, - {__NR_bind, "bind"}, - {__NR_bpf, "bpf"}, - {__NR_brk, "brk"}, - {__NR_capget, "capget"}, - {__NR_capset, "capset"}, - {__NR_chdir, "chdir"}, - {__NR_chmod, "chmod"}, - {__NR_chown, "chown"}, - {__NR_chroot, "chroot"}, - {__NR_clock_adjtime, "clock_adjtime"}, - {__NR_clock_getres, "clock_getres"}, - {__NR_clock_gettime, "clock_gettime"}, - {__NR_clock_nanosleep, "clock_nanosleep"}, - {__NR_clock_settime, "clock_settime"}, - {__NR_clone, "clone"}, - {__NR_clone3, "clone3"}, - {__NR_close, "close"}, - {__NR_connect, "connect"}, - {__NR_copy_file_range, "copy_file_range"}, - {__NR_creat, "creat"}, - {__NR_create_module, "create_module"}, - {__NR_delete_module, "delete_module"}, - {__NR_dup, "dup"}, - {__NR_dup2, "dup2"}, - {__NR_dup3, "dup3"}, - {__NR_epoll_create, "epoll_create"}, - {__NR_epoll_create1, "epoll_create1"}, - {__NR_epoll_ctl, "epoll_ctl"}, - {__NR_epoll_ctl_old, "epoll_ctl_old"}, - {__NR_epoll_pwait, "epoll_pwait"}, - {__NR_epoll_wait, "epoll_wait"}, - {__NR_epoll_wait_old, "epoll_wait_old"}, - {__NR_eventfd, "eventfd"}, - {__NR_eventfd2, "eventfd2"}, - {__NR_execve, "execve"}, - {__NR_execveat, "execveat"}, - {__NR_exit, "exit"}, - {__NR_exit_group, "exit_group"}, - {__NR_faccessat, "faccessat"}, - {__NR_fadvise64, "fadvise64"}, - {__NR_fallocate, "fallocate"}, - {__NR_fanotify_init, "fanotify_init"}, - {__NR_fanotify_mark, "fanotify_mark"}, - {__NR_fchdir, "fchdir"}, - {__NR_fchmod, "fchmod"}, - {__NR_fchmodat, "fchmodat"}, - {__NR_fchown, "fchown"}, - {__NR_fchownat, "fchownat"}, - {__NR_fcntl, "fcntl"}, - {__NR_fdatasync, "fdatasync"}, - {__NR_fgetxattr, "fgetxattr"}, - {__NR_finit_module, "finit_module"}, - {__NR_flistxattr, "flistxattr"}, - {__NR_flock, "flock"}, - {__NR_fork, "fork"}, - {__NR_fremovexattr, "fremovexattr"}, - {__NR_fsetxattr, "fsetxattr"}, - {__NR_fstat, "fstat"}, - {__NR_fstatfs, "fstatfs"}, - {__NR_fsync, "fsync"}, - {__NR_ftruncate, "ftruncate"}, - {__NR_futex, "futex"}, - {__NR_futimesat, "futimesat"}, - {__NR_get_kernel_syms, "get_kernel_syms"}, - {__NR_get_mempolicy, "get_mempolicy"}, - {__NR_get_robust_list, "get_robust_list"}, - {__NR_get_thread_area, "get_thread_area"}, - {__NR_getcpu, "getcpu"}, - {__NR_getcwd, "getcwd"}, - {__NR_getdents, "getdents"}, - {__NR_getdents64, "getdents64"}, - {__NR_getegid, "getegid"}, - {__NR_geteuid, "geteuid"}, - {__NR_getgid, "getgid"}, - {__NR_getgroups, "getgroups"}, - {__NR_getitimer, "getitimer"}, - {__NR_getpeername, "getpeername"}, - {__NR_getpgid, "getpgid"}, - {__NR_getpgrp, "getpgrp"}, - {__NR_getpid, "getpid"}, - {__NR_getpmsg, "getpmsg"}, - {__NR_getppid, "getppid"}, - {__NR_getpriority, "getpriority"}, - {__NR_getrandom, "getrandom"}, - {__NR_getresgid, "getresgid"}, - {__NR_getresuid, "getresuid"}, - {__NR_getrlimit, "getrlimit"}, - {__NR_getrusage, "getrusage"}, - {__NR_getsid, "getsid"}, - {__NR_getsockname, "getsockname"}, - {__NR_getsockopt, "getsockopt"}, - {__NR_gettid, "gettid"}, - {__NR_gettimeofday, "gettimeofday"}, - {__NR_getuid, "getuid"}, - {__NR_getxattr, "getxattr"}, - {__NR_init_module, "init_module"}, - {__NR_inotify_add_watch, "inotify_add_watch"}, - {__NR_inotify_init, "inotify_init"}, - {__NR_inotify_init1, "inotify_init1"}, - {__NR_inotify_rm_watch, "inotify_rm_watch"}, - {__NR_io_cancel, "io_cancel"}, - {__NR_io_destroy, "io_destroy"}, - {__NR_io_getevents, "io_getevents"}, - {__NR_io_setup, "io_setup"}, - {__NR_io_submit, "io_submit"}, - {__NR_ioctl, "ioctl"}, - {__NR_ioperm, "ioperm"}, - {__NR_iopl, "iopl"}, - {__NR_ioprio_get, "ioprio_get"}, - {__NR_ioprio_set, "ioprio_set"}, - {__NR_kcmp, "kcmp"}, - {__NR_kexec_file_load, "kexec_file_load"}, - {__NR_kexec_load, "kexec_load"}, - {__NR_keyctl, "keyctl"}, - {__NR_kill, "kill"}, - {__NR_lchown, "lchown"}, - {__NR_lgetxattr, "lgetxattr"}, - {__NR_link, "link"}, - {__NR_linkat, "linkat"}, - {__NR_listen, "listen"}, - {__NR_listxattr, "listxattr"}, - {__NR_llistxattr, "llistxattr"}, - {__NR_lookup_dcookie, "lookup_dcookie"}, - {__NR_lremovexattr, "lremovexattr"}, - {__NR_lseek, "lseek"}, - {__NR_lsetxattr, "lsetxattr"}, - {__NR_lstat, "lstat"}, - {__NR_madvise, "madvise"}, - {__NR_mbind, "mbind"}, - {__NR_membarrier, "membarrier"}, - {__NR_memfd_create, "memfd_create"}, - {__NR_migrate_pages, "migrate_pages"}, - {__NR_mincore, "mincore"}, - {__NR_mkdir, "mkdir"}, - {__NR_mkdirat, "mkdirat"}, - {__NR_mknod, "mknod"}, - {__NR_mknodat, "mknodat"}, - {__NR_mlock, "mlock"}, - {__NR_mlock2, "mlock2"}, - {__NR_mlockall, "mlockall"}, - {__NR_mmap, "mmap"}, - {__NR_modify_ldt, "modify_ldt"}, - {__NR_mount, "mount"}, - {__NR_move_pages, "move_pages"}, - {__NR_mprotect, "mprotect"}, - {__NR_mq_getsetattr, "mq_getsetattr"}, - {__NR_mq_notify, "mq_notify"}, - {__NR_mq_open, "mq_open"}, - {__NR_mq_timedreceive, "mq_timedreceive"}, - {__NR_mq_timedsend, "mq_timedsend"}, - {__NR_mq_unlink, "mq_unlink"}, - {__NR_mremap, "mremap"}, - {__NR_msgctl, "msgctl"}, - {__NR_msgget, "msgget"}, - {__NR_msgrcv, "msgrcv"}, - {__NR_msgsnd, "msgsnd"}, - {__NR_msync, "msync"}, - {__NR_munlock, "munlock"}, - {__NR_munlockall, "munlockall"}, - {__NR_munmap, "munmap"}, - {__NR_name_to_handle_at, "name_to_handle_at"}, - {__NR_nanosleep, "nanosleep"}, - {__NR_newfstatat, "newfstatat"}, - {__NR_nfsservctl, "nfsservctl"}, - {__NR_open, "open"}, - {__NR_open_by_handle_at, "open_by_handle_at"}, - {__NR_openat, "openat"}, - {__NR_pause, "pause"}, - {__NR_perf_event_open, "perf_event_open"}, - {__NR_personality, "personality"}, - {__NR_pipe, "pipe"}, - {__NR_pipe2, "pipe2"}, - {__NR_pivot_root, "pivot_root"}, -#ifdef __NR_pkey_alloc - {__NR_pkey_alloc, "pkey_alloc"}, -#endif -#ifdef __NR_pkey_free - {__NR_pkey_free, "pkey_free"}, -#endif -#ifdef __NR_pkey_mprotect - {__NR_pkey_mprotect, "pkey_mprotect"}, -#endif - {__NR_poll, "poll"}, - {__NR_ppoll, "ppoll"}, - {__NR_prctl, "prctl"}, - {__NR_pread64, "pread64"}, - {__NR_preadv, "preadv"}, -#ifdef __NR_preadv2 - {__NR_preadv2, "preadv2"}, -#endif - {__NR_prlimit64, "prlimit64"}, - {__NR_process_vm_readv, "process_vm_readv"}, - {__NR_process_vm_writev, "process_vm_writev"}, - {__NR_pselect6, "pselect6"}, - {__NR_ptrace, "ptrace"}, - {__NR_putpmsg, "putpmsg"}, - {__NR_pwrite64, "pwrite64"}, - {__NR_pwritev, "pwritev"}, -#ifdef __NR_pwritev2 - {__NR_pwritev2, "pwritev2"}, -#endif - {__NR__sysctl, "_sysctl"}, - {__NR_query_module, "query_module"}, - {__NR_quotactl, "quotactl"}, - {__NR_read, "read"}, - {__NR_readahead, "readahead"}, - {__NR_readlink, "readlink"}, - {__NR_readlinkat, "readlinkat"}, - {__NR_readv, "readv"}, - {__NR_reboot, "reboot"}, - {__NR_recvfrom, "recvfrom"}, - {__NR_recvmmsg, "recvmmsg"}, - {__NR_recvmsg, "recvmsg"}, - {__NR_remap_file_pages, "remap_file_pages"}, - {__NR_removexattr, "removexattr"}, - {__NR_rename, "rename"}, - {__NR_renameat, "renameat"}, - {__NR_renameat2, "renameat2"}, - {__NR_request_key, "request_key"}, - {__NR_restart_syscall, "restart_syscall"}, - {__NR_rmdir, "rmdir"}, - {__NR_rseq, "rseq"}, - {__NR_rt_sigaction, "rt_sigaction"}, - {__NR_rt_sigpending, "rt_sigpending"}, - {__NR_rt_sigprocmask, "rt_sigprocmask"}, - {__NR_rt_sigqueueinfo, "rt_sigqueueinfo"}, - {__NR_rt_sigreturn, "rt_sigreturn"}, - {__NR_rt_sigsuspend, "rt_sigsuspend"}, - {__NR_rt_sigtimedwait, "rt_sigtimedwait"}, - {__NR_rt_tgsigqueueinfo, "rt_tgsigqueueinfo"}, - {__NR_sched_get_priority_max, "sched_get_priority_max"}, - {__NR_sched_get_priority_min, "sched_get_priority_min"}, - {__NR_sched_getaffinity, "sched_getaffinity"}, - {__NR_sched_getattr, "sched_getattr"}, - {__NR_sched_getparam, "sched_getparam"}, - {__NR_sched_getscheduler, "sched_getscheduler"}, - {__NR_sched_rr_get_interval, "sched_rr_get_interval"}, - {__NR_sched_setaffinity, "sched_setaffinity"}, - {__NR_sched_setattr, "sched_setattr"}, - {__NR_sched_setparam, "sched_setparam"}, - {__NR_sched_setscheduler, "sched_setscheduler"}, - {__NR_sched_yield, "sched_yield"}, - {__NR_seccomp, "seccomp"}, - {__NR_security, "security"}, - {__NR_select, "select"}, - {__NR_semctl, "semctl"}, - {__NR_semget, "semget"}, - {__NR_semop, "semop"}, - {__NR_semtimedop, "semtimedop"}, - {__NR_sendfile, "sendfile"}, - {__NR_sendmmsg, "sendmmsg"}, - {__NR_sendmsg, "sendmsg"}, - {__NR_sendto, "sendto"}, - {__NR_set_mempolicy, "set_mempolicy"}, - {__NR_set_robust_list, "set_robust_list"}, - {__NR_set_thread_area, "set_thread_area"}, - {__NR_set_tid_address, "set_tid_address"}, - {__NR_setdomainname, "setdomainname"}, - {__NR_setfsgid, "setfsgid"}, - {__NR_setfsuid, "setfsuid"}, - {__NR_setgid, "setgid"}, - {__NR_setgroups, "setgroups"}, - {__NR_sethostname, "sethostname"}, - {__NR_setitimer, "setitimer"}, - {__NR_setns, "setns"}, - {__NR_setpgid, "setpgid"}, - {__NR_setpriority, "setpriority"}, - {__NR_setregid, "setregid"}, - {__NR_setresgid, "setresgid"}, - {__NR_setresuid, "setresuid"}, - {__NR_setreuid, "setreuid"}, - {__NR_setrlimit, "setrlimit"}, - {__NR_setsid, "setsid"}, - {__NR_setsockopt, "setsockopt"}, - {__NR_settimeofday, "settimeofday"}, - {__NR_setuid, "setuid"}, - {__NR_setxattr, "setxattr"}, - {__NR_shmat, "shmat"}, - {__NR_shmctl, "shmctl"}, - {__NR_shmdt, "shmdt"}, - {__NR_shmget, "shmget"}, - {__NR_shutdown, "shutdown"}, - {__NR_sigaltstack, "sigaltstack"}, - {__NR_signalfd, "signalfd"}, - {__NR_signalfd4, "signalfd4"}, - {__NR_socket, "socket"}, - {__NR_socketpair, "socketpair"}, - {__NR_splice, "splice"}, - {__NR_stat, "stat"}, - {__NR_statfs, "statfs"}, - {__NR_statx, "statx"}, - {__NR_swapoff, "swapoff"}, - {__NR_swapon, "swapon"}, - {__NR_symlink, "symlink"}, - {__NR_symlinkat, "symlinkat"}, - {__NR_sync, "sync"}, - {__NR_sync_file_range, "sync_file_range"}, - {__NR_syncfs, "syncfs"}, - {__NR_sysfs, "sysfs"}, - {__NR_sysinfo, "sysinfo"}, - {__NR_syslog, "syslog"}, - {__NR_tee, "tee"}, - {__NR_tgkill, "tgkill"}, - {__NR_time, "time"}, - {__NR_timer_create, "timer_create"}, - {__NR_timer_delete, "timer_delete"}, - {__NR_timer_getoverrun, "timer_getoverrun"}, - {__NR_timer_gettime, "timer_gettime"}, - {__NR_timer_settime, "timer_settime"}, - {__NR_timerfd_create, "timerfd_create"}, - {__NR_timerfd_gettime, "timerfd_gettime"}, - {__NR_timerfd_settime, "timerfd_settime"}, - {__NR_times, "times"}, - {__NR_tkill, "tkill"}, - {__NR_truncate, "truncate"}, - {__NR_tuxcall, "tuxcall"}, - {__NR_umask, "umask"}, - {__NR_umount2, "umount2"}, - {__NR_uname, "uname"}, - {__NR_unlink, "unlink"}, - {__NR_unlinkat, "unlinkat"}, - {__NR_unshare, "unshare"}, - {__NR_uselib, "uselib"}, - {__NR_userfaultfd, "userfaultfd"}, - {__NR_ustat, "ustat"}, - {__NR_utime, "utime"}, - {__NR_utimensat, "utimensat"}, - {__NR_utimes, "utimes"}, - {__NR_vfork, "vfork"}, - {__NR_vhangup, "vhangup"}, - {__NR_vmsplice, "vmsplice"}, - {__NR_vserver, "vserver"}, - {__NR_wait4, "wait4"}, - {__NR_waitid, "waitid"}, - {__NR_write, "write"}, - {__NR_writev, "writev"}, -}; - -std::string GetLinuxSyscallName(uint32_t syscall_number) -{ - const auto element = LINUX_SYSCALLS.find(syscall_number); - if (element != LINUX_SYSCALLS.end()) { - return element->second; - } - return "*unknown*"; -} - -// See Linux kernel developer Kees Cook's seccomp guide at <https://outflux.net/teach-seccomp/> for -// an accessible introduction to using seccomp. -// -// This function largely follows <https://outflux.net/teach-seccomp/step-3/syscall-reporter.c> and -// <https://outflux.net/teach-seccomp/step-3/seccomp-bpf.h>. -// -// Seccomp BPF resources: -// * Seccomp BPF documentation: <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> -// * seccomp(2) manual page: <https://www.kernel.org/doc/man-pages/online/pages/man2/seccomp.2.html> -// * Seccomp BPF demo code samples: <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/samples/seccomp> -void SyscallSandboxDebugSignalHandler(int, siginfo_t* signal_info, void* void_signal_context) -{ - // The si_code field inside the siginfo_t argument that is passed to a SA_SIGINFO signal handler - // is a value indicating why the signal was sent. - // - // The following value can be placed in si_code for a SIGSYS signal: - // * SYS_SECCOMP (since Linux 3.5): Triggered by a seccomp(2) filter rule. - constexpr int32_t SYS_SECCOMP_SI_CODE{1}; - assert(signal_info->si_code == SYS_SECCOMP_SI_CODE); - - // The ucontext_t structure contains signal context information that was saved on the user-space - // stack by the kernel. - const ucontext_t* signal_context = static_cast<ucontext_t*>(void_signal_context); - assert(signal_context != nullptr); - - std::set_new_handler(std::terminate); - // Portability note: REG_RAX is Linux x86_64 specific. - const uint32_t syscall_number = static_cast<uint32_t>(signal_context->uc_mcontext.gregs[REG_RAX]); - const std::string syscall_name = GetLinuxSyscallName(syscall_number); - const std::string thread_name = !util::ThreadGetInternalName().empty() ? util::ThreadGetInternalName() : "*unnamed*"; - const std::string error_message = strprintf("ERROR: The syscall \"%s\" (syscall number %d) is not allowed by the syscall sandbox in thread \"%s\". Please report.", syscall_name, syscall_number, thread_name); - tfm::format(std::cerr, "%s\n", error_message); - LogPrintf("%s\n", error_message); - std::terminate(); -} - -// This function largely follows install_syscall_reporter from Kees Cook's seccomp guide: -// <https://outflux.net/teach-seccomp/step-3/syscall-reporter.c> -bool SetupSyscallSandboxDebugHandler() -{ - struct sigaction action = {}; - sigset_t mask; - sigemptyset(&mask); - sigaddset(&mask, SIGSYS); - action.sa_sigaction = &SyscallSandboxDebugSignalHandler; - action.sa_flags = SA_SIGINFO; - if (sigaction(SIGSYS, &action, nullptr) < 0) { - return false; - } - if (sigprocmask(SIG_UNBLOCK, &mask, nullptr)) { - return false; - } - return true; -} - -enum class SyscallSandboxAction { - KILL_PROCESS, - INVOKE_SIGNAL_HANDLER, -}; - -class SeccompPolicyBuilder -{ - std::set<uint32_t> allowed_syscalls; - -public: - SeccompPolicyBuilder() - { - // Allowed by default. - AllowAddressSpaceAccess(); - AllowEpoll(); - AllowEventFd(); - AllowFutex(); - AllowGeneralIo(); - AllowGetRandom(); - AllowGetSimpleId(); - AllowGetTime(); - AllowGlobalProcessEnvironment(); - AllowGlobalSystemStatus(); - AllowKernelInternalApi(); - AllowNetworkSocketInformation(); - AllowOperationOnExistingFileDescriptor(); - AllowPipe(); - AllowPrctl(); - AllowProcessStartOrDeath(); - AllowScheduling(); - AllowSignalHandling(); - AllowSleep(); - AllowUmask(); - } - - void AllowAddressSpaceAccess() - { - allowed_syscalls.insert(__NR_brk); // change data segment size - allowed_syscalls.insert(__NR_madvise); // give advice about use of memory - allowed_syscalls.insert(__NR_membarrier); // issue memory barriers on a set of threads - allowed_syscalls.insert(__NR_mincore); // check if virtual memory is in RAM - allowed_syscalls.insert(__NR_mlock); // lock memory - allowed_syscalls.insert(__NR_mmap); // map files or devices into memory - allowed_syscalls.insert(__NR_mprotect); // set protection on a region of memory - allowed_syscalls.insert(__NR_mremap); // remap a file in memory - allowed_syscalls.insert(__NR_munlock); // unlock memory - allowed_syscalls.insert(__NR_munmap); // unmap files or devices into memory - } - - void AllowEpoll() - { - allowed_syscalls.insert(__NR_epoll_create1); // open an epoll file descriptor - allowed_syscalls.insert(__NR_epoll_ctl); // control interface for an epoll file descriptor - allowed_syscalls.insert(__NR_epoll_pwait); // wait for an I/O event on an epoll file descriptor - allowed_syscalls.insert(__NR_epoll_wait); // wait for an I/O event on an epoll file descriptor - } - - void AllowEventFd() - { - allowed_syscalls.insert(__NR_eventfd2); // create a file descriptor for event notification - } - - void AllowFileSystem() - { - allowed_syscalls.insert(__NR_access); // check user's permissions for a file - allowed_syscalls.insert(__NR_chdir); // change working directory - allowed_syscalls.insert(__NR_chmod); // change permissions of a file - allowed_syscalls.insert(__NR_copy_file_range); // copy a range of data from one file to another - allowed_syscalls.insert(__NR_fallocate); // manipulate file space - allowed_syscalls.insert(__NR_fchmod); // change permissions of a file - allowed_syscalls.insert(__NR_fchown); // change ownership of a file - allowed_syscalls.insert(__NR_fdatasync); // synchronize a file's in-core state with storage device - allowed_syscalls.insert(__NR_flock); // apply or remove an advisory lock on an open file - allowed_syscalls.insert(__NR_fstat); // get file status - allowed_syscalls.insert(__NR_fstatfs); // get file system status - allowed_syscalls.insert(__NR_fsync); // synchronize a file's in-core state with storage device - allowed_syscalls.insert(__NR_ftruncate); // truncate a file to a specified length - allowed_syscalls.insert(__NR_getcwd); // get current working directory - allowed_syscalls.insert(__NR_getdents); // get directory entries - allowed_syscalls.insert(__NR_getdents64); // get directory entries - allowed_syscalls.insert(__NR_lstat); // get file status - allowed_syscalls.insert(__NR_mkdir); // create a directory - allowed_syscalls.insert(__NR_newfstatat); // get file status - allowed_syscalls.insert(__NR_open); // open and possibly create a file - allowed_syscalls.insert(__NR_openat); // open and possibly create a file - allowed_syscalls.insert(__NR_readlink); // read value of a symbolic link - allowed_syscalls.insert(__NR_rename); // change the name or location of a file - allowed_syscalls.insert(__NR_rmdir); // delete a directory - allowed_syscalls.insert(__NR_sendfile); // transfer data between file descriptors - allowed_syscalls.insert(__NR_stat); // get file status - allowed_syscalls.insert(__NR_statfs); // get filesystem statistics - allowed_syscalls.insert(__NR_statx); // get file status (extended) - allowed_syscalls.insert(__NR_unlink); // delete a name and possibly the file it refers to - allowed_syscalls.insert(__NR_unlinkat); // delete relative to a directory file descriptor - } - - void AllowFutex() - { - allowed_syscalls.insert(__NR_futex); // fast user-space locking - allowed_syscalls.insert(__NR_set_robust_list); // set list of robust futexes - } - - void AllowGeneralIo() - { - allowed_syscalls.insert(__NR_ioctl); // control device - allowed_syscalls.insert(__NR_lseek); // reposition read/write file offset - allowed_syscalls.insert(__NR_poll); // wait for some event on a file descriptor - allowed_syscalls.insert(__NR_ppoll); // wait for some event on a file descriptor - allowed_syscalls.insert(__NR_pread64); // read from a file descriptor at a given offset - allowed_syscalls.insert(__NR_pwrite64); // write to a file descriptor at a given offset - allowed_syscalls.insert(__NR_read); // read from a file descriptor - allowed_syscalls.insert(__NR_readv); // read data into multiple buffers - allowed_syscalls.insert(__NR_recvfrom); // receive a message from a socket - allowed_syscalls.insert(__NR_recvmsg); // receive a message from a socket - allowed_syscalls.insert(__NR_select); // synchronous I/O multiplexing - allowed_syscalls.insert(__NR_sendmmsg); // send multiple messages on a socket - allowed_syscalls.insert(__NR_sendmsg); // send a message on a socket - allowed_syscalls.insert(__NR_sendto); // send a message on a socket - allowed_syscalls.insert(__NR_write); // write to a file descriptor - allowed_syscalls.insert(__NR_writev); // write data into multiple buffers - } - - void AllowGetRandom() - { - allowed_syscalls.insert(__NR_getrandom); // obtain a series of random bytes - } - - void AllowGetSimpleId() - { - allowed_syscalls.insert(__NR_getegid); // get group identity - allowed_syscalls.insert(__NR_geteuid); // get user identity - allowed_syscalls.insert(__NR_getgid); // get group identity - allowed_syscalls.insert(__NR_getpgid); // get process group - allowed_syscalls.insert(__NR_getpid); // get process identification - allowed_syscalls.insert(__NR_getppid); // get process identification - allowed_syscalls.insert(__NR_getresgid); // get real, effective and saved group IDs - allowed_syscalls.insert(__NR_getresuid); // get real, effective and saved user IDs - allowed_syscalls.insert(__NR_getsid); // get session ID - allowed_syscalls.insert(__NR_gettid); // get thread identification - allowed_syscalls.insert(__NR_getuid); // get user identity - } - - void AllowGetTime() - { - allowed_syscalls.insert(__NR_clock_getres); // find the resolution (precision) of the specified clock - allowed_syscalls.insert(__NR_clock_gettime); // retrieve the time of the specified clock - allowed_syscalls.insert(__NR_gettimeofday); // get timeval - } - - void AllowGlobalProcessEnvironment() - { - allowed_syscalls.insert(__NR_getrlimit); // get resource limits - allowed_syscalls.insert(__NR_getrusage); // get resource usage - allowed_syscalls.insert(__NR_prlimit64); // get/set resource limits - } - - void AllowGlobalSystemStatus() - { - allowed_syscalls.insert(__NR_sysinfo); // return system information - allowed_syscalls.insert(__NR_uname); // get name and information about current kernel - } - - void AllowKernelInternalApi() - { - allowed_syscalls.insert(__NR_restart_syscall); // restart a system call after interruption by a stop signal - } - - void AllowNetwork() - { - allowed_syscalls.insert(__NR_accept); // accept a connection on a socket - allowed_syscalls.insert(__NR_accept4); // accept a connection on a socket - allowed_syscalls.insert(__NR_bind); // bind a name to a socket - allowed_syscalls.insert(__NR_connect); // initiate a connection on a socket - allowed_syscalls.insert(__NR_listen); // listen for connections on a socket - allowed_syscalls.insert(__NR_setsockopt); // set options on sockets - allowed_syscalls.insert(__NR_socket); // create an endpoint for communication - allowed_syscalls.insert(__NR_socketpair); // create a pair of connected sockets - } - - void AllowNetworkSocketInformation() - { - allowed_syscalls.insert(__NR_getpeername); // get name of connected peer socket - allowed_syscalls.insert(__NR_getsockname); // get socket name - allowed_syscalls.insert(__NR_getsockopt); // get options on sockets - } - - void AllowOperationOnExistingFileDescriptor() - { - allowed_syscalls.insert(__NR_close); // close a file descriptor - allowed_syscalls.insert(__NR_dup); // duplicate a file descriptor - allowed_syscalls.insert(__NR_dup2); // duplicate a file descriptor - allowed_syscalls.insert(__NR_fcntl); // manipulate file descriptor - allowed_syscalls.insert(__NR_shutdown); // shut down part of a full-duplex connection - } - - void AllowPipe() - { - allowed_syscalls.insert(__NR_pipe); // create pipe - allowed_syscalls.insert(__NR_pipe2); // create pipe - } - - void AllowPrctl() - { - allowed_syscalls.insert(__NR_arch_prctl); // set architecture-specific thread state - allowed_syscalls.insert(__NR_prctl); // operations on a process - } - - void AllowProcessStartOrDeath() - { - allowed_syscalls.insert(__NR_clone); // create a child process - allowed_syscalls.insert(__NR_clone3); // create a child process - allowed_syscalls.insert(__NR_exit); // terminate the calling process - allowed_syscalls.insert(__NR_exit_group); // exit all threads in a process - allowed_syscalls.insert(__NR_fork); // create a child process - allowed_syscalls.insert(__NR_tgkill); // send a signal to a thread - allowed_syscalls.insert(__NR_wait4); // wait for process to change state, BSD style - allowed_syscalls.insert(__NR_rseq); // register restartable sequence for thread - } - - void AllowScheduling() - { - allowed_syscalls.insert(__NR_sched_getaffinity); // set a thread's CPU affinity mask - allowed_syscalls.insert(__NR_sched_getparam); // get scheduling parameters - allowed_syscalls.insert(__NR_sched_getscheduler); // get scheduling policy/parameters - allowed_syscalls.insert(__NR_sched_setscheduler); // set scheduling policy/parameters - allowed_syscalls.insert(__NR_sched_yield); // yield the processor - } - - void AllowSignalHandling() - { - allowed_syscalls.insert(__NR_rt_sigaction); // examine and change a signal action - allowed_syscalls.insert(__NR_rt_sigprocmask); // examine and change blocked signals - allowed_syscalls.insert(__NR_rt_sigreturn); // return from signal handler and cleanup stack frame - allowed_syscalls.insert(__NR_sigaltstack); // set and/or get signal stack context - } - - void AllowSleep() - { - allowed_syscalls.insert(__NR_clock_nanosleep); // high-resolution sleep with specifiable clock - allowed_syscalls.insert(__NR_nanosleep); // high-resolution sleep - } - - void AllowUmask() - { - allowed_syscalls.insert(__NR_umask); // set file mode creation mask - } - - // See Linux kernel developer Kees Cook's seccomp guide at <https://outflux.net/teach-seccomp/> - // for an accessible introduction to using seccomp. - // - // This function largely follows <https://outflux.net/teach-seccomp/step-3/seccomp-bpf.h>. - std::vector<sock_filter> BuildFilter(SyscallSandboxAction default_action) - { - std::vector<sock_filter> bpf_policy; - // See VALIDATE_ARCHITECTURE in seccomp-bpf.h referenced above. - bpf_policy.push_back(BPF_STMT(BPF_LD + BPF_W + BPF_ABS, offsetof(struct seccomp_data, arch))); - // Portability note: AUDIT_ARCH_X86_64 is Linux x86_64 specific. - bpf_policy.push_back(BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, AUDIT_ARCH_X86_64, 1, 0)); - bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_KILL_PROCESS)); - // See EXAMINE_SYSCALL in seccomp-bpf.h referenced above. - bpf_policy.push_back(BPF_STMT(BPF_LD + BPF_W + BPF_ABS, offsetof(struct seccomp_data, nr))); - for (const uint32_t allowed_syscall : allowed_syscalls) { - // See ALLOW_SYSCALL in seccomp-bpf.h referenced above. - bpf_policy.push_back(BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, allowed_syscall, 0, 1)); - bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ALLOW)); - } - switch (default_action) { - case SyscallSandboxAction::KILL_PROCESS: - // Disallow syscall and kill the process. - // - // See KILL_PROCESS in seccomp-bpf.h referenced above. - // - // Note that we're using SECCOMP_RET_KILL_PROCESS (kill the process) instead - // of SECCOMP_RET_KILL_THREAD (kill the thread). The SECCOMP_RET_KILL_PROCESS - // action was introduced in Linux 4.14. - // - // SECCOMP_RET_KILL_PROCESS: Results in the entire process exiting immediately without - // executing the system call. - // - // SECCOMP_RET_KILL_PROCESS documentation: - // <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> - bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_KILL_PROCESS)); - break; - case SyscallSandboxAction::INVOKE_SIGNAL_HANDLER: - // Disallow syscall and force a SIGSYS to trigger syscall debug reporter. - // - // SECCOMP_RET_TRAP: Results in the kernel sending a SIGSYS signal to the triggering - // task without executing the system call. - // - // SECCOMP_RET_TRAP documentation: - // <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> - bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_TRAP)); - break; - } - return bpf_policy; - } -}; -} // namespace - -bool SetupSyscallSandbox(bool log_syscall_violation_before_terminating) -{ - assert(!g_syscall_sandbox_enabled && "SetupSyscallSandbox(...) should only be called once."); - g_syscall_sandbox_enabled = true; - g_syscall_sandbox_log_violation_before_terminating = log_syscall_violation_before_terminating; - if (log_syscall_violation_before_terminating) { - if (!SetupSyscallSandboxDebugHandler()) { - return false; - } - } - return true; -} - -void TestDisallowedSandboxCall() -{ - // The getgroups syscall is assumed NOT to be allowed by the syscall sandbox policy. - std::array<gid_t, 1> groups; - [[maybe_unused]] int32_t ignored = getgroups(groups.size(), groups.data()); -} -#endif // defined(USE_SYSCALL_SANDBOX) - -void SetSyscallSandboxPolicy(SyscallSandboxPolicy syscall_policy) -{ -#if defined(USE_SYSCALL_SANDBOX) - if (!g_syscall_sandbox_enabled) { - return; - } - SeccompPolicyBuilder seccomp_policy_builder; - switch (syscall_policy) { - case SyscallSandboxPolicy::INITIALIZATION: // Thread: main thread (state: init) - // SyscallSandboxPolicy::INITIALIZATION is the first policy loaded. - // - // Subsequently loaded policies can reduce the abilities further, but - // abilities can never be regained. - // - // SyscallSandboxPolicy::INITIALIZATION must thus be a superset of all - // other policies. - seccomp_policy_builder.AllowFileSystem(); - seccomp_policy_builder.AllowNetwork(); - break; - case SyscallSandboxPolicy::INITIALIZATION_DNS_SEED: // Thread: dnsseed - seccomp_policy_builder.AllowFileSystem(); - seccomp_policy_builder.AllowNetwork(); - break; - case SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS: // Thread: loadblk - seccomp_policy_builder.AllowFileSystem(); - break; - case SyscallSandboxPolicy::INITIALIZATION_MAP_PORT: // Thread: mapport - seccomp_policy_builder.AllowFileSystem(); - seccomp_policy_builder.AllowNetwork(); - break; - case SyscallSandboxPolicy::MESSAGE_HANDLER: // Thread: msghand - seccomp_policy_builder.AllowFileSystem(); - break; - case SyscallSandboxPolicy::NET: // Thread: net - seccomp_policy_builder.AllowFileSystem(); - seccomp_policy_builder.AllowNetwork(); - break; - case SyscallSandboxPolicy::NET_ADD_CONNECTION: // Thread: addcon - seccomp_policy_builder.AllowFileSystem(); - seccomp_policy_builder.AllowNetwork(); - break; - case SyscallSandboxPolicy::NET_HTTP_SERVER: // Thread: http - seccomp_policy_builder.AllowFileSystem(); - seccomp_policy_builder.AllowNetwork(); - break; - case SyscallSandboxPolicy::NET_HTTP_SERVER_WORKER: // Thread: httpworker.<N> - seccomp_policy_builder.AllowFileSystem(); - seccomp_policy_builder.AllowNetwork(); - break; - case SyscallSandboxPolicy::NET_OPEN_CONNECTION: // Thread: opencon - seccomp_policy_builder.AllowFileSystem(); - seccomp_policy_builder.AllowNetwork(); - break; - case SyscallSandboxPolicy::SCHEDULER: // Thread: scheduler - seccomp_policy_builder.AllowFileSystem(); - break; - case SyscallSandboxPolicy::TOR_CONTROL: // Thread: torcontrol - seccomp_policy_builder.AllowFileSystem(); - seccomp_policy_builder.AllowNetwork(); - break; - case SyscallSandboxPolicy::TX_INDEX: // Thread: txindex - seccomp_policy_builder.AllowFileSystem(); - break; - case SyscallSandboxPolicy::VALIDATION_SCRIPT_CHECK: // Thread: scriptch.<N> - break; - case SyscallSandboxPolicy::SHUTOFF: // Thread: main thread (state: shutoff) - seccomp_policy_builder.AllowFileSystem(); - break; - } - - const SyscallSandboxAction default_action = g_syscall_sandbox_log_violation_before_terminating ? SyscallSandboxAction::INVOKE_SIGNAL_HANDLER : SyscallSandboxAction::KILL_PROCESS; - std::vector<sock_filter> filter = seccomp_policy_builder.BuildFilter(default_action); - const sock_fprog prog = { - .len = static_cast<uint16_t>(filter.size()), - .filter = filter.data(), - }; - // Do not allow abilities to be regained after being dropped. - // - // PR_SET_NO_NEW_PRIVS documentation: <https://www.kernel.org/doc/html/latest/userspace-api/no_new_privs.html> - if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) { - throw std::runtime_error("Syscall sandbox enforcement failed: prctl(PR_SET_NO_NEW_PRIVS)"); - } - // Install seccomp-bpf syscall filter. - // - // PR_SET_SECCOMP documentation: <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> - if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog) != 0) { - throw std::runtime_error("Syscall sandbox enforcement failed: prctl(PR_SET_SECCOMP)"); - } - - const std::string thread_name = !util::ThreadGetInternalName().empty() ? util::ThreadGetInternalName() : "*unnamed*"; - LogPrint(BCLog::UTIL, "Syscall filter installed for thread \"%s\"\n", thread_name); -#endif // defined(USE_SYSCALL_SANDBOX) -} diff --git a/src/util/syscall_sandbox.h b/src/util/syscall_sandbox.h deleted file mode 100644 index 3e56ebe937..0000000000 --- a/src/util/syscall_sandbox.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) 2020-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_UTIL_SYSCALL_SANDBOX_H -#define BITCOIN_UTIL_SYSCALL_SANDBOX_H - -enum class SyscallSandboxPolicy { - // 1. Initialization - INITIALIZATION, - INITIALIZATION_DNS_SEED, - INITIALIZATION_LOAD_BLOCKS, - INITIALIZATION_MAP_PORT, - - // 2. Steady state (non-initialization, non-shutdown) - MESSAGE_HANDLER, - NET, - NET_ADD_CONNECTION, - NET_HTTP_SERVER, - NET_HTTP_SERVER_WORKER, - NET_OPEN_CONNECTION, - SCHEDULER, - TOR_CONTROL, - TX_INDEX, - VALIDATION_SCRIPT_CHECK, - - // 3. Shutdown - SHUTOFF, -}; - -//! Force the current thread (and threads created from the current thread) into a restricted-service -//! operating mode where only a subset of all syscalls are available. -//! -//! Subsequent calls to this function can reduce the abilities further, but abilities can never be -//! regained. -//! -//! This function is a no-op unless SetupSyscallSandbox(...) has been called. -//! -//! SetupSyscallSandbox(...) is called during bitcoind initialization if Bitcoin Core was compiled -//! with seccomp-bpf support (--with-seccomp) *and* the parameter -sandbox=<mode> was passed to -//! bitcoind. -//! -//! This experimental feature is available under Linux x86_64 only. -void SetSyscallSandboxPolicy(SyscallSandboxPolicy syscall_policy); - -#if defined(USE_SYSCALL_SANDBOX) -//! Setup and enable the experimental syscall sandbox for the running process. -[[nodiscard]] bool SetupSyscallSandbox(bool log_syscall_violation_before_terminating); - -//! Invoke a disallowed syscall. Use for testing purposes. -void TestDisallowedSandboxCall(); -#endif // defined(USE_SYSCALL_SANDBOX) - -#endif // BITCOIN_UTIL_SYSCALL_SANDBOX_H diff --git a/src/util/system.h b/src/util/system.h deleted file mode 100644 index e2fc3450f6..0000000000 --- a/src/util/system.h +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_UTIL_SYSTEM_H -#define BITCOIN_UTIL_SYSTEM_H - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#include <compat/assumptions.h> -#include <compat/compat.h> - -#include <any> -#include <set> -#include <stdint.h> -#include <string> - -// Application startup time (used for uptime calculation) -int64_t GetStartupTime(); - -void SetupEnvironment(); -bool SetupNetworking(); -#ifndef WIN32 -std::string ShellEscape(const std::string& arg); -#endif -#if HAVE_SYSTEM -void runCommand(const std::string& strCommand); -#endif - -/** - * Return the number of cores available on the current system. - * @note This does count virtual cores, such as those provided by HyperThreading. - */ -int GetNumCores(); - -/** - * On platforms that support it, tell the kernel the calling thread is - * CPU-intensive and non-interactive. See SCHED_BATCH in sched(7) for details. - * - */ -void ScheduleBatchPriority(); - -namespace util { - -//! Simplification of std insertion -template <typename Tdst, typename Tsrc> -inline void insert(Tdst& dst, const Tsrc& src) { - dst.insert(dst.begin(), src.begin(), src.end()); -} -template <typename TsetT, typename Tsrc> -inline void insert(std::set<TsetT>& dst, const Tsrc& src) { - dst.insert(src.begin(), src.end()); -} - -/** - * Helper function to access the contained object of a std::any instance. - * Returns a pointer to the object if passed instance has a value and the type - * matches, nullptr otherwise. - */ -template<typename T> -T* AnyPtr(const std::any& any) noexcept -{ - T* const* ptr = std::any_cast<T*>(&any); - return ptr ? *ptr : nullptr; -} - -} // namespace util - -#endif // BITCOIN_UTIL_SYSTEM_H diff --git a/src/util/translation.h b/src/util/translation.h index d2b49d00b0..d33fd2d0a0 100644 --- a/src/util/translation.h +++ b/src/util/translation.h @@ -49,22 +49,18 @@ inline bilingual_str Untranslated(std::string original) { return {original, orig // Provide an overload of tinyformat::format which can take bilingual_str arguments. namespace tinyformat { -inline std::string TranslateArg(const bilingual_str& arg, bool translated) -{ - return translated ? arg.translated : arg.original; -} - -template <typename T> -inline T const& TranslateArg(const T& arg, bool translated) -{ - return arg; -} - template <typename... Args> bilingual_str format(const bilingual_str& fmt, const Args&... args) { - return bilingual_str{format(fmt.original, TranslateArg(args, false)...), - format(fmt.translated, TranslateArg(args, true)...)}; + const auto translate_arg{[](const auto& arg, bool translated) -> const auto& { + if constexpr (std::is_same_v<decltype(arg), const bilingual_str&>) { + return translated ? arg.translated : arg.original; + } else { + return arg; + } + }}; + return bilingual_str{tfm::format(fmt.original, translate_arg(args, false)...), + tfm::format(fmt.translated, translate_arg(args, true)...)}; } } // namespace tinyformat diff --git a/src/validation.cpp b/src/validation.cpp index e536dfb4eb..6836498a64 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -11,7 +11,6 @@ #include <arith_uint256.h> #include <chain.h> #include <checkqueue.h> -#include <common/args.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/merkle.h> @@ -23,10 +22,10 @@ #include <hash.h> #include <kernel/chainparams.h> #include <kernel/mempool_entry.h> +#include <kernel/notifications_interface.h> #include <logging.h> #include <logging/timer.h> #include <node/blockstorage.h> -#include <node/interface_ui.h> #include <node/utxo_snapshot.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -52,7 +51,6 @@ #include <util/moneystr.h> #include <util/rbf.h> #include <util/strencodings.h> -#include <util/system.h> #include <util/time.h> #include <util/trace.h> #include <util/translation.h> @@ -72,6 +70,7 @@ using kernel::CCoinsStats; using kernel::CoinStatsHashType; using kernel::ComputeUTXOStats; using kernel::LoadMempool; +using kernel::Notifications; using fsbridge::FopenFn; using node::BlockManager; @@ -1639,26 +1638,6 @@ bool Chainstate::IsInitialBlockDownload() const return false; } -static void AlertNotify(const std::string& strMessage) -{ - uiInterface.NotifyAlertChanged(); -#if HAVE_SYSTEM - std::string strCmd = gArgs.GetArg("-alertnotify", ""); - if (strCmd.empty()) return; - - // Alert text should be plain ascii coming from a trusted source, but to - // be safe we first strip anything not in safeChars, then add single quotes around - // the whole string before passing it to the shell: - std::string singleQuote("'"); - std::string safeStatus = SanitizeString(strMessage); - safeStatus = singleQuote+safeStatus+singleQuote; - ReplaceAll(strCmd, "%s", safeStatus); - - std::thread t(runCommand, strCmd); - t.detach(); // thread runs free -#endif -} - void Chainstate::CheckForkWarningConditions() { AssertLockHeld(cs_main); @@ -2014,8 +1993,6 @@ public: } }; -static std::array<ThresholdConditionCache, VERSIONBITS_NUM_BITS> warningcache GUARDED_BY(cs_main); - static unsigned int GetBlockScriptFlags(const CBlockIndex& block_index, const ChainstateManager& chainman) { const Consensus::Params& consensusparams = chainman.GetConsensus(); @@ -2520,7 +2497,7 @@ bool Chainstate::FlushStateToDisk( // Write blocks and block index to disk. if (fDoFullFlush || fPeriodicWrite) { // Ensure we can write block index - if (!CheckDiskSpace(gArgs.GetBlocksDirPath())) { + if (!CheckDiskSpace(m_blockman.m_opts.blocks_dir)) { return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); } { @@ -2556,7 +2533,7 @@ bool Chainstate::FlushStateToDisk( // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(gArgs.GetDataDirNet(), 48 * 2 * 2 * CoinsTip().GetCacheSize())) { + if (!CheckDiskSpace(m_chainman.m_options.datadir, 48 * 2 * 2 * CoinsTip().GetCacheSize())) { return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); } // Flush the chainstate (which may refer to block index entries). @@ -2599,16 +2576,6 @@ void Chainstate::PruneAndFlush() } } -static void DoWarning(const bilingual_str& warning) -{ - static bool fWarned = false; - SetMiscWarning(warning); - if (!fWarned) { - AlertNotify(warning.original); - fWarned = true; - } -} - /** Private helper function that concatenates warning messages. */ static void AppendWarning(bilingual_str& res, const bilingual_str& warn) { @@ -2671,11 +2638,11 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(m_chainman, bit); - ThresholdState state = checker.GetStateFor(pindex, params.GetConsensus(), warningcache.at(bit)); + ThresholdState state = checker.GetStateFor(pindex, params.GetConsensus(), m_chainman.m_warningcache.at(bit)); if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { const bilingual_str warning = strprintf(_("Unknown new rules activated (versionbit %i)"), bit); if (state == ThresholdState::ACTIVE) { - DoWarning(warning); + m_chainman.GetNotifications().warning(warning); } else { AppendWarning(warning_messages, warning); } @@ -2760,7 +2727,6 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra return true; } -static SteadyClock::duration time_read_from_disk_total{}; static SteadyClock::duration time_connect_total{}; static SteadyClock::duration time_flush{}; static SteadyClock::duration time_chainstate{}; @@ -2834,12 +2800,11 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, const CBlock& blockConnecting = *pthisBlock; // Apply the block atomically to the chain state. const auto time_2{SteadyClock::now()}; - time_read_from_disk_total += time_2 - time_1; SteadyClock::time_point time_3; - LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs (%.2fms/blk)]\n", - Ticks<MillisecondsDouble>(time_2 - time_1), - Ticks<SecondsDouble>(time_read_from_disk_total), - Ticks<MillisecondsDouble>(time_read_from_disk_total) / num_blocks_total); + // When adding aggregate statistics in the future, keep in mind that + // num_blocks_total may be zero until the ConnectBlock() call below. + LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms\n", + Ticks<MillisecondsDouble>(time_2 - time_1)); { CCoinsViewCache view(&CoinsTip()); bool rv = ConnectBlock(blockConnecting, state, pindexNew, view); @@ -2949,6 +2914,7 @@ CBlockIndex* Chainstate::FindMostWorkChain() while (pindexTest != pindexFailed) { if (fFailedChain) { pindexFailed->nStatus |= BLOCK_FAILED_CHILD; + m_blockman.m_dirty_blockindex.insert(pindexFailed); } else if (fMissingData) { // If we're missing data, then add back to m_blocks_unlinked, // so that if the block arrives in the future we can try adding @@ -3097,7 +3063,7 @@ static bool NotifyHeaderTip(Chainstate& chainstate) LOCKS_EXCLUDED(cs_main) { } // Send block tip changed notifications without cs_main if (fNotify) { - uiInterface.NotifyHeaderTip(GetSynchronizationState(fInitialBlockDownload), pindexHeader->nHeight, pindexHeader->nTime, false); + chainstate.m_chainman.GetNotifications().headerTip(GetSynchronizationState(fInitialBlockDownload), pindexHeader->nHeight, pindexHeader->nTime, false); } return fNotify; } @@ -3136,7 +3102,6 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< CBlockIndex *pindexMostWork = nullptr; CBlockIndex *pindexNewTip = nullptr; - int nStopAtHeight = gArgs.GetIntArg("-stopatheight", DEFAULT_STOPATHEIGHT); do { // Block until the validation queue drains. This should largely // never happen in normal operation, however may happen during @@ -3206,12 +3171,12 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); // Always notify the UI if a new block tip was connected - uiInterface.NotifyBlockTip(GetSynchronizationState(fInitialDownload), pindexNewTip); + m_chainman.GetNotifications().blockTip(GetSynchronizationState(fInitialDownload), *pindexNewTip); } } // When we reach this point, we switched to a new tip (stored in pindexNewTip). - if (nStopAtHeight && pindexNewTip && pindexNewTip->nHeight >= nStopAtHeight) StartShutdown(); + if (m_chainman.StopAtHeight() && pindexNewTip && pindexNewTip->nHeight >= m_chainman.StopAtHeight()) StartShutdown(); if (WITH_LOCK(::cs_main, return m_disabled)) { // Background chainstate has reached the snapshot base block, so exit. @@ -3403,7 +3368,7 @@ bool Chainstate::InvalidateBlock(BlockValidationState& state, CBlockIndex* pinde // Only notify about a new block tip if the active chain was modified. if (pindex_was_in_chain) { - uiInterface.NotifyBlockTip(GetSynchronizationState(IsInitialBlockDownload()), to_mark_failed->pprev); + m_chainman.GetNotifications().blockTip(GetSynchronizationState(IsInitialBlockDownload()), *to_mark_failed->pprev); } return true; } @@ -3920,7 +3885,7 @@ void ChainstateManager::ReportHeadersPresync(const arith_uint256& work, int64_t m_last_presync_update = now; } bool initial_download = chainstate.IsInitialBlockDownload(); - uiInterface.NotifyHeaderTip(GetSynchronizationState(initial_download), height, timestamp, /*presync=*/true); + GetNotifications().headerTip(GetSynchronizationState(initial_download), height, timestamp, /*presync=*/true); if (initial_download) { const int64_t blocks_left{(GetTime() - timestamp) / GetConsensus().nPowTargetSpacing}; const double progress{100.0 * height / (height + blocks_left)}; @@ -4145,14 +4110,15 @@ bool Chainstate::LoadChainTip() return true; } -CVerifyDB::CVerifyDB() +CVerifyDB::CVerifyDB(Notifications& notifications) + : m_notifications{notifications} { - uiInterface.ShowProgress(_("Verifying blocks…").translated, 0, false); + m_notifications.progress(_("Verifying blocks…"), 0, false); } CVerifyDB::~CVerifyDB() { - uiInterface.ShowProgress("", 100, false); + m_notifications.progress(bilingual_str{}, 100, false); } VerifyDBResult CVerifyDB::VerifyDB( @@ -4192,7 +4158,7 @@ VerifyDBResult CVerifyDB::VerifyDB( LogPrintf("Verification progress: %d%%\n", percentageDone); reportDone = percentageDone / 10; } - uiInterface.ShowProgress(_("Verifying blocks…").translated, percentageDone, false); + m_notifications.progress(_("Verifying blocks…"), percentageDone, false); if (pindex->nHeight <= chainstate.m_chain.Height() - nCheckDepth) { break; } @@ -4268,7 +4234,7 @@ VerifyDBResult CVerifyDB::VerifyDB( LogPrintf("Verification progress: %d%%\n", percentageDone); reportDone = percentageDone / 10; } - uiInterface.ShowProgress(_("Verifying blocks…").translated, percentageDone, false); + m_notifications.progress(_("Verifying blocks…"), percentageDone, false); pindex = chainstate.m_chain.Next(pindex); CBlock block; if (!chainstate.m_blockman.ReadBlockFromDisk(block, *pindex)) { @@ -4327,7 +4293,7 @@ bool Chainstate::ReplayBlocks() if (hashHeads.empty()) return true; // We're already in a consistent state. if (hashHeads.size() != 2) return error("ReplayBlocks(): unknown inconsistent state"); - uiInterface.ShowProgress(_("Replaying blocks…").translated, 0, false); + m_chainman.GetNotifications().progress(_("Replaying blocks…"), 0, false); LogPrintf("Replaying blocks\n"); const CBlockIndex* pindexOld = nullptr; // Old tip during the interrupted flush. @@ -4374,13 +4340,13 @@ bool Chainstate::ReplayBlocks() const CBlockIndex& pindex{*Assert(pindexNew->GetAncestor(nHeight))}; LogPrintf("Rolling forward %s (%i)\n", pindex.GetBlockHash().ToString(), nHeight); - uiInterface.ShowProgress(_("Replaying blocks…").translated, (int) ((nHeight - nForkHeight) * 100.0 / (pindexNew->nHeight - nForkHeight)) , false); + m_chainman.GetNotifications().progress(_("Replaying blocks…"), (int)((nHeight - nForkHeight) * 100.0 / (pindexNew->nHeight - nForkHeight)), false); if (!RollforwardBlock(&pindex, cache)) return false; } cache.SetBestBlock(pindexNew->GetBlockHash()); cache.Flush(); - uiInterface.ShowProgress("", 100, false); + m_chainman.GetNotifications().progress(bilingual_str{}, 100, false); return true; } @@ -5033,15 +4999,15 @@ static bool DeleteCoinsDBFromDisk(const fs::path db_path, bool is_snapshot) if (is_snapshot) { fs::path base_blockhash_path = db_path / node::SNAPSHOT_BLOCKHASH_FILENAME; - if (fs::exists(base_blockhash_path)) { - bool removed = fs::remove(base_blockhash_path); - if (!removed) { - LogPrintf("[snapshot] failed to remove file %s\n", - fs::PathToString(base_blockhash_path)); + try { + bool existed = fs::remove(base_blockhash_path); + if (!existed) { + LogPrintf("[snapshot] snapshot chainstate dir being removed lacks %s file\n", + fs::PathToString(node::SNAPSHOT_BLOCKHASH_FILENAME)); } - } else { - LogPrintf("[snapshot] snapshot chainstate dir being removed lacks %s file\n", - fs::PathToString(node::SNAPSHOT_BLOCKHASH_FILENAME)); + } catch (const fs::filesystem_error& e) { + LogPrintf("[snapshot] failed to remove file %s: %s\n", + fs::PathToString(base_blockhash_path), fsbridge::get_filesystem_error_message(e)); } } @@ -5139,7 +5105,7 @@ bool ChainstateManager::ActivateSnapshot( // PopulateAndValidateSnapshot can return (in error) before the leveldb datadir // has been created, so only attempt removal if we got that far. - if (auto snapshot_datadir = node::FindSnapshotChainstateDir()) { + if (auto snapshot_datadir = node::FindSnapshotChainstateDir(m_options.datadir)) { // We have to destruct leveldb::DB in order to release the db lock, otherwise // DestroyDB() (in DeleteCoinsDBFromDisk()) will fail. See `leveldb::~DBImpl()`. // Destructing the chainstate (and so resetting the coinsviews object) does this. @@ -5446,7 +5412,7 @@ SnapshotCompletionResult ChainstateManager::MaybeCompleteSnapshotValidation( "restart, the node will resume syncing from %d " "without using any snapshot data. " "Please report this incident to %s, including how you obtained the snapshot. " - "The invalid snapshot chainstate has been left on disk in case it is " + "The invalid snapshot chainstate will be left on disk in case it is " "helpful in diagnosing the issue that caused this error."), PACKAGE_NAME, snapshot_tip_height, snapshot_base_height, snapshot_base_height, PACKAGE_BUGREPORT ); @@ -5459,7 +5425,10 @@ SnapshotCompletionResult ChainstateManager::MaybeCompleteSnapshotValidation( assert(!this->IsUsable(m_snapshot_chainstate.get())); assert(this->IsUsable(m_ibd_chainstate.get())); - m_snapshot_chainstate->InvalidateCoinsDBOnDisk(); + auto rename_result = m_snapshot_chainstate->InvalidateCoinsDBOnDisk(); + if (!rename_result) { + user_error = strprintf(Untranslated("%s\n%s"), user_error, util::ErrorString(rename_result)); + } shutdown_fnc(user_error); }; @@ -5619,17 +5588,12 @@ ChainstateManager::~ChainstateManager() LOCK(::cs_main); m_versionbitscache.Clear(); - - // TODO: The warning cache should probably become non-global - for (auto& i : warningcache) { - i.clear(); - } } bool ChainstateManager::DetectSnapshotChainstate(CTxMemPool* mempool) { assert(!m_snapshot_chainstate); - std::optional<fs::path> path = node::FindSnapshotChainstateDir(); + std::optional<fs::path> path = node::FindSnapshotChainstateDir(m_options.datadir); if (!path) { return false; } @@ -5666,7 +5630,7 @@ bool IsBIP30Unspendable(const CBlockIndex& block_index) (block_index.nHeight==91812 && block_index.GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f")); } -void Chainstate::InvalidateCoinsDBOnDisk() +util::Result<void> Chainstate::InvalidateCoinsDBOnDisk() { AssertLockHeld(::cs_main); // Should never be called on a non-snapshot chainstate. @@ -5695,13 +5659,14 @@ void Chainstate::InvalidateCoinsDBOnDisk() LogPrintf("%s: error renaming file '%s' -> '%s': %s\n", __func__, src_str, dest_str, e.what()); - AbortNode(strprintf( + return util::Error{strprintf(_( "Rename of '%s' -> '%s' failed. " "You should resolve this by manually moving or deleting the invalid " "snapshot directory %s, otherwise you will encounter the same error again " - "on the next startup.", - src_str, dest_str, src_str)); + "on the next startup."), + src_str, dest_str, src_str)}; } + return {}; } const CBlockIndex* ChainstateManager::GetSnapshotBaseBlock() const diff --git a/src/validation.h b/src/validation.h index fd0e2115f7..8bc8842c54 100644 --- a/src/validation.h +++ b/src/validation.h @@ -31,6 +31,7 @@ #include <util/check.h> #include <util/fs.h> #include <util/hasher.h> +#include <util/result.h> #include <util/translation.h> #include <versionbits.h> @@ -42,6 +43,7 @@ #include <stdint.h> #include <string> #include <thread> +#include <type_traits> #include <utility> #include <vector> @@ -65,8 +67,6 @@ struct Params; static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; -/** Default for -stopatheight */ -static const int DEFAULT_STOPATHEIGHT = 0; /** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ static const unsigned int MIN_BLOCKS_TO_KEEP = 288; static const signed int DEFAULT_CHECKBLOCKS = 6; @@ -331,6 +331,11 @@ public: ScriptError GetScriptError() const { return error; } }; +// CScriptCheck is used a lot in std::vector, make sure that's efficient +static_assert(std::is_nothrow_move_assignable_v<CScriptCheck>); +static_assert(std::is_nothrow_move_constructible_v<CScriptCheck>); +static_assert(std::is_nothrow_destructible_v<CScriptCheck>); + /** Initializes the script-execution cache */ [[nodiscard]] bool InitScriptExecutionCache(size_t max_size_bytes); @@ -364,9 +369,13 @@ enum class VerifyDBResult { }; /** RAII wrapper for VerifyDB: Verify consistency of the block and coin databases */ -class CVerifyDB { +class CVerifyDB +{ +private: + kernel::Notifications& m_notifications; + public: - CVerifyDB(); + explicit CVerifyDB(kernel::Notifications& notifications); ~CVerifyDB(); [[nodiscard]] VerifyDBResult VerifyDB( Chainstate& chainstate, @@ -808,7 +817,7 @@ private: * In case of an invalid snapshot, rename the coins leveldb directory so * that it can be examined for issue diagnosis. */ - void InvalidateCoinsDBOnDisk() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + [[nodiscard]] util::Result<void> InvalidateCoinsDBOnDisk() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); friend ChainstateManager; }; @@ -936,6 +945,8 @@ private: //! nullopt. std::optional<int> GetSnapshotBaseHeight() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + std::array<ThresholdConditionCache, VERSIONBITS_NUM_BITS> m_warningcache GUARDED_BY(::cs_main); + //! Return true if a chainstate is considered usable. //! //! This is false when a background validation chainstate has completed its @@ -955,6 +966,8 @@ public: bool ShouldCheckBlockIndex() const { return *Assert(m_options.check_block_index); } const arith_uint256& MinimumChainWork() const { return *Assert(m_options.minimum_chain_work); } const uint256& AssumedValidBlock() const { return *Assert(m_options.assumed_valid_block); } + kernel::Notifications& GetNotifications() const { return m_options.notifications; }; + int StopAtHeight() const { return m_options.stop_at_height; }; /** * Alias for ::cs_main. diff --git a/src/version.h b/src/version.h index ee646eefc3..611a670314 100644 --- a/src/version.h +++ b/src/version.h @@ -20,9 +20,6 @@ static const int MIN_PEER_PROTO_VERSION = 31800; //! BIP 0031, pong message, is enabled for all versions AFTER this one static const int BIP0031_VERSION = 60000; -//! "filter*" commands are disabled without NODE_BLOOM after and including this version -static const int NO_BLOOM_VERSION = 70011; - //! "sendheaders" command and announcing blocks with headers starts with this version static const int SENDHEADERS_VERSION = 70012; diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 6dce51fc12..658500e456 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -31,6 +31,10 @@ namespace wallet { namespace { +Span<const std::byte> SpanFromDbt(const SafeDbt& dbt) +{ + return {reinterpret_cast<const std::byte*>(dbt.get_data()), dbt.get_size()}; +} //! Make sure database has a unique fileid within the environment. If it //! doesn't, throw an error. BDB caches do not work properly when more than one @@ -668,7 +672,8 @@ void BerkeleyDatabase::ReloadDbEnv() env->ReloadDbEnv(); } -BerkeleyCursor::BerkeleyCursor(BerkeleyDatabase& database, const BerkeleyBatch& batch) +BerkeleyCursor::BerkeleyCursor(BerkeleyDatabase& database, const BerkeleyBatch& batch, Span<const std::byte> prefix) + : m_key_prefix(prefix.begin(), prefix.end()) { if (!database.m_db.get()) { throw std::runtime_error(STR_INTERNAL_BUG("BerkeleyDatabase does not exist")); @@ -685,21 +690,32 @@ DatabaseCursor::Status BerkeleyCursor::Next(DataStream& ssKey, DataStream& ssVal { if (m_cursor == nullptr) return Status::FAIL; // Read at cursor - SafeDbt datKey; + SafeDbt datKey(m_key_prefix.data(), m_key_prefix.size()); SafeDbt datValue; - int ret = m_cursor->get(datKey, datValue, DB_NEXT); + int ret = -1; + if (m_first && !m_key_prefix.empty()) { + ret = m_cursor->get(datKey, datValue, DB_SET_RANGE); + } else { + ret = m_cursor->get(datKey, datValue, DB_NEXT); + } + m_first = false; if (ret == DB_NOTFOUND) { return Status::DONE; } - if (ret != 0 || datKey.get_data() == nullptr || datValue.get_data() == nullptr) { + if (ret != 0) { return Status::FAIL; } + Span<const std::byte> raw_key = SpanFromDbt(datKey); + if (!m_key_prefix.empty() && std::mismatch(raw_key.begin(), raw_key.end(), m_key_prefix.begin(), m_key_prefix.end()).second != m_key_prefix.end()) { + return Status::DONE; + } + // Convert to streams ssKey.clear(); - ssKey.write({AsBytePtr(datKey.get_data()), datKey.get_size()}); + ssKey.write(raw_key); ssValue.clear(); - ssValue.write({AsBytePtr(datValue.get_data()), datValue.get_size()}); + ssValue.write(SpanFromDbt(datValue)); return Status::MORE; } @@ -716,6 +732,12 @@ std::unique_ptr<DatabaseCursor> BerkeleyBatch::GetNewCursor() return std::make_unique<BerkeleyCursor>(m_database, *this); } +std::unique_ptr<DatabaseCursor> BerkeleyBatch::GetNewPrefixCursor(Span<const std::byte> prefix) +{ + if (!pdb) return nullptr; + return std::make_unique<BerkeleyCursor>(m_database, *this, prefix); +} + bool BerkeleyBatch::TxnBegin() { if (!pdb || activeTxn) @@ -777,7 +799,8 @@ bool BerkeleyBatch::ReadKey(DataStream&& key, DataStream& value) SafeDbt datValue; int ret = pdb->get(activeTxn, datKey, datValue, 0); if (ret == 0 && datValue.get_data() != nullptr) { - value.write({AsBytePtr(datValue.get_data()), datValue.get_size()}); + value.clear(); + value.write(SpanFromDbt(datValue)); return true; } return false; diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index 9134643b23..1073d32e0f 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -7,10 +7,10 @@ #define BITCOIN_WALLET_BDB_H #include <clientversion.h> +#include <common/system.h> #include <serialize.h> #include <streams.h> #include <util/fs.h> -#include <util/system.h> #include <wallet/db.h> #include <atomic> @@ -21,14 +21,7 @@ #include <unordered_map> #include <vector> -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wsuggest-override" -#endif #include <db_cxx.h> -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif struct bilingual_str; @@ -190,9 +183,13 @@ class BerkeleyCursor : public DatabaseCursor { private: Dbc* m_cursor; + std::vector<std::byte> m_key_prefix; + bool m_first{true}; public: - explicit BerkeleyCursor(BerkeleyDatabase& database, const BerkeleyBatch& batch); + // Constructor for cursor for records matching the prefix + // To match all records, an empty prefix may be provided. + explicit BerkeleyCursor(BerkeleyDatabase& database, const BerkeleyBatch& batch, Span<const std::byte> prefix = {}); ~BerkeleyCursor() override; Status Next(DataStream& key, DataStream& value) override; @@ -229,6 +226,7 @@ public: void Close() override; std::unique_ptr<DatabaseCursor> GetNewCursor() override; + std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override; bool TxnBegin() override; bool TxnCommit() override; bool TxnAbort() override; diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 832d24746f..d6b9b68e1f 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -4,13 +4,13 @@ #include <wallet/coinselection.h> +#include <common/system.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <logging.h> #include <policy/feerate.h> #include <util/check.h> #include <util/moneystr.h> -#include <util/system.h> #include <numeric> #include <optional> @@ -191,7 +191,7 @@ public: } }; -util::Result<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value, FastRandomContext& rng, +util::Result<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value, CAmount change_fee, FastRandomContext& rng, int max_weight) { SelectionResult result(target_value, SelectionAlgorithm::SRD); @@ -201,7 +201,7 @@ util::Result<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utx // barely meets the target. Just use the lower bound change target instead of the randomly // generated one, since SRD will result in a random change amount anyway; avoid making the // target needlessly large. - target_value += CHANGE_LOWER; + target_value += CHANGE_LOWER + change_fee; std::vector<size_t> indexes; indexes.resize(utxo_pool.size()); diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index 723f5bbfb3..afd868fc89 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -11,8 +11,8 @@ #include <policy/feerate.h> #include <primitives/transaction.h> #include <random.h> -#include <util/system.h> #include <util/check.h> +#include <util/insert.h> #include <util/result.h> #include <optional> @@ -421,7 +421,7 @@ util::Result<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool * @param[in] max_weight The maximum allowed weight for a selection result to be valid * @returns If successful, a valid SelectionResult, otherwise, util::Error */ -util::Result<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value, FastRandomContext& rng, +util::Result<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value, CAmount change_fee, FastRandomContext& rng, int max_weight); // Original coin selection algorithm as a fallback diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index cd414b3d44..e2799c2d05 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -4,9 +4,9 @@ #include <wallet/crypter.h> +#include <common/system.h> #include <crypto/aes.h> #include <crypto/sha512.h> -#include <util/system.h> #include <vector> diff --git a/src/wallet/db.h b/src/wallet/db.h index b4ccd13a9a..9d684225c3 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -113,6 +113,7 @@ public: virtual bool ErasePrefix(Span<const std::byte> prefix) = 0; virtual std::unique_ptr<DatabaseCursor> GetNewCursor() = 0; + virtual std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) = 0; virtual bool TxnBegin() = 0; virtual bool TxnCommit() = 0; virtual bool TxnAbort() = 0; diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index 865af0bb23..3ac5cf03b1 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -57,12 +57,12 @@ bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error) // Write out a magic string with version std::string line = strprintf("%s,%u\n", DUMP_MAGIC, DUMP_VERSION); dump_file.write(line.data(), line.size()); - hasher.write(MakeByteSpan(line)); + hasher << Span{line}; // Write out the file format line = strprintf("%s,%s\n", "format", db.Format()); dump_file.write(line.data(), line.size()); - hasher.write(MakeByteSpan(line)); + hasher << Span{line}; if (ret) { @@ -83,7 +83,7 @@ bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error) std::string value_str = HexStr(ss_value); line = strprintf("%s,%s\n", key_str, value_str); dump_file.write(line.data(), line.size()); - hasher.write(MakeByteSpan(line)); + hasher << Span{line}; } } @@ -160,7 +160,7 @@ bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs:: return false; } std::string magic_hasher_line = strprintf("%s,%s\n", magic_key, version_value); - hasher.write(MakeByteSpan(magic_hasher_line)); + hasher << Span{magic_hasher_line}; // Get the stored file format std::string format_key; @@ -191,7 +191,7 @@ bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs:: warnings.push_back(strprintf(_("Warning: Dumpfile wallet format \"%s\" does not match command line specified format \"%s\"."), format_value, file_format)); } std::string format_hasher_line = strprintf("%s,%s\n", format_key, format_value); - hasher.write(MakeByteSpan(format_hasher_line)); + hasher << Span{format_hasher_line}; DatabaseOptions options; DatabaseStatus status; @@ -236,7 +236,7 @@ bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs:: } std::string line = strprintf("%s,%s\n", key, value); - hasher.write(MakeByteSpan(line)); + hasher << Span{line}; if (key.empty() || value.empty()) { continue; @@ -255,11 +255,7 @@ bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs:: std::vector<unsigned char> k = ParseHex(key); std::vector<unsigned char> v = ParseHex(value); - - DataStream ss_key{k}; - DataStream ss_value{v}; - - if (!batch->Write(ss_key, ss_value)) { + if (!batch->Write(Span{k}, Span{v})) { error = strprintf(_("Error: Unable to write record to new wallet")); ret = false; break; diff --git a/src/wallet/external_signer_scriptpubkeyman.cpp b/src/wallet/external_signer_scriptpubkeyman.cpp index e2852c5d52..6d026d02c1 100644 --- a/src/wallet/external_signer_scriptpubkeyman.cpp +++ b/src/wallet/external_signer_scriptpubkeyman.cpp @@ -4,8 +4,8 @@ #include <chainparams.h> #include <common/args.h> +#include <common/system.h> #include <external_signer.h> -#include <util/system.h> #include <wallet/external_signer_scriptpubkeyman.h> #include <iostream> diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index b6b1fa1d3e..0e1f1f9847 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <consensus/validation.h> #include <interfaces/chain.h> #include <policy/fees.h> #include <policy/policy.h> #include <util/moneystr.h> #include <util/rbf.h> -#include <util/system.h> #include <util/translation.h> #include <wallet/coincontrol.h> #include <wallet/feebumper.h> diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 2560dda87c..4cdfadbee2 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -62,7 +62,7 @@ bool VerifyWallets(WalletContext& context) options.require_existing = true; options.verify = false; if (MakeWalletDatabase("", options, status, error_string)) { - util::SettingsValue wallets(util::SettingsValue::VARR); + common::SettingsValue wallets(common::SettingsValue::VARR); wallets.push_back(""); // Default wallet name is "" // Pass write=false because no need to write file and probably // better not to. If unnamed wallet needs to be added next startup diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index 0bd6a9670c..a8ef0a5731 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -677,11 +677,11 @@ RPCHelpMan getaddressesbylabel() CHECK_NONFATAL(unique); // UniValue::pushKV checks if the key exists in O(N) // and since duplicate addresses are unexpected (checked with - // std::set in O(log(N))), UniValue::__pushKV is used instead, + // std::set in O(log(N))), UniValue::pushKVEnd is used instead, // which currently is O(1). UniValue value(UniValue::VOBJ); value.pushKV("purpose", _purpose ? PurposeToString(*_purpose) : "unknown"); - ret.__pushKV(address, value); + ret.pushKVEnd(address, value); } }); diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 43da15b36e..af8043f158 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -1298,7 +1298,7 @@ RPCHelpMan importmulti() }, }, RPCArgOptions{.oneline_description="\"requests\""}}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions after all imports."}, }, @@ -1862,7 +1862,7 @@ RPCHelpMan listdescriptors() RPCHelpMan backupwallet() { return RPCHelpMan{"backupwallet", - "\nSafely copies current wallet file to destination, which can be a directory or a path with filename.\n", + "\nSafely copies the current wallet file to the specified destination, which can either be a directory or a path with a filename.\n", { {"destination", RPCArg::Type::STR, RPCArg::Optional::NO, "The destination directory or file"}, }, @@ -1897,7 +1897,7 @@ RPCHelpMan restorewallet() { return RPCHelpMan{ "restorewallet", - "\nRestore and loads a wallet from backup.\n" + "\nRestores and loads a wallet from backup.\n" "\nThe rescan is significantly faster if a descriptor wallet is restored" "\nand block filters are available (using startup option \"-blockfilterindex=1\").\n", { @@ -1909,8 +1909,7 @@ RPCHelpMan restorewallet() RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR, "name", "The wallet name if restored successfully."}, - {RPCResult::Type::STR, "warning", /*optional=*/true, "Warning messages, if any, related to restoring the wallet. Multiple messages will be delimited by newlines. (DEPRECATED, returned only if config option -deprecatedrpc=walletwarningfield is passed.)"}, - {RPCResult::Type::ARR, "warnings", /*optional=*/true, "Warning messages, if any, related to restoring the wallet.", + {RPCResult::Type::ARR, "warnings", /*optional=*/true, "Warning messages, if any, related to restoring and loading the wallet.", { {RPCResult::Type::STR, "", ""}, }}, @@ -1943,9 +1942,6 @@ RPCHelpMan restorewallet() UniValue obj(UniValue::VOBJ); obj.pushKV("name", wallet->GetName()); - if (wallet->chain().rpcEnableDeprecated("walletwarningfield")) { - obj.pushKV("warning", Join(warnings, Untranslated("\n")).original); - } PushWarnings(warnings, obj); return obj; diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index 5bdd3c9e72..22f0f0b83c 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -513,7 +513,7 @@ RPCHelpMan listunspent() }, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{true}, "Include outputs that are not safe to spend\n" "See description of \"safe\" attribute below."}, - {"query_options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "JSON with query options", + {"query_options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"minimumAmount", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(0)}, "Minimum value of each UTXO in " + CURRENCY_UNIT + ""}, {"maximumAmount", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"unlimited"}, "Maximum value of each UTXO in " + CURRENCY_UNIT + ""}, diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index a5b1f594bf..feee643f26 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -453,9 +453,9 @@ RPCHelpMan settxfee() static std::vector<RPCArg> FundTxDoc(bool solving_data = true) { std::vector<RPCArg> args = { - {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, + {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks", RPCArgOptions{.also_positional = true}}, {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, "The fee estimate mode, must be one of (case insensitive):\n" - "\"" + FeeModes("\"\n\"") + "\""}, + "\"" + FeeModes("\"\n\"") + "\"", RPCArgOptions{.also_positional = true}}, { "replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125-replaceable.\n" "Allows this transaction to be replaced by a transaction with higher fees" @@ -758,7 +758,7 @@ RPCHelpMan fundrawtransaction() "Only pay-to-pubkey, multisig, and P2SH versions thereof are currently supported for watch-only\n", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "For backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::Default{true}, "For a transaction with existing inputs, automatically include more if they are not enough."}, @@ -997,7 +997,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) "* WARNING: before version 0.21, fee_rate was in " + CURRENCY_UNIT + "/kvB. As of 0.21, fee_rate is in " + CURRENCY_ATOM + "/vB. *\n", { {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The txid to be bumped"}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks\n"}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, @@ -1187,7 +1187,7 @@ RPCHelpMan send() {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, "The fee estimate mode, must be one of (case insensitive):\n" "\"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::DefaultHint{"false when \"inputs\" are specified, true otherwise"},"Automatically include coins from the wallet to cover the target amount.\n"}, @@ -1200,7 +1200,7 @@ RPCHelpMan send() {"change_address", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"change_position", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if change_address is not specified. Options are \"legacy\", \"p2sh-segwit\", \"bech32\" and \"bech32m\"."}, - {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, + {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB.", RPCArgOptions{.also_positional = true}}, {"include_watching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch only.\n" "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" "e.g. with 'importpubkey' or 'importmulti' with the 'pubkeys' or 'desc' field."}, @@ -1302,11 +1302,11 @@ RPCHelpMan sendall() "\"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, { - "options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + "options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", Cat<std::vector<RPCArg>>( { {"add_to_wallet", RPCArg::Type::BOOL, RPCArg::Default{true}, "When false, returns the serialized transaction without broadcasting or adding it to the wallet"}, - {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, + {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB.", RPCArgOptions{.also_positional = true}}, {"include_watching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch-only.\n" "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" "e.g. with 'importpubkey' or 'importmulti' with the 'pubkeys' or 'desc' field."}, @@ -1635,7 +1635,7 @@ RPCHelpMan walletcreatefundedpsbt() OutputsDoc(), RPCArgOptions{.skip_type_check = true}}, {"locktime", RPCArg::Type::NUM, RPCArg::Default{0}, "Raw locktime. Non-0 value also locktime-activates inputs"}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::DefaultHint{"false when \"inputs\" are specified, true otherwise"}, "Automatically include coins from the wallet to cover the target amount.\n"}, diff --git a/src/wallet/rpc/util.cpp b/src/wallet/rpc/util.cpp index 4ff44b84b0..06ec7db1bc 100644 --- a/src/wallet/rpc/util.cpp +++ b/src/wallet/rpc/util.cpp @@ -6,7 +6,7 @@ #include <common/url.h> #include <rpc/util.h> -#include <util/system.h> +#include <util/any.h> #include <util/translation.h> #include <wallet/context.h> #include <wallet/wallet.h> diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index a22862bfa9..fb4b642bba 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -68,6 +68,7 @@ static RPCHelpMan getwalletinfo() }, /*skip_type_check=*/true}, {RPCResult::Type::BOOL, "descriptors", "whether this wallet uses descriptors for scriptPubKey management"}, {RPCResult::Type::BOOL, "external_signer", "whether this wallet is configured to use an external signer such as a hardware wallet"}, + {RPCResult::Type::BOOL, "blank", "Whether this wallet intentionally does not contain any keys, scripts, or descriptors"}, RESULT_LAST_PROCESSED_BLOCK, }}, }, @@ -130,6 +131,7 @@ static RPCHelpMan getwalletinfo() } obj.pushKV("descriptors", pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); obj.pushKV("external_signer", pwallet->IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)); + obj.pushKV("blank", pwallet->IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); AppendLastProcessedBlock(obj, *pwallet); return obj; @@ -219,7 +221,6 @@ static RPCHelpMan loadwallet() RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR, "name", "The wallet name if loaded successfully."}, - {RPCResult::Type::STR, "warning", /*optional=*/true, "Warning messages, if any, related to loading the wallet. Multiple messages will be delimited by newlines. (DEPRECATED, returned only if config option -deprecatedrpc=walletwarningfield is passed.)"}, {RPCResult::Type::ARR, "warnings", /*optional=*/true, "Warning messages, if any, related to loading the wallet.", { {RPCResult::Type::STR, "", ""}, @@ -256,9 +257,6 @@ static RPCHelpMan loadwallet() UniValue obj(UniValue::VOBJ); obj.pushKV("name", wallet->GetName()); - if (wallet->chain().rpcEnableDeprecated("walletwarningfield")) { - obj.pushKV("warning", Join(warnings, Untranslated("\n")).original); - } PushWarnings(warnings, obj); return obj; @@ -354,8 +352,7 @@ static RPCHelpMan createwallet() RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR, "name", "The wallet name if created successfully. If the wallet was created using a full path, the wallet_name will be the full path."}, - {RPCResult::Type::STR, "warning", /*optional=*/true, "Warning messages, if any, related to creating the wallet. Multiple messages will be delimited by newlines. (DEPRECATED, returned only if config option -deprecatedrpc=walletwarningfield is passed.)"}, - {RPCResult::Type::ARR, "warnings", /*optional=*/true, "Warning messages, if any, related to creating the wallet.", + {RPCResult::Type::ARR, "warnings", /*optional=*/true, "Warning messages, if any, related to creating and loading the wallet.", { {RPCResult::Type::STR, "", ""}, }}, @@ -428,9 +425,6 @@ static RPCHelpMan createwallet() UniValue obj(UniValue::VOBJ); obj.pushKV("name", wallet->GetName()); - if (wallet->chain().rpcEnableDeprecated("walletwarningfield")) { - obj.pushKV("warning", Join(warnings, Untranslated("\n")).original); - } PushWarnings(warnings, obj); return obj; @@ -441,14 +435,13 @@ static RPCHelpMan createwallet() static RPCHelpMan unloadwallet() { return RPCHelpMan{"unloadwallet", - "Unloads the wallet referenced by the request endpoint otherwise unloads the wallet specified in the argument.\n" + "Unloads the wallet referenced by the request endpoint, otherwise unloads the wallet specified in the argument.\n" "Specifying the wallet name on a wallet endpoint is invalid.", { {"wallet_name", RPCArg::Type::STR, RPCArg::DefaultHint{"the wallet name from the RPC endpoint"}, "The name of the wallet to unload. If provided both here and in the RPC endpoint, the two must be identical."}, {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, }, RPCResult{RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::STR, "warning", /*optional=*/true, "Warning messages, if any, related to unloading the wallet. Multiple messages will be delimited by newlines. (DEPRECATED, returned only if config option -deprecatedrpc=walletwarningfield is passed.)"}, {RPCResult::Type::ARR, "warnings", /*optional=*/true, "Warning messages, if any, related to unloading the wallet.", { {RPCResult::Type::STR, "", ""}, @@ -490,13 +483,12 @@ static RPCHelpMan unloadwallet() throw JSONRPCError(RPC_MISC_ERROR, "Requested wallet already unloaded"); } } + + UnloadWallet(std::move(wallet)); + UniValue result(UniValue::VOBJ); - if (wallet->chain().rpcEnableDeprecated("walletwarningfield")) { - result.pushKV("warning", Join(warnings, Untranslated("\n")).original); - } PushWarnings(warnings, result); - UnloadWallet(std::move(wallet)); return result; }, }; @@ -645,7 +637,7 @@ RPCHelpMan simulaterawtransaction() {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, }, }, - {"options", RPCArg::Type::OBJ_USER_KEYS, RPCArg::Optional::OMITTED, "Options", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"include_watchonly", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Whether to include watch-only addresses (see RPC importaddress)"}, }, diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index ab73e67285..da16435f04 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -18,11 +18,6 @@ static const char *HEADER_END = "HEADER=END"; static const char *DATA_END = "DATA=END"; typedef std::pair<std::vector<unsigned char>, std::vector<unsigned char> > KeyValPair; -static bool KeyFilter(const std::string& type) -{ - return WalletBatch::IsKeyType(type) || type == DBKeys::HDCHAIN; -} - class DummyCursor : public DatabaseCursor { Status Next(DataStream& key, DataStream& value) override { return Status::FAIL; } @@ -43,6 +38,7 @@ public: void Close() override {} std::unique_ptr<DatabaseCursor> GetNewCursor() override { return std::make_unique<DummyCursor>(); } + std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override { return GetNewCursor(); } bool TxnBegin() override { return true; } bool TxnCommit() override { return true; } bool TxnAbort() override { return true; } @@ -185,17 +181,24 @@ bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bil { /* Filter for only private key type KV pairs to be added to the salvaged wallet */ DataStream ssKey{row.first}; - CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); + DataStream ssValue(row.second); std::string strType, strErr; - bool fReadOK; - { - // Required in LoadKeyMetadata(): - LOCK(dummyWallet.cs_wallet); - fReadOK = ReadKeyValue(&dummyWallet, ssKey, ssValue, strType, strErr, KeyFilter); - } - if (!KeyFilter(strType)) { + + // We only care about KEY, MASTER_KEY, CRYPTED_KEY, and HDCHAIN types + ssKey >> strType; + bool fReadOK = false; + if (strType == DBKeys::KEY) { + fReadOK = LoadKey(&dummyWallet, ssKey, ssValue, strErr); + } else if (strType == DBKeys::CRYPTED_KEY) { + fReadOK = LoadCryptedKey(&dummyWallet, ssKey, ssValue, strErr); + } else if (strType == DBKeys::MASTER_KEY) { + fReadOK = LoadEncryptionKey(&dummyWallet, ssKey, ssValue, strErr); + } else if (strType == DBKeys::HDCHAIN) { + fReadOK = LoadHDChain(&dummyWallet, ssValue, strErr); + } else { continue; } + if (!fReadOK) { warnings.push_back(strprintf(Untranslated("WARNING: WalletBatch::Recover skipping %s: %s"), strType, strErr)); diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 953fdb63b6..796b7f11c5 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -710,6 +710,8 @@ void LegacyScriptPubKeyMan::UpdateTimeFirstKey(int64_t nCreateTime) } else if (!nTimeFirstKey || nCreateTime < nTimeFirstKey) { nTimeFirstKey = nCreateTime; } + + NotifyFirstKeyTimeChanged(this, nTimeFirstKey); } bool LegacyScriptPubKeyMan::LoadKey(const CKey& key, const CPubKey &pubkey) @@ -755,12 +757,12 @@ bool LegacyScriptPubKeyMan::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& s RemoveWatchOnly(script); } + m_storage.UnsetBlankWalletFlag(batch); if (!m_storage.HasEncryptionKeys()) { return batch.WriteKey(pubkey, secret.GetPrivKey(), mapKeyMetadata[pubkey.GetID()]); } - m_storage.UnsetBlankWalletFlag(batch); return true; } @@ -2736,6 +2738,8 @@ void DescriptorScriptPubKeyMan::UpdateWalletDescriptor(WalletDescriptor& descrip m_map_script_pub_keys.clear(); m_max_cached_index = -1; m_wallet_descriptor = descriptor; + + NotifyFirstKeyTimeChanged(this, m_wallet_descriptor.creation_time); } bool DescriptorScriptPubKeyMan::CanUpdateToWalletDescriptor(const WalletDescriptor& descriptor, std::string& error) diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 22b67c88e9..bf35c776ae 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_WALLET_SCRIPTPUBKEYMAN_H #define BITCOIN_WALLET_SCRIPTPUBKEYMAN_H +#include <logging.h> #include <psbt.h> #include <script/descriptor.h> #include <script/signingprovider.h> @@ -27,6 +28,8 @@ enum class OutputType; struct bilingual_str; namespace wallet { +struct MigrationData; + // Wallet storage things that ScriptPubKeyMans need in order to be able to store things to the wallet database. // It provides access to things that are part of the entire wallet and not specific to a ScriptPubKeyMan such as // wallet flags, wallet version, encryption keys, encryption status, and the database itself. This allows a @@ -256,6 +259,9 @@ public: /** Keypool has new keys */ boost::signals2::signal<void ()> NotifyCanGetAddressesChanged; + + /** Birth time changed */ + boost::signals2::signal<void (const ScriptPubKeyMan* spkm, int64_t new_birth_time)> NotifyFirstKeyTimeChanged; }; /** OutputTypes supported by the LegacyScriptPubKeyMan */ @@ -658,6 +664,18 @@ public: void UpgradeDescriptorCache(); }; + +/** struct containing information needed for migrating legacy wallets to descriptor wallets */ +struct MigrationData +{ + CExtKey master_key; + std::vector<std::pair<std::string, int64_t>> watch_descs; + std::vector<std::pair<std::string, int64_t>> solvable_descs; + std::vector<std::unique_ptr<DescriptorScriptPubKeyMan>> desc_spkms; + std::shared_ptr<CWallet> watchonly_wallet{nullptr}; + std::shared_ptr<CWallet> solvable_wallet{nullptr}; +}; + } // namespace wallet #endif // BITCOIN_WALLET_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index b14a30921b..a3728223fb 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -4,6 +4,7 @@ #include <algorithm> #include <common/args.h> +#include <common/system.h> #include <consensus/amount.h> #include <consensus/validation.h> #include <interfaces/chain.h> @@ -15,7 +16,6 @@ #include <util/fees.h> #include <util/moneystr.h> #include <util/rbf.h> -#include <util/system.h> #include <util/trace.h> #include <util/translation.h> #include <wallet/coincontrol.h> @@ -583,7 +583,7 @@ util::Result<SelectionResult> ChooseSelectionResult(const CAmount& nTargetValue, results.push_back(*knapsack_result); } else append_error(knapsack_result); - if (auto srd_result{SelectCoinsSRD(groups.positive_group, nTargetValue, coin_selection_params.rng_fast, max_inputs_weight)}) { + if (auto srd_result{SelectCoinsSRD(groups.positive_group, nTargetValue, coin_selection_params.m_change_fee, coin_selection_params.rng_fast, max_inputs_weight)}) { srd_result->ComputeAndSetWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); results.push_back(*srd_result); } else append_error(srd_result); diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 77e8a4e9c1..ecd34bb96a 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -9,6 +9,7 @@ #include <logging.h> #include <sync.h> #include <util/fs_helpers.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/translation.h> #include <wallet/db.h> @@ -23,6 +24,12 @@ namespace wallet { static constexpr int32_t WALLET_SCHEMA_VERSION = 0; +static Span<const std::byte> SpanFromBlob(sqlite3_stmt* stmt, int col) +{ + return {reinterpret_cast<const std::byte*>(sqlite3_column_blob(stmt, col)), + static_cast<size_t>(sqlite3_column_bytes(stmt, col))}; +} + static void ErrorLogCallback(void* arg, int code, const char* msg) { // From sqlite3_config() documentation for the SQLITE_CONFIG_LOG option: @@ -34,12 +41,31 @@ static void ErrorLogCallback(void* arg, int code, const char* msg) LogPrintf("SQLite Error. Code: %d. Message: %s\n", code, msg); } +static int TraceSqlCallback(unsigned code, void* context, void* param1, void* param2) +{ + auto* db = static_cast<SQLiteDatabase*>(context); + if (code == SQLITE_TRACE_STMT) { + auto* stmt = static_cast<sqlite3_stmt*>(param1); + // To be conservative and avoid leaking potentially secret information + // in the log file, only expand statements that query the database, not + // statements that update the database. + char* expanded{sqlite3_stmt_readonly(stmt) ? sqlite3_expanded_sql(stmt) : nullptr}; + LogPrintf("[%s] SQLite Statement: %s\n", db->Filename(), expanded ? expanded : sqlite3_sql(stmt)); + if (expanded) sqlite3_free(expanded); + } + return SQLITE_OK; +} + static bool BindBlobToStatement(sqlite3_stmt* stmt, int index, Span<const std::byte> blob, const std::string& description) { - int res = sqlite3_bind_blob(stmt, index, blob.data(), blob.size(), SQLITE_STATIC); + // Pass a pointer to the empty string "" below instead of passing the + // blob.data() pointer if the blob.data() pointer is null. Passing a null + // data pointer to bind_blob would cause sqlite to bind the SQL NULL value + // instead of the empty blob value X'', which would mess up SQL comparisons. + int res = sqlite3_bind_blob(stmt, index, blob.data() ? static_cast<const void*>(blob.data()) : "", blob.size(), SQLITE_STATIC); if (res != SQLITE_OK) { LogPrintf("Unable to bind %s to statement: %s\n", description, sqlite3_errstr(res)); sqlite3_clear_bindings(stmt); @@ -235,6 +261,13 @@ void SQLiteDatabase::Open() if (ret != SQLITE_OK) { throw std::runtime_error(strprintf("SQLiteDatabase: Failed to enable extended result codes: %s\n", sqlite3_errstr(ret))); } + // Trace SQL statements if tracing is enabled with -debug=walletdb -loglevel=walletdb:trace + if (LogAcceptCategory(BCLog::WALLETDB, BCLog::Level::Trace)) { + ret = sqlite3_trace_v2(m_db, SQLITE_TRACE_STMT, TraceSqlCallback, this); + if (ret != SQLITE_OK) { + LogPrintf("Failed to enable SQL tracing for %s\n", Filename()); + } + } } if (sqlite3_db_readonly(m_db, "main") != 0) { @@ -407,9 +440,8 @@ bool SQLiteBatch::ReadKey(DataStream&& key, DataStream& value) return false; } // Leftmost column in result is index 0 - const std::byte* data{AsBytePtr(sqlite3_column_blob(m_read_stmt, 0))}; - size_t data_size(sqlite3_column_bytes(m_read_stmt, 0)); - value.write({data, data_size}); + value.clear(); + value.write(SpanFromBlob(m_read_stmt, 0)); sqlite3_clear_bindings(m_read_stmt); sqlite3_reset(m_read_stmt); @@ -495,18 +527,18 @@ DatabaseCursor::Status SQLiteCursor::Next(DataStream& key, DataStream& value) return Status::FAIL; } + key.clear(); + value.clear(); + // Leftmost column in result is index 0 - const std::byte* key_data{AsBytePtr(sqlite3_column_blob(m_cursor_stmt, 0))}; - size_t key_data_size(sqlite3_column_bytes(m_cursor_stmt, 0)); - key.write({key_data, key_data_size}); - const std::byte* value_data{AsBytePtr(sqlite3_column_blob(m_cursor_stmt, 1))}; - size_t value_data_size(sqlite3_column_bytes(m_cursor_stmt, 1)); - value.write({value_data, value_data_size}); + key.write(SpanFromBlob(m_cursor_stmt, 0)); + value.write(SpanFromBlob(m_cursor_stmt, 1)); return Status::MORE; } SQLiteCursor::~SQLiteCursor() { + sqlite3_clear_bindings(m_cursor_stmt); sqlite3_reset(m_cursor_stmt); int res = sqlite3_finalize(m_cursor_stmt); if (res != SQLITE_OK) { @@ -530,6 +562,48 @@ std::unique_ptr<DatabaseCursor> SQLiteBatch::GetNewCursor() return cursor; } +std::unique_ptr<DatabaseCursor> SQLiteBatch::GetNewPrefixCursor(Span<const std::byte> prefix) +{ + if (!m_database.m_db) return nullptr; + + // To get just the records we want, the SQL statement does a comparison of the binary data + // where the data must be greater than or equal to the prefix, and less than + // the prefix incremented by one (when interpreted as an integer) + std::vector<std::byte> start_range(prefix.begin(), prefix.end()); + std::vector<std::byte> end_range(prefix.begin(), prefix.end()); + auto it = end_range.rbegin(); + for (; it != end_range.rend(); ++it) { + if (*it == std::byte(std::numeric_limits<unsigned char>::max())) { + *it = std::byte(0); + continue; + } + *it = std::byte(std::to_integer<unsigned char>(*it) + 1); + break; + } + if (it == end_range.rend()) { + // If the prefix is all 0xff bytes, clear end_range as we won't need it + end_range.clear(); + } + + auto cursor = std::make_unique<SQLiteCursor>(start_range, end_range); + if (!cursor) return nullptr; + + const char* stmt_text = end_range.empty() ? "SELECT key, value FROM main WHERE key >= ?" : + "SELECT key, value FROM main WHERE key >= ? AND key < ?"; + int res = sqlite3_prepare_v2(m_database.m_db, stmt_text, -1, &cursor->m_cursor_stmt, nullptr); + if (res != SQLITE_OK) { + throw std::runtime_error(strprintf( + "SQLiteDatabase: Failed to setup cursor SQL statement: %s\n", sqlite3_errstr(res))); + } + + if (!BindBlobToStatement(cursor->m_cursor_stmt, 1, cursor->m_prefix_range_start, "prefix_start")) return nullptr; + if (!end_range.empty()) { + if (!BindBlobToStatement(cursor->m_cursor_stmt, 2, cursor->m_prefix_range_end, "prefix_end")) return nullptr; + } + + return cursor; +} + bool SQLiteBatch::TxnBegin() { if (!m_database.m_db || sqlite3_get_autocommit(m_database.m_db) == 0) return false; diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index d9de40569b..0378bbb8d6 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -15,12 +15,21 @@ struct bilingual_str; namespace wallet { class SQLiteDatabase; +/** RAII class that provides a database cursor */ class SQLiteCursor : public DatabaseCursor { public: sqlite3_stmt* m_cursor_stmt{nullptr}; + // Copies of the prefix things for the prefix cursor. + // Prevents SQLite from accessing temp variables for the prefix things. + std::vector<std::byte> m_prefix_range_start; + std::vector<std::byte> m_prefix_range_end; explicit SQLiteCursor() {} + explicit SQLiteCursor(std::vector<std::byte> start_range, std::vector<std::byte> end_range) + : m_prefix_range_start(std::move(start_range)), + m_prefix_range_end(std::move(end_range)) + {} ~SQLiteCursor() override; Status Next(DataStream& key, DataStream& value) override; @@ -57,6 +66,7 @@ public: void Close() override; std::unique_ptr<DatabaseCursor> GetNewCursor() override; + std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override; bool TxnBegin() override; bool TxnCommit() override; bool TxnAbort() override; diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index a5394d8816..c8283f453a 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -432,7 +432,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) CAmount selection_target = 16 * CENT; const auto& no_res = SelectCoinsBnB(GroupCoins(available_coins.All(), /*subtract_fee_outputs*/true), selection_target, /*cost_of_change=*/0, MAX_STANDARD_TX_WEIGHT); - BOOST_ASSERT(!no_res); + BOOST_REQUIRE(!no_res); BOOST_CHECK(util::ErrorString(no_res).original.find("The inputs size exceeds the maximum weight") != std::string::npos); // Now add same coin value with a good size and check that it gets selected @@ -968,7 +968,7 @@ static util::Result<SelectionResult> SelectCoinsSRD(const CAmount& target, std::unique_ptr<CWallet> wallet = NewWallet(m_node); CoinEligibilityFilter filter(0, 0, 0); // accept all coins without ancestors Groups group = GroupOutputs(*wallet, coin_setup(*wallet), cs_params, {{filter}})[filter].all_groups; - return SelectCoinsSRD(group.positive_group, target, cs_params.rng_fast, max_weight); + return SelectCoinsSRD(group.positive_group, target, cs_params.m_change_fee, cs_params.rng_fast, max_weight); } BOOST_AUTO_TEST_CASE(srd_tests) diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 7761308bbc..d341e84d9b 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -5,14 +5,58 @@ #include <boost/test/unit_test.hpp> #include <test/util/setup_common.h> +#include <util/check.h> #include <util/fs.h> +#include <util/translation.h> +#ifdef USE_BDB #include <wallet/bdb.h> +#endif +#ifdef USE_SQLITE +#include <wallet/sqlite.h> +#endif +#include <wallet/test/util.h> +#include <wallet/walletutil.h> // for WALLET_FLAG_DESCRIPTORS #include <fstream> #include <memory> #include <string> +inline std::ostream& operator<<(std::ostream& os, const std::pair<const SerializeData, SerializeData>& kv) +{ + Span key{kv.first}, value{kv.second}; + os << "(\"" << std::string_view{reinterpret_cast<const char*>(key.data()), key.size()} << "\", \"" + << std::string_view{reinterpret_cast<const char*>(key.data()), key.size()} << "\")"; + return os; +} + namespace wallet { + +static Span<const std::byte> StringBytes(std::string_view str) +{ + return AsBytes<const char>({str.data(), str.size()}); +} + +static SerializeData StringData(std::string_view str) +{ + auto bytes = StringBytes(str); + return SerializeData{bytes.begin(), bytes.end()}; +} + +static void CheckPrefix(DatabaseBatch& batch, Span<const std::byte> prefix, MockableData expected) +{ + std::unique_ptr<DatabaseCursor> cursor = batch.GetNewPrefixCursor(prefix); + MockableData actual; + while (true) { + DataStream key, value; + DatabaseCursor::Status status = cursor->Next(key, value); + if (status == DatabaseCursor::Status::DONE) break; + BOOST_CHECK(status == DatabaseCursor::Status::MORE); + BOOST_CHECK( + actual.emplace(SerializeData(key.begin(), key.end()), SerializeData(value.begin(), value.end())).second); + } + BOOST_CHECK_EQUAL_COLLECTIONS(actual.begin(), actual.end(), expected.begin(), expected.end()); +} + BOOST_FIXTURE_TEST_SUITE(db_tests, BasicTestingSetup) static std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& path, fs::path& database_filename) @@ -78,5 +122,88 @@ BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_free_instance) BOOST_CHECK(env_2_a == env_2_b); } +static std::vector<std::unique_ptr<WalletDatabase>> TestDatabases(const fs::path& path_root) +{ + std::vector<std::unique_ptr<WalletDatabase>> dbs; + DatabaseOptions options; + DatabaseStatus status; + bilingual_str error; +#ifdef USE_BDB + dbs.emplace_back(MakeBerkeleyDatabase(path_root / "bdb", options, status, error)); +#endif +#ifdef USE_SQLITE + dbs.emplace_back(MakeSQLiteDatabase(path_root / "sqlite", options, status, error)); +#endif + dbs.emplace_back(CreateMockableWalletDatabase()); + return dbs; +} + +BOOST_AUTO_TEST_CASE(db_cursor_prefix_range_test) +{ + // Test each supported db + for (const auto& database : TestDatabases(m_path_root)) { + std::vector<std::string> prefixes = {"", "FIRST", "SECOND", "P\xfe\xff", "P\xff\x01", "\xff\xff"}; + + // Write elements to it + std::unique_ptr<DatabaseBatch> handler = Assert(database)->MakeBatch(); + for (unsigned int i = 0; i < 10; i++) { + for (const auto& prefix : prefixes) { + BOOST_CHECK(handler->Write(std::make_pair(prefix, i), i)); + } + } + + // Now read all the items by prefix and verify that each element gets parsed correctly + for (const auto& prefix : prefixes) { + DataStream s_prefix; + s_prefix << prefix; + std::unique_ptr<DatabaseCursor> cursor = handler->GetNewPrefixCursor(s_prefix); + DataStream key; + DataStream value; + for (int i = 0; i < 10; i++) { + DatabaseCursor::Status status = cursor->Next(key, value); + BOOST_CHECK_EQUAL(status, DatabaseCursor::Status::MORE); + + std::string key_back; + unsigned int i_back; + key >> key_back >> i_back; + BOOST_CHECK_EQUAL(key_back, prefix); + + unsigned int value_back; + value >> value_back; + BOOST_CHECK_EQUAL(value_back, i_back); + } + + // Let's now read it once more, it should return DONE + BOOST_CHECK(cursor->Next(key, value) == DatabaseCursor::Status::DONE); + } + } +} + +// Lower level DatabaseBase::GetNewPrefixCursor test, to cover cases that aren't +// covered in the higher level test above. The higher level test uses +// serialized strings which are prefixed with string length, so it doesn't test +// truly empty prefixes or prefixes that begin with \xff +BOOST_AUTO_TEST_CASE(db_cursor_prefix_byte_test) +{ + const MockableData::value_type + e{StringData(""), StringData("e")}, + p{StringData("prefix"), StringData("p")}, + ps{StringData("prefixsuffix"), StringData("ps")}, + f{StringData("\xff"), StringData("f")}, + fs{StringData("\xffsuffix"), StringData("fs")}, + ff{StringData("\xff\xff"), StringData("ff")}, + ffs{StringData("\xff\xffsuffix"), StringData("ffs")}; + for (const auto& database : TestDatabases(m_path_root)) { + std::unique_ptr<DatabaseBatch> batch = database->MakeBatch(); + for (const auto& [k, v] : {e, p, ps, f, fs, ff, ffs}) { + batch->Write(Span{k}, Span{v}); + } + CheckPrefix(*batch, StringBytes(""), {e, p, ps, f, fs, ff, ffs}); + CheckPrefix(*batch, StringBytes("prefix"), {p, ps}); + CheckPrefix(*batch, StringBytes("\xff"), {f, fs, ff, ffs}); + CheckPrefix(*batch, StringBytes("\xff\xff"), {ff, ffs}); + } +} + BOOST_AUTO_TEST_SUITE_END() } // namespace wallet diff --git a/src/wallet/test/fuzz/coincontrol.cpp b/src/wallet/test/fuzz/coincontrol.cpp new file mode 100644 index 0000000000..7dabdfb472 --- /dev/null +++ b/src/wallet/test/fuzz/coincontrol.cpp @@ -0,0 +1,87 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <wallet/coincontrol.h> +#include <wallet/test/util.h> + +namespace wallet { +namespace { + +const TestingSetup* g_setup; + +void initialize_coincontrol() +{ + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); +} + +FUZZ_TARGET_INIT(coincontrol, initialize_coincontrol) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const auto& node = g_setup->m_node; + ArgsManager& args = *node.args; + + // for GetBoolArg to return true sometimes + args.ForceSetArg("-avoidpartialspends", fuzzed_data_provider.ConsumeBool()?"1":"0"); + + CCoinControl coin_control; + COutPoint out_point; + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + CallOneOf( + fuzzed_data_provider, + [&] { + std::optional<COutPoint> optional_out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + if (!optional_out_point) { + return; + } + out_point = *optional_out_point; + }, + [&] { + (void)coin_control.HasSelected(); + }, + [&] { + (void)coin_control.IsSelected(out_point); + }, + [&] { + (void)coin_control.IsExternalSelected(out_point); + }, + [&] { + (void)coin_control.GetExternalOutput(out_point); + }, + [&] { + (void)coin_control.Select(out_point); + }, + [&] { + const CTxOut tx_out{ConsumeMoney(fuzzed_data_provider), ConsumeScript(fuzzed_data_provider)}; + (void)coin_control.SelectExternal(out_point, tx_out); + }, + [&] { + (void)coin_control.UnSelect(out_point); + }, + [&] { + (void)coin_control.UnSelectAll(); + }, + [&] { + (void)coin_control.ListSelected(); + }, + [&] { + int64_t weight{fuzzed_data_provider.ConsumeIntegral<int64_t>()}; + (void)coin_control.SetInputWeight(out_point, weight); + }, + [&] { + // Condition to avoid the assertion in GetInputWeight + if (coin_control.HasInputWeight(out_point)) { + (void)coin_control.GetInputWeight(out_point); + } + }); + } +} +} // namespace +} // namespace wallet diff --git a/src/wallet/test/fuzz/coinselection.cpp b/src/wallet/test/fuzz/coinselection.cpp index 9be8efab62..bc935157b1 100644 --- a/src/wallet/test/fuzz/coinselection.cpp +++ b/src/wallet/test/fuzz/coinselection.cpp @@ -90,8 +90,11 @@ FUZZ_TARGET(coinselection) // Run coinselection algorithms const auto result_bnb = SelectCoinsBnB(group_pos, target, cost_of_change, MAX_STANDARD_TX_WEIGHT); - auto result_srd = SelectCoinsSRD(group_pos, target, fast_random_context, MAX_STANDARD_TX_WEIGHT); - if (result_srd) result_srd->ComputeAndSetWaste(cost_of_change, cost_of_change, 0); + auto result_srd = SelectCoinsSRD(group_pos, target, coin_params.m_change_fee, fast_random_context, MAX_STANDARD_TX_WEIGHT); + if (result_srd) { + assert(result_srd->GetChange(CHANGE_LOWER, coin_params.m_change_fee) > 0); // Demonstrate that SRD creates change of at least CHANGE_LOWER + result_srd->ComputeAndSetWaste(cost_of_change, cost_of_change, 0); + } CAmount change_target{GenerateChangeTarget(target, coin_params.m_change_fee, fast_random_context)}; auto result_knapsack = KnapsackSolver(group_all, target, change_target, fast_random_context, MAX_STANDARD_TX_WEIGHT); diff --git a/src/wallet/test/fuzz/fees.cpp b/src/wallet/test/fuzz/fees.cpp new file mode 100644 index 0000000000..24e1098941 --- /dev/null +++ b/src/wallet/test/fuzz/fees.cpp @@ -0,0 +1,68 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <wallet/coincontrol.h> +#include <wallet/fees.h> +#include <wallet/wallet.h> +#include <wallet/test/util.h> +#include <validation.h> + +namespace wallet { +namespace { +const TestingSetup* g_setup; +static std::unique_ptr<CWallet> g_wallet_ptr; + +void initialize_setup() +{ + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); + const auto& node{g_setup->m_node}; + g_wallet_ptr = std::make_unique<CWallet>(node.chain.get(), "", CreateMockableWalletDatabase()); +} + +FUZZ_TARGET_INIT(wallet_fees, initialize_setup) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + const auto& node{g_setup->m_node}; + Chainstate* chainstate = &node.chainman->ActiveChainstate(); + CWallet& wallet = *g_wallet_ptr; + { + LOCK(wallet.cs_wallet); + wallet.SetLastBlockProcessed(chainstate->m_chain.Height(), chainstate->m_chain.Tip()->GetBlockHash()); + } + + if (fuzzed_data_provider.ConsumeBool()) { + wallet.m_discard_rate = CFeeRate{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; + } + (void)GetDiscardRate(wallet); + + const auto tx_bytes{fuzzed_data_provider.ConsumeIntegral<unsigned int>()}; + + if (fuzzed_data_provider.ConsumeBool()) { + wallet.m_pay_tx_fee = CFeeRate{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; + wallet.m_min_fee = CFeeRate{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; + } + + (void)GetRequiredFee(wallet, tx_bytes); + (void)GetRequiredFeeRate(wallet); + + CCoinControl coin_control; + if (fuzzed_data_provider.ConsumeBool()) { + coin_control.m_feerate = CFeeRate{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; + } + if (fuzzed_data_provider.ConsumeBool()) { + coin_control.m_confirm_target = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, 999'000); + } + + FeeCalculation fee_calculation; + FeeCalculation* maybe_fee_calculation{fuzzed_data_provider.ConsumeBool() ? nullptr : &fee_calculation}; + (void)GetMinimumFeeRate(wallet, coin_control, maybe_fee_calculation); + (void)GetMinimumFee(wallet, tx_bytes, coin_control, maybe_fee_calculation); +} +} // namespace +} // namespace wallet diff --git a/src/wallet/test/fuzz/notifications.cpp b/src/wallet/test/fuzz/notifications.cpp index de381a5ec9..f4b69f7403 100644 --- a/src/wallet/test/fuzz/notifications.cpp +++ b/src/wallet/test/fuzz/notifications.cpp @@ -141,6 +141,10 @@ FUZZ_TARGET_INIT(wallet_notifications, initialize_setup) info.prev_hash = &block.hashPrevBlock; info.height = chain.size(); info.data = █ + // Ensure that no blocks are skipped by the wallet by setting the chain's accumulated + // time to the maximum value. This ensures that the wallet's birth time is always + // earlier than this maximum time. + info.chain_time_max = std::numeric_limits<unsigned int>::max(); a.wallet->blockConnected(info); b.wallet->blockConnected(info); // Store the coins for the next block diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index 09e7979c02..069ab25f26 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -9,6 +9,7 @@ #include <key_io.h> #include <streams.h> #include <test/util/setup_common.h> +#include <wallet/context.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> @@ -45,6 +46,36 @@ std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cc return wallet; } +std::shared_ptr<CWallet> TestLoadWallet(std::unique_ptr<WalletDatabase> database, WalletContext& context, uint64_t create_flags) +{ + bilingual_str error; + std::vector<bilingual_str> warnings; + auto wallet = CWallet::Create(context, "", std::move(database), create_flags, error, warnings); + NotifyWalletLoaded(context, wallet); + if (context.chain) { + wallet->postInitProcess(); + } + return wallet; +} + +std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) +{ + DatabaseOptions options; + options.create_flags = WALLET_FLAG_DESCRIPTORS; + DatabaseStatus status; + bilingual_str error; + std::vector<bilingual_str> warnings; + auto database = MakeWalletDatabase("", options, status, error); + return TestLoadWallet(std::move(database), context, options.create_flags); +} + +void TestUnloadWallet(std::shared_ptr<CWallet>&& wallet) +{ + SyncWithValidationInterfaceQueue(); + wallet->m_chain_notifications_handler.reset(); + UnloadWallet(std::move(wallet)); +} + std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database) { return std::make_unique<MockableDatabase>(dynamic_cast<MockableDatabase&>(database).m_records); @@ -61,6 +92,17 @@ CTxDestination getNewDestination(CWallet& w, OutputType output_type) return *Assert(w.GetNewDestination(output_type, "")); } +// BytePrefix compares equality with other byte spans that begin with the same prefix. +struct BytePrefix { Span<const std::byte> prefix; }; +bool operator<(BytePrefix a, Span<const std::byte> b) { return a.prefix < b.subspan(0, std::min(a.prefix.size(), b.size())); } +bool operator<(Span<const std::byte> a, BytePrefix b) { return a.subspan(0, std::min(a.size(), b.prefix.size())) < b.prefix; } + +MockableCursor::MockableCursor(const MockableData& records, bool pass, Span<const std::byte> prefix) +{ + m_pass = pass; + std::tie(m_cursor, m_cursor_end) = records.equal_range(BytePrefix{prefix}); +} + DatabaseCursor::Status MockableCursor::Next(DataStream& key, DataStream& value) { if (!m_pass) { @@ -69,6 +111,8 @@ DatabaseCursor::Status MockableCursor::Next(DataStream& key, DataStream& value) if (m_cursor == m_cursor_end) { return Status::DONE; } + key.clear(); + value.clear(); const auto& [key_data, value_data] = *m_cursor; key.write(key_data); value.write(value_data); @@ -86,6 +130,7 @@ bool MockableBatch::ReadKey(DataStream&& key, DataStream& value) if (it == m_records.end()) { return false; } + value.clear(); value.write(it->second); return true; } @@ -141,7 +186,7 @@ bool MockableBatch::ErasePrefix(Span<const std::byte> prefix) return true; } -std::unique_ptr<WalletDatabase> CreateMockableWalletDatabase(std::map<SerializeData, SerializeData> records) +std::unique_ptr<WalletDatabase> CreateMockableWalletDatabase(MockableData records) { return std::make_unique<MockableDatabase>(records); } diff --git a/src/wallet/test/util.h b/src/wallet/test/util.h index eb1cfd9e21..2a1fe639de 100644 --- a/src/wallet/test/util.h +++ b/src/wallet/test/util.h @@ -21,6 +21,7 @@ class Chain; namespace wallet { class CWallet; class WalletDatabase; +struct WalletContext; static const DatabaseFormat DATABASE_FORMATS[] = { #ifdef USE_SQLITE @@ -31,8 +32,14 @@ static const DatabaseFormat DATABASE_FORMATS[] = { #endif }; +const std::string ADDRESS_BCRT1_UNSPENDABLE = "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj"; + std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, const CKey& key); +std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context); +std::shared_ptr<CWallet> TestLoadWallet(std::unique_ptr<WalletDatabase> database, WalletContext& context, uint64_t create_flags); +void TestUnloadWallet(std::shared_ptr<CWallet>&& wallet); + // Creates a copy of the provided database std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database); @@ -41,14 +48,17 @@ std::string getnewaddress(CWallet& w); /** Returns a new destination, of an specific type, from the wallet */ CTxDestination getNewDestination(CWallet& w, OutputType output_type); +using MockableData = std::map<SerializeData, SerializeData, std::less<>>; + class MockableCursor: public DatabaseCursor { public: - std::map<SerializeData, SerializeData>::const_iterator m_cursor; - std::map<SerializeData, SerializeData>::const_iterator m_cursor_end; + MockableData::const_iterator m_cursor; + MockableData::const_iterator m_cursor_end; bool m_pass; - explicit MockableCursor(const std::map<SerializeData, SerializeData>& records, bool pass) : m_cursor(records.begin()), m_cursor_end(records.end()), m_pass(pass) {} + explicit MockableCursor(const MockableData& records, bool pass) : m_cursor(records.begin()), m_cursor_end(records.end()), m_pass(pass) {} + MockableCursor(const MockableData& records, bool pass, Span<const std::byte> prefix); ~MockableCursor() {} Status Next(DataStream& key, DataStream& value) override; @@ -57,7 +67,7 @@ public: class MockableBatch : public DatabaseBatch { private: - std::map<SerializeData, SerializeData>& m_records; + MockableData& m_records; bool m_pass; bool ReadKey(DataStream&& key, DataStream& value) override; @@ -67,7 +77,7 @@ private: bool ErasePrefix(Span<const std::byte> prefix) override; public: - explicit MockableBatch(std::map<SerializeData, SerializeData>& records, bool pass) : m_records(records), m_pass(pass) {} + explicit MockableBatch(MockableData& records, bool pass) : m_records(records), m_pass(pass) {} ~MockableBatch() {} void Flush() override {} @@ -77,6 +87,9 @@ public: { return std::make_unique<MockableCursor>(m_records, m_pass); } + std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override { + return std::make_unique<MockableCursor>(m_records, m_pass, prefix); + } bool TxnBegin() override { return m_pass; } bool TxnCommit() override { return m_pass; } bool TxnAbort() override { return m_pass; } @@ -87,10 +100,10 @@ public: class MockableDatabase : public WalletDatabase { public: - std::map<SerializeData, SerializeData> m_records; + MockableData m_records; bool m_pass{true}; - MockableDatabase(std::map<SerializeData, SerializeData> records = {}) : WalletDatabase(), m_records(records) {} + MockableDatabase(MockableData records = {}) : WalletDatabase(), m_records(records) {} ~MockableDatabase() {}; void Open() override {} @@ -110,7 +123,7 @@ public: std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override { return std::make_unique<MockableBatch>(m_records, m_pass); } }; -std::unique_ptr<WalletDatabase> CreateMockableWalletDatabase(std::map<SerializeData, SerializeData> records = {}); +std::unique_ptr<WalletDatabase> CreateMockableWalletDatabase(MockableData records = {}); MockableDatabase& GetMockableDatabase(CWallet& wallet); } // namespace wallet diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 194c8663db..65b02c267d 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -42,26 +42,6 @@ static_assert(WALLET_INCREMENTAL_RELAY_FEE >= DEFAULT_INCREMENTAL_RELAY_FEE, "wa BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) -static std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) -{ - DatabaseOptions options; - options.create_flags = WALLET_FLAG_DESCRIPTORS; - DatabaseStatus status; - bilingual_str error; - std::vector<bilingual_str> warnings; - auto database = MakeWalletDatabase("", options, status, error); - auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); - NotifyWalletLoaded(context, wallet); - return wallet; -} - -static void TestUnloadWallet(std::shared_ptr<CWallet>&& wallet) -{ - SyncWithValidationInterfaceQueue(); - wallet->m_chain_notifications_handler.reset(); - UnloadWallet(std::move(wallet)); -} - static CMutableTransaction TestSimpleSpend(const CTransaction& from, uint32_t index, const CKey& key, const CScript& pubkey) { CMutableTransaction mtx; @@ -845,10 +825,11 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // Reload wallet and make sure new transactions are detected despite events // being blocked + // Loading will also ask for current mempool transactions wallet = TestLoadWallet(context); BOOST_CHECK(rescan_completed); - // AddToWallet events for block_tx and mempool_tx - BOOST_CHECK_EQUAL(addtx_count, 2); + // AddToWallet events for block_tx and mempool_tx (x2) + BOOST_CHECK_EQUAL(addtx_count, 3); { LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); @@ -862,7 +843,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) SyncWithValidationInterfaceQueue(); // AddToWallet events for block_tx and mempool_tx events are counted a // second time as the notification queue is processed - BOOST_CHECK_EQUAL(addtx_count, 4); + BOOST_CHECK_EQUAL(addtx_count, 5); TestUnloadWallet(std::move(wallet)); @@ -885,7 +866,9 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) SyncWithValidationInterfaceQueue(); }); wallet = TestLoadWallet(context); - BOOST_CHECK_EQUAL(addtx_count, 2); + // Since mempool transactions are requested at the end of loading, there will + // be 2 additional AddToWallet calls, one from the previous test, and a duplicate for mempool_tx + BOOST_CHECK_EQUAL(addtx_count, 2 + 2); { LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); @@ -955,11 +938,10 @@ BOOST_FIXTURE_TEST_CASE(wallet_sync_tx_invalid_state_test, TestingSetup) } // Add tx to wallet - const auto& op_dest = wallet.GetNewDestination(OutputType::BECH32M, ""); - BOOST_ASSERT(op_dest); + const auto op_dest{*Assert(wallet.GetNewDestination(OutputType::BECH32M, ""))}; CMutableTransaction mtx; - mtx.vout.push_back({COIN, GetScriptForDestination(*op_dest)}); + mtx.vout.push_back({COIN, GetScriptForDestination(op_dest)}); mtx.vin.push_back(CTxIn(g_insecure_rand_ctx.rand256(), 0)); const auto& tx_id_to_spend = wallet.AddToWallet(MakeTransactionRef(mtx), TxStateInMempool{})->GetHash(); diff --git a/src/wallet/test/walletdb_tests.cpp b/src/wallet/test/walletdb_tests.cpp index 00b2f47e14..17b6c4f7ed 100644 --- a/src/wallet/test/walletdb_tests.cpp +++ b/src/wallet/test/walletdb_tests.cpp @@ -39,7 +39,7 @@ BOOST_AUTO_TEST_CASE(walletdb_read_write_deadlock) DatabaseStatus status; bilingual_str error_string; std::unique_ptr<WalletDatabase> db = MakeDatabase(m_path_root / strprintf("wallet_%d_.dat", db_format).c_str(), options, status, error_string); - BOOST_ASSERT(status == DatabaseStatus::SUCCESS); + BOOST_CHECK_EQUAL(status, DatabaseStatus::SUCCESS); std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", std::move(db))); wallet->m_keypool_size = 4; diff --git a/src/wallet/test/walletload_tests.cpp b/src/wallet/test/walletload_tests.cpp index 6823eafdfa..73a4b77188 100644 --- a/src/wallet/test/walletload_tests.cpp +++ b/src/wallet/test/walletload_tests.cpp @@ -83,7 +83,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_load_ckey, TestingSetup) { SerializeData ckey_record_key; SerializeData ckey_record_value; - std::map<SerializeData, SerializeData> records; + MockableData records; { // Context setup. @@ -169,7 +169,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_load_ckey, TestingSetup) // Fourth test case: // Verify that loading up a 'ckey' with an invalid pubkey throws an error CPubKey invalid_key; - BOOST_ASSERT(!invalid_key.IsValid()); + BOOST_CHECK(!invalid_key.IsValid()); SerializeData key = MakeSerializeData(DBKeys::CRYPTED_KEY, invalid_key); records[key] = ckey_record_value; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index b3eed8abc7..5d99fb92a2 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -42,6 +42,7 @@ #include <wallet/context.h> #include <wallet/external_signer_scriptpubkeyman.h> #include <wallet/fees.h> +#include <wallet/scriptpubkeyman.h> #include <univalue.h> @@ -55,9 +56,9 @@ namespace wallet { bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name) { - util::SettingsValue setting_value = chain.getRwSetting("wallet"); + common::SettingsValue setting_value = chain.getRwSetting("wallet"); if (!setting_value.isArray()) setting_value.setArray(); - for (const util::SettingsValue& value : setting_value.getValues()) { + for (const common::SettingsValue& value : setting_value.getValues()) { if (value.isStr() && value.get_str() == wallet_name) return true; } setting_value.push_back(wallet_name); @@ -66,10 +67,10 @@ bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name) bool RemoveWalletSetting(interfaces::Chain& chain, const std::string& wallet_name) { - util::SettingsValue setting_value = chain.getRwSetting("wallet"); + common::SettingsValue setting_value = chain.getRwSetting("wallet"); if (!setting_value.isArray()) return true; - util::SettingsValue new_value(util::SettingsValue::VARR); - for (const util::SettingsValue& value : setting_value.getValues()) { + common::SettingsValue new_value(common::SettingsValue::VARR); + for (const common::SettingsValue& value : setting_value.getValues()) { if (!value.isStr() || value.get_str() != wallet_name) new_value.push_back(value); } if (new_value.size() == setting_value.size()) return true; @@ -1266,11 +1267,6 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) { LOCK(cs_wallet); - WalletBatch batch(GetDatabase()); - - std::set<uint256> todo; - std::set<uint256> done; - // Can't mark abandoned if confirmed or in mempool auto it = mapWallet.find(hashTx); assert(it != mapWallet.end()); @@ -1279,44 +1275,25 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) return false; } - todo.insert(hashTx); - - while (!todo.empty()) { - uint256 now = *todo.begin(); - todo.erase(now); - done.insert(now); - auto it = mapWallet.find(now); - assert(it != mapWallet.end()); - CWalletTx& wtx = it->second; - int currentconfirm = GetTxDepthInMainChain(wtx); - // If the orig tx was not in block, none of its spends can be - assert(currentconfirm <= 0); - // if (currentconfirm < 0) {Tx and spends are already conflicted, no need to abandon} - if (currentconfirm == 0 && !wtx.isAbandoned()) { - // If the orig tx was not in block/mempool, none of its spends can be in mempool - assert(!wtx.InMempool()); + auto try_updating_state = [](CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { + // If the orig tx was not in block/mempool, none of its spends can be. + assert(!wtx.isConfirmed()); + assert(!wtx.InMempool()); + // If already conflicted or abandoned, no need to set abandoned + if (!wtx.isConflicted() && !wtx.isAbandoned()) { wtx.m_state = TxStateInactive{/*abandoned=*/true}; - wtx.MarkDirty(); - batch.WriteTx(wtx); - NotifyTransactionChanged(wtx.GetHash(), CT_UPDATED); - // Iterate over all its outputs, and mark transactions in the wallet that spend them abandoned too. - // States are not permanent, so these transactions can become unabandoned if they are re-added to the - // mempool, or confirmed in a block, or conflicted. - // Note: If the reorged coinbase is re-added to the main chain, the descendants that have not had their - // states change will remain abandoned and will require manual broadcast if the user wants them. - for (unsigned int i = 0; i < wtx.tx->vout.size(); ++i) { - std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(COutPoint(now, i)); - for (TxSpends::const_iterator iter = range.first; iter != range.second; ++iter) { - if (!done.count(iter->second)) { - todo.insert(iter->second); - } - } - } - // If a transaction changes 'conflicted' state, that changes the balance - // available of the outputs it spends. So force those to be recomputed - MarkInputsDirty(wtx.tx); + return TxUpdate::NOTIFY_CHANGED; } - } + return TxUpdate::UNCHANGED; + }; + + // Iterate over all its outputs, and mark transactions in the wallet that spend them abandoned too. + // States are not permanent, so these transactions can become unabandoned if they are re-added to the + // mempool, or confirmed in a block, or conflicted. + // Note: If the reorged coinbase is re-added to the main chain, the descendants that have not had their + // states change will remain abandoned and will require manual broadcast if the user wants them. + + RecursiveUpdateTxState(hashTx, try_updating_state); return true; } @@ -1333,13 +1310,29 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c if (conflictconfirms >= 0) return; + auto try_updating_state = [&](CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { + if (conflictconfirms < GetTxDepthInMainChain(wtx)) { + // Block is 'more conflicted' than current confirm; update. + // Mark transaction as conflicted with this block. + wtx.m_state = TxStateConflicted{hashBlock, conflicting_height}; + return TxUpdate::CHANGED; + } + return TxUpdate::UNCHANGED; + }; + + // Iterate over all its outputs, and mark transactions in the wallet that spend them conflicted too. + RecursiveUpdateTxState(hashTx, try_updating_state); + +} + +void CWallet::RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { // Do not flush the wallet here for performance reasons WalletBatch batch(GetDatabase(), false); std::set<uint256> todo; std::set<uint256> done; - todo.insert(hashTx); + todo.insert(tx_hash); while (!todo.empty()) { uint256 now = *todo.begin(); @@ -1348,14 +1341,12 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c auto it = mapWallet.find(now); assert(it != mapWallet.end()); CWalletTx& wtx = it->second; - int currentconfirm = GetTxDepthInMainChain(wtx); - if (conflictconfirms < currentconfirm) { - // Block is 'more conflicted' than current confirm; update. - // Mark transaction as conflicted with this block. - wtx.m_state = TxStateConflicted{hashBlock, conflicting_height}; + + TxUpdate update_state = try_updating_state(wtx); + if (update_state != TxUpdate::UNCHANGED) { wtx.MarkDirty(); batch.WriteTx(wtx); - // Iterate over all its outputs, and mark transactions in the wallet that spend them conflicted too + // Iterate over all its outputs, and update those tx states as well (if applicable) for (unsigned int i = 0; i < wtx.tx->vout.size(); ++i) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(COutPoint(now, i)); for (TxSpends::const_iterator iter = range.first; iter != range.second; ++iter) { @@ -1364,7 +1355,12 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c } } } - // If a transaction changes 'conflicted' state, that changes the balance + + if (update_state == TxUpdate::NOTIFY_CHANGED) { + NotifyTransactionChanged(wtx.GetHash(), CT_UPDATED); + } + + // If a transaction changes its tx state, that usually changes the balance // available of the outputs it spends. So force those to be recomputed MarkInputsDirty(wtx.tx); } @@ -1436,6 +1432,12 @@ void CWallet::blockConnected(const interfaces::BlockInfo& block) m_last_block_processed_height = block.height; m_last_block_processed = block.hash; + + // No need to scan block if it was created before the wallet birthday. + // Uses chain max time and twice the grace period to adjust time for block time variability. + if (block.chain_time_max < m_birth_time.load() - (TIMESTAMP_WINDOW * 2)) return; + + // Scan block for (size_t index = 0; index < block.data->vtx.size(); index++) { SyncTransaction(block.data->vtx[index], TxStateConfirmed{block.hash, block.height, static_cast<int>(index)}); transactionRemovedFromMempool(block.data->vtx[index], MemPoolRemovalReason::BLOCK); @@ -1453,8 +1455,36 @@ void CWallet::blockDisconnected(const interfaces::BlockInfo& block) // future with a stickier abandoned state or even removing abandontransaction call. m_last_block_processed_height = block.height - 1; m_last_block_processed = *Assert(block.prev_hash); + + int disconnect_height = block.height; + for (const CTransactionRef& ptx : Assert(block.data)->vtx) { SyncTransaction(ptx, TxStateInactive{}); + + for (const CTxIn& tx_in : ptx->vin) { + // No other wallet transactions conflicted with this transaction + if (mapTxSpends.count(tx_in.prevout) < 1) continue; + + std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(tx_in.prevout); + + // For all of the spends that conflict with this transaction + for (TxSpends::const_iterator _it = range.first; _it != range.second; ++_it) { + CWalletTx& wtx = mapWallet.find(_it->second)->second; + + if (!wtx.isConflicted()) continue; + + auto try_updating_state = [&](CWalletTx& tx) { + if (!tx.isConflicted()) return TxUpdate::UNCHANGED; + if (tx.state<TxStateConflicted>()->conflicting_block_height >= disconnect_height) { + tx.m_state = TxStateInactive{}; + return TxUpdate::CHANGED; + } + return TxUpdate::UNCHANGED; + }; + + RecursiveUpdateTxState(wtx.tx->GetHash(), try_updating_state); + } + } } } @@ -1779,6 +1809,14 @@ bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScri return true; } +void CWallet::FirstKeyTimeChanged(const ScriptPubKeyMan* spkm, int64_t new_birth_time) +{ + int64_t birthtime = m_birth_time.load(); + if (new_birth_time < birthtime) { + m_birth_time = new_birth_time; + } +} + /** * Scan active chain for relevant transactions after importing keys. This should * be called whenever new keys are added to the wallet, with the oldest key @@ -2794,7 +2832,7 @@ bool CWallet::SetAddressPreviouslySpent(WalletBatch& batch, const CTxDestination return false; if (!used) { - if (auto* data{util::FindKey(m_address_book, dest)}) data->previously_spent = false; + if (auto* data{common::FindKey(m_address_book, dest)}) data->previously_spent = false; return batch.WriteAddressPreviouslySpent(dest, false); } @@ -2814,7 +2852,7 @@ void CWallet::LoadAddressReceiveRequest(const CTxDestination& dest, const std::s bool CWallet::IsAddressPreviouslySpent(const CTxDestination& dest) const { - if (auto* data{util::FindKey(m_address_book, dest)}) return data->previously_spent; + if (auto* data{common::FindKey(m_address_book, dest)}) return data->previously_spent; return false; } @@ -2891,7 +2929,7 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri else if (nLoadWalletRet == DBErrors::NONCRITICAL_ERROR) { warnings.push_back(strprintf(_("Error reading %s! All keys read correctly, but transaction data" - " or address book entries might be missing or incorrect."), + " or address metadata may be missing or incorrect."), walletFile)); } else if (nLoadWalletRet == DBErrors::TOO_NEW) { @@ -3107,6 +3145,14 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri // Try to top up keypool. No-op if the wallet is locked. walletInstance->TopUpKeyPool(); + // Cache the first key time + std::optional<int64_t> time_first_key; + for (auto spk_man : walletInstance->GetAllScriptPubKeyMans()) { + int64_t time = spk_man->GetTimeFirstKey(); + if (!time_first_key || time < *time_first_key) time_first_key = time; + } + if (time_first_key) walletInstance->m_birth_time = *time_first_key; + if (chain && !AttachChain(walletInstance, *chain, rescan_required, error, warnings)) { return nullptr; } @@ -3182,11 +3228,7 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf { // No need to read and scan block if block was created before // our wallet birthday (as adjusted for block time variability) - std::optional<int64_t> time_first_key; - for (auto spk_man : walletInstance->GetAllScriptPubKeyMans()) { - int64_t time = spk_man->GetTimeFirstKey(); - if (!time_first_key || time < *time_first_key) time_first_key = time; - } + std::optional<int64_t> time_first_key = walletInstance->m_birth_time.load(); if (time_first_key) { FoundBlock found = FoundBlock().height(rescan_height); chain.findFirstBlockWithTimeAndHeight(*time_first_key - TIMESTAMP_WINDOW, rescan_height, found); @@ -3498,6 +3540,14 @@ LegacyScriptPubKeyMan* CWallet::GetOrCreateLegacyScriptPubKeyMan() return GetLegacyScriptPubKeyMan(); } +void CWallet::AddScriptPubKeyMan(const uint256& id, std::unique_ptr<ScriptPubKeyMan> spkm_man) +{ + const auto& spkm = m_spk_managers[id] = std::move(spkm_man); + + // Update birth time if needed + FirstKeyTimeChanged(spkm.get(), spkm->GetTimeFirstKey()); +} + void CWallet::SetupLegacyScriptPubKeyMan() { if (!m_internal_spk_managers.empty() || !m_external_spk_managers.empty() || !m_spk_managers.empty() || IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { @@ -3509,7 +3559,8 @@ void CWallet::SetupLegacyScriptPubKeyMan() m_internal_spk_managers[type] = spk_manager.get(); m_external_spk_managers[type] = spk_manager.get(); } - m_spk_managers[spk_manager->GetID()] = std::move(spk_manager); + uint256 id = spk_manager->GetID(); + AddScriptPubKeyMan(id, std::move(spk_manager)); } const CKeyingMaterial& CWallet::GetEncryptionKey() const @@ -3527,6 +3578,7 @@ void CWallet::ConnectScriptPubKeyManNotifiers() for (const auto& spk_man : GetActiveScriptPubKeyMans()) { spk_man->NotifyWatchonlyChanged.connect(NotifyWatchonlyChanged); spk_man->NotifyCanGetAddressesChanged.connect(NotifyCanGetAddressesChanged); + spk_man->NotifyFirstKeyTimeChanged.connect(std::bind(&CWallet::FirstKeyTimeChanged, this, std::placeholders::_1, std::placeholders::_2)); } } @@ -3534,10 +3586,10 @@ void CWallet::LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc) { if (IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new ExternalSignerScriptPubKeyMan(*this, desc, m_keypool_size)); - m_spk_managers[id] = std::move(spk_manager); + AddScriptPubKeyMan(id, std::move(spk_manager)); } else { auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, desc, m_keypool_size)); - m_spk_managers[id] = std::move(spk_manager); + AddScriptPubKeyMan(id, std::move(spk_manager)); } } @@ -3558,7 +3610,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans(const CExtKey& master_key) } spk_manager->SetupDescriptorGeneration(master_key, t, internal); uint256 id = spk_manager->GetID(); - m_spk_managers[id] = std::move(spk_manager); + AddScriptPubKeyMan(id, std::move(spk_manager)); AddActiveScriptPubKeyMan(id, t, internal); } } @@ -3606,7 +3658,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() auto spk_manager = std::unique_ptr<ExternalSignerScriptPubKeyMan>(new ExternalSignerScriptPubKeyMan(*this, m_keypool_size)); spk_manager->SetupDescriptor(std::move(desc)); uint256 id = spk_manager->GetID(); - m_spk_managers[id] = std::move(spk_manager); + AddScriptPubKeyMan(id, std::move(spk_manager)); AddActiveScriptPubKeyMan(id, t, internal); } } @@ -3723,7 +3775,8 @@ ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const Flat spk_man = new_spk_man.get(); // Save the descriptor to memory - m_spk_managers[new_spk_man->GetID()] = std::move(new_spk_man); + uint256 id = new_spk_man->GetID(); + AddScriptPubKeyMan(id, std::move(new_spk_man)); } // Add the private keys to the descriptor @@ -3803,16 +3856,19 @@ bool CWallet::MigrateToSQLite(bilingual_str& error) // Close this database and delete the file fs::path db_path = fs::PathFromString(m_database->Filename()); - fs::path db_dir = db_path.parent_path(); m_database->Close(); fs::remove(db_path); + // Generate the path for the location of the migrated wallet + // Wallets that are plain files rather than wallet directories will be migrated to be wallet directories. + const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), fs::PathFromString(m_name)); + // Make new DB DatabaseOptions opts; opts.require_create = true; opts.require_format = DatabaseFormat::SQLITE; DatabaseStatus db_status; - std::unique_ptr<WalletDatabase> new_db = MakeDatabase(db_dir, opts, db_status, error); + std::unique_ptr<WalletDatabase> new_db = MakeDatabase(wallet_path, opts, db_status, error); assert(new_db); // This is to prevent doing anything further with this wallet. The original file was deleted, but a backup exists. m_database.reset(); m_database = std::move(new_db); @@ -3822,9 +3878,7 @@ bool CWallet::MigrateToSQLite(bilingual_str& error) bool began = batch->TxnBegin(); assert(began); // This is a critical error, the new db could not be written to. The original db exists as a backup, but we should not continue execution. for (const auto& [key, value] : records) { - DataStream ss_key{key}; - DataStream ss_value{value}; - if (!batch->Write(ss_key, ss_value)) { + if (!batch->Write(Span{key}, Span{value})) { batch->TxnAbort(); m_database->Close(); fs::remove(m_database->Filename()); @@ -3866,7 +3920,8 @@ bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error) error = _("Error: Duplicate descriptors created during migration. Your wallet may be corrupted."); return false; } - m_spk_managers[desc_spkm->GetID()] = std::move(desc_spkm); + uint256 id = desc_spkm->GetID(); + AddScriptPubKeyMan(id, std::move(desc_spkm)); } // Remove the LegacyScriptPubKeyMan from disk diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 79f4c43456..cbd5008366 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -299,6 +299,10 @@ private: // Local time that the tip block was received. Used to schedule wallet rebroadcasts. std::atomic<int64_t> m_best_block_time {0}; + // First created key time. Used to skip blocks prior to this time. + // 'std::numeric_limits<int64_t>::max()' if wallet is blank. + std::atomic<int64_t> m_birth_time{std::numeric_limits<int64_t>::max()}; + /** * Used to keep track of spent outpoints, and * detect and report conflicts (double-spends or @@ -330,6 +334,13 @@ private: /** Mark a transaction (and its in-wallet descendants) as conflicting with a particular block. */ void MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx); + enum class TxUpdate { UNCHANGED, CHANGED, NOTIFY_CHANGED }; + + using TryUpdatingStateFn = std::function<TxUpdate(CWalletTx& wtx)>; + + /** Mark a transaction (and its in-wallet descendants) as a particular tx state. */ + void RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** Mark a transaction's inputs dirty, thus forcing the outputs to be recomputed */ void MarkInputsDirty(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -380,6 +391,10 @@ private: // ScriptPubKeyMan::GetID. In many cases it will be the hash of an internal structure std::map<uint256, std::unique_ptr<ScriptPubKeyMan>> m_spk_managers; + // Appends spk managers into the main 'm_spk_managers'. + // Must be the only method adding data to it. + void AddScriptPubKeyMan(const uint256& id, std::unique_ptr<ScriptPubKeyMan> spkm_man); + /** * Catch wallet up to current chain, scanning new blocks, updating the best * block locator and m_last_block_processed, and registering for @@ -641,6 +656,9 @@ public: bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool ImportScriptPubKeys(const std::string& label, const std::set<CScript>& script_pub_keys, const bool have_solving_data, const bool apply_label, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** Updates wallet birth time if 'new_birth_time' is below it */ + void FirstKeyTimeChanged(const ScriptPubKeyMan* spkm, int64_t new_birth_time); + CFeeRate m_pay_tx_fee{DEFAULT_PAY_TX_FEE}; unsigned int m_confirm_target{DEFAULT_TX_CONFIRM_TARGET}; /** Allow Coin Selection to pick unconfirmed UTXOs that were sent from our own wallet if it @@ -1056,7 +1074,7 @@ struct MigrationResult { }; //! Do all steps to migrate a legacy wallet to a descriptor wallet -util::Result<MigrationResult> MigrateLegacyToDescriptor(const std::string& wallet_name, const SecureString& passphrase, WalletContext& context); +[[nodiscard]] util::Result<MigrationResult> MigrateLegacyToDescriptor(const std::string& wallet_name, const SecureString& passphrase, WalletContext& context); } // namespace wallet #endif // BITCOIN_WALLET_WALLET_H diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index ddabdac99f..ff0b83dc38 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -5,13 +5,14 @@ #include <wallet/walletdb.h> +#include <common/system.h> #include <key_io.h> #include <protocol.h> #include <serialize.h> #include <sync.h> #include <util/bip32.h> +#include <util/check.h> #include <util/fs.h> -#include <util/system.h> #include <util/time.h> #include <util/translation.h> #ifdef USE_BDB @@ -297,426 +298,590 @@ bool WalletBatch::EraseLockedUTXO(const COutPoint& output) return EraseIC(std::make_pair(DBKeys::LOCKED_UTXO, std::make_pair(output.hash, output.n))); } -class CWalletScanState { -public: - unsigned int nKeys{0}; - unsigned int nCKeys{0}; - unsigned int nWatchKeys{0}; - unsigned int nKeyMeta{0}; - unsigned int m_unknown_records{0}; - bool fIsEncrypted{false}; - bool fAnyUnordered{false}; - std::vector<uint256> vWalletUpgrade; - std::map<OutputType, uint256> m_active_external_spks; - std::map<OutputType, uint256> m_active_internal_spks; - std::map<uint256, DescriptorCache> m_descriptor_caches; - std::map<std::pair<uint256, CKeyID>, CKey> m_descriptor_keys; - std::map<std::pair<uint256, CKeyID>, std::pair<CPubKey, std::vector<unsigned char>>> m_descriptor_crypt_keys; - std::map<uint160, CHDChain> m_hd_chains; - bool tx_corrupt{false}; - bool descriptor_unknown{false}; - bool unexpected_legacy_entry{false}; - - CWalletScanState() = default; -}; - -static bool -ReadKeyValue(CWallet* pwallet, DataStream& ssKey, CDataStream& ssValue, - CWalletScanState &wss, std::string& strType, std::string& strErr, const KeyFilterFn& filter_fn = nullptr) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +bool LoadKey(CWallet* pwallet, DataStream& ssKey, DataStream& ssValue, std::string& strErr) { + LOCK(pwallet->cs_wallet); try { - // Unserialize - // Taking advantage of the fact that pair serialization - // is just the two items serialized one after the other - ssKey >> strType; - // If we have a filter, check if this matches the filter - if (filter_fn && !filter_fn(strType)) { - return true; - } - // Legacy entries in descriptor wallets are not allowed, abort immediately - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS) && DBKeys::LEGACY_TYPES.count(strType) > 0) { - wss.unexpected_legacy_entry = true; + CPubKey vchPubKey; + ssKey >> vchPubKey; + if (!vchPubKey.IsValid()) + { + strErr = "Error reading wallet database: CPubKey corrupt"; return false; } - if (strType == DBKeys::NAME) { - std::string strAddress; - ssKey >> strAddress; - std::string label; - ssValue >> label; - pwallet->m_address_book[DecodeDestination(strAddress)].SetLabel(label); - } else if (strType == DBKeys::PURPOSE) { - std::string strAddress; - ssKey >> strAddress; - std::string purpose_str; - ssValue >> purpose_str; - std::optional<AddressPurpose> purpose{PurposeFromString(purpose_str)}; - if (!purpose) { - pwallet->WalletLogPrintf("Warning: nonstandard purpose string '%s' for address '%s'\n", purpose_str, strAddress); - } - pwallet->m_address_book[DecodeDestination(strAddress)].purpose = purpose; - } else if (strType == DBKeys::TX) { - uint256 hash; - ssKey >> hash; - // LoadToWallet call below creates a new CWalletTx that fill_wtx - // callback fills with transaction metadata. - auto fill_wtx = [&](CWalletTx& wtx, bool new_tx) { - if(!new_tx) { - // There's some corruption here since the tx we just tried to load was already in the wallet. - // We don't consider this type of corruption critical, and can fix it by removing tx data and - // rescanning. - wss.tx_corrupt = true; - return false; - } - ssValue >> wtx; - if (wtx.GetHash() != hash) - return false; + CKey key; + CPrivKey pkey; + uint256 hash; + + ssValue >> pkey; + + // Old wallets store keys as DBKeys::KEY [pubkey] => [privkey] + // ... which was slow for wallets with lots of keys, because the public key is re-derived from the private key + // using EC operations as a checksum. + // Newer wallets store keys as DBKeys::KEY [pubkey] => [privkey][hash(pubkey,privkey)], which is much faster while + // remaining backwards-compatible. + try + { + ssValue >> hash; + } + catch (const std::ios_base::failure&) {} - // Undo serialize changes in 31600 - if (31404 <= wtx.fTimeReceivedIsTxTime && wtx.fTimeReceivedIsTxTime <= 31703) - { - if (!ssValue.empty()) - { - uint8_t fTmp; - uint8_t fUnused; - std::string unused_string; - ssValue >> fTmp >> fUnused >> unused_string; - strErr = strprintf("LoadWallet() upgrading tx ver=%d %d %s", - wtx.fTimeReceivedIsTxTime, fTmp, hash.ToString()); - wtx.fTimeReceivedIsTxTime = fTmp; - } - else - { - strErr = strprintf("LoadWallet() repairing tx ver=%d %s", wtx.fTimeReceivedIsTxTime, hash.ToString()); - wtx.fTimeReceivedIsTxTime = 0; - } - wss.vWalletUpgrade.push_back(hash); - } + bool fSkipCheck = false; - if (wtx.nOrderPos == -1) - wss.fAnyUnordered = true; + if (!hash.IsNull()) + { + // hash pubkey/privkey to accelerate wallet load + std::vector<unsigned char> vchKey; + vchKey.reserve(vchPubKey.size() + pkey.size()); + vchKey.insert(vchKey.end(), vchPubKey.begin(), vchPubKey.end()); + vchKey.insert(vchKey.end(), pkey.begin(), pkey.end()); - return true; - }; - if (!pwallet->LoadToWallet(hash, fill_wtx)) { - return false; - } - } else if (strType == DBKeys::WATCHS) { - wss.nWatchKeys++; - CScript script; - ssKey >> script; - uint8_t fYes; - ssValue >> fYes; - if (fYes == '1') { - pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadWatchOnly(script); - } - } else if (strType == DBKeys::KEY) { - CPubKey vchPubKey; - ssKey >> vchPubKey; - if (!vchPubKey.IsValid()) + if (Hash(vchKey) != hash) { - strErr = "Error reading wallet database: CPubKey corrupt"; + strErr = "Error reading wallet database: CPubKey/CPrivKey corrupt"; return false; } - CKey key; - CPrivKey pkey; - uint256 hash; - wss.nKeys++; - ssValue >> pkey; + fSkipCheck = true; + } - // Old wallets store keys as DBKeys::KEY [pubkey] => [privkey] - // ... which was slow for wallets with lots of keys, because the public key is re-derived from the private key - // using EC operations as a checksum. - // Newer wallets store keys as DBKeys::KEY [pubkey] => [privkey][hash(pubkey,privkey)], which is much faster while - // remaining backwards-compatible. - try - { - ssValue >> hash; + if (!key.Load(pkey, vchPubKey, fSkipCheck)) + { + strErr = "Error reading wallet database: CPrivKey corrupt"; + return false; + } + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKey(key, vchPubKey)) + { + strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadKey failed"; + return false; + } + } catch (const std::exception& e) { + if (strErr.empty()) { + strErr = e.what(); + } + return false; + } + return true; +} + +bool LoadCryptedKey(CWallet* pwallet, DataStream& ssKey, DataStream& ssValue, std::string& strErr) +{ + LOCK(pwallet->cs_wallet); + try { + CPubKey vchPubKey; + ssKey >> vchPubKey; + if (!vchPubKey.IsValid()) + { + strErr = "Error reading wallet database: CPubKey corrupt"; + return false; + } + std::vector<unsigned char> vchPrivKey; + ssValue >> vchPrivKey; + + // Get the checksum and check it + bool checksum_valid = false; + if (!ssValue.eof()) { + uint256 checksum; + ssValue >> checksum; + if (!(checksum_valid = Hash(vchPrivKey) == checksum)) { + strErr = "Error reading wallet database: Encrypted key corrupt"; + return false; } - catch (const std::ios_base::failure&) {} + } - bool fSkipCheck = false; + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCryptedKey(vchPubKey, vchPrivKey, checksum_valid)) + { + strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCryptedKey failed"; + return false; + } + } catch (const std::exception& e) { + if (strErr.empty()) { + strErr = e.what(); + } + return false; + } + return true; +} - if (!hash.IsNull()) - { - // hash pubkey/privkey to accelerate wallet load - std::vector<unsigned char> vchKey; - vchKey.reserve(vchPubKey.size() + pkey.size()); - vchKey.insert(vchKey.end(), vchPubKey.begin(), vchPubKey.end()); - vchKey.insert(vchKey.end(), pkey.begin(), pkey.end()); +bool LoadEncryptionKey(CWallet* pwallet, DataStream& ssKey, DataStream& ssValue, std::string& strErr) +{ + LOCK(pwallet->cs_wallet); + try { + // Master encryption key is loaded into only the wallet and not any of the ScriptPubKeyMans. + unsigned int nID; + ssKey >> nID; + CMasterKey kMasterKey; + ssValue >> kMasterKey; + if(pwallet->mapMasterKeys.count(nID) != 0) + { + strErr = strprintf("Error reading wallet database: duplicate CMasterKey id %u", nID); + return false; + } + pwallet->mapMasterKeys[nID] = kMasterKey; + if (pwallet->nMasterKeyMaxID < nID) + pwallet->nMasterKeyMaxID = nID; - if (Hash(vchKey) != hash) - { - strErr = "Error reading wallet database: CPubKey/CPrivKey corrupt"; - return false; - } + } catch (const std::exception& e) { + if (strErr.empty()) { + strErr = e.what(); + } + return false; + } + return true; +} - fSkipCheck = true; - } +bool LoadHDChain(CWallet* pwallet, DataStream& ssValue, std::string& strErr) +{ + LOCK(pwallet->cs_wallet); + try { + CHDChain chain; + ssValue >> chain; + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadHDChain(chain); + } catch (const std::exception& e) { + if (strErr.empty()) { + strErr = e.what(); + } + return false; + } + return true; +} - if (!key.Load(pkey, vchPubKey, fSkipCheck)) - { - strErr = "Error reading wallet database: CPrivKey corrupt"; - return false; - } - if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKey(key, vchPubKey)) - { - strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadKey failed"; - return false; - } - } else if (strType == DBKeys::MASTER_KEY) { - // Master encryption key is loaded into only the wallet and not any of the ScriptPubKeyMans. - unsigned int nID; - ssKey >> nID; - CMasterKey kMasterKey; - ssValue >> kMasterKey; - if(pwallet->mapMasterKeys.count(nID) != 0) - { - strErr = strprintf("Error reading wallet database: duplicate CMasterKey id %u", nID); - return false; - } - pwallet->mapMasterKeys[nID] = kMasterKey; - if (pwallet->nMasterKeyMaxID < nID) - pwallet->nMasterKeyMaxID = nID; - } else if (strType == DBKeys::CRYPTED_KEY) { - CPubKey vchPubKey; - ssKey >> vchPubKey; - if (!vchPubKey.IsValid()) - { - strErr = "Error reading wallet database: CPubKey corrupt"; - return false; +static DBErrors LoadMinVersion(CWallet* pwallet, DatabaseBatch& batch) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +{ + AssertLockHeld(pwallet->cs_wallet); + int nMinVersion = 0; + if (batch.Read(DBKeys::MINVERSION, nMinVersion)) { + if (nMinVersion > FEATURE_LATEST) + return DBErrors::TOO_NEW; + pwallet->LoadMinVersion(nMinVersion); + } + return DBErrors::LOAD_OK; +} + +static DBErrors LoadWalletFlags(CWallet* pwallet, DatabaseBatch& batch) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +{ + AssertLockHeld(pwallet->cs_wallet); + uint64_t flags; + if (batch.Read(DBKeys::FLAGS, flags)) { + if (!pwallet->LoadWalletFlags(flags)) { + pwallet->WalletLogPrintf("Error reading wallet database: Unknown non-tolerable wallet flags found\n"); + return DBErrors::TOO_NEW; + } + } + return DBErrors::LOAD_OK; +} + +struct LoadResult +{ + DBErrors m_result{DBErrors::LOAD_OK}; + int m_records{0}; +}; + +using LoadFunc = std::function<DBErrors(CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err)>; +static LoadResult LoadRecords(CWallet* pwallet, DatabaseBatch& batch, const std::string& key, DataStream& prefix, LoadFunc load_func) +{ + LoadResult result; + DataStream ssKey; + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + + Assume(!prefix.empty()); + std::unique_ptr<DatabaseCursor> cursor = batch.GetNewPrefixCursor(prefix); + if (!cursor) { + pwallet->WalletLogPrintf("Error getting database cursor for '%s' records\n", key); + result.m_result = DBErrors::CORRUPT; + return result; + } + + while (true) { + DatabaseCursor::Status status = cursor->Next(ssKey, ssValue); + if (status == DatabaseCursor::Status::DONE) { + break; + } else if (status == DatabaseCursor::Status::FAIL) { + pwallet->WalletLogPrintf("Error reading next '%s' record for wallet database\n", key); + result.m_result = DBErrors::CORRUPT; + return result; + } + std::string type; + ssKey >> type; + assert(type == key); + std::string error; + DBErrors record_res = load_func(pwallet, ssKey, ssValue, error); + if (record_res != DBErrors::LOAD_OK) { + pwallet->WalletLogPrintf("%s\n", error); + } + result.m_result = std::max(result.m_result, record_res); + ++result.m_records; + } + return result; +} + +static LoadResult LoadRecords(CWallet* pwallet, DatabaseBatch& batch, const std::string& key, LoadFunc load_func) +{ + DataStream prefix; + prefix << key; + return LoadRecords(pwallet, batch, key, prefix, load_func); +} + +static DBErrors LoadLegacyWalletRecords(CWallet* pwallet, DatabaseBatch& batch, int last_client) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +{ + AssertLockHeld(pwallet->cs_wallet); + DBErrors result = DBErrors::LOAD_OK; + + // Make sure descriptor wallets don't have any legacy records + if (pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + for (const auto& type : DBKeys::LEGACY_TYPES) { + DataStream key; + CDataStream value(SER_DISK, CLIENT_VERSION); + + DataStream prefix; + prefix << type; + std::unique_ptr<DatabaseCursor> cursor = batch.GetNewPrefixCursor(prefix); + if (!cursor) { + pwallet->WalletLogPrintf("Error getting database cursor for '%s' records\n", type); + return DBErrors::CORRUPT; } - std::vector<unsigned char> vchPrivKey; - ssValue >> vchPrivKey; - - // Get the checksum and check it - bool checksum_valid = false; - if (!ssValue.eof()) { - uint256 checksum; - ssValue >> checksum; - if (!(checksum_valid = Hash(vchPrivKey) == checksum)) { - strErr = "Error reading wallet database: Encrypted key corrupt"; - return false; - } + + DatabaseCursor::Status status = cursor->Next(key, value); + if (status != DatabaseCursor::Status::DONE) { + pwallet->WalletLogPrintf("Error: Unexpected legacy entry found in descriptor wallet %s. The wallet might have been tampered with or created with malicious intent.\n", pwallet->GetName()); + return DBErrors::UNEXPECTED_LEGACY_ENTRY; } + } - wss.nCKeys++; + return DBErrors::LOAD_OK; + } - if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCryptedKey(vchPubKey, vchPrivKey, checksum_valid)) - { - strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCryptedKey failed"; - return false; - } - wss.fIsEncrypted = true; - } else if (strType == DBKeys::KEYMETA) { - CPubKey vchPubKey; - ssKey >> vchPubKey; - CKeyMetadata keyMeta; - ssValue >> keyMeta; - wss.nKeyMeta++; - pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); - - // Extract some CHDChain info from this metadata if it has any - if (keyMeta.nVersion >= CKeyMetadata::VERSION_WITH_HDDATA && !keyMeta.hd_seed_id.IsNull() && keyMeta.hdKeypath.size() > 0) { - // Get the path from the key origin or from the path string - // Not applicable when path is "s" or "m" as those indicate a seed - // See https://github.com/bitcoin/bitcoin/pull/12924 - bool internal = false; - uint32_t index = 0; - if (keyMeta.hdKeypath != "s" && keyMeta.hdKeypath != "m") { - std::vector<uint32_t> path; - if (keyMeta.has_key_origin) { - // We have a key origin, so pull it from its path vector - path = keyMeta.key_origin.path; - } else { - // No key origin, have to parse the string - if (!ParseHDKeypath(keyMeta.hdKeypath, path)) { - strErr = "Error reading wallet database: keymeta with invalid HD keypath"; - return false; - } - } + // Load HD Chain + // Note: There should only be one HDCHAIN record with no data following the type + LoadResult hd_chain_res = LoadRecords(pwallet, batch, DBKeys::HDCHAIN, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + return LoadHDChain(pwallet, value, err) ? DBErrors:: LOAD_OK : DBErrors::CORRUPT; + }); + result = std::max(result, hd_chain_res.m_result); + + // Load unencrypted keys + LoadResult key_res = LoadRecords(pwallet, batch, DBKeys::KEY, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + return LoadKey(pwallet, key, value, err) ? DBErrors::LOAD_OK : DBErrors::CORRUPT; + }); + result = std::max(result, key_res.m_result); + + // Load encrypted keys + LoadResult ckey_res = LoadRecords(pwallet, batch, DBKeys::CRYPTED_KEY, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + return LoadCryptedKey(pwallet, key, value, err) ? DBErrors::LOAD_OK : DBErrors::CORRUPT; + }); + result = std::max(result, ckey_res.m_result); + + // Load scripts + LoadResult script_res = LoadRecords(pwallet, batch, DBKeys::CSCRIPT, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& strErr) { + uint160 hash; + key >> hash; + CScript script; + value >> script; + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCScript(script)) + { + strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCScript failed"; + return DBErrors::NONCRITICAL_ERROR; + } + return DBErrors::LOAD_OK; + }); + result = std::max(result, script_res.m_result); + + // Check whether rewrite is needed + if (ckey_res.m_records > 0) { + // Rewrite encrypted wallets of versions 0.4.0 and 0.5.0rc: + if (last_client == 40000 || last_client == 50000) result = std::max(result, DBErrors::NEED_REWRITE); + } - // Extract the index and internal from the path - // Path string is m/0'/k'/i' - // Path vector is [0', k', i'] (but as ints OR'd with the hardened bit - // k == 0 for external, 1 for internal. i is the index - if (path.size() != 3) { - strErr = "Error reading wallet database: keymeta found with unexpected path"; - return false; - } - if (path[0] != 0x80000000) { - strErr = strprintf("Unexpected path index of 0x%08x (expected 0x80000000) for the element at index 0", path[0]); - return false; - } - if (path[1] != 0x80000000 && path[1] != (1 | 0x80000000)) { - strErr = strprintf("Unexpected path index of 0x%08x (expected 0x80000000 or 0x80000001) for the element at index 1", path[1]); - return false; - } - if ((path[2] & 0x80000000) == 0) { - strErr = strprintf("Unexpected path index of 0x%08x (expected to be greater than or equal to 0x80000000)", path[2]); - return false; + // Load keymeta + std::map<uint160, CHDChain> hd_chains; + LoadResult keymeta_res = LoadRecords(pwallet, batch, DBKeys::KEYMETA, + [&hd_chains] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& strErr) { + CPubKey vchPubKey; + key >> vchPubKey; + CKeyMetadata keyMeta; + value >> keyMeta; + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); + + // Extract some CHDChain info from this metadata if it has any + if (keyMeta.nVersion >= CKeyMetadata::VERSION_WITH_HDDATA && !keyMeta.hd_seed_id.IsNull() && keyMeta.hdKeypath.size() > 0) { + // Get the path from the key origin or from the path string + // Not applicable when path is "s" or "m" as those indicate a seed + // See https://github.com/bitcoin/bitcoin/pull/12924 + bool internal = false; + uint32_t index = 0; + if (keyMeta.hdKeypath != "s" && keyMeta.hdKeypath != "m") { + std::vector<uint32_t> path; + if (keyMeta.has_key_origin) { + // We have a key origin, so pull it from its path vector + path = keyMeta.key_origin.path; + } else { + // No key origin, have to parse the string + if (!ParseHDKeypath(keyMeta.hdKeypath, path)) { + strErr = "Error reading wallet database: keymeta with invalid HD keypath"; + return DBErrors::NONCRITICAL_ERROR; } - internal = path[1] == (1 | 0x80000000); - index = path[2] & ~0x80000000; } - // Insert a new CHDChain, or get the one that already exists - auto ins = wss.m_hd_chains.emplace(keyMeta.hd_seed_id, CHDChain()); - CHDChain& chain = ins.first->second; - if (ins.second) { - // For new chains, we want to default to VERSION_HD_BASE until we see an internal - chain.nVersion = CHDChain::VERSION_HD_BASE; - chain.seed_id = keyMeta.hd_seed_id; + // Extract the index and internal from the path + // Path string is m/0'/k'/i' + // Path vector is [0', k', i'] (but as ints OR'd with the hardened bit + // k == 0 for external, 1 for internal. i is the index + if (path.size() != 3) { + strErr = "Error reading wallet database: keymeta found with unexpected path"; + return DBErrors::NONCRITICAL_ERROR; } - if (internal) { - chain.nVersion = CHDChain::VERSION_HD_CHAIN_SPLIT; - chain.nInternalChainCounter = std::max(chain.nInternalChainCounter, index + 1); - } else { - chain.nExternalChainCounter = std::max(chain.nExternalChainCounter, index + 1); + if (path[0] != 0x80000000) { + strErr = strprintf("Unexpected path index of 0x%08x (expected 0x80000000) for the element at index 0", path[0]); + return DBErrors::NONCRITICAL_ERROR; } + if (path[1] != 0x80000000 && path[1] != (1 | 0x80000000)) { + strErr = strprintf("Unexpected path index of 0x%08x (expected 0x80000000 or 0x80000001) for the element at index 1", path[1]); + return DBErrors::NONCRITICAL_ERROR; + } + if ((path[2] & 0x80000000) == 0) { + strErr = strprintf("Unexpected path index of 0x%08x (expected to be greater than or equal to 0x80000000)", path[2]); + return DBErrors::NONCRITICAL_ERROR; + } + internal = path[1] == (1 | 0x80000000); + index = path[2] & ~0x80000000; } - } else if (strType == DBKeys::WATCHMETA) { - CScript script; - ssKey >> script; - CKeyMetadata keyMeta; - ssValue >> keyMeta; - wss.nKeyMeta++; - pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadScriptMetadata(CScriptID(script), keyMeta); - } else if (strType == DBKeys::DEFAULTKEY) { - // We don't want or need the default key, but if there is one set, - // we want to make sure that it is valid so that we can detect corruption - CPubKey vchPubKey; - ssValue >> vchPubKey; - if (!vchPubKey.IsValid()) { - strErr = "Error reading wallet database: Default Key corrupt"; - return false; - } - } else if (strType == DBKeys::POOL) { - int64_t nIndex; - ssKey >> nIndex; - CKeyPool keypool; - ssValue >> keypool; - - pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKeyPool(nIndex, keypool); - } else if (strType == DBKeys::CSCRIPT) { - uint160 hash; - ssKey >> hash; - CScript script; - ssValue >> script; - if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCScript(script)) - { - strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCScript failed"; - return false; + + // Insert a new CHDChain, or get the one that already exists + auto [ins, inserted] = hd_chains.emplace(keyMeta.hd_seed_id, CHDChain()); + CHDChain& chain = ins->second; + if (inserted) { + // For new chains, we want to default to VERSION_HD_BASE until we see an internal + chain.nVersion = CHDChain::VERSION_HD_BASE; + chain.seed_id = keyMeta.hd_seed_id; } - } else if (strType == DBKeys::ORDERPOSNEXT) { - ssValue >> pwallet->nOrderPosNext; - } else if (strType == DBKeys::DESTDATA) { - std::string strAddress, strKey, strValue; - ssKey >> strAddress; - ssKey >> strKey; - ssValue >> strValue; - const CTxDestination& dest{DecodeDestination(strAddress)}; - if (strKey.compare("used") == 0) { - // Load "used" key indicating if an IsMine address has - // previously been spent from with avoid_reuse option enabled. - // The strValue is not used for anything currently, but could - // hold more information in the future. Current values are just - // "1" or "p" for present (which was written prior to - // f5ba424cd44619d9b9be88b8593d69a7ba96db26). - pwallet->LoadAddressPreviouslySpent(dest); - } else if (strKey.compare(0, 2, "rr") == 0) { - // Load "rr##" keys where ## is a decimal number, and strValue - // is a serialized RecentRequestEntry object. - pwallet->LoadAddressReceiveRequest(dest, strKey.substr(2), strValue); + if (internal) { + chain.nVersion = CHDChain::VERSION_HD_CHAIN_SPLIT; + chain.nInternalChainCounter = std::max(chain.nInternalChainCounter, index + 1); + } else { + chain.nExternalChainCounter = std::max(chain.nExternalChainCounter, index + 1); } - } else if (strType == DBKeys::HDCHAIN) { - CHDChain chain; - ssValue >> chain; - pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadHDChain(chain); - } else if (strType == DBKeys::OLD_KEY) { - strErr = "Found unsupported 'wkey' record, try loading with version 0.18"; - return false; - } else if (strType == DBKeys::ACTIVEEXTERNALSPK || strType == DBKeys::ACTIVEINTERNALSPK) { - uint8_t type; - ssKey >> type; - uint256 id; - ssValue >> id; + } + return DBErrors::LOAD_OK; + }); + result = std::max(result, keymeta_res.m_result); - bool internal = strType == DBKeys::ACTIVEINTERNALSPK; - auto& spk_mans = internal ? wss.m_active_internal_spks : wss.m_active_external_spks; - if (spk_mans.count(static_cast<OutputType>(type)) > 0) { - strErr = "Multiple ScriptPubKeyMans specified for a single type"; - return false; - } - spk_mans[static_cast<OutputType>(type)] = id; - } else if (strType == DBKeys::WALLETDESCRIPTOR) { - uint256 id; - ssKey >> id; - WalletDescriptor desc; - try { - ssValue >> desc; - } catch (const std::ios_base::failure& e) { - strErr = e.what(); - wss.descriptor_unknown = true; - return false; + // Set inactive chains + if (!hd_chains.empty()) { + LegacyScriptPubKeyMan* legacy_spkm = pwallet->GetLegacyScriptPubKeyMan(); + if (legacy_spkm) { + for (const auto& [hd_seed_id, chain] : hd_chains) { + if (hd_seed_id != legacy_spkm->GetHDChain().seed_id) { + legacy_spkm->AddInactiveHDChain(chain); + } } - if (wss.m_descriptor_caches.count(id) == 0) { - wss.m_descriptor_caches[id] = DescriptorCache(); + } else { + pwallet->WalletLogPrintf("Inactive HD Chains found but no Legacy ScriptPubKeyMan\n"); + result = DBErrors::CORRUPT; + } + } + + // Load watchonly scripts + LoadResult watch_script_res = LoadRecords(pwallet, batch, DBKeys::WATCHS, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + CScript script; + key >> script; + uint8_t fYes; + value >> fYes; + if (fYes == '1') { + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadWatchOnly(script); + } + return DBErrors::LOAD_OK; + }); + result = std::max(result, watch_script_res.m_result); + + // Load watchonly meta + LoadResult watch_meta_res = LoadRecords(pwallet, batch, DBKeys::WATCHMETA, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + CScript script; + key >> script; + CKeyMetadata keyMeta; + value >> keyMeta; + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadScriptMetadata(CScriptID(script), keyMeta); + return DBErrors::LOAD_OK; + }); + result = std::max(result, watch_meta_res.m_result); + + // Load keypool + LoadResult pool_res = LoadRecords(pwallet, batch, DBKeys::POOL, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + int64_t nIndex; + key >> nIndex; + CKeyPool keypool; + value >> keypool; + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKeyPool(nIndex, keypool); + return DBErrors::LOAD_OK; + }); + result = std::max(result, pool_res.m_result); + + // Deal with old "wkey" and "defaultkey" records. + // These are not actually loaded, but we need to check for them + + // We don't want or need the default key, but if there is one set, + // we want to make sure that it is valid so that we can detect corruption + // Note: There should only be one DEFAULTKEY with nothing trailing the type + LoadResult default_key_res = LoadRecords(pwallet, batch, DBKeys::DEFAULTKEY, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + CPubKey default_pubkey; + try { + value >> default_pubkey; + } catch (const std::exception& e) { + err = e.what(); + return DBErrors::CORRUPT; + } + if (!default_pubkey.IsValid()) { + err = "Error reading wallet database: Default Key corrupt"; + return DBErrors::CORRUPT; + } + return DBErrors::LOAD_OK; + }); + result = std::max(result, default_key_res.m_result); + + // "wkey" records are unsupported, if we see any, throw an error + LoadResult wkey_res = LoadRecords(pwallet, batch, DBKeys::OLD_KEY, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + err = "Found unsupported 'wkey' record, try loading with version 0.18"; + return DBErrors::LOAD_FAIL; + }); + result = std::max(result, wkey_res.m_result); + + if (result <= DBErrors::NONCRITICAL_ERROR) { + // Only do logging and time first key update if there were no critical errors + pwallet->WalletLogPrintf("Legacy Wallet Keys: %u plaintext, %u encrypted, %u w/ metadata, %u total.\n", + key_res.m_records, ckey_res.m_records, keymeta_res.m_records, key_res.m_records + ckey_res.m_records); + + // nTimeFirstKey is only reliable if all keys have metadata + if (pwallet->IsLegacy() && (key_res.m_records + ckey_res.m_records + watch_script_res.m_records) != (keymeta_res.m_records + watch_meta_res.m_records)) { + auto spk_man = pwallet->GetOrCreateLegacyScriptPubKeyMan(); + if (spk_man) { + LOCK(spk_man->cs_KeyStore); + spk_man->UpdateTimeFirstKey(1); } - pwallet->LoadDescriptorScriptPubKeyMan(id, desc); - } else if (strType == DBKeys::WALLETDESCRIPTORCACHE) { + } + } + + return result; +} + +template<typename... Args> +static DataStream PrefixStream(const Args&... args) +{ + DataStream prefix; + SerializeMany(prefix, args...); + return prefix; +} + +static DBErrors LoadDescriptorWalletRecords(CWallet* pwallet, DatabaseBatch& batch, int last_client) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +{ + AssertLockHeld(pwallet->cs_wallet); + + // Load descriptor record + int num_keys = 0; + int num_ckeys= 0; + LoadResult desc_res = LoadRecords(pwallet, batch, DBKeys::WALLETDESCRIPTOR, + [&batch, &num_keys, &num_ckeys, &last_client] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& strErr) { + DBErrors result = DBErrors::LOAD_OK; + + uint256 id; + key >> id; + WalletDescriptor desc; + try { + value >> desc; + } catch (const std::ios_base::failure&) { + strErr = strprintf("Error: Unrecognized descriptor found in wallet %s. ", pwallet->GetName()); + strErr += (last_client > CLIENT_VERSION) ? "The wallet might had been created on a newer version. " : + "The database might be corrupted or the software version is not compatible with one of your wallet descriptors. "; + strErr += "Please try running the latest software version"; + return DBErrors::UNKNOWN_DESCRIPTOR; + } + pwallet->LoadDescriptorScriptPubKeyMan(id, desc); + + DescriptorCache cache; + + // Get key cache for this descriptor + DataStream prefix = PrefixStream(DBKeys::WALLETDESCRIPTORCACHE, id); + LoadResult key_cache_res = LoadRecords(pwallet, batch, DBKeys::WALLETDESCRIPTORCACHE, prefix, + [&id, &cache] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { bool parent = true; uint256 desc_id; uint32_t key_exp_index; uint32_t der_index; - ssKey >> desc_id; - ssKey >> key_exp_index; + key >> desc_id; + assert(desc_id == id); + key >> key_exp_index; // if the der_index exists, it's a derived xpub try { - ssKey >> der_index; + key >> der_index; parent = false; } catch (...) {} std::vector<unsigned char> ser_xpub(BIP32_EXTKEY_SIZE); - ssValue >> ser_xpub; + value >> ser_xpub; CExtPubKey xpub; xpub.Decode(ser_xpub.data()); if (parent) { - wss.m_descriptor_caches[desc_id].CacheParentExtPubKey(key_exp_index, xpub); + cache.CacheParentExtPubKey(key_exp_index, xpub); } else { - wss.m_descriptor_caches[desc_id].CacheDerivedExtPubKey(key_exp_index, der_index, xpub); + cache.CacheDerivedExtPubKey(key_exp_index, der_index, xpub); } - } else if (strType == DBKeys::WALLETDESCRIPTORLHCACHE) { + return DBErrors::LOAD_OK; + }); + result = std::max(result, key_cache_res.m_result); + + // Get last hardened cache for this descriptor + prefix = PrefixStream(DBKeys::WALLETDESCRIPTORLHCACHE, id); + LoadResult lh_cache_res = LoadRecords(pwallet, batch, DBKeys::WALLETDESCRIPTORLHCACHE, prefix, + [&id, &cache] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { uint256 desc_id; uint32_t key_exp_index; - ssKey >> desc_id; - ssKey >> key_exp_index; + key >> desc_id; + assert(desc_id == id); + key >> key_exp_index; std::vector<unsigned char> ser_xpub(BIP32_EXTKEY_SIZE); - ssValue >> ser_xpub; + value >> ser_xpub; CExtPubKey xpub; xpub.Decode(ser_xpub.data()); - wss.m_descriptor_caches[desc_id].CacheLastHardenedExtPubKey(key_exp_index, xpub); - } else if (strType == DBKeys::WALLETDESCRIPTORKEY) { + cache.CacheLastHardenedExtPubKey(key_exp_index, xpub); + return DBErrors::LOAD_OK; + }); + result = std::max(result, lh_cache_res.m_result); + + // Set the cache for this descriptor + auto spk_man = (DescriptorScriptPubKeyMan*)pwallet->GetScriptPubKeyMan(id); + assert(spk_man); + spk_man->SetCache(cache); + + // Get unencrypted keys + prefix = PrefixStream(DBKeys::WALLETDESCRIPTORKEY, id); + LoadResult key_res = LoadRecords(pwallet, batch, DBKeys::WALLETDESCRIPTORKEY, prefix, + [&id, &spk_man] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& strErr) { uint256 desc_id; CPubKey pubkey; - ssKey >> desc_id; - ssKey >> pubkey; + key >> desc_id; + assert(desc_id == id); + key >> pubkey; if (!pubkey.IsValid()) { - strErr = "Error reading wallet database: CPubKey corrupt"; - return false; + strErr = "Error reading wallet database: descriptor unencrypted key CPubKey corrupt"; + return DBErrors::CORRUPT; } - CKey key; + CKey privkey; CPrivKey pkey; uint256 hash; - wss.nKeys++; - ssValue >> pkey; - ssValue >> hash; + value >> pkey; + value >> hash; // hash pubkey/privkey to accelerate wallet load std::vector<unsigned char> to_hash; @@ -726,77 +891,254 @@ ReadKeyValue(CWallet* pwallet, DataStream& ssKey, CDataStream& ssValue, if (Hash(to_hash) != hash) { - strErr = "Error reading wallet database: CPubKey/CPrivKey corrupt"; - return false; + strErr = "Error reading wallet database: descriptor unencrypted key CPubKey/CPrivKey corrupt"; + return DBErrors::CORRUPT; } - if (!key.Load(pkey, pubkey, true)) + if (!privkey.Load(pkey, pubkey, true)) { - strErr = "Error reading wallet database: CPrivKey corrupt"; - return false; + strErr = "Error reading wallet database: descriptor unencrypted key CPrivKey corrupt"; + return DBErrors::CORRUPT; } - wss.m_descriptor_keys.insert(std::make_pair(std::make_pair(desc_id, pubkey.GetID()), key)); - } else if (strType == DBKeys::WALLETDESCRIPTORCKEY) { + spk_man->AddKey(pubkey.GetID(), privkey); + return DBErrors::LOAD_OK; + }); + result = std::max(result, key_res.m_result); + num_keys = key_res.m_records; + + // Get encrypted keys + prefix = PrefixStream(DBKeys::WALLETDESCRIPTORCKEY, id); + LoadResult ckey_res = LoadRecords(pwallet, batch, DBKeys::WALLETDESCRIPTORCKEY, prefix, + [&id, &spk_man] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { uint256 desc_id; CPubKey pubkey; - ssKey >> desc_id; - ssKey >> pubkey; + key >> desc_id; + assert(desc_id == id); + key >> pubkey; if (!pubkey.IsValid()) { - strErr = "Error reading wallet database: CPubKey corrupt"; - return false; + err = "Error reading wallet database: descriptor encrypted key CPubKey corrupt"; + return DBErrors::CORRUPT; } std::vector<unsigned char> privkey; - ssValue >> privkey; - wss.nCKeys++; + value >> privkey; - wss.m_descriptor_crypt_keys.insert(std::make_pair(std::make_pair(desc_id, pubkey.GetID()), std::make_pair(pubkey, privkey))); - wss.fIsEncrypted = true; - } else if (strType == DBKeys::LOCKED_UTXO) { - uint256 hash; - uint32_t n; - ssKey >> hash; - ssKey >> n; - pwallet->LockCoin(COutPoint(hash, n)); - } else if (strType != DBKeys::BESTBLOCK && strType != DBKeys::BESTBLOCK_NOMERKLE && - strType != DBKeys::MINVERSION && strType != DBKeys::ACENTRY && - strType != DBKeys::VERSION && strType != DBKeys::SETTINGS && - strType != DBKeys::FLAGS) { - wss.m_unknown_records++; + spk_man->AddCryptedKey(pubkey.GetID(), pubkey, privkey); + return DBErrors::LOAD_OK; + }); + result = std::max(result, ckey_res.m_result); + num_ckeys = ckey_res.m_records; + + return result; + }); + + if (desc_res.m_result <= DBErrors::NONCRITICAL_ERROR) { + // Only log if there are no critical errors + pwallet->WalletLogPrintf("Descriptors: %u, Descriptor Keys: %u plaintext, %u encrypted, %u total.\n", + desc_res.m_records, num_keys, num_ckeys, num_keys + num_ckeys); + } + + return desc_res.m_result; +} + +static DBErrors LoadAddressBookRecords(CWallet* pwallet, DatabaseBatch& batch) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +{ + AssertLockHeld(pwallet->cs_wallet); + DBErrors result = DBErrors::LOAD_OK; + + // Load name record + LoadResult name_res = LoadRecords(pwallet, batch, DBKeys::NAME, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { + std::string strAddress; + key >> strAddress; + std::string label; + value >> label; + pwallet->m_address_book[DecodeDestination(strAddress)].SetLabel(label); + return DBErrors::LOAD_OK; + }); + result = std::max(result, name_res.m_result); + + // Load purpose record + LoadResult purpose_res = LoadRecords(pwallet, batch, DBKeys::PURPOSE, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { + std::string strAddress; + key >> strAddress; + std::string purpose_str; + value >> purpose_str; + std::optional<AddressPurpose> purpose{PurposeFromString(purpose_str)}; + if (!purpose) { + pwallet->WalletLogPrintf("Warning: nonstandard purpose string '%s' for address '%s'\n", purpose_str, strAddress); } - } catch (const std::exception& e) { - if (strErr.empty()) { - strErr = e.what(); + pwallet->m_address_book[DecodeDestination(strAddress)].purpose = purpose; + return DBErrors::LOAD_OK; + }); + result = std::max(result, purpose_res.m_result); + + // Load destination data record + LoadResult dest_res = LoadRecords(pwallet, batch, DBKeys::DESTDATA, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { + std::string strAddress, strKey, strValue; + key >> strAddress; + key >> strKey; + value >> strValue; + const CTxDestination& dest{DecodeDestination(strAddress)}; + if (strKey.compare("used") == 0) { + // Load "used" key indicating if an IsMine address has + // previously been spent from with avoid_reuse option enabled. + // The strValue is not used for anything currently, but could + // hold more information in the future. Current values are just + // "1" or "p" for present (which was written prior to + // f5ba424cd44619d9b9be88b8593d69a7ba96db26). + pwallet->LoadAddressPreviouslySpent(dest); + } else if (strKey.compare(0, 2, "rr") == 0) { + // Load "rr##" keys where ## is a decimal number, and strValue + // is a serialized RecentRequestEntry object. + pwallet->LoadAddressReceiveRequest(dest, strKey.substr(2), strValue); } - return false; - } catch (...) { - if (strErr.empty()) { - strErr = "Caught unknown exception in ReadKeyValue"; + return DBErrors::LOAD_OK; + }); + result = std::max(result, dest_res.m_result); + + return result; +} + +static DBErrors LoadTxRecords(CWallet* pwallet, DatabaseBatch& batch, std::vector<uint256>& upgraded_txs, bool& any_unordered) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +{ + AssertLockHeld(pwallet->cs_wallet); + DBErrors result = DBErrors::LOAD_OK; + + // Load tx record + any_unordered = false; + LoadResult tx_res = LoadRecords(pwallet, batch, DBKeys::TX, + [&any_unordered, &upgraded_txs] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { + DBErrors result = DBErrors::LOAD_OK; + uint256 hash; + key >> hash; + // LoadToWallet call below creates a new CWalletTx that fill_wtx + // callback fills with transaction metadata. + auto fill_wtx = [&](CWalletTx& wtx, bool new_tx) { + if(!new_tx) { + // There's some corruption here since the tx we just tried to load was already in the wallet. + err = "Error: Corrupt transaction found. This can be fixed by removing transactions from wallet and rescanning."; + result = DBErrors::CORRUPT; + return false; + } + value >> wtx; + if (wtx.GetHash() != hash) + return false; + + // Undo serialize changes in 31600 + if (31404 <= wtx.fTimeReceivedIsTxTime && wtx.fTimeReceivedIsTxTime <= 31703) + { + if (!value.empty()) + { + uint8_t fTmp; + uint8_t fUnused; + std::string unused_string; + value >> fTmp >> fUnused >> unused_string; + pwallet->WalletLogPrintf("LoadWallet() upgrading tx ver=%d %d %s\n", + wtx.fTimeReceivedIsTxTime, fTmp, hash.ToString()); + wtx.fTimeReceivedIsTxTime = fTmp; + } + else + { + pwallet->WalletLogPrintf("LoadWallet() repairing tx ver=%d %s\n", wtx.fTimeReceivedIsTxTime, hash.ToString()); + wtx.fTimeReceivedIsTxTime = 0; + } + upgraded_txs.push_back(hash); + } + + if (wtx.nOrderPos == -1) + any_unordered = true; + + return true; + }; + if (!pwallet->LoadToWallet(hash, fill_wtx)) { + // Use std::max as fill_wtx may have already set result to CORRUPT + result = std::max(result, DBErrors::NEED_RESCAN); } - return false; - } - return true; + return result; + }); + result = std::max(result, tx_res.m_result); + + // Load locked utxo record + LoadResult locked_utxo_res = LoadRecords(pwallet, batch, DBKeys::LOCKED_UTXO, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { + uint256 hash; + uint32_t n; + key >> hash; + key >> n; + pwallet->LockCoin(COutPoint(hash, n)); + return DBErrors::LOAD_OK; + }); + result = std::max(result, locked_utxo_res.m_result); + + // Load orderposnext record + // Note: There should only be one ORDERPOSNEXT record with nothing trailing the type + LoadResult order_pos_res = LoadRecords(pwallet, batch, DBKeys::ORDERPOSNEXT, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { + try { + value >> pwallet->nOrderPosNext; + } catch (const std::exception& e) { + err = e.what(); + return DBErrors::NONCRITICAL_ERROR; + } + return DBErrors::LOAD_OK; + }); + result = std::max(result, order_pos_res.m_result); + + return result; } -bool ReadKeyValue(CWallet* pwallet, DataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr, const KeyFilterFn& filter_fn) +static DBErrors LoadActiveSPKMs(CWallet* pwallet, DatabaseBatch& batch) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { - CWalletScanState dummy_wss; - LOCK(pwallet->cs_wallet); - return ReadKeyValue(pwallet, ssKey, ssValue, dummy_wss, strType, strErr, filter_fn); + AssertLockHeld(pwallet->cs_wallet); + DBErrors result = DBErrors::LOAD_OK; + + // Load spk records + std::set<std::pair<OutputType, bool>> seen_spks; + for (const auto& spk_key : {DBKeys::ACTIVEEXTERNALSPK, DBKeys::ACTIVEINTERNALSPK}) { + LoadResult spkm_res = LoadRecords(pwallet, batch, spk_key, + [&seen_spks, &spk_key] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& strErr) { + uint8_t output_type; + key >> output_type; + uint256 id; + value >> id; + + bool internal = spk_key == DBKeys::ACTIVEINTERNALSPK; + auto [it, insert] = seen_spks.emplace(static_cast<OutputType>(output_type), internal); + if (!insert) { + strErr = "Multiple ScriptpubKeyMans specified for a single type"; + return DBErrors::CORRUPT; + } + pwallet->LoadActiveScriptPubKeyMan(id, static_cast<OutputType>(output_type), /*internal=*/internal); + return DBErrors::LOAD_OK; + }); + result = std::max(result, spkm_res.m_result); + } + return result; } -bool WalletBatch::IsKeyType(const std::string& strType) +static DBErrors LoadDecryptionKeys(CWallet* pwallet, DatabaseBatch& batch) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { - return (strType == DBKeys::KEY || - strType == DBKeys::MASTER_KEY || strType == DBKeys::CRYPTED_KEY); + AssertLockHeld(pwallet->cs_wallet); + + // Load decryption key (mkey) records + LoadResult mkey_res = LoadRecords(pwallet, batch, DBKeys::MASTER_KEY, + [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + if (!LoadEncryptionKey(pwallet, key, value, err)) { + return DBErrors::CORRUPT; + } + return DBErrors::LOAD_OK; + }); + return mkey_res.m_result; } DBErrors WalletBatch::LoadWallet(CWallet* pwallet) { - CWalletScanState wss; - bool fNoncriticalErrors = false; - bool rescan_required = false; DBErrors result = DBErrors::LOAD_OK; + bool any_unordered = false; + std::vector<uint256> upgraded_txs; LOCK(pwallet->cs_wallet); @@ -806,22 +1148,11 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) pwallet->WalletLogPrintf("Wallet file version = %d, last client version = %d\n", pwallet->GetVersion(), last_client); try { - int nMinVersion = 0; - if (m_batch->Read(DBKeys::MINVERSION, nMinVersion)) { - if (nMinVersion > FEATURE_LATEST) - return DBErrors::TOO_NEW; - pwallet->LoadMinVersion(nMinVersion); - } + if ((result = LoadMinVersion(pwallet, *m_batch)) != DBErrors::LOAD_OK) return result; // Load wallet flags, so they are known when processing other records. // The FLAGS key is absent during wallet creation. - uint64_t flags; - if (m_batch->Read(DBKeys::FLAGS, flags)) { - if (!pwallet->LoadWalletFlags(flags)) { - pwallet->WalletLogPrintf("Error reading wallet database: Unknown non-tolerable wallet flags found\n"); - return DBErrors::CORRUPT; - } - } + if ((result = LoadWalletFlags(pwallet, *m_batch)) != DBErrors::LOAD_OK) return result; #ifndef ENABLE_EXTERNAL_SIGNER if (pwallet->IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { @@ -830,101 +1161,31 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) } #endif - // Get cursor - std::unique_ptr<DatabaseCursor> cursor = m_batch->GetNewCursor(); - if (!cursor) - { - pwallet->WalletLogPrintf("Error getting wallet database cursor\n"); - return DBErrors::CORRUPT; - } + // Load legacy wallet keys + result = std::max(LoadLegacyWalletRecords(pwallet, *m_batch, last_client), result); - while (true) - { - // Read next record - DataStream ssKey{}; - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - DatabaseCursor::Status status = cursor->Next(ssKey, ssValue); - if (status == DatabaseCursor::Status::DONE) { - break; - } else if (status == DatabaseCursor::Status::FAIL) { - cursor.reset(); - pwallet->WalletLogPrintf("Error reading next record from wallet database\n"); - return DBErrors::CORRUPT; - } + // Load descriptors + result = std::max(LoadDescriptorWalletRecords(pwallet, *m_batch, last_client), result); + // Early return if there are unknown descriptors. Later loading of ACTIVEINTERNALSPK and ACTIVEEXTERNALEXPK + // may reference the unknown descriptor's ID which can result in a misleading corruption error + // when in reality the wallet is simply too new. + if (result == DBErrors::UNKNOWN_DESCRIPTOR) return result; - // Try to be tolerant of single corrupt records: - std::string strType, strErr; - if (!ReadKeyValue(pwallet, ssKey, ssValue, wss, strType, strErr)) - { - if (wss.unexpected_legacy_entry) { - strErr = strprintf("Error: Unexpected legacy entry found in descriptor wallet %s. ", pwallet->GetName()); - strErr += "The wallet might have been tampered with or created with malicious intent."; - pwallet->WalletLogPrintf("%s\n", strErr); - return DBErrors::UNEXPECTED_LEGACY_ENTRY; - } - // losing keys is considered a catastrophic error, anything else - // we assume the user can live with: - if (IsKeyType(strType) || strType == DBKeys::DEFAULTKEY) { - result = DBErrors::CORRUPT; - } else if (strType == DBKeys::FLAGS) { - // reading the wallet flags can only fail if unknown flags are present - result = DBErrors::TOO_NEW; - } else if (wss.tx_corrupt) { - pwallet->WalletLogPrintf("Error: Corrupt transaction found. This can be fixed by removing transactions from wallet and rescanning.\n"); - // Set tx_corrupt back to false so that the error is only printed once (per corrupt tx) - wss.tx_corrupt = false; - result = DBErrors::CORRUPT; - } else if (wss.descriptor_unknown) { - strErr = strprintf("Error: Unrecognized descriptor found in wallet %s. ", pwallet->GetName()); - strErr += (last_client > CLIENT_VERSION) ? "The wallet might had been created on a newer version. " : - "The database might be corrupted or the software version is not compatible with one of your wallet descriptors. "; - strErr += "Please try running the latest software version"; - pwallet->WalletLogPrintf("%s\n", strErr); - return DBErrors::UNKNOWN_DESCRIPTOR; - } else { - // Leave other errors alone, if we try to fix them we might make things worse. - fNoncriticalErrors = true; // ... but do warn the user there is something wrong. - if (strType == DBKeys::TX) - // Rescan if there is a bad transaction record: - rescan_required = true; - } - } - if (!strErr.empty()) - pwallet->WalletLogPrintf("%s\n", strErr); - } - } catch (...) { - result = DBErrors::CORRUPT; - } + // Load address book + result = std::max(LoadAddressBookRecords(pwallet, *m_batch), result); - // Set the active ScriptPubKeyMans - for (auto spk_man_pair : wss.m_active_external_spks) { - pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /*internal=*/false); - } - for (auto spk_man_pair : wss.m_active_internal_spks) { - pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /*internal=*/true); - } + // Load tx records + result = std::max(LoadTxRecords(pwallet, *m_batch, upgraded_txs, any_unordered), result); - // Set the descriptor caches - for (const auto& desc_cache_pair : wss.m_descriptor_caches) { - auto spk_man = pwallet->GetScriptPubKeyMan(desc_cache_pair.first); - assert(spk_man); - ((DescriptorScriptPubKeyMan*)spk_man)->SetCache(desc_cache_pair.second); - } + // Load SPKMs + result = std::max(LoadActiveSPKMs(pwallet, *m_batch), result); - // Set the descriptor keys - for (const auto& desc_key_pair : wss.m_descriptor_keys) { - auto spk_man = pwallet->GetScriptPubKeyMan(desc_key_pair.first.first); - ((DescriptorScriptPubKeyMan*)spk_man)->AddKey(desc_key_pair.first.second, desc_key_pair.second); - } - for (const auto& desc_key_pair : wss.m_descriptor_crypt_keys) { - auto spk_man = pwallet->GetScriptPubKeyMan(desc_key_pair.first.first); - ((DescriptorScriptPubKeyMan*)spk_man)->AddCryptedKey(desc_key_pair.first.second, desc_key_pair.second.first, desc_key_pair.second.second); - } - - if (rescan_required && result == DBErrors::LOAD_OK) { - result = DBErrors::NEED_RESCAN; - } else if (fNoncriticalErrors && result == DBErrors::LOAD_OK) { - result = DBErrors::NONCRITICAL_ERROR; + // Load decryption keys + result = std::max(LoadDecryptionKeys(pwallet, *m_batch), result); + } catch (...) { + // Exceptions that can be ignored or treated as non-critical are handled by the individual loading functions. + // Any uncaught exceptions will be caught here and treated as critical. + result = DBErrors::CORRUPT; } // Any wallet corruption at all: skip any rewriting or @@ -932,29 +1193,13 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (result != DBErrors::LOAD_OK) return result; - pwallet->WalletLogPrintf("Keys: %u plaintext, %u encrypted, %u w/ metadata, %u total. Unknown wallet records: %u\n", - wss.nKeys, wss.nCKeys, wss.nKeyMeta, wss.nKeys + wss.nCKeys, wss.m_unknown_records); - - // nTimeFirstKey is only reliable if all keys have metadata - if (pwallet->IsLegacy() && (wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) { - auto spk_man = pwallet->GetOrCreateLegacyScriptPubKeyMan(); - if (spk_man) { - LOCK(spk_man->cs_KeyStore); - spk_man->UpdateTimeFirstKey(1); - } - } - - for (const uint256& hash : wss.vWalletUpgrade) + for (const uint256& hash : upgraded_txs) WriteTx(pwallet->mapWallet.at(hash)); - // Rewrite encrypted wallets of versions 0.4.0 and 0.5.0rc: - if (wss.fIsEncrypted && (last_client == 40000 || last_client == 50000)) - return DBErrors::NEED_REWRITE; - if (!has_last_client || last_client != CLIENT_VERSION) // Update m_batch->Write(DBKeys::VERSION, CLIENT_VERSION); - if (wss.fAnyUnordered) + if (any_unordered) result = pwallet->ReorderTransactions(); // Upgrade all of the wallet keymetadata to have the hd master key id @@ -973,20 +1218,6 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) result = DBErrors::CORRUPT; } - // Set the inactive chain - if (wss.m_hd_chains.size() > 0) { - LegacyScriptPubKeyMan* legacy_spkm = pwallet->GetLegacyScriptPubKeyMan(); - if (!legacy_spkm) { - pwallet->WalletLogPrintf("Inactive HD Chains found but no Legacy ScriptPubKeyMan\n"); - return DBErrors::CORRUPT; - } - for (const auto& chain_pair : wss.m_hd_chains) { - if (chain_pair.first != pwallet->GetLegacyScriptPubKeyMan()->GetHDChain().seed_id) { - pwallet->GetLegacyScriptPubKeyMan()->AddInactiveHDChain(chain_pair.second); - } - } - } - return result; } @@ -1161,7 +1392,7 @@ bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) } // Make a copy of key to avoid data being deleted by the following read of the type - Span<const unsigned char> key_data = MakeUCharSpan(key); + Span key_data{key}; std::string type; key >> type; diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index f84a89b23f..8f7c2f030c 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -42,19 +42,21 @@ struct WalletContext; static const bool DEFAULT_FLUSHWALLET = true; -/** Error statuses for the wallet database */ -enum class DBErrors +/** Error statuses for the wallet database. + * Values are in order of severity. When multiple errors occur, the most severe (highest value) will be returned. + */ +enum class DBErrors : int { - LOAD_OK, - CORRUPT, - NONCRITICAL_ERROR, - TOO_NEW, - EXTERNAL_SIGNER_SUPPORT_REQUIRED, - LOAD_FAIL, - NEED_REWRITE, - NEED_RESCAN, - UNKNOWN_DESCRIPTOR, - UNEXPECTED_LEGACY_ENTRY + LOAD_OK = 0, + NEED_RESCAN = 1, + NEED_REWRITE = 2, + EXTERNAL_SIGNER_SUPPORT_REQUIRED = 3, + NONCRITICAL_ERROR = 4, + TOO_NEW = 5, + UNKNOWN_DESCRIPTOR = 6, + LOAD_FAIL = 7, + UNEXPECTED_LEGACY_ENTRY = 8, + CORRUPT = 9, }; namespace DBKeys { @@ -276,8 +278,6 @@ public: DBErrors LoadWallet(CWallet* pwallet); DBErrors FindWalletTxHashes(std::vector<uint256>& tx_hashes); DBErrors ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut); - /* Function to determine if a certain KV/key-type is a key (cryptographical key) type */ - static bool IsKeyType(const std::string& strType); //! write the hdchain model (external chain child index counter) bool WriteHDChain(const CHDChain& chain); @@ -300,11 +300,10 @@ private: //! Compacts BDB state so that wallet.dat is self-contained (if there are changes) void MaybeCompactWalletDB(WalletContext& context); -//! Callback for filtering key types to deserialize in ReadKeyValue -using KeyFilterFn = std::function<bool(const std::string&)>; - -//! Unserialize a given Key-Value pair and load it into the wallet -bool ReadKeyValue(CWallet* pwallet, DataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr, const KeyFilterFn& filter_fn = nullptr); +bool LoadKey(CWallet* pwallet, DataStream& ssKey, DataStream& ssValue, std::string& strErr); +bool LoadCryptedKey(CWallet* pwallet, DataStream& ssKey, DataStream& ssValue, std::string& strErr); +bool LoadEncryptionKey(CWallet* pwallet, DataStream& ssKey, DataStream& ssValue, std::string& strErr); +bool LoadHDChain(CWallet* pwallet, DataStream& ssValue, std::string& strErr); } // namespace wallet #endif // BITCOIN_WALLET_WALLETDB_H diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index f639078de5..c5975144c1 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -53,10 +53,18 @@ enum WalletFlags : uint64_t { //! Flag set when a wallet contains no HD seed and no private keys, scripts, //! addresses, and other watch only things, and is therefore "blank." //! - //! The only function this flag serves is to distinguish a blank wallet from + //! The main function this flag serves is to distinguish a blank wallet from //! a newly created wallet when the wallet database is loaded, to avoid //! initialization that should only happen on first run. //! + //! A secondary function of this flag, which applies to descriptor wallets + //! only, is to serve as an ongoing indication that descriptors in the + //! wallet should be created manually, and that the wallet should not + //! generate automatically generate new descriptors if it is later + //! encrypted. To support this behavior, descriptor wallets unlike legacy + //! wallets do not automatically unset the BLANK flag when things are + //! imported. + //! //! This flag is also a mandatory flag to prevent previous versions of //! bitcoin from opening the wallet, thinking it was newly created, and //! then improperly reinitializing it. @@ -104,20 +112,6 @@ public: WalletDescriptor() {} WalletDescriptor(std::shared_ptr<Descriptor> descriptor, uint64_t creation_time, int32_t range_start, int32_t range_end, int32_t next_index) : descriptor(descriptor), creation_time(creation_time), range_start(range_start), range_end(range_end), next_index(next_index) {} }; - -class CWallet; -class DescriptorScriptPubKeyMan; - -/** struct containing information needed for migrating legacy wallets to descriptor wallets */ -struct MigrationData -{ - CExtKey master_key; - std::vector<std::pair<std::string, int64_t>> watch_descs; - std::vector<std::pair<std::string, int64_t>> solvable_descs; - std::vector<std::unique_ptr<DescriptorScriptPubKeyMan>> desc_spkms; - std::shared_ptr<CWallet> watchonly_wallet{nullptr}; - std::shared_ptr<CWallet> solvable_wallet{nullptr}; -}; } // namespace wallet #endif // BITCOIN_WALLET_WALLETUTIL_H diff --git a/src/warnings.cpp b/src/warnings.cpp index d0de706953..cb73c7aea2 100644 --- a/src/warnings.cpp +++ b/src/warnings.cpp @@ -5,9 +5,9 @@ #include <warnings.h> +#include <common/system.h> #include <sync.h> #include <util/string.h> -#include <util/system.h> #include <util/translation.h> #include <vector> diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 21aa44c309..1241431523 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -97,9 +97,8 @@ static bool IsZMQAddressIPV6(const std::string &zmq_address) const size_t colon_index = zmq_address.rfind(':'); if (tcp_index == 0 && colon_index != std::string::npos) { const std::string ip = zmq_address.substr(tcp_prefix.length(), colon_index - tcp_prefix.length()); - CNetAddr addr; - LookupHost(ip, addr, false); - if (addr.IsIPv6()) return true; + const std::optional<CNetAddr> addr{LookupHost(ip, false)}; + if (addr.has_value() && addr.value().IsIPv6()) return true; } return false; } diff --git a/test/README.md b/test/README.md index 0eddb72e1f..fee8828d34 100644 --- a/test/README.md +++ b/test/README.md @@ -331,6 +331,7 @@ Use the `-v` option for verbose output. | Lint test | Dependency | |-----------|:----------:| | [`lint-python.py`](lint/lint-python.py) | [flake8](https://gitlab.com/pycqa/flake8) +| [`lint-python.py`](lint/lint-python.py) | [lief](https://github.com/lief-project/LIEF) | [`lint-python.py`](lint/lint-python.py) | [mypy](https://github.com/python/mypy) | [`lint-python.py`](lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq) | [`lint-python-dead-code.py`](lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) diff --git a/test/config.ini.in b/test/config.ini.in index 5888ef443b..291599da45 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -22,8 +22,7 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @BUILD_BITCOIN_UTIL_TRUE@ENABLE_BITCOIN_UTIL=true @BUILD_BITCOIN_WALLET_TRUE@ENABLE_WALLET_TOOL=true @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true -@ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true +@ENABLE_FUZZ_BINARY_TRUE@ENABLE_FUZZ_BINARY=true @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true @ENABLE_EXTERNAL_SIGNER_TRUE@ENABLE_EXTERNAL_SIGNER=true -@ENABLE_SYSCALL_SANDBOX_TRUE@ENABLE_SYSCALL_SANDBOX=true @ENABLE_USDT_TRACEPOINTS_TRUE@ENABLE_USDT_TRACEPOINTS=true diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index fa1bb6506a..afee9597ad 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -9,10 +9,7 @@ - Mine a fork that requires disconnecting the tip. - Verify that bitcoind AbortNode's. """ - from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import get_datadir_path -import os class AbortNodeTest(BitcoinTestFramework): @@ -26,10 +23,9 @@ class AbortNodeTest(BitcoinTestFramework): def run_test(self): self.generate(self.nodes[0], 3, sync_fun=self.no_op) - datadir = get_datadir_path(self.options.tmpdir, 0) # Deleting the undo file will result in reorg failure - os.unlink(os.path.join(datadir, self.chain, 'blocks', 'rev00000.dat')) + (self.nodes[0].chain_path / "blocks" / "rev00000.dat").unlink() # Connecting to a node with a more work chain will trigger a reorg # attempt. @@ -40,7 +36,7 @@ class AbortNodeTest(BitcoinTestFramework): # Check that node0 aborted self.log.info("Waiting for crash") - self.nodes[0].wait_until_stopped(timeout=5) + self.nodes[0].wait_until_stopped(timeout=5, expect_error=True) self.log.info("Node crashed - now verifying restart fails") self.nodes[0].assert_start_raises_init_error() diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py index 28c3880513..7877f9d302 100755 --- a/test/functional/feature_addrman.py +++ b/test/functional/feature_addrman.py @@ -53,7 +53,7 @@ class AddrmanTest(BitcoinTestFramework): self.num_nodes = 1 def run_test(self): - peers_dat = os.path.join(self.nodes[0].datadir, self.chain, "peers.dat") + peers_dat = os.path.join(self.nodes[0].chain_path, "peers.dat") init_error = lambda reason: ( f"Error: Invalid or corrupt peers.dat \\({reason}\\). If you believe this " f"is a bug, please report it to {self.config['environment']['PACKAGE_BUGREPORT']}. " diff --git a/test/functional/feature_anchors.py b/test/functional/feature_anchors.py index 468ad1eafa..0961f21a40 100755 --- a/test/functional/feature_anchors.py +++ b/test/functional/feature_anchors.py @@ -20,9 +20,7 @@ class AnchorsTest(BitcoinTestFramework): self.disable_autoconnect = False def run_test(self): - node_anchors_path = os.path.join( - self.nodes[0].datadir, "regtest", "anchors.dat" - ) + node_anchors_path = self.nodes[0].chain_path / "anchors.dat" self.log.info("When node starts, check if anchors.dat doesn't exist") assert not os.path.exists(node_anchors_path) diff --git a/test/functional/feature_asmap.py b/test/functional/feature_asmap.py index 9440ba11f5..9cff8042a8 100755 --- a/test/functional/feature_asmap.py +++ b/test/functional/feature_asmap.py @@ -113,7 +113,7 @@ class AsmapTest(BitcoinTestFramework): def run_test(self): self.node = self.nodes[0] - self.datadir = os.path.join(self.node.datadir, self.chain) + self.datadir = self.node.chain_path self.default_asmap = os.path.join(self.datadir, DEFAULT_ASMAP_FILENAME) self.asmap_raw = os.path.join(os.path.dirname(os.path.realpath(__file__)), ASMAP) diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index 36ee79dab9..613d2eab14 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -35,7 +35,6 @@ from test_framework.blocktools import ( create_block, create_coinbase, ) -from test_framework.key import ECKey from test_framework.messages import ( CBlockHeader, COutPoint, @@ -46,9 +45,13 @@ from test_framework.messages import ( msg_headers, ) from test_framework.p2p import P2PInterface -from test_framework.script import (CScript, OP_TRUE) +from test_framework.script import ( + CScript, + OP_TRUE, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal +from test_framework.wallet_util import generate_keypair class BaseNode(P2PInterface): @@ -90,9 +93,7 @@ class AssumeValidTest(BitcoinTestFramework): self.blocks = [] # Get a pubkey for the coinbase TXO - coinbase_key = ECKey() - coinbase_key.generate() - coinbase_pubkey = coinbase_key.get_pubkey().get_bytes() + _, coinbase_pubkey = generate_keypair() # Create the first block with a coinbase output to our key height = 1 diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 1080e77c40..765db97445 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -14,7 +14,6 @@ from test_framework.blocktools import ( get_legacy_sigopcount_block, MAX_BLOCK_SIGOPS, ) -from test_framework.key import ECKey from test_framework.messages import ( CBlock, COIN, @@ -55,6 +54,7 @@ from test_framework.util import ( assert_equal, assert_greater_than, ) +from test_framework.wallet_util import generate_keypair from data import invalid_txs @@ -98,9 +98,7 @@ class FullBlockTest(BitcoinTestFramework): self.bootstrap_p2p() # Add one p2p connection to the node self.block_heights = {} - self.coinbase_key = ECKey() - self.coinbase_key.generate() - self.coinbase_pubkey = self.coinbase_key.get_pubkey().get_bytes() + self.coinbase_key, self.coinbase_pubkey = generate_keypair() self.tip = None self.blocks = {} self.genesis_hash = int(self.nodes[0].getbestblockhash(), 16) diff --git a/test/functional/feature_blocksdir.py b/test/functional/feature_blocksdir.py index e8d2ec3676..99763ab97f 100755 --- a/test/functional/feature_blocksdir.py +++ b/test/functional/feature_blocksdir.py @@ -18,7 +18,7 @@ class BlocksdirTest(BitcoinTestFramework): def run_test(self): self.stop_node(0) - assert os.path.isdir(os.path.join(self.nodes[0].datadir, self.chain, "blocks")) + assert os.path.isdir(os.path.join(self.nodes[0].chain_path, "blocks")) assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "blocks")) shutil.rmtree(self.nodes[0].datadir) initialize_datadir(self.options.tmpdir, 0, self.chain) @@ -31,7 +31,7 @@ class BlocksdirTest(BitcoinTestFramework): self.log.info("mining blocks..") self.generatetoaddress(self.nodes[0], 10, self.nodes[0].get_deterministic_priv_key().address) assert os.path.isfile(os.path.join(blocksdir_path, self.chain, "blocks", "blk00000.dat")) - assert os.path.isdir(os.path.join(self.nodes[0].datadir, self.chain, "blocks", "index")) + assert os.path.isdir(os.path.join(self.nodes[0].chain_path, "blocks", "index")) if __name__ == '__main__': diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index f9730b48c5..2927355bda 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -5,9 +5,14 @@ """Test various command line arguments and configuration file parameters.""" import os +import pathlib +import re +import sys +import tempfile import time from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_node import ErrorMatch from test_framework import util @@ -27,7 +32,7 @@ class ConfArgsTest(BitcoinTestFramework): self.stop_node(0) # Check that startup fails if conf= is set in bitcoin.conf or in an included conf file - bad_conf_file_path = os.path.join(self.options.tmpdir, 'node0', 'bitcoin_bad.conf') + bad_conf_file_path = self.nodes[0].datadir_path / "bitcoin_bad.conf" util.write_config(bad_conf_file_path, n=0, chain='', extra_config=f'conf=some.conf\n') conf_in_config_file_err = 'Error: Error reading configuration file: conf cannot be set in the configuration file; use includeconf= if you want to include additional config files' self.nodes[0].assert_start_raises_init_error( @@ -70,11 +75,11 @@ class ConfArgsTest(BitcoinTestFramework): conf.write("wallet=foo\n") self.nodes[0].assert_start_raises_init_error(expected_msg=f'Error: Config setting for -wallet only applied on {self.chain} network when in [{self.chain}] section.') - main_conf_file_path = os.path.join(self.options.tmpdir, 'node0', 'bitcoin_main.conf') + main_conf_file_path = self.nodes[0].datadir_path / "bitcoin_main.conf" util.write_config(main_conf_file_path, n=0, chain='', extra_config=f'includeconf={inc_conf_file_path}\n') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('acceptnonstdtxn=1\n') - self.nodes[0].assert_start_raises_init_error(extra_args=[f"-conf={main_conf_file_path}"], expected_msg='Error: acceptnonstdtxn is not currently supported for main chain') + self.nodes[0].assert_start_raises_init_error(extra_args=[f"-conf={main_conf_file_path}", "-allowignoredconf"], expected_msg='Error: acceptnonstdtxn is not currently supported for main chain') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('nono\n') @@ -108,6 +113,41 @@ class ConfArgsTest(BitcoinTestFramework): with open(inc_conf_file2_path, 'w', encoding='utf-8') as conf: conf.write('') # clear + def test_config_file_log(self): + # Disable this test for windows currently because trying to override + # the default datadir through the environment does not seem to work. + if sys.platform == "win32": + return + + self.log.info('Test that correct configuration path is changed when configuration file changes the datadir') + + # Create a temporary directory that will be treated as the default data + # directory by bitcoind. + env, default_datadir = util.get_temp_default_datadir(pathlib.Path(self.options.tmpdir, "test_config_file_log")) + default_datadir.mkdir(parents=True) + + # Write a bitcoin.conf file in the default data directory containing a + # datadir= line pointing at the node datadir. + node = self.nodes[0] + conf_text = pathlib.Path(node.bitcoinconf).read_text() + conf_path = default_datadir / "bitcoin.conf" + conf_path.write_text(f"datadir={node.datadir}\n{conf_text}") + + # Drop the node -datadir= argument during this test, because if it is + # specified it would take precedence over the datadir setting in the + # config file. + node_args = node.args + node.args = [arg for arg in node.args if not arg.startswith("-datadir=")] + + # Check that correct configuration file path is actually logged + # (conf_path, not node.bitcoinconf) + with self.nodes[0].assert_debug_log(expected_msgs=[f"Config file: {conf_path}"]): + self.start_node(0, ["-allowignoredconf"], env=env) + self.stop_node(0) + + # Restore node arguments after the test + node.args = node_args + def test_invalid_command_line_options(self): self.nodes[0].assert_start_raises_init_error( expected_msg='Error: Error parsing command line arguments: Can not set -proxy with no value. Please specify value with -proxy=value.', @@ -213,7 +253,7 @@ class ConfArgsTest(BitcoinTestFramework): with self.nodes[0].assert_debug_log(expected_msgs=[ "Loaded 0 addresses from peers.dat", "DNS seeding disabled", - "Adding fixed seeds as -dnsseed=0 (or IPv4/IPv6 connections are disabled via -onlynet), -addnode is not provided and all -seednode(s) attempted\n", + "Adding fixed seeds as -dnsseed=0 (or IPv4/IPv6 connections are disabled via -onlynet) and neither -addnode nor -seednode are provided\n", ]): self.start_node(0, extra_args=['-dnsseed=0', '-fixedseeds=1']) assert time.time() - start < 60 @@ -282,6 +322,55 @@ class ConfArgsTest(BitcoinTestFramework): unexpected_msgs=seednode_ignored): self.restart_node(0, extra_args=[connect_arg, '-seednode=fakeaddress2']) + def test_ignored_conf(self): + self.log.info('Test error is triggered when the datadir in use contains a bitcoin.conf file that would be ignored ' + 'because a conflicting -conf file argument is passed.') + node = self.nodes[0] + with tempfile.NamedTemporaryFile(dir=self.options.tmpdir, mode="wt", delete=False) as temp_conf: + temp_conf.write(f"datadir={node.datadir}\n") + node.assert_start_raises_init_error([f"-conf={temp_conf.name}"], re.escape( + f'Error: Data directory "{node.datadir}" contains a "bitcoin.conf" file which is ignored, because a ' + f'different configuration file "{temp_conf.name}" from command line argument "-conf={temp_conf.name}" ' + f'is being used instead.') + r"[\s\S]*", match=ErrorMatch.FULL_REGEX) + + # Test that passing a redundant -conf command line argument pointing to + # the same bitcoin.conf that would be loaded anyway does not trigger an + # error. + self.start_node(0, [f'-conf={node.datadir}/bitcoin.conf']) + self.stop_node(0) + + def test_ignored_default_conf(self): + # Disable this test for windows currently because trying to override + # the default datadir through the environment does not seem to work. + if sys.platform == "win32": + return + + self.log.info('Test error is triggered when bitcoin.conf in the default data directory sets another datadir ' + 'and it contains a different bitcoin.conf file that would be ignored') + + # Create a temporary directory that will be treated as the default data + # directory by bitcoind. + env, default_datadir = util.get_temp_default_datadir(pathlib.Path(self.options.tmpdir, "home")) + default_datadir.mkdir(parents=True) + + # Write a bitcoin.conf file in the default data directory containing a + # datadir= line pointing at the node datadir. This will trigger a + # startup error because the node datadir contains a different + # bitcoin.conf that would be ignored. + node = self.nodes[0] + (default_datadir / "bitcoin.conf").write_text(f"datadir={node.datadir}\n") + + # Drop the node -datadir= argument during this test, because if it is + # specified it would take precedence over the datadir setting in the + # config file. + node_args = node.args + node.args = [arg for arg in node.args if not arg.startswith("-datadir=")] + node.assert_start_raises_init_error([], re.escape( + f'Error: Data directory "{node.datadir}" contains a "bitcoin.conf" file which is ignored, because a ' + f'different configuration file "{default_datadir}/bitcoin.conf" from data directory "{default_datadir}" ' + f'is being used instead.') + r"[\s\S]*", env=env, match=ErrorMatch.FULL_REGEX) + node.args = node_args + def run_test(self): self.test_log_buffer() self.test_args_log() @@ -290,7 +379,10 @@ class ConfArgsTest(BitcoinTestFramework): self.test_connect_with_seednode() self.test_config_file_parser() + self.test_config_file_log() self.test_invalid_command_line_options() + self.test_ignored_conf() + self.test_ignored_default_conf() # Remove the -datadir argument so it doesn't override the config file self.nodes[0].args = [arg for arg in self.nodes[0].args if not arg.startswith("-datadir")] diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 05ee556ece..4f56d585d3 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -7,6 +7,7 @@ from copy import deepcopy from decimal import Decimal import os import random +import time from test_framework.messages import ( COIN, @@ -21,6 +22,8 @@ from test_framework.util import ( ) from test_framework.wallet import MiniWallet +MAX_FILE_AGE = 60 +SECONDS_PER_HOUR = 60 * 60 def small_txpuzzle_randfee( wallet, from_node, conflist, unconflist, amount, min_fee, fee_increment, batch_reqs @@ -290,6 +293,95 @@ class EstimateFeeTest(BitcoinTestFramework): est_feerate = node.estimatesmartfee(2)["feerate"] assert_equal(est_feerate, high_feerate_kvb) + def test_old_fee_estimate_file(self): + # Get the initial fee rate while node is running + fee_rate = self.nodes[0].estimatesmartfee(1)["feerate"] + + # Restart node to ensure fee_estimate.dat file is read + self.restart_node(0) + assert_equal(self.nodes[0].estimatesmartfee(1)["feerate"], fee_rate) + + fee_dat = self.nodes[0].chain_path / "fee_estimates.dat" + + # Stop the node and backdate the fee_estimates.dat file more than MAX_FILE_AGE + self.stop_node(0) + last_modified_time = time.time() - (MAX_FILE_AGE + 1) * SECONDS_PER_HOUR + os.utime(fee_dat, (last_modified_time, last_modified_time)) + + # Start node and ensure the fee_estimates.dat file was not read + self.start_node(0) + assert_equal(self.nodes[0].estimatesmartfee(1)["errors"], ["Insufficient data or no feerate found"]) + + + def test_estimate_dat_is_flushed_periodically(self): + fee_dat = self.nodes[0].chain_path / "fee_estimates.dat" + os.remove(fee_dat) if os.path.exists(fee_dat) else None + + # Verify that fee_estimates.dat does not exist + assert_equal(os.path.isfile(fee_dat), False) + + # Verify if the string "Flushed fee estimates to fee_estimates.dat." is present in the debug log file. + # If present, it indicates that fee estimates have been successfully flushed to disk. + with self.nodes[0].assert_debug_log(expected_msgs=["Flushed fee estimates to fee_estimates.dat."], timeout=1): + # Mock the scheduler for an hour to flush fee estimates to fee_estimates.dat + self.nodes[0].mockscheduler(SECONDS_PER_HOUR) + + # Verify that fee estimates were flushed and fee_estimates.dat file is created + assert_equal(os.path.isfile(fee_dat), True) + + # Verify that the estimates remain the same if there are no blocks in the flush interval + block_hash_before = self.nodes[0].getbestblockhash() + fee_dat_initial_content = open(fee_dat, "rb").read() + with self.nodes[0].assert_debug_log(expected_msgs=["Flushed fee estimates to fee_estimates.dat."], timeout=1): + # Mock the scheduler for an hour to flush fee estimates to fee_estimates.dat + self.nodes[0].mockscheduler(SECONDS_PER_HOUR) + + # Verify that there were no blocks in between the flush interval + assert_equal(block_hash_before, self.nodes[0].getbestblockhash()) + + fee_dat_current_content = open(fee_dat, "rb").read() + assert_equal(fee_dat_current_content, fee_dat_initial_content) + + # Verify that the estimates remain the same after shutdown with no blocks before shutdown + self.restart_node(0) + fee_dat_current_content = open(fee_dat, "rb").read() + assert_equal(fee_dat_current_content, fee_dat_initial_content) + + # Verify that the estimates are not the same if new blocks were produced in the flush interval + with self.nodes[0].assert_debug_log(expected_msgs=["Flushed fee estimates to fee_estimates.dat."], timeout=1): + # Mock the scheduler for an hour to flush fee estimates to fee_estimates.dat + self.generate(self.nodes[0], 5, sync_fun=self.no_op) + self.nodes[0].mockscheduler(SECONDS_PER_HOUR) + + fee_dat_current_content = open(fee_dat, "rb").read() + assert fee_dat_current_content != fee_dat_initial_content + + fee_dat_initial_content = fee_dat_current_content + + # Generate blocks before shutdown and verify that the fee estimates are not the same + self.generate(self.nodes[0], 5, sync_fun=self.no_op) + self.restart_node(0) + fee_dat_current_content = open(fee_dat, "rb").read() + assert fee_dat_current_content != fee_dat_initial_content + + + def test_acceptstalefeeestimates_option(self): + # Get the initial fee rate while node is running + fee_rate = self.nodes[0].estimatesmartfee(1)["feerate"] + + self.stop_node(0) + + fee_dat = self.nodes[0].chain_path / "fee_estimates.dat" + + # Stop the node and backdate the fee_estimates.dat file more than MAX_FILE_AGE + last_modified_time = time.time() - (MAX_FILE_AGE + 1) * SECONDS_PER_HOUR + os.utime(fee_dat, (last_modified_time, last_modified_time)) + + # Restart node with -acceptstalefeeestimates option to ensure fee_estimate.dat file is read + self.start_node(0,extra_args=["-acceptstalefeeestimates"]) + assert_equal(self.nodes[0].estimatesmartfee(1)["feerate"], fee_rate) + + def run_test(self): self.log.info("This test is time consuming, please be patient") self.log.info("Splitting inputs so we can generate tx's") @@ -312,15 +404,24 @@ class EstimateFeeTest(BitcoinTestFramework): self.log.info("Testing estimates with single transactions.") self.sanity_check_estimates_range() + self.log.info("Test fee_estimates.dat is flushed periodically") + self.test_estimate_dat_is_flushed_periodically() + # check that the effective feerate is greater than or equal to the mempoolminfee even for high mempoolminfee self.log.info( "Test fee rate estimation after restarting node with high MempoolMinFee" ) self.test_feerate_mempoolminfee() + self.log.info("Test acceptstalefeeestimates option") + self.test_acceptstalefeeestimates_option() + + self.log.info("Test reading old fee_estimates.dat") + self.test_old_fee_estimate_file() + self.log.info("Restarting node with fresh estimation") self.stop_node(0) - fee_dat = os.path.join(self.nodes[0].datadir, self.chain, "fee_estimates.dat") + fee_dat = os.path.join(self.nodes[0].chain_path, "fee_estimates.dat") os.remove(fee_dat) self.start_node(0) self.connect_nodes(0, 1) diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index bb4104bf8e..cf2f21d553 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -3,7 +3,6 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Check that it's not possible to start a second bitcoind instance using the same datadir or wallet.""" -import os import random import string @@ -24,7 +23,7 @@ class FilelockTest(BitcoinTestFramework): self.nodes[0].wait_for_rpc_connection() def run_test(self): - datadir = os.path.join(self.nodes[0].datadir, self.chain) + datadir = self.nodes[0].chain_path self.log.info(f"Using datadir {datadir}") self.log.info("Check that we can't start a second bitcoind instance using the same datadir") @@ -35,7 +34,7 @@ class FilelockTest(BitcoinTestFramework): def check_wallet_filelock(descriptors): wallet_name = ''.join([random.choice(string.ascii_lowercase) for _ in range(6)]) self.nodes[0].createwallet(wallet_name=wallet_name, descriptors=descriptors) - wallet_dir = os.path.join(datadir, 'wallets') + wallet_dir = self.nodes[0].wallets_path self.log.info("Check that we can't start a second bitcoind instance using the same wallet") if descriptors: expected_msg = f"Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['PACKAGE_NAME']}?" diff --git a/test/functional/feature_includeconf.py b/test/functional/feature_includeconf.py index 818e4c923b..58ab063e71 100755 --- a/test/functional/feature_includeconf.py +++ b/test/functional/feature_includeconf.py @@ -14,27 +14,25 @@ Verify that: 4. multiple includeconf arguments can be specified in the main config file. """ -import os - from test_framework.test_framework import BitcoinTestFramework + class IncludeConfTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - def setup_chain(self): - super().setup_chain() + def run_test(self): # Create additional config files # - tmpdir/node0/relative.conf - with open(os.path.join(self.options.tmpdir, "node0", "relative.conf"), "w", encoding="utf8") as f: + with open(self.nodes[0].datadir_path / "relative.conf", "w", encoding="utf8") as f: f.write("uacomment=relative\n") # - tmpdir/node0/relative2.conf - with open(os.path.join(self.options.tmpdir, "node0", "relative2.conf"), "w", encoding="utf8") as f: + with open(self.nodes[0].datadir_path / "relative2.conf", "w", encoding="utf8") as f: f.write("uacomment=relative2\n") - with open(os.path.join(self.options.tmpdir, "node0", "bitcoin.conf"), "a", encoding='utf8') as f: + with open(self.nodes[0].datadir_path / "bitcoin.conf", "a", encoding="utf8") as f: f.write("uacomment=main\nincludeconf=relative.conf\n") + self.restart_node(0) - def run_test(self): self.log.info("-includeconf works from config file. subversion should end with 'main; relative)/'") subversion = self.nodes[0].getnetworkinfo()["subversion"] @@ -52,7 +50,7 @@ class IncludeConfTest(BitcoinTestFramework): ) self.log.info("-includeconf cannot be used recursively. subversion should end with 'main; relative)/'") - with open(os.path.join(self.options.tmpdir, "node0", "relative.conf"), "a", encoding="utf8") as f: + with open(self.nodes[0].datadir_path / "relative.conf", "a", encoding="utf8") as f: f.write("includeconf=relative2.conf\n") self.start_node(0) @@ -63,20 +61,20 @@ class IncludeConfTest(BitcoinTestFramework): self.log.info("-includeconf cannot contain invalid arg") # Commented out as long as we ignore invalid arguments in configuration files - #with open(os.path.join(self.options.tmpdir, "node0", "relative.conf"), "w", encoding="utf8") as f: + #with open(self.nodes[0].datadir_path / "relative.conf", "w", encoding="utf8") as f: # f.write("foo=bar\n") #self.nodes[0].assert_start_raises_init_error(expected_msg="Error: Error reading configuration file: Invalid configuration value foo") self.log.info("-includeconf cannot be invalid path") - os.remove(os.path.join(self.options.tmpdir, "node0", "relative.conf")) + (self.nodes[0].datadir_path / "relative.conf").unlink() self.nodes[0].assert_start_raises_init_error(expected_msg="Error: Error reading configuration file: Failed to include configuration file relative.conf") self.log.info("multiple -includeconf args can be used from the base config file. subversion should end with 'main; relative; relative2)/'") - with open(os.path.join(self.options.tmpdir, "node0", "relative.conf"), "w", encoding="utf8") as f: + with open(self.nodes[0].datadir_path / "relative.conf", "w", encoding="utf8") as f: # Restore initial file contents f.write("uacomment=relative\n") - with open(os.path.join(self.options.tmpdir, "node0", "bitcoin.conf"), "a", encoding='utf8') as f: + with open(self.nodes[0].datadir_path / "bitcoin.conf", "a", encoding="utf8") as f: f.write("includeconf=relative2.conf\n") self.start_node(0) diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py index 7f030c6773..c90ccc4936 100755 --- a/test/functional/feature_loadblock.py +++ b/test/functional/feature_loadblock.py @@ -37,7 +37,7 @@ class LoadblockTest(BitcoinTestFramework): cfg_file = os.path.join(data_dir, "linearize.cfg") bootstrap_file = os.path.join(self.options.tmpdir, "bootstrap.dat") genesis_block = self.nodes[0].getblockhash(0) - blocks_dir = os.path.join(data_dir, self.chain, "blocks") + blocks_dir = self.nodes[0].chain_path / "blocks" hash_list = tempfile.NamedTemporaryFile(dir=data_dir, mode='w', delete=False, diff --git a/test/functional/feature_logging.py b/test/functional/feature_logging.py index fe4f02dfe6..0e9aca358d 100755 --- a/test/functional/feature_logging.py +++ b/test/functional/feature_logging.py @@ -16,7 +16,7 @@ class LoggingTest(BitcoinTestFramework): self.setup_clean_chain = True def relative_log_path(self, name): - return os.path.join(self.nodes[0].datadir, self.chain, name) + return os.path.join(self.nodes[0].chain_path, name) def run_test(self): # test default log file name @@ -69,6 +69,36 @@ class LoggingTest(BitcoinTestFramework): # just sanity check no crash here self.restart_node(0, [f"-debuglogfile={os.devnull}"]) + self.log.info("Test -debug and -debugexclude raise when invalid values are passed") + self.stop_node(0) + self.nodes[0].assert_start_raises_init_error( + extra_args=["-debug=abc"], + expected_msg="Error: Unsupported logging category -debug=abc.", + match=ErrorMatch.FULL_REGEX, + ) + self.nodes[0].assert_start_raises_init_error( + extra_args=["-debugexclude=abc"], + expected_msg="Error: Unsupported logging category -debugexclude=abc.", + match=ErrorMatch.FULL_REGEX, + ) + + self.log.info("Test -loglevel raises when invalid values are passed") + self.nodes[0].assert_start_raises_init_error( + extra_args=["-loglevel=abc"], + expected_msg="Error: Unsupported global logging level -loglevel=abc. Valid values: info, debug, trace.", + match=ErrorMatch.FULL_REGEX, + ) + self.nodes[0].assert_start_raises_init_error( + extra_args=["-loglevel=net:abc"], + expected_msg="Error: Unsupported category-specific logging level -loglevel=net:abc.", + match=ErrorMatch.PARTIAL_REGEX, + ) + self.nodes[0].assert_start_raises_init_error( + extra_args=["-loglevel=net:info:abc"], + expected_msg="Error: Unsupported category-specific logging level -loglevel=net:info:abc.", + match=ErrorMatch.PARTIAL_REGEX, + ) + if __name__ == '__main__': LoggingTest().main() diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index 8cb633d454..adf6c13973 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -30,9 +30,6 @@ class NotificationsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True - # The experimental syscall sandbox feature (-sandbox) is not compatible with -alertnotify, - # -blocknotify, -walletnotify or -shutdownnotify (which all invoke execve). - self.disable_syscall_sandbox = True def setup_network(self): self.wallet = ''.join(chr(i) for i in range(FILE_CHAR_START, FILE_CHAR_END) if chr(i) not in FILE_CHARS_DISALLOWED) diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index c95657dbbb..7b2a29bdb4 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -35,8 +35,7 @@ from test_framework.util import ( assert_raises_rpc_error, ) from test_framework.wallet import getnewdestination -from test_framework.key import ECKey -from test_framework.wallet_util import bytes_to_wif +from test_framework.wallet_util import generate_keypair NULLDUMMY_ERROR = "non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero)" @@ -71,12 +70,9 @@ class NULLDUMMYTest(BitcoinTestFramework): return tx_from_hex(signedtx["hex"]) def run_test(self): - eckey = ECKey() - eckey.generate() - self.privkey = bytes_to_wif(eckey.get_bytes()) - self.pubkey = eckey.get_pubkey().get_bytes().hex() - cms = self.nodes[0].createmultisig(1, [self.pubkey]) - wms = self.nodes[0].createmultisig(1, [self.pubkey], 'p2sh-segwit') + self.privkey, self.pubkey = generate_keypair(wif=True) + cms = self.nodes[0].createmultisig(1, [self.pubkey.hex()]) + wms = self.nodes[0].createmultisig(1, [self.pubkey.hex()], 'p2sh-segwit') self.ms_address = cms["address"] ms_unlock_details = {"scriptPubKey": address_to_scriptpubkey(self.ms_address).hex(), "redeemScript": cms["redeemScript"]} diff --git a/test/functional/feature_posix_fs_permissions.py b/test/functional/feature_posix_fs_permissions.py index c5a543e97a..40528779e6 100755 --- a/test/functional/feature_posix_fs_permissions.py +++ b/test/functional/feature_posix_fs_permissions.py @@ -31,11 +31,11 @@ class PosixFsPermissionsTest(BitcoinTestFramework): def run_test(self): self.stop_node(0) - datadir = os.path.join(self.nodes[0].datadir, self.chain) + datadir = self.nodes[0].chain_path self.check_directory_permissions(datadir) - walletsdir = os.path.join(datadir, "wallets") + walletsdir = self.nodes[0].wallets_path self.check_directory_permissions(walletsdir) - debuglog = os.path.join(datadir, "debug.log") + debuglog = self.nodes[0].debug_log_path self.check_file_permissions(debuglog) diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index b0c6138bcf..15dd4827ae 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -91,7 +91,7 @@ class PruneTest(BitcoinTestFramework): def setup_network(self): self.setup_nodes() - self.prunedir = os.path.join(self.nodes[2].datadir, self.chain, 'blocks', '') + self.prunedir = os.path.join(self.nodes[2].chain_path, 'blocks', '') self.connect_nodes(0, 1) self.connect_nodes(1, 2) @@ -290,7 +290,7 @@ class PruneTest(BitcoinTestFramework): assert_equal(ret + 1, node.getblockchaininfo()['pruneheight']) def has_block(index): - return os.path.isfile(os.path.join(self.nodes[node_number].datadir, self.chain, "blocks", f"blk{index:05}.dat")) + return os.path.isfile(os.path.join(self.nodes[node_number].chain_path, "blocks", f"blk{index:05}.dat")) # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000) assert_raises_rpc_error(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500)) diff --git a/test/functional/feature_reindex.py b/test/functional/feature_reindex.py index 0f6a8fd0d2..fcbb49d420 100755 --- a/test/functional/feature_reindex.py +++ b/test/functional/feature_reindex.py @@ -10,7 +10,6 @@ - Verify that out-of-order blocks are correctly processed, see LoadExternalBlockFile() """ -import os from test_framework.test_framework import BitcoinTestFramework from test_framework.p2p import MAGIC_BYTES from test_framework.util import assert_equal @@ -39,7 +38,7 @@ class ReindexTest(BitcoinTestFramework): # In this test environment, blocks will always be in order (since # we're generating them rather than getting them from peers), so to # test out-of-order handling, swap blocks 1 and 2 on disk. - blk0 = os.path.join(self.nodes[0].datadir, self.nodes[0].chain, 'blocks', 'blk00000.dat') + blk0 = self.nodes[0].chain_path / "blocks" / "blk00000.dat" with open(blk0, 'r+b') as bf: # Read at least the first few blocks (including genesis) b = bf.read(2000) diff --git a/test/functional/feature_remove_pruned_files_on_startup.py b/test/functional/feature_remove_pruned_files_on_startup.py index ca0e5ace9f..a55e08ef1a 100755 --- a/test/functional/feature_remove_pruned_files_on_startup.py +++ b/test/functional/feature_remove_pruned_files_on_startup.py @@ -20,10 +20,10 @@ class FeatureRemovePrunedFilesOnStartupTest(BitcoinTestFramework): self.sync_blocks() def run_test(self): - blk0 = os.path.join(self.nodes[0].datadir, self.nodes[0].chain, 'blocks', 'blk00000.dat') - rev0 = os.path.join(self.nodes[0].datadir, self.nodes[0].chain, 'blocks', 'rev00000.dat') - blk1 = os.path.join(self.nodes[0].datadir, self.nodes[0].chain, 'blocks', 'blk00001.dat') - rev1 = os.path.join(self.nodes[0].datadir, self.nodes[0].chain, 'blocks', 'rev00001.dat') + blk0 = self.nodes[0].chain_path / "blocks" / "blk00000.dat" + rev0 = self.nodes[0].chain_path / "blocks" / "rev00000.dat" + blk1 = self.nodes[0].chain_path / "blocks" / "blk00001.dat" + rev1 = self.nodes[0].chain_path / "blocks" / "rev00001.dat" self.mine_batches(800) fo1 = os.open(blk0, os.O_RDONLY) fo2 = os.open(rev1, os.O_RDONLY) diff --git a/test/functional/feature_settings.py b/test/functional/feature_settings.py index 20018f010f..bcae963428 100755 --- a/test/functional/feature_settings.py +++ b/test/functional/feature_settings.py @@ -21,7 +21,7 @@ class SettingsTest(BitcoinTestFramework): def run_test(self): node, = self.nodes - settings = Path(node.datadir, self.chain, "settings.json") + settings = Path(node.chain_path, "settings.json") conf = Path(node.datadir, "bitcoin.conf") # Assert empty settings file was created diff --git a/test/functional/feature_signet.py b/test/functional/feature_signet.py index b41fe378af..a90a2a8e5e 100755 --- a/test/functional/feature_signet.py +++ b/test/functional/feature_signet.py @@ -76,6 +76,9 @@ class SignetBasicTest(BitcoinTestFramework): self.log.info("test that signet logs the network magic on node start") with self.nodes[0].assert_debug_log(["Signet derived magic (message start)"]): self.restart_node(0) + self.stop_node(0) + self.nodes[0].assert_start_raises_init_error(extra_args=["-signetchallenge=abc"], expected_msg="Error: -signetchallenge must be hex, not 'abc'.") + self.nodes[0].assert_start_raises_init_error(extra_args=["-signetchallenge=abc"] * 2, expected_msg="Error: -signetchallenge cannot be multiple values.") if __name__ == '__main__': diff --git a/test/functional/feature_startupnotify.py b/test/functional/feature_startupnotify.py index ff5272b281..a8e62c6244 100755 --- a/test/functional/feature_startupnotify.py +++ b/test/functional/feature_startupnotify.py @@ -3,9 +3,6 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test -startupnotify.""" - -import os - from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -18,15 +15,14 @@ FILE_NAME = "test.txt" class StartupNotifyTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.disable_syscall_sandbox = True def run_test(self): - tmpdir_file = os.path.join(self.options.tmpdir, NODE_DIR, FILE_NAME) - assert not os.path.exists(tmpdir_file) + tmpdir_file = self.nodes[0].datadir_path / FILE_NAME + assert not tmpdir_file.exists() self.log.info("Test -startupnotify command is run when node starts") self.restart_node(0, extra_args=[f"-startupnotify=echo '{FILE_NAME}' >> {NODE_DIR}/{FILE_NAME}"]) - self.wait_until(lambda: os.path.exists(tmpdir_file)) + self.wait_until(lambda: tmpdir_file.exists()) self.log.info("Test -startupnotify is executed once") diff --git a/test/functional/feature_syscall_sandbox.py b/test/functional/feature_syscall_sandbox.py deleted file mode 100755 index 2200f6c2e6..0000000000 --- a/test/functional/feature_syscall_sandbox.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2021-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test bitcoind aborts if a disallowed syscall is used when compiled with the syscall sandbox.""" - -from test_framework.test_framework import BitcoinTestFramework, SkipTest - - -class SyscallSandboxTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 1 - - def skip_test_if_missing_module(self): - if not self.is_syscall_sandbox_compiled(): - raise SkipTest("bitcoind has not been built with syscall sandbox enabled.") - if self.disable_syscall_sandbox: - raise SkipTest("--nosandbox passed to test runner.") - - def run_test(self): - disallowed_syscall_terminated_bitcoind = False - expected_log_entry = 'ERROR: The syscall "getgroups" (syscall number 115) is not allowed by the syscall sandbox' - with self.nodes[0].assert_debug_log([expected_log_entry]): - self.log.info("Invoking disallowed syscall") - try: - self.nodes[0].invokedisallowedsyscall() - except ConnectionError: - disallowed_syscall_terminated_bitcoind = True - assert disallowed_syscall_terminated_bitcoind - self.nodes = [] - - -if __name__ == "__main__": - SyscallSandboxTest().main() diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 8be2040d91..e32319961e 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -97,14 +97,15 @@ from test_framework.util import ( assert_equal, random_bytes, ) +from test_framework.wallet_util import generate_keypair from test_framework.key import ( generate_privkey, compute_xonly_pubkey, sign_schnorr, tweak_add_privkey, ECKey, - SECP256K1 ) +from test_framework import secp256k1 from test_framework.address import ( hash160, program_to_witness, @@ -694,7 +695,7 @@ def spenders_taproot_active(): # Generate an invalid public key while True: invalid_pub = random_bytes(32) - if not SECP256K1.is_x_coord(int.from_bytes(invalid_pub, 'big')): + if not secp256k1.GE.is_valid_x(int.from_bytes(invalid_pub, 'big')): break # Implement a test case that detects validation logic which maps invalid public keys to the @@ -738,7 +739,11 @@ def spenders_taproot_active(): scripts = [ ("pk_codesep", CScript(random_checksig_style(pubs[1]) + bytes([OP_CODESEPARATOR]))), # codesep after checksig ("codesep_pk", CScript(bytes([OP_CODESEPARATOR]) + random_checksig_style(pubs[1]))), # codesep before checksig - ("branched_codesep", CScript([random_bytes(random.randrange(511)), OP_DROP, OP_IF, OP_CODESEPARATOR, pubs[0], OP_ELSE, OP_CODESEPARATOR, pubs[1], OP_ENDIF, OP_CHECKSIG])), # branch dependent codesep + ("branched_codesep", CScript([random_bytes(random.randrange(2, 511)), OP_DROP, OP_IF, OP_CODESEPARATOR, pubs[0], OP_ELSE, OP_CODESEPARATOR, pubs[1], OP_ENDIF, OP_CHECKSIG])), # branch dependent codesep + # Note that the first data push in the "branched_codesep" script has the purpose of + # randomizing the sighash, both by varying script size and content. In order to + # avoid MINIMALDATA script verification errors caused by not-minimal-encoded data + # pushes (e.g. `OP_PUSH1 1` instead of `OP_1`), we set a minimum data size of 2 bytes. ] random.shuffle(scripts) tap = taproot_construct(pubs[0], scripts) @@ -1186,11 +1191,8 @@ def spenders_taproot_active(): # Also add a few legacy spends into the mix, so that transactions which combine taproot and pre-taproot spends get tested too. for compressed in [False, True]: - eckey1 = ECKey() - eckey1.set(generate_privkey(), compressed) - pubkey1 = eckey1.get_pubkey().get_bytes() - eckey2 = ECKey() - eckey2.set(generate_privkey(), compressed) + eckey1, pubkey1 = generate_keypair(compressed=compressed) + eckey2, _ = generate_keypair(compressed=compressed) for p2sh in [False, True]: for witv0 in [False, True]: for hashtype in VALID_SIGHASHES_ECDSA + [random.randrange(0x04, 0x80), random.randrange(0x84, 0x100)]: diff --git a/test/functional/feature_txindex_compatibility.py b/test/functional/feature_txindex_compatibility.py index 48fefaa0ba..a5b25cbd71 100755 --- a/test/functional/feature_txindex_compatibility.py +++ b/test/functional/feature_txindex_compatibility.py @@ -50,10 +50,10 @@ class TxindexCompatibilityTest(BitcoinTestFramework): self.nodes[0].getrawtransaction(txid=spend_utxo["txid"]) # Requires -txindex self.stop_nodes() - legacy_chain_dir = os.path.join(self.nodes[0].datadir, self.chain) + legacy_chain_dir = self.nodes[0].chain_path self.log.info("Migrate legacy txindex") - migrate_chain_dir = os.path.join(self.nodes[2].datadir, self.chain) + migrate_chain_dir = self.nodes[2].chain_path shutil.rmtree(migrate_chain_dir) shutil.copytree(legacy_chain_dir, migrate_chain_dir) with self.nodes[2].assert_debug_log([ @@ -64,7 +64,7 @@ class TxindexCompatibilityTest(BitcoinTestFramework): self.nodes[2].getrawtransaction(txid=spend_utxo["txid"]) # Requires -txindex self.log.info("Drop legacy txindex") - drop_index_chain_dir = os.path.join(self.nodes[1].datadir, self.chain) + drop_index_chain_dir = self.nodes[1].chain_path shutil.rmtree(drop_index_chain_dir) shutil.copytree(legacy_chain_dir, drop_index_chain_dir) self.nodes[1].assert_start_raises_init_error( diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index 0a9e1d4448..073d3de812 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -28,9 +28,6 @@ class VersionBitsWarningTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 - # The experimental syscall sandbox feature (-sandbox) is not compatible with -alertnotify - # (which invokes execve). - self.disable_syscall_sandbox = True def setup_network(self): self.alert_filename = os.path.join(self.options.tmpdir, "alert.txt") diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index 315d717986..c0679c5ba9 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -26,6 +26,7 @@ from test_framework.wallet import ( MiniWallet, getnewdestination, ) +from typing import Optional INVALID_PARAM = "abc" @@ -64,7 +65,7 @@ class RESTTest (BitcoinTestFramework): body: str = '', status: int = 200, ret_type: RetType = RetType.JSON, - query_params: typing.Dict[str, typing.Any] = None, + query_params: Optional[typing.Dict[str, typing.Any]] = None, ) -> typing.Union[http.client.HTTPResponse, bytes, str, None]: rest_uri = '/rest' + uri if req_type in ReqType: @@ -421,6 +422,10 @@ class RESTTest (BitcoinTestFramework): deployment_info = self.nodes[0].getdeploymentinfo() assert_equal(deployment_info, self.test_rest_request('/deploymentinfo')) + previous_bb_hash = self.nodes[0].getblockhash(self.nodes[0].getblockcount() - 1) + deployment_info = self.nodes[0].getdeploymentinfo(previous_bb_hash) + assert_equal(deployment_info, self.test_rest_request(f"/deploymentinfo/{previous_bb_hash}")) + non_existing_blockhash = '42759cde25462784395a337460bde75f58e73d3f08bd31fdc3507cbac856a2c4' resp = self.test_rest_request(f'/deploymentinfo/{non_existing_blockhash}', ret_type=RetType.OBJ, status=400) assert_equal(resp.read().decode('utf-8').rstrip(), "Block not found") diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index 3725c89719..e873e2da0b 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -46,7 +46,7 @@ class RPCInterfaceTest(BitcoinTestFramework): command = info['active_commands'][0] assert_equal(command['method'], 'getrpcinfo') assert_greater_than_or_equal(command['duration'], 0) - assert_equal(info['logpath'], os.path.join(self.nodes[0].datadir, self.chain, 'debug.log')) + assert_equal(info['logpath'], os.path.join(self.nodes[0].chain_path, 'debug.log')) def test_batch_request(self): self.log.info("Testing basic JSON-RPC batch request...") diff --git a/test/functional/interface_usdt_mempool.py b/test/functional/interface_usdt_mempool.py index 542849d558..f138fa44cc 100755 --- a/test/functional/interface_usdt_mempool.py +++ b/test/functional/interface_usdt_mempool.py @@ -35,7 +35,7 @@ MEMPOOL_TRACEPOINTS_PROGRAM = """ struct added_event { u8 hash[HASH_LENGTH]; - u64 vsize; + s32 vsize; s64 fee; }; @@ -43,7 +43,7 @@ struct removed_event { u8 hash[HASH_LENGTH]; char reason[MAX_REMOVAL_REASON_LENGTH]; - u64 vsize; + s32 vsize; s64 fee; u64 entry_time; }; @@ -57,11 +57,11 @@ struct rejected_event struct replaced_event { u8 replaced_hash[HASH_LENGTH]; - u64 replaced_vsize; + s32 replaced_vsize; s64 replaced_fee; u64 replaced_entry_time; u8 replacement_hash[HASH_LENGTH]; - u64 replacement_vsize; + s32 replacement_vsize; s64 replacement_fee; }; @@ -139,6 +139,7 @@ class MempoolTracepointTest(BitcoinTestFramework): EXPECTED_ADDED_EVENTS = 1 handled_added_events = 0 + event = None self.log.info("Hooking into mempool:added tracepoint...") node = self.nodes[0] @@ -147,11 +148,8 @@ class MempoolTracepointTest(BitcoinTestFramework): bpf = BPF(text=MEMPOOL_TRACEPOINTS_PROGRAM, usdt_contexts=[ctx], debug=0) def handle_added_event(_, data, __): - nonlocal handled_added_events + nonlocal event, handled_added_events event = bpf["added_events"].event(data) - assert_equal(txid, bytes(event.hash)[::-1].hex()) - assert_equal(vsize, event.vsize) - assert_equal(fee, event.fee) handled_added_events += 1 bpf["added_events"].open_perf_buffer(handle_added_event) @@ -159,9 +157,6 @@ class MempoolTracepointTest(BitcoinTestFramework): self.log.info("Sending transaction...") fee = Decimal(31200) tx = self.wallet.send_self_transfer(from_node=node, fee=fee / COIN) - # expected data - txid = tx["txid"] - vsize = tx["tx"].get_vsize() self.log.info("Polling buffer...") bpf.perf_buffer_poll(timeout=200) @@ -169,10 +164,13 @@ class MempoolTracepointTest(BitcoinTestFramework): self.log.info("Cleaning up mempool...") self.generate(node, 1) - bpf.cleanup() - self.log.info("Ensuring mempool:added event was handled successfully...") assert_equal(EXPECTED_ADDED_EVENTS, handled_added_events) + assert_equal(bytes(event.hash)[::-1].hex(), tx["txid"]) + assert_equal(event.vsize, tx["tx"].get_vsize()) + assert_equal(event.fee, fee) + + bpf.cleanup() self.generate(self.wallet, 1) def removed_test(self): @@ -181,6 +179,7 @@ class MempoolTracepointTest(BitcoinTestFramework): EXPECTED_REMOVED_EVENTS = 1 handled_removed_events = 0 + event = None self.log.info("Hooking into mempool:removed tracepoint...") node = self.nodes[0] @@ -189,13 +188,8 @@ class MempoolTracepointTest(BitcoinTestFramework): bpf = BPF(text=MEMPOOL_TRACEPOINTS_PROGRAM, usdt_contexts=[ctx], debug=0) def handle_removed_event(_, data, __): - nonlocal handled_removed_events + nonlocal event, handled_removed_events event = bpf["removed_events"].event(data) - assert_equal(txid, bytes(event.hash)[::-1].hex()) - assert_equal(reason, event.reason.decode("UTF-8")) - assert_equal(vsize, event.vsize) - assert_equal(fee, event.fee) - assert_equal(entry_time, event.entry_time) handled_removed_events += 1 bpf["removed_events"].open_perf_buffer(handle_removed_event) @@ -203,10 +197,7 @@ class MempoolTracepointTest(BitcoinTestFramework): self.log.info("Sending transaction...") fee = Decimal(31200) tx = self.wallet.send_self_transfer(from_node=node, fee=fee / COIN) - # expected data txid = tx["txid"] - reason = "expiry" - vsize = tx["tx"].get_vsize() self.log.info("Fast-forwarding time to mempool expiry...") entry_time = node.getmempoolentry(txid)["time"] @@ -220,10 +211,15 @@ class MempoolTracepointTest(BitcoinTestFramework): self.log.info("Polling buffer...") bpf.perf_buffer_poll(timeout=200) - bpf.cleanup() - self.log.info("Ensuring mempool:removed event was handled successfully...") assert_equal(EXPECTED_REMOVED_EVENTS, handled_removed_events) + assert_equal(bytes(event.hash)[::-1].hex(), txid) + assert_equal(event.reason.decode("UTF-8"), "expiry") + assert_equal(event.vsize, tx["tx"].get_vsize()) + assert_equal(event.fee, fee) + assert_equal(event.entry_time, entry_time) + + bpf.cleanup() self.generate(self.wallet, 1) def replaced_test(self): @@ -232,6 +228,7 @@ class MempoolTracepointTest(BitcoinTestFramework): EXPECTED_REPLACED_EVENTS = 1 handled_replaced_events = 0 + event = None self.log.info("Hooking into mempool:replaced tracepoint...") node = self.nodes[0] @@ -240,15 +237,8 @@ class MempoolTracepointTest(BitcoinTestFramework): bpf = BPF(text=MEMPOOL_TRACEPOINTS_PROGRAM, usdt_contexts=[ctx], debug=0) def handle_replaced_event(_, data, __): - nonlocal handled_replaced_events + nonlocal event, handled_replaced_events event = bpf["replaced_events"].event(data) - assert_equal(replaced_txid, bytes(event.replaced_hash)[::-1].hex()) - assert_equal(replaced_vsize, event.replaced_vsize) - assert_equal(replaced_fee, event.replaced_fee) - assert_equal(replaced_entry_time, event.replaced_entry_time) - assert_equal(replacement_txid, bytes(event.replacement_hash)[::-1].hex()) - assert_equal(replacement_vsize, event.replacement_vsize) - assert_equal(replacement_fee, event.replacement_fee) handled_replaced_events += 1 bpf["replaced_events"].open_perf_buffer(handle_replaced_event) @@ -267,21 +257,20 @@ class MempoolTracepointTest(BitcoinTestFramework): from_node=node, utxo_to_spend=utxo, fee=replacement_fee / COIN ) - # expected data - replaced_txid = original_tx["txid"] - replaced_vsize = original_tx["tx"].get_vsize() - replaced_fee = original_fee - replaced_entry_time = entry_time - replacement_txid = replacement_tx["txid"] - replacement_vsize = replacement_tx["tx"].get_vsize() - self.log.info("Polling buffer...") bpf.perf_buffer_poll(timeout=200) - bpf.cleanup() - self.log.info("Ensuring mempool:replaced event was handled successfully...") assert_equal(EXPECTED_REPLACED_EVENTS, handled_replaced_events) + assert_equal(bytes(event.replaced_hash)[::-1].hex(), original_tx["txid"]) + assert_equal(event.replaced_vsize, original_tx["tx"].get_vsize()) + assert_equal(event.replaced_fee, original_fee) + assert_equal(event.replaced_entry_time, entry_time) + assert_equal(bytes(event.replacement_hash)[::-1].hex(), replacement_tx["txid"]) + assert_equal(event.replacement_vsize, replacement_tx["tx"].get_vsize()) + assert_equal(event.replacement_fee, replacement_fee) + + bpf.cleanup() self.generate(self.wallet, 1) def rejected_test(self): @@ -290,6 +279,7 @@ class MempoolTracepointTest(BitcoinTestFramework): EXPECTED_REJECTED_EVENTS = 1 handled_rejected_events = 0 + event = None self.log.info("Adding P2P connection...") node = self.nodes[0] @@ -301,10 +291,8 @@ class MempoolTracepointTest(BitcoinTestFramework): bpf = BPF(text=MEMPOOL_TRACEPOINTS_PROGRAM, usdt_contexts=[ctx], debug=0) def handle_rejected_event(_, data, __): - nonlocal handled_rejected_events + nonlocal event, handled_rejected_events event = bpf["rejected_events"].event(data) - assert_equal(txid, bytes(event.hash)[::-1].hex()) - assert_equal(reason, event.reason.decode("UTF-8")) handled_rejected_events += 1 bpf["rejected_events"].open_perf_buffer(handle_rejected_event) @@ -313,17 +301,15 @@ class MempoolTracepointTest(BitcoinTestFramework): tx = self.wallet.create_self_transfer(fee_rate=Decimal(0)) node.p2ps[0].send_txs_and_test([tx["tx"]], node, success=False) - # expected data - txid = tx["tx"].hash - reason = "min relay fee not met" - self.log.info("Polling buffer...") bpf.perf_buffer_poll(timeout=200) - bpf.cleanup() - self.log.info("Ensuring mempool:rejected event was handled successfully...") assert_equal(EXPECTED_REJECTED_EVENTS, handled_rejected_events) + assert_equal(bytes(event.hash)[::-1].hex(), tx["tx"].hash) + assert_equal(event.reason.decode("UTF-8"), "min relay fee not met") + + bpf.cleanup() self.generate(self.wallet, 1) def run_test(self): diff --git a/test/functional/interface_usdt_net.py b/test/functional/interface_usdt_net.py index 2235da702b..d1f94637c9 100755 --- a/test/functional/interface_usdt_net.py +++ b/test/functional/interface_usdt_net.py @@ -116,13 +116,10 @@ class NetTracepointTest(BitcoinTestFramework): fn_name="trace_outbound_message") bpf = BPF(text=net_tracepoints_program, usdt_contexts=[ctx], debug=0) - # The handle_* function is a ctypes callback function called from C. When - # we assert in the handle_* function, the AssertError doesn't propagate - # back to Python. The exception is ignored. We manually count and assert - # that the handle_* functions succeeded. EXPECTED_INOUTBOUND_VERSION_MSG = 1 checked_inbound_version_msg = 0 checked_outbound_version_msg = 0 + events = [] def check_p2p_message(event, inbound): nonlocal checked_inbound_version_msg, checked_outbound_version_msg @@ -142,12 +139,13 @@ class NetTracepointTest(BitcoinTestFramework): checked_outbound_version_msg += 1 def handle_inbound(_, data, __): + nonlocal events event = ctypes.cast(data, ctypes.POINTER(P2PMessage)).contents - check_p2p_message(event, True) + events.append((event, True)) def handle_outbound(_, data, __): event = ctypes.cast(data, ctypes.POINTER(P2PMessage)).contents - check_p2p_message(event, False) + events.append((event, False)) bpf["inbound_messages"].open_perf_buffer(handle_inbound) bpf["outbound_messages"].open_perf_buffer(handle_outbound) @@ -158,12 +156,15 @@ class NetTracepointTest(BitcoinTestFramework): bpf.perf_buffer_poll(timeout=200) self.log.info( - "check that we got both an inbound and outbound version message") + "check receipt and content of in- and outbound version messages") + for event, inbound in events: + check_p2p_message(event, inbound) assert_equal(EXPECTED_INOUTBOUND_VERSION_MSG, checked_inbound_version_msg) assert_equal(EXPECTED_INOUTBOUND_VERSION_MSG, checked_outbound_version_msg) + bpf.cleanup() diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py index 6774db7c5f..5f2ba49026 100755 --- a/test/functional/interface_usdt_utxocache.py +++ b/test/functional/interface_usdt_utxocache.py @@ -188,13 +188,16 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): nonlocal handle_uncache_succeeds event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents self.log.info(f"handle_utxocache_uncache(): {event}") - assert_equal(block_1_coinbase_txid, bytes(event.txid[::-1]).hex()) - assert_equal(0, event.index) # prevout index - assert_equal(EARLY_BLOCK_HEIGHT, event.height) - assert_equal(50 * COIN, event.value) - assert_equal(True, event.is_coinbase) - - handle_uncache_succeeds += 1 + try: + assert_equal(block_1_coinbase_txid, bytes(event.txid[::-1]).hex()) + assert_equal(0, event.index) # prevout index + assert_equal(EARLY_BLOCK_HEIGHT, event.height) + assert_equal(50 * COIN, event.value) + assert_equal(True, event.is_coinbase) + except AssertionError: + self.log.exception("Assertion failed") + else: + handle_uncache_succeeds += 1 bpf["utxocache_uncache"].open_perf_buffer(handle_utxocache_uncache) @@ -260,24 +263,32 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents self.log.info(f"handle_utxocache_add(): {event}") add = expected_utxocache_adds.pop(0) - assert_equal(add["txid"], bytes(event.txid[::-1]).hex()) - assert_equal(add["index"], event.index) - assert_equal(add["height"], event.height) - assert_equal(add["value"], event.value) - assert_equal(add["is_coinbase"], event.is_coinbase) - handle_add_succeeds += 1 + try: + assert_equal(add["txid"], bytes(event.txid[::-1]).hex()) + assert_equal(add["index"], event.index) + assert_equal(add["height"], event.height) + assert_equal(add["value"], event.value) + assert_equal(add["is_coinbase"], event.is_coinbase) + except AssertionError: + self.log.exception("Assertion failed") + else: + handle_add_succeeds += 1 def handle_utxocache_spent(_, data, __): nonlocal handle_spent_succeeds event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents self.log.info(f"handle_utxocache_spent(): {event}") spent = expected_utxocache_spents.pop(0) - assert_equal(spent["txid"], bytes(event.txid[::-1]).hex()) - assert_equal(spent["index"], event.index) - assert_equal(spent["height"], event.height) - assert_equal(spent["value"], event.value) - assert_equal(spent["is_coinbase"], event.is_coinbase) - handle_spent_succeeds += 1 + try: + assert_equal(spent["txid"], bytes(event.txid[::-1]).hex()) + assert_equal(spent["index"], event.index) + assert_equal(spent["height"], event.height) + assert_equal(spent["value"], event.value) + assert_equal(spent["is_coinbase"], event.is_coinbase) + except AssertionError: + self.log.exception("Assertion failed") + else: + handle_spent_succeeds += 1 bpf["utxocache_add"].open_perf_buffer(handle_utxocache_add) bpf["utxocache_spent"].open_perf_buffer(handle_utxocache_spent) diff --git a/test/functional/interface_usdt_validation.py b/test/functional/interface_usdt_validation.py index 4323aef771..f9d9b525cd 100755 --- a/test/functional/interface_usdt_validation.py +++ b/test/functional/interface_usdt_validation.py @@ -85,13 +85,10 @@ class ValidationTracepointTest(BitcoinTestFramework): self.sigops, self.duration) - # The handle_* function is a ctypes callback function called from C. When - # we assert in the handle_* function, the AssertError doesn't propagate - # back to Python. The exception is ignored. We manually count and assert - # that the handle_* functions succeeded. BLOCKS_EXPECTED = 2 blocks_checked = 0 expected_blocks = dict() + events = [] self.log.info("hook into the validation:block_connected tracepoint") ctx = USDT(pid=self.nodes[0].process.pid) @@ -101,19 +98,10 @@ class ValidationTracepointTest(BitcoinTestFramework): usdt_contexts=[ctx], debug=0) def handle_blockconnected(_, data, __): - nonlocal expected_blocks, blocks_checked + nonlocal events, blocks_checked event = ctypes.cast(data, ctypes.POINTER(Block)).contents self.log.info(f"handle_blockconnected(): {event}") - block_hash = bytes(event.hash[::-1]).hex() - block = expected_blocks[block_hash] - assert_equal(block["hash"], block_hash) - assert_equal(block["height"], event.height) - assert_equal(len(block["tx"]), event.transactions) - assert_equal(len([tx["vin"] for tx in block["tx"]]), event.inputs) - assert_equal(0, event.sigops) # no sigops in coinbase tx - # only plausibility checks - assert event.duration > 0 - del expected_blocks[block_hash] + events.append(event) blocks_checked += 1 bpf["block_connected"].open_perf_buffer( @@ -126,12 +114,24 @@ class ValidationTracepointTest(BitcoinTestFramework): expected_blocks[block_hash] = self.nodes[0].getblock(block_hash, 2) bpf.perf_buffer_poll(timeout=200) - bpf.cleanup() - self.log.info(f"check that we traced {BLOCKS_EXPECTED} blocks") + self.log.info(f"check that we correctly traced {BLOCKS_EXPECTED} blocks") + for event in events: + block_hash = bytes(event.hash[::-1]).hex() + block = expected_blocks[block_hash] + assert_equal(block["hash"], block_hash) + assert_equal(block["height"], event.height) + assert_equal(len(block["tx"]), event.transactions) + assert_equal(len([tx["vin"] for tx in block["tx"]]), event.inputs) + assert_equal(0, event.sigops) # no sigops in coinbase tx + # only plausibility checks + assert event.duration > 0 + del expected_blocks[block_hash] assert_equal(BLOCKS_EXPECTED, blocks_checked) assert_equal(0, len(expected_blocks)) + bpf.cleanup() + if __name__ == '__main__': ValidationTracepointTest().main() diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 737a8d0a2e..8f3aec96a7 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -9,7 +9,6 @@ from decimal import Decimal import math from test_framework.test_framework import BitcoinTestFramework -from test_framework.key import ECKey from test_framework.messages import ( MAX_BIP125_RBF_SEQUENCE, COIN, @@ -44,6 +43,7 @@ from test_framework.util import ( assert_raises_rpc_error, ) from test_framework.wallet import MiniWallet +from test_framework.wallet_util import generate_keypair class MempoolAcceptanceTest(BitcoinTestFramework): @@ -283,9 +283,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): rawtxs=[tx.serialize().hex()], ) tx = tx_from_hex(raw_tx_reference) - key = ECKey() - key.generate() - pubkey = key.get_pubkey().get_bytes() + _, pubkey = generate_keypair() tx.vout[0].scriptPubKey = keys_to_multisig_script([pubkey] * 3, k=2) # Some bare multisig script (2-of-3) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bare-multisig'}], diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index 7337802aea..3f632d3d56 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -55,8 +55,8 @@ class MempoolCompatibilityTest(BitcoinTestFramework): self.stop_node(1) self.log.info("Move mempool.dat from old to new node") - old_node_mempool = os.path.join(old_node.datadir, self.chain, 'mempool.dat') - new_node_mempool = os.path.join(new_node.datadir, self.chain, 'mempool.dat') + old_node_mempool = os.path.join(old_node.chain_path, 'mempool.dat') + new_node_mempool = os.path.join(new_node.chain_path, 'mempool.dat') os.rename(old_node_mempool, new_node_mempool) self.log.info("Start new node and verify mempool contains the tx") diff --git a/test/functional/mempool_dust.py b/test/functional/mempool_dust.py index 41a26e82da..f4e385a112 100755 --- a/test/functional/mempool_dust.py +++ b/test/functional/mempool_dust.py @@ -5,7 +5,6 @@ """Test dust limit mempool policy (`-dustrelayfee` parameter)""" from decimal import Decimal -from test_framework.key import ECKey from test_framework.messages import ( COIN, CTxOut, @@ -32,6 +31,7 @@ from test_framework.util import ( get_fee, ) from test_framework.wallet import MiniWallet +from test_framework.wallet_util import generate_keypair DUST_RELAY_TX_FEE = 3000 # default setting [sat/kvB] @@ -74,11 +74,8 @@ class DustRelayFeeTest(BitcoinTestFramework): self.wallet = MiniWallet(self.nodes[0]) # prepare output scripts of each standard type - key = ECKey() - key.generate(compressed=False) - uncompressed_pubkey = key.get_pubkey().get_bytes() - key.generate(compressed=True) - pubkey = key.get_pubkey().get_bytes() + _, uncompressed_pubkey = generate_keypair(compressed=False) + _, pubkey = generate_keypair(compressed=True) output_scripts = ( (key_to_p2pk_script(uncompressed_pubkey), "P2PK (uncompressed)"), diff --git a/test/functional/mempool_expiry.py b/test/functional/mempool_expiry.py index 15a5f765df..18b3a8def4 100755 --- a/test/functional/mempool_expiry.py +++ b/test/functional/mempool_expiry.py @@ -12,7 +12,10 @@ definable expiry timeout via the '-mempoolexpiry=<n>' command line argument from datetime import timedelta -from test_framework.messages import DEFAULT_MEMPOOL_EXPIRY_HOURS +from test_framework.messages import ( + COIN, + DEFAULT_MEMPOOL_EXPIRY_HOURS, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -37,6 +40,10 @@ class MempoolExpiryTest(BitcoinTestFramework): parent_utxo = self.wallet.get_utxo(txid=parent_txid) independent_utxo = self.wallet.get_utxo() + # Add prioritisation to this transaction to check that it persists after the expiry + node.prioritisetransaction(parent_txid, 0, COIN) + assert_equal(node.getprioritisedtransactions()[parent_txid], { "fee_delta" : COIN, "in_mempool" : True}) + # Ensure the transactions we send to trigger the mempool check spend utxos that are independent of # the transactions being tested for expiration. trigger_utxo1 = self.wallet.get_utxo() @@ -79,6 +86,9 @@ class MempoolExpiryTest(BitcoinTestFramework): assert_raises_rpc_error(-5, 'Transaction not in mempool', node.getmempoolentry, parent_txid) + # Prioritisation does not disappear when transaction expires + assert_equal(node.getprioritisedtransactions()[parent_txid], { "fee_delta" : COIN, "in_mempool" : False}) + # The child transaction should be removed from the mempool as well. self.log.info('Test child tx is evicted as well.') assert_raises_rpc_error(-5, 'Transaction not in mempool', diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 0387282862..95f7939412 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -7,7 +7,6 @@ from decimal import Decimal from test_framework.messages import ( - COIN, DEFAULT_ANCESTOR_LIMIT, DEFAULT_DESCENDANT_LIMIT, ) @@ -26,9 +25,6 @@ assert CUSTOM_DESCENDANT_LIMIT >= CUSTOM_ANCESTOR_LIMIT class MempoolPackagesTest(BitcoinTestFramework): - def add_options(self, parser): - self.add_wallet_options(parser) - def set_test_params(self): self.num_nodes = 2 self.extra_args = [ @@ -47,10 +43,6 @@ class MempoolPackagesTest(BitcoinTestFramework): self.wallet = MiniWallet(self.nodes[0]) self.wallet.rescan_utxos() - if self.is_specified_wallet_compiled(): - self.nodes[0].createwallet("watch_wallet", disable_private_keys=True) - self.nodes[0].importaddress(self.wallet.get_address()) - peer_inv_store = self.nodes[0].add_p2p_connection(P2PTxInvStore()) # keep track of invs # DEFAULT_ANCESTOR_LIMIT transactions off a confirmed tx should be fine @@ -63,13 +55,6 @@ class MempoolPackagesTest(BitcoinTestFramework): ancestor_vsize += t["tx"].get_vsize() ancestor_fees += t["fee"] self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=t["hex"]) - # Check that listunspent ancestor{count, size, fees} yield the correct results - if self.is_specified_wallet_compiled(): - wallet_unspent = self.nodes[0].listunspent(minconf=0) - this_unspent = next(utxo_info for utxo_info in wallet_unspent if utxo_info["txid"] == t["txid"]) - assert_equal(this_unspent['ancestorcount'], i + 1) - assert_equal(this_unspent['ancestorsize'], ancestor_vsize) - assert_equal(this_unspent['ancestorfees'], ancestor_fees * COIN) # Wait until mempool transactions have passed initial broadcast (sent inv and received getdata) # Otherwise, getrawmempool may be inconsistent with getmempoolentry if unbroadcast changes in between diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 8f74d9de20..a1335ff069 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -143,8 +143,8 @@ class MempoolPersistTest(BitcoinTestFramework): self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet assert_equal(node2_balance, wallet_watch.getbalance()) - mempooldat0 = os.path.join(self.nodes[0].datadir, self.chain, 'mempool.dat') - mempooldat1 = os.path.join(self.nodes[1].datadir, self.chain, 'mempool.dat') + mempooldat0 = os.path.join(self.nodes[0].chain_path, 'mempool.dat') + mempooldat1 = os.path.join(self.nodes[1].chain_path, 'mempool.dat') self.log.debug("Force -persistmempool=0 node1 to savemempool to disk via RPC") assert not os.path.exists(mempooldat1) diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 332099516c..aabf06ee53 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -145,6 +145,7 @@ class MiningTest(BitcoinTestFramework): assert_template(node, bad_block, 'bad-cb-missing') self.log.info("submitblock: Test invalid coinbase transaction") + assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, CBlock().serialize().hex()) assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, bad_block.serialize().hex()) self.log.info("getblocktemplate: Test truncated final transaction") diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index a4481c15a0..099c0e418c 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -30,6 +30,33 @@ class PrioritiseTransactionTest(BitcoinTestFramework): ]] * self.num_nodes self.supports_cli = False + def clear_prioritisation(self, node): + for txid, info in node.getprioritisedtransactions().items(): + delta = info["fee_delta"] + node.prioritisetransaction(txid, 0, -delta) + assert_equal(node.getprioritisedtransactions(), {}) + + def test_replacement(self): + self.log.info("Test tx prioritisation stays after a tx is replaced") + conflicting_input = self.wallet.get_utxo() + tx_replacee = self.wallet.create_self_transfer(utxo_to_spend=conflicting_input, fee_rate=Decimal("0.0001")) + tx_replacement = self.wallet.create_self_transfer(utxo_to_spend=conflicting_input, fee_rate=Decimal("0.005")) + # Add 1 satoshi fee delta to replacee + self.nodes[0].prioritisetransaction(tx_replacee["txid"], 0, 100) + assert_equal(self.nodes[0].getprioritisedtransactions(), { tx_replacee["txid"] : { "fee_delta" : 100, "in_mempool" : False}}) + self.nodes[0].sendrawtransaction(tx_replacee["hex"]) + assert_equal(self.nodes[0].getprioritisedtransactions(), { tx_replacee["txid"] : { "fee_delta" : 100, "in_mempool" : True}}) + self.nodes[0].sendrawtransaction(tx_replacement["hex"]) + assert tx_replacee["txid"] not in self.nodes[0].getrawmempool() + assert_equal(self.nodes[0].getprioritisedtransactions(), { tx_replacee["txid"] : { "fee_delta" : 100, "in_mempool" : False}}) + + # PrioritiseTransaction is additive + self.nodes[0].prioritisetransaction(tx_replacee["txid"], 0, COIN) + self.nodes[0].sendrawtransaction(tx_replacee["hex"]) + assert_equal(self.nodes[0].getprioritisedtransactions(), { tx_replacee["txid"] : { "fee_delta" : COIN + 100, "in_mempool" : True}}) + self.generate(self.nodes[0], 1) + assert_equal(self.nodes[0].getprioritisedtransactions(), {}) + def test_diamond(self): self.log.info("Test diamond-shape package with priority") mock_time = int(time.time()) @@ -84,6 +111,13 @@ class PrioritiseTransactionTest(BitcoinTestFramework): raw_after = self.nodes[0].getrawmempool(verbose=True) assert_equal(raw_before[txid_a], raw_after[txid_a]) assert_equal(raw_before, raw_after) + prioritisation_map_in_mempool = self.nodes[0].getprioritisedtransactions() + assert_equal(prioritisation_map_in_mempool[txid_b], {"fee_delta" : fee_delta_b*COIN, "in_mempool" : True}) + assert_equal(prioritisation_map_in_mempool[txid_c], {"fee_delta" : (fee_delta_c_1 + fee_delta_c_2)*COIN, "in_mempool" : True}) + # Clear prioritisation, otherwise the transactions' fee deltas are persisted to mempool.dat and loaded again when the node + # is restarted at the end of this subtest. Deltas are removed when a transaction is mined, but only at that time. We do + # not check whether mapDeltas transactions were mined when loading from mempool.dat. + self.clear_prioritisation(node=self.nodes[0]) self.log.info("Test priority while txs are not in mempool") self.restart_node(0, extra_args=["-nopersistmempool"]) @@ -92,17 +126,26 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.nodes[0].prioritisetransaction(txid=txid_b, fee_delta=int(fee_delta_b * COIN)) self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_1 * COIN)) self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_2 * COIN)) + prioritisation_map_not_in_mempool = self.nodes[0].getprioritisedtransactions() + assert_equal(prioritisation_map_not_in_mempool[txid_b], {"fee_delta" : fee_delta_b*COIN, "in_mempool" : False}) + assert_equal(prioritisation_map_not_in_mempool[txid_c], {"fee_delta" : (fee_delta_c_1 + fee_delta_c_2)*COIN, "in_mempool" : False}) for t in [tx_o_a["hex"], tx_o_b["hex"], tx_o_c["hex"], tx_o_d["hex"]]: self.nodes[0].sendrawtransaction(t) raw_after = self.nodes[0].getrawmempool(verbose=True) assert_equal(raw_before[txid_a], raw_after[txid_a]) assert_equal(raw_before, raw_after) + prioritisation_map_in_mempool = self.nodes[0].getprioritisedtransactions() + assert_equal(prioritisation_map_in_mempool[txid_b], {"fee_delta" : fee_delta_b*COIN, "in_mempool" : True}) + assert_equal(prioritisation_map_in_mempool[txid_c], {"fee_delta" : (fee_delta_c_1 + fee_delta_c_2)*COIN, "in_mempool" : True}) # Clear mempool self.generate(self.nodes[0], 1) + # Prioritisation for transactions is automatically deleted after they are mined. + assert_equal(self.nodes[0].getprioritisedtransactions(), {}) # Use default extra_args self.restart_node(0) + assert_equal(self.nodes[0].getprioritisedtransactions(), {}) def run_test(self): self.wallet = MiniWallet(self.nodes[0]) @@ -115,6 +158,10 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Test `prioritisetransaction` invalid extra parameters assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction, '', 0, 0, 0) + # Test `getprioritisedtransactions` invalid parameters + assert_raises_rpc_error(-1, "getprioritisedtransactions", + self.nodes[0].getprioritisedtransactions, True) + # Test `prioritisetransaction` invalid `txid` assert_raises_rpc_error(-8, "txid must be of length 64 (not 3, for 'foo')", self.nodes[0].prioritisetransaction, txid='foo', fee_delta=0) assert_raises_rpc_error(-8, "txid must be hexadecimal string (not 'Zd1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000')", self.nodes[0].prioritisetransaction, txid='Zd1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000', fee_delta=0) @@ -127,6 +174,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Test `prioritisetransaction` invalid `fee_delta` assert_raises_rpc_error(-3, "JSON value of type string is not of expected type number", self.nodes[0].prioritisetransaction, txid=txid, fee_delta='foo') + self.test_replacement() self.test_diamond() self.txouts = gen_return_txouts() @@ -165,9 +213,18 @@ class PrioritiseTransactionTest(BitcoinTestFramework): sizes[i] += mempool[j]['vsize'] assert sizes[i] > MAX_BLOCK_WEIGHT // 4 # Fail => raise utxo_count + assert_equal(self.nodes[0].getprioritisedtransactions(), {}) # add a fee delta to something in the cheapest bucket and make sure it gets mined # also check that a different entry in the cheapest bucket is NOT mined self.nodes[0].prioritisetransaction(txid=txids[0][0], fee_delta=int(3*base_fee*COIN)) + assert_equal(self.nodes[0].getprioritisedtransactions(), {txids[0][0] : { "fee_delta" : 3*base_fee*COIN, "in_mempool" : True}}) + + # Priority disappears when prioritisetransaction is called with an inverse value... + self.nodes[0].prioritisetransaction(txid=txids[0][0], fee_delta=int(-3*base_fee*COIN)) + assert txids[0][0] not in self.nodes[0].getprioritisedtransactions() + # ... and reappears when prioritisetransaction is called again. + self.nodes[0].prioritisetransaction(txid=txids[0][0], fee_delta=int(3*base_fee*COIN)) + assert txids[0][0] in self.nodes[0].getprioritisedtransactions() self.generate(self.nodes[0], 1) @@ -187,6 +244,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Add a prioritisation before a tx is in the mempool (de-prioritising a # high-fee transaction so that it's now low fee). self.nodes[0].prioritisetransaction(txid=high_fee_tx, fee_delta=-int(2*base_fee*COIN)) + assert_equal(self.nodes[0].getprioritisedtransactions()[high_fee_tx], { "fee_delta" : -2*base_fee*COIN, "in_mempool" : False}) # Add everything back to mempool self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) @@ -206,6 +264,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): mempool = self.nodes[0].getrawmempool() self.log.info("Assert that de-prioritised transaction is still in mempool") assert high_fee_tx in mempool + assert_equal(self.nodes[0].getprioritisedtransactions()[high_fee_tx], { "fee_delta" : -2*base_fee*COIN, "in_mempool" : True}) for x in txids[2]: if (x != high_fee_tx): assert x not in mempool @@ -223,10 +282,12 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # to be the minimum for a 1000-byte transaction and check that it is # accepted. self.nodes[0].prioritisetransaction(txid=tx_id, fee_delta=int(self.relayfee*COIN)) + assert_equal(self.nodes[0].getprioritisedtransactions()[tx_id], { "fee_delta" : self.relayfee*COIN, "in_mempool" : False}) self.log.info("Assert that prioritised free transaction is accepted to mempool") assert_equal(self.nodes[0].sendrawtransaction(tx_hex), tx_id) assert tx_id in self.nodes[0].getrawmempool() + assert_equal(self.nodes[0].getprioritisedtransactions()[tx_id], { "fee_delta" : self.relayfee*COIN, "in_mempool" : True}) # Test that calling prioritisetransaction is sufficient to trigger # getblocktemplate to (eventually) return a new block. @@ -234,6 +295,10 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.nodes[0].setmocktime(mock_time) template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) self.nodes[0].prioritisetransaction(txid=tx_id, fee_delta=-int(self.relayfee*COIN)) + + # Calling prioritisetransaction with the inverse amount should delete its prioritisation entry + assert tx_id not in self.nodes[0].getprioritisedtransactions() + self.nodes[0].setmocktime(mock_time+10) new_template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 644abda914..2fb88b828f 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -4,6 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid network messages.""" +import random import struct import time @@ -75,6 +76,7 @@ class InvalidMessagesTest(BitcoinTestFramework): self.test_oversized_getdata_msg() self.test_oversized_headers_msg() self.test_invalid_pow_headers_msg() + self.test_noncontinuous_headers_msg() self.test_resource_exhaustion() def test_buffer(self): @@ -283,6 +285,25 @@ class InvalidMessagesTest(BitcoinTestFramework): peer.send_message(msg_headers([blockheader])) peer.wait_for_disconnect() + def test_noncontinuous_headers_msg(self): + self.log.info("Test headers message with non-continuous headers sequence is logged as misbehaving") + block_hashes = self.generate(self.nodes[0], 10) + block_headers = [] + for block_hash in block_hashes: + block_headers.append(from_hex(CBlockHeader(), self.nodes[0].getblockheader(block_hash, False))) + + # continuous headers sequence should be fine + MISBEHAVING_NONCONTINUOUS_HEADERS_MSGS = ['Misbehaving', 'non-continuous headers sequence'] + peer = self.nodes[0].add_p2p_connection(P2PInterface()) + with self.nodes[0].assert_debug_log([], unexpected_msgs=MISBEHAVING_NONCONTINUOUS_HEADERS_MSGS): + peer.send_and_ping(msg_headers(block_headers)) + + # delete arbitrary block header somewhere in the middle to break link + del block_headers[random.randrange(1, len(block_headers)-1)] + with self.nodes[0].assert_debug_log(expected_msgs=MISBEHAVING_NONCONTINUOUS_HEADERS_MSGS): + peer.send_and_ping(msg_headers(block_headers)) + self.nodes[0].disconnect_p2ps() + def test_resource_exhaustion(self): self.log.info("Test node stays up despite many large junk messages") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py index ef327c7ce8..f53f98e06d 100755 --- a/test/functional/p2p_leak_tx.py +++ b/test/functional/p2p_leak_tx.py @@ -4,8 +4,8 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test transaction upload""" -from test_framework.messages import msg_getdata, CInv, MSG_TX -from test_framework.p2p import p2p_lock, P2PDataStore +from test_framework.messages import msg_getdata, CInv, MSG_TX, MSG_WTX +from test_framework.p2p import p2p_lock, P2PDataStore, P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -27,6 +27,7 @@ class P2PLeakTxTest(BitcoinTestFramework): self.miniwallet = MiniWallet(self.gen_node) self.test_tx_in_block() + self.test_notfound_on_replaced_tx() self.test_notfound_on_unannounced_tx() def test_tx_in_block(self): @@ -45,8 +46,37 @@ class P2PLeakTxTest(BitcoinTestFramework): inbound_peer.send_and_ping(want_tx) assert_equal(inbound_peer.last_message.get("tx").tx.getwtxid(), wtxid) + def test_notfound_on_replaced_tx(self): + self.gen_node.disconnect_p2ps() + inbound_peer = self.gen_node.add_p2p_connection(P2PTxInvStore()) + + self.log.info("Transaction tx_a is broadcast") + tx_a = self.miniwallet.send_self_transfer(from_node=self.gen_node) + inbound_peer.wait_for_broadcast(txns=[tx_a["wtxid"]]) + + tx_b = tx_a["tx"] + tx_b.vout[0].nValue -= 9000 + self.gen_node.sendrawtransaction(tx_b.serialize().hex()) + inbound_peer.wait_until(lambda: "tx" in inbound_peer.last_message and inbound_peer.last_message.get("tx").tx.getwtxid() == tx_b.getwtxid()) + + self.log.info("Re-request of tx_a after replacement is answered with notfound") + req_vec = [ + CInv(t=MSG_TX, h=int(tx_a["txid"], 16)), + CInv(t=MSG_WTX, h=int(tx_a["wtxid"], 16)), + ] + want_tx = msg_getdata() + want_tx.inv = req_vec + with p2p_lock: + inbound_peer.last_message.pop("notfound", None) + inbound_peer.last_message.pop("tx", None) + inbound_peer.send_and_ping(want_tx) + + assert_equal(inbound_peer.last_message.get("notfound").vec, req_vec) + assert "tx" not in inbound_peer.last_message + def test_notfound_on_unannounced_tx(self): self.log.info("Check that we don't leak txs to inbound peers that we haven't yet announced to") + self.gen_node.disconnect_p2ps() inbound_peer = self.gen_node.add_p2p_connection(P2PNode()) # An "attacking" inbound peer MAX_REPEATS = 100 diff --git a/test/functional/p2p_message_capture.py b/test/functional/p2p_message_capture.py index 3ab0b79ba2..691a0b6409 100755 --- a/test/functional/p2p_message_capture.py +++ b/test/functional/p2p_message_capture.py @@ -19,7 +19,7 @@ TIME_SIZE = 8 LENGTH_SIZE = 4 MSGTYPE_SIZE = 12 -def mini_parser(dat_file): +def mini_parser(dat_file: str) -> None: """Parse a data file created by CaptureMessageToFile. From the data file we'll only check the structure. @@ -58,7 +58,7 @@ class MessageCaptureTest(BitcoinTestFramework): self.setup_clean_chain = True def run_test(self): - capturedir = os.path.join(self.nodes[0].datadir, "regtest/message_capture") + capturedir = self.nodes[0].chain_path / "message_capture" # Connect a node so that the handshake occurs self.nodes[0].add_p2p_connection(P2PDataStore()) self.nodes[0].disconnect_p2ps() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index b0900e49b8..bfae190c66 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -14,7 +14,6 @@ from test_framework.blocktools import ( create_block, create_coinbase, ) -from test_framework.key import ECKey from test_framework.messages import ( MAX_BIP125_RBF_SEQUENCE, CBlockHeader, @@ -89,6 +88,7 @@ from test_framework.util import ( assert_raises_rpc_error, ) from test_framework.wallet import MiniWallet +from test_framework.wallet_util import generate_keypair MAX_SIGOP_COST = 80000 @@ -1448,9 +1448,7 @@ class SegWitTest(BitcoinTestFramework): # Segwit transactions using uncompressed pubkeys are not accepted # under default policy, but should still pass consensus. - key = ECKey() - key.generate(False) - pubkey = key.get_pubkey().get_bytes() + key, pubkey = generate_keypair(compressed=False) assert_equal(len(pubkey), 65) # This should be an uncompressed pubkey utxo = self.utxo.pop(0) @@ -1544,11 +1542,7 @@ class SegWitTest(BitcoinTestFramework): @subtest def test_signature_version_1(self): - - key = ECKey() - key.generate() - pubkey = key.get_pubkey().get_bytes() - + key, pubkey = generate_keypair() witness_script = key_to_p2pk_script(pubkey) script_pubkey = script_to_p2wsh_script(witness_script) diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 6022042c11..5f2bece733 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -49,7 +49,6 @@ from test_framework.util import ( assert_raises_rpc_error, assert_is_hex_string, assert_is_hash_string, - get_datadir_path, ) from test_framework.wallet import MiniWallet @@ -572,16 +571,15 @@ class BlockchainTest(BitcoinTestFramework): self.log.info("Test that getblock with verbosity 3 includes prevout") assert_vin_contains_prevout(3) - self.log.info("Test that getblock with verbosity 2 and 3 still works with pruned Undo data") - datadir = get_datadir_path(self.options.tmpdir, 0) - self.log.info("Test getblock with invalid verbosity type returns proper error message") assert_raises_rpc_error(-3, "JSON value of type string is not of expected type number", node.getblock, blockhash, "2") + self.log.info("Test that getblock with verbosity 2 and 3 still works with pruned Undo data") + def move_block_file(old, new): - old_path = os.path.join(datadir, self.chain, 'blocks', old) - new_path = os.path.join(datadir, self.chain, 'blocks', new) - os.rename(old_path, new_path) + old_path = self.nodes[0].chain_path / "blocks" / old + new_path = self.nodes[0].chain_path / "blocks" / new + old_path.rename(new_path) # Move instead of deleting so we can restore chain state afterwards move_block_file('rev00000.dat', 'rev_wrong') diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 279fb01a57..65d7b4c422 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -12,13 +12,13 @@ from test_framework.address import address_to_scriptpubkey from test_framework.blocktools import COINBASE_MATURITY from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create, drop_origins -from test_framework.key import ECPubKey, ECKey +from test_framework.key import ECPubKey from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_raises_rpc_error, assert_equal, ) -from test_framework.wallet_util import bytes_to_wif +from test_framework.wallet_util import generate_keypair from test_framework.wallet import ( MiniWallet, getnewdestination, @@ -38,10 +38,9 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): self.priv = [] node0, node1, node2 = self.nodes for _ in range(self.nkeys): - k = ECKey() - k.generate() - self.pub.append(k.get_pubkey().get_bytes().hex()) - self.priv.append(bytes_to_wif(k.get_bytes(), k.is_compressed)) + privkey, pubkey = generate_keypair(wif=True) + self.pub.append(pubkey.hex()) + self.priv.append(privkey) if self.is_bdb_compiled(): self.final = node2.getnewaddress() else: @@ -158,7 +157,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): try: node1.loadwallet('wmulti') except JSONRPCException as e: - path = os.path.join(self.options.tmpdir, "node1", "regtest", "wallets", "wmulti") + path = self.nodes[1].wallets_path / "wmulti" if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']: node1.createwallet(wallet_name='wmulti', disable_private_keys=True) else: diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 39a931be03..4260e95629 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -52,7 +52,7 @@ class DumptxoutsetTest(BitcoinTestFramework): # Specifying a path to an existing or invalid file will fail. assert_raises_rpc_error( -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME) - invalid_path = str(Path(node.datadir) / "invalid" / "path") + invalid_path = node.datadir_path / "invalid" / "path" assert_raises_rpc_error( -8, "Couldn't open file {}.incomplete for writing".format(invalid_path), node.dumptxoutset, invalid_path) diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index 2f093bebff..1ab1023cf1 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -117,9 +117,11 @@ class GetBlockFromPeerTest(BitcoinTestFramework): assert_raises_rpc_error(-1, error_msg, self.nodes[1].getblockfrompeer, blockhash, node1_interface_id) self.log.info("Connect pruned node") - # We need to generate more blocks to be able to prune self.connect_nodes(0, 2) pruned_node = self.nodes[2] + self.sync_blocks([self.nodes[0], pruned_node]) + + # We need to generate more blocks to be able to prune self.generate(self.nodes[0], 400, sync_fun=self.no_op) self.sync_blocks([self.nodes[0], pruned_node]) pruneheight = pruned_node.pruneblockchain(300) diff --git a/test/functional/rpc_help.py b/test/functional/rpc_help.py index 7acc3cbbd5..53c5aa05e5 100755 --- a/test/functional/rpc_help.py +++ b/test/functional/rpc_help.py @@ -85,8 +85,8 @@ class HelpRpcTest(BitcoinTestFramework): for argname, convert in converts_by_argname.items(): if all(convert) != any(convert): - # Only allow dummy to fail consistency check - assert argname == 'dummy', ('WARNING: conversion mismatch for argument named %s (%s)' % (argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname])))) + # Only allow dummy and psbt to fail consistency check + assert argname in ['dummy', "psbt"], ('WARNING: conversion mismatch for argument named %s (%s)' % (argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname])))) def test_categories(self): node = self.nodes[0] diff --git a/test/functional/rpc_invalid_address_message.py b/test/functional/rpc_invalid_address_message.py index 59b45bbb9b..6759b69dd1 100755 --- a/test/functional/rpc_invalid_address_message.py +++ b/test/functional/rpc_invalid_address_message.py @@ -63,12 +63,12 @@ class InvalidAddressErrorMessageTest(BitcoinTestFramework): def test_validateaddress(self): # Invalid Bech32 - self.check_invalid(BECH32_INVALID_SIZE, "Invalid Bech32 address program size (41 byte)") + self.check_invalid(BECH32_INVALID_SIZE, "Invalid Bech32 address program size (41 bytes)") self.check_invalid(BECH32_INVALID_PREFIX, 'Invalid or unsupported Segwit (Bech32) or Base58 encoding.') self.check_invalid(BECH32_INVALID_BECH32, 'Version 1+ witness address must use Bech32m checksum') self.check_invalid(BECH32_INVALID_BECH32M, 'Version 0 witness address must use Bech32 checksum') self.check_invalid(BECH32_INVALID_VERSION, 'Invalid Bech32 address witness version') - self.check_invalid(BECH32_INVALID_V0_SIZE, "Invalid Bech32 v0 address program size (21 byte), per BIP141") + self.check_invalid(BECH32_INVALID_V0_SIZE, "Invalid Bech32 v0 address program size (21 bytes), per BIP141") self.check_invalid(BECH32_TOO_LONG, 'Bech32 string too long', list(range(90, 108))) self.check_invalid(BECH32_ONE_ERROR, 'Invalid Bech32 checksum', [9]) self.check_invalid(BECH32_TWO_ERRORS, 'Invalid Bech32 checksum', [22, 43]) @@ -105,7 +105,7 @@ class InvalidAddressErrorMessageTest(BitcoinTestFramework): def test_getaddressinfo(self): node = self.nodes[0] - assert_raises_rpc_error(-5, "Invalid Bech32 address program size (41 byte)", node.getaddressinfo, BECH32_INVALID_SIZE) + assert_raises_rpc_error(-5, "Invalid Bech32 address program size (41 bytes)", node.getaddressinfo, BECH32_INVALID_SIZE) assert_raises_rpc_error(-5, "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", node.getaddressinfo, BECH32_INVALID_PREFIX) assert_raises_rpc_error(-5, "Invalid or unsupported Base58-encoded address.", node.getaddressinfo, BASE58_INVALID_PREFIX) assert_raises_rpc_error(-5, "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", node.getaddressinfo, INVALID_ADDRESS) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 7c653cc315..c4ed4da0f2 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -8,7 +8,7 @@ from decimal import Decimal from itertools import product from test_framework.descriptors import descsum_create -from test_framework.key import ECKey, H_POINT +from test_framework.key import H_POINT from test_framework.messages import ( COutPoint, CTransaction, @@ -43,8 +43,8 @@ from test_framework.util import ( random_bytes, ) from test_framework.wallet_util import ( - bytes_to_wif, - get_generate_key + generate_keypair, + get_generate_key, ) import json @@ -376,7 +376,7 @@ class PSBTTest(BitcoinTestFramework): self.log.info("Test various PSBT operations") # partially sign multisig things with node 1 - psbtx = wmulti.walletcreatefundedpsbt(inputs=[{"txid":txid,"vout":p2wsh_pos},{"txid":txid,"vout":p2sh_pos},{"txid":txid,"vout":p2sh_p2wsh_pos}], outputs={self.nodes[1].getnewaddress():29.99}, options={'changeAddress': self.nodes[1].getrawchangeaddress()})['psbt'] + psbtx = wmulti.walletcreatefundedpsbt(inputs=[{"txid":txid,"vout":p2wsh_pos},{"txid":txid,"vout":p2sh_pos},{"txid":txid,"vout":p2sh_p2wsh_pos}], outputs={self.nodes[1].getnewaddress():29.99}, changeAddress=self.nodes[1].getrawchangeaddress())['psbt'] walletprocesspsbt_out = self.nodes[1].walletprocesspsbt(psbtx) psbtx = walletprocesspsbt_out['psbt'] assert_equal(walletprocesspsbt_out['complete'], False) @@ -710,9 +710,7 @@ class PSBTTest(BitcoinTestFramework): self.log.info("Test that we can fund psbts with external inputs specified") - eckey = ECKey() - eckey.generate() - privkey = bytes_to_wif(eckey.get_bytes()) + privkey, _ = generate_keypair(wif=True) self.nodes[1].createwallet("extfund") wallet = self.nodes[1].get_wallet_rpc("extfund") @@ -779,7 +777,7 @@ class PSBTTest(BitcoinTestFramework): psbt = wallet.walletcreatefundedpsbt( inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": input_weight}], outputs={self.nodes[0].getnewaddress(): 15}, - options={"add_inputs": True} + add_inputs=True, ) signed = wallet.walletprocesspsbt(psbt["psbt"]) signed = self.nodes[0].walletprocesspsbt(signed["psbt"]) @@ -789,21 +787,21 @@ class PSBTTest(BitcoinTestFramework): psbt2 = wallet.walletcreatefundedpsbt( inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": low_input_weight}], outputs={self.nodes[0].getnewaddress(): 15}, - options={"add_inputs": True} + add_inputs=True, ) assert_greater_than(psbt["fee"], psbt2["fee"]) # Increasing the weight should have a higher fee psbt2 = wallet.walletcreatefundedpsbt( inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], outputs={self.nodes[0].getnewaddress(): 15}, - options={"add_inputs": True} + add_inputs=True, ) assert_greater_than(psbt2["fee"], psbt["fee"]) # The provided weight should override the calculated weight when solving data is provided psbt3 = wallet.walletcreatefundedpsbt( inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], outputs={self.nodes[0].getnewaddress(): 15}, - options={'add_inputs': True, "solving_data":{"descriptors": [desc]}} + add_inputs=True, solving_data={"descriptors": [desc]}, ) assert_equal(psbt2["fee"], psbt3["fee"]) @@ -817,7 +815,7 @@ class PSBTTest(BitcoinTestFramework): psbt3 = wallet.walletcreatefundedpsbt( inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], outputs={self.nodes[0].getnewaddress(): 15}, - options={"add_inputs": True} + add_inputs=True, ) assert_equal(psbt2["fee"], psbt3["fee"]) @@ -825,11 +823,9 @@ class PSBTTest(BitcoinTestFramework): self.nodes[1].createwallet(wallet_name="scriptwatchonly", disable_private_keys=True) watchonly = self.nodes[1].get_wallet_rpc("scriptwatchonly") - eckey = ECKey() - eckey.generate() - privkey = bytes_to_wif(eckey.get_bytes()) + privkey, pubkey = generate_keypair(wif=True) - desc = descsum_create("wsh(pkh({}))".format(eckey.get_pubkey().get_bytes().hex())) + desc = descsum_create("wsh(pkh({}))".format(pubkey.hex())) if self.options.descriptors: res = watchonly.importdescriptors([{"desc": desc, "timestamp": "now"}]) else: @@ -846,11 +842,9 @@ class PSBTTest(BitcoinTestFramework): # Same test but for taproot if self.options.descriptors: - eckey = ECKey() - eckey.generate() - privkey = bytes_to_wif(eckey.get_bytes()) + privkey, pubkey = generate_keypair(wif=True) - desc = descsum_create("tr({},pk({}))".format(H_POINT, eckey.get_pubkey().get_bytes().hex())) + desc = descsum_create("tr({},pk({}))".format(H_POINT, pubkey.hex())) res = watchonly.importdescriptors([{"desc": desc, "timestamp": "now"}]) assert res[0]["success"] addr = self.nodes[0].deriveaddresses(desc)[0] diff --git a/test/functional/rpc_signer.py b/test/functional/rpc_signer.py index 4300190387..5ba0d35835 100755 --- a/test/functional/rpc_signer.py +++ b/test/functional/rpc_signer.py @@ -27,9 +27,6 @@ class RPCSignerTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 - # The experimental syscall sandbox feature (-sandbox) is not compatible with -signer (which - # invokes execve). - self.disable_syscall_sandbox = True self.extra_args = [ [], diff --git a/test/functional/rpc_signrawtransactionwithkey.py b/test/functional/rpc_signrawtransactionwithkey.py index 580f63063d..ac7a86704f 100755 --- a/test/functional/rpc_signrawtransactionwithkey.py +++ b/test/functional/rpc_signrawtransactionwithkey.py @@ -11,7 +11,6 @@ from test_framework.address import ( address_to_scriptpubkey, script_to_p2sh, ) -from test_framework.key import ECKey from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -23,16 +22,16 @@ from test_framework.script_util import ( script_to_p2sh_p2wsh_script, script_to_p2wsh_script, ) +from test_framework.wallet import ( + getnewdestination, +) from test_framework.wallet_util import ( - bytes_to_wif, + generate_keypair, ) from decimal import ( Decimal, ) -from test_framework.wallet import ( - getnewdestination, -) class SignRawTransactionWithKeyTest(BitcoinTestFramework): @@ -80,11 +79,8 @@ class SignRawTransactionWithKeyTest(BitcoinTestFramework): def witness_script_test(self): self.log.info("Test signing transaction to P2SH-P2WSH addresses without wallet") # Create a new P2SH-P2WSH 1-of-1 multisig address: - eckey = ECKey() - eckey.generate() - embedded_privkey = bytes_to_wif(eckey.get_bytes()) - embedded_pubkey = eckey.get_pubkey().get_bytes().hex() - p2sh_p2wsh_address = self.nodes[1].createmultisig(1, [embedded_pubkey], "p2sh-segwit") + embedded_privkey, embedded_pubkey = generate_keypair(wif=True) + p2sh_p2wsh_address = self.nodes[1].createmultisig(1, [embedded_pubkey.hex()], "p2sh-segwit") # send transaction to P2SH-P2WSH 1-of-1 multisig address self.block_hash = self.generate(self.nodes[0], COINBASE_MATURITY + 1) self.blk_idx = 0 @@ -109,10 +105,7 @@ class SignRawTransactionWithKeyTest(BitcoinTestFramework): def verify_txn_with_witness_script(self, tx_type): self.log.info("Test with a {} script as the witnessScript".format(tx_type)) - eckey = ECKey() - eckey.generate() - embedded_privkey = bytes_to_wif(eckey.get_bytes()) - embedded_pubkey = eckey.get_pubkey().get_bytes().hex() + embedded_privkey, embedded_pubkey = generate_keypair(wif=True) witness_script = { 'P2PKH': key_to_p2pkh_script(embedded_pubkey).hex(), 'P2PK': key_to_p2pk_script(embedded_pubkey).hex() diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index 8cc3ec401e..66cdd7cf9a 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -7,11 +7,9 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - get_datadir_path, str_to_b64str, ) -import os import http.client import urllib.parse import subprocess @@ -38,8 +36,7 @@ class HTTPBasicsTest(BitcoinTestFramework): self.num_nodes = 2 self.supports_cli = False - def setup_chain(self): - super().setup_chain() + def conf_setup(self): #Append rpcauth to bitcoin.conf before initialization self.rtpassword = "cA773lm788buwYe4g4WT+05pKyNruVKjQ25x3n0DQcM=" rpcauth = "rpcauth=rt:93648e835a54c573682c2eb19f882535$7681e9c5b74bdd85e78166031d2058e1069b3ed7ed967c93fc63abba06f31144" @@ -64,13 +61,15 @@ class HTTPBasicsTest(BitcoinTestFramework): rpcauth3 = lines[1] self.password = lines[3] - with open(os.path.join(get_datadir_path(self.options.tmpdir, 0), "bitcoin.conf"), 'a', encoding='utf8') as f: + with open(self.nodes[0].datadir_path / "bitcoin.conf", "a", encoding="utf8") as f: f.write(rpcauth + "\n") f.write(rpcauth2 + "\n") f.write(rpcauth3 + "\n") - with open(os.path.join(get_datadir_path(self.options.tmpdir, 1), "bitcoin.conf"), 'a', encoding='utf8') as f: + with open(self.nodes[1].datadir_path / "bitcoin.conf", "a", encoding="utf8") as f: f.write("rpcuser={}\n".format(self.rpcuser)) f.write("rpcpassword={}\n".format(self.rpcpassword)) + self.restart_node(0) + self.restart_node(1) def test_auth(self, node, user, password): self.log.info('Correct...') @@ -86,6 +85,7 @@ class HTTPBasicsTest(BitcoinTestFramework): assert_equal(401, call_with_auth(node, user + 'wrong', password + 'wrong').status) def run_test(self): + self.conf_setup() self.log.info('Check correctness of the rpcauth config option') url = urllib.parse.urlparse(self.nodes[0].url) @@ -112,8 +112,7 @@ class HTTPBasicsTest(BitcoinTestFramework): self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo$bar$baz']) self.log.info('Check that failure to write cookie file will abort the node gracefully') - cookie_file = os.path.join(get_datadir_path(self.options.tmpdir, 0), self.chain, '.cookie.tmp') - os.mkdir(cookie_file) + (self.nodes[0].chain_path / ".cookie.tmp").mkdir() self.nodes[0].assert_start_raises_init_error(expected_msg=init_error) diff --git a/test/functional/rpc_validateaddress.py b/test/functional/rpc_validateaddress.py index 2ce6b35d04..d87ba098c3 100755 --- a/test/functional/rpc_validateaddress.py +++ b/test/functional/rpc_validateaddress.py @@ -33,7 +33,7 @@ INVALID_DATA = [ ), ( "BC1QR508D6QEJXTDG4Y5R3ZARVARYV98GJ9P", - "Invalid Bech32 v0 address program size (16 byte), per BIP141", + "Invalid Bech32 v0 address program size (16 bytes), per BIP141", [], ), ( @@ -101,12 +101,12 @@ INVALID_DATA = [ ("bc1pw5dgrnzv", "Invalid Bech32 address program size (1 byte)", []), ( "bc1p0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7v8n0nx0muaewav253zgeav", - "Invalid Bech32 address program size (41 byte)", + "Invalid Bech32 address program size (41 bytes)", [], ), ( "BC1QR508D6QEJXTDG4Y5R3ZARVARYV98GJ9P", - "Invalid Bech32 v0 address program size (16 byte), per BIP141", + "Invalid Bech32 v0 address program size (16 bytes), per BIP141", [], ), ( diff --git a/test/functional/rpc_whitelist.py b/test/functional/rpc_whitelist.py index 219132410b..fb404fb479 100755 --- a/test/functional/rpc_whitelist.py +++ b/test/functional/rpc_whitelist.py @@ -6,11 +6,9 @@ A test for RPC users with restricted permissions """ from test_framework.test_framework import BitcoinTestFramework -import os from test_framework.util import ( - get_datadir_path, assert_equal, - str_to_b64str + str_to_b64str, ) import http.client import urllib.parse @@ -30,8 +28,7 @@ class RPCWhitelistTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - def setup_chain(self): - super().setup_chain() + def run_test(self): # 0 => Username # 1 => Password (Hashed) # 2 => Permissions @@ -55,7 +52,7 @@ class RPCWhitelistTest(BitcoinTestFramework): ] # These commands shouldn't be allowed for any user to test failures self.never_allowed = ["getnetworkinfo"] - with open(os.path.join(get_datadir_path(self.options.tmpdir, 0), "bitcoin.conf"), 'a', encoding='utf8') as f: + with open(self.nodes[0].datadir_path / "bitcoin.conf", "a", encoding="utf8") as f: f.write("\nrpcwhitelistdefault=0\n") for user in self.users: f.write("rpcauth=" + user[0] + ":" + user[1] + "\n") @@ -64,9 +61,8 @@ class RPCWhitelistTest(BitcoinTestFramework): for strangedude in self.strange_users: f.write("rpcauth=" + strangedude[0] + ":" + strangedude[1] + "\n") f.write("rpcwhitelist=" + strangedude[0] + strangedude[2] + "\n") + self.restart_node(0) - - def run_test(self): for user in self.users: permissions = user[2].replace(" ", "").split(",") # Pop all empty items diff --git a/test/functional/test-shell.md b/test/functional/test-shell.md index 80f4e88109..b89b40f13d 100644 --- a/test/functional/test-shell.md +++ b/test/functional/test-shell.md @@ -37,13 +37,13 @@ importing the `TestShell` class from the `test_shell` sub-package. The following `TestShell` methods manage the lifetime of the underlying bitcoind processes and logging utilities. -* `TestShell.setup()` -* `TestShell.shutdown()` +* `TestShell().setup()` +* `TestShell().shutdown()` The `TestShell` inherits all `BitcoinTestFramework` members and methods, such as: -* `TestShell.nodes[index].rpc_method()` -* `TestShell.log.info("Custom log message")` +* `TestShell().nodes[index].rpc_method()` +* `TestShell().log.info("Custom log message")` The following sections demonstrate how to initialize, run, and shut down a `TestShell` object. @@ -143,7 +143,7 @@ instances and remove all temporary data and logging directories. 20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Tests successful ``` To prevent the logs from being removed after a shutdown, simply set the -`TestShell.options.nocleanup` member to `True`. +`TestShell().options.nocleanup` member to `True`. ``` >>> test.options.nocleanup = True >>> test.shutdown() @@ -162,9 +162,9 @@ underlying `BitcoinTestFramework`: The `TestShell` object initializes with the default settings inherited from the `BitcoinTestFramework` class. The user can override these in -`TestShell.setup(key=value)`. +`TestShell().setup(key=value)`. -**Note:** `TestShell.reset()` will reset test parameters to default values and +**Note:** `TestShell().reset()` will reset test parameters to default values and can be called after the TestShell is shut down. | Test parameter key | Default Value | Description | @@ -181,7 +181,7 @@ can be called after the TestShell is shut down. | `perf` | False | Profiles running nodes with `perf` for the duration of the test if set to `True`. | | `rpc_timeout` | `60` | Sets the RPC server timeout for the underlying bitcoind processes. | | `setup_clean_chain` | `False` | A 200-block-long chain is initialized from cache by default. Instead, `setup_clean_chain` initializes an empty blockchain if set to `True`. | -| `randomseed` | Random Integer | `TestShell.options.randomseed` is a member of `TestShell` which can be accessed during a test to seed a random generator. User can override default with a constant value for reproducible test runs. | +| `randomseed` | Random Integer | `TestShell().options.randomseed` is a member of `TestShell` which can be accessed during a test to seed a random generator. User can override default with a constant value for reproducible test runs. | | `supports_cli` | `False` | Whether the bitcoin-cli utility is compiled and available for the test. | | `tmpdir` | `"/var/folders/.../"` | Sets directory for test logs. Will be deleted upon a successful test run unless `nocleanup` is set to `True` | | `trace_rpc` | `False` | Logs all RPC calls if set to `True`. | diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index f7765a9dfa..03042877b2 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -39,6 +39,7 @@ from http import HTTPStatus import http.client import json import logging +import pathlib import socket import time import urllib.parse @@ -59,9 +60,11 @@ class JSONRPCException(Exception): self.http_status = http_status -def EncodeDecimal(o): +def serialization_fallback(o): if isinstance(o, decimal.Decimal): return str(o) + if isinstance(o, pathlib.Path): + return str(o) raise TypeError(repr(o) + " is not JSON serializable") class AuthServiceProxy(): @@ -108,7 +111,7 @@ class AuthServiceProxy(): log.debug("-{}-> {} {}".format( AuthServiceProxy.__id_count, self._service_name, - json.dumps(args or argsn, default=EncodeDecimal, ensure_ascii=self.ensure_ascii), + json.dumps(args or argsn, default=serialization_fallback, ensure_ascii=self.ensure_ascii), )) if args and argsn: params = dict(args=args, **argsn) @@ -120,7 +123,7 @@ class AuthServiceProxy(): 'id': AuthServiceProxy.__id_count} def __call__(self, *args, **argsn): - postdata = json.dumps(self.get_request(*args, **argsn), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) + postdata = json.dumps(self.get_request(*args, **argsn), default=serialization_fallback, ensure_ascii=self.ensure_ascii) response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) if response['error'] is not None: raise JSONRPCException(response['error'], status) @@ -134,7 +137,7 @@ class AuthServiceProxy(): return response['result'] def batch(self, rpc_call_list): - postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) + postdata = json.dumps(list(rpc_call_list), default=serialization_fallback, ensure_ascii=self.ensure_ascii) log.debug("--> " + postdata) response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) if status != HTTPStatus.OK: @@ -167,7 +170,7 @@ class AuthServiceProxy(): response = json.loads(responsedata, parse_float=decimal.Decimal) elapsed = time.time() - req_start_time if "error" in response and response["error"] is None: - log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) + log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, json.dumps(response["result"], default=serialization_fallback, ensure_ascii=self.ensure_ascii))) else: log.debug("<-- [%.6f] %s" % (elapsed, responsedata)) return response, http_response.status diff --git a/test/functional/test_framework/coverage.py b/test/functional/test_framework/coverage.py index 4fb4f8bb82..912a945d95 100644 --- a/test/functional/test_framework/coverage.py +++ b/test/functional/test_framework/coverage.py @@ -11,6 +11,7 @@ testing. import os from .authproxy import AuthServiceProxy +from typing import Optional REFERENCE_FILENAME = 'rpc_interface.txt' @@ -20,7 +21,7 @@ class AuthServiceProxyWrapper(): An object that wraps AuthServiceProxy to record specific RPC calls. """ - def __init__(self, auth_service_proxy_instance: AuthServiceProxy, rpc_url: str, coverage_logfile: str=None): + def __init__(self, auth_service_proxy_instance: AuthServiceProxy, rpc_url: str, coverage_logfile: Optional[str]=None): """ Kwargs: auth_service_proxy_instance: the instance being wrapped. diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index efb4934ff0..c250fc6fe8 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -1,7 +1,7 @@ # Copyright (c) 2019-2020 Pieter Wuille # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test-only secp256k1 elliptic curve implementation +"""Test-only secp256k1 elliptic curve protocols implementation WARNING: This code is slow, uses bad randomness, does not properly protect keys, and is trivially vulnerable to side channel attacks. Do not use for @@ -13,9 +13,13 @@ import os import random import unittest +from test_framework import secp256k1 + # Point with no known discrete log. H_POINT = "50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0" +# Order of the secp256k1 curve +ORDER = secp256k1.GE.ORDER def TaggedHash(tag, data): ss = hashlib.sha256(tag.encode('utf-8')).digest() @@ -23,233 +27,18 @@ def TaggedHash(tag, data): ss += data return hashlib.sha256(ss).digest() -def jacobi_symbol(n, k): - """Compute the Jacobi symbol of n modulo k - - See https://en.wikipedia.org/wiki/Jacobi_symbol - - For our application k is always prime, so this is the same as the Legendre symbol.""" - assert k > 0 and k & 1, "jacobi symbol is only defined for positive odd k" - n %= k - t = 0 - while n != 0: - while n & 1 == 0: - n >>= 1 - r = k & 7 - t ^= (r == 3 or r == 5) - n, k = k, n - t ^= (n & k & 3 == 3) - n = n % k - if k == 1: - return -1 if t else 1 - return 0 - -def modsqrt(a, p): - """Compute the square root of a modulo p when p % 4 = 3. - - The Tonelli-Shanks algorithm can be used. See https://en.wikipedia.org/wiki/Tonelli-Shanks_algorithm - - Limiting this function to only work for p % 4 = 3 means we don't need to - iterate through the loop. The highest n such that p - 1 = 2^n Q with Q odd - is n = 1. Therefore Q = (p-1)/2 and sqrt = a^((Q+1)/2) = a^((p+1)/4) - - secp256k1's is defined over field of size 2**256 - 2**32 - 977, which is 3 mod 4. - """ - if p % 4 != 3: - raise NotImplementedError("modsqrt only implemented for p % 4 = 3") - sqrt = pow(a, (p + 1)//4, p) - if pow(sqrt, 2, p) == a % p: - return sqrt - return None - -class EllipticCurve: - def __init__(self, p, a, b): - """Initialize elliptic curve y^2 = x^3 + a*x + b over GF(p).""" - self.p = p - self.a = a % p - self.b = b % p - - def affine(self, p1): - """Convert a Jacobian point tuple p1 to affine form, or None if at infinity. - - An affine point is represented as the Jacobian (x, y, 1)""" - x1, y1, z1 = p1 - if z1 == 0: - return None - inv = pow(z1, -1, self.p) - inv_2 = (inv**2) % self.p - inv_3 = (inv_2 * inv) % self.p - return ((inv_2 * x1) % self.p, (inv_3 * y1) % self.p, 1) - - def has_even_y(self, p1): - """Whether the point p1 has an even Y coordinate when expressed in affine coordinates.""" - return not (p1[2] == 0 or self.affine(p1)[1] & 1) - - def negate(self, p1): - """Negate a Jacobian point tuple p1.""" - x1, y1, z1 = p1 - return (x1, (self.p - y1) % self.p, z1) - - def on_curve(self, p1): - """Determine whether a Jacobian tuple p is on the curve (and not infinity)""" - x1, y1, z1 = p1 - z2 = pow(z1, 2, self.p) - z4 = pow(z2, 2, self.p) - return z1 != 0 and (pow(x1, 3, self.p) + self.a * x1 * z4 + self.b * z2 * z4 - pow(y1, 2, self.p)) % self.p == 0 - - def is_x_coord(self, x): - """Test whether x is a valid X coordinate on the curve.""" - x_3 = pow(x, 3, self.p) - return jacobi_symbol(x_3 + self.a * x + self.b, self.p) != -1 - - def lift_x(self, x): - """Given an X coordinate on the curve, return a corresponding affine point for which the Y coordinate is even.""" - x_3 = pow(x, 3, self.p) - v = x_3 + self.a * x + self.b - y = modsqrt(v, self.p) - if y is None: - return None - return (x, self.p - y if y & 1 else y, 1) - - def double(self, p1): - """Double a Jacobian tuple p1 - - See https://en.wikibooks.org/wiki/Cryptography/Prime_Curve/Jacobian_Coordinates - Point Doubling""" - x1, y1, z1 = p1 - if z1 == 0: - return (0, 1, 0) - y1_2 = (y1**2) % self.p - y1_4 = (y1_2**2) % self.p - x1_2 = (x1**2) % self.p - s = (4*x1*y1_2) % self.p - m = 3*x1_2 - if self.a: - m += self.a * pow(z1, 4, self.p) - m = m % self.p - x2 = (m**2 - 2*s) % self.p - y2 = (m*(s - x2) - 8*y1_4) % self.p - z2 = (2*y1*z1) % self.p - return (x2, y2, z2) - - def add_mixed(self, p1, p2): - """Add a Jacobian tuple p1 and an affine tuple p2 - - See https://en.wikibooks.org/wiki/Cryptography/Prime_Curve/Jacobian_Coordinates - Point Addition (with affine point)""" - x1, y1, z1 = p1 - x2, y2, z2 = p2 - assert z2 == 1 - # Adding to the point at infinity is a no-op - if z1 == 0: - return p2 - z1_2 = (z1**2) % self.p - z1_3 = (z1_2 * z1) % self.p - u2 = (x2 * z1_2) % self.p - s2 = (y2 * z1_3) % self.p - if x1 == u2: - if (y1 != s2): - # p1 and p2 are inverses. Return the point at infinity. - return (0, 1, 0) - # p1 == p2. The formulas below fail when the two points are equal. - return self.double(p1) - h = u2 - x1 - r = s2 - y1 - h_2 = (h**2) % self.p - h_3 = (h_2 * h) % self.p - u1_h_2 = (x1 * h_2) % self.p - x3 = (r**2 - h_3 - 2*u1_h_2) % self.p - y3 = (r*(u1_h_2 - x3) - y1*h_3) % self.p - z3 = (h*z1) % self.p - return (x3, y3, z3) - - def add(self, p1, p2): - """Add two Jacobian tuples p1 and p2 - - See https://en.wikibooks.org/wiki/Cryptography/Prime_Curve/Jacobian_Coordinates - Point Addition""" - x1, y1, z1 = p1 - x2, y2, z2 = p2 - # Adding the point at infinity is a no-op - if z1 == 0: - return p2 - if z2 == 0: - return p1 - # Adding an Affine to a Jacobian is more efficient since we save field multiplications and squarings when z = 1 - if z1 == 1: - return self.add_mixed(p2, p1) - if z2 == 1: - return self.add_mixed(p1, p2) - z1_2 = (z1**2) % self.p - z1_3 = (z1_2 * z1) % self.p - z2_2 = (z2**2) % self.p - z2_3 = (z2_2 * z2) % self.p - u1 = (x1 * z2_2) % self.p - u2 = (x2 * z1_2) % self.p - s1 = (y1 * z2_3) % self.p - s2 = (y2 * z1_3) % self.p - if u1 == u2: - if (s1 != s2): - # p1 and p2 are inverses. Return the point at infinity. - return (0, 1, 0) - # p1 == p2. The formulas below fail when the two points are equal. - return self.double(p1) - h = u2 - u1 - r = s2 - s1 - h_2 = (h**2) % self.p - h_3 = (h_2 * h) % self.p - u1_h_2 = (u1 * h_2) % self.p - x3 = (r**2 - h_3 - 2*u1_h_2) % self.p - y3 = (r*(u1_h_2 - x3) - s1*h_3) % self.p - z3 = (h*z1*z2) % self.p - return (x3, y3, z3) - - def mul(self, ps): - """Compute a (multi) point multiplication - - ps is a list of (Jacobian tuple, scalar) pairs. - """ - r = (0, 1, 0) - for i in range(255, -1, -1): - r = self.double(r) - for (p, n) in ps: - if ((n >> i) & 1): - r = self.add(r, p) - return r - -SECP256K1_FIELD_SIZE = 2**256 - 2**32 - 977 -SECP256K1 = EllipticCurve(SECP256K1_FIELD_SIZE, 0, 7) -SECP256K1_G = (0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798, 0x483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8, 1) -SECP256K1_ORDER = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 -SECP256K1_ORDER_HALF = SECP256K1_ORDER // 2 - -class ECPubKey(): + +class ECPubKey: """A secp256k1 public key""" def __init__(self): """Construct an uninitialized public key""" - self.valid = False + self.p = None def set(self, data): """Construct a public key from a serialization in compressed or uncompressed format""" - if (len(data) == 65 and data[0] == 0x04): - p = (int.from_bytes(data[1:33], 'big'), int.from_bytes(data[33:65], 'big'), 1) - self.valid = SECP256K1.on_curve(p) - if self.valid: - self.p = p - self.compressed = False - elif (len(data) == 33 and (data[0] == 0x02 or data[0] == 0x03)): - x = int.from_bytes(data[1:33], 'big') - if SECP256K1.is_x_coord(x): - p = SECP256K1.lift_x(x) - # Make the Y coordinate odd if required (lift_x always produces - # a point with an even Y coordinate). - if data[0] & 1: - p = SECP256K1.negate(p) - self.p = p - self.valid = True - self.compressed = True - else: - self.valid = False - else: - self.valid = False + self.p = secp256k1.GE.from_bytes(data) + self.compressed = len(data) == 33 @property def is_compressed(self): @@ -257,24 +46,21 @@ class ECPubKey(): @property def is_valid(self): - return self.valid + return self.p is not None def get_bytes(self): - assert self.valid - p = SECP256K1.affine(self.p) - if p is None: - return None + assert self.is_valid if self.compressed: - return bytes([0x02 + (p[1] & 1)]) + p[0].to_bytes(32, 'big') + return self.p.to_bytes_compressed() else: - return bytes([0x04]) + p[0].to_bytes(32, 'big') + p[1].to_bytes(32, 'big') + return self.p.to_bytes_uncompressed() def verify_ecdsa(self, sig, msg, low_s=True): """Verify a strictly DER-encoded ECDSA signature against this pubkey. See https://en.wikipedia.org/wiki/Elliptic_Curve_Digital_Signature_Algorithm for the ECDSA verifier algorithm""" - assert self.valid + assert self.is_valid # Extract r and s from the DER formatted signature. Return false for # any DER encoding errors. @@ -310,24 +96,22 @@ class ECPubKey(): s = int.from_bytes(sig[6+rlen:6+rlen+slen], 'big') # Verify that r and s are within the group order - if r < 1 or s < 1 or r >= SECP256K1_ORDER or s >= SECP256K1_ORDER: + if r < 1 or s < 1 or r >= ORDER or s >= ORDER: return False - if low_s and s >= SECP256K1_ORDER_HALF: + if low_s and s >= secp256k1.GE.ORDER_HALF: return False z = int.from_bytes(msg, 'big') # Run verifier algorithm on r, s - w = pow(s, -1, SECP256K1_ORDER) - u1 = z*w % SECP256K1_ORDER - u2 = r*w % SECP256K1_ORDER - R = SECP256K1.affine(SECP256K1.mul([(SECP256K1_G, u1), (self.p, u2)])) - if R is None or (R[0] % SECP256K1_ORDER) != r: + w = pow(s, -1, ORDER) + R = secp256k1.GE.mul((z * w, secp256k1.G), (r * w, self.p)) + if R.infinity or (int(R.x) % ORDER) != r: return False return True def generate_privkey(): """Generate a valid random 32-byte private key.""" - return random.randrange(1, SECP256K1_ORDER).to_bytes(32, 'big') + return random.randrange(1, ORDER).to_bytes(32, 'big') def rfc6979_nonce(key): """Compute signing nonce using RFC6979.""" @@ -339,7 +123,7 @@ def rfc6979_nonce(key): v = hmac.new(k, v, 'sha256').digest() return hmac.new(k, v, 'sha256').digest() -class ECKey(): +class ECKey: """A secp256k1 private key""" def __init__(self): @@ -349,7 +133,7 @@ class ECKey(): """Construct a private key object with given 32-byte secret and compressed flag.""" assert len(secret) == 32 secret = int.from_bytes(secret, 'big') - self.valid = (secret > 0 and secret < SECP256K1_ORDER) + self.valid = (secret > 0 and secret < ORDER) if self.valid: self.secret = secret self.compressed = compressed @@ -375,9 +159,7 @@ class ECKey(): """Compute an ECPubKey object for this secret key.""" assert self.valid ret = ECPubKey() - p = SECP256K1.mul([(SECP256K1_G, self.secret)]) - ret.p = p - ret.valid = True + ret.p = self.secret * secp256k1.G ret.compressed = self.compressed return ret @@ -392,12 +174,12 @@ class ECKey(): if rfc6979: k = int.from_bytes(rfc6979_nonce(self.secret.to_bytes(32, 'big') + msg), 'big') else: - k = random.randrange(1, SECP256K1_ORDER) - R = SECP256K1.affine(SECP256K1.mul([(SECP256K1_G, k)])) - r = R[0] % SECP256K1_ORDER - s = (pow(k, -1, SECP256K1_ORDER) * (z + self.secret * r)) % SECP256K1_ORDER - if low_s and s > SECP256K1_ORDER_HALF: - s = SECP256K1_ORDER - s + k = random.randrange(1, ORDER) + R = k * secp256k1.G + r = int(R.x) % ORDER + s = (pow(k, -1, ORDER) * (z + self.secret * r)) % ORDER + if low_s and s > secp256k1.GE.ORDER_HALF: + s = ORDER - s # Represent in DER format. The byte representations of r and s have # length rounded up (255 bits becomes 32 bytes and 256 bits becomes 33 # bytes). @@ -413,10 +195,10 @@ def compute_xonly_pubkey(key): assert len(key) == 32 x = int.from_bytes(key, 'big') - if x == 0 or x >= SECP256K1_ORDER: + if x == 0 or x >= ORDER: return (None, None) - P = SECP256K1.affine(SECP256K1.mul([(SECP256K1_G, x)])) - return (P[0].to_bytes(32, 'big'), not SECP256K1.has_even_y(P)) + P = x * secp256k1.G + return (P.to_bytes_xonly(), not P.y.is_even()) def tweak_add_privkey(key, tweak): """Tweak a private key (after negating it if needed).""" @@ -425,14 +207,14 @@ def tweak_add_privkey(key, tweak): assert len(tweak) == 32 x = int.from_bytes(key, 'big') - if x == 0 or x >= SECP256K1_ORDER: + if x == 0 or x >= ORDER: return None - if not SECP256K1.has_even_y(SECP256K1.mul([(SECP256K1_G, x)])): - x = SECP256K1_ORDER - x + if not (x * secp256k1.G).y.is_even(): + x = ORDER - x t = int.from_bytes(tweak, 'big') - if t >= SECP256K1_ORDER: + if t >= ORDER: return None - x = (x + t) % SECP256K1_ORDER + x = (x + t) % ORDER if x == 0: return None return x.to_bytes(32, 'big') @@ -443,19 +225,16 @@ def tweak_add_pubkey(key, tweak): assert len(key) == 32 assert len(tweak) == 32 - x_coord = int.from_bytes(key, 'big') - if x_coord >= SECP256K1_FIELD_SIZE: - return None - P = SECP256K1.lift_x(x_coord) + P = secp256k1.GE.from_bytes_xonly(key) if P is None: return None t = int.from_bytes(tweak, 'big') - if t >= SECP256K1_ORDER: + if t >= ORDER: return None - Q = SECP256K1.affine(SECP256K1.mul([(SECP256K1_G, t), (P, 1)])) - if Q is None: + Q = t * secp256k1.G + P + if Q.infinity: return None - return (Q[0].to_bytes(32, 'big'), not SECP256K1.has_even_y(Q)) + return (Q.to_bytes_xonly(), not Q.y.is_even()) def verify_schnorr(key, sig, msg): """Verify a Schnorr signature (see BIP 340). @@ -468,23 +247,20 @@ def verify_schnorr(key, sig, msg): assert len(msg) == 32 assert len(sig) == 64 - x_coord = int.from_bytes(key, 'big') - if x_coord == 0 or x_coord >= SECP256K1_FIELD_SIZE: - return False - P = SECP256K1.lift_x(x_coord) + P = secp256k1.GE.from_bytes_xonly(key) if P is None: return False r = int.from_bytes(sig[0:32], 'big') - if r >= SECP256K1_FIELD_SIZE: + if r >= secp256k1.FE.SIZE: return False s = int.from_bytes(sig[32:64], 'big') - if s >= SECP256K1_ORDER: + if s >= ORDER: return False - e = int.from_bytes(TaggedHash("BIP0340/challenge", sig[0:32] + key + msg), 'big') % SECP256K1_ORDER - R = SECP256K1.mul([(SECP256K1_G, s), (P, SECP256K1_ORDER - e)]) - if not SECP256K1.has_even_y(R): + e = int.from_bytes(TaggedHash("BIP0340/challenge", sig[0:32] + key + msg), 'big') % ORDER + R = secp256k1.GE.mul((s, secp256k1.G), (-e, P)) + if R.infinity or not R.y.is_even(): return False - if ((r * R[2] * R[2]) % SECP256K1_FIELD_SIZE) != R[0]: + if r != R.x: return False return True @@ -499,23 +275,24 @@ def sign_schnorr(key, msg, aux=None, flip_p=False, flip_r=False): assert len(aux) == 32 sec = int.from_bytes(key, 'big') - if sec == 0 or sec >= SECP256K1_ORDER: + if sec == 0 or sec >= ORDER: return None - P = SECP256K1.affine(SECP256K1.mul([(SECP256K1_G, sec)])) - if SECP256K1.has_even_y(P) == flip_p: - sec = SECP256K1_ORDER - sec + P = sec * secp256k1.G + if P.y.is_even() == flip_p: + sec = ORDER - sec t = (sec ^ int.from_bytes(TaggedHash("BIP0340/aux", aux), 'big')).to_bytes(32, 'big') - kp = int.from_bytes(TaggedHash("BIP0340/nonce", t + P[0].to_bytes(32, 'big') + msg), 'big') % SECP256K1_ORDER + kp = int.from_bytes(TaggedHash("BIP0340/nonce", t + P.to_bytes_xonly() + msg), 'big') % ORDER assert kp != 0 - R = SECP256K1.affine(SECP256K1.mul([(SECP256K1_G, kp)])) - k = kp if SECP256K1.has_even_y(R) != flip_r else SECP256K1_ORDER - kp - e = int.from_bytes(TaggedHash("BIP0340/challenge", R[0].to_bytes(32, 'big') + P[0].to_bytes(32, 'big') + msg), 'big') % SECP256K1_ORDER - return R[0].to_bytes(32, 'big') + ((k + e * sec) % SECP256K1_ORDER).to_bytes(32, 'big') + R = kp * secp256k1.G + k = kp if R.y.is_even() != flip_r else ORDER - kp + e = int.from_bytes(TaggedHash("BIP0340/challenge", R.to_bytes_xonly() + P.to_bytes_xonly() + msg), 'big') % ORDER + return R.to_bytes_xonly() + ((k + e * sec) % ORDER).to_bytes(32, 'big') + class TestFrameworkKey(unittest.TestCase): def test_schnorr(self): """Test the Python Schnorr implementation.""" - byte_arrays = [generate_privkey() for _ in range(3)] + [v.to_bytes(32, 'big') for v in [0, SECP256K1_ORDER - 1, SECP256K1_ORDER, 2**256 - 1]] + byte_arrays = [generate_privkey() for _ in range(3)] + [v.to_bytes(32, 'big') for v in [0, ORDER - 1, ORDER, 2**256 - 1]] keys = {} for privkey in byte_arrays: # build array of key/pubkey pairs pubkey, _ = compute_xonly_pubkey(privkey) diff --git a/test/functional/test_framework/secp256k1.py b/test/functional/test_framework/secp256k1.py new file mode 100644 index 0000000000..2e9e419da5 --- /dev/null +++ b/test/functional/test_framework/secp256k1.py @@ -0,0 +1,346 @@ +# Copyright (c) 2022-2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only implementation of low-level secp256k1 field and group arithmetic + +It is designed for ease of understanding, not performance. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. + +Exports: +* FE: class for secp256k1 field elements +* GE: class for secp256k1 group elements +* G: the secp256k1 generator point +""" + + +class FE: + """Objects of this class represent elements of the field GF(2**256 - 2**32 - 977). + + They are represented internally in numerator / denominator form, in order to delay inversions. + """ + + # The size of the field (also its modulus and characteristic). + SIZE = 2**256 - 2**32 - 977 + + def __init__(self, a=0, b=1): + """Initialize a field element a/b; both a and b can be ints or field elements.""" + if isinstance(a, FE): + num = a._num + den = a._den + else: + num = a % FE.SIZE + den = 1 + if isinstance(b, FE): + den = (den * b._num) % FE.SIZE + num = (num * b._den) % FE.SIZE + else: + den = (den * b) % FE.SIZE + assert den != 0 + if num == 0: + den = 1 + self._num = num + self._den = den + + def __add__(self, a): + """Compute the sum of two field elements (second may be int).""" + if isinstance(a, FE): + return FE(self._num * a._den + self._den * a._num, self._den * a._den) + return FE(self._num + self._den * a, self._den) + + def __radd__(self, a): + """Compute the sum of an integer and a field element.""" + return FE(a) + self + + def __sub__(self, a): + """Compute the difference of two field elements (second may be int).""" + if isinstance(a, FE): + return FE(self._num * a._den - self._den * a._num, self._den * a._den) + return FE(self._num - self._den * a, self._den) + + def __rsub__(self, a): + """Compute the difference of an integer and a field element.""" + return FE(a) - self + + def __mul__(self, a): + """Compute the product of two field elements (second may be int).""" + if isinstance(a, FE): + return FE(self._num * a._num, self._den * a._den) + return FE(self._num * a, self._den) + + def __rmul__(self, a): + """Compute the product of an integer with a field element.""" + return FE(a) * self + + def __truediv__(self, a): + """Compute the ratio of two field elements (second may be int).""" + return FE(self, a) + + def __pow__(self, a): + """Raise a field element to an integer power.""" + return FE(pow(self._num, a, FE.SIZE), pow(self._den, a, FE.SIZE)) + + def __neg__(self): + """Negate a field element.""" + return FE(-self._num, self._den) + + def __int__(self): + """Convert a field element to an integer in range 0..p-1. The result is cached.""" + if self._den != 1: + self._num = (self._num * pow(self._den, -1, FE.SIZE)) % FE.SIZE + self._den = 1 + return self._num + + def sqrt(self): + """Compute the square root of a field element if it exists (None otherwise). + + Due to the fact that our modulus is of the form (p % 4) == 3, the Tonelli-Shanks + algorithm (https://en.wikipedia.org/wiki/Tonelli-Shanks_algorithm) is simply + raising the argument to the power (p + 1) / 4. + + To see why: (p-1) % 2 = 0, so 2 divides the order of the multiplicative group, + and thus only half of the non-zero field elements are squares. An element a is + a (nonzero) square when Euler's criterion, a^((p-1)/2) = 1 (mod p), holds. We're + looking for x such that x^2 = a (mod p). Given a^((p-1)/2) = 1, that is equivalent + to x^2 = a^(1 + (p-1)/2) mod p. As (1 + (p-1)/2) is even, this is equivalent to + x = a^((1 + (p-1)/2)/2) mod p, or x = a^((p+1)/4) mod p.""" + v = int(self) + s = pow(v, (FE.SIZE + 1) // 4, FE.SIZE) + if s**2 % FE.SIZE == v: + return FE(s) + return None + + def is_square(self): + """Determine if this field element has a square root.""" + # A more efficient algorithm is possible here (Jacobi symbol). + return self.sqrt() is not None + + def is_even(self): + """Determine whether this field element, represented as integer in 0..p-1, is even.""" + return int(self) & 1 == 0 + + def __eq__(self, a): + """Check whether two field elements are equal (second may be an int).""" + if isinstance(a, FE): + return (self._num * a._den - self._den * a._num) % FE.SIZE == 0 + return (self._num - self._den * a) % FE.SIZE == 0 + + def to_bytes(self): + """Convert a field element to a 32-byte array (BE byte order).""" + return int(self).to_bytes(32, 'big') + + @staticmethod + def from_bytes(b): + """Convert a 32-byte array to a field element (BE byte order, no overflow allowed).""" + v = int.from_bytes(b, 'big') + if v >= FE.SIZE: + return None + return FE(v) + + def __str__(self): + """Convert this field element to a 64 character hex string.""" + return f"{int(self):064x}" + + def __repr__(self): + """Get a string representation of this field element.""" + return f"FE(0x{int(self):x})" + + +class GE: + """Objects of this class represent secp256k1 group elements (curve points or infinity) + + Normal points on the curve have fields: + * x: the x coordinate (a field element) + * y: the y coordinate (a field element, satisfying y^2 = x^3 + 7) + * infinity: False + + The point at infinity has field: + * infinity: True + """ + + # Order of the group (number of points on the curve, plus 1 for infinity) + ORDER = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 + + # Number of valid distinct x coordinates on the curve. + ORDER_HALF = ORDER // 2 + + def __init__(self, x=None, y=None): + """Initialize a group element with specified x and y coordinates, or infinity.""" + if x is None: + # Initialize as infinity. + assert y is None + self.infinity = True + else: + # Initialize as point on the curve (and check that it is). + fx = FE(x) + fy = FE(y) + assert fy**2 == fx**3 + 7 + self.infinity = False + self.x = fx + self.y = fy + + def __add__(self, a): + """Add two group elements together.""" + # Deal with infinity: a + infinity == infinity + a == a. + if self.infinity: + return a + if a.infinity: + return self + if self.x == a.x: + if self.y != a.y: + # A point added to its own negation is infinity. + assert self.y + a.y == 0 + return GE() + else: + # For identical inputs, use the tangent (doubling formula). + lam = (3 * self.x**2) / (2 * self.y) + else: + # For distinct inputs, use the line through both points (adding formula). + lam = (self.y - a.y) / (self.x - a.x) + # Determine point opposite to the intersection of that line with the curve. + x = lam**2 - (self.x + a.x) + y = lam * (self.x - x) - self.y + return GE(x, y) + + @staticmethod + def mul(*aps): + """Compute a (batch) scalar group element multiplication. + + GE.mul((a1, p1), (a2, p2), (a3, p3)) is identical to a1*p1 + a2*p2 + a3*p3, + but more efficient.""" + # Reduce all the scalars modulo order first (so we can deal with negatives etc). + naps = [(a % GE.ORDER, p) for a, p in aps] + # Start with point at infinity. + r = GE() + # Iterate over all bit positions, from high to low. + for i in range(255, -1, -1): + # Double what we have so far. + r = r + r + # Add then add the points for which the corresponding scalar bit is set. + for (a, p) in naps: + if (a >> i) & 1: + r += p + return r + + def __rmul__(self, a): + """Multiply an integer with a group element.""" + if self == G: + return FAST_G.mul(a) + return GE.mul((a, self)) + + def __neg__(self): + """Compute the negation of a group element.""" + if self.infinity: + return self + return GE(self.x, -self.y) + + def to_bytes_compressed(self): + """Convert a non-infinite group element to 33-byte compressed encoding.""" + assert not self.infinity + return bytes([3 - self.y.is_even()]) + self.x.to_bytes() + + def to_bytes_uncompressed(self): + """Convert a non-infinite group element to 65-byte uncompressed encoding.""" + assert not self.infinity + return b'\x04' + self.x.to_bytes() + self.y.to_bytes() + + def to_bytes_xonly(self): + """Convert (the x coordinate of) a non-infinite group element to 32-byte xonly encoding.""" + assert not self.infinity + return self.x.to_bytes() + + @staticmethod + def lift_x(x): + """Return group element with specified field element as x coordinate (and even y).""" + y = (FE(x)**3 + 7).sqrt() + if y is None: + return None + if not y.is_even(): + y = -y + return GE(x, y) + + @staticmethod + def from_bytes(b): + """Convert a compressed or uncompressed encoding to a group element.""" + assert len(b) in (33, 65) + if len(b) == 33: + if b[0] != 2 and b[0] != 3: + return None + x = FE.from_bytes(b[1:]) + if x is None: + return None + r = GE.lift_x(x) + if r is None: + return None + if b[0] == 3: + r = -r + return r + else: + if b[0] != 4: + return None + x = FE.from_bytes(b[1:33]) + y = FE.from_bytes(b[33:]) + if y**2 != x**3 + 7: + return None + return GE(x, y) + + @staticmethod + def from_bytes_xonly(b): + """Convert a point given in xonly encoding to a group element.""" + assert len(b) == 32 + x = FE.from_bytes(b) + if x is None: + return None + return GE.lift_x(x) + + @staticmethod + def is_valid_x(x): + """Determine whether the provided field element is a valid X coordinate.""" + return (FE(x)**3 + 7).is_square() + + def __str__(self): + """Convert this group element to a string.""" + if self.infinity: + return "(inf)" + return f"({self.x},{self.y})" + + def __repr__(self): + """Get a string representation for this group element.""" + if self.infinity: + return "GE()" + return f"GE(0x{int(self.x):x},0x{int(self.y):x})" + +# The secp256k1 generator point +G = GE.lift_x(0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798) + + +class FastGEMul: + """Table for fast multiplication with a constant group element. + + Speed up scalar multiplication with a fixed point P by using a precomputed lookup table with + its powers of 2: + + table = [P, 2*P, 4*P, (2^3)*P, (2^4)*P, ..., (2^255)*P] + + During multiplication, the points corresponding to each bit set in the scalar are added up, + i.e. on average ~128 point additions take place. + """ + + def __init__(self, p): + self.table = [p] # table[i] = (2^i) * p + for _ in range(255): + p = p + p + self.table.append(p) + + def mul(self, a): + result = GE() + a = a % GE.ORDER + for bit in range(a.bit_length()): + if a & (1 << bit): + result += self.table[bit] + return result + +# Precomputed table with multiples of G for fast multiplication +FAST_G = FastGEMul(G) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index d4dc90a517..73e7516ea7 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -92,7 +92,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): This class also contains various public and private helper methods.""" - def __init__(self): + def __init__(self) -> None: """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method""" self.chain: str = 'regtest' self.setup_clean_chain: bool = False @@ -103,7 +103,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.supports_cli = True self.bind_to_localhost_only = True self.parse_args() - self.disable_syscall_sandbox = self.options.nosandbox or self.options.valgrind self.default_wallet_name = "default_wallet" if self.options.descriptors else "" self.wallet_data_filename = "wallet.dat" # Optional list of wallet names that can be set in set_test_params to @@ -160,8 +159,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") - parser.add_argument("--nosandbox", dest="nosandbox", default=False, action="store_true", - help="Don't use the syscall sandbox") parser.add_argument("--noshutdown", dest="noshutdown", default=False, action="store_true", help="Don't stop bitcoinds after the test execution") parser.add_argument("--cachedir", dest="cachedir", default=os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../../cache"), @@ -188,7 +185,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): parser.add_argument("--perf", dest="perf", default=False, action="store_true", help="profile running nodes with perf for the duration of the test") parser.add_argument("--valgrind", dest="valgrind", default=False, action="store_true", - help="run nodes under the valgrind memory error detector: expect at least a ~10x slowdown. valgrind 3.14 or later required. Forces --nosandbox.") + help="run nodes under the valgrind memory error detector: expect at least a ~10x slowdown. valgrind 3.14 or later required.") parser.add_argument("--randomseed", type=int, help="set a random seed for deterministically reproducing a previous test run") parser.add_argument("--timeout-factor", dest="timeout_factor", type=float, help="adjust test timeouts by a factor. Setting it to 0 disables all timeouts") @@ -497,11 +494,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): extra_args = [[]] * num_nodes if versions is None: versions = [None] * num_nodes - if self.is_syscall_sandbox_compiled() and not self.disable_syscall_sandbox: - for i in range(len(extra_args)): - # The -sandbox argument is not present in the v22.0 release. - if versions[i] is None or versions[i] >= 229900: - extra_args[i] = extra_args[i] + ["-sandbox=log-and-abort"] if binary is None: binary = [get_bin_from_version(v, 'bitcoind', self.options.bitcoind) for v in versions] if binary_cli is None: @@ -987,7 +979,3 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def is_bdb_compiled(self): """Checks whether the wallet module was compiled with BDB support.""" return self.config["components"].getboolean("USE_BDB") - - def is_syscall_sandbox_compiled(self): - """Checks whether the syscall sandbox was compiled.""" - return self.config["components"].getboolean("ENABLE_SYSCALL_SANDBOX") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 56abe5f26a..5111d88e15 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -22,7 +22,10 @@ import shlex import sys from pathlib import Path -from .authproxy import JSONRPCException +from .authproxy import ( + JSONRPCException, + serialization_fallback, +) from .descriptors import descsum_create from .p2p import P2P_SUBVERSION from .util import ( @@ -35,7 +38,6 @@ from .util import ( rpc_url, wait_until_helper, p2p_port, - EncodeDecimal, ) BITCOIND_PROC_WAIT_TIMEOUT = 60 @@ -190,7 +192,7 @@ class TestNode(): assert self.rpc_connected and self.rpc is not None, self._node_msg("Error: no RPC connection") return getattr(RPCOverloadWrapper(self.rpc, descriptors=self.descriptors), name) - def start(self, extra_args=None, *, cwd=None, stdout=None, stderr=None, **kwargs): + def start(self, extra_args=None, *, cwd=None, stdout=None, stderr=None, env=None, **kwargs): """Start the node.""" if extra_args is None: extra_args = self.extra_args @@ -213,6 +215,8 @@ class TestNode(): # add environment variable LIBC_FATAL_STDERR_=1 so that libc errors are written to stderr and not the terminal subp_env = dict(os.environ, LIBC_FATAL_STDERR_="1") + if env is not None: + subp_env.update(env) self.process = subprocess.Popen(self.args + extra_args, env=subp_env, stdout=stdout, stderr=stderr, cwd=cwd, **kwargs) @@ -363,7 +367,7 @@ class TestNode(): if wait_until_stopped: self.wait_until_stopped() - def is_node_stopped(self): + def is_node_stopped(self, expected_ret_code=0): """Checks whether the node has stopped. Returns True if the node has stopped. False otherwise. @@ -375,8 +379,8 @@ class TestNode(): return False # process has stopped. Assert that it didn't return an error code. - assert return_code == 0, self._node_msg( - "Node returned non-zero exit code (%d) when stopping" % return_code) + assert return_code == expected_ret_code, self._node_msg( + f"Node returned unexpected exit code ({return_code}) vs ({expected_ret_code}) when stopping") self.running = False self.process = None self.rpc_connected = False @@ -384,8 +388,9 @@ class TestNode(): self.log.debug("Node stopped") return True - def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT): - wait_until_helper(self.is_node_stopped, timeout=timeout, timeout_factor=self.timeout_factor) + def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT, expect_error=False): + expected_ret_code = 1 if expect_error else 0 # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS + wait_until_helper(lambda: self.is_node_stopped(expected_ret_code=expected_ret_code), timeout=timeout, timeout_factor=self.timeout_factor) def replace_in_config(self, replacements): """ @@ -403,13 +408,21 @@ class TestNode(): conf.write(conf_data) @property + def datadir_path(self) -> Path: + return Path(self.datadir) + + @property def chain_path(self) -> Path: - return Path(self.datadir) / self.chain + return self.datadir_path / self.chain @property def debug_log_path(self) -> Path: return self.chain_path / 'debug.log' + @property + def wallets_path(self) -> Path: + return self.chain_path / "wallets" + def debug_log_bytes(self) -> int: with open(self.debug_log_path, encoding='utf-8') as dl: dl.seek(0, 2) @@ -701,7 +714,7 @@ def arg_to_cli(arg): elif arg is None: return 'null' elif isinstance(arg, dict) or isinstance(arg, list): - return json.dumps(arg, default=EncodeDecimal) + return json.dumps(arg, default=serialization_fallback) else: return str(arg) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 2c227922c5..9143397042 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -12,13 +12,15 @@ import inspect import json import logging import os +import pathlib import random import re +import sys import time from . import coverage from .authproxy import AuthServiceProxy, JSONRPCException -from typing import Callable, Optional +from typing import Callable, Optional, Tuple logger = logging.getLogger("TestFramework.utils") @@ -209,12 +211,6 @@ def check_json_precision(): raise RuntimeError("JSON encode/decode loses precision") -def EncodeDecimal(o): - if isinstance(o, Decimal): - return str(o) - raise TypeError(repr(o) + " is not JSON serializable") - - def count_bytes(hex_string): return len(bytearray.fromhex(hex_string)) @@ -313,7 +309,7 @@ class PortSeed: n = None -def get_rpc_proxy(url: str, node_number: int, *, timeout: int=None, coveragedir: str=None) -> coverage.AuthServiceProxyWrapper: +def get_rpc_proxy(url: str, node_number: int, *, timeout: Optional[int]=None, coveragedir: Optional[str]=None) -> coverage.AuthServiceProxyWrapper: """ Args: url: URL of the RPC server to call @@ -419,6 +415,22 @@ def get_datadir_path(dirname, n): return os.path.join(dirname, "node" + str(n)) +def get_temp_default_datadir(temp_dir: pathlib.Path) -> Tuple[dict, pathlib.Path]: + """Return os-specific environment variables that can be set to make the + GetDefaultDataDir() function return a datadir path under the provided + temp_dir, as well as the complete path it would return.""" + if sys.platform == "win32": + env = dict(APPDATA=str(temp_dir)) + datadir = temp_dir / "Bitcoin" + else: + env = dict(HOME=str(temp_dir)) + if sys.platform == "darwin": + datadir = temp_dir / "Library/Application Support/Bitcoin" + else: + datadir = temp_dir / ".bitcoin" + return env, datadir + + def append_config(datadir, options): with open(os.path.join(datadir, "bitcoin.conf"), 'a', encoding='utf8') as f: for option in options: diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 1d546e12bd..271095ea21 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -20,6 +20,7 @@ from test_framework.address import ( key_to_p2wpkh, output_key_to_p2tr, ) +from test_framework.blocktools import COINBASE_MATURITY from test_framework.descriptors import descsum_create from test_framework.key import ( ECKey, @@ -53,7 +54,7 @@ from test_framework.util import ( assert_equal, assert_greater_than_or_equal, ) -from test_framework.blocktools import COINBASE_MATURITY +from test_framework.wallet_util import generate_keypair DEFAULT_FEE = Decimal("0.0001") @@ -395,9 +396,7 @@ def getnewdestination(address_type='bech32m'): 'legacy', 'p2sh-segwit', 'bech32' and 'bech32m'. Can be used when a random destination is needed, but no compiled wallet is available (e.g. as replacement to the getnewaddress/getaddressinfo RPCs).""" - key = ECKey() - key.generate() - pubkey = key.get_pubkey().get_bytes() + key, pubkey = generate_keypair() if address_type == 'legacy': scriptpubkey = key_to_p2pkh_script(pubkey) address = key_to_p2pkh(pubkey) diff --git a/test/functional/test_framework/wallet_util.py b/test/functional/test_framework/wallet_util.py index 410d85cd8c..319f120297 100755 --- a/test/functional/test_framework/wallet_util.py +++ b/test/functional/test_framework/wallet_util.py @@ -63,12 +63,9 @@ def get_generate_key(): """Generate a fresh key Returns a named tuple of privkey, pubkey and all address and scripts.""" - eckey = ECKey() - eckey.generate() - privkey = bytes_to_wif(eckey.get_bytes()) - pubkey = eckey.get_pubkey().get_bytes().hex() + privkey, pubkey = generate_keypair(wif=True) return Key(privkey=privkey, - pubkey=pubkey, + pubkey=pubkey.hex(), p2pkh_script=key_to_p2pkh_script(pubkey).hex(), p2pkh_addr=key_to_p2pkh(pubkey), p2wpkh_script=key_to_p2wpkh_script(pubkey).hex(), @@ -114,8 +111,14 @@ def bytes_to_wif(b, compressed=True): b += b'\x01' return byte_to_base58(b, 239) -def generate_wif_key(): - # Makes a WIF privkey for imports - k = ECKey() - k.generate() - return bytes_to_wif(k.get_bytes(), k.is_compressed) +def generate_keypair(compressed=True, wif=False): + """Generate a new random keypair and return the corresponding ECKey / + bytes objects. The private key can also be provided as WIF (wallet + import format) string instead, which is often useful for wallet RPC + interaction.""" + privkey = ECKey() + privkey.generate(compressed) + pubkey = privkey.get_pubkey().get_bytes() + if wif: + privkey = bytes_to_wif(privkey.get_bytes(), compressed) + return privkey, pubkey diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index a6f1bbae7d..e5df29b135 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -166,6 +166,8 @@ BASE_SCRIPTS = [ 'p2p_compactblocks_blocksonly.py', 'wallet_hd.py --legacy-wallet', 'wallet_hd.py --descriptors', + 'wallet_blank.py --legacy-wallet', + 'wallet_blank.py --descriptors', 'wallet_keypool_topup.py --legacy-wallet', 'wallet_keypool_topup.py --descriptors', 'wallet_fast_rescan.py --descriptors', @@ -196,6 +198,8 @@ BASE_SCRIPTS = [ 'wallet_watchonly.py --legacy-wallet', 'wallet_watchonly.py --usecli --legacy-wallet', 'wallet_reorgsrestore.py', + 'wallet_conflicts.py --legacy-wallet', + 'wallet_conflicts.py --descriptors', 'interface_http.py', 'interface_rpc.py', 'interface_usdt_coinselection.py', @@ -206,7 +210,6 @@ BASE_SCRIPTS = [ 'rpc_users.py', 'rpc_whitelist.py', 'feature_proxy.py', - 'feature_syscall_sandbox.py', 'wallet_signrawtransactionwithwallet.py --legacy-wallet', 'wallet_signrawtransactionwithwallet.py --descriptors', 'rpc_signrawtransactionwithkey.py', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 95999649b4..327dd43e5a 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -173,12 +173,12 @@ class ToolWalletTest(BitcoinTestFramework): if file_format is not None and file_format != dump_data["format"]: load_output += "Warning: Dumpfile wallet format \"{}\" does not match command line specified format \"{}\".\n".format(dump_data["format"], file_format) self.assert_tool_output(load_output, *args) - assert os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", wallet_name)) + assert (self.nodes[0].wallets_path / wallet_name).is_dir() self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", '-wallet={}'.format(wallet_name), '-dumpfile={}'.format(rt_dumppath), 'dump') rt_dump_data = self.read_dump(rt_dumppath) - wallet_dat = os.path.join(self.nodes[0].datadir, "regtest/wallets/", wallet_name, "wallet.dat") + wallet_dat = self.nodes[0].wallets_path / wallet_name / "wallet.dat" if rt_dump_data["format"] == "bdb": self.assert_is_bdb(wallet_dat) else: @@ -193,7 +193,7 @@ class ToolWalletTest(BitcoinTestFramework): self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') self.assert_raises_tool_error('No method provided. Run `bitcoin-wallet -help` for valid methods.') self.assert_raises_tool_error('Wallet name must be provided when creating a new wallet.', 'create') - locked_dir = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets") + locked_dir = self.nodes[0].wallets_path error = 'Error initializing wallet database environment "{}"!'.format(locked_dir) if self.options.descriptors: error = f"SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['PACKAGE_NAME']}?" @@ -202,7 +202,7 @@ class ToolWalletTest(BitcoinTestFramework): '-wallet=' + self.default_wallet_name, 'info', ) - path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "nonexistent.dat") + path = self.nodes[0].wallets_path / "nonexistent.dat" self.assert_raises_tool_error("Failed to load database path '{}'. Path does not exist.".format(path), '-wallet=nonexistent.dat', 'info') def test_tool_wallet_info(self): @@ -347,7 +347,7 @@ class ToolWalletTest(BitcoinTestFramework): non_exist_dump = os.path.join(self.nodes[0].datadir, "wallet.nodump") self.assert_raises_tool_error('Unknown wallet file format "notaformat" provided. Please provide one of "bdb" or "sqlite".', '-wallet=todump', '-format=notaformat', '-dumpfile={}'.format(wallet_dump), 'createfromdump') self.assert_raises_tool_error('Dump file {} does not exist.'.format(non_exist_dump), '-wallet=todump', '-dumpfile={}'.format(non_exist_dump), 'createfromdump') - wallet_path = os.path.join(self.nodes[0].datadir, 'regtest', 'wallets', 'todump2') + wallet_path = self.nodes[0].wallets_path / "todump2" self.assert_raises_tool_error('Failed to create database path \'{}\'. Database already exists.'.format(wallet_path), '-wallet=todump2', '-dumpfile={}'.format(wallet_dump), 'createfromdump') self.assert_raises_tool_error("The -descriptors option can only be used with the 'create' command.", '-descriptors', '-wallet=todump2', '-dumpfile={}'.format(wallet_dump), 'createfromdump') @@ -363,18 +363,18 @@ class ToolWalletTest(BitcoinTestFramework): dump_data["BITCOIN_CORE_WALLET_DUMP"] = "0" self.write_dump(dump_data, bad_ver_wallet_dump) self.assert_raises_tool_error('Error: Dumpfile version is not supported. This version of bitcoin-wallet only supports version 1 dumpfiles. Got dumpfile with version 0', '-wallet=badload', '-dumpfile={}'.format(bad_ver_wallet_dump), 'createfromdump') - assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + assert not (self.nodes[0].wallets_path / "badload").is_dir() bad_ver_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_ver2.dump") dump_data["BITCOIN_CORE_WALLET_DUMP"] = "2" self.write_dump(dump_data, bad_ver_wallet_dump) self.assert_raises_tool_error('Error: Dumpfile version is not supported. This version of bitcoin-wallet only supports version 1 dumpfiles. Got dumpfile with version 2', '-wallet=badload', '-dumpfile={}'.format(bad_ver_wallet_dump), 'createfromdump') - assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + assert not (self.nodes[0].wallets_path / "badload").is_dir() bad_magic_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_magic.dump") del dump_data["BITCOIN_CORE_WALLET_DUMP"] dump_data["not_the_right_magic"] = "1" self.write_dump(dump_data, bad_magic_wallet_dump, "not_the_right_magic") self.assert_raises_tool_error('Error: Dumpfile identifier record is incorrect. Got "not_the_right_magic", expected "BITCOIN_CORE_WALLET_DUMP".', '-wallet=badload', '-dumpfile={}'.format(bad_magic_wallet_dump), 'createfromdump') - assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + assert not (self.nodes[0].wallets_path / "badload").is_dir() self.log.info('Checking createfromdump handling of checksums') bad_sum_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_sum1.dump") @@ -383,25 +383,25 @@ class ToolWalletTest(BitcoinTestFramework): dump_data["checksum"] = "1" * 64 self.write_dump(dump_data, bad_sum_wallet_dump) self.assert_raises_tool_error('Error: Dumpfile checksum does not match. Computed {}, expected {}'.format(checksum, "1" * 64), '-wallet=bad', '-dumpfile={}'.format(bad_sum_wallet_dump), 'createfromdump') - assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + assert not (self.nodes[0].wallets_path / "badload").is_dir() bad_sum_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_sum2.dump") del dump_data["checksum"] self.write_dump(dump_data, bad_sum_wallet_dump, skip_checksum=True) self.assert_raises_tool_error('Error: Missing checksum', '-wallet=badload', '-dumpfile={}'.format(bad_sum_wallet_dump), 'createfromdump') - assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + assert not (self.nodes[0].wallets_path / "badload").is_dir() bad_sum_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_sum3.dump") dump_data["checksum"] = "2" * 10 self.write_dump(dump_data, bad_sum_wallet_dump) self.assert_raises_tool_error('Error: Checksum is not the correct size', '-wallet=badload', '-dumpfile={}'.format(bad_sum_wallet_dump), 'createfromdump') - assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + assert not (self.nodes[0].wallets_path / "badload").is_dir() dump_data["checksum"] = "3" * 66 self.write_dump(dump_data, bad_sum_wallet_dump) self.assert_raises_tool_error('Error: Checksum is not the correct size', '-wallet=badload', '-dumpfile={}'.format(bad_sum_wallet_dump), 'createfromdump') - assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + assert not (self.nodes[0].wallets_path / "badload").is_dir() def run_test(self): - self.wallet_path = os.path.join(self.nodes[0].datadir, self.chain, 'wallets', self.default_wallet_name, self.wallet_data_filename) + self.wallet_path = os.path.join(self.nodes[0].wallets_path, self.default_wallet_name, self.wallet_data_filename) self.test_invalid_tool_commands_and_args() # Warning: The following tests are order-dependent. self.test_tool_wallet_info() diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index 934f44588d..2691507773 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -226,20 +226,16 @@ class AbandonConflictTest(BitcoinTestFramework): assert_equal(double_spend["walletconflicts"], [txAB1]) # Verify that B and C's 10 BTC outputs are available for spending again because AB1 is now conflicted + assert_equal(alice.gettransaction(txAB1)["confirmations"], -1) newbalance = alice.getbalance() assert_equal(newbalance, balance + Decimal("20")) balance = newbalance - # There is currently a minor bug around this and so this test doesn't work. See Issue #7315 - # Invalidate the block with the double spend and B's 10 BTC output should no longer be available - # Don't think C's should either + # Invalidate the block with the double spend. B & C's 10 BTC outputs should no longer be available self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) + assert_equal(alice.gettransaction(txAB1)["confirmations"], 0) newbalance = alice.getbalance() - #assert_equal(newbalance, balance - Decimal("10")) - self.log.info("If balance has not declined after invalidateblock then out of mempool wallet tx which is no longer") - self.log.info("conflicted has not resumed causing its inputs to be seen as spent. See Issue #7315") - assert_equal(balance, newbalance) - + assert_equal(newbalance, balance - Decimal("20")) if __name__ == '__main__': AbandonConflictTest().main() diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index 4ad25d964e..fa92ebd436 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -109,16 +109,16 @@ class WalletBackupTest(BitcoinTestFramework): self.stop_node(2) def erase_three(self): - os.remove(os.path.join(self.nodes[0].datadir, self.chain, 'wallets', self.default_wallet_name, self.wallet_data_filename)) - os.remove(os.path.join(self.nodes[1].datadir, self.chain, 'wallets', self.default_wallet_name, self.wallet_data_filename)) - os.remove(os.path.join(self.nodes[2].datadir, self.chain, 'wallets', self.default_wallet_name, self.wallet_data_filename)) + os.remove(os.path.join(self.nodes[0].wallets_path, self.default_wallet_name, self.wallet_data_filename)) + os.remove(os.path.join(self.nodes[1].wallets_path, self.default_wallet_name, self.wallet_data_filename)) + os.remove(os.path.join(self.nodes[2].wallets_path, self.default_wallet_name, self.wallet_data_filename)) def restore_invalid_wallet(self): node = self.nodes[3] invalid_wallet_file = os.path.join(self.nodes[0].datadir, 'invalid_wallet_file.bak') open(invalid_wallet_file, 'a', encoding="utf8").write('invald wallet') wallet_name = "res0" - not_created_wallet_file = os.path.join(node.datadir, self.chain, 'wallets', wallet_name) + not_created_wallet_file = os.path.join(node.wallets_path, wallet_name) error_message = "Wallet file verification failed. Failed to load database path '{}'. Data is not in recognized format.".format(not_created_wallet_file) assert_raises_rpc_error(-18, error_message, node.restorewallet, wallet_name, invalid_wallet_file) assert not os.path.exists(not_created_wallet_file) @@ -128,14 +128,14 @@ class WalletBackupTest(BitcoinTestFramework): nonexistent_wallet_file = os.path.join(self.nodes[0].datadir, 'nonexistent_wallet.bak') wallet_name = "res0" assert_raises_rpc_error(-8, "Backup file does not exist", node.restorewallet, wallet_name, nonexistent_wallet_file) - not_created_wallet_file = os.path.join(node.datadir, self.chain, 'wallets', wallet_name) + not_created_wallet_file = os.path.join(node.wallets_path, wallet_name) assert not os.path.exists(not_created_wallet_file) def restore_wallet_existent_name(self): node = self.nodes[3] backup_file = os.path.join(self.nodes[0].datadir, 'wallet.bak') wallet_name = "res0" - wallet_file = os.path.join(node.datadir, self.chain, 'wallets', wallet_name) + wallet_file = os.path.join(node.wallets_path, wallet_name) error_message = "Failed to create database path '{}'. Database already exists.".format(wallet_file) assert_raises_rpc_error(-36, error_message, node.restorewallet, wallet_name, backup_file) assert os.path.exists(wallet_file) @@ -206,9 +206,9 @@ class WalletBackupTest(BitcoinTestFramework): self.nodes[3].restorewallet("res1", backup_file_1) self.nodes[3].restorewallet("res2", backup_file_2) - assert os.path.exists(os.path.join(self.nodes[3].datadir, self.chain, 'wallets', "res0")) - assert os.path.exists(os.path.join(self.nodes[3].datadir, self.chain, 'wallets', "res1")) - assert os.path.exists(os.path.join(self.nodes[3].datadir, self.chain, 'wallets', "res2")) + assert os.path.exists(os.path.join(self.nodes[3].wallets_path, "res0")) + assert os.path.exists(os.path.join(self.nodes[3].wallets_path, "res1")) + assert os.path.exists(os.path.join(self.nodes[3].wallets_path, "res2")) res0_rpc = self.nodes[3].get_wallet_rpc("res0") res1_rpc = self.nodes[3].get_wallet_rpc("res1") @@ -226,8 +226,8 @@ class WalletBackupTest(BitcoinTestFramework): self.erase_three() #start node2 with no chain - shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'blocks')) - shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'chainstate')) + shutil.rmtree(os.path.join(self.nodes[2].chain_path, 'blocks')) + shutil.rmtree(os.path.join(self.nodes[2].chain_path, 'chainstate')) self.start_three(["-nowallet"]) self.init_three() @@ -248,10 +248,10 @@ class WalletBackupTest(BitcoinTestFramework): # Backup to source wallet file must fail sourcePaths = [ - os.path.join(self.nodes[0].datadir, self.chain, 'wallets', self.default_wallet_name, self.wallet_data_filename), - os.path.join(self.nodes[0].datadir, self.chain, '.', 'wallets', self.default_wallet_name, self.wallet_data_filename), - os.path.join(self.nodes[0].datadir, self.chain, 'wallets', self.default_wallet_name), - os.path.join(self.nodes[0].datadir, self.chain, 'wallets')] + os.path.join(self.nodes[0].wallets_path, self.default_wallet_name, self.wallet_data_filename), + os.path.join(self.nodes[0].wallets_path, '.', self.default_wallet_name, self.wallet_data_filename), + os.path.join(self.nodes[0].wallets_path, self.default_wallet_name), + os.path.join(self.nodes[0].wallets_path)] for sourcePath in sourcePaths: assert_raises_rpc_error(-4, "backup failed", self.nodes[0].backupwallet, sourcePath) diff --git a/test/functional/wallet_backwards_compatibility.py b/test/functional/wallet_backwards_compatibility.py index 5088e11eda..7d88e009c7 100755 --- a/test/functional/wallet_backwards_compatibility.py +++ b/test/functional/wallet_backwards_compatibility.py @@ -74,8 +74,8 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): def nodes_wallet_dir(self, node): if node.version < 170000: - return os.path.join(node.datadir, "regtest") - return os.path.join(node.datadir, "regtest/wallets") + return node.chain_path + return node.wallets_path def run_test(self): node_miner = self.nodes[0] @@ -157,10 +157,10 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): assert info['keypoolsize'] == 0 # Unload wallets and copy to older nodes: - node_master_wallets_dir = os.path.join(node_master.datadir, "regtest/wallets") - node_v19_wallets_dir = os.path.join(node_v19.datadir, "regtest/wallets") - node_v17_wallets_dir = os.path.join(node_v17.datadir, "regtest/wallets") - node_v16_wallets_dir = os.path.join(node_v16.datadir, "regtest") + node_master_wallets_dir = node_master.wallets_path + node_v19_wallets_dir = node_v19.wallets_path + node_v17_wallets_dir = node_v17.wallets_path + node_v16_wallets_dir = node_v16.chain_path node_master.unloadwallet("w1") node_master.unloadwallet("w2") node_master.unloadwallet("w3") diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index e23ec7bc01..a1b805c09e 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -8,6 +8,10 @@ from itertools import product from test_framework.blocktools import COINBASE_MATURITY from test_framework.descriptors import descsum_create +from test_framework.messages import ( + COIN, + DEFAULT_ANCESTOR_LIMIT, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_array_result, @@ -17,6 +21,7 @@ from test_framework.util import ( find_vout_for_address, ) from test_framework.wallet_util import test_address +from test_framework.wallet import MiniWallet NOT_A_NUMBER_OR_STRING = "Amount is not a number or string" OUT_OF_RANGE = "Amount out of range" @@ -784,6 +789,34 @@ class WalletTest(BitcoinTestFramework): zeroconf_wallet.sendtoaddress(zeroconf_wallet.getnewaddress(), Decimal('0.5')) + self.test_chain_listunspent() + + def test_chain_listunspent(self): + if not self.options.descriptors: + return + self.wallet = MiniWallet(self.nodes[0]) + self.nodes[0].get_wallet_rpc(self.default_wallet_name).sendtoaddress(self.wallet.get_address(), "5") + self.generate(self.wallet, 1, sync_fun=self.no_op) + self.nodes[0].createwallet("watch_wallet", disable_private_keys=True) + watch_wallet = self.nodes[0].get_wallet_rpc("watch_wallet") + watch_wallet.importaddress(self.wallet.get_address()) + + # DEFAULT_ANCESTOR_LIMIT transactions off a confirmed tx should be fine + chain = self.wallet.create_self_transfer_chain(chain_length=DEFAULT_ANCESTOR_LIMIT) + ancestor_vsize = 0 + ancestor_fees = Decimal(0) + + for i, t in enumerate(chain): + ancestor_vsize += t["tx"].get_vsize() + ancestor_fees += t["fee"] + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=t["hex"]) + # Check that listunspent ancestor{count, size, fees} yield the correct results + wallet_unspent = watch_wallet.listunspent(minconf=0) + this_unspent = next(utxo_info for utxo_info in wallet_unspent if utxo_info["txid"] == t["txid"]) + assert_equal(this_unspent['ancestorcount'], i + 1) + assert_equal(this_unspent['ancestorsize'], ancestor_vsize) + assert_equal(this_unspent['ancestorfees'], ancestor_fees * COIN) + if __name__ == '__main__': WalletTest().main() diff --git a/test/functional/wallet_blank.py b/test/functional/wallet_blank.py new file mode 100755 index 0000000000..4836eba3b2 --- /dev/null +++ b/test/functional/wallet_blank.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://www.opensource.org/licenses/mit-license.php. + +import os + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.address import ( + ADDRESS_BCRT1_UNSPENDABLE, + ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR, +) +from test_framework.util import ( + assert_equal, +) +from test_framework.wallet_util import generate_keypair + + +class WalletBlankTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def add_options(self, options): + self.add_wallet_options(options) + + def test_importaddress(self): + if self.options.descriptors: + return + self.log.info("Test that importaddress unsets the blank flag") + self.nodes[0].createwallet(wallet_name="iaddr", disable_private_keys=True, blank=True) + wallet = self.nodes[0].get_wallet_rpc("iaddr") + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], False) + assert_equal(info["blank"], True) + wallet.importaddress(ADDRESS_BCRT1_UNSPENDABLE) + assert_equal(wallet.getwalletinfo()["blank"], False) + + def test_importpubkey(self): + if self.options.descriptors: + return + self.log.info("Test that importpubkey unsets the blank flag") + for i, comp in enumerate([True, False]): + self.nodes[0].createwallet(wallet_name=f"ipub{i}", disable_private_keys=True, blank=True) + wallet = self.nodes[0].get_wallet_rpc(f"ipub{i}") + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], False) + assert_equal(info["blank"], True) + + _, pubkey = generate_keypair(compressed=comp) + wallet.importpubkey(pubkey.hex()) + assert_equal(wallet.getwalletinfo()["blank"], False) + + def test_importprivkey(self): + if self.options.descriptors: + return + self.log.info("Test that importprivkey unsets the blank flag") + for i, comp in enumerate([True, False]): + self.nodes[0].createwallet(wallet_name=f"ipriv{i}", blank=True) + wallet = self.nodes[0].get_wallet_rpc(f"ipriv{i}") + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], False) + assert_equal(info["blank"], True) + + wif, _ = generate_keypair(compressed=comp, wif=True) + wallet.importprivkey(wif) + assert_equal(wallet.getwalletinfo()["blank"], False) + + def test_importmulti(self): + if self.options.descriptors: + return + self.log.info("Test that importmulti unsets the blank flag") + self.nodes[0].createwallet(wallet_name="imulti", disable_private_keys=True, blank=True) + wallet = self.nodes[0].get_wallet_rpc("imulti") + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], False) + assert_equal(info["blank"], True) + wallet.importmulti([{ + "desc": ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR, + "timestamp": "now", + }]) + assert_equal(wallet.getwalletinfo()["blank"], False) + + def test_importdescriptors(self): + if not self.options.descriptors: + return + self.log.info("Test that importdescriptors preserves the blank flag") + self.nodes[0].createwallet(wallet_name="idesc", disable_private_keys=True, blank=True) + wallet = self.nodes[0].get_wallet_rpc("idesc") + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], True) + assert_equal(info["blank"], True) + wallet.importdescriptors([{ + "desc": ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR, + "timestamp": "now", + }]) + assert_equal(wallet.getwalletinfo()["blank"], True) + + def test_importwallet(self): + if self.options.descriptors: + return + self.log.info("Test that importwallet unsets the blank flag") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + self.nodes[0].createwallet(wallet_name="iwallet", blank=True) + wallet = self.nodes[0].get_wallet_rpc("iwallet") + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], False) + assert_equal(info["blank"], True) + + wallet_dump_path = os.path.join(self.nodes[0].datadir, "wallet.dump") + def_wallet.dumpwallet(wallet_dump_path) + + wallet.importwallet(wallet_dump_path) + assert_equal(wallet.getwalletinfo()["blank"], False) + + def test_encrypt_legacy(self): + if self.options.descriptors: + return + self.log.info("Test that encrypting a blank legacy wallet preserves the blank flag and does not generate a seed") + self.nodes[0].createwallet(wallet_name="encblanklegacy", blank=True) + wallet = self.nodes[0].get_wallet_rpc("encblanklegacy") + + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], False) + assert_equal(info["blank"], True) + assert "hdseedid" not in info + + wallet.encryptwallet("pass") + info = wallet.getwalletinfo() + assert_equal(info["blank"], True) + assert "hdseedid" not in info + + def test_encrypt_descriptors(self): + if not self.options.descriptors: + return + self.log.info("Test that encrypting a blank descriptor wallet preserves the blank flag and descriptors remain the same") + self.nodes[0].createwallet(wallet_name="encblankdesc", blank=True) + wallet = self.nodes[0].get_wallet_rpc("encblankdesc") + + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], True) + assert_equal(info["blank"], True) + descs = wallet.listdescriptors() + + wallet.encryptwallet("pass") + assert_equal(wallet.getwalletinfo()["blank"], True) + assert_equal(descs, wallet.listdescriptors()) + + def run_test(self): + self.test_importaddress() + self.test_importpubkey() + self.test_importprivkey() + self.test_importmulti() + self.test_importdescriptors() + self.test_importwallet() + self.test_encrypt_legacy() + self.test_encrypt_descriptors() + + +if __name__ == '__main__': + WalletBlankTest().main() diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 19c8022600..b9ebf64c22 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -123,36 +123,36 @@ class BumpFeeTest(BitcoinTestFramework): assert_raises_rpc_error(-3, "Unexpected key {}".format(key), rbf_node.bumpfee, rbfid, {key: NORMAL}) # Bumping to just above minrelay should fail to increase the total fee enough. - assert_raises_rpc_error(-8, "Insufficient total fee 0.00000141", rbf_node.bumpfee, rbfid, {"fee_rate": INSUFFICIENT}) + assert_raises_rpc_error(-8, "Insufficient total fee 0.00000141", rbf_node.bumpfee, rbfid, fee_rate=INSUFFICIENT) self.log.info("Test invalid fee rate settings") assert_raises_rpc_error(-4, "Specified or calculated fee 0.141 is too high (cannot be higher than -maxtxfee 0.10", - rbf_node.bumpfee, rbfid, {"fee_rate": TOO_HIGH}) + rbf_node.bumpfee, rbfid, fee_rate=TOO_HIGH) # Test fee_rate with zero values. msg = "Insufficient total fee 0.00" for zero_value in [0, 0.000, 0.00000000, "0", "0.000", "0.00000000"]: - assert_raises_rpc_error(-8, msg, rbf_node.bumpfee, rbfid, {"fee_rate": zero_value}) + assert_raises_rpc_error(-8, msg, rbf_node.bumpfee, rbfid, fee_rate=zero_value) msg = "Invalid amount" # Test fee_rate values that don't pass fixed-point parsing checks. for invalid_value in ["", 0.000000001, 1e-09, 1.111111111, 1111111111111111, "31.999999999999999999999"]: - assert_raises_rpc_error(-3, msg, rbf_node.bumpfee, rbfid, {"fee_rate": invalid_value}) + assert_raises_rpc_error(-3, msg, rbf_node.bumpfee, rbfid, fee_rate=invalid_value) # Test fee_rate values that cannot be represented in sat/vB. for invalid_value in [0.0001, 0.00000001, 0.00099999, 31.99999999, "0.0001", "0.00000001", "0.00099999", "31.99999999"]: - assert_raises_rpc_error(-3, msg, rbf_node.bumpfee, rbfid, {"fee_rate": invalid_value}) + assert_raises_rpc_error(-3, msg, rbf_node.bumpfee, rbfid, fee_rate=invalid_value) # Test fee_rate out of range (negative number). - assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, {"fee_rate": -1}) + assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, fee_rate=-1) # Test type error. for value in [{"foo": "bar"}, True]: - assert_raises_rpc_error(-3, "Amount is not a number or string", rbf_node.bumpfee, rbfid, {"fee_rate": value}) + assert_raises_rpc_error(-3, "Amount is not a number or string", rbf_node.bumpfee, rbfid, fee_rate=value) self.log.info("Test explicit fee rate raises RPC error if both fee_rate and conf_target are passed") assert_raises_rpc_error(-8, "Cannot specify both conf_target and fee_rate. Please provide either a confirmation " "target in blocks for automatic fee estimation, or an explicit fee rate.", - rbf_node.bumpfee, rbfid, {"conf_target": NORMAL, "fee_rate": NORMAL}) + rbf_node.bumpfee, rbfid, conf_target=NORMAL, fee_rate=NORMAL) self.log.info("Test explicit fee rate raises RPC error if both fee_rate and estimate_mode are passed") assert_raises_rpc_error(-8, "Cannot specify both estimate_mode and fee_rate", - rbf_node.bumpfee, rbfid, {"estimate_mode": "economical", "fee_rate": NORMAL}) + rbf_node.bumpfee, rbfid, estimate_mode="economical", fee_rate=NORMAL) self.log.info("Test invalid conf_target settings") assert_raises_rpc_error(-8, "confTarget and conf_target options should not both be set", @@ -161,10 +161,10 @@ class BumpFeeTest(BitcoinTestFramework): self.log.info("Test invalid estimate_mode settings") for k, v in {"number": 42, "object": {"foo": "bar"}}.items(): assert_raises_rpc_error(-3, f"JSON value of type {k} for field estimate_mode is not of expected type string", - rbf_node.bumpfee, rbfid, {"estimate_mode": v}) + rbf_node.bumpfee, rbfid, estimate_mode=v) for mode in ["foo", Decimal("3.1415"), "sat/B", "BTC/kB"]: assert_raises_rpc_error(-8, 'Invalid estimate_mode parameter, must be one of: "unset", "economical", "conservative"', - rbf_node.bumpfee, rbfid, {"estimate_mode": mode}) + rbf_node.bumpfee, rbfid, estimate_mode=mode) self.log.info("Test invalid outputs values") assert_raises_rpc_error(-8, "Invalid parameter, output argument cannot be an empty array", @@ -232,12 +232,12 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): self.sync_mempools((rbf_node, peer_node)) assert rbfid in rbf_node.getrawmempool() and rbfid in peer_node.getrawmempool() if mode == "fee_rate": - bumped_psbt = rbf_node.psbtbumpfee(rbfid, {"fee_rate": str(NORMAL)}) - bumped_tx = rbf_node.bumpfee(rbfid, {"fee_rate": NORMAL}) + bumped_psbt = rbf_node.psbtbumpfee(rbfid, fee_rate=str(NORMAL)) + bumped_tx = rbf_node.bumpfee(rbfid, fee_rate=NORMAL) elif mode == "new_outputs": new_address = peer_node.getnewaddress() - bumped_psbt = rbf_node.psbtbumpfee(rbfid, {"outputs": {new_address: 0.0003}}) - bumped_tx = rbf_node.bumpfee(rbfid, {"outputs": {new_address: 0.0003}}) + bumped_psbt = rbf_node.psbtbumpfee(rbfid, outputs={new_address: 0.0003}) + bumped_tx = rbf_node.bumpfee(rbfid, outputs={new_address: 0.0003}) else: bumped_psbt = rbf_node.psbtbumpfee(rbfid) bumped_tx = rbf_node.bumpfee(rbfid) @@ -305,7 +305,7 @@ def test_notmine_bumpfee(self, rbf_node, peer_node, dest_address): # Note that this test depends upon the RPC code checking input ownership prior to change outputs # (since it can't use fundrawtransaction, it lacks a proper change output) fee = Decimal("0.001") - utxos = [node.listunspent(query_options={'minimumAmount': fee})[-1] for node in (rbf_node, peer_node)] + utxos = [node.listunspent(minimumAmount=fee)[-1] for node in (rbf_node, peer_node)] inputs = [{ "txid": utxo["txid"], "vout": utxo["vout"], @@ -335,7 +335,7 @@ def test_notmine_bumpfee(self, rbf_node, peer_node, dest_address): psbt = rbf_node.psbtbumpfee(txid=rbfid) finish_psbtbumpfee(psbt["psbt"]) - psbt = rbf_node.psbtbumpfee(txid=rbfid, options={"fee_rate": old_feerate + 10}) + psbt = rbf_node.psbtbumpfee(txid=rbfid, fee_rate=old_feerate + 10) finish_psbtbumpfee(psbt["psbt"]) self.clear_mempool() @@ -445,7 +445,7 @@ def test_dust_to_fee(self, rbf_node, dest_address): # Expected fee is 141 vbytes * fee_rate 0.00350250 BTC / 1000 vbytes = 0.00049385 BTC. # or occasionally 140 vbytes * fee_rate 0.00350250 BTC / 1000 vbytes = 0.00049035 BTC. # Dust should be dropped to the fee, so actual bump fee is 0.00050000 BTC. - bumped_tx = rbf_node.bumpfee(rbfid, {"fee_rate": 350.25}) + bumped_tx = rbf_node.bumpfee(rbfid, fee_rate=350.25) full_bumped_tx = rbf_node.getrawtransaction(bumped_tx["txid"], 1) assert_equal(bumped_tx["fee"], Decimal("0.00050000")) assert_equal(len(fulltx["vout"]), 2) @@ -569,7 +569,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): assert_raises_rpc_error(-4, "bumpfee is not available with wallets that have private keys disabled. Use psbtbumpfee instead.", watcher.bumpfee, original_txid) # Bump fee, obnoxiously high to add additional watchonly input - bumped_psbt = watcher.psbtbumpfee(original_txid, {"fee_rate": HIGH}) + bumped_psbt = watcher.psbtbumpfee(original_txid, fee_rate=HIGH) assert_greater_than(len(watcher.decodepsbt(bumped_psbt['psbt'])["tx"]["vin"]), 1) assert "txid" not in bumped_psbt assert_equal(bumped_psbt["origfee"], -watcher.gettransaction(original_txid)["fee"]) @@ -593,17 +593,17 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): def test_rebumping(self, rbf_node, dest_address): self.log.info('Test that re-bumping the original tx fails, but bumping successor works') rbfid = spend_one_input(rbf_node, dest_address) - bumped = rbf_node.bumpfee(rbfid, {"fee_rate": ECONOMICAL}) + bumped = rbf_node.bumpfee(rbfid, fee_rate=ECONOMICAL) assert_raises_rpc_error(-4, f"Cannot bump transaction {rbfid} which was already bumped by transaction {bumped['txid']}", - rbf_node.bumpfee, rbfid, {"fee_rate": NORMAL}) - rbf_node.bumpfee(bumped["txid"], {"fee_rate": NORMAL}) + rbf_node.bumpfee, rbfid, fee_rate=NORMAL) + rbf_node.bumpfee(bumped["txid"], fee_rate=NORMAL) self.clear_mempool() def test_rebumping_not_replaceable(self, rbf_node, dest_address): self.log.info('Test that re-bumping non-replaceable fails') rbfid = spend_one_input(rbf_node, dest_address) - bumped = rbf_node.bumpfee(rbfid, {"fee_rate": ECONOMICAL, "replaceable": False}) + bumped = rbf_node.bumpfee(rbfid, fee_rate=ECONOMICAL, replaceable=False) assert_raises_rpc_error(-4, "Transaction is not BIP 125 replaceable", rbf_node.bumpfee, bumped["txid"], {"fee_rate": NORMAL}) self.clear_mempool() @@ -615,7 +615,7 @@ def test_bumpfee_already_spent(self, rbf_node, dest_address): self.generate(rbf_node, 1) # spend coin simply by mining block with tx spent_input = rbf_node.gettransaction(txid=txid, verbose=True)['decoded']['vin'][0] assert_raises_rpc_error(-1, f"{spent_input['txid']}:{spent_input['vout']} is already spent", - rbf_node.bumpfee, txid, {"fee_rate": NORMAL}) + rbf_node.bumpfee, txid, fee_rate=NORMAL) def test_unconfirmed_not_spendable(self, rbf_node, rbf_node_address): @@ -694,7 +694,7 @@ def test_change_script_match(self, rbf_node, dest_address): assert_equal(len(change_addresses), 1) # Now find that address in each subsequent tx, and no other change - bumped_total_tx = rbf_node.bumpfee(rbfid, {"fee_rate": ECONOMICAL}) + bumped_total_tx = rbf_node.bumpfee(rbfid, fee_rate=ECONOMICAL) assert_equal(change_addresses, get_change_address(bumped_total_tx['txid'], rbf_node)) bumped_rate_tx = rbf_node.bumpfee(bumped_total_tx["txid"]) assert_equal(change_addresses, get_change_address(bumped_rate_tx['txid'], rbf_node)) diff --git a/test/functional/wallet_conflicts.py b/test/functional/wallet_conflicts.py new file mode 100755 index 0000000000..802b718cd5 --- /dev/null +++ b/test/functional/wallet_conflicts.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Test that wallet correctly tracks transactions that have been conflicted by blocks, particularly during reorgs. +""" + +from decimal import Decimal + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) + +class TxConflicts(BitcoinTestFramework): + def add_options(self, parser): + self.add_wallet_options(parser) + + def set_test_params(self): + self.num_nodes = 3 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def get_utxo_of_value(self, from_tx_id, search_value): + return next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(from_tx_id)["details"] if tx_out["amount"] == Decimal(f"{search_value}")) + + def run_test(self): + self.log.info("Send tx from which to conflict outputs later") + txid_conflict_from_1 = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) + txid_conflict_from_2 = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) + self.generate(self.nodes[0], 1) + self.sync_blocks() + + self.log.info("Disconnect nodes to broadcast conflicts on their respective chains") + self.disconnect_nodes(0, 1) + self.disconnect_nodes(2, 1) + + self.log.info("Create transactions that conflict with each other") + output_A = self.get_utxo_of_value(from_tx_id=txid_conflict_from_1, search_value=10) + output_B = self.get_utxo_of_value(from_tx_id=txid_conflict_from_2, search_value=10) + + # First create a transaction that consumes both A and B outputs. + # + # | tx1 | -----> | | | | + # | AB_parent_tx | ----> | Child_Tx | + # | tx2 | -----> | | | | + # + inputs_tx_AB_parent = [{"txid": txid_conflict_from_1, "vout": output_A}, {"txid": txid_conflict_from_2, "vout": output_B}] + tx_AB_parent = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs_tx_AB_parent, {self.nodes[0].getnewaddress(): Decimal("19.99998")})) + + # Secondly, create two transactions: One consuming output_A, and another one consuming output_B + # + # | tx1 | -----> | Tx_A_1 | + # ---------------- + # | tx2 | -----> | Tx_B_1 | + # + inputs_tx_A_1 = [{"txid": txid_conflict_from_1, "vout": output_A}] + inputs_tx_B_1 = [{"txid": txid_conflict_from_2, "vout": output_B}] + tx_A_1 = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs_tx_A_1, {self.nodes[0].getnewaddress(): Decimal("9.99998")})) + tx_B_1 = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs_tx_B_1, {self.nodes[0].getnewaddress(): Decimal("9.99998")})) + + self.log.info("Broadcast conflicted transaction") + txid_AB_parent = self.nodes[0].sendrawtransaction(tx_AB_parent["hex"]) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) + + # Now that 'AB_parent_tx' was broadcast, build 'Child_Tx' + output_c = self.get_utxo_of_value(from_tx_id=txid_AB_parent, search_value=19.99998) + inputs_tx_C_child = [({"txid": txid_AB_parent, "vout": output_c})] + + tx_C_child = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs_tx_C_child, {self.nodes[0].getnewaddress() : Decimal("19.99996")})) + tx_C_child_txid = self.nodes[0].sendrawtransaction(tx_C_child["hex"]) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) + + self.log.info("Broadcast conflicting tx to node 1 and generate a longer chain") + conflicting_txid_A = self.nodes[1].sendrawtransaction(tx_A_1["hex"]) + self.generate(self.nodes[1], 4, sync_fun=self.no_op) + conflicting_txid_B = self.nodes[1].sendrawtransaction(tx_B_1["hex"]) + self.generate(self.nodes[1], 4, sync_fun=self.no_op) + + self.log.info("Connect nodes 0 and 1, trigger reorg and ensure that the tx is effectively conflicted") + self.connect_nodes(0, 1) + self.sync_blocks([self.nodes[0], self.nodes[1]]) + conflicted_AB_tx = self.nodes[0].gettransaction(txid_AB_parent) + tx_C_child = self.nodes[0].gettransaction(tx_C_child_txid) + conflicted_A_tx = self.nodes[0].gettransaction(conflicting_txid_A) + + self.log.info("Verify, after the reorg, that Tx_A was accepted, and tx_AB and its Child_Tx are conflicting now") + # Tx A was accepted, Tx AB was not. + assert conflicted_AB_tx["confirmations"] < 0 + assert conflicted_A_tx["confirmations"] > 0 + + # Conflicted tx should have confirmations set to the confirmations of the most conflicting tx + assert_equal(-conflicted_AB_tx["confirmations"], conflicted_A_tx["confirmations"]) + # Child should inherit conflicted state from parent + assert_equal(-tx_C_child["confirmations"], conflicted_A_tx["confirmations"]) + # Check the confirmations of the conflicting transactions + assert_equal(conflicted_A_tx["confirmations"], 8) + assert_equal(self.nodes[0].gettransaction(conflicting_txid_B)["confirmations"], 4) + + self.log.info("Now generate a longer chain that does not contain any tx") + # Node2 chain without conflicts + self.generate(self.nodes[2], 15, sync_fun=self.no_op) + + # Connect node0 and node2 and wait reorg + self.connect_nodes(0, 2) + self.sync_blocks() + conflicted = self.nodes[0].gettransaction(txid_AB_parent) + tx_C_child = self.nodes[0].gettransaction(tx_C_child_txid) + + self.log.info("Test that formerly conflicted transaction are inactive after reorg") + # Former conflicted tx should be unconfirmed as it hasn't been yet rebroadcast + assert_equal(conflicted["confirmations"], 0) + # Former conflicted child tx should be unconfirmed as it hasn't been rebroadcast + assert_equal(tx_C_child["confirmations"], 0) + # Rebroadcast former conflicted tx and check it confirms smoothly + self.nodes[2].sendrawtransaction(conflicted["hex"]) + self.generate(self.nodes[2], 1) + self.sync_blocks() + former_conflicted = self.nodes[0].gettransaction(txid_AB_parent) + assert_equal(former_conflicted["confirmations"], 1) + assert_equal(former_conflicted["blockheight"], 217) + +if __name__ == '__main__': + TxConflicts().main() diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index 58cc6befbd..75b507c387 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -7,13 +7,13 @@ from test_framework.address import key_to_p2wpkh from test_framework.descriptors import descsum_create -from test_framework.key import ECKey from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) -from test_framework.wallet_util import bytes_to_wif, generate_wif_key +from test_framework.wallet_util import generate_keypair + EMPTY_PASSPHRASE_MSG = "Empty string given as passphrase, wallet will not be encrypted." LEGACY_WALLET_MSG = "Wallet created successfully. The legacy wallet type is being deprecated and support for creating and opening legacy wallets will be removed in the future." @@ -25,7 +25,6 @@ class CreateWalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [["-deprecatedrpc=walletwarningfield"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -51,14 +50,12 @@ class CreateWalletTest(BitcoinTestFramework): w1.importpubkey(w0.getaddressinfo(address1)['pubkey']) self.log.info('Test that private keys cannot be imported') - eckey = ECKey() - eckey.generate() - privkey = bytes_to_wif(eckey.get_bytes()) + privkey, pubkey = generate_keypair(wif=True) assert_raises_rpc_error(-4, 'Cannot import private keys to a wallet with private keys disabled', w1.importprivkey, privkey) if self.options.descriptors: result = w1.importdescriptors([{'desc': descsum_create('wpkh(' + privkey + ')'), 'timestamp': 'now'}]) else: - result = w1.importmulti([{'scriptPubKey': {'address': key_to_p2wpkh(eckey.get_pubkey().get_bytes())}, 'timestamp': 'now', 'keys': [privkey]}]) + result = w1.importmulti([{'scriptPubKey': {'address': key_to_p2wpkh(pubkey)}, 'timestamp': 'now', 'keys': [privkey]}]) assert not result[0]['success'] assert 'warnings' not in result[0] assert_equal(result[0]['error']['code'], -4) @@ -78,7 +75,7 @@ class CreateWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getnewaddress) assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getrawchangeaddress) # Import private key - w3.importprivkey(generate_wif_key()) + w3.importprivkey(generate_keypair(wif=True)[0]) # Imported private keys are currently ignored by the keypool assert_equal(w3.getwalletinfo()['keypoolsize'], 0) assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getnewaddress) @@ -164,7 +161,6 @@ class CreateWalletTest(BitcoinTestFramework): assert_equal(walletinfo['keypoolsize_hd_internal'], keys) # Allow empty passphrase, but there should be a warning resp = self.nodes[0].createwallet(wallet_name='w7', disable_private_keys=False, blank=False, passphrase='') - assert_equal(resp["warning"], EMPTY_PASSPHRASE_MSG if self.options.descriptors else f"{EMPTY_PASSPHRASE_MSG}\n{LEGACY_WALLET_MSG}") assert_equal(resp["warnings"], [EMPTY_PASSPHRASE_MSG] if self.options.descriptors else [EMPTY_PASSPHRASE_MSG, LEGACY_WALLET_MSG]) w7 = node.get_wallet_rpc('w7') @@ -184,21 +180,14 @@ class CreateWalletTest(BitcoinTestFramework): result = self.nodes[0].createwallet(wallet_name="legacy_w0", descriptors=False, passphrase=None) assert_equal(result, { "name": "legacy_w0", - "warning": LEGACY_WALLET_MSG, "warnings": [LEGACY_WALLET_MSG], }) result = self.nodes[0].createwallet(wallet_name="legacy_w1", descriptors=False, passphrase="") assert_equal(result, { "name": "legacy_w1", - "warning": f"{EMPTY_PASSPHRASE_MSG}\n{LEGACY_WALLET_MSG}", "warnings": [EMPTY_PASSPHRASE_MSG, LEGACY_WALLET_MSG], }) - self.log.info('Test "warning" field deprecation, i.e. not returned without -deprecatedrpc=walletwarningfield') - self.restart_node(0, extra_args=[]) - result = self.nodes[0].createwallet(wallet_name="w7_again", disable_private_keys=False, blank=False, passphrase="") - assert "warning" not in result - if __name__ == '__main__': CreateWalletTest().main() diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index 4673eb091c..f4b67bae1b 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -234,7 +234,7 @@ class WalletDescriptorTest(BitcoinTestFramework): self.log.info("Test that loading descriptor wallet containing legacy key types throws error") self.nodes[0].createwallet(wallet_name="crashme", descriptors=True) self.nodes[0].unloadwallet("crashme") - wallet_db = os.path.join(self.nodes[0].datadir, self.chain, "wallets", "crashme", self.wallet_data_filename) + wallet_db = os.path.join(self.nodes[0].wallets_path, "crashme", self.wallet_data_filename) with sqlite3.connect(wallet_db) as conn: # add "cscript" entry: key type is uint160 (20 bytes), value type is CScript (zero-length here) conn.execute('INSERT INTO main VALUES(?, ?)', (b'\x07cscript' + b'\x00'*20, b'\x00')) diff --git a/test/functional/wallet_fundrawtransaction.py b/test/functional/wallet_fundrawtransaction.py index 29ddb77b41..46706d6ad2 100755 --- a/test/functional/wallet_fundrawtransaction.py +++ b/test/functional/wallet_fundrawtransaction.py @@ -10,7 +10,6 @@ from itertools import product from math import ceil from test_framework.descriptors import descsum_create -from test_framework.key import ECKey from test_framework.messages import ( COIN, ) @@ -25,7 +24,7 @@ from test_framework.util import ( count_bytes, find_vout_for_address, ) -from test_framework.wallet_util import bytes_to_wif +from test_framework.wallet_util import generate_keypair ERR_NOT_ENOUGH_PRESET_INPUTS = "The preselected coins total amount does not cover the transaction target. " \ "Please allow other inputs to be automatically selected or include more coins manually" @@ -154,7 +153,7 @@ class RawTransactionsTest(BitcoinTestFramework): """Ensure setting changePosition in fundraw with an exact match is handled properly.""" self.log.info("Test fundrawtxn changePosition option") rawmatch = self.nodes[2].createrawtransaction([], {self.nodes[2].getnewaddress():50}) - rawmatch = self.nodes[2].fundrawtransaction(rawmatch, {"changePosition":1, "subtractFeeFromOutputs":[0]}) + rawmatch = self.nodes[2].fundrawtransaction(rawmatch, changePosition=1, subtractFeeFromOutputs=[0]) assert_equal(rawmatch["changepos"], -1) self.nodes[3].createwallet(wallet_name="wwatch", disable_private_keys=True) @@ -268,10 +267,10 @@ class RawTransactionsTest(BitcoinTestFramework): dec_tx = self.nodes[2].decoderawtransaction(rawtx) assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) - assert_raises_rpc_error(-3, "Unexpected key foo", self.nodes[2].fundrawtransaction, rawtx, {'foo':'bar'}) + assert_raises_rpc_error(-8, "Unknown named parameter foo", self.nodes[2].fundrawtransaction, rawtx, foo='bar') # reserveChangeKey was deprecated and is now removed - assert_raises_rpc_error(-3, "Unexpected key reserveChangeKey", lambda: self.nodes[2].fundrawtransaction(hexstring=rawtx, options={'reserveChangeKey': True})) + assert_raises_rpc_error(-8, "Unknown named parameter reserveChangeKey", lambda: self.nodes[2].fundrawtransaction(hexstring=rawtx, reserveChangeKey=True)) def test_invalid_change_address(self): self.log.info("Test fundrawtxn with an invalid change address") @@ -283,7 +282,7 @@ class RawTransactionsTest(BitcoinTestFramework): dec_tx = self.nodes[2].decoderawtransaction(rawtx) assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) - assert_raises_rpc_error(-5, "Change address must be a valid bitcoin address", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':'foobar'}) + assert_raises_rpc_error(-5, "Change address must be a valid bitcoin address", self.nodes[2].fundrawtransaction, rawtx, changeAddress='foobar') def test_valid_change_address(self): self.log.info("Test fundrawtxn with a provided change address") @@ -296,8 +295,8 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) change = self.nodes[2].getnewaddress() - assert_raises_rpc_error(-8, "changePosition out of bounds", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':change, 'changePosition':2}) - rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {'changeAddress': change, 'changePosition': 0}) + assert_raises_rpc_error(-8, "changePosition out of bounds", self.nodes[2].fundrawtransaction, rawtx, changeAddress=change, changePosition=2) + rawtxfund = self.nodes[2].fundrawtransaction(rawtx, changeAddress=change, changePosition=0) dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) out = dec_tx['vout'][0] assert_equal(change, out['scriptPubKey']['address']) @@ -309,9 +308,9 @@ class RawTransactionsTest(BitcoinTestFramework): inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] outputs = { self.nodes[0].getnewaddress() : Decimal(4.0) } rawtx = self.nodes[2].createrawtransaction(inputs, outputs) - assert_raises_rpc_error(-3, "JSON value of type null is not of expected type string", self.nodes[2].fundrawtransaction, rawtx, {'change_type': None}) - assert_raises_rpc_error(-5, "Unknown change type ''", self.nodes[2].fundrawtransaction, rawtx, {'change_type': ''}) - rawtx = self.nodes[2].fundrawtransaction(rawtx, {'change_type': 'bech32'}) + assert_raises_rpc_error(-3, "JSON value of type null is not of expected type string", self.nodes[2].fundrawtransaction, rawtx, change_type=None) + assert_raises_rpc_error(-5, "Unknown change type ''", self.nodes[2].fundrawtransaction, rawtx, change_type='') + rawtx = self.nodes[2].fundrawtransaction(rawtx, change_type='bech32') dec_tx = self.nodes[2].decoderawtransaction(rawtx['hex']) assert_equal('witness_v0_keyhash', dec_tx['vout'][rawtx['changepos']]['scriptPubKey']['type']) @@ -331,7 +330,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal("00", dec_tx['vin'][0]['scriptSig']['hex']) # Should fail without add_inputs: - assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, self.nodes[2].fundrawtransaction, rawtx, {"add_inputs": False}) + assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, self.nodes[2].fundrawtransaction, rawtx, add_inputs=False) # add_inputs is enabled by default rawtxfund = self.nodes[2].fundrawtransaction(rawtx) @@ -363,8 +362,8 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) # Should fail without add_inputs: - assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, self.nodes[2].fundrawtransaction, rawtx, {"add_inputs": False}) - rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {"add_inputs": True}) + assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, self.nodes[2].fundrawtransaction, rawtx, add_inputs=False) + rawtxfund = self.nodes[2].fundrawtransaction(rawtx, add_inputs=True) dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) totalOut = 0 @@ -397,8 +396,8 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) # Should fail without add_inputs: - assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, self.nodes[2].fundrawtransaction, rawtx, {"add_inputs": False}) - rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {"add_inputs": True}) + assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, self.nodes[2].fundrawtransaction, rawtx, add_inputs=False) + rawtxfund = self.nodes[2].fundrawtransaction(rawtx, add_inputs=True) dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) totalOut = 0 @@ -567,7 +566,7 @@ class RawTransactionsTest(BitcoinTestFramework): oldBalance = self.nodes[1].getbalance() inputs = [] outputs = {self.nodes[1].getnewaddress():1.1} - funded_psbt = wmulti.walletcreatefundedpsbt(inputs=inputs, outputs=outputs, options={'changeAddress': w2.getrawchangeaddress()})['psbt'] + funded_psbt = wmulti.walletcreatefundedpsbt(inputs=inputs, outputs=outputs, changeAddress=w2.getrawchangeaddress())['psbt'] signed_psbt = w2.walletprocesspsbt(funded_psbt) final_psbt = w2.finalizepsbt(signed_psbt['psbt']) @@ -750,7 +749,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[3].loadwallet('wwatch') wwatch = self.nodes[3].get_wallet_rpc('wwatch') w3 = self.nodes[3].get_wallet_rpc(self.default_wallet_name) - result = wwatch.fundrawtransaction(rawtx, {'includeWatching': True, 'changeAddress': w3.getrawchangeaddress(), 'subtractFeeFromOutputs': [0]}) + result = wwatch.fundrawtransaction(rawtx, includeWatching=True, changeAddress=w3.getrawchangeaddress(), subtractFeeFromOutputs=[0]) res_dec = self.nodes[0].decoderawtransaction(result["hex"]) assert_equal(len(res_dec["vin"]), 1) assert res_dec["vin"][0]["txid"] == self.watchonly_txid @@ -779,10 +778,10 @@ class RawTransactionsTest(BitcoinTestFramework): result = node.fundrawtransaction(rawtx) # uses self.min_relay_tx_fee (set by settxfee) btc_kvb_to_sat_vb = 100000 # (1e5) - result1 = node.fundrawtransaction(rawtx, {"fee_rate": str(2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee)}) - result2 = node.fundrawtransaction(rawtx, {"feeRate": 2 * self.min_relay_tx_fee}) - result3 = node.fundrawtransaction(rawtx, {"fee_rate": 10 * btc_kvb_to_sat_vb * self.min_relay_tx_fee}) - result4 = node.fundrawtransaction(rawtx, {"feeRate": str(10 * self.min_relay_tx_fee)}) + result1 = node.fundrawtransaction(rawtx, fee_rate=str(2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee)) + result2 = node.fundrawtransaction(rawtx, feeRate=2 * self.min_relay_tx_fee) + result3 = node.fundrawtransaction(rawtx, fee_rate=10 * btc_kvb_to_sat_vb * self.min_relay_tx_fee) + result4 = node.fundrawtransaction(rawtx, feeRate=str(10 * self.min_relay_tx_fee)) result_fee_rate = result['fee'] * 1000 / count_bytes(result['hex']) assert_fee_amount(result1['fee'], count_bytes(result1['hex']), 2 * result_fee_rate) @@ -797,54 +796,54 @@ class RawTransactionsTest(BitcoinTestFramework): # With no arguments passed, expect fee of 141 satoshis. assert_approx(node.fundrawtransaction(rawtx)["fee"], vexp=0.00000141, vspan=0.00000001) # Expect fee to be 10,000x higher when an explicit fee rate 10,000x greater is specified. - result = node.fundrawtransaction(rawtx, {"fee_rate": 10000}) + result = node.fundrawtransaction(rawtx, fee_rate=10000) assert_approx(result["fee"], vexp=0.0141, vspan=0.0001) self.log.info("Test fundrawtxn with invalid estimate_mode settings") for k, v in {"number": 42, "object": {"foo": "bar"}}.items(): assert_raises_rpc_error(-3, f"JSON value of type {k} for field estimate_mode is not of expected type string", - node.fundrawtransaction, rawtx, {"estimate_mode": v, "conf_target": 0.1, "add_inputs": True}) + node.fundrawtransaction, rawtx, estimate_mode=v, conf_target=0.1, add_inputs=True) for mode in ["", "foo", Decimal("3.141592")]: assert_raises_rpc_error(-8, 'Invalid estimate_mode parameter, must be one of: "unset", "economical", "conservative"', - node.fundrawtransaction, rawtx, {"estimate_mode": mode, "conf_target": 0.1, "add_inputs": True}) + node.fundrawtransaction, rawtx, estimate_mode=mode, conf_target=0.1, add_inputs=True) self.log.info("Test fundrawtxn with invalid conf_target settings") for mode in ["unset", "economical", "conservative"]: self.log.debug("{}".format(mode)) for k, v in {"string": "", "object": {"foo": "bar"}}.items(): assert_raises_rpc_error(-3, f"JSON value of type {k} for field conf_target is not of expected type number", - node.fundrawtransaction, rawtx, {"estimate_mode": mode, "conf_target": v, "add_inputs": True}) + node.fundrawtransaction, rawtx, estimate_mode=mode, conf_target=v, add_inputs=True) for n in [-1, 0, 1009]: assert_raises_rpc_error(-8, "Invalid conf_target, must be between 1 and 1008", # max value of 1008 per src/policy/fees.h - node.fundrawtransaction, rawtx, {"estimate_mode": mode, "conf_target": n, "add_inputs": True}) + node.fundrawtransaction, rawtx, estimate_mode=mode, conf_target=n, add_inputs=True) self.log.info("Test invalid fee rate settings") for param, value in {("fee_rate", 100000), ("feeRate", 1.000)}: assert_raises_rpc_error(-4, "Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)", - node.fundrawtransaction, rawtx, {param: value, "add_inputs": True}) + node.fundrawtransaction, rawtx, add_inputs=True, **{param: value}) assert_raises_rpc_error(-3, "Amount out of range", - node.fundrawtransaction, rawtx, {param: -1, "add_inputs": True}) + node.fundrawtransaction, rawtx, add_inputs=True, **{param: -1}) assert_raises_rpc_error(-3, "Amount is not a number or string", - node.fundrawtransaction, rawtx, {param: {"foo": "bar"}, "add_inputs": True}) + node.fundrawtransaction, rawtx, add_inputs=True, **{param: {"foo": "bar"}}) # Test fee rate values that don't pass fixed-point parsing checks. for invalid_value in ["", 0.000000001, 1e-09, 1.111111111, 1111111111111111, "31.999999999999999999999"]: - assert_raises_rpc_error(-3, "Invalid amount", node.fundrawtransaction, rawtx, {param: invalid_value, "add_inputs": True}) + assert_raises_rpc_error(-3, "Invalid amount", node.fundrawtransaction, rawtx, add_inputs=True, **{param: invalid_value}) # Test fee_rate values that cannot be represented in sat/vB. for invalid_value in [0.0001, 0.00000001, 0.00099999, 31.99999999, "0.0001", "0.00000001", "0.00099999", "31.99999999"]: assert_raises_rpc_error(-3, "Invalid amount", - node.fundrawtransaction, rawtx, {"fee_rate": invalid_value, "add_inputs": True}) + node.fundrawtransaction, rawtx, fee_rate=invalid_value, add_inputs=True) self.log.info("Test min fee rate checks are bypassed with fundrawtxn, e.g. a fee_rate under 1 sat/vB is allowed") - node.fundrawtransaction(rawtx, {"fee_rate": 0.999, "add_inputs": True}) - node.fundrawtransaction(rawtx, {"feeRate": 0.00000999, "add_inputs": True}) + node.fundrawtransaction(rawtx, fee_rate=0.999, add_inputs=True) + node.fundrawtransaction(rawtx, feeRate=0.00000999, add_inputs=True) self.log.info("- raises RPC error if both feeRate and fee_rate are passed") assert_raises_rpc_error(-8, "Cannot specify both fee_rate (sat/vB) and feeRate (BTC/kvB)", - node.fundrawtransaction, rawtx, {"fee_rate": 0.1, "feeRate": 0.1, "add_inputs": True}) + node.fundrawtransaction, rawtx, fee_rate=0.1, feeRate=0.1, add_inputs=True) self.log.info("- raises RPC error if both feeRate and estimate_mode passed") assert_raises_rpc_error(-8, "Cannot specify both estimate_mode and feeRate", - node.fundrawtransaction, rawtx, {"estimate_mode": "economical", "feeRate": 0.1, "add_inputs": True}) + node.fundrawtransaction, rawtx, estimate_mode="economical", feeRate=0.1, add_inputs=True) for param in ["feeRate", "fee_rate"]: self.log.info("- raises RPC error if both {} and conf_target are passed".format(param)) @@ -854,7 +853,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.log.info("- raises RPC error if both fee_rate and estimate_mode are passed") assert_raises_rpc_error(-8, "Cannot specify both estimate_mode and fee_rate", - node.fundrawtransaction, rawtx, {"fee_rate": 1, "estimate_mode": "economical", "add_inputs": True}) + node.fundrawtransaction, rawtx, fee_rate=1, estimate_mode="economical", add_inputs=True) def test_address_reuse(self): """Test no address reuse occurs.""" @@ -884,10 +883,10 @@ class RawTransactionsTest(BitcoinTestFramework): # Test subtract fee from outputs with feeRate (BTC/kvB) result = [self.nodes[3].fundrawtransaction(rawtx), # uses self.min_relay_tx_fee (set by settxfee) - self.nodes[3].fundrawtransaction(rawtx, {"subtractFeeFromOutputs": []}), # empty subtraction list - self.nodes[3].fundrawtransaction(rawtx, {"subtractFeeFromOutputs": [0]}), # uses self.min_relay_tx_fee (set by settxfee) - self.nodes[3].fundrawtransaction(rawtx, {"feeRate": 2 * self.min_relay_tx_fee}), - self.nodes[3].fundrawtransaction(rawtx, {"feeRate": 2 * self.min_relay_tx_fee, "subtractFeeFromOutputs": [0]}),] + self.nodes[3].fundrawtransaction(rawtx, subtractFeeFromOutputs=[]), # empty subtraction list + self.nodes[3].fundrawtransaction(rawtx, subtractFeeFromOutputs=[0]), # uses self.min_relay_tx_fee (set by settxfee) + self.nodes[3].fundrawtransaction(rawtx, feeRate=2 * self.min_relay_tx_fee), + self.nodes[3].fundrawtransaction(rawtx, feeRate=2 * self.min_relay_tx_fee, subtractFeeFromOutputs=[0]),] dec_tx = [self.nodes[3].decoderawtransaction(tx_['hex']) for tx_ in result] output = [d['vout'][1 - r['changepos']]['value'] for d, r in zip(dec_tx, result)] change = [d['vout'][r['changepos']]['value'] for d, r in zip(dec_tx, result)] @@ -904,10 +903,10 @@ class RawTransactionsTest(BitcoinTestFramework): # Test subtract fee from outputs with fee_rate (sat/vB) btc_kvb_to_sat_vb = 100000 # (1e5) result = [self.nodes[3].fundrawtransaction(rawtx), # uses self.min_relay_tx_fee (set by settxfee) - self.nodes[3].fundrawtransaction(rawtx, {"subtractFeeFromOutputs": []}), # empty subtraction list - self.nodes[3].fundrawtransaction(rawtx, {"subtractFeeFromOutputs": [0]}), # uses self.min_relay_tx_fee (set by settxfee) - self.nodes[3].fundrawtransaction(rawtx, {"fee_rate": 2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee}), - self.nodes[3].fundrawtransaction(rawtx, {"fee_rate": 2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee, "subtractFeeFromOutputs": [0]}),] + self.nodes[3].fundrawtransaction(rawtx, subtractFeeFromOutputs=[]), # empty subtraction list + self.nodes[3].fundrawtransaction(rawtx, subtractFeeFromOutputs=[0]), # uses self.min_relay_tx_fee (set by settxfee) + self.nodes[3].fundrawtransaction(rawtx, fee_rate=2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee), + self.nodes[3].fundrawtransaction(rawtx, fee_rate=2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee, subtractFeeFromOutputs=[0]),] dec_tx = [self.nodes[3].decoderawtransaction(tx_['hex']) for tx_ in result] output = [d['vout'][1 - r['changepos']]['value'] for d, r in zip(dec_tx, result)] change = [d['vout'][r['changepos']]['value'] for d, r in zip(dec_tx, result)] @@ -927,7 +926,7 @@ class RawTransactionsTest(BitcoinTestFramework): result = [self.nodes[3].fundrawtransaction(rawtx), # Split the fee between outputs 0, 2, and 3, but not output 1. - self.nodes[3].fundrawtransaction(rawtx, {"subtractFeeFromOutputs": [0, 2, 3]})] + self.nodes[3].fundrawtransaction(rawtx, subtractFeeFromOutputs=[0, 2, 3])] dec_tx = [self.nodes[3].decoderawtransaction(result[0]['hex']), self.nodes[3].decoderawtransaction(result[1]['hex'])] @@ -969,7 +968,7 @@ class RawTransactionsTest(BitcoinTestFramework): vout = find_vout_for_address(self.nodes[0], txid, addr) rawtx = self.nodes[0].createrawtransaction([{'txid': txid, 'vout': vout}], [{self.nodes[0].getnewaddress(): 5}]) - fundedtx = self.nodes[0].fundrawtransaction(rawtx, {'subtractFeeFromOutputs': [0]}) + fundedtx = self.nodes[0].fundrawtransaction(rawtx, subtractFeeFromOutputs=[0]) signedtx = self.nodes[0].signrawtransactionwithwallet(fundedtx['hex']) self.nodes[0].sendrawtransaction(signedtx['hex']) @@ -999,11 +998,7 @@ class RawTransactionsTest(BitcoinTestFramework): def test_external_inputs(self): self.log.info("Test funding with external inputs") - - eckey = ECKey() - eckey.generate() - privkey = bytes_to_wif(eckey.get_bytes()) - + privkey, _ = generate_keypair(wif=True) self.nodes[2].createwallet("extfund") wallet = self.nodes[2].get_wallet_rpc("extfund") @@ -1027,25 +1022,25 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Not solvable pre-selected input COutPoint(%s, %s)" % (ext_utxo["txid"][0:10], ext_utxo["vout"]), wallet.fundrawtransaction, raw_tx) # Error conditions - assert_raises_rpc_error(-5, "'not a pubkey' is not hex", wallet.fundrawtransaction, raw_tx, {"solving_data": {"pubkeys":["not a pubkey"]}}) - assert_raises_rpc_error(-5, "'01234567890a0b0c0d0e0f' is not a valid public key", wallet.fundrawtransaction, raw_tx, {"solving_data": {"pubkeys":["01234567890a0b0c0d0e0f"]}}) - assert_raises_rpc_error(-5, "'not a script' is not hex", wallet.fundrawtransaction, raw_tx, {"solving_data": {"scripts":["not a script"]}}) - assert_raises_rpc_error(-8, "Unable to parse descriptor 'not a descriptor'", wallet.fundrawtransaction, raw_tx, {"solving_data": {"descriptors":["not a descriptor"]}}) - assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"]}]}) - assert_raises_rpc_error(-8, "Invalid parameter, vout cannot be negative", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": -1}]}) - assert_raises_rpc_error(-8, "Invalid parameter, missing weight key", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"]}]}) - assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be less than 165", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 164}]}) - assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be less than 165", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": -1}]}) - assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be greater than", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 400001}]}) + assert_raises_rpc_error(-5, "'not a pubkey' is not hex", wallet.fundrawtransaction, raw_tx, solving_data={"pubkeys":["not a pubkey"]}) + assert_raises_rpc_error(-5, "'01234567890a0b0c0d0e0f' is not a valid public key", wallet.fundrawtransaction, raw_tx, solving_data={"pubkeys":["01234567890a0b0c0d0e0f"]}) + assert_raises_rpc_error(-5, "'not a script' is not hex", wallet.fundrawtransaction, raw_tx, solving_data={"scripts":["not a script"]}) + assert_raises_rpc_error(-8, "Unable to parse descriptor 'not a descriptor'", wallet.fundrawtransaction, raw_tx, solving_data={"descriptors":["not a descriptor"]}) + assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", wallet.fundrawtransaction, raw_tx, input_weights=[{"txid": ext_utxo["txid"]}]) + assert_raises_rpc_error(-8, "Invalid parameter, vout cannot be negative", wallet.fundrawtransaction, raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": -1}]) + assert_raises_rpc_error(-8, "Invalid parameter, missing weight key", wallet.fundrawtransaction, raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"]}]) + assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be less than 165", wallet.fundrawtransaction, raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 164}]) + assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be less than 165", wallet.fundrawtransaction, raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": -1}]) + assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be greater than", wallet.fundrawtransaction, raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 400001}]) # But funding should work when the solving data is provided - funded_tx = wallet.fundrawtransaction(raw_tx, {"solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}}) + funded_tx = wallet.fundrawtransaction(raw_tx, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}) signed_tx = wallet.signrawtransactionwithwallet(funded_tx['hex']) assert not signed_tx['complete'] signed_tx = self.nodes[0].signrawtransactionwithwallet(signed_tx['hex']) assert signed_tx['complete'] - funded_tx = wallet.fundrawtransaction(raw_tx, {"solving_data": {"descriptors": [desc]}}) + funded_tx = wallet.fundrawtransaction(raw_tx, solving_data={"descriptors": [desc]}) signed_tx1 = wallet.signrawtransactionwithwallet(funded_tx['hex']) assert not signed_tx1['complete'] signed_tx2 = self.nodes[0].signrawtransactionwithwallet(signed_tx1['hex']) @@ -1060,30 +1055,30 @@ class RawTransactionsTest(BitcoinTestFramework): high_input_weight = input_weight * 2 # Funding should also work if the input weight is provided - funded_tx = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": input_weight}]}) + funded_tx = wallet.fundrawtransaction(raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": input_weight}]) signed_tx = wallet.signrawtransactionwithwallet(funded_tx["hex"]) signed_tx = self.nodes[0].signrawtransactionwithwallet(signed_tx["hex"]) assert_equal(self.nodes[0].testmempoolaccept([signed_tx["hex"]])[0]["allowed"], True) assert_equal(signed_tx["complete"], True) # Reducing the weight should have a lower fee - funded_tx2 = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": low_input_weight}]}) + funded_tx2 = wallet.fundrawtransaction(raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": low_input_weight}]) assert_greater_than(funded_tx["fee"], funded_tx2["fee"]) # Increasing the weight should have a higher fee - funded_tx2 = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}]}) + funded_tx2 = wallet.fundrawtransaction(raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}]) assert_greater_than(funded_tx2["fee"], funded_tx["fee"]) # The provided weight should override the calculated weight when solving data is provided - funded_tx3 = wallet.fundrawtransaction(raw_tx, {"solving_data": {"descriptors": [desc]}, "input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}]}) + funded_tx3 = wallet.fundrawtransaction(raw_tx, solving_data={"descriptors": [desc]}, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}]) assert_equal(funded_tx2["fee"], funded_tx3["fee"]) # The feerate should be met - funded_tx4 = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], "fee_rate": 10}) + funded_tx4 = wallet.fundrawtransaction(raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], fee_rate=10) input_add_weight = high_input_weight - (41 * 4) tx4_weight = wallet.decoderawtransaction(funded_tx4["hex"])["weight"] + input_add_weight tx4_vsize = int(ceil(tx4_weight / 4)) assert_fee_amount(funded_tx4["fee"], tx4_vsize, Decimal(0.0001)) # Funding with weight at csuint boundaries should not cause problems - funded_tx = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 255}]}) - funded_tx = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 65539}]}) + funded_tx = wallet.fundrawtransaction(raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 255}]) + funded_tx = wallet.fundrawtransaction(raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 65539}]) self.nodes[2].unloadwallet("extfund") @@ -1123,7 +1118,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Fund wallet with 2 outputs, 5 BTC each. addr2 = wallet.getnewaddress(address_type="bech32") - source_tx = self.nodes[0].send(outputs=[{addr1: 5}, {addr2: 5}], options={"change_position": 0}) + source_tx = self.nodes[0].send(outputs=[{addr1: 5}, {addr2: 5}], change_position=0) self.generate(self.nodes[0], 1) # Select only one input. @@ -1135,12 +1130,12 @@ class RawTransactionsTest(BitcoinTestFramework): } ] } - assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, wallet.send, outputs=[{addr1: 8}], options=options) + assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, wallet.send, outputs=[{addr1: 8}], **options) # Case (3), Explicit add_inputs=true and preset inputs (with preset inputs not-covering the target amount) options["add_inputs"] = True options["add_to_wallet"] = False - tx = wallet.send(outputs=[{addr1: 8}], options=options) + tx = wallet.send(outputs=[{addr1: 8}], **options) assert tx["complete"] # Case (4), Explicit add_inputs=true and preset inputs (with preset inputs covering the target amount) @@ -1148,7 +1143,7 @@ class RawTransactionsTest(BitcoinTestFramework): "txid": source_tx["txid"], "vout": 2 # change position was hardcoded to index 0 }) - tx = wallet.send(outputs=[{addr1: 8}], options=options) + tx = wallet.send(outputs=[{addr1: 8}], **options) assert tx["complete"] # Check that only the preset inputs were added to the tx decoded_psbt_inputs = self.nodes[0].decodepsbt(tx["psbt"])['tx']['vin'] @@ -1158,12 +1153,12 @@ class RawTransactionsTest(BitcoinTestFramework): # Case (5), assert that inputs are added to the tx by explicitly setting add_inputs=true options = {"add_inputs": True, "add_to_wallet": True} - tx = wallet.send(outputs=[{addr1: 8}], options=options) + tx = wallet.send(outputs=[{addr1: 8}], **options) assert tx["complete"] # 6. Explicit add_inputs=false, no preset inputs: options = {"add_inputs": False} - assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, wallet.send, outputs=[{addr1: 3}], options=options) + assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, wallet.send, outputs=[{addr1: 3}], **options) ################################################ @@ -1184,14 +1179,14 @@ class RawTransactionsTest(BitcoinTestFramework): # Case (3), Explicit add_inputs=true and preset inputs (with preset inputs not-covering the target amount) options["add_inputs"] = True - assert "psbt" in wallet.walletcreatefundedpsbt(outputs=[{addr1: 8}], inputs=inputs, options=options) + assert "psbt" in wallet.walletcreatefundedpsbt(outputs=[{addr1: 8}], inputs=inputs, **options) # Case (4), Explicit add_inputs=true and preset inputs (with preset inputs covering the target amount) inputs.append({ "txid": source_tx["txid"], "vout": 2 # change position was hardcoded to index 0 }) - psbt_tx = wallet.walletcreatefundedpsbt(outputs=[{addr1: 8}], inputs=inputs, options=options) + psbt_tx = wallet.walletcreatefundedpsbt(outputs=[{addr1: 8}], inputs=inputs, **options) # Check that only the preset inputs were added to the tx decoded_psbt_inputs = self.nodes[0].decodepsbt(psbt_tx["psbt"])['tx']['vin'] assert_equal(len(decoded_psbt_inputs), 2) @@ -1203,11 +1198,11 @@ class RawTransactionsTest(BitcoinTestFramework): options = { "add_inputs": True } - assert "psbt" in wallet.walletcreatefundedpsbt(inputs=[], outputs=outputs, options=options) + assert "psbt" in wallet.walletcreatefundedpsbt(inputs=[], outputs=outputs, **options) # Case (6). Explicit add_inputs=false, no preset inputs: options = {"add_inputs": False} - assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, wallet.walletcreatefundedpsbt, inputs=[], outputs=outputs, options=options) + assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, wallet.walletcreatefundedpsbt, inputs=[], outputs=outputs, **options) self.nodes[2].unloadwallet("test_preset_inputs") @@ -1271,7 +1266,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.generate(self.nodes[0], 1) rawtx = wallet.createrawtransaction([{'txid': txid, 'vout': vout}], [{self.nodes[0].getnewaddress(address_type="bech32"): 8}]) - fundedtx = wallet.fundrawtransaction(rawtx, {'fee_rate': 10, "change_type": "bech32"}) + fundedtx = wallet.fundrawtransaction(rawtx, fee_rate=10, change_type="bech32") # with 71-byte signatures we should expect following tx size # tx overhead (10) + 2 inputs (41 each) + 2 p2wpkh (31 each) + (segwit marker and flag (2) + 2 p2wpkh 71 byte sig witnesses (107 each)) / witness scaling factor (4) tx_size = ceil(10 + 41*2 + 31*2 + (2 + 107*2)/4) @@ -1280,7 +1275,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Using the other output should have 72 byte sigs rawtx = wallet.createrawtransaction([{'txid': txid, 'vout': ext_vout}], [{self.nodes[0].getnewaddress(): 13}]) ext_desc = self.nodes[0].getaddressinfo(ext_addr)["desc"] - fundedtx = wallet.fundrawtransaction(rawtx, {'fee_rate': 10, "change_type": "bech32", "solving_data": {"descriptors": [ext_desc]}}) + fundedtx = wallet.fundrawtransaction(rawtx, fee_rate=10, change_type="bech32", solving_data={"descriptors": [ext_desc]}) # tx overhead (10) + 3 inputs (41 each) + 2 p2wpkh(31 each) + (segwit marker and flag (2) + 2 p2wpkh 71 bytes sig witnesses (107 each) + p2wpkh 72 byte sig witness (108)) / witness scaling factor (4) tx_size = ceil(10 + 41*3 + 31*2 + (2 + 107*2 + 108)/4) assert_equal(fundedtx['fee'] * COIN, tx_size * 10) @@ -1307,7 +1302,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Insufficient funds", wallet.fundrawtransaction, rawtx) # But we can opt-in to use them for funding. - fundedtx = wallet.fundrawtransaction(rawtx, {"include_unsafe": True}) + fundedtx = wallet.fundrawtransaction(rawtx, include_unsafe=True) tx_dec = wallet.decoderawtransaction(fundedtx['hex']) assert all((txin["txid"], txin["vout"]) in inputs for txin in tx_dec["vin"]) signedtx = wallet.signrawtransactionwithwallet(fundedtx['hex']) @@ -1315,7 +1310,7 @@ class RawTransactionsTest(BitcoinTestFramework): # And we can also use them once they're confirmed. self.generate(self.nodes[0], 1) - fundedtx = wallet.fundrawtransaction(rawtx, {"include_unsafe": False}) + fundedtx = wallet.fundrawtransaction(rawtx, include_unsafe=False) tx_dec = wallet.decoderawtransaction(fundedtx['hex']) assert all((txin["txid"], txin["vout"]) in inputs for txin in tx_dec["vin"]) signedtx = wallet.signrawtransactionwithwallet(fundedtx['hex']) @@ -1350,7 +1345,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Create transactions in order to calculate fees for the target bounds that can trigger this bug change_tx = tester.fundrawtransaction(tester.createrawtransaction([], [{funds.getnewaddress(): 1.5}])) tx = tester.createrawtransaction([], [{funds.getnewaddress(): 2}]) - no_change_tx = tester.fundrawtransaction(tx, {"subtractFeeFromOutputs": [0]}) + no_change_tx = tester.fundrawtransaction(tx, subtractFeeFromOutputs=[0]) overhead_fees = feerate * len(tx) / 2 / 1000 cost_of_change = change_tx["fee"] - no_change_tx["fee"] @@ -1402,7 +1397,7 @@ class RawTransactionsTest(BitcoinTestFramework): # To test this does not happen, we subtract 202 sats from the input value. If working correctly, this should # fail with insufficient funds rather than bitcoind asserting. rawtx = w.createrawtransaction(inputs=[], outputs=[{self.nodes[0].getnewaddress(address_type="bech32"): 1 - 0.00000202}]) - assert_raises_rpc_error(-4, "Insufficient funds", w.fundrawtransaction, rawtx, {"fee_rate": 1.85}) + assert_raises_rpc_error(-4, "Insufficient funds", w.fundrawtransaction, rawtx, fee_rate=1.85) def test_input_confs_control(self): self.nodes[0].createwallet("minconf") diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 8f84d8ed60..0fb0d7ea97 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -87,11 +87,11 @@ class WalletHDTest(BitcoinTestFramework): self.stop_node(1) # we need to delete the complete chain directory # otherwise node1 would auto-recover all funds in flag the keypool keys as used - shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "blocks")) - shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "chainstate")) + shutil.rmtree(os.path.join(self.nodes[1].chain_path, "blocks")) + shutil.rmtree(os.path.join(self.nodes[1].chain_path, "chainstate")) shutil.copyfile( os.path.join(self.nodes[1].datadir, "hd.bak"), - os.path.join(self.nodes[1].datadir, self.chain, 'wallets', self.default_wallet_name, self.wallet_data_filename), + os.path.join(self.nodes[1].wallets_path, self.default_wallet_name, self.wallet_data_filename), ) self.start_node(1) @@ -115,11 +115,11 @@ class WalletHDTest(BitcoinTestFramework): # Try a RPC based rescan self.stop_node(1) - shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "blocks")) - shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "chainstate")) + shutil.rmtree(os.path.join(self.nodes[1].chain_path, "blocks")) + shutil.rmtree(os.path.join(self.nodes[1].chain_path, "chainstate")) shutil.copyfile( os.path.join(self.nodes[1].datadir, "hd.bak"), - os.path.join(self.nodes[1].datadir, self.chain, "wallets", self.default_wallet_name, self.wallet_data_filename), + os.path.join(self.nodes[1].wallets_path, self.default_wallet_name, self.wallet_data_filename), ) self.start_node(1, extra_args=self.extra_args[1]) self.connect_nodes(0, 1) diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index 211e939a39..0ac67607e1 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -75,7 +75,7 @@ class Variant(collections.namedtuple("Variant", "call data address_type rescan p request.update({"redeemscript": self.address['embedded']['scriptPubKey']}) response = self.node.importmulti( requests=[request], - options={"rescan": self.rescan in (Rescan.yes, Rescan.late_timestamp)}, + rescan=self.rescan in (Rescan.yes, Rescan.late_timestamp), ) assert_equal(response, [{"success": True}]) diff --git a/test/functional/wallet_importprunedfunds.py b/test/functional/wallet_importprunedfunds.py index 77b407579f..5fe7c4b591 100755 --- a/test/functional/wallet_importprunedfunds.py +++ b/test/functional/wallet_importprunedfunds.py @@ -7,7 +7,6 @@ from decimal import Decimal from test_framework.address import key_to_p2wpkh from test_framework.blocktools import COINBASE_MATURITY -from test_framework.key import ECKey from test_framework.messages import ( CMerkleBlock, from_hex, @@ -17,7 +16,7 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) -from test_framework.wallet_util import bytes_to_wif +from test_framework.wallet_util import generate_keypair class ImportPrunedFundsTest(BitcoinTestFramework): @@ -40,10 +39,8 @@ class ImportPrunedFundsTest(BitcoinTestFramework): # pubkey address2 = self.nodes[0].getnewaddress() # privkey - eckey = ECKey() - eckey.generate() - address3_privkey = bytes_to_wif(eckey.get_bytes()) - address3 = key_to_p2wpkh(eckey.get_pubkey().get_bytes()) + address3_privkey, address3_pubkey = generate_keypair(wif=True) + address3 = key_to_p2wpkh(address3_pubkey) self.nodes[0].importprivkey(address3_privkey) # Check only one address diff --git a/test/functional/wallet_inactive_hdchains.py b/test/functional/wallet_inactive_hdchains.py index c0b3fea1c0..c6d22ab90b 100755 --- a/test/functional/wallet_inactive_hdchains.py +++ b/test/functional/wallet_inactive_hdchains.py @@ -5,7 +5,6 @@ """ Test Inactive HD Chains. """ -import os import shutil import time @@ -130,8 +129,8 @@ class InactiveHDChainsTest(BitcoinTestFramework): # Copy test wallet to node 0 test_wallet.unloadwallet() - test_wallet_dir = os.path.join(self.nodes[1].datadir, "regtest/wallets/keymeta_test") - new_test_wallet_dir = os.path.join(self.nodes[0].datadir, "regtest/wallets/keymeta_test") + test_wallet_dir = self.nodes[1].wallets_path / "keymeta_test" + new_test_wallet_dir = self.nodes[0].wallets_path / "keymeta_test" shutil.copytree(test_wallet_dir, new_test_wallet_dir) self.nodes[0].loadwallet("keymeta_test") test_wallet = self.nodes[0].get_wallet_rpc("keymeta_test") diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index bd97851153..a39db3bfb8 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -178,30 +178,30 @@ class KeyPoolTest(BitcoinTestFramework): # Using a fee rate (10 sat / byte) well above the minimum relay rate # creating a 5,000 sat transaction with change should not be possible - assert_raises_rpc_error(-4, "Transaction needs a change address, but we can't generate it.", w2.walletcreatefundedpsbt, inputs=[], outputs=[{addr.pop(): 0.00005000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010}) + assert_raises_rpc_error(-4, "Transaction needs a change address, but we can't generate it.", w2.walletcreatefundedpsbt, inputs=[], outputs=[{addr.pop(): 0.00005000}], subtractFeeFromOutputs=[0], feeRate=0.00010) # creating a 10,000 sat transaction without change, with a manual input, should still be possible - res = w2.walletcreatefundedpsbt(inputs=w2.listunspent(), outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010}) + res = w2.walletcreatefundedpsbt(inputs=w2.listunspent(), outputs=[{destination: 0.00010000}], subtractFeeFromOutputs=[0], feeRate=0.00010) assert_equal("psbt" in res, True) # creating a 10,000 sat transaction without change should still be possible - res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010}) + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], subtractFeeFromOutputs=[0], feeRate=0.00010) assert_equal("psbt" in res, True) # should work without subtractFeeFromOutputs if the exact fee is subtracted from the amount - res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00008900}], options={"feeRate": 0.00010}) + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00008900}], feeRate=0.00010) assert_equal("psbt" in res, True) # dust change should be removed - res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00008800}], options={"feeRate": 0.00010}) + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00008800}], feeRate=0.00010) assert_equal("psbt" in res, True) # create a transaction without change at the maximum fee rate, such that the output is still spendable: - res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.0008823}) + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], subtractFeeFromOutputs=[0], feeRate=0.0008823) assert_equal("psbt" in res, True) assert_equal(res["fee"], Decimal("0.00009706")) # creating a 10,000 sat transaction with a manual change address should be possible - res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010, "changeAddress": addr.pop()}) + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], subtractFeeFromOutputs=[0], feeRate=0.00010, changeAddress=addr.pop()) assert_equal("psbt" in res, True) if not self.options.descriptors: diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index f1458bb374..0f1c33a0c2 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -33,7 +33,7 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.skip_if_no_wallet() def run_test(self): - wallet_path = os.path.join(self.nodes[1].datadir, self.chain, "wallets", self.default_wallet_name, self.wallet_data_filename) + wallet_path = os.path.join(self.nodes[1].wallets_path, self.default_wallet_name, self.wallet_data_filename) wallet_backup_path = os.path.join(self.nodes[1].datadir, "wallet.bak") self.generate(self.nodes[0], COINBASE_MATURITY + 1) diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index bfca344fd1..a19a3ac2cb 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -7,7 +7,6 @@ from test_framework.address import key_to_p2wpkh from test_framework.blocktools import COINBASE_MATURITY from test_framework.descriptors import descsum_create -from test_framework.key import ECKey from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import MAX_BIP125_RBF_SEQUENCE from test_framework.util import ( @@ -15,7 +14,7 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) -from test_framework.wallet_util import bytes_to_wif +from test_framework.wallet_util import generate_keypair from decimal import Decimal @@ -202,10 +201,8 @@ class ListSinceBlockTest(BitcoinTestFramework): self.sync_all() # share utxo between nodes[1] and nodes[2] - eckey = ECKey() - eckey.generate() - privkey = bytes_to_wif(eckey.get_bytes()) - address = key_to_p2wpkh(eckey.get_pubkey().get_bytes()) + privkey, pubkey = generate_keypair(wif=True) + address = key_to_p2wpkh(pubkey) self.nodes[2].sendtoaddress(address, 10) self.generate(self.nodes[2], 6) self.nodes[2].importprivkey(privkey) diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index a44c129c87..18bb8a0cd8 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -234,8 +234,8 @@ class ListTransactionsTest(BitcoinTestFramework): # refill keypool otherwise the second node wouldn't recognize addresses generated on the first nodes self.nodes[0].keypoolrefill(1000) self.stop_nodes() - wallet0 = os.path.join(self.nodes[0].datadir, self.chain, self.default_wallet_name, "wallet.dat") - wallet2 = os.path.join(self.nodes[2].datadir, self.chain, self.default_wallet_name, "wallet.dat") + wallet0 = os.path.join(self.nodes[0].chain_path, self.default_wallet_name, "wallet.dat") + wallet2 = os.path.join(self.nodes[2].chain_path, self.default_wallet_name, "wallet.dat") shutil.copyfile(wallet0, wallet2) self.start_nodes() # reconnect nodes diff --git a/test/functional/wallet_migration.py b/test/functional/wallet_migration.py index 7a04e11370..b64e5a8cd8 100755 --- a/test/functional/wallet_migration.py +++ b/test/functional/wallet_migration.py @@ -4,8 +4,8 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Migrating a wallet from legacy to descriptor.""" -import os import random +import shutil from test_framework.descriptors import descsum_create from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -34,7 +34,7 @@ class WalletMigrationTest(BitcoinTestFramework): self.skip_if_no_bdb() def assert_is_sqlite(self, wallet_name): - wallet_file_path = os.path.join(self.nodes[0].datadir, "regtest/wallets", wallet_name, self.wallet_data_filename) + wallet_file_path = self.nodes[0].wallets_path / wallet_name / self.wallet_data_filename with open(wallet_file_path, 'rb') as f: file_magic = f.read(16) assert_equal(file_magic, b'SQLite format 3\x00') @@ -281,7 +281,7 @@ class WalletMigrationTest(BitcoinTestFramework): imports0.importaddress(import_sent_addr) received_sent_watchonly_txid = default.sendtoaddress(import_sent_addr, 10) received_sent_watchonly_vout = find_vout_for_address(self.nodes[0], received_sent_watchonly_txid, import_sent_addr) - send = default.sendall(recipients=[default.getnewaddress()], options={"inputs": [{"txid": received_sent_watchonly_txid, "vout": received_sent_watchonly_vout}]}) + send = default.sendall(recipients=[default.getnewaddress()], inputs=[{"txid": received_sent_watchonly_txid, "vout": received_sent_watchonly_vout}]) sent_watchonly_txid = send["txid"] self.generate(self.nodes[0], 1) @@ -457,11 +457,11 @@ class WalletMigrationTest(BitcoinTestFramework): wallet.unloadwallet() - wallet_file_path = os.path.join(self.nodes[0].datadir, "regtest", "wallets", "notloaded2") + wallet_file_path = self.nodes[0].wallets_path / "notloaded2" self.nodes[0].migratewallet(wallet_file_path) # Because we gave the name by full path, the loaded wallet's name is that path too. - wallet = self.nodes[0].get_wallet_rpc(wallet_file_path) + wallet = self.nodes[0].get_wallet_rpc(str(wallet_file_path)) info = wallet.getwalletinfo() assert_equal(info["descriptors"], True) @@ -470,6 +470,40 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(bals, wallet.getbalances()) + def test_default_wallet(self): + self.log.info("Test migration of the wallet named as the empty string") + wallet = self.create_legacy_wallet("") + + wallet.migratewallet() + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], True) + assert_equal(info["format"], "sqlite") + + def test_direct_file(self): + self.log.info("Test migration of a wallet that is not in a wallet directory") + wallet = self.create_legacy_wallet("plainfile") + wallet.unloadwallet() + + wallets_dir = self.nodes[0].wallets_path + wallet_path = wallets_dir / "plainfile" + wallet_dat_path = wallet_path / "wallet.dat" + shutil.copyfile(wallet_dat_path, wallets_dir / "plainfile.bak") + shutil.rmtree(wallet_path) + shutil.move(wallets_dir / "plainfile.bak", wallet_path) + + self.nodes[0].loadwallet("plainfile") + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], False) + assert_equal(info["format"], "bdb") + + wallet.migratewallet() + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], True) + assert_equal(info["format"], "sqlite") + + assert wallet_path.is_dir() + assert wallet_dat_path.is_file() + def run_test(self): self.generate(self.nodes[0], 101) @@ -482,6 +516,8 @@ class WalletMigrationTest(BitcoinTestFramework): self.test_encrypted() self.test_unloaded() self.test_unloaded_by_path() + self.test_default_wallet() + self.test_direct_file() if __name__ == '__main__': WalletMigrationTest().main() diff --git a/test/functional/wallet_multisig_descriptor_psbt.py b/test/functional/wallet_multisig_descriptor_psbt.py index 12069fb00d..28bee1911e 100755 --- a/test/functional/wallet_multisig_descriptor_psbt.py +++ b/test/functional/wallet_multisig_descriptor_psbt.py @@ -121,7 +121,7 @@ class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): to = participants["signers"][self.N - 1].getnewaddress() value = 1 self.log.info("First, make a sending transaction, created using `walletcreatefundedpsbt` (anyone can initiate this)...") - psbt = participants["multisigs"][0].walletcreatefundedpsbt(inputs=[], outputs={to: value}, options={"feeRate": 0.00010}) + psbt = participants["multisigs"][0].walletcreatefundedpsbt(inputs=[], outputs={to: value}, feeRate=0.00010) psbts = [] self.log.info("Now at least M users check the psbt with decodepsbt and (if OK) signs it with walletprocesspsbt...") @@ -143,7 +143,7 @@ class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): assert_equal(participants["signers"][self.N - 1].getbalance(), value) self.log.info("Send another transaction from the multisig, this time with a daisy chained signing flow (one after another in series)!") - psbt = participants["multisigs"][0].walletcreatefundedpsbt(inputs=[], outputs={to: value}, options={"feeRate": 0.00010}) + psbt = participants["multisigs"][0].walletcreatefundedpsbt(inputs=[], outputs={to: value}, feeRate=0.00010) for m in range(self.M): signers_multisig = participants["multisigs"][m] self._check_psbt(psbt["psbt"], to, value, signers_multisig) diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 2faf6cad8b..10bc516d8f 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -62,7 +62,7 @@ class MultiWalletTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] - data_dir = lambda *p: os.path.join(node.datadir, self.chain, *p) + data_dir = lambda *p: os.path.join(node.chain_path, *p) wallet_dir = lambda *p: data_dir('wallets', *p) wallet = lambda name: node.get_wallet_rpc(name) @@ -299,7 +299,7 @@ class MultiWalletTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].listwallets()), set(wallet_names)) # Fail to load if wallet doesn't exist - path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "wallets") + path = wallet_dir("wallets") assert_raises_rpc_error(-18, "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path), self.nodes[0].loadwallet, 'wallets') # Fail to load duplicate wallets @@ -307,7 +307,7 @@ class MultiWalletTest(BitcoinTestFramework): if not self.options.descriptors: # This tests the default wallet that BDB makes, so SQLite wallet doesn't need to test this # Fail to load duplicate wallets by different ways (directory and filepath) - path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "wallet.dat") + path = wallet_dir("wallet.dat") assert_raises_rpc_error(-35, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, 'wallet.dat') # Only BDB doesn't open duplicate wallet files. SQLite does not have this limitation. While this may be desired in the future, it is not necessary @@ -322,13 +322,13 @@ class MultiWalletTest(BitcoinTestFramework): # Fail to load if a directory is specified that doesn't contain a wallet os.mkdir(wallet_dir('empty_wallet_dir')) - path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "empty_wallet_dir") + path = wallet_dir("empty_wallet_dir") assert_raises_rpc_error(-18, "Wallet file verification failed. Failed to load database path '{}'. Data is not in recognized format.".format(path), self.nodes[0].loadwallet, 'empty_wallet_dir') self.log.info("Test dynamic wallet creation.") # Fail to create a wallet if it already exists. - path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "w2") + path = wallet_dir("w2") assert_raises_rpc_error(-4, "Failed to create database path '{}'. Database already exists.".format(path), self.nodes[0].createwallet, 'w2') # Successfully create a wallet with a new name diff --git a/test/functional/wallet_pruning.py b/test/functional/wallet_pruning.py index 1ceceaee93..9e6061287c 100755 --- a/test/functional/wallet_pruning.py +++ b/test/functional/wallet_pruning.py @@ -106,7 +106,7 @@ class WalletPruningTest(BitcoinTestFramework): def has_block(self, block_index): """Checks if the pruned node has the specific blk0000*.dat file""" - return os.path.isfile(os.path.join(self.nodes[1].datadir, self.chain, "blocks", f"blk{block_index:05}.dat")) + return os.path.isfile(os.path.join(self.nodes[1].chain_path, "blocks", f"blk{block_index:05}.dat")) def create_wallet(self, wallet_name, *, unload=False): """Creates and dumps a wallet on the non-pruned node0 to be later import by the pruned node""" diff --git a/test/functional/wallet_reorgsrestore.py b/test/functional/wallet_reorgsrestore.py index 1c79c6816c..af01b9439f 100755 --- a/test/functional/wallet_reorgsrestore.py +++ b/test/functional/wallet_reorgsrestore.py @@ -89,7 +89,7 @@ class ReorgsRestoreTest(BitcoinTestFramework): # Node0 wallet file is loaded on longest sync'ed node1 self.stop_node(1) self.nodes[0].backupwallet(os.path.join(self.nodes[0].datadir, 'wallet.bak')) - shutil.copyfile(os.path.join(self.nodes[0].datadir, 'wallet.bak'), os.path.join(self.nodes[1].datadir, self.chain, self.default_wallet_name, self.wallet_data_filename)) + shutil.copyfile(os.path.join(self.nodes[0].datadir, 'wallet.bak'), os.path.join(self.nodes[1].chain_path, self.default_wallet_name, self.wallet_data_filename)) self.start_node(1) tx_after_reorg = self.nodes[1].gettransaction(txid) # Check that normal confirmed tx is confirmed again but with different blockhash diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 7e4a4002b2..7bdb6f5e3a 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -35,7 +35,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): self.log.info("Create a new transaction and wait until it's broadcast") parent_utxo, indep_utxo = node.listunspent()[:2] addr = node.getnewaddress() - txid = node.send(outputs=[{addr: 1}], options={"inputs": [parent_utxo]})["txid"] + txid = node.send(outputs=[{addr: 1}], inputs=[parent_utxo])["txid"] # Can take a few seconds due to transaction trickling peer_first.wait_for_broadcast([txid]) @@ -86,7 +86,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): # ordering of mapWallet is, if the child is not before the parent, we will create a new # child (via bumpfee) and remove the old child (via removeprunedfunds) until we get the # ordering of child before parent. - child_txid = node.send(outputs=[{addr: 0.5}], options={"inputs": [{"txid":txid, "vout":0}]})["txid"] + child_txid = node.send(outputs=[{addr: 0.5}], inputs=[{"txid":txid, "vout":0}])["txid"] while True: txids = node.listreceivedbyaddress(minconf=0, address_filter=addr)[0]["txids"] if txids == [child_txid, txid]: @@ -111,7 +111,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): # Evict these txs from the mempool evict_time = block_time + 60 * 60 * DEFAULT_MEMPOOL_EXPIRY_HOURS + 5 node.setmocktime(evict_time) - indep_send = node.send(outputs=[{node.getnewaddress(): 1}], options={"inputs": [indep_utxo]}) + indep_send = node.send(outputs=[{node.getnewaddress(): 1}], inputs=[indep_utxo]) node.getmempoolentry(indep_send["txid"]) assert_raises_rpc_error(-5, "Transaction not in mempool", node.getmempoolentry, txid) assert_raises_rpc_error(-5, "Transaction not in mempool", node.getmempoolentry, child_txid) diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index ac3ec06eec..d7bb6ab1e7 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -9,7 +9,6 @@ from itertools import product from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create -from test_framework.key import ECKey from test_framework.messages import ( ser_compact_size, WITNESS_SCALE_FACTOR, @@ -22,7 +21,8 @@ from test_framework.util import ( assert_raises_rpc_error, count_bytes, ) -from test_framework.wallet_util import bytes_to_wif +from test_framework.wallet_util import generate_keypair + class WalletSendTest(BitcoinTestFramework): def add_options(self, parser): @@ -500,9 +500,7 @@ class WalletSendTest(BitcoinTestFramework): assert res["complete"] self.log.info("External outputs") - eckey = ECKey() - eckey.generate() - privkey = bytes_to_wif(eckey.get_bytes()) + privkey, _ = generate_keypair(wif=True) self.nodes[1].createwallet("extsend") ext_wallet = self.nodes[1].get_wallet_rpc("extsend") diff --git a/test/functional/wallet_sendall.py b/test/functional/wallet_sendall.py index f6440f07d7..c2b800df21 100755 --- a/test/functional/wallet_sendall.py +++ b/test/functional/wallet_sendall.py @@ -151,7 +151,7 @@ class SendallTest(BitcoinTestFramework): self.log.info("Test sending more than balance") pre_sendall_balance = self.add_utxos([7, 14]) - expected_tx = self.wallet.sendall(recipients=[{self.recipient: 5}, self.remainder_target], options={"add_to_wallet": False}) + expected_tx = self.wallet.sendall(recipients=[{self.recipient: 5}, self.remainder_target], add_to_wallet=False) tx = self.wallet.decoderawtransaction(expected_tx['hex']) fee = 21 - sum([o["value"] for o in tx["vout"]]) @@ -190,7 +190,7 @@ class SendallTest(BitcoinTestFramework): self.add_utxos([0.00000400, 0.00000300, 1]) # sendall with send_max - sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], fee_rate=300, options={"send_max": True}) + sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], fee_rate=300, send_max=True) tx_from_wallet = self.wallet.gettransaction(txid = sendall_tx_receipt["txid"], verbose = True) assert_equal(len(tx_from_wallet["decoded"]["vin"]), 1) @@ -206,7 +206,7 @@ class SendallTest(BitcoinTestFramework): self.add_utxos([17, 4]) utxo = self.wallet.listunspent()[0] - sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], options={"inputs": [utxo]}) + sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], inputs=[utxo]) tx_from_wallet = self.wallet.gettransaction(txid = sendall_tx_receipt["txid"], verbose = True) assert_equal(len(tx_from_wallet["decoded"]["vin"]), 1) assert_equal(len(tx_from_wallet["decoded"]["vout"]), 1) @@ -227,26 +227,26 @@ class SendallTest(BitcoinTestFramework): # fails on out of bounds vout assert_raises_rpc_error(-8, "Input not found. UTXO ({}:{}) is not part of wallet.".format(spent_utxo["txid"], 1000), - self.wallet.sendall, recipients=[self.remainder_target], options={"inputs": [{"txid": spent_utxo["txid"], "vout": 1000}]}) + self.wallet.sendall, recipients=[self.remainder_target], inputs=[{"txid": spent_utxo["txid"], "vout": 1000}]) # fails on unconfirmed spent UTXO self.wallet.sendall(recipients=[self.remainder_target]) assert_raises_rpc_error(-8, "Input not available. UTXO ({}:{}) was already spent.".format(spent_utxo["txid"], spent_utxo["vout"]), - self.wallet.sendall, recipients=[self.remainder_target], options={"inputs": [spent_utxo]}) + self.wallet.sendall, recipients=[self.remainder_target], inputs=[spent_utxo]) # fails on specific previously spent UTXO, while other UTXOs exist self.generate(self.nodes[0], 1) self.add_utxos([19, 2]) assert_raises_rpc_error(-8, "Input not available. UTXO ({}:{}) was already spent.".format(spent_utxo["txid"], spent_utxo["vout"]), - self.wallet.sendall, recipients=[self.remainder_target], options={"inputs": [spent_utxo]}) + self.wallet.sendall, recipients=[self.remainder_target], inputs=[spent_utxo]) # fails because UTXO is unknown, while other UTXOs exist foreign_utxo = self.def_wallet.listunspent()[0] assert_raises_rpc_error(-8, "Input not found. UTXO ({}:{}) is not part of wallet.".format(foreign_utxo["txid"], foreign_utxo["vout"]), self.wallet.sendall, recipients=[self.remainder_target], - options={"inputs": [foreign_utxo]}) + inputs=[foreign_utxo]) @cleanup def sendall_fails_on_no_address(self): @@ -270,7 +270,7 @@ class SendallTest(BitcoinTestFramework): "Cannot combine send_max with specific inputs.", self.wallet.sendall, recipients=[self.remainder_target], - options={"inputs": [utxo], "send_max": True}) + inputs=[utxo], send_max=True) @cleanup def sendall_fails_on_high_fee(self): @@ -308,7 +308,7 @@ class SendallTest(BitcoinTestFramework): else: watchonly.importmulti(import_req) - sendall_tx_receipt = watchonly.sendall(recipients=[self.remainder_target], options={"inputs": [utxo]}) + sendall_tx_receipt = watchonly.sendall(recipients=[self.remainder_target], inputs=[utxo]) psbt = sendall_tx_receipt["psbt"] decoded = self.nodes[0].decodepsbt(psbt) assert_equal(len(decoded["inputs"]), 1) diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 6aaec65b86..3e7c613e55 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -45,9 +45,6 @@ class WalletSignerTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - # The experimental syscall sandbox feature (-sandbox) is not compatible with -signer (which - # invokes execve). - self.disable_syscall_sandbox = True self.extra_args = [ [], @@ -211,13 +208,13 @@ class WalletSignerTest(BitcoinTestFramework): self.log.info('Test send using hww1') # Don't broadcast transaction yet so the RPC returns the raw hex - res = hww.send(outputs={dest:0.5},options={"add_to_wallet": False}) + res = hww.send(outputs={dest:0.5},add_to_wallet=False) assert res["complete"] assert_equal(res["hex"], mock_tx) self.log.info('Test sendall using hww1') - res = hww.sendall(recipients=[{dest:0.5}, hww.getrawchangeaddress()],options={"add_to_wallet": False}) + res = hww.sendall(recipients=[{dest:0.5}, hww.getrawchangeaddress()], add_to_wallet=False) assert res["complete"] assert_equal(res["hex"], mock_tx) # Broadcast transaction so we can bump the fee diff --git a/test/functional/wallet_taproot.py b/test/functional/wallet_taproot.py index b52892704f..a5d7445ce8 100755 --- a/test/functional/wallet_taproot.py +++ b/test/functional/wallet_taproot.py @@ -378,7 +378,7 @@ class WalletTaprootTest(BitcoinTestFramework): assert psbt_online.gettransaction(txid)['confirmations'] > 0 # Cleanup - psbt = psbt_online.sendall(recipients=[self.boring.getnewaddress()], options={"psbt": True})["psbt"] + psbt = psbt_online.sendall(recipients=[self.boring.getnewaddress()], psbt=True)["psbt"] res = psbt_offline.walletprocesspsbt(psbt=psbt, finalize=False) rawtx = self.nodes[0].finalizepsbt(res['psbt'])['hex'] txid = self.nodes[0].sendrawtransaction(rawtx) diff --git a/test/functional/wallet_timelock.py b/test/functional/wallet_timelock.py index 57a7c3907a..0a622979a4 100755 --- a/test/functional/wallet_timelock.py +++ b/test/functional/wallet_timelock.py @@ -29,7 +29,7 @@ class WalletLocktimeTest(BitcoinTestFramework): self.log.info("Send to new address with locktime") node.send( outputs={address: 5}, - options={"locktime": mtp_tip - 1}, + locktime=mtp_tip - 1, ) self.generate(node, 1) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index 4495a7d778..a4f2a9b74d 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -138,11 +138,11 @@ class UpgradeWalletTest(BitcoinTestFramework): self.log.info("Test upgradewallet RPC...") # Prepare for copying of the older wallet - node_master_wallet_dir = os.path.join(node_master.datadir, "regtest/wallets", self.default_wallet_name) - node_master_wallet = os.path.join(node_master_wallet_dir, self.default_wallet_name, self.wallet_data_filename) - v16_3_wallet = os.path.join(v16_3_node.datadir, "regtest/wallets/wallet.dat") - v15_2_wallet = os.path.join(v15_2_node.datadir, "regtest/wallet.dat") - split_hd_wallet = os.path.join(v15_2_node.datadir, "regtest/splithd") + node_master_wallet_dir = node_master.wallets_path / self.default_wallet_name + node_master_wallet = node_master_wallet_dir / self.default_wallet_name / self.wallet_data_filename + v16_3_wallet = v16_3_node.wallets_path / "wallet.dat" + v15_2_wallet = v15_2_node.chain_path / "wallet.dat" + split_hd_wallet = v15_2_node.chain_path / "splithd" self.stop_nodes() # Make split hd wallet diff --git a/test/functional/wallet_watchonly.py b/test/functional/wallet_watchonly.py index dd4514318c..b473f5d65c 100755 --- a/test/functional/wallet_watchonly.py +++ b/test/functional/wallet_watchonly.py @@ -98,13 +98,13 @@ class CreateWalletWatchonlyTest(BitcoinTestFramework): options = {'changeAddress': wo_change} no_wo_options = {'changeAddress': wo_change, 'includeWatching': False} - result = wo_wallet.walletcreatefundedpsbt(inputs=inputs, outputs=outputs, options=options) + result = wo_wallet.walletcreatefundedpsbt(inputs=inputs, outputs=outputs, **options) assert_equal("psbt" in result, True) assert_raises_rpc_error(-4, "Insufficient funds", wo_wallet.walletcreatefundedpsbt, inputs, outputs, 0, no_wo_options) self.log.info('Testing fundrawtransaction watch-only defaults') rawtx = wo_wallet.createrawtransaction(inputs=inputs, outputs=outputs) - result = wo_wallet.fundrawtransaction(hexstring=rawtx, options=options) + result = wo_wallet.fundrawtransaction(hexstring=rawtx, **options) assert_equal("hex" in result, True) assert_raises_rpc_error(-4, "Insufficient funds", wo_wallet.fundrawtransaction, rawtx, no_wo_options) diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index af21e7b956..84028f3dc5 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -6,6 +6,7 @@ """ from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path import argparse import configparser import logging @@ -42,6 +43,11 @@ def main(): help='If true, run fuzzing binaries under the valgrind memory error detector', ) parser.add_argument( + "--empty_min_time", + type=int, + help="If set, run at least this long, if the existing fuzz inputs directory is empty.", + ) + parser.add_argument( '-x', '--exclude', help="A comma-separated list of targets to exclude", @@ -76,6 +82,7 @@ def main(): ) args = parser.parse_args() + args.corpus_dir = Path(args.corpus_dir) # Set up logging logging.basicConfig( @@ -88,8 +95,8 @@ def main(): configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" config.read_file(open(configfile, encoding="utf8")) - if not config["components"].getboolean("ENABLE_FUZZ"): - logging.error("Must have fuzz targets built") + if not config["components"].getboolean("ENABLE_FUZZ_BINARY"): + logging.error("Must have fuzz executable built") sys.exit(1) # Build list of tests @@ -141,11 +148,12 @@ def main(): ], env=get_fuzz_env(target=test_list_selection[0], source_dir=config['environment']['SRCDIR']), timeout=20, - check=True, + check=False, stderr=subprocess.PIPE, text=True, ).stderr - if "libFuzzer" not in help_output: + using_libfuzzer = "libFuzzer" in help_output + if (args.generate or args.m_dir) and not using_libfuzzer: logging.error("Must be built with libFuzzer") sys.exit(1) except subprocess.TimeoutExpired: @@ -179,7 +187,9 @@ def main(): test_list=test_list_selection, src_dir=config['environment']['SRCDIR'], build_dir=config["environment"]["BUILDDIR"], + using_libfuzzer=using_libfuzzer, use_valgrind=args.valgrind, + empty_min_time=args.empty_min_time, ) @@ -251,16 +261,25 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, build_dir, merge_dir) future.result() -def run_once(*, fuzz_pool, corpus, test_list, src_dir, build_dir, use_valgrind): +def run_once(*, fuzz_pool, corpus, test_list, src_dir, build_dir, using_libfuzzer, use_valgrind, empty_min_time): jobs = [] for t in test_list: - corpus_path = os.path.join(corpus, t) + corpus_path = corpus / t os.makedirs(corpus_path, exist_ok=True) args = [ os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), - '-runs=1', - corpus_path, ] + empty_dir = not any(corpus_path.iterdir()) + if using_libfuzzer: + if empty_min_time and empty_dir: + args += [f"-max_total_time={empty_min_time}"] + else: + args += [ + "-runs=1", + corpus_path, + ] + else: + args += [corpus_path] if use_valgrind: args = ['valgrind', '--quiet', '--error-exitcode=1'] + args @@ -287,7 +306,7 @@ def run_once(*, fuzz_pool, corpus, test_list, src_dir, build_dir, use_valgrind): logging.info(e.stdout) if e.stderr: logging.info(e.stderr) - logging.info("Target \"{}\" failed with exit code {}".format(" ".join(result.args), e.returncode)) + logging.info(f"Target {result.args} failed with exit code {e.returncode}") sys.exit(1) diff --git a/test/lint/lint-assertions.py b/test/lint/lint-assertions.py index e7eecebce5..6da59b0d48 100755 --- a/test/lint/lint-assertions.py +++ b/test/lint/lint-assertions.py @@ -45,6 +45,16 @@ def main(): ":(exclude)src/rpc/server.cpp", ], "CHECK_NONFATAL(condition) or NONFATAL_UNREACHABLE should be used instead of assert for RPC code.") + # The `BOOST_ASSERT` macro requires to `#include boost/assert.hpp`, + # which is an unnecessary Boost dependency. + exit_code |= git_grep([ + "-E", + r"BOOST_ASSERT *\(.*\);", + "--", + "*.cpp", + "*.h", + ], "BOOST_ASSERT must be replaced with Assert, BOOST_REQUIRE, or BOOST_CHECK.") + sys.exit(exit_code) diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py index 459030bb0b..b14caa4855 100755 --- a/test/lint/lint-includes.py +++ b/test/lint/lint-includes.py @@ -23,15 +23,20 @@ EXCLUDED_DIRS = ["src/leveldb/", EXPECTED_BOOST_INCLUDES = ["boost/date_time/posix_time/posix_time.hpp", "boost/multi_index/hashed_index.hpp", + "boost/multi_index/identity.hpp", + "boost/multi_index/indexed_by.hpp", "boost/multi_index/ordered_index.hpp", "boost/multi_index/sequenced_index.hpp", + "boost/multi_index/tag.hpp", "boost/multi_index_container.hpp", "boost/process.hpp", "boost/signals2/connection.hpp", "boost/signals2/optional_last_value.hpp", "boost/signals2/signal.hpp", "boost/test/included/unit_test.hpp", - "boost/test/unit_test.hpp"] + "boost/test/unit_test.hpp", + "boost/tuple/tuple.hpp", + ] def get_toplevel(): diff --git a/test/lint/lint-python.py b/test/lint/lint-python.py index 9de13e44e2..539d0acb5d 100755 --- a/test/lint/lint-python.py +++ b/test/lint/lint-python.py @@ -13,7 +13,7 @@ import pkg_resources import subprocess import sys -DEPS = ['flake8', 'mypy', 'pyzmq'] +DEPS = ['flake8', 'lief', 'mypy', 'pyzmq'] MYPY_CACHE_DIR = f"{os.getenv('BASE_ROOT_DIR', '')}/test/.mypy_cache" # All .py files, except those in src/ (to exclude subtrees there) diff --git a/test/lint/run-lint-format-strings.py b/test/lint/run-lint-format-strings.py index 91915f05f9..d1896dba84 100755 --- a/test/lint/run-lint-format-strings.py +++ b/test/lint/run-lint-format-strings.py @@ -241,20 +241,32 @@ def count_format_specifiers(format_string): 3 >>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo") 4 + >>> count_format_specifiers("foo %5$d") + 5 + >>> count_format_specifiers("foo %5$*7$d") + 7 """ assert type(format_string) is str format_string = format_string.replace('%%', 'X') - n = 0 - in_specifier = False - for i, char in enumerate(format_string): - if char == "%": - in_specifier = True + n = max_pos = 0 + for m in re.finditer("%(.*?)[aAcdeEfFgGinopsuxX]", format_string, re.DOTALL): + # Increase the max position if the argument has a position number like + # "5$", otherwise increment the argument count. + pos_num, = re.match(r"(?:(^\d+)\$)?", m.group(1)).groups() + if pos_num is not None: + max_pos = max(max_pos, int(pos_num)) + else: n += 1 - elif char in "aAcdeEfFgGinopsuxX": - in_specifier = False - elif in_specifier and char == "*": + + # Increase the max position if there is a "*" width argument with a + # position like "*7$", and increment the argument count if there is a + # "*" width argument with no position. + star, star_pos_num = re.match(r"(?:.*?(\*(?:(\d+)\$)?)|)", m.group(1)).groups() + if star_pos_num is not None: + max_pos = max(max_pos, int(star_pos_num)) + elif star is not None: n += 1 - return n + return max(n, max_pos) def main(): diff --git a/test/lint/spelling.ignore-words.txt b/test/lint/spelling.ignore-words.txt index fa47a31725..ecc2a553d2 100644 --- a/test/lint/spelling.ignore-words.txt +++ b/test/lint/spelling.ignore-words.txt @@ -1,3 +1,4 @@ +afile asend ba blockin @@ -15,6 +16,7 @@ lief mor nd nin +requestor ser siz stap diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 6b891c462e..74703b04ec 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -23,6 +23,7 @@ implicit-integer-sign-change:crc32c/ # implicit-integer-sign-change in FuzzedDataProvider's ConsumeIntegralInRange implicit-integer-sign-change:FuzzedDataProvider.h implicit-integer-sign-change:minisketch/ +implicit-signed-integer-truncation:*/include/c++/ implicit-signed-integer-truncation:leveldb/ implicit-unsigned-integer-truncation:*/include/c++/ implicit-unsigned-integer-truncation:leveldb/ @@ -46,7 +47,6 @@ unsigned-integer-overflow:hash.cpp unsigned-integer-overflow:policy/fees.cpp unsigned-integer-overflow:prevector.h unsigned-integer-overflow:script/interpreter.cpp -unsigned-integer-overflow:txmempool.cpp unsigned-integer-overflow:xoroshiro128plusplus.h implicit-integer-sign-change:compat/stdin.cpp implicit-integer-sign-change:compressor.h @@ -56,7 +56,6 @@ implicit-integer-sign-change:prevector.h implicit-integer-sign-change:script/bitcoinconsensus.cpp implicit-integer-sign-change:script/interpreter.cpp implicit-integer-sign-change:serialize.h -implicit-integer-sign-change:txmempool.cpp implicit-signed-integer-truncation:crypto/ implicit-unsigned-integer-truncation:crypto/ shift-base:arith_uint256.cpp |