diff options
498 files changed, 5683 insertions, 3658 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 4f0ad2d04a..64fcd82c2b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,4 +1,5 @@ env: # Global defaults + CIRRUS_CLONE_DEPTH: 1 PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" MAKEJOBS: "-j10" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache @@ -27,7 +28,7 @@ base_template: &BASE_TEMPLATE # Unconditionally install git (used in fingerprint_script). - bash -c "$PACKAGE_MANAGER_INSTALL git" - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi - - git fetch $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge" + - git fetch --depth=1 $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge" - git checkout FETCH_HEAD # Use merged changes to detect silent merge conflicts # Also, the merge commit is used to lint COMMIT_RANGE="HEAD~..HEAD" @@ -65,14 +66,19 @@ compute_credits_template: &CREDITS_TEMPLATE use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != "" task: - name: 'lint [bionic]' + name: 'lint [jammy]' << : *BASE_TEMPLATE container: - image: ubuntu:bionic # For python 3.6, oldest supported version according to doc/dependencies.md + image: ubuntu:jammy cpu: 1 memory: 1G # For faster CI feedback, immediately schedule the linters << : *CREDITS_TEMPLATE + python_cache: + folder: "/tmp/python" + fingerprint_script: cat .python-version /etc/os-release + unshallow_script: + - git fetch --unshallow --no-tags lint_script: - ./ci/lint_run_all.sh env: @@ -102,7 +108,7 @@ task: env: PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Current\Bin;%PATH%' PYTHONUTF8: 1 - CI_VCPKG_TAG: '2022.09.27' + CI_VCPKG_TAG: '2023.01.09' VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache' @@ -153,7 +159,7 @@ task: ccache_cache: folder: '%CCACHE_DIR%' install_tools_script: - - choco install --yes --no-progress ccache --version=4.6.1 + - choco install --yes --no-progress ccache --version=4.7.4 - choco install --yes --no-progress python3 --version=3.9.6 - pip install zmq - ccache --version @@ -185,7 +191,7 @@ task: - netsh int ipv4 set dynamicport tcp start=1025 num=64511 - netsh int ipv6 set dynamicport tcp start=1025 num=64511 # Exclude feature_dbcrash for now due to timeout - - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash + - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=99999999 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash task: name: 'ARM [unit tests, no functional tests] [bullseye]' @@ -292,10 +298,10 @@ task: FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" task: - name: '[no wallet, libbitcoinkernel] [bionic]' + name: '[no wallet, libbitcoinkernel] [buster]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:bionic + image: debian:buster env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh" diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.md b/.github/ISSUE_TEMPLATE/good_first_issue.md index d32e22d360..ff943032ba 100644 --- a/.github/ISSUE_TEMPLATE/good_first_issue.md +++ b/.github/ISSUE_TEMPLATE/good_first_issue.md @@ -15,7 +15,7 @@ assignees: '' #### Useful skills: -<!-- (For example, “C++11 std::thread”, “Qt5 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”.) --> +<!-- (For example, “std::thread”, “Qt5 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”.) --> #### Want to work on this issue? diff --git a/.python-version b/.python-version index cd337510be..36f601f10e 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.6.15 +3.7.16 diff --git a/build-aux/m4/ax_boost_base.m4 b/build-aux/m4/ax_boost_base.m4 index 6c944b160f..f6620882a2 100644 --- a/build-aux/m4/ax_boost_base.m4 +++ b/build-aux/m4/ax_boost_base.m4 @@ -8,7 +8,7 @@ # # DESCRIPTION # -# Test for the Boost C++ libraries of a particular version (or newer) +# Test for the Boost C++ headers of a particular version (or newer) # # If no path to the installed boost library is given the macro searchs # under /usr, /usr/local, /opt, /opt/local and /opt/homebrew and evaluates @@ -17,12 +17,14 @@ # # This macro calls: # -# AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS) +# AC_SUBST(BOOST_CPPFLAGS) # # And sets: # # HAVE_BOOST # +# Note that this macro has been modified compared to upstream. +# # LICENSE # # Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de> @@ -59,26 +61,10 @@ AC_ARG_WITH([boost], ], [want_boost="yes"]) - -AC_ARG_WITH([boost-libdir], - [AS_HELP_STRING([--with-boost-libdir=LIB_DIR], - [Force given directory for boost libraries. - Note that this will override library path detection, - so use this parameter only if default library detection fails - and you know exactly where your boost libraries are located.])], - [ - AS_IF([test -d "$withval"], - [_AX_BOOST_BASE_boost_lib_path="$withval"], - [AC_MSG_ERROR([--with-boost-libdir expected directory name])]) - ], - [_AX_BOOST_BASE_boost_lib_path=""]) - -BOOST_LDFLAGS="" BOOST_CPPFLAGS="" AS_IF([test "x$want_boost" = "xyes"], [_AX_BOOST_BASE_RUNDETECT([$1],[$2],[$3])]) AC_SUBST(BOOST_CPPFLAGS) -AC_SUBST(BOOST_LDFLAGS) ]) @@ -139,7 +125,6 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ AC_MSG_CHECKING([for boostlib >= $1 ($WANT_BOOST_VERSION) lib path in "$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp"]) AS_IF([test -d "$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp" && test -r "$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp" ],[ AC_MSG_RESULT([yes]) - BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp"; break; ], [AC_MSG_RESULT([no])]) @@ -156,27 +141,17 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ for libsubdir in $search_libsubdirs ; do if ls "$_AX_BOOST_BASE_boost_path_tmp/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi done - BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_path_tmp/$libsubdir" BOOST_CPPFLAGS="-I$_AX_BOOST_BASE_boost_path_tmp/include" break; fi done ]) - dnl overwrite ld flags if we have required special directory with - dnl --with-boost-libdir parameter - AS_IF([test "x$_AX_BOOST_BASE_boost_lib_path" != "x"], - [BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_lib_path"]) - - AC_MSG_CHECKING([for boostlib >= $1 ($WANT_BOOST_VERSION)]) + AC_MSG_CHECKING([for Boost headers >= $1 ($WANT_BOOST_VERSION)]) CPPFLAGS_SAVED="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" export CPPFLAGS - LDFLAGS_SAVED="$LDFLAGS" - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS - AC_REQUIRE([AC_PROG_CXX]) AC_LANG_PUSH(C++) AC_COMPILE_IFELSE([_AX_BOOST_BASE_PROGRAM($WANT_BOOST_VERSION)],[ @@ -193,11 +168,8 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ dnl built and installed without the --layout=system option or for a staged(not installed) version if test "x$succeeded" != "xyes" ; then CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" BOOST_CPPFLAGS= - if test -z "$_AX_BOOST_BASE_boost_lib_path" ; then - BOOST_LDFLAGS= - fi + _version=0 if test -n "$_AX_BOOST_BASE_boost_path" ; then if test -d "$_AX_BOOST_BASE_boost_path" && test -r "$_AX_BOOST_BASE_boost_path"; then @@ -216,14 +188,6 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ BOOST_CPPFLAGS="-I$_AX_BOOST_BASE_boost_path" fi fi - dnl if we found something and BOOST_LDFLAGS was unset before - dnl (because "$_AX_BOOST_BASE_boost_lib_path" = ""), set it here. - if test -n "$BOOST_CPPFLAGS" && test -z "$BOOST_LDFLAGS"; then - for libsubdir in $libsubdirs ; do - if ls "$_AX_BOOST_BASE_boost_path/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi - done - BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_path/$libsubdir" - fi fi else if test "x$cross_compiling" != "xyes" ; then @@ -242,12 +206,6 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE" - if test -z "$_AX_BOOST_BASE_boost_lib_path" ; then - for libsubdir in $libsubdirs ; do - if ls "$best_path/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi - done - BOOST_LDFLAGS="-L$best_path/$libsubdir" - fi fi if test -n "$BOOST_ROOT" ; then @@ -259,10 +217,9 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'` stage_version_shorten=`expr $stage_version : '\([[0-9]]*\.[[0-9]]*\)'` V_CHECK=`expr $stage_version_shorten \>\= $_version` - if test "x$V_CHECK" = "x1" && test -z "$_AX_BOOST_BASE_boost_lib_path" ; then + if test "x$V_CHECK" = "x1" ; then AC_MSG_NOTICE(We will use a staged boost library from $BOOST_ROOT) BOOST_CPPFLAGS="-I$BOOST_ROOT" - BOOST_LDFLAGS="-L$BOOST_ROOT/stage/$libsubdir" fi fi fi @@ -270,8 +227,6 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" export CPPFLAGS - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS AC_LANG_PUSH(C++) AC_COMPILE_IFELSE([_AX_BOOST_BASE_PROGRAM($WANT_BOOST_VERSION)],[ @@ -298,6 +253,4 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ fi CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" - ]) diff --git a/build_msvc/libsecp256k1/libsecp256k1.vcxproj b/build_msvc/libsecp256k1/libsecp256k1.vcxproj index 16ee32d87e..0b90f341a7 100644 --- a/build_msvc/libsecp256k1/libsecp256k1.vcxproj +++ b/build_msvc/libsecp256k1/libsecp256k1.vcxproj @@ -14,7 +14,7 @@ </ItemGroup> <ItemDefinitionGroup> <ClCompile> - <PreprocessorDefinitions>ENABLE_MODULE_ECDH;ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;%(PreprocessorDefinitions)</PreprocessorDefinitions> <AdditionalIncludeDirectories>..\..\src\secp256k1;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <DisableSpecificWarnings>4146;4244;4267;4334</DisableSpecificWarnings> </ClCompile> diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index b459b321f1..f7147582dc 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -7,14 +7,31 @@ export LC_ALL=C ${CI_RETRY_EXE} apt-get update -${CI_RETRY_EXE} apt-get install -y python3-pip curl git gawk jq -( - # Temporary workaround for https://github.com/bitcoin/bitcoin/pull/26130#issuecomment-1260499544 - # Can be removed once the underlying image is bumped to something that includes git2.34 or later - sed -i -e 's/bionic/jammy/g' /etc/apt/sources.list - ${CI_RETRY_EXE} apt-get update - ${CI_RETRY_EXE} apt-get install -y --reinstall git -) +# Lint dependencies: +# - curl/xz-utils (to install shellcheck) +# - git (used in many lint scripts) +# - gpg (used by verify-commits) +${CI_RETRY_EXE} apt-get install -y curl xz-utils git gpg + +if [ -z "${SKIP_PYTHON_INSTALL}" ]; then + PYTHON_PATH=/tmp/python + if [ ! -d "${PYTHON_PATH}/bin" ]; then + ( + git clone https://github.com/pyenv/pyenv.git + cd pyenv/plugins/python-build || exit 1 + ./install.sh + ) + # For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment + ${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \ + libbz2-dev libreadline-dev libsqlite3-dev curl llvm \ + libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ + clang + env CC=clang python-build "$(cat "${BASE_ROOT_DIR}/.python-version")" "${PYTHON_PATH}" + fi + export PATH="${PYTHON_PATH}/bin:${PATH}" + command -v python3 + python3 --version +fi ${CI_RETRY_EXE} pip3 install codespell==2.2.1 ${CI_RETRY_EXE} pip3 install flake8==5.0.4 @@ -23,5 +40,6 @@ ${CI_RETRY_EXE} pip3 install pyzmq==24.0.1 ${CI_RETRY_EXE} pip3 install vulture==2.6 SHELLCHECK_VERSION=v0.8.0 -curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ -export PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}" +curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | \ + tar --xz -xf - --directory /tmp/ +mv "/tmp/shellcheck-${SHELLCHECK_VERSION}/shellcheck" /usr/bin/ diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index c14d7473d3..fa28f6126c 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -6,7 +6,11 @@ export LC_ALL=C -if [ -n "$CIRRUS_PR" ]; then +if [ -n "$LOCAL_BRANCH" ]; then + # To faithfully recreate CI linting locally, specify all commits on the current + # branch. + COMMIT_RANGE="$(git merge-base HEAD master)..HEAD" +elif [ -n "$CIRRUS_PR" ]; then COMMIT_RANGE="HEAD~..HEAD" echo git log --no-merges --oneline "$COMMIT_RANGE" diff --git a/ci/lint/Dockerfile b/ci/lint/Dockerfile new file mode 100644 index 0000000000..03c20c7286 --- /dev/null +++ b/ci/lint/Dockerfile @@ -0,0 +1,29 @@ +# See test/lint/README.md for usage. +# +# This container basically has to live in this directory in order to pull in the CI +# install scripts. If it lived in the root directory, it would have to pull in the +# entire repo as docker context during build; if it lived elsewhere, it wouldn't be +# able to make back-references to pull in the install scripts. So here it lives. + +FROM python:3.7-buster + +ENV DEBIAN_FRONTEND=noninteractive +ENV LC_ALL=C.UTF-8 + +# This is used by the 04_install.sh script; we can't read the Python version from +# .python-version for the same reasons as above, and it's more efficient to pull a +# preexisting Python image than it is to build from source. +ENV SKIP_PYTHON_INSTALL=1 + +# Must be built from ./ci/lint/ for these paths to work. +COPY ./docker-entrypoint.sh /entrypoint.sh +COPY ./04_install.sh /install.sh + +RUN /install.sh && \ + echo 'alias lint="./ci/lint/06_script.sh"' >> ~/.bashrc && \ + chmod 755 /entrypoint.sh && \ + rm -rf /var/lib/apt/lists/* + + +WORKDIR /bitcoin +ENTRYPOINT ["/entrypoint.sh"] diff --git a/ci/lint/docker-entrypoint.sh b/ci/lint/docker-entrypoint.sh new file mode 100755 index 0000000000..3fdbbb0761 --- /dev/null +++ b/ci/lint/docker-entrypoint.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +export LC_ALL=C + +# Fixes permission issues when there is a container UID/GID mismatch with the owner +# of the mounted bitcoin src dir. +git config --global --add safe.directory /bitcoin + +if [ -z "$1" ]; then + LOCAL_BRANCH=1 bash -ic "./ci/lint/06_script.sh" +else + exec "$@" +fi diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 366080975e..07c20f632d 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -48,7 +48,7 @@ export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000} export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} -export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:20.04} +export CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG:-ubuntu:20.04} # Randomize test order. # See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/random.html export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} @@ -66,7 +66,7 @@ export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build} export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} -export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} +export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH diff --git a/ci/test/00_setup_env_android.sh b/ci/test/00_setup_env_android.sh index 7a97a57295..e1830b4f49 100755 --- a/ci/test/00_setup_env_android.sh +++ b/ci/test/00_setup_env_android.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export HOST=aarch64-linux-android export PACKAGES="unzip openjdk-8-jdk gradle" export CONTAINER_NAME=ci_android -export DOCKER_NAME_TAG="ubuntu:focal" +export CI_IMAGE_NAME_TAG="ubuntu:focal" export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 932be4b43d..ac0c0be96a 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -18,7 +18,7 @@ if [ -n "$QEMU_USER_CMD" ]; then fi export CONTAINER_NAME=ci_arm_linux # Use debian to avoid 404 apt errors when cross compiling -export DOCKER_NAME_TAG="debian:bullseye" +export CI_IMAGE_NAME_TAG="debian:bullseye" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 9b4169c746..8a931d44e5 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -8,10 +8,11 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_centos -export DOCKER_NAME_TAG=quay.io/centos/centos:stream8 -export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison" +export CI_IMAGE_NAME_TAG=quay.io/centos/centos:stream8 +export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python38 python38-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison" export PIP_PACKAGES="pyzmq" export GOAL="install" +export NO_WERROR=1 # GCC 8 export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports" export CONFIG_SHELL="/bin/dash" export TEST_RUNNER_ENV="LC_ALL=en_US.UTF-8" diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 8246f869fb..9e3ea0d383 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_multiprocess -export DOCKER_NAME_TAG=ubuntu:20.04 +export CI_IMAGE_NAME_TAG=ubuntu:20.04 export PACKAGES="cmake python3 llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index c4f22c8f9e..fe42871c31 100755 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross -export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos +export CI_IMAGE_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos export HOST=x86_64-apple-darwin export PACKAGES="cmake libz-dev libtinfo5 python3-setuptools xorriso" export XCODE_VERSION=12.2 diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 66bbdb8cb5..bb3f6997f3 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -20,7 +20,7 @@ fi export CONTAINER_NAME=ci_native_asan export PACKAGES="systemtap-sdt-dev clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" -export DOCKER_NAME_TAG=ubuntu:22.04 +export CI_IMAGE_NAME_TAG=ubuntu:22.04 export NO_DEPENDS=1 export GOAL="install" export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index fd30ea068f..a2d393bb5a 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_fuzz export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev" export NO_DEPENDS=1 diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 82419708d2..7886f6efc9 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:20.04" +export CI_IMAGE_NAME_TAG="ubuntu:20.04" LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" @@ -15,7 +15,7 @@ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_fuzz_msan" export PACKAGES="clang-12 llvm-12 cmake" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 1e419245f0..75c10046b7 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_fuzz_valgrind export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev valgrind" export NO_DEPENDS=1 diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 9707f4cfb8..1f9209bafb 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:20.04" +export CI_IMAGE_NAME_TAG="ubuntu:20.04" LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" @@ -15,7 +15,7 @@ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" export PACKAGES="clang-12 llvm-12 cmake" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index a38bbf20b5..06bc2401c5 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -7,8 +7,11 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel -export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md -export PACKAGES="python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" # Use clang-8 to test C++17 compatibility, see doc/dependencies.md +export CI_IMAGE_NAME_TAG=debian:buster +# Use minimum supported python3.7 and clang-8, see doc/dependencies.md +export PACKAGES="-t buster-backports python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" +export APPEND_APT_SOURCES_LIST="deb http://deb.debian.org/debian buster-backports main" export DEP_OPTS="NO_WALLET=1 CC=clang-8 CXX='clang++-8 -stdlib=libc++'" export GOAL="install" +export NO_WERROR=1 export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 154e27064d..5cc0addd33 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -7,13 +7,15 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 -export DOCKER_NAME_TAG=debian:buster # Check that buster gcc-8 can compile our C++17 and run our functional tests in python3, see doc/dependencies.md +export CI_IMAGE_NAME_TAG=debian:buster +# Use minimum supported python3.7 and gcc-8, see doc/dependencies.md export PACKAGES="gcc-8 g++-8 python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-8 CXX=g++-8" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" +export NO_WERROR=1 export DOWNLOAD_PREVIOUS_RELEASES="true" export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports \ --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index e4d3468473..06ffe14636 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_tidy export PACKAGES="clang libclang-dev llvm-dev clang-tidy bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" export NO_DEPENDS=1 diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 0e0a4b8f18..61bcd98f0a 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export DOCKER_NAME_TAG=ubuntu:22.04 +export CI_IMAGE_NAME_TAG=ubuntu:22.04 export PACKAGES="clang-13 llvm-13 libc++abi-13-dev libc++-13-dev python3-zmq" export DEP_OPTS="CC=clang-13 CXX='clang++-13 -stdlib=libc++'" export GOAL="install" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index d881c41cdf..039a22b02e 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_valgrind export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 82c1e89220..af18703ce1 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -18,7 +18,7 @@ if [ -n "$QEMU_USER_CMD" ]; then fi # Use debian to avoid 404 apt errors export CONTAINER_NAME=ci_s390x -export DOCKER_NAME_TAG="debian:bookworm" +export CI_IMAGE_NAME_TAG="debian:bookworm" export TEST_RUNNER_ENV="LC_ALL=C" export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index b637086b06..817ee724c2 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export DOCKER_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to win64 +export CI_IMAGE_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to win64 export HOST=x86_64-w64-mingw32 export DPKG_ADD_ARCH="i386" export PACKAGES="python3 nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh new file mode 100755 index 0000000000..c2469d7ca9 --- /dev/null +++ b/ci/test/01_base_install.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +CI_EXEC_ROOT () { bash -c "$*"; } +export -f CI_EXEC_ROOT + +if [ -n "$DPKG_ADD_ARCH" ]; then + CI_EXEC_ROOT dpkg --add-architecture "$DPKG_ADD_ARCH" +fi + +if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then + ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y install epel-release + ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y --allowerasing install "$CI_BASE_PACKAGES" "$PACKAGES" +elif [ "$CI_USE_APT_INSTALL" != "no" ]; then + if [[ "${ADD_UNTRUSTED_BPFCC_PPA}" == "true" ]]; then + # Ubuntu 22.04 LTS and Debian 11 both have an outdated bpfcc-tools packages. + # The iovisor PPA is outdated as well. The next Ubuntu and Debian releases will contain updated + # packages. Meanwhile, use an untrusted PPA to install an up-to-date version of the bpfcc-tools + # package. + # TODO: drop this once we can use newer images in GCE + CI_EXEC_ROOT add-apt-repository ppa:hadret/bpfcc + fi + if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then + CI_EXEC_ROOT echo "${APPEND_APT_SOURCES_LIST}" \>\> /etc/apt/sources.list + fi + ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get update + ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$CI_BASE_PACKAGES" +fi diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index f8ff5aa37a..05bef79a3d 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -20,20 +20,25 @@ export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/t export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) - DOCKER_ADMIN="--cap-add SYS_PTRACE" + CI_CONTAINER_CAP="--cap-add SYS_PTRACE" fi export P_CI_DIR="$PWD" export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/" if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - echo "Creating $DOCKER_NAME_TAG container to run in" + echo "Creating $CI_IMAGE_NAME_TAG container to run in" LOCAL_UID=$(id -u) LOCAL_GID=$(id -g) # the name isn't important, so long as we use the same UID LOCAL_USER=nonroot - ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" + DOCKER_BUILDKIT=1 ${CI_RETRY_EXE} docker build \ + --file "${BASE_ROOT_DIR}/ci/test_imagefile" \ + --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ + --build-arg "FILE_ENV=${FILE_ENV}" \ + --tag="${CONTAINER_NAME}" \ + "${BASE_ROOT_DIR}" if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then echo "Restart docker before run to stop and clear all containers started with --rm" @@ -41,7 +46,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then fi # shellcheck disable=SC2086 - DOCKER_ID=$(docker run $DOCKER_ADMIN --rm --interactive --detach --tty \ + CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \ --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \ @@ -49,52 +54,34 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then -w $BASE_ROOT_DIR \ --env-file /tmp/env \ --name $CONTAINER_NAME \ - $DOCKER_NAME_TAG) + $CONTAINER_NAME) + export CI_CONTAINER_ID # Create a non-root user inside the container which matches the local user. # # This prevents the root user in the container modifying the local file system permissions # on the mounted directories - docker exec "$DOCKER_ID" useradd -u "$LOCAL_UID" -o -m "$LOCAL_USER" - docker exec "$DOCKER_ID" groupmod -o -g "$LOCAL_GID" "$LOCAL_USER" - docker exec "$DOCKER_ID" chown -R "$LOCAL_USER":"$LOCAL_USER" "${BASE_ROOT_DIR}" - export DOCKER_CI_CMD_PREFIX_ROOT="docker exec -u 0 $DOCKER_ID" - export DOCKER_CI_CMD_PREFIX="docker exec -u $LOCAL_UID $DOCKER_ID" + docker exec "$CI_CONTAINER_ID" useradd -u "$LOCAL_UID" -o -m "$LOCAL_USER" + docker exec "$CI_CONTAINER_ID" groupmod -o -g "$LOCAL_GID" "$LOCAL_USER" + docker exec "$CI_CONTAINER_ID" chown -R "$LOCAL_USER":"$LOCAL_USER" "${BASE_ROOT_DIR}" + export CI_EXEC_CMD_PREFIX_ROOT="docker exec -u 0 $CI_CONTAINER_ID" + export CI_EXEC_CMD_PREFIX="docker exec -u $LOCAL_UID $CI_CONTAINER_ID" else echo "Running on host system without docker wrapper" + "${BASE_ROOT_DIR}/ci/test/01_base_install.sh" fi CI_EXEC () { - $DOCKER_CI_CMD_PREFIX bash -c "export PATH=${BINS_SCRATCH_DIR}:\$PATH && cd \"$P_CI_DIR\" && $*" + $CI_EXEC_CMD_PREFIX bash -c "export PATH=${BINS_SCRATCH_DIR}:\$PATH && cd \"$P_CI_DIR\" && $*" } CI_EXEC_ROOT () { - $DOCKER_CI_CMD_PREFIX_ROOT bash -c "export PATH=${BINS_SCRATCH_DIR}:\$PATH && cd \"$P_CI_DIR\" && $*" + $CI_EXEC_CMD_PREFIX_ROOT bash -c "export PATH=${BINS_SCRATCH_DIR}:\$PATH && cd \"$P_CI_DIR\" && $*" } export -f CI_EXEC export -f CI_EXEC_ROOT CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}" -if [ -n "$DPKG_ADD_ARCH" ]; then - CI_EXEC_ROOT dpkg --add-architecture "$DPKG_ADD_ARCH" -fi - -if [[ $DOCKER_NAME_TAG == *centos* ]]; then - ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y install epel-release - ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y --allowerasing install "$DOCKER_PACKAGES" "$PACKAGES" -elif [ "$CI_USE_APT_INSTALL" != "no" ]; then - if [[ "${ADD_UNTRUSTED_BPFCC_PPA}" == "true" ]]; then - # Ubuntu 22.04 LTS and Debian 11 both have an outdated bpfcc-tools packages. - # The iovisor PPA is outdated as well. The next Ubuntu and Debian releases will contain updated - # packages. Meanwhile, use an untrusted PPA to install an up-to-date version of the bpfcc-tools - # package. - # TODO: drop this once we can use newer images in GCE - CI_EXEC_ROOT add-apt-repository ppa:hadret/bpfcc - fi - ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get update - ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$DOCKER_PACKAGES" -fi - if [ -n "$PIP_PACKAGES" ]; then if [ "$CI_OS_NAME" == "macos" ]; then sudo -H pip3 install --upgrade pip diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index c0cf957f79..f49305597d 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -38,7 +38,7 @@ if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then fi if [ -z "$NO_DEPENDS" ]; then - if [[ $DOCKER_NAME_TAG == *centos* ]]; then + if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then # CentOS has problems building the depends if the config shell is not explicitly set # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to # an error as the first command is executed) diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index ac2df5fbfb..49cc48aa15 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -9,12 +9,14 @@ export LC_ALL=C.UTF-8 if [[ $HOST = *-mingw32 ]]; then # Generate all binaries, so that they can be wrapped CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + CI_EXEC make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1 CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" fi if [ -n "$QEMU_USER_CMD" ]; then # Generate all binaries, so that they can be wrapped CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + CI_EXEC make "$MAKEJOBS" -C src minisketch/test VERBOSE=1 CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" fi @@ -31,7 +33,7 @@ if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then - CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast + CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=99999999 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast fi if [ "${RUN_TIDY}" = "true" ]; then @@ -48,6 +50,7 @@ if [ "${RUN_TIDY}" = "true" ]; then " src/node/chainstate.cpp"\ " src/node/chainstatemanager_args.cpp"\ " src/node/mempool_args.cpp"\ + " src/node/utxo_snapshot.cpp"\ " src/node/validation_cache_args.cpp"\ " src/policy/feerate.cpp"\ " src/policy/packages.cpp"\ @@ -72,7 +75,12 @@ if [ "${RUN_TIDY}" = "true" ]; then " src/util/syserror.cpp"\ " src/util/threadinterrupt.cpp"\ " src/zmq"\ - " -p . ${MAKEJOBS} -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" + " -p . ${MAKEJOBS}"\ + " -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp"\ + " |& tee /tmp/iwyu_ci.out" + export P_CI_DIR="${BASE_ROOT_DIR}/src" + CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/fix_includes.py --nosafe_headers < /tmp/iwyu_ci.out" + CI_EXEC "git --no-pager diff" fi if [ "$RUN_SECURITY_TESTS" = "true" ]; then @@ -82,3 +90,8 @@ fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py "${FUZZ_TESTS_CONFIG}" "$MAKEJOBS" -l DEBUG "${DIR_FUZZ_IN}" fi + +if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then + echo "Stop and remove CI container by ID" + docker container kill "${CI_CONTAINER_ID}" +fi diff --git a/ci/test_imagefile b/ci/test_imagefile new file mode 100644 index 0000000000..4854708d1a --- /dev/null +++ b/ci/test_imagefile @@ -0,0 +1,10 @@ +ARG CI_IMAGE_NAME_TAG +FROM ${CI_IMAGE_NAME_TAG} + +ARG FILE_ENV +ENV FILE_ENV=${FILE_ENV} + +COPY ./ci/retry/retry /usr/bin/retry +COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_base_install/ci/test/ + +RUN ["bash", "-c", "cd /ci_base_install/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"] diff --git a/configure.ac b/configure.ac index 4b55bef5c7..aa1a45c527 100644 --- a/configure.ac +++ b/configure.ac @@ -120,8 +120,8 @@ AC_PATH_TOOL([AR], [ar]) AC_PATH_TOOL([GCOV], [gcov]) AC_PATH_TOOL([LLVM_COV], [llvm-cov]) AC_PATH_PROG([LCOV], [lcov]) -dnl Python 3.6 is specified in .python-version and should be used if available, see doc/dependencies.md -AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9 python3.10 python3.11 python3 python]) +dnl Python 3.7 is specified in .python-version and should be used if available, see doc/dependencies.md +AC_PATH_PROGS([PYTHON], [python3.7 python3.8 python3.9 python3.10 python3.11 python3.12 python3 python]) AC_PATH_PROG([GENHTML], [genhtml]) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG([CCACHE], [ccache]) @@ -163,24 +163,12 @@ AC_ARG_WITH([miniupnpc], [use_upnp=$withval], [use_upnp=auto]) -AC_ARG_ENABLE([upnp-default], - [AS_HELP_STRING([--enable-upnp-default], - [if UPNP is enabled, turn it on at startup (default is no)])], - [use_upnp_default=$enableval], - [use_upnp_default=no]) - AC_ARG_WITH([natpmp], [AS_HELP_STRING([--with-natpmp], [enable NAT-PMP (default is yes if libnatpmp is found)])], [use_natpmp=$withval], [use_natpmp=auto]) -AC_ARG_ENABLE([natpmp-default], - [AS_HELP_STRING([--enable-natpmp-default], - [if NAT-PMP is enabled, turn it on at startup (default is no)])], - [use_natpmp_default=$enableval], - [use_natpmp_default=no]) - AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[do not compile tests (default is to compile)]), [use_tests=$enableval], @@ -448,7 +436,7 @@ if test "$CXXFLAGS_overridden" = "no"; then AX_CHECK_COMPILE_FLAG([-Wvla], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wshadow-field], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wthread-safety], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wloop-analysis], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wloop-analysis], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wloop-analysis"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wunused-member-function], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wdate-time], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"], [], [$CXXFLAG_WERROR]) @@ -1006,7 +994,7 @@ if test "$TARGET_OS" = "darwin"; then AX_CHECK_LINK_FLAG([-Wl,-bind_at_load], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"], [], [$LDFLAG_WERROR]) fi -AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h unistd.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) +AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) AC_CHECK_DECLS([getifaddrs, freeifaddrs],[CHECK_SOCKET],, [#include <sys/types.h> @@ -1461,7 +1449,7 @@ if test "$use_natpmp" != "no"; then CPPFLAGS="$TEMP_CPPFLAGS" fi -if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" = "nonononononono"; then +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" = "nononononononono"; then use_boost=no else use_boost=yes @@ -1479,9 +1467,11 @@ if test "$use_boost" = "yes"; then BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_DISABLE_SERIALIZATION" dnl Prevent use of std::unary_function, which was removed in C++17, - dnl and will generate warnings with newer compilers. - dnl See: https://github.com/boostorg/container_hash/issues/22. - BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_NO_CXX98_FUNCTION_BASE" + dnl and will generate warnings with newer compilers for Boost + dnl older than 1.80. + dnl See: https://github.com/boostorg/config/pull/430. + AX_CHECK_PREPROC_FLAG([-DBOOST_NO_CXX98_FUNCTION_BASE], [BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_NO_CXX98_FUNCTION_BASE"], [], [$CXXFLAG_WERROR], + [AC_LANG_PROGRAM([[#include <boost/config.hpp>]])]) if test "$enable_debug" = "yes" || test "$enable_fuzz" = "yes"; then BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE" @@ -1765,15 +1755,8 @@ if test "$have_miniupnpc" = "no"; then else if test "$use_upnp" != "no"; then AC_MSG_RESULT([yes]) - AC_MSG_CHECKING([whether to build with UPnP enabled by default]) use_upnp=yes - upnp_setting=0 - if test "$use_upnp_default" != "no"; then - use_upnp_default=yes - upnp_setting=1 - fi - AC_MSG_RESULT([$use_upnp_default]) - AC_DEFINE_UNQUOTED([USE_UPNP],[$upnp_setting],[UPnP support not compiled if undefined, otherwise value (0 or 1) determines default state]) + AC_DEFINE([USE_UPNP], [1], [Define to 1 if UPnP support should be compiled in.]) if test "$TARGET_OS" = "windows"; then MINIUPNPC_CPPFLAGS="$MINIUPNPC_CPPFLAGS -DSTATICLIB -DMINIUPNP_STATICLIB" fi @@ -1793,15 +1776,8 @@ if test "$have_natpmp" = "no"; then else if test "$use_natpmp" != "no"; then AC_MSG_RESULT([yes]) - AC_MSG_CHECKING([whether to build with NAT-PMP enabled by default]) use_natpmp=yes - natpmp_setting=0 - if test "$use_natpmp_default" != "no"; then - use_natpmp_default=yes - natpmp_setting=1 - fi - AC_MSG_RESULT($use_natpmp_default) - AC_DEFINE_UNQUOTED([USE_NATPMP], [$natpmp_setting], [NAT-PMP support not compiled if undefined, otherwise value (0 or 1) determines default state]) + AC_DEFINE([USE_NATPMP], [1], [Define to 1 if UPnP support should be compiled in.]) if test "$TARGET_OS" = "windows"; then NATPMP_CPPFLAGS="$NATPMP_CPPFLAGS -DSTATICLIB -DNATPMP_STATICLIB" fi @@ -1870,7 +1846,7 @@ else AC_MSG_RESULT([no]) fi -if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_bench$use_tests" = "nonononononononono"; then +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_bench$use_tests" = "nononononononononono"; then AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-fuzz(-binary) --enable-bench or --enable-tests]) fi @@ -2004,15 +1980,7 @@ CPPFLAGS_TEMP="$CPPFLAGS" unset CPPFLAGS CPPFLAGS="$CPPFLAGS_TEMP" -LDFLAGS_TEMP="$LDFLAGS" -unset LDFLAGS -LDFLAGS="$LDFLAGS_TEMP" - -LIBS_TEMP="$LIBS" -unset LIBS -LIBS="$LIBS_TEMP" - -ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --enable-module-schnorrsig" +ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --disable-module-ecdh" AC_CONFIG_SUBDIRS([src/secp256k1]) AC_OUTPUT diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 60b3bcb424..ca430170a1 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -7,22 +7,14 @@ Source: https://github.com/bitcoin/bitcoin Files: * Copyright: 2009-2023, Bitcoin Core Developers License: Expat -Comment: The Bitcoin Core Developers encompasses the current developers listed on bitcoin.org, - as well as the numerous contributors to the project. +Comment: The Bitcoin Core Developers encompasses all contributors to the + project, listed in the release notes or the git log. Files: debian/* Copyright: 2010-2011, Jonas Smedegaard <dr@jones.dk> 2011, Matt Corallo <matt@bluematt.me> License: GPL-2+ -Files: src/secp256k1/build-aux/m4/ax_jni_include_dir.m4 -Copyright: 2008 Don Anderson <dda@sleepycat.com> -License: GNU-All-permissive-License - -Files: src/secp256k1/build-aux/m4/ax_prog_cc_for_build.m4 -Copyright: 2008 Paolo Bonzini <bonzini@gnu.org> -License: GNU-All-permissive-License - Files: src/qt/res/icons/add.png src/qt/res/icons/address-book.png src/qt/res/icons/chevron.png @@ -112,12 +104,6 @@ License: Expat TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -License: GNU-All-permissive-License - Copying and distribution of this file, with or without modification, are - permitted in any medium without royalty provided the copyright notice - and this notice are preserved. This file is offered as-is, without any - warranty. - License: GPL-2+ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py index 98eee67f43..420bf7ff33 100755 --- a/contrib/devtools/clang-format-diff.py +++ b/contrib/devtools/clang-format-diff.py @@ -146,7 +146,7 @@ def main(): stdout=subprocess.PIPE, stderr=None, stdin=subprocess.PIPE, - universal_newlines=True) + text=True) stdout, stderr = p.communicate() if p.returncode != 0: sys.exit(p.returncode) diff --git a/contrib/devtools/gen-manpages.py b/contrib/devtools/gen-manpages.py index 38cdb3acb8..2860e7db99 100755 --- a/contrib/devtools/gen-manpages.py +++ b/contrib/devtools/gen-manpages.py @@ -23,7 +23,7 @@ help2man = os.getenv('HELP2MAN', 'help2man') # If not otherwise specified, get top directory from git. topdir = os.getenv('TOPDIR') if not topdir: - r = subprocess.run([git, 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, check=True, universal_newlines=True) + r = subprocess.run([git, 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, check=True, text=True) topdir = r.stdout.rstrip() # Get input and output directories. @@ -36,7 +36,7 @@ versions = [] for relpath in BINARIES: abspath = os.path.join(builddir, relpath) try: - r = subprocess.run([abspath, "--version"], stdout=subprocess.PIPE, check=True, universal_newlines=True) + r = subprocess.run([abspath, "--version"], stdout=subprocess.PIPE, check=True, text=True) except IOError: print(f'{abspath} not found or not an executable', file=sys.stderr) sys.exit(1) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 336e2c9e2e..8377b92736 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -146,6 +146,12 @@ def check_PE_control_flow(binary) -> bool: return True return False +def check_PE_Canary(binary) -> bool: + ''' + Check for use of stack canary + ''' + return binary.has_symbol('__stack_chk_fail') + def check_MACHO_NOUNDEFS(binary) -> bool: ''' Check for no undefined references. @@ -203,6 +209,7 @@ BASE_PE = [ ('NX', check_NX), ('RELOC_SECTION', check_PE_RELOC_SECTION), ('CONTROL_FLOW', check_PE_control_flow), + ('Canary', check_PE_Canary), ] BASE_MACHO = [ diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 6e0e0ae1b5..54718fd7a1 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -39,7 +39,7 @@ def call_security_check(cc, source, executable, options): env_flags += filter(None, os.environ.get(var, '').split(' ')) subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) - p = subprocess.run([os.path.join(os.path.dirname(__file__), 'security-check.py'), executable], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.run([os.path.join(os.path.dirname(__file__), 'security-check.py'), executable], stdout=subprocess.PIPE, text=True) return (p.returncode, p.stdout.rstrip()) def get_arch(cc, source, executable): @@ -94,19 +94,19 @@ class TestSecurityChecks(unittest.TestCase): cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fno-stack-protector']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fstack-protector-all', '-lssp']), (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fstack-protector-all', '-lssp']), (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), (1, executable+': failed CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full','-fstack-protector-all', '-lssp']), (0, '')) clean_files(source, executable) diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index 624870ebcb..de73b02090 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -23,13 +23,13 @@ def call_symbol_check(cc: List[str], source, executable, options): env_flags += filter(None, os.environ.get(var, '').split(' ')) subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) - p = subprocess.run([os.path.join(os.path.dirname(__file__), 'symbol-check.py'), executable], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.run([os.path.join(os.path.dirname(__file__), 'symbol-check.py'), executable], stdout=subprocess.PIPE, text=True) os.remove(source) os.remove(executable) return (p.returncode, p.stdout.rstrip()) def get_machine(cc: List[str]): - p = subprocess.run([*cc,'-dumpmachine'], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.run([*cc,'-dumpmachine'], stdout=subprocess.PIPE, text=True) return p.stdout.rstrip() class TestSymbolChecks(unittest.TestCase): diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh deleted file mode 100755 index c7d39f5b99..0000000000 --- a/contrib/install_db4.sh +++ /dev/null @@ -1,259 +0,0 @@ -#!/bin/sh -# Copyright (c) 2017-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# Install libdb4.8 (Berkeley DB). - -export LC_ALL=C -set -e - -if [ -z "${1}" ]; then - echo "Usage: $0 <base-dir> [<extra-bdb-configure-flag> ...]" - echo - echo "Must specify a single argument: the directory in which db4 will be built." - echo "This is probably \`pwd\` if you're at the root of the bitcoin repository." - exit 1 -fi - -expand_path() { - cd "${1}" && pwd -P -} - -BDB_PREFIX="$(expand_path "${1}")/db4"; shift; -BDB_VERSION='db-4.8.30.NC' -BDB_HASH='12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef' -BDB_URL="https://download.oracle.com/berkeley-db/${BDB_VERSION}.tar.gz" - -check_exists() { - command -v "$1" >/dev/null -} - -sha256_check() { - # Args: <sha256_hash> <filename> - # - if [ "$(uname)" = "FreeBSD" ]; then - # sha256sum exists on FreeBSD, but takes different arguments than the GNU version - sha256 -c "${1}" "${2}" - elif check_exists sha256sum; then - echo "${1} ${2}" | sha256sum -c - elif check_exists sha256; then - echo "${1} ${2}" | sha256 -c - else - echo "${1} ${2}" | shasum -a 256 -c - fi -} - -http_get() { - # Args: <url> <filename> <sha256_hash> - # - # It's acceptable that we don't require SSL here because we manually verify - # content hashes below. - # - if [ -f "${2}" ]; then - echo "File ${2} already exists; not downloading again" - elif check_exists curl; then - curl --insecure --retry 5 "${1}" -o "${2}" - elif check_exists wget; then - wget --no-check-certificate "${1}" -O "${2}" - else - echo "Simple transfer utilities 'curl' and 'wget' not found. Please install one of them and try again." - exit 1 - fi - - sha256_check "${3}" "${2}" -} - -# Ensure the commands we use exist on the system -if ! check_exists patch; then - echo "Command-line tool 'patch' not found. Install patch and try again." - exit 1 -fi - -mkdir -p "${BDB_PREFIX}" -http_get "${BDB_URL}" "${BDB_VERSION}.tar.gz" "${BDB_HASH}" -tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX" -cd "${BDB_PREFIX}/${BDB_VERSION}/" - -# Apply a patch necessary when building with clang and c++11 (see https://community.oracle.com/thread/3952592) -patch --ignore-whitespace -p1 << 'EOF' -commit 3311d68f11d1697565401eee6efc85c34f022ea7 -Author: fanquake <fanquake@gmail.com> -Date: Mon Aug 17 20:03:56 2020 +0800 - - Fix C++11 compatibility - -diff --git a/dbinc/atomic.h b/dbinc/atomic.h -index 0034dcc..7c11d4a 100644 ---- a/dbinc/atomic.h -+++ b/dbinc/atomic.h -@@ -70,7 +70,7 @@ typedef struct { - * These have no memory barriers; the caller must include them when necessary. - */ - #define atomic_read(p) ((p)->value) --#define atomic_init(p, val) ((p)->value = (val)) -+#define atomic_init_db(p, val) ((p)->value = (val)) - - #ifdef HAVE_ATOMIC_SUPPORT - -@@ -144,7 +144,7 @@ typedef LONG volatile *interlocked_val; - #define atomic_inc(env, p) __atomic_inc(p) - #define atomic_dec(env, p) __atomic_dec(p) - #define atomic_compare_exchange(env, p, o, n) \ -- __atomic_compare_exchange((p), (o), (n)) -+ __atomic_compare_exchange_db((p), (o), (n)) - static inline int __atomic_inc(db_atomic_t *p) - { - int temp; -@@ -176,7 +176,7 @@ static inline int __atomic_dec(db_atomic_t *p) - * http://gcc.gnu.org/onlinedocs/gcc-4.1.0/gcc/Atomic-Builtins.html - * which configure could be changed to use. - */ --static inline int __atomic_compare_exchange( -+static inline int __atomic_compare_exchange_db( - db_atomic_t *p, atomic_value_t oldval, atomic_value_t newval) - { - atomic_value_t was; -@@ -206,7 +206,7 @@ static inline int __atomic_compare_exchange( - #define atomic_dec(env, p) (--(p)->value) - #define atomic_compare_exchange(env, p, oldval, newval) \ - (DB_ASSERT(env, atomic_read(p) == (oldval)), \ -- atomic_init(p, (newval)), 1) -+ atomic_init_db(p, (newval)), 1) - #else - #define atomic_inc(env, p) __atomic_inc(env, p) - #define atomic_dec(env, p) __atomic_dec(env, p) -diff --git a/mp/mp_fget.c b/mp/mp_fget.c -index 5fdee5a..0b75f57 100644 ---- a/mp/mp_fget.c -+++ b/mp/mp_fget.c -@@ -617,7 +617,7 @@ alloc: /* Allocate a new buffer header and data space. */ - - /* Initialize enough so we can call __memp_bhfree. */ - alloc_bhp->flags = 0; -- atomic_init(&alloc_bhp->ref, 1); -+ atomic_init_db(&alloc_bhp->ref, 1); - #ifdef DIAGNOSTIC - if ((uintptr_t)alloc_bhp->buf & (sizeof(size_t) - 1)) { - __db_errx(env, -@@ -911,7 +911,7 @@ alloc: /* Allocate a new buffer header and data space. */ - MVCC_MPROTECT(bhp->buf, mfp->stat.st_pagesize, - PROT_READ); - -- atomic_init(&alloc_bhp->ref, 1); -+ atomic_init_db(&alloc_bhp->ref, 1); - MUTEX_LOCK(env, alloc_bhp->mtx_buf); - alloc_bhp->priority = bhp->priority; - alloc_bhp->pgno = bhp->pgno; -diff --git a/mp/mp_mvcc.c b/mp/mp_mvcc.c -index 34467d2..f05aa0c 100644 ---- a/mp/mp_mvcc.c -+++ b/mp/mp_mvcc.c -@@ -276,7 +276,7 @@ __memp_bh_freeze(dbmp, infop, hp, bhp, need_frozenp) - #else - memcpy(frozen_bhp, bhp, SSZA(BH, buf)); - #endif -- atomic_init(&frozen_bhp->ref, 0); -+ atomic_init_db(&frozen_bhp->ref, 0); - if (mutex != MUTEX_INVALID) - frozen_bhp->mtx_buf = mutex; - else if ((ret = __mutex_alloc(env, MTX_MPOOL_BH, -@@ -428,7 +428,7 @@ __memp_bh_thaw(dbmp, infop, hp, frozen_bhp, alloc_bhp) - #endif - alloc_bhp->mtx_buf = mutex; - MUTEX_LOCK(env, alloc_bhp->mtx_buf); -- atomic_init(&alloc_bhp->ref, 1); -+ atomic_init_db(&alloc_bhp->ref, 1); - F_CLR(alloc_bhp, BH_FROZEN); - } - -diff --git a/mp/mp_region.c b/mp/mp_region.c -index e6cece9..ddbe906 100644 ---- a/mp/mp_region.c -+++ b/mp/mp_region.c -@@ -224,7 +224,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) - MTX_MPOOL_FILE_BUCKET, 0, &htab[i].mtx_hash)) != 0) - return (ret); - SH_TAILQ_INIT(&htab[i].hash_bucket); -- atomic_init(&htab[i].hash_page_dirty, 0); -+ atomic_init_db(&htab[i].hash_page_dirty, 0); - } - - /* -@@ -269,7 +269,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) - hp->mtx_hash = (mtx_base == MUTEX_INVALID) ? MUTEX_INVALID : - mtx_base + i; - SH_TAILQ_INIT(&hp->hash_bucket); -- atomic_init(&hp->hash_page_dirty, 0); -+ atomic_init_db(&hp->hash_page_dirty, 0); - #ifdef HAVE_STATISTICS - hp->hash_io_wait = 0; - hp->hash_frozen = hp->hash_thawed = hp->hash_frozen_freed = 0; -diff --git a/mutex/mut_method.c b/mutex/mut_method.c -index 2588763..5c6d516 100644 ---- a/mutex/mut_method.c -+++ b/mutex/mut_method.c -@@ -426,7 +426,7 @@ atomic_compare_exchange(env, v, oldval, newval) - MUTEX_LOCK(env, mtx); - ret = atomic_read(v) == oldval; - if (ret) -- atomic_init(v, newval); -+ atomic_init_db(v, newval); - MUTEX_UNLOCK(env, mtx); - - return (ret); -diff --git a/mutex/mut_tas.c b/mutex/mut_tas.c -index f3922e0..e40fcdf 100644 ---- a/mutex/mut_tas.c -+++ b/mutex/mut_tas.c -@@ -46,7 +46,7 @@ __db_tas_mutex_init(env, mutex, flags) - - #ifdef HAVE_SHARED_LATCHES - if (F_ISSET(mutexp, DB_MUTEX_SHARED)) -- atomic_init(&mutexp->sharecount, 0); -+ atomic_init_db(&mutexp->sharecount, 0); - else - #endif - if (MUTEX_INIT(&mutexp->tas)) { -@@ -486,7 +486,7 @@ __db_tas_mutex_unlock(env, mutex) - F_CLR(mutexp, DB_MUTEX_LOCKED); - /* Flush flag update before zeroing count */ - MEMBAR_EXIT(); -- atomic_init(&mutexp->sharecount, 0); -+ atomic_init_db(&mutexp->sharecount, 0); - } else { - DB_ASSERT(env, sharecount > 0); - MEMBAR_EXIT(); -EOF - -# The packaged config.guess and config.sub are ancient (2009) and can cause build issues. -# Replace them with modern versions. -# See https://github.com/bitcoin/bitcoin/issues/16064 -CONFIG_GUESS_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' -CONFIG_GUESS_HASH='c8f530e01840719871748a8071113435bdfdf75b74c57e78e47898edea8754ae' -CONFIG_SUB_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' -CONFIG_SUB_HASH='3969f7d5f6967ccc6f792401b8ef3916a1d1b1d0f0de5a4e354c95addb8b800e' - -rm -f "dist/config.guess" -rm -f "dist/config.sub" - -http_get "${CONFIG_GUESS_URL}" dist/config.guess "${CONFIG_GUESS_HASH}" -http_get "${CONFIG_SUB_URL}" dist/config.sub "${CONFIG_SUB_HASH}" - -cd build_unix/ - -"${BDB_PREFIX}/${BDB_VERSION}/dist/configure" \ - --enable-cxx --disable-shared --disable-replication --with-pic --prefix="${BDB_PREFIX}" \ - "${@}" - -make install - -echo -echo "db4 build complete." -echo -# shellcheck disable=SC2016 -echo 'When compiling bitcoind, run `./configure` in the following way:' -echo -echo " export BDB_PREFIX='${BDB_PREFIX}'" -# shellcheck disable=SC2016 -echo ' ./configure BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" BDB_CFLAGS="-I${BDB_PREFIX}/include" ...' diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 2420539b7c..f8677ba7b8 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -187,7 +187,7 @@ def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: if verbose: print(f"Inspecting with otool: {binaryPath}") otoolbin=os.getenv("OTOOL", "otool") - otool = run([otoolbin, "-L", binaryPath], stdout=PIPE, stderr=PIPE, universal_newlines=True) + otool = run([otoolbin, "-L", binaryPath], stdout=PIPE, stderr=PIPE, text=True) if otool.returncode != 0: sys.stderr.write(otool.stderr) sys.stderr.flush() @@ -577,17 +577,17 @@ if config.dmg is not None: tempname: str = appname + ".temp.dmg" - run(["hdiutil", "create", tempname, "-srcfolder", "dist", "-format", "UDRW", "-size", str(size), "-volname", appname], check=True, universal_newlines=True) + run(["hdiutil", "create", tempname, "-srcfolder", "dist", "-format", "UDRW", "-size", str(size), "-volname", appname], check=True, text=True) if verbose: print("Attaching temp image...") - output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, universal_newlines=True, stdout=PIPE).stdout + output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, text=True, stdout=PIPE).stdout print("+ Finalizing .dmg disk image +") - run(["hdiutil", "detach", f"/Volumes/{appname}"], universal_newlines=True) + run(["hdiutil", "detach", f"/Volumes/{appname}"], text=True) - run(["hdiutil", "convert", tempname, "-format", "UDZO", "-o", appname, "-imagekey", "zlib-level=9"], check=True, universal_newlines=True) + run(["hdiutil", "convert", tempname, "-format", "UDZO", "-o", appname, "-imagekey", "zlib-level=9"], check=True, text=True) os.unlink(tempname) diff --git a/contrib/signet/getcoins.py b/contrib/signet/getcoins.py index ff99d60679..d4e436626f 100755 --- a/contrib/signet/getcoins.py +++ b/contrib/signet/getcoins.py @@ -129,7 +129,7 @@ if args.captcha != '': # Retrieve a captcha # Convert SVG image to PPM, and load it try: - rv = subprocess.run([args.imagemagick, 'svg:-', '-depth', '8', 'ppm:-'], input=res.content, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + rv = subprocess.run([args.imagemagick, 'svg:-', '-depth', '8', 'ppm:-'], input=res.content, check=True, capture_output=True) except FileNotFoundError: raise SystemExit(f"The binary {args.imagemagick} could not be found. Please make sure ImageMagick (or a compatible fork) is installed and that the correct path is specified.") diff --git a/depends/Makefile b/depends/Makefile index 11fdd6dd53..27bf804c6b 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -32,6 +32,8 @@ SOURCES_PATH ?= $(BASEDIR)/sources WORK_PATH = $(BASEDIR)/work BASE_CACHE ?= $(BASEDIR)/built SDK_PATH ?= $(BASEDIR)/SDKs +NO_BOOST ?= +NO_LIBEVENT ?= NO_QT ?= NO_QR ?= NO_BDB ?= @@ -147,6 +149,10 @@ include packages/packages.mk build_id:=$(shell env CC='$(build_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(build_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') $(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(host_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') +boost_packages_$(NO_BOOST) = $(boost_packages) + +libevent_packages_$(NO_LIBEVENT) = $(libevent_packages) + qrencode_packages_$(NO_QR) = $(qrencode_$(host_os)_packages) qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) $(qrencode_packages_) @@ -162,7 +168,7 @@ zmq_packages_$(NO_ZMQ) = $(zmq_packages) multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) usdt_packages_$(NO_USDT) = $(usdt_$(host_os)_packages) -packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) +packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) ifneq ($(zmq_packages_),) diff --git a/depends/README.md b/depends/README.md index a8831eb0fc..11abbbd90d 100644 --- a/depends/README.md +++ b/depends/README.md @@ -98,6 +98,8 @@ The following can be set when running make: `make FOO=bar` - `FALLBACK_DOWNLOAD_PATH`: If a source file can't be fetched, try here before giving up - `C_STANDARD`: Set the C standard version used. Defaults to `c11`. - `CXX_STANDARD`: Set the C++ standard version used. Defaults to `c++17`. +- `NO_BOOST`: Don't download/build/cache Boost +- `NO_LIBEVENT`: Don't download/build/cache Libevent - `NO_QT`: Don't download/build/cache Qt and its dependencies - `NO_QR`: Don't download/build/cache packages needed for enabling qrencode - `NO_ZMQ`: Don't download/build/cache packages needed for enabling ZeroMQ diff --git a/depends/funcs.mk b/depends/funcs.mk index 2b21d053b1..6b333f940e 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -76,7 +76,7 @@ $(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted $(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed $(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned $(1)_built=$$($(1)_build_dir)/.stamp_built -$(1)_configured=$$($(1)_build_dir)/.stamp_configured +$(1)_configured=$(host_prefix)/.$(1)_stamp_configured $(1)_staged=$$($(1)_staging_dir)/.stamp_staged $(1)_postprocessed=$$($(1)_staging_prefix_dir)/.stamp_postprocessed $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path)) @@ -214,8 +214,8 @@ $($(1)_preprocessed): | $($(1)_extracted) $($(1)_configured): | $($(1)_dependencies) $($(1)_preprocessed) echo Configuring $(1)... rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), $(build_TAR) --no-same-owner -xf $($(package)_cached); ) - mkdir -p $$(@D) - +{ cd $$(@D); export $($(1)_config_env); $($(1)_config_cmds); } $$($(1)_logging) + mkdir -p $$($(1)_build_dir) + +{ cd $$($(1)_build_dir); export $($(1)_config_env); $($(1)_config_cmds); } $$($(1)_logging) touch $$@ $($(1)_built): | $($(1)_configured) echo Building $(1)... diff --git a/depends/hosts/default.mk b/depends/hosts/default.mk index 7c76331ab4..bad4568bcb 100644 --- a/depends/hosts/default.mk +++ b/depends/hosts/default.mk @@ -28,8 +28,13 @@ host_$1=$$($(host_arch)_$(host_os)_$1) endef define add_host_flags_func +ifeq ($(filter $(origin $1),undefined default),) +$(host_arch)_$(host_os)_$1 = +$(host_arch)_$(host_os)_$(release_type)_$1 = $($1) +else $(host_arch)_$(host_os)_$1 += $($(host_os)_$1) $(host_arch)_$(host_os)_$(release_type)_$1 += $($(host_os)_$(release_type)_$1) +endif host_$1 = $$($(host_arch)_$(host_os)_$1) host_$(release_type)_$1 = $$($(host_arch)_$(host_os)_$(release_type)_$1) endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 998cc0221c..b3600b72d0 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,4 +1,8 @@ -packages:=boost libevent +packages:= + +boost_packages = boost + +libevent_packages = libevent qrencode_linux_packages = qrencode qrencode_android_packages = qrencode diff --git a/depends/packages/sqlite.mk b/depends/packages/sqlite.mk index 820d724214..a8ec89c6c6 100644 --- a/depends/packages/sqlite.mk +++ b/depends/packages/sqlite.mk @@ -6,10 +6,15 @@ $(package)_sha256_hash=5af07de982ba658fd91a03170c945f99c971f6955bc79df3266544373 define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-readline --disable-dynamic-extensions --enable-option-checking +$(package)_config_opts+= --disable-rtree --disable-fts4 --disable-fts5 $(package)_config_opts_linux=--with-pic $(package)_config_opts_freebsd=--with-pic $(package)_config_opts_netbsd=--with-pic $(package)_config_opts_openbsd=--with-pic +$(package)_config_opts_debug=--enable-debug +$(package)_cflags+=-DSQLITE_DQS=0 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED +$(package)_cflags+=-DSQLITE_OMIT_SHARED_CACHE -DSQLITE_OMIT_JSON -DSQLITE_LIKE_DOESNT_MATCH_BLOBS +$(package)_cflags+=-DSQLITE_OMIT_DECLTYPE -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_AUTOINIT endef define $(package)_preprocess_cmds diff --git a/depends/packages/systemtap.mk b/depends/packages/systemtap.mk index a57f1b6d36..541ebeee01 100644 --- a/depends/packages/systemtap.mk +++ b/depends/packages/systemtap.mk @@ -1,12 +1,13 @@ package=systemtap -$(package)_version=4.7 -$(package)_download_path=https://sourceware.org/systemtap/ftp/releases/ +$(package)_version=4.8 +$(package)_download_path=https://sourceware.org/ftp/systemtap/releases/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=43a0a3db91aa4d41e28015b39a65e62059551f3cc7377ebf3a3a5ca7339e7b1f -$(package)_patches=remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch +$(package)_sha256_hash=cbd50a4eba5b261394dc454c12448ddec73e55e6742fda7f508f9fbc1331c223 +$(package)_patches=remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch fix_variadic_warning.patch define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch && \ + patch -p1 < $($(package)_patch_dir)/fix_variadic_warning.patch && \ mkdir -p $($(package)_staging_prefix_dir)/include/sys && \ cp includes/sys/sdt.h $($(package)_staging_prefix_dir)/include/sys/sdt.h endef diff --git a/depends/patches/systemtap/fix_variadic_warning.patch b/depends/patches/systemtap/fix_variadic_warning.patch new file mode 100644 index 0000000000..93cc2d6081 --- /dev/null +++ b/depends/patches/systemtap/fix_variadic_warning.patch @@ -0,0 +1,16 @@ +Could be dropped after a migration to C++20. +See: https://github.com/bitcoin/bitcoin/issues/26916. + +diff --git a/includes/sys/sdt.h b/includes/sys/sdt.h +index 4075a5f..7c6138c 100644 +--- a/includes/sys/sdt.h ++++ b/includes/sys/sdt.h +@@ -276,7 +276,7 @@ __extension__ extern unsigned long long __sdt_unsp; + _SDT_ASM_1(.purgem _SDT_TYPE_) \ + _SDT_ASM_1(.purgem _SDT_TYPE) + +-#define _SDT_ASM_BODY(provider, name, pack_args, args, ...) \ ++#define _SDT_ASM_BODY(provider, name, pack_args, args) \ + _SDT_DEF_MACROS \ + _SDT_ASM_1(990: _SDT_NOP) \ + _SDT_ASM_3( .pushsection .note.stapsdt,_SDT_ASM_AUTOGROUP,"note") \ diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index a8e643a2ab..d45e9c4d0d 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -72,7 +72,7 @@ There is an included test suite that is useful for testing code changes when dev To run the test suite (recommended), you will need to have Python 3 installed: ```bash -pkg install python3 +pkg install python3 databases/py-sqlite3 ``` --- diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index afbb5c8e75..255995a517 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -41,16 +41,18 @@ pkg_add sqlite3 BerkeleyDB is only required to support legacy wallets. It is recommended to use Berkeley DB 4.8. You cannot use the BerkeleyDB library -from ports. However you can build it yourself, [using the installation script included in contrib/](/contrib/install_db4.sh), like so, from the root of the repository. +from ports. However you can build it yourself, [using depends](/depends). ```bash -./contrib/install_db4.sh `pwd` +gmake -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 +... +to: /path/to/bitcoin/depends/x86_64-unknown-openbsd ``` Then set `BDB_PREFIX`: ```bash -export BDB_PREFIX="$PWD/db4" +export BDB_PREFIX="/path/to/bitcoin/depends/x86_64-unknown-openbsd" ``` #### GUI Dependencies diff --git a/doc/build-unix.md b/doc/build-unix.md index 874015707a..0960ae1577 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -72,7 +72,7 @@ executables, which are based on BerkeleyDB 4.8. If you do not care about wallet To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) -Optional port mapping libraries (see: `--with-miniupnpc`, `--enable-upnp-default`, and `--with-natpmp`, `--enable-natpmp-default`): +Optional port mapping libraries (see: `--with-miniupnpc` and `--with-natpmp`): sudo apt install libminiupnpc-dev libnatpmp-dev @@ -133,7 +133,7 @@ pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley D To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) -Optional port mapping libraries (see: `--with-miniupnpc`, `--enable-upnp-default`, and `--with-natpmp`, `--enable-natpmp-default`): +Optional port mapping libraries (see: `--with-miniupnpc` and `--with-natpmp`): sudo dnf install miniupnpc-devel libnatpmp-devel @@ -176,38 +176,34 @@ miniupnpc [miniupnpc](https://miniupnp.tuxfamily.org) may be used for UPnP port mapping. It can be downloaded from [here]( https://miniupnp.tuxfamily.org/files/). UPnP support is compiled in and -turned off by default. See the configure options for UPnP behavior desired: - - --without-miniupnpc No UPnP support, miniupnp not required - --disable-upnp-default (the default) UPnP support turned off by default at runtime - --enable-upnp-default UPnP support turned on by default at runtime +turned off by default. libnatpmp --------- [libnatpmp](https://miniupnp.tuxfamily.org/libnatpmp.html) may be used for NAT-PMP port mapping. It can be downloaded from [here](https://miniupnp.tuxfamily.org/files/). NAT-PMP support is compiled in and -turned off by default. See the configure options for NAT-PMP behavior desired: - - --without-natpmp No NAT-PMP support, libnatpmp not required - --disable-natpmp-default (the default) NAT-PMP support turned off by default at runtime - --enable-natpmp-default NAT-PMP support turned on by default at runtime +turned off by default. Berkeley DB ----------- The legacy wallet uses Berkeley DB. To ensure backwards compatibility it is -recommended to use Berkeley DB 4.8. If you have to build it yourself, you can -use [the installation script included in contrib/](/contrib/install_db4.sh) -like so: - -```shell -./contrib/install_db4.sh `pwd` +recommended to use Berkeley DB 4.8. If you have to build it yourself, and don't +want to use any other libraries built in depends, you can do: +```bash +make -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 +... +to: /path/to/bitcoin/depends/x86_64-pc-linux-gnu ``` +and configure using the following: +```bash +export BDB_PREFIX="/path/to/bitcoin/depends/x86_64-pc-linux-gnu" -from the root of the repository. - -Otherwise, you can build Bitcoin Core from self-compiled [depends](/depends/README.md). +./configure \ + BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ + BDB_CFLAGS="-I${BDB_PREFIX}/include" +``` **Note**: You only need Berkeley DB if the legacy wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). diff --git a/doc/dependencies.md b/doc/dependencies.md index c1f641c162..3349c81c46 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -10,7 +10,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Automake](https://www.gnu.org/software/automake/) | [1.13](https://github.com/bitcoin/bitcoin/pull/18290) | | [Clang](https://clang.llvm.org) | [8.0](https://github.com/bitcoin/bitcoin/pull/24164) | | [GCC](https://gcc.gnu.org) | [8.1](https://github.com/bitcoin/bitcoin/pull/23060) | -| [Python](https://www.python.org) (tests) | [3.6](https://github.com/bitcoin/bitcoin/pull/19504) | +| [Python](https://www.python.org) (tests) | [3.7](https://github.com/bitcoin/bitcoin/pull/26226) | | [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | ## Required diff --git a/doc/managing-wallets.md b/doc/managing-wallets.md index 366d7ec54b..22e006c963 100644 --- a/doc/managing-wallets.md +++ b/doc/managing-wallets.md @@ -88,7 +88,7 @@ In the RPC, the destination parameter must include the name of the file. Otherwi $ bitcoin-cli -rpcwallet="wallet-01" backupwallet /home/node01/Backups/backup-01.dat ``` -In the GUI, the wallet is selected in the `Wallet` drop-down list in the upper right corner. If this list is not present, the wallet can be loaded in `File` ->`Open wallet` if necessary. Then, the backup can be done in `File` -> `Backup Wallet...`. +In the GUI, the wallet is selected in the `Wallet` drop-down list in the upper right corner. If this list is not present, the wallet can be loaded in `File` ->`Open Wallet` if necessary. Then, the backup can be done in `File` -> `Backup Wallet…`. This backup file can be stored on one or multiple offline devices, which must be reliable enough to work in an emergency and be malware free. Backup files can be regularly tested to avoid problems in the future. @@ -108,7 +108,7 @@ Wallets created before version 0.13 are not HD and must be backed up every 100 k ### 1.6 Restoring the Wallet From a Backup -To restore a wallet, the `restorewallet` RPC must be used. +To restore a wallet, the `restorewallet` RPC or the `Restore Wallet` GUI menu item (`File` -> `Restore Wallet…`) must be used. ``` $ bitcoin-cli restorewallet "restored-wallet" /home/node01/Backups/backup-01.dat @@ -144,5 +144,5 @@ unforeseen configurations which result in some scripts being excluded. If a migr unexpectedly or otherwise misses any scripts, please create an issue on GitHub. A backup of the original wallet can be found in the wallet directory with the name `<name>-<timestamp>.legacy.bak`. -The backup can be restored using the `restorewallet` command as discussed in the -[Restoring the Wallet From a Backup](#16-restoring-the-wallet-from-a-backup) section +The backup can be restored using the methods discussed in the +[Restoring the Wallet From a Backup](#16-restoring-the-wallet-from-a-backup) section. diff --git a/doc/reduce-memory.md b/doc/reduce-memory.md index 296b172bde..25205258b8 100644 --- a/doc/reduce-memory.md +++ b/doc/reduce-memory.md @@ -16,11 +16,11 @@ The size of some in-memory caches can be reduced. As caches trade off memory usa - The minimum value for `-maxmempool` is 5. - A lower maximum mempool size means that transactions will be evicted sooner. This will affect any uses of `bitcoind` that process unconfirmed transactions. -- To completely disable mempool functionality there is the option `-blocksonly`. This will make the client opt out of receiving (and thus relaying) transactions completely, except as part of blocks. +- Since `0.14.0`, unused memory allocated to the mempool (default: 300MB) is shared with the UTXO cache, so when trying to reduce memory usage you should limit the mempool, with the `-maxmempool` command line argument. - - Do not use this when using the client to broadcast transactions as any transaction sent will stick out like a sore thumb, affecting privacy. When used with the wallet it should be combined with `-walletbroadcast=0` and `-spendzeroconfchange=0`. Another mechanism for broadcasting outgoing transactions (if any) should be used. +- To disable most of the mempool functionality there is the `-blocksonly` option. This will reduce the default memory usage to 5MB and make the client opt out of receiving (and thus relaying) transactions, except from peers who have the `relay` permission set (e.g. whitelisted peers), and as part of blocks. -- Since `0.14.0`, unused memory allocated to the mempool (default: 300MB) is shared with the UTXO cache, so when trying to reduce memory usage you should limit the mempool, with the `-maxmempool` command line argument. + - Do not use this when using the client to broadcast transactions as any transaction sent will stick out like a sore thumb, affecting privacy. When used with the wallet it should be combined with `-walletbroadcast=0` and `-spendzeroconfchange=0`. Another mechanism for broadcasting outgoing transactions (if any) should be used. ## Number of peers diff --git a/doc/release-notes-23395.md b/doc/release-notes-23395.md new file mode 100644 index 0000000000..b9d7d9409c --- /dev/null +++ b/doc/release-notes-23395.md @@ -0,0 +1,8 @@ +Notable changes +=============== + +New settings +------------ + +- The `shutdownnotify` option is used to specify a command to execute synchronously +before Bitcoin Core has begun its shutdown sequence. (#23395) diff --git a/doc/release-notes-25375.md b/doc/release-notes-25375.md new file mode 100644 index 0000000000..504a2644f4 --- /dev/null +++ b/doc/release-notes-25375.md @@ -0,0 +1,11 @@ +Updated RPCs +-------- + +The `minconf` option, which allows a user to specify the minimum number +of confirmations a UTXO being spent has, and the `maxconf` option, +which allows specifying the maximum number of confirmations, have been +added to the following RPCs: +- `fundrawtransaction` +- `send` +- `walletcreatefundedpsbt` +- `sendall` diff --git a/doc/release-notes-25957.md b/doc/release-notes-25957.md new file mode 100644 index 0000000000..c71afa2c2e --- /dev/null +++ b/doc/release-notes-25957.md @@ -0,0 +1,9 @@ +Wallet +------ + +- Rescans for descriptor wallets are now significantly faster if compact + block filters (BIP158) are available. Since those are not constructed + by default, the configuration option "-blockfilterindex=1" has to be + provided to take advantage of the optimization. This improves the + performance of the RPC calls `rescanblockchain`, `importdescriptors` + and `restorewallet`. (#25957) diff --git a/doc/release-notes-26471.md b/doc/release-notes-26471.md new file mode 100644 index 0000000000..2cb74804ca --- /dev/null +++ b/doc/release-notes-26471.md @@ -0,0 +1,13 @@ +Updated settings +---------------- + +- Setting `-blocksonly` will now reduce the maximum mempool memory + to 5MB (users may still use `-maxmempool` to override). Previously, + the default 300MB would be used, leading to unexpected memory usage + for users running with `-blocksonly` expecting it to eliminate + mempool memory usage. + + As unused mempool memory is shared with dbcache, this also reduces + the dbcache size for users running with `-blocksonly`, potentially + impacting performance. + diff --git a/release-notes-26646.md b/doc/release-notes-26646.md index 7f94505a01..7f94505a01 100644 --- a/release-notes-26646.md +++ b/doc/release-notes-26646.md diff --git a/doc/release-notes-26896.md b/doc/release-notes-26896.md new file mode 100644 index 0000000000..ff4ab44e27 --- /dev/null +++ b/doc/release-notes-26896.md @@ -0,0 +1,7 @@ +Build System +------------ + +The --enable-upnp-default and --enable-natpmp-default options +have been removed. If you want to use port mapping, you can +configure it using a .conf file, or by passing the relevant +options at runtime.
\ No newline at end of file diff --git a/src/.clang-tidy b/src/.clang-tidy index bea5b53668..b2c1b49588 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -9,22 +9,12 @@ performance-for-range-copy, performance-move-const-arg, performance-no-automatic-move, performance-unnecessary-copy-initialization, +readability-const-return-type, readability-redundant-declaration, readability-redundant-string-init, ' -WarningsAsErrors: ' -bugprone-argument-comment, -bugprone-use-after-move, -misc-unused-using-decls, -modernize-use-default-member-init, -modernize-use-nullptr, -performance-for-range-copy, -performance-move-const-arg, -performance-no-automatic-move, -performance-unnecessary-copy-initialization, -readability-redundant-declaration, -readability-redundant-string-init, -' +WarningsAsErrors: '*' CheckOptions: - key: performance-move-const-arg.CheckTriviallyCopyableMove value: false +HeaderFilterRegex: '.' diff --git a/src/Makefile.am b/src/Makefile.am index e73245daeb..5830090ada 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -20,6 +20,7 @@ noinst_LTLIBRARIES = bin_PROGRAMS = noinst_PROGRAMS = +check_PROGRAMS = TESTS = BENCHMARKS = @@ -177,6 +178,7 @@ BITCOIN_CORE_H = \ kernel/checks.h \ kernel/coinstats.h \ kernel/context.h \ + kernel/cs_main.h \ kernel/mempool_entry.h \ kernel/mempool_limits.h \ kernel/mempool_options.h \ @@ -375,6 +377,7 @@ libbitcoin_node_a_SOURCES = \ kernel/checks.cpp \ kernel/coinstats.cpp \ kernel/context.cpp \ + kernel/cs_main.cpp \ kernel/mempool_persist.cpp \ mapport.cpp \ net.cpp \ @@ -653,8 +656,9 @@ libbitcoin_common_a_SOURCES = \ policy/policy.cpp \ protocol.cpp \ psbt.cpp \ - rpc/rawtransaction_util.cpp \ rpc/external_signer.cpp \ + rpc/rawtransaction_util.cpp \ + rpc/request.cpp \ rpc/util.cpp \ scheduler.cpp \ script/descriptor.cpp \ @@ -682,7 +686,6 @@ libbitcoin_util_a_SOURCES = \ logging.cpp \ random.cpp \ randomenv.cpp \ - rpc/request.cpp \ support/cleanse.cpp \ sync.cpp \ util/asmap.cpp \ @@ -906,6 +909,7 @@ libbitcoinkernel_la_SOURCES = \ kernel/checks.cpp \ kernel/coinstats.cpp \ kernel/context.cpp \ + kernel/cs_main.cpp \ kernel/mempool_persist.cpp \ key.cpp \ logging.cpp \ diff --git a/src/Makefile.minisketch.include b/src/Makefile.minisketch.include index b337f48349..1363bec34e 100644 --- a/src/Makefile.minisketch.include +++ b/src/Makefile.minisketch.include @@ -31,7 +31,7 @@ if ENABLE_TESTS if !ENABLE_FUZZ MINISKETCH_TEST = minisketch/test TESTS += $(MINISKETCH_TEST) -noinst_PROGRAMS += $(MINISKETCH_TEST) +check_PROGRAMS += $(MINISKETCH_TEST) minisketch_test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT) minisketch_test_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 1a29e9a47a..4d867fdc2f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -293,6 +293,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/parse_numbers.cpp \ test/fuzz/parse_script.cpp \ test/fuzz/parse_univalue.cpp \ + test/fuzz/partially_downloaded_block.cpp \ test/fuzz/policy_estimator.cpp \ test/fuzz/policy_estimator_io.cpp \ test/fuzz/pow.cpp \ diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index a4e8b3f842..8496b3698a 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -10,6 +10,7 @@ EXTRA_LIBRARIES += \ TEST_UTIL_H = \ test/util/blockfilter.h \ test/util/chainstate.h \ + test/util/json.h \ test/util/logging.h \ test/util/mining.h \ test/util/net.h \ @@ -28,6 +29,7 @@ libtest_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libtest_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_util_a_SOURCES = \ test/util/blockfilter.cpp \ + test/util/json.cpp \ test/util/logging.cpp \ test/util/mining.cpp \ test/util/net.cpp \ diff --git a/src/addrdb.cpp b/src/addrdb.cpp index d95c07d6a8..7be13c8f1e 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -34,10 +34,9 @@ bool SerializeDB(Stream& stream, const Data& data) { // Write and commit header, data try { - CHashWriter hasher(stream.GetType(), stream.GetVersion()); - stream << Params().MessageStart() << data; - hasher << Params().MessageStart() << data; - stream << hasher.GetHash(); + HashedSourceWriter hashwriter{stream}; + hashwriter << Params().MessageStart() << data; + stream << hashwriter.GetHash(); } catch (const std::exception& e) { return error("%s: Serialize or I/O error - %s", __func__, e.what()); } @@ -191,7 +190,7 @@ std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, con const auto path_addr{args.GetDataDirNet() / "peers.dat"}; try { DeserializeFileDB(path_addr, *addrman, CLIENT_VERSION); - LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->size(), Ticks<std::chrono::milliseconds>(SteadyClock::now() - start)); + LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->Size(), Ticks<std::chrono::milliseconds>(SteadyClock::now() - start)); } catch (const DbNotFoundError&) { // Addrman can be in an inconsistent state after failure, reset it addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); diff --git a/src/addrman.cpp b/src/addrman.cpp index 91eedeebe1..a740760faf 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -291,6 +291,7 @@ void AddrManImpl::Unserialize(Stream& s_) mapAddr[info] = n; info.nRandomPos = vRandom.size(); vRandom.push_back(n); + m_network_counts[info.GetNetwork()].n_new++; } nIdCount = nNew; @@ -310,6 +311,7 @@ void AddrManImpl::Unserialize(Stream& s_) mapAddr[info] = nIdCount; vvTried[nKBucket][nKBucketPos] = nIdCount; nIdCount++; + m_network_counts[info.GetNetwork()].n_tried++; } else { nLost++; } @@ -425,6 +427,8 @@ AddrInfo* AddrManImpl::Create(const CAddress& addr, const CNetAddr& addrSource, mapAddr[addr] = nId; mapInfo[nId].nRandomPos = vRandom.size(); vRandom.push_back(nId); + nNew++; + m_network_counts[addr.GetNetwork()].n_new++; if (pnId) *pnId = nId; return &mapInfo[nId]; @@ -464,6 +468,7 @@ void AddrManImpl::Delete(int nId) assert(info.nRefCount == 0); SwapRandom(info.nRandomPos, vRandom.size() - 1); + m_network_counts[info.GetNetwork()].n_new--; vRandom.pop_back(); mapAddr.erase(info); mapInfo.erase(nId); @@ -504,6 +509,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) } } nNew--; + m_network_counts[info.GetNetwork()].n_new--; assert(info.nRefCount == 0); @@ -522,6 +528,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) infoOld.fInTried = false; vvTried[nKBucket][nKBucketPos] = -1; nTried--; + m_network_counts[infoOld.GetNetwork()].n_tried--; // find which new bucket it belongs to int nUBucket = infoOld.GetNewBucket(nKey, m_netgroupman); @@ -533,6 +540,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) infoOld.nRefCount = 1; vvNew[nUBucket][nUBucketPos] = nIdEvict; nNew++; + m_network_counts[infoOld.GetNetwork()].n_new++; LogPrint(BCLog::ADDRMAN, "Moved %s from tried[%i][%i] to new[%i][%i] to make space\n", infoOld.ToString(), nKBucket, nKBucketPos, nUBucket, nUBucketPos); } @@ -541,6 +549,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) vvTried[nKBucket][nKBucketPos] = nId; nTried++; info.fInTried = true; + m_network_counts[info.GetNetwork()].n_tried++; } bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, std::chrono::seconds time_penalty) @@ -591,7 +600,6 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, std::c } else { pinfo = Create(addr, source, &nId); pinfo->nTime = std::max(NodeSeconds{0s}, pinfo->nTime - time_penalty); - nNew++; } int nUBucket = pinfo->GetNewBucket(nKey, source, m_netgroupman); @@ -962,6 +970,28 @@ std::optional<AddressPosition> AddrManImpl::FindAddressEntry_(const CAddress& ad } } +size_t AddrManImpl::Size_(std::optional<Network> net, std::optional<bool> in_new) const +{ + AssertLockHeld(cs); + + if (!net.has_value()) { + if (in_new.has_value()) { + return *in_new ? nNew : nTried; + } else { + return vRandom.size(); + } + } + if (auto it = m_network_counts.find(*net); it != m_network_counts.end()) { + auto net_count = it->second; + if (in_new.has_value()) { + return *in_new ? net_count.n_new : net_count.n_tried; + } else { + return net_count.n_new + net_count.n_tried; + } + } + return 0; +} + void AddrManImpl::Check() const { AssertLockHeld(cs); @@ -986,6 +1016,7 @@ int AddrManImpl::CheckAddrman() const std::unordered_set<int> setTried; std::unordered_map<int, int> mapNew; + std::unordered_map<Network, NewTriedCount> local_counts; if (vRandom.size() != (size_t)(nTried + nNew)) return -7; @@ -1000,12 +1031,14 @@ int AddrManImpl::CheckAddrman() const if (info.nRefCount) return -2; setTried.insert(n); + local_counts[info.GetNetwork()].n_tried++; } else { if (info.nRefCount < 0 || info.nRefCount > ADDRMAN_NEW_BUCKETS_PER_ADDRESS) return -3; if (!info.nRefCount) return -4; mapNew[n] = info.nRefCount; + local_counts[info.GetNetwork()].n_new++; } const auto it{mapAddr.find(info)}; if (it == mapAddr.end() || it->second != n) { @@ -1065,13 +1098,27 @@ int AddrManImpl::CheckAddrman() const if (nKey.IsNull()) return -16; + // It's possible that m_network_counts may have all-zero entries that local_counts + // doesn't have if addrs from a network were being added and then removed again in the past. + if (m_network_counts.size() < local_counts.size()) { + return -20; + } + for (const auto& [net, count] : m_network_counts) { + if (local_counts[net].n_new != count.n_new || local_counts[net].n_tried != count.n_tried) { + return -21; + } + } + return 0; } -size_t AddrManImpl::size() const +size_t AddrManImpl::Size(std::optional<Network> net, std::optional<bool> in_new) const { - LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead - return vRandom.size(); + LOCK(cs); + Check(); + auto ret = Size_(net, in_new); + Check(); + return ret; } bool AddrManImpl::Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) @@ -1178,17 +1225,16 @@ void AddrMan::Unserialize(Stream& s_) } // explicit instantiation -template void AddrMan::Serialize(CHashWriter& s) const; -template void AddrMan::Serialize(CAutoFile& s) const; +template void AddrMan::Serialize(HashedSourceWriter<CAutoFile>& s) const; template void AddrMan::Serialize(CDataStream& s) const; template void AddrMan::Unserialize(CAutoFile& s); template void AddrMan::Unserialize(CHashVerifier<CAutoFile>& s); template void AddrMan::Unserialize(CDataStream& s); template void AddrMan::Unserialize(CHashVerifier<CDataStream>& s); -size_t AddrMan::size() const +size_t AddrMan::Size(std::optional<Network> net, std::optional<bool> in_new) const { - return m_impl->size(); + return m_impl->Size(net, in_new); } bool AddrMan::Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) diff --git a/src/addrman.h b/src/addrman.h index 0f1f808fa1..4985fc764c 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -99,8 +99,14 @@ public: template <typename Stream> void Unserialize(Stream& s_); - //! Return the number of (unique) addresses in all tables. - size_t size() const; + /** + * Return size information about addrman. + * + * @param[in] net Select addresses only from specified network (nullopt = all) + * @param[in] in_new Select addresses only from one table (true = new, false = tried, nullopt = both) + * @return Number of unique addresses that match specified options. + */ + size_t Size(std::optional<Network> net = std::nullopt, std::optional<bool> in_new = std::nullopt) const; /** * Attempt to add one or more addresses to addrman's new table. diff --git a/src/addrman_impl.h b/src/addrman_impl.h index 39754b673e..94fe81aca9 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -112,7 +112,7 @@ public: template <typename Stream> void Unserialize(Stream& s_) EXCLUSIVE_LOCKS_REQUIRED(!cs); - size_t size() const EXCLUSIVE_LOCKS_REQUIRED(!cs); + size_t Size(std::optional<Network> net, std::optional<bool> in_new) const EXCLUSIVE_LOCKS_REQUIRED(!cs); bool Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) EXCLUSIVE_LOCKS_REQUIRED(!cs); @@ -215,6 +215,14 @@ private: /** Reference to the netgroup manager. netgroupman must be constructed before addrman and destructed after. */ const NetGroupManager& m_netgroupman; + struct NewTriedCount { + size_t n_new; + size_t n_tried; + }; + + /** Number of entries in addrman per network and new/tried table. */ + std::unordered_map<Network, NewTriedCount> m_network_counts GUARDED_BY(cs); + //! Find an entry. AddrInfo* Find(const CService& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -257,6 +265,8 @@ private: std::optional<AddressPosition> FindAddressEntry_(const CAddress& addr) EXCLUSIVE_LOCKS_REQUIRED(cs); + size_t Size_(std::optional<Network> net, std::optional<bool> in_new) const EXCLUSIVE_LOCKS_REQUIRED(cs); + //! Consistency check, taking into account m_consistency_check_ratio. //! Will std::abort if an inconsistency is detected. void Check() const EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/arith_uint256.h b/src/arith_uint256.h index a6065dd9bd..c710fe9471 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -58,7 +58,7 @@ public: explicit base_uint(const std::string& str); - const base_uint operator~() const + base_uint operator~() const { base_uint ret; for (int i = 0; i < WIDTH; i++) @@ -66,7 +66,7 @@ public: return ret; } - const base_uint operator-() const + base_uint operator-() const { base_uint ret; for (int i = 0; i < WIDTH; i++) @@ -171,7 +171,7 @@ public: return *this; } - const base_uint operator++(int) + base_uint operator++(int) { // postfix operator const base_uint ret = *this; @@ -188,7 +188,7 @@ public: return *this; } - const base_uint operator--(int) + base_uint operator--(int) { // postfix operator const base_uint ret = *this; @@ -199,16 +199,16 @@ public: int CompareTo(const base_uint& b) const; bool EqualTo(uint64_t b) const; - friend inline const base_uint operator+(const base_uint& a, const base_uint& b) { return base_uint(a) += b; } - friend inline const base_uint operator-(const base_uint& a, const base_uint& b) { return base_uint(a) -= b; } - friend inline const base_uint operator*(const base_uint& a, const base_uint& b) { return base_uint(a) *= b; } - friend inline const base_uint operator/(const base_uint& a, const base_uint& b) { return base_uint(a) /= b; } - friend inline const base_uint operator|(const base_uint& a, const base_uint& b) { return base_uint(a) |= b; } - friend inline const base_uint operator&(const base_uint& a, const base_uint& b) { return base_uint(a) &= b; } - friend inline const base_uint operator^(const base_uint& a, const base_uint& b) { return base_uint(a) ^= b; } - friend inline const base_uint operator>>(const base_uint& a, int shift) { return base_uint(a) >>= shift; } - friend inline const base_uint operator<<(const base_uint& a, int shift) { return base_uint(a) <<= shift; } - friend inline const base_uint operator*(const base_uint& a, uint32_t b) { return base_uint(a) *= b; } + friend inline base_uint operator+(const base_uint& a, const base_uint& b) { return base_uint(a) += b; } + friend inline base_uint operator-(const base_uint& a, const base_uint& b) { return base_uint(a) -= b; } + friend inline base_uint operator*(const base_uint& a, const base_uint& b) { return base_uint(a) *= b; } + friend inline base_uint operator/(const base_uint& a, const base_uint& b) { return base_uint(a) /= b; } + friend inline base_uint operator|(const base_uint& a, const base_uint& b) { return base_uint(a) |= b; } + friend inline base_uint operator&(const base_uint& a, const base_uint& b) { return base_uint(a) &= b; } + friend inline base_uint operator^(const base_uint& a, const base_uint& b) { return base_uint(a) ^= b; } + friend inline base_uint operator>>(const base_uint& a, int shift) { return base_uint(a) >>= shift; } + friend inline base_uint operator<<(const base_uint& a, int shift) { return base_uint(a) <<= shift; } + friend inline base_uint operator*(const base_uint& a, uint32_t b) { return base_uint(a) *= b; } friend inline bool operator==(const base_uint& a, const base_uint& b) { return memcmp(a.pn, b.pn, sizeof(a.pn)) == 0; } friend inline bool operator!=(const base_uint& a, const base_uint& b) { return memcmp(a.pn, b.pn, sizeof(a.pn)) != 0; } friend inline bool operator>(const base_uint& a, const base_uint& b) { return a.CompareTo(b) > 0; } diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 63acb6c24d..8dd4117a3e 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -5,6 +5,7 @@ #include <bench/bench.h> #include <consensus/validation.h> #include <crypto/sha256.h> +#include <node/miner.h> #include <test/util/mining.h> #include <test/util/script.h> #include <test/util/setup_common.h> @@ -45,5 +46,18 @@ static void AssembleBlock(benchmark::Bench& bench) PrepareBlock(test_setup->m_node, P2WSH_OP_TRUE); }); } +static void BlockAssemblerAddPackageTxns(benchmark::Bench& bench) +{ + FastRandomContext det_rand{true}; + auto testing_setup{MakeNoLogFileContext<TestChain100Setup>()}; + testing_setup->PopulateMempool(det_rand, /*num_transactions=*/1000, /*submit=*/true); + node::BlockAssembler::Options assembler_options; + assembler_options.test_block_validity = false; + + bench.run([&] { + PrepareBlock(testing_setup->m_node, P2WSH_OP_TRUE, assembler_options); + }); +} BENCHMARK(AssembleBlock, benchmark::PriorityLevel::HIGH); +BENCHMARK(BlockAssemblerAddPackageTxns, benchmark::PriorityLevel::LOW); diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 5243391d9e..4a3ec67c2b 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -18,7 +18,6 @@ // (https://github.com/bitcoin/bitcoin/issues/7883#issuecomment-224807484) static void CCoinsCaching(benchmark::Bench& bench) { - const ECCVerifyHandle verify_handle; ECC_Start(); FillableSigningProvider keystore; diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 8df25d94c3..dfd7275f46 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -25,7 +25,6 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) // We shouldn't ever be running with the checkqueue on a single core machine. if (GetNumCores() <= 1) return; - const ECCVerifyHandle verify_handle; ECC_Start(); struct PrevectorJob { diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index bd524e7458..cf8d807d7b 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -18,7 +18,7 @@ /* Number of bytes to hash per iteration */ static const uint64_t BUFFER_SIZE = 1000*1000; -static void RIPEMD160(benchmark::Bench& bench) +static void BenchRIPEMD160(benchmark::Bench& bench) { uint8_t hash[CRIPEMD160::OUTPUT_SIZE]; std::vector<uint8_t> in(BUFFER_SIZE,0); @@ -150,7 +150,7 @@ static void MuHashPrecompute(benchmark::Bench& bench) }); } -BENCHMARK(RIPEMD160, benchmark::PriorityLevel::HIGH); +BENCHMARK(BenchRIPEMD160, benchmark::PriorityLevel::HIGH); BENCHMARK(SHA1, benchmark::PriorityLevel::HIGH); BENCHMARK(SHA256, benchmark::PriorityLevel::HIGH); BENCHMARK(SHA512, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/descriptors.cpp b/src/bench/descriptors.cpp index 40a0a20db8..5d28d26909 100644 --- a/src/bench/descriptors.cpp +++ b/src/bench/descriptors.cpp @@ -13,7 +13,6 @@ static void ExpandDescriptor(benchmark::Bench& bench) { - const ECCVerifyHandle verify_handle; ECC_Start(); const auto desc_str = "sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))"; diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp index 51fbe15760..0af4ee98fe 100644 --- a/src/bench/gcs_filter.cpp +++ b/src/bench/gcs_filter.cpp @@ -5,7 +5,7 @@ #include <bench/bench.h> #include <blockfilter.h> -static const GCSFilter::ElementSet GenerateGCSTestElements() +static GCSFilter::ElementSet GenerateGCSTestElements() { GCSFilter::ElementSet elements; diff --git a/src/bench/load_external.cpp b/src/bench/load_external.cpp index be01b2a483..0fd842c7c3 100644 --- a/src/bench/load_external.cpp +++ b/src/bench/load_external.cpp @@ -27,7 +27,7 @@ static void LoadExternalBlockFile(benchmark::Bench& bench) // Create a single block as in the blocks files (magic bytes, block size, // block data) as a stream object. const fs::path blkfile{testing_setup.get()->m_path_root / "blk.dat"}; - CDataStream ss(SER_DISK, 0); + DataStream ss{}; auto params{testing_setup->m_node.chainman->GetParams()}; ss << params.MessageStart(); ss << static_cast<uint32_t>(benchmark::data::block413567.size()); diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index ef1ea1162b..59c4af086e 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -61,7 +61,7 @@ static void PrevectorResize(benchmark::Bench& bench) template <typename T> static void PrevectorDeserialize(benchmark::Bench& bench) { - CDataStream s0(SER_NETWORK, 0); + DataStream s0{}; prevector<28, T> t0; t0.resize(28); for (auto x = 0; x < 900; ++x) { diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index 3d82e495fa..e3e1a07c83 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <chainparamsbase.h> +#include <kernel/cs_main.h> #include <kernel/mempool_entry.h> #include <rpc/mempool.h> #include <test/util/setup_common.h> diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 2c1abf0846..757094167a 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -18,7 +18,6 @@ // modified to measure performance of other types of scripts. static void VerifyScriptBench(benchmark::Bench& bench) { - const ECCVerifyHandle verify_handle; ECC_Start(); const uint32_t flags{SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH}; diff --git a/src/bench/wallet_loading.cpp b/src/bench/wallet_loading.cpp index 2f7dc53b0c..6b09adcc9d 100644 --- a/src/bench/wallet_loading.cpp +++ b/src/bench/wallet_loading.cpp @@ -24,7 +24,7 @@ using wallet::WALLET_FLAG_DESCRIPTORS; using wallet::WalletContext; using wallet::WalletDatabase; -static const std::shared_ptr<CWallet> BenchLoadWallet(std::unique_ptr<WalletDatabase> database, WalletContext& context, DatabaseOptions& options) +static std::shared_ptr<CWallet> BenchLoadWallet(std::unique_ptr<WalletDatabase> database, WalletContext& context, DatabaseOptions& options) { bilingual_str error; std::vector<bilingual_str> warnings; diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index dc30383167..57ca2bbe8a 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -681,8 +681,6 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) class Secp256k1Init { - ECCVerifyHandle globalVerifyHandle; - public: Secp256k1Init() { ECC_Start(); diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp index 7327875b64..61d4b9c6f1 100644 --- a/src/bitcoin-util.cpp +++ b/src/bitcoin-util.cpp @@ -139,7 +139,7 @@ static int Grind(const std::vector<std::string>& args, std::string& strPrint) return EXIT_FAILURE; } - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + DataStream ss{}; ss << header; strPrint = HexStr(ss); return EXIT_SUCCESS; diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 2d94be0192..12cb60e6de 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -130,7 +130,6 @@ MAIN_FUNCTION return EXIT_FAILURE; } - ECCVerifyHandle globalVerifyHandle; ECC_Start(); if (!wallet::WalletTool::ExecuteWalletToolFunc(args, command->command)) { return EXIT_FAILURE; diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index bcb86d75cc..a29e4f794e 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -29,7 +29,7 @@ CBlockHeaderAndShortTxIDs::CBlockHeaderAndShortTxIDs(const CBlock& block) : } void CBlockHeaderAndShortTxIDs::FillShortTxIDSelector() const { - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << header << nonce; CSHA256 hasher; hasher.Write((unsigned char*)&(*stream.begin()), stream.end() - stream.begin()); @@ -52,7 +52,8 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c if (cmpctblock.shorttxids.size() + cmpctblock.prefilledtxn.size() > MAX_BLOCK_WEIGHT / MIN_SERIALIZABLE_TRANSACTION_WEIGHT) return READ_STATUS_INVALID; - assert(header.IsNull() && txn_available.empty()); + if (!header.IsNull() || !txn_available.empty()) return READ_STATUS_INVALID; + header = cmpctblock.header; txn_available.resize(cmpctblock.BlockTxCount()); @@ -167,14 +168,18 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c return READ_STATUS_OK; } -bool PartiallyDownloadedBlock::IsTxAvailable(size_t index) const { - assert(!header.IsNull()); +bool PartiallyDownloadedBlock::IsTxAvailable(size_t index) const +{ + if (header.IsNull()) return false; + assert(index < txn_available.size()); return txn_available[index] != nullptr; } -ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector<CTransactionRef>& vtx_missing) { - assert(!header.IsNull()); +ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector<CTransactionRef>& vtx_missing) +{ + if (header.IsNull()) return READ_STATUS_INVALID; + uint256 hash = header.GetHash(); block = header; block.vtx.resize(txn_available.size()); @@ -197,7 +202,8 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< return READ_STATUS_INVALID; BlockValidationState state; - if (!CheckBlock(block, state, Params().GetConsensus())) { + CheckBlockFn check_block = m_check_block_mock ? m_check_block_mock : CheckBlock; + if (!check_block(block, state, Params().GetConsensus(), /*fCheckPoW=*/true, /*fCheckMerkleRoot=*/true)) { // TODO: We really want to just check merkle tree manually here, // but that is expensive, and CheckBlock caches a block's // "checked-status" (in the CBlock?). CBlock should be able to diff --git a/src/blockencodings.h b/src/blockencodings.h index e60c1e3db4..7207ff1ae2 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -7,8 +7,13 @@ #include <primitives/block.h> +#include <functional> class CTxMemPool; +class BlockValidationState; +namespace Consensus { +struct Params; +}; // Transaction compression schemes for compact block relay can be introduced by writing // an actual formatter here. @@ -129,6 +134,11 @@ protected: const CTxMemPool* pool; public: CBlockHeader header; + + // Can be overriden for testing + using CheckBlockFn = std::function<bool(const CBlock&, BlockValidationState&, const Consensus::Params&, bool, bool)>; + CheckBlockFn m_check_block_mock{nullptr}; + explicit PartiallyDownloadedBlock(CTxMemPool* poolIn) : pool(poolIn) {} // extra_txn is a list of extra transactions to look at, in <witness hash, reference> form diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index fc6dde20f9..88c7526b9e 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -247,21 +247,10 @@ bool BlockFilter::BuildParams(GCSFilter::Params& params) const uint256 BlockFilter::GetHash() const { - const std::vector<unsigned char>& data = GetEncodedFilter(); - - uint256 result; - CHash256().Write(data).Finalize(result); - return result; + return Hash(GetEncodedFilter()); } uint256 BlockFilter::ComputeHeader(const uint256& prev_header) const { - const uint256& filter_hash = GetHash(); - - uint256 result; - CHash256() - .Write(filter_hash) - .Write(prev_header) - .Finalize(result); - return result; + return Hash(GetHash(), prev_header); } diff --git a/src/chain.h b/src/chain.h index fbbb715986..f5dd0fd315 100644 --- a/src/chain.h +++ b/src/chain.h @@ -9,6 +9,7 @@ #include <arith_uint256.h> #include <consensus/params.h> #include <flatfile.h> +#include <kernel/cs_main.h> #include <primitives/block.h> #include <sync.h> #include <uint256.h> @@ -38,8 +39,6 @@ static constexpr int64_t TIMESTAMP_WINDOW = MAX_FUTURE_BLOCK_TIME; */ static constexpr int64_t MAX_BLOCK_TIME_GAP = 90 * 60; -extern RecursiveMutex cs_main; - class CBlockFileInfo { public: diff --git a/src/coins.cpp b/src/coins.cpp index 976118e23c..e98bf816ab 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -13,7 +13,7 @@ bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } std::vector<uint256> CCoinsView::GetHeadBlocks() const { return std::vector<uint256>(); } -bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return false; } +bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, bool erase) { return false; } std::unique_ptr<CCoinsViewCursor> CCoinsView::Cursor() const { return nullptr; } bool CCoinsView::HaveCoin(const COutPoint &outpoint) const @@ -28,11 +28,11 @@ bool CCoinsViewBacked::HaveCoin(const COutPoint &outpoint) const { return base-> uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } std::vector<uint256> CCoinsViewBacked::GetHeadBlocks() const { return base->GetHeadBlocks(); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } -bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } +bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, bool erase) { return base->BatchWrite(mapCoins, hashBlock, erase); } std::unique_ptr<CCoinsViewCursor> CCoinsViewBacked::Cursor() const { return base->Cursor(); } size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); } -CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) {} +CCoinsViewCache::CCoinsViewCache(CCoinsView* baseIn) : CCoinsViewBacked(baseIn) {} size_t CCoinsViewCache::DynamicMemoryUsage() const { return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; @@ -176,8 +176,10 @@ void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) { hashBlock = hashBlockIn; } -bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn) { - for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); it = mapCoins.erase(it)) { +bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn, bool erase) { + for (CCoinsMap::iterator it = mapCoins.begin(); + it != mapCoins.end(); + it = erase ? mapCoins.erase(it) : std::next(it)) { // Ignore non-dirty entries (optimization). if (!(it->second.flags & CCoinsCacheEntry::DIRTY)) { continue; @@ -190,7 +192,14 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn // Create the coin in the parent cache, move the data up // and mark it as dirty. CCoinsCacheEntry& entry = cacheCoins[it->first]; - entry.coin = std::move(it->second.coin); + if (erase) { + // The `move` call here is purely an optimization; we rely on the + // `mapCoins.erase` call in the `for` expression to actually remove + // the entry from the child map. + entry.coin = std::move(it->second.coin); + } else { + entry.coin = it->second.coin; + } cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); entry.flags = CCoinsCacheEntry::DIRTY; // We can mark it FRESH in the parent if it was FRESH in the child @@ -218,7 +227,14 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn } else { // A normal modification. cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); - itUs->second.coin = std::move(it->second.coin); + if (erase) { + // The `move` call here is purely an optimization; we rely on the + // `mapCoins.erase` call in the `for` expression to actually remove + // the entry from the child map. + itUs->second.coin = std::move(it->second.coin); + } else { + itUs->second.coin = it->second.coin; + } cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; // NOTE: It isn't safe to mark the coin as FRESH in the parent @@ -233,12 +249,32 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn } bool CCoinsViewCache::Flush() { - bool fOk = base->BatchWrite(cacheCoins, hashBlock); - cacheCoins.clear(); + bool fOk = base->BatchWrite(cacheCoins, hashBlock, /*erase=*/true); + if (fOk && !cacheCoins.empty()) { + /* BatchWrite must erase all cacheCoins elements when erase=true. */ + throw std::logic_error("Not all cached coins were erased"); + } cachedCoinsUsage = 0; return fOk; } +bool CCoinsViewCache::Sync() +{ + bool fOk = base->BatchWrite(cacheCoins, hashBlock, /*erase=*/false); + // Instead of clearing `cacheCoins` as we would in Flush(), just clear the + // FRESH/DIRTY flags of any coin that isn't spent. + for (auto it = cacheCoins.begin(); it != cacheCoins.end(); ) { + if (it->second.coin.IsSpent()) { + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + it = cacheCoins.erase(it); + } else { + it->second.flags = 0; + ++it; + } + } + return fOk; +} + void CCoinsViewCache::Uncache(const COutPoint& hash) { CCoinsMap::iterator it = cacheCoins.find(hash); diff --git a/src/coins.h b/src/coins.h index b0d6bdf333..710b8c7c83 100644 --- a/src/coins.h +++ b/src/coins.h @@ -176,7 +176,7 @@ public: //! Do a bulk modification (multiple Coin changes + BestBlock change). //! The passed mapCoins can be modified. - virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); + virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, bool erase = true); //! Get a cursor to iterate over the whole state virtual std::unique_ptr<CCoinsViewCursor> Cursor() const; @@ -202,7 +202,7 @@ public: uint256 GetBestBlock() const override; std::vector<uint256> GetHeadBlocks() const override; void SetBackend(CCoinsView &viewIn); - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, bool erase = true) override; std::unique_ptr<CCoinsViewCursor> Cursor() const override; size_t EstimateSize() const override; }; @@ -220,7 +220,7 @@ protected: mutable CCoinsMap cacheCoins; /* Cached dynamic memory usage for the inner Coin objects. */ - mutable size_t cachedCoinsUsage; + mutable size_t cachedCoinsUsage{0}; public: CCoinsViewCache(CCoinsView *baseIn); @@ -235,7 +235,7 @@ public: bool HaveCoin(const COutPoint &outpoint) const override; uint256 GetBestBlock() const override; void SetBestBlock(const uint256 &hashBlock); - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, bool erase = true) override; std::unique_ptr<CCoinsViewCursor> Cursor() const override { throw std::logic_error("CCoinsViewCache cursor iteration not supported."); } @@ -282,13 +282,23 @@ public: bool SpendCoin(const COutPoint &outpoint, Coin* moveto = nullptr); /** - * Push the modifications applied to this cache to its base. - * Failure to call this method before destruction will cause the changes to be forgotten. + * Push the modifications applied to this cache to its base and wipe local state. + * Failure to call this method or Sync() before destruction will cause the changes + * to be forgotten. * If false is returned, the state of this cache (and its backing view) will be undefined. */ bool Flush(); /** + * Push the modifications applied to this cache to its base while retaining + * the contents of this cache (except for spent coins, which we erase). + * Failure to call this method or Flush() before destruction will cause the changes + * to be forgotten. + * If false is returned, the state of this cache (and its backing view) will be undefined. + */ + bool Sync(); + + /** * Removes the UTXO with the given outpoint from the cache, if it is * not modified. */ diff --git a/src/common/bloom.cpp b/src/common/bloom.cpp index 3ba0414b31..fd3276b5a7 100644 --- a/src/common/bloom.cpp +++ b/src/common/bloom.cpp @@ -60,7 +60,7 @@ void CBloomFilter::insert(Span<const unsigned char> vKey) void CBloomFilter::insert(const COutPoint& outpoint) { - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << outpoint; insert(MakeUCharSpan(stream)); } @@ -81,7 +81,7 @@ bool CBloomFilter::contains(Span<const unsigned char> vKey) const bool CBloomFilter::contains(const COutPoint& outpoint) const { - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << outpoint; return contains(MakeUCharSpan(stream)); } diff --git a/src/core_read.cpp b/src/core_read.cpp index 7bab171c89..84cd559b7f 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -207,7 +207,7 @@ bool DecodeHexBlockHeader(CBlockHeader& header, const std::string& hex_header) if (!IsHex(hex_header)) return false; const std::vector<unsigned char> header_data{ParseHex(hex_header)}; - CDataStream ser_header(header_data, SER_NETWORK, PROTOCOL_VERSION); + DataStream ser_header{header_data}; try { ser_header >> header; } catch (const std::exception&) { diff --git a/src/core_write.cpp b/src/core_write.cpp index 91a6eb2864..300bd30e43 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -170,6 +170,8 @@ void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex, bool i void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo, TxVerbosity verbosity) { + CHECK_NONFATAL(verbosity >= TxVerbosity::SHOW_DETAILS); + entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); // Transaction version is actually unsigned in consensus checks, just signed in memory, diff --git a/src/crypto/ripemd160.cpp b/src/crypto/ripemd160.cpp index 29a4ad906f..a2f7c6e156 100644 --- a/src/crypto/ripemd160.cpp +++ b/src/crypto/ripemd160.cpp @@ -239,7 +239,7 @@ void Transform(uint32_t* s, const unsigned char* chunk) ////// RIPEMD160 -CRIPEMD160::CRIPEMD160() : bytes(0) +CRIPEMD160::CRIPEMD160() { ripemd160::Initialize(s); } diff --git a/src/crypto/ripemd160.h b/src/crypto/ripemd160.h index ae9c339181..fb631a66d2 100644 --- a/src/crypto/ripemd160.h +++ b/src/crypto/ripemd160.h @@ -14,7 +14,7 @@ class CRIPEMD160 private: uint32_t s[5]; unsigned char buf[64]; - uint64_t bytes; + uint64_t bytes{0}; public: static const size_t OUTPUT_SIZE = 20; diff --git a/src/crypto/sha1.cpp b/src/crypto/sha1.cpp index 1fb9bb2b72..2610108f60 100644 --- a/src/crypto/sha1.cpp +++ b/src/crypto/sha1.cpp @@ -146,7 +146,7 @@ void Transform(uint32_t* s, const unsigned char* chunk) ////// SHA1 -CSHA1::CSHA1() : bytes(0) +CSHA1::CSHA1() { sha1::Initialize(s); } diff --git a/src/crypto/sha1.h b/src/crypto/sha1.h index 4bd6c331a8..741cdaad58 100644 --- a/src/crypto/sha1.h +++ b/src/crypto/sha1.h @@ -14,7 +14,7 @@ class CSHA1 private: uint32_t s[5]; unsigned char buf[64]; - uint64_t bytes; + uint64_t bytes{0}; public: static const size_t OUTPUT_SIZE = 20; diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 7cd5b3661b..a4eef36480 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -673,7 +673,7 @@ std::string SHA256AutoDetect() ////// SHA-256 -CSHA256::CSHA256() : bytes(0) +CSHA256::CSHA256() { sha256::Initialize(s); } diff --git a/src/crypto/sha256.h b/src/crypto/sha256.h index 9fd73becfd..7625508665 100644 --- a/src/crypto/sha256.h +++ b/src/crypto/sha256.h @@ -15,7 +15,7 @@ class CSHA256 private: uint32_t s[8]; unsigned char buf[64]; - uint64_t bytes; + uint64_t bytes{0}; public: static const size_t OUTPUT_SIZE = 32; diff --git a/src/crypto/sha512.cpp b/src/crypto/sha512.cpp index 8a822e0e7e..2713f06210 100644 --- a/src/crypto/sha512.cpp +++ b/src/crypto/sha512.cpp @@ -151,7 +151,7 @@ void Transform(uint64_t* s, const unsigned char* chunk) ////// SHA-512 -CSHA512::CSHA512() : bytes(0) +CSHA512::CSHA512() { sha512::Initialize(s); } diff --git a/src/crypto/sha512.h b/src/crypto/sha512.h index d8fa8d2e39..d2f7d6a05e 100644 --- a/src/crypto/sha512.h +++ b/src/crypto/sha512.h @@ -14,7 +14,7 @@ class CSHA512 private: uint64_t s[8]; unsigned char buf[128]; - uint64_t bytes; + uint64_t bytes{0}; public: static constexpr size_t OUTPUT_SIZE = 64; diff --git a/src/cuckoocache.h b/src/cuckoocache.h index 6adcc74516..cb0b362143 100644 --- a/src/cuckoocache.h +++ b/src/cuckoocache.h @@ -166,7 +166,7 @@ private: std::vector<Element> table; /** size stores the total available slots in the hash table */ - uint32_t size; + uint32_t size{0}; /** The bit_packed_atomic_flags array is marked mutable because we want * garbage collection to be allowed to occur from const methods */ @@ -183,7 +183,7 @@ private: * decremented on insert and reset to the new number of inserts which would * cause the epoch to reach epoch_size when it reaches zero. */ - uint32_t epoch_heuristic_counter; + uint32_t epoch_heuristic_counter{0}; /** epoch_size is set to be the number of elements supposed to be in a * epoch. When the number of non-erased elements in an epoch @@ -193,12 +193,12 @@ private: * one "dead" which has been erased, one "dying" which has been marked to be * erased next, and one "living" which new inserts add to. */ - uint32_t epoch_size; + uint32_t epoch_size{0}; /** depth_limit determines how many elements insert should try to replace. * Should be set to log2(n). */ - uint8_t depth_limit; + uint8_t depth_limit{0}; /** hash_function is a const instance of the hash function. It cannot be * static or initialized at call time as it may have internal state (such as @@ -322,8 +322,7 @@ public: /** You must always construct a cache with some elements via a subsequent * call to setup or setup_bytes, otherwise operations may segfault. */ - cache() : table(), size(), collection_flags(0), epoch_flags(), - epoch_heuristic_counter(), epoch_size(), depth_limit(0), hash_function() + cache() : table(), collection_flags(0), epoch_flags(), hash_function() { } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 3d3eee32ce..b389d039fb 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -68,16 +68,16 @@ private: const CDBWrapper &parent; leveldb::WriteBatch batch; - CDataStream ssKey; + DataStream ssKey{}; CDataStream ssValue; - size_t size_estimate; + size_t size_estimate{0}; public: /** * @param[in] _parent CDBWrapper that this batch is to be submitted to */ - explicit CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0) { }; + explicit CDBBatch(const CDBWrapper& _parent) : parent(_parent), ssValue(SER_DISK, CLIENT_VERSION){}; void Clear() { @@ -151,7 +151,7 @@ public: void SeekToFirst(); template<typename K> void Seek(const K& key) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); + DataStream ssKey{}; ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); @@ -163,7 +163,7 @@ public: template<typename K> bool GetKey(K& key) { leveldb::Slice slKey = piter->key(); try { - CDataStream ssKey{MakeByteSpan(slKey), SER_DISK, CLIENT_VERSION}; + DataStream ssKey{MakeByteSpan(slKey)}; ssKey >> key; } catch (const std::exception&) { return false; @@ -247,7 +247,7 @@ public: template <typename K, typename V> bool Read(const K& key, V& value) const { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); + DataStream ssKey{}; ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); @@ -289,7 +289,7 @@ public: template <typename K> bool Exists(const K& key) const { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); + DataStream ssKey{}; ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); @@ -331,7 +331,7 @@ public: template<typename K> size_t EstimateSize(const K& key_begin, const K& key_end) const { - CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION); + DataStream ssKey1{}, ssKey2{}; ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey1 << key_begin; diff --git a/src/external_signer.cpp b/src/external_signer.cpp index 8a3e17a292..5524b943f4 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -16,7 +16,7 @@ ExternalSigner::ExternalSigner(const std::string& command, const std::string chain, const std::string& fingerprint, const std::string name): m_command(command), m_chain(chain), m_fingerprint(fingerprint), m_name(name) {} -const std::string ExternalSigner::NetworkArg() const +std::string ExternalSigner::NetworkArg() const { return " --chain " + m_chain; } diff --git a/src/external_signer.h b/src/external_signer.h index e40fd7f010..90f07478e3 100644 --- a/src/external_signer.h +++ b/src/external_signer.h @@ -24,7 +24,7 @@ private: //! Bitcoin mainnet, testnet, etc std::string m_chain; - const std::string NetworkArg() const; + std::string NetworkArg() const; public: //! @param[in] command the command which handles interaction with the external signer @@ -35,7 +35,7 @@ public: // Allow path objects arguments for compatibility. path(std::filesystem::path path) : std::filesystem::path::path(std::move(path)) {} path& operator=(std::filesystem::path path) { std::filesystem::path::operator=(std::move(path)); return *this; } - path& operator/=(std::filesystem::path path) { std::filesystem::path::operator/=(std::move(path)); return *this; } + path& operator/=(std::filesystem::path path) { std::filesystem::path::operator/=(path); return *this; } // Allow literal string arguments, which are safe as long as the literals are ASCII. path(const char* c) : std::filesystem::path(c) {} diff --git a/src/hash.h b/src/hash.h index b18a031268..2e3ed11b43 100644 --- a/src/hash.h +++ b/src/hash.h @@ -6,11 +6,13 @@ #ifndef BITCOIN_HASH_H #define BITCOIN_HASH_H +#include <attributes.h> #include <crypto/common.h> #include <crypto/ripemd160.h> #include <crypto/sha256.h> #include <prevector.h> #include <serialize.h> +#include <span.h> #include <uint256.h> #include <version.h> @@ -165,6 +167,39 @@ public: }; /** Reads data from an underlying stream, while hashing the read data. */ +template <typename Source> +class HashVerifier : public HashWriter +{ +private: + Source& m_source; + +public: + explicit HashVerifier(Source& source LIFETIMEBOUND) : m_source{source} {} + + void read(Span<std::byte> dst) + { + m_source.read(dst); + this->write(dst); + } + + void ignore(size_t num_bytes) + { + std::byte data[1024]; + while (num_bytes > 0) { + size_t now = std::min<size_t>(num_bytes, 1024); + read({data, now}); + num_bytes -= now; + } + } + + template <typename T> + HashVerifier<Source>& operator>>(T&& obj) + { + ::Unserialize(*this, obj); + return *this; + } +}; + template<typename Source> class CHashVerifier : public CHashWriter { @@ -199,6 +234,30 @@ public: } }; +/** Writes data to an underlying source stream, while hashing the written data. */ +template <typename Source> +class HashedSourceWriter : public CHashWriter +{ +private: + Source& m_source; + +public: + explicit HashedSourceWriter(Source& source LIFETIMEBOUND) : CHashWriter{source.GetType(), source.GetVersion()}, m_source{source} {} + + void write(Span<const std::byte> src) + { + m_source.write(src); + CHashWriter::write(src); + } + + template <typename T> + HashedSourceWriter& operator<<(const T& obj) + { + ::Serialize(*this, obj); + return *this; + } +}; + /** Compute the 256-bit hash of an object's serialization. */ template<typename T> uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL_VERSION) @@ -223,4 +282,12 @@ void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char he */ HashWriter TaggedHash(const std::string& tag); +/** Compute the 160-bit RIPEMD-160 hash of an array. */ +inline uint160 RIPEMD160(Span<const unsigned char> data) +{ + uint160 result; + CRIPEMD160().Write(data.data(), data.size()).Finalize(result.begin()); + return result; +} + #endif // BITCOIN_HASH_H diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 6597ed71af..720f5c9353 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -279,7 +279,7 @@ static void http_reject_request_cb(struct evhttp_request* req, void*) } /** Event dispatcher thread */ -static bool ThreadHTTP(struct event_base* base) +static void ThreadHTTP(struct event_base* base) { util::ThreadRename("http"); SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_HTTP_SERVER); @@ -287,7 +287,6 @@ static bool ThreadHTTP(struct event_base* base) event_base_dispatch(base); // Event loop will be interrupted by InterruptHTTPServer() LogPrint(BCLog::HTTP, "Exited http event loop\n"); - return event_base_got_break(base) == 0; } /** Bind HTTP server to specified addresses */ diff --git a/src/index/base.cpp b/src/index/base.cpp index a8b8cbe8a9..1d5c0dbe24 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -415,8 +415,9 @@ IndexSummary BaseIndex::GetSummary() const return summary; } -void BaseIndex::SetBestBlockIndex(const CBlockIndex* block) { - assert(!node::fPruneMode || AllowPrune()); +void BaseIndex::SetBestBlockIndex(const CBlockIndex* block) +{ + assert(!m_chainstate->m_blockman.IsPruneMode() || AllowPrune()); if (AllowPrune() && block) { node::PruneLockInfo prune_lock; diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index 07b4cdc06b..59bf6d34cf 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -157,9 +157,7 @@ bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, const uint256& std::vector<uint8_t> encoded_filter; try { filein >> block_hash >> encoded_filter; - uint256 result; - CHash256().Write(encoded_filter).Finalize(result); - if (result != hash) return error("Checksum mismatch in filter decode."); + if (Hash(encoded_filter) != hash) return error("Checksum mismatch in filter decode."); filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter), /*skip_decode_check=*/true); } catch (const std::exception& e) { diff --git a/src/init.cpp b/src/init.cpp index 5160718eaa..5b486854e0 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -191,8 +191,24 @@ static fs::path GetPidFile(const ArgsManager& args) // shutdown thing. // +#if HAVE_SYSTEM +static void ShutdownNotify(const ArgsManager& args) +{ + std::vector<std::thread> threads; + for (const auto& cmd : args.GetArgs("-shutdownnotify")) { + threads.emplace_back(runCommand, cmd); + } + for (auto& t : threads) { + t.join(); + } +} +#endif + void Interrupt(NodeContext& node) { +#if HAVE_SYSTEM + ShutdownNotify(*node.args); +#endif InterruptHTTPServer(); InterruptHTTPRPC(); InterruptRPC(); @@ -441,6 +457,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-settings=<file>", strprintf("Specify path to dynamic settings data file. Can be disabled with -nosettings. File is written at runtime and not meant to be edited by users (use %s instead for custom settings). Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME, BITCOIN_SETTINGS_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #if HAVE_SYSTEM argsman.AddArg("-startupnotify=<cmd>", "Execute command on startup.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-shutdownnotify=<cmd>", "Execute command immediately before beginning shutdown. The need for shutdown may be urgent, so be careful not to delay it long (if the command doesn't require interaction with the server, consider having it fork into the background).", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #endif #ifndef WIN32 argsman.AddArg("-sysperms", "Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -586,7 +603,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-rest", strprintf("Accept public REST requests (default: %u)", DEFAULT_REST_ENABLE), ArgsManager::ALLOW_ANY, OptionsCategory::RPC); argsman.AddArg("-rpcallowip=<ip>", "Allow JSON-RPC connections from specified source. Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); argsman.AddArg("-rpcauth=<userpw>", "Username and HMAC-SHA-256 hashed password for JSON-RPC connections. The field <userpw> comes in the format: <USERNAME>:<SALT>$<HASH>. A canonical python script is included in share/rpcauth. The client then connects normally using the rpcuser=<USERNAME>/rpcpassword=<PASSWORD> pair of arguments. This option can be specified multiple times", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::RPC); - argsman.AddArg("-rpcbind=<addr>[:port]", "Bind to given address to listen for JSON-RPC connections. Do not expose the RPC server to untrusted networks such as the public internet! This option is ignored unless -rpcallowip is also passed. Port is optional and overrides -rpcport. Use [host]:port notation for IPv6. This option can be specified multiple times (default: 127.0.0.1 and ::1 i.e., localhost)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY | ArgsManager::SENSITIVE, OptionsCategory::RPC); + argsman.AddArg("-rpcbind=<addr>[:port]", "Bind to given address to listen for JSON-RPC connections. Do not expose the RPC server to untrusted networks such as the public internet! This option is ignored unless -rpcallowip is also passed. Port is optional and overrides -rpcport. Use [host]:port notation for IPv6. This option can be specified multiple times (default: 127.0.0.1 and ::1 i.e., localhost)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::RPC); argsman.AddArg("-rpcdoccheck", strprintf("Throw a non-fatal error at runtime if the documentation for an RPC is incorrect (default: %u)", DEFAULT_RPC_DOC_CHECK), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::RPC); argsman.AddArg("-rpccookiefile=<loc>", "Location of the auth cookie. Relative paths will be prefixed by a net-specific datadir location. (default: data dir)", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); argsman.AddArg("-rpcpassword=<pw>", "Password for JSON-RPC connections", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::RPC); @@ -719,10 +736,13 @@ void InitParameterInteraction(ArgsManager& args) LogPrintf("%s: parameter interaction: -externalip set -> setting -discover=0\n", __func__); } - // disable whitelistrelay in blocksonly mode if (args.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY)) { + // disable whitelistrelay in blocksonly mode if (args.SoftSetBoolArg("-whitelistrelay", false)) LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -whitelistrelay=0\n", __func__); + // Reduce default mempool size in blocksonly mode to avoid unexpected resource usage + if (args.SoftSetArg("-maxmempool", ToString(DEFAULT_BLOCKSONLY_MAX_MEMPOOL_SIZE_MB))) + LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -maxmempool=%d\n", __func__, DEFAULT_BLOCKSONLY_MAX_MEMPOOL_SIZE_MB); } // Forcing relay from whitelisted hosts implies we will accept relays from them in the first place. @@ -1479,7 +1499,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) options.mempool = Assert(node.mempool.get()); options.reindex = node::fReindex; options.reindex_chainstate = fReindexChainState; - options.prune = node::fPruneMode; + options.prune = chainman.m_blockman.IsPruneMode(); options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); options.check_level = args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); options.check_interrupt = ShutdownRequested; @@ -1589,7 +1609,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // if pruning, perform the initial blockstore prune // after any wallet rescanning has taken place. - if (fPruneMode) { + if (chainman.m_blockman.IsPruneMode()) { if (!fReindex) { LOCK(cs_main); for (Chainstate* chainstate : chainman.GetAll()) { @@ -1617,8 +1637,10 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // On first startup, warn on low block storage space if (!fReindex && !fReindexChainState && chain_active_height <= 1) { - uint64_t additional_bytes_needed = fPruneMode ? nPruneTarget - : chainparams.AssumedBlockchainSize() * 1024 * 1024 * 1024; + uint64_t additional_bytes_needed{ + chainman.m_blockman.IsPruneMode() ? + chainman.m_blockman.GetPruneTarget() : + chainparams.AssumedBlockchainSize() * 1024 * 1024 * 1024}; if (!CheckDiskSpace(args.GetBlocksDirPath(), additional_bytes_needed)) { InitWarning(strprintf(_( diff --git a/src/kernel/coinstats.cpp b/src/kernel/coinstats.cpp index 06a4b8c974..82d7d8c46b 100644 --- a/src/kernel/coinstats.cpp +++ b/src/kernel/coinstats.cpp @@ -48,8 +48,9 @@ uint64_t GetBogoSize(const CScript& script_pub_key) script_pub_key.size() /* scriptPubKey */; } -CDataStream TxOutSer(const COutPoint& outpoint, const Coin& coin) { - CDataStream ss(SER_DISK, PROTOCOL_VERSION); +DataStream TxOutSer(const COutPoint& outpoint, const Coin& coin) +{ + DataStream ss{}; ss << outpoint; ss << static_cast<uint32_t>(coin.nHeight * 2 + coin.fCoinBase); ss << coin.out; diff --git a/src/kernel/coinstats.h b/src/kernel/coinstats.h index b7c1328e93..54d0e4f664 100644 --- a/src/kernel/coinstats.h +++ b/src/kernel/coinstats.h @@ -72,7 +72,7 @@ struct CCoinsStats { uint64_t GetBogoSize(const CScript& script_pub_key); -CDataStream TxOutSer(const COutPoint& outpoint, const Coin& coin); +DataStream TxOutSer(const COutPoint& outpoint, const Coin& coin); std::optional<CCoinsStats> ComputeUTXOStats(CoinStatsHashType hash_type, CCoinsView* view, node::BlockManager& blockman, const std::function<void()>& interruption_point = {}); } // namespace kernel diff --git a/src/kernel/context.cpp b/src/kernel/context.cpp index 15413c1840..1205da869e 100644 --- a/src/kernel/context.cpp +++ b/src/kernel/context.cpp @@ -21,12 +21,10 @@ Context::Context() LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo); RandomInit(); ECC_Start(); - ecc_verify_handle.reset(new ECCVerifyHandle()); } Context::~Context() { - ecc_verify_handle.reset(); ECC_Stop(); } diff --git a/src/kernel/context.h b/src/kernel/context.h index 9746ef994b..f11b7b54f0 100644 --- a/src/kernel/context.h +++ b/src/kernel/context.h @@ -7,8 +7,6 @@ #include <memory> -class ECCVerifyHandle; - namespace kernel { //! Context struct holding the kernel library's logically global state, and //! passed to external libbitcoin_kernel functions which need access to this @@ -18,8 +16,6 @@ namespace kernel { //! State stored directly in this struct should be simple. More complex state //! should be stored to std::unique_ptr members pointing to opaque types. struct Context { - std::unique_ptr<ECCVerifyHandle> ecc_verify_handle; - //! Declare default constructor and destructor that are not inline, so code //! instantiating the kernel::Context struct doesn't need to #include class //! definitions for all the unique_ptr members. diff --git a/src/kernel/cs_main.cpp b/src/kernel/cs_main.cpp new file mode 100644 index 0000000000..c3a08c9695 --- /dev/null +++ b/src/kernel/cs_main.cpp @@ -0,0 +1,7 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <sync.h> + +RecursiveMutex cs_main; diff --git a/src/kernel/cs_main.h b/src/kernel/cs_main.h new file mode 100644 index 0000000000..8d03903b8e --- /dev/null +++ b/src/kernel/cs_main.h @@ -0,0 +1,22 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_CS_MAIN_H +#define BITCOIN_KERNEL_CS_MAIN_H + +#include <sync.h> + +/** + * Mutex to guard access to validation specific variables, such as reading + * or changing the chainstate. + * + * This may also need to be locked when updating the transaction pool, e.g. on + * AcceptToMemoryPool. See CTxMemPool::cs comment for details. + * + * The transaction pool has a separate lock to allow reading from it and the + * chainstate at the same time. + */ +extern RecursiveMutex cs_main; + +#endif // BITCOIN_KERNEL_CS_MAIN_H diff --git a/src/kernel/mempool_options.h b/src/kernel/mempool_options.h index dad6f14c39..beb5fca5e9 100644 --- a/src/kernel/mempool_options.h +++ b/src/kernel/mempool_options.h @@ -18,6 +18,8 @@ class CBlockPolicyEstimator; /** Default for -maxmempool, maximum megabytes of mempool memory usage */ static constexpr unsigned int DEFAULT_MAX_MEMPOOL_SIZE_MB{300}; +/** Default for -maxmempool when blocksonly is set */ +static constexpr unsigned int DEFAULT_BLOCKSONLY_MAX_MEMPOOL_SIZE_MB{5}; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static constexpr unsigned int DEFAULT_MEMPOOL_EXPIRY_HOURS{336}; /** Default for -mempoolfullrbf, if the transaction replaceability signaling is ignored */ diff --git a/src/key.cpp b/src/key.cpp index 0cb51cb9be..3a3f0b2bc2 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -233,7 +233,7 @@ bool CKey::Sign(const uint256 &hash, std::vector<unsigned char>& vchSig, bool gr secp256k1_pubkey pk; ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &pk, begin()); assert(ret); - ret = secp256k1_ecdsa_verify(GetVerifyContext(), &sig, hash.begin(), &pk); + ret = secp256k1_ecdsa_verify(secp256k1_context_static, &sig, hash.begin(), &pk); assert(ret); return true; } @@ -245,8 +245,7 @@ bool CKey::VerifyPubKey(const CPubKey& pubkey) const { unsigned char rnd[8]; std::string str = "Bitcoin key verification\n"; GetRandBytes(rnd); - uint256 hash; - CHash256().Write(MakeUCharSpan(str)).Write(rnd).Finalize(hash); + uint256 hash{Hash(str, rnd)}; std::vector<unsigned char> vchSig; Sign(hash, vchSig); return pubkey.Verify(hash, vchSig); @@ -268,9 +267,9 @@ bool CKey::SignCompact(const uint256 &hash, std::vector<unsigned char>& vchSig) secp256k1_pubkey epk, rpk; ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &epk, begin()); assert(ret); - ret = secp256k1_ecdsa_recover(GetVerifyContext(), &rpk, &rsig, hash.begin()); + ret = secp256k1_ecdsa_recover(secp256k1_context_static, &rpk, &rsig, hash.begin()); assert(ret); - ret = secp256k1_ec_pubkey_cmp(GetVerifyContext(), &epk, &rpk); + ret = secp256k1_ec_pubkey_cmp(secp256k1_context_static, &epk, &rpk); assert(ret == 0); return true; } @@ -286,14 +285,14 @@ bool CKey::SignSchnorr(const uint256& hash, Span<unsigned char> sig, const uint2 unsigned char pubkey_bytes[32]; if (!secp256k1_xonly_pubkey_serialize(secp256k1_context_sign, pubkey_bytes, &pubkey)) return false; uint256 tweak = XOnlyPubKey(pubkey_bytes).ComputeTapTweakHash(merkle_root->IsNull() ? nullptr : merkle_root); - if (!secp256k1_keypair_xonly_tweak_add(GetVerifyContext(), &keypair, tweak.data())) return false; + if (!secp256k1_keypair_xonly_tweak_add(secp256k1_context_static, &keypair, tweak.data())) return false; } bool ret = secp256k1_schnorrsig_sign32(secp256k1_context_sign, sig.data(), hash.data(), &keypair, aux.data()); if (ret) { // Additional verification step to prevent using a potentially corrupted signature secp256k1_xonly_pubkey pubkey_verify; - ret = secp256k1_keypair_xonly_pub(GetVerifyContext(), &pubkey_verify, nullptr, &keypair); - ret &= secp256k1_schnorrsig_verify(GetVerifyContext(), sig.data(), hash.begin(), 32, &pubkey_verify); + ret = secp256k1_keypair_xonly_pub(secp256k1_context_static, &pubkey_verify, nullptr, &keypair); + ret &= secp256k1_schnorrsig_verify(secp256k1_context_static, sig.data(), hash.begin(), 32, &pubkey_verify); } if (!ret) memory_cleanse(sig.data(), sig.size()); memory_cleanse(&keypair, sizeof(keypair)); @@ -392,7 +391,7 @@ bool ECC_InitSanityCheck() { void ECC_Start() { assert(secp256k1_context_sign == nullptr); - secp256k1_context *ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + secp256k1_context *ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); assert(ctx != nullptr); { @@ -42,10 +42,10 @@ public: private: //! Whether this private key is valid. We check for correctness when modifying the key //! data, so fValid should always correspond to the actual state. - bool fValid; + bool fValid{false}; //! Whether the public key corresponding to this private key is (to be) compressed. - bool fCompressed; + bool fCompressed{false}; //! The actual byte data std::vector<unsigned char, secure_allocator<unsigned char> > keydata; @@ -55,7 +55,7 @@ private: public: //! Construct an invalid private key. - CKey() : fValid(false), fCompressed(false) + CKey() { // Important: vch must be 32 bytes in length to not break serialization keydata.resize(32); diff --git a/src/logging/timer.h b/src/logging/timer.h index ea0821dede..993ba99c25 100644 --- a/src/logging/timer.h +++ b/src/logging/timer.h @@ -12,6 +12,7 @@ #include <util/types.h> #include <chrono> +#include <optional> #include <string> @@ -28,14 +29,14 @@ public: std::string prefix, std::string end_msg, BCLog::LogFlags log_category = BCLog::LogFlags::ALL, - bool msg_on_completion = true) : - m_prefix(std::move(prefix)), - m_title(std::move(end_msg)), - m_log_category(log_category), - m_message_on_completion(msg_on_completion) + bool msg_on_completion = true) + : m_prefix(std::move(prefix)), + m_title(std::move(end_msg)), + m_log_category(log_category), + m_message_on_completion(msg_on_completion) { this->Log(strprintf("%s started", m_title)); - m_start_t = GetTime<std::chrono::microseconds>(); + m_start_t = std::chrono::steady_clock::now(); } ~Timer() @@ -60,24 +61,25 @@ public: std::string LogMsg(const std::string& msg) { - const auto end_time = GetTime<std::chrono::microseconds>() - m_start_t; - if (m_start_t.count() <= 0) { + const auto end_time{std::chrono::steady_clock::now()}; + if (!m_start_t) { return strprintf("%s: %s", m_prefix, msg); } + const auto duration{end_time - *m_start_t}; if constexpr (std::is_same<TimeType, std::chrono::microseconds>::value) { - return strprintf("%s: %s (%iμs)", m_prefix, msg, end_time.count()); + return strprintf("%s: %s (%iμs)", m_prefix, msg, Ticks<std::chrono::microseconds>(duration)); } else if constexpr (std::is_same<TimeType, std::chrono::milliseconds>::value) { - return strprintf("%s: %s (%.2fms)", m_prefix, msg, end_time.count() * 0.001); + return strprintf("%s: %s (%.2fms)", m_prefix, msg, Ticks<MillisecondsDouble>(duration)); } else if constexpr (std::is_same<TimeType, std::chrono::seconds>::value) { - return strprintf("%s: %s (%.2fs)", m_prefix, msg, end_time.count() * 0.000001); + return strprintf("%s: %s (%.2fs)", m_prefix, msg, Ticks<SecondsDouble>(duration)); } else { static_assert(ALWAYS_FALSE<TimeType>, "Error: unexpected time type"); } } private: - std::chrono::microseconds m_start_t{}; + std::optional<std::chrono::steady_clock::time_point> m_start_t{}; //! Log prefix; usually the name of the function this was created in. const std::string m_prefix; diff --git a/src/mapport.h b/src/mapport.h index 279d65167f..6f55c46f6c 100644 --- a/src/mapport.h +++ b/src/mapport.h @@ -5,17 +5,9 @@ #ifndef BITCOIN_MAPPORT_H #define BITCOIN_MAPPORT_H -#ifdef USE_UPNP -static constexpr bool DEFAULT_UPNP = USE_UPNP; -#else static constexpr bool DEFAULT_UPNP = false; -#endif // USE_UPNP -#ifdef USE_NATPMP -static constexpr bool DEFAULT_NATPMP = USE_NATPMP; -#else static constexpr bool DEFAULT_NATPMP = false; -#endif // USE_NATPMP enum MapPortProtoFlag : unsigned int { NONE = 0x00, diff --git a/src/net.cpp b/src/net.cpp index 960d0ee841..98500bd30e 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1305,15 +1305,14 @@ void CConnman::SocketHandlerConnected(const std::vector<CNode*>& nodes, RecordBytesRecv(nBytes); if (notify) { size_t nSizeAdded = 0; - auto it(pnode->vRecvMsg.begin()); - for (; it != pnode->vRecvMsg.end(); ++it) { + for (const auto& msg : pnode->vRecvMsg) { // vRecvMsg contains only completed CNetMessage // the single possible partially deserialized message are held by TransportDeserializer - nSizeAdded += it->m_raw_message_size; + nSizeAdded += msg.m_raw_message_size; } { LOCK(pnode->cs_vProcessMsg); - pnode->vProcessMsg.splice(pnode->vProcessMsg.end(), pnode->vRecvMsg, pnode->vRecvMsg.begin(), it); + pnode->vProcessMsg.splice(pnode->vProcessMsg.end(), pnode->vRecvMsg); pnode->nProcessQueueSize += nSizeAdded; pnode->fPauseRecv = pnode->nProcessQueueSize > nReceiveFloodSize; } @@ -1399,7 +1398,7 @@ void CConnman::ThreadDNSAddressSeed() if (gArgs.GetBoolArg("-forcednsseed", DEFAULT_FORCEDNSSEED)) { // When -forcednsseed is provided, query all. seeds_right_now = seeds.size(); - } else if (addrman.size() == 0) { + } else if (addrman.Size() == 0) { // If we have no known peers, query all. // This will occur on the first run, or if peers.dat has been // deleted. @@ -1418,13 +1417,13 @@ void CConnman::ThreadDNSAddressSeed() // * If we continue having problems, eventually query all the // DNS seeds, and if that fails too, also try the fixed seeds. // (done in ThreadOpenConnections) - const std::chrono::seconds seeds_wait_time = (addrman.size() >= DNSSEEDS_DELAY_PEER_THRESHOLD ? DNSSEEDS_DELAY_MANY_PEERS : DNSSEEDS_DELAY_FEW_PEERS); + const std::chrono::seconds seeds_wait_time = (addrman.Size() >= DNSSEEDS_DELAY_PEER_THRESHOLD ? DNSSEEDS_DELAY_MANY_PEERS : DNSSEEDS_DELAY_FEW_PEERS); for (const std::string& seed : seeds) { if (seeds_right_now == 0) { seeds_right_now += DNSSEEDS_TO_QUERY_AT_ONCE; - if (addrman.size() > 0) { + if (addrman.Size() > 0) { LogPrintf("Waiting %d seconds before querying DNS seeds.\n", seeds_wait_time.count()); std::chrono::seconds to_wait = seeds_wait_time; while (to_wait.count() > 0) { @@ -1504,7 +1503,7 @@ void CConnman::DumpAddresses() DumpPeerAddresses(::gArgs, addrman); LogPrint(BCLog::NET, "Flushed %d addresses to peers.dat %dms\n", - addrman.size(), Ticks<std::chrono::milliseconds>(SteadyClock::now() - start)); + addrman.Size(), Ticks<std::chrono::milliseconds>(SteadyClock::now() - start)); } void CConnman::ProcessAddrFetch() @@ -1575,6 +1574,19 @@ int CConnman::GetExtraBlockRelayCount() const return std::max(block_relay_peers - m_max_outbound_block_relay, 0); } +std::unordered_set<Network> CConnman::GetReachableEmptyNetworks() const +{ + std::unordered_set<Network> networks{}; + for (int n = 0; n < NET_MAX; n++) { + enum Network net = (enum Network)n; + if (net == NET_UNROUTABLE || net == NET_INTERNAL) continue; + if (IsReachable(net) && addrman.Size(net, std::nullopt) == 0) { + networks.insert(net); + } + } + return networks; +} + void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) { SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_OPEN_CONNECTION); @@ -1624,7 +1636,8 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) if (interruptNet) return; - if (add_fixed_seeds && addrman.size() == 0) { + const std::unordered_set<Network> fixed_seed_networks{GetReachableEmptyNetworks()}; + if (add_fixed_seeds && !fixed_seed_networks.empty()) { // When the node starts with an empty peers.dat, there are a few other sources of peers before // we fallback on to fixed seeds: -dnsseed, -seednode, -addnode // If none of those are available, we fallback on to fixed seeds immediately, else we allow @@ -1633,7 +1646,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // It is cheapest to check if enough time has passed first. if (GetTime<std::chrono::seconds>() > start + std::chrono::minutes{1}) { add_fixed_seeds_now = true; - LogPrintf("Adding fixed seeds as 60 seconds have passed and addrman is empty\n"); + LogPrintf("Adding fixed seeds as 60 seconds have passed and addrman is empty for at least one reachable network\n"); } // Checking !dnsseed is cheaper before locking 2 mutexes. @@ -1650,14 +1663,12 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // We will not make outgoing connections to peers that are unreachable // (e.g. because of -onlynet configuration). // Therefore, we do not add them to addrman in the first place. - // Note that if you change -onlynet setting from one network to another, - // peers.dat will contain only peers of unreachable networks and - // manual intervention will be needed (either delete peers.dat after - // configuration change or manually add some reachable peer using addnode), - // see <https://github.com/bitcoin/bitcoin/issues/26035> for details. + // In case previously unreachable networks become reachable + // (e.g. in case of -onlynet changes by the user), fixed seeds will + // be loaded only for networks for which we have no addressses. seed_addrs.erase(std::remove_if(seed_addrs.begin(), seed_addrs.end(), - [](const CAddress& addr) { return !IsReachable(addr); }), - seed_addrs.end()); + [&fixed_seed_networks](const CAddress& addr) { return fixed_seed_networks.count(addr.GetNetwork()) == 0; }), + seed_addrs.end()); CNetAddr local; local.SetInternal("fixedseeds"); addrman.Add(seed_addrs, local); @@ -39,6 +39,7 @@ #include <memory> #include <optional> #include <thread> +#include <unordered_set> #include <vector> class AddrMan; @@ -970,6 +971,12 @@ private: void RecordBytesSent(uint64_t bytes) EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); /** + Return reachable networks for which we have no addresses in addrman and therefore + may require loading fixed seeds. + */ + std::unordered_set<Network> GetReachableEmptyNetworks() const; + + /** * Return vector of current BLOCK_RELAY peers. */ std::vector<CAddress> GetCurrentBlockRelayOnlyConns() const; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 21a49bdebd..a659300a0d 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -53,9 +53,6 @@ using node::ReadBlockFromDisk; using node::ReadRawBlockFromDisk; -using node::fImporting; -using node::fPruneMode; -using node::fReindex; /** How long to cache transactions in mapRelay for normal relay */ static constexpr auto RELAY_TX_CACHE_TIME = 15min; @@ -113,8 +110,11 @@ static constexpr auto GETDATA_TX_INTERVAL{60s}; static const unsigned int MAX_GETDATA_SZ = 1000; /** Number of blocks that can be requested at any given time from a single peer. */ static const int MAX_BLOCKS_IN_TRANSIT_PER_PEER = 16; -/** Time during which a peer must stall block download progress before being disconnected. */ -static constexpr auto BLOCK_STALLING_TIMEOUT{2s}; +/** Default time during which a peer must stall block download progress before being disconnected. + * the actual timeout is increased temporarily if peers are disconnected for hitting the timeout */ +static constexpr auto BLOCK_STALLING_TIMEOUT_DEFAULT{2s}; +/** Maximum timeout for stalling block download. */ +static constexpr auto BLOCK_STALLING_TIMEOUT_MAX{64s}; /** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ static const unsigned int MAX_HEADERS_RESULTS = 2000; @@ -584,14 +584,17 @@ private: /** * Reconsider orphan transactions after a parent has been accepted to the mempool. * - * @peer[in] peer The peer whose orphan transactions we will reconsider. Generally only one - * orphan will be reconsidered on each call of this function. This set - * may be added to if accepting an orphan causes its children to be - * reconsidered. - * @return True if there are still orphans in this peer's work set. + * @peer[in] peer The peer whose orphan transactions we will reconsider. Generally only + * one orphan will be reconsidered on each call of this function. If an + * accepted orphan has orphaned children, those will need to be + * reconsidered, creating more work, possibly for other peers. + * @return True if meaningful work was done (an orphan was accepted/rejected). + * If no meaningful work was done, then the work set for this peer + * will be empty. */ bool ProcessOrphanTx(Peer& peer) - EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, cs_main); + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex); + /** Process a single headers message from a peer. * * @param[in] pfrom CNode of the peer @@ -771,6 +774,9 @@ private: /** Number of preferable block download peers. */ int m_num_preferred_download_peers GUARDED_BY(cs_main){0}; + /** Stalling timeout for blocks in IBD */ + std::atomic<std::chrono::seconds> m_block_stalling_timeout{BLOCK_STALLING_TIMEOUT_DEFAULT}; + bool AlreadyHaveTx(const GenTxid& gtxid) EXCLUSIVE_LOCKS_REQUIRED(cs_main, !m_recent_confirmed_transactions_mutex); @@ -1730,8 +1736,7 @@ bool PeerManagerImpl::BlockRequestAllowed(const CBlockIndex* pindex) std::optional<std::string> PeerManagerImpl::FetchBlock(NodeId peer_id, const CBlockIndex& block_index) { - if (fImporting) return "Importing..."; - if (fReindex) return "Reindexing..."; + if (m_chainman.m_blockman.LoadingBlocks()) return "Loading blocks ..."; // Ensure this peer exists and hasn't been disconnected PeerRef peer = GetPeerRef(peer_id); @@ -1809,7 +1814,8 @@ void PeerManagerImpl::StartScheduledTasks(CScheduler& scheduler) /** * Evict orphan txn pool entries based on a newly connected * block, remember the recently confirmed transactions, and delete tracked - * announcements for them. Also save the time of the last tip update. + * announcements for them. Also save the time of the last tip update and + * possibly reduce dynamic block stalling timeout. */ void PeerManagerImpl::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) { @@ -1832,6 +1838,16 @@ void PeerManagerImpl::BlockConnected(const std::shared_ptr<const CBlock>& pblock m_txrequest.ForgetTxHash(ptx->GetWitnessHash()); } } + + // In case the dynamic timeout was doubled once or more, reduce it slowly back to its default value + auto stalling_timeout = m_block_stalling_timeout.load(); + Assume(stalling_timeout >= BLOCK_STALLING_TIMEOUT_DEFAULT); + if (stalling_timeout != BLOCK_STALLING_TIMEOUT_DEFAULT) { + const auto new_timeout = std::max(std::chrono::duration_cast<std::chrono::seconds>(stalling_timeout * 0.85), BLOCK_STALLING_TIMEOUT_DEFAULT); + if (m_block_stalling_timeout.compare_exchange_strong(stalling_timeout, new_timeout)) { + LogPrint(BCLog::NET, "Decreased stalling timeout to %d seconds\n", count_seconds(new_timeout)); + } + } } void PeerManagerImpl::BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) @@ -2897,13 +2913,11 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) { AssertLockHeld(g_msgproc_mutex); - AssertLockHeld(cs_main); + LOCK(cs_main); CTransactionRef porphanTx = nullptr; - NodeId from_peer = -1; - bool more = false; - while (CTransactionRef porphanTx = m_orphanage.GetTxToReconsider(peer.m_id, from_peer, more)) { + while (CTransactionRef porphanTx = m_orphanage.GetTxToReconsider(peer.m_id)) { const MempoolAcceptResult result = m_chainman.ProcessTransaction(porphanTx); const TxValidationState& state = result.m_state; const uint256& orphanHash = porphanTx->GetHash(); @@ -2911,20 +2925,20 @@ bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); RelayTransaction(orphanHash, porphanTx->GetWitnessHash()); - m_orphanage.AddChildrenToWorkSet(*porphanTx, peer.m_id); + m_orphanage.AddChildrenToWorkSet(*porphanTx); m_orphanage.EraseTx(orphanHash); for (const CTransactionRef& removedTx : result.m_replaced_transactions.value()) { AddToCompactExtraTransactions(removedTx); } - break; + return true; } else if (state.GetResult() != TxValidationResult::TX_MISSING_INPUTS) { if (state.IsInvalid()) { LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s from peer=%d. %s\n", orphanHash.ToString(), - from_peer, + peer.m_id, state.ToString()); // Maybe punish peer that gave us an invalid orphan tx - MaybePunishNodeForTx(from_peer, state); + MaybePunishNodeForTx(peer.m_id, state); } // Has inputs but not accepted to mempool // Probably non-standard or insufficient fee @@ -2959,11 +2973,11 @@ bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) } } m_orphanage.EraseTx(orphanHash); - break; + return true; } } - return more; + return false; } bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& node, Peer& peer, @@ -3361,7 +3375,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // If the peer is old enough to have the old alert system, send it the final alert. if (greatest_common_version <= 70012) { - CDataStream finalAlert(ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50"), SER_NETWORK, PROTOCOL_VERSION); + DataStream finalAlert{ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50")}; m_connman.PushMessage(&pfrom, CNetMsgMaker(greatest_common_version).Make("alert", finalAlert)); } @@ -3679,7 +3693,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); UpdateBlockAvailability(pfrom.GetId(), inv.hash); - if (!fAlreadyHave && !fImporting && !fReindex && !IsBlockRequested(inv.hash)) { + if (!fAlreadyHave && !m_chainman.m_blockman.LoadingBlocks() && !IsBlockRequested(inv.hash)) { // Headers-first is the primary method of announcement on // the network. If a node fell back to sending blocks by // inv, it may be for a re-org, or because we haven't @@ -3812,8 +3826,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // If pruning, don't inv blocks unless we have on disk and are likely to still have // for some reasonable time window (1 hour) that block relay might require. const int nPrunedBlocksLikelyToHave = MIN_BLOCKS_TO_KEEP - 3600 / m_chainparams.GetConsensus().nPowTargetSpacing; - if (fPruneMode && (!(pindex->nStatus & BLOCK_HAVE_DATA) || pindex->nHeight <= m_chainman.ActiveChain().Tip()->nHeight - nPrunedBlocksLikelyToHave)) - { + if (m_chainman.m_blockman.IsPruneMode() && (!(pindex->nStatus & BLOCK_HAVE_DATA) || pindex->nHeight <= m_chainman.ActiveChain().Tip()->nHeight - nPrunedBlocksLikelyToHave)) { LogPrint(BCLog::NET, " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); break; } @@ -3889,7 +3902,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - if (fImporting || fReindex) { + if (m_chainman.m_blockman.LoadingBlocks()) { LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d while importing/reindexing\n", pfrom.GetId()); return; } @@ -4033,7 +4046,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_txrequest.ForgetTxHash(tx.GetHash()); m_txrequest.ForgetTxHash(tx.GetWitnessHash()); RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); - m_orphanage.AddChildrenToWorkSet(tx, peer->m_id); + m_orphanage.AddChildrenToWorkSet(tx); pfrom.m_last_tx_time = GetTime<std::chrono::seconds>(); @@ -4045,9 +4058,6 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, for (const CTransactionRef& removedTx : result.m_replaced_transactions.value()) { AddToCompactExtraTransactions(removedTx); } - - // Recursively process any orphan transactions that depended on this one - ProcessOrphanTx(*peer); } else if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { @@ -4171,7 +4181,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::CMPCTBLOCK) { // Ignore cmpctblock received while importing - if (fImporting || fReindex) { + if (m_chainman.m_blockman.LoadingBlocks()) { LogPrint(BCLog::NET, "Unexpected cmpctblock message received from peer %d\n", pfrom.GetId()); return; } @@ -4387,7 +4397,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::BLOCKTXN) { // Ignore blocktxn received while importing - if (fImporting || fReindex) { + if (m_chainman.m_blockman.LoadingBlocks()) { LogPrint(BCLog::NET, "Unexpected blocktxn message received from peer %d\n", pfrom.GetId()); return; } @@ -4462,7 +4472,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::HEADERS) { // Ignore headers received while importing - if (fImporting || fReindex) { + if (m_chainman.m_blockman.LoadingBlocks()) { LogPrint(BCLog::NET, "Unexpected headers message received from peer %d\n", pfrom.GetId()); return; } @@ -4507,7 +4517,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::BLOCK) { // Ignore block received while importing - if (fImporting || fReindex) { + if (m_chainman.m_blockman.LoadingBlocks()) { LogPrint(BCLog::NET, "Unexpected block message received from peer %d\n", pfrom.GetId()); return; } @@ -4856,16 +4866,12 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt } } - bool has_more_orphans; - { - LOCK(cs_main); - has_more_orphans = ProcessOrphanTx(*peer); - } + const bool processed_orphan = ProcessOrphanTx(*peer); if (pfrom->fDisconnect) return false; - if (has_more_orphans) return true; + if (processed_orphan) return true; // this maintains the order of responses // and prevents m_getdata_requests to grow unbounded @@ -4911,6 +4917,12 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt LOCK(peer->m_getdata_requests_mutex); if (!peer->m_getdata_requests.empty()) fMoreWork = true; } + // Does this peer has an orphan ready to reconsider? + // (Note: we may have provided a parent for an orphan provided + // by another peer that was already processed; in that case, + // the extra work may not be noticed, possibly resulting in an + // unnecessary 100ms delay) + if (m_orphanage.HaveTxToReconsider(peer->m_id)) fMoreWork = true; } catch (const std::exception& e) { LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' (%s) caught\n", __func__, SanitizeString(msg.m_type), msg.m_message_size, e.what(), typeid(e).name()); } catch (...) { @@ -5092,7 +5104,7 @@ void PeerManagerImpl::CheckForStaleTipAndEvictPeers() if (now > m_stale_tip_check_time) { // Check whether our tip is stale, and if so, allow using an extra // outbound peer - if (!fImporting && !fReindex && m_connman.GetNetworkActive() && m_connman.GetUseAddrmanOutgoing() && TipMayBeStale()) { + if (!m_chainman.m_blockman.LoadingBlocks() && m_connman.GetNetworkActive() && m_connman.GetUseAddrmanOutgoing() && TipMayBeStale()) { LogPrintf("Potential stale tip detected, will try using extra outbound peer (last tip update: %d seconds ago)\n", count_seconds(now - m_last_tip_update.load())); m_connman.SetTryNewOutboundPeer(true); @@ -5399,7 +5411,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) } } - if (!state.fSyncStarted && CanServeBlocks(*peer) && !fImporting && !fReindex) { + if (!state.fSyncStarted && CanServeBlocks(*peer) && !m_chainman.m_blockman.LoadingBlocks()) { // Only actively request headers from a single peer, unless we're close to today. if ((nSyncStarted == 0 && sync_blocks_and_headers_from_peer) || m_chainman.m_best_header->Time() > GetAdjustedTime() - 24h) { const CBlockIndex* pindexStart = m_chainman.m_best_header; @@ -5713,12 +5725,19 @@ bool PeerManagerImpl::SendMessages(CNode* pto) m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); // Detect whether we're stalling - if (state.m_stalling_since.count() && state.m_stalling_since < current_time - BLOCK_STALLING_TIMEOUT) { + auto stalling_timeout = m_block_stalling_timeout.load(); + if (state.m_stalling_since.count() && state.m_stalling_since < current_time - stalling_timeout) { // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection // should only happen during initial block download. LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->GetId()); pto->fDisconnect = true; + // Increase timeout for the next peer so that we don't disconnect multiple peers if our own + // bandwidth is insufficient. + const auto new_timeout = std::min(2 * stalling_timeout, BLOCK_STALLING_TIMEOUT_MAX); + if (stalling_timeout != new_timeout && m_block_stalling_timeout.compare_exchange_strong(stalling_timeout, new_timeout)) { + LogPrint(BCLog::NET, "Increased stalling timeout temporarily to %d seconds\n", count_seconds(new_timeout)); + } return true; } // In case there is a block that has been in flight from this peer for block_interval * (1 + 0.5 * N) diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index b8a57acf80..a81099a26c 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -352,7 +352,7 @@ bool BlockManager::LoadBlockIndexDB(const Consensus::Params& consensus_params) } for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) { FlatFilePos pos(*it, 0); - if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) { + if (AutoFile{OpenBlockFile(pos, true)}.IsNull()) { return false; } } @@ -454,13 +454,13 @@ CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n) static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append - CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); + AutoFile fileout{OpenUndoFile(pos)}; if (fileout.IsNull()) { return error("%s: OpenUndoFile failed", __func__); } // Write index header - unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion()); + unsigned int nSize = GetSerializeSize(blockundo, CLIENT_VERSION); fileout << messageStart << nSize; // Write undo data @@ -489,14 +489,14 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) } // Open history file to read - CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); + AutoFile filein{OpenUndoFile(pos, true)}; if (filein.IsNull()) { return error("%s: OpenUndoFile failed", __func__); } // Read block uint256 hashChecksum; - CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data + HashVerifier verifier{filein}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a try { verifier << pindex->pprev->GetBlockHash(); verifier >> blockundo; @@ -768,7 +768,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c { FlatFilePos hpos = pos; hpos.nPos -= 8; // Seek back 8 bytes for meta header - CAutoFile filein(OpenBlockFile(hpos, true), SER_DISK, CLIENT_VERSION); + AutoFile filein{OpenBlockFile(hpos, true)}; if (filein.IsNull()) { return error("%s: OpenBlockFile failed for %s", __func__, pos.ToString()); } diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index d4411a7776..b6007897df 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -8,6 +8,7 @@ #include <attributes.h> #include <chain.h> #include <fs.h> +#include <kernel/cs_main.h> #include <protocol.h> #include <sync.h> #include <txdb.h> @@ -17,8 +18,6 @@ #include <unordered_map> #include <vector> -extern RecursiveMutex cs_main; - class ArgsManager; class BlockValidationState; class CBlock; @@ -49,10 +48,7 @@ static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAG extern std::atomic_bool fImporting; extern std::atomic_bool fReindex; -/** Pruning-related variables and constants */ -/** True if we're running in -prune mode. */ extern bool fPruneMode; -/** Number of bytes of block files that we're trying to stay below. */ extern uint64_t nPruneTarget; // Because validation code takes pointers to the map's CBlockIndex objects, if @@ -177,6 +173,17 @@ public: /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */ FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); + /** Whether running in -prune mode. */ + [[nodiscard]] bool IsPruneMode() const { return fPruneMode; } + + /** Attempt to stay below this number of bytes of block files. */ + [[nodiscard]] uint64_t GetPruneTarget() const { return nPruneTarget; } + + [[nodiscard]] bool LoadingBlocks() const + { + return fImporting || fReindex; + } + /** Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage(); diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index 99dc319ec0..ba1024d22e 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -44,10 +44,10 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize if (chainman.MinimumChainWork() < UintToArith256(chainman.GetConsensus().nMinimumChainWork)) { LogPrintf("Warning: nMinimumChainWork set below default value of %s\n", chainman.GetConsensus().nMinimumChainWork.GetHex()); } - if (nPruneTarget == std::numeric_limits<uint64_t>::max()) { + if (chainman.m_blockman.GetPruneTarget() == std::numeric_limits<uint64_t>::max()) { LogPrintf("Block pruning enabled. Use RPC call pruneblockchain(height) to manually prune block and undo files.\n"); - } else if (nPruneTarget) { - LogPrintf("Prune configured to target %u MiB on disk for block and undo files.\n", nPruneTarget / 1024 / 1024); + } else if (chainman.m_blockman.GetPruneTarget()) { + LogPrintf("Prune configured to target %u MiB on disk for block and undo files.\n", chainman.m_blockman.GetPruneTarget() / 1024 / 1024); } LOCK(cs_main); diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 4f3dc99bbf..eda359568f 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -711,8 +711,9 @@ public: LOCK(::cs_main); return chainman().m_blockman.m_have_pruned; } - bool isReadyToBroadcast() override { return !node::fImporting && !node::fReindex && !isInitialBlockDownload(); } - bool isInitialBlockDownload() override { + bool isReadyToBroadcast() override { return !chainman().m_blockman.LoadingBlocks() && !isInitialBlockDownload(); } + bool isInitialBlockDownload() override + { return chainman().ActiveChainstate().IsInitialBlockDownload(); } bool shutdownRequested() override { return ShutdownRequested(); } diff --git a/src/node/miner.cpp b/src/node/miner.cpp index dc6849e0d2..c2b6fd1dc3 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -60,10 +60,12 @@ BlockAssembler::Options::Options() { blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; + test_block_validity = true; } BlockAssembler::BlockAssembler(Chainstate& chainstate, const CTxMemPool* mempool, const Options& options) - : chainparams{chainstate.m_chainman.GetParams()}, + : test_block_validity{options.test_block_validity}, + chainparams{chainstate.m_chainman.GetParams()}, m_mempool(mempool), m_chainstate(chainstate) { @@ -72,11 +74,10 @@ BlockAssembler::BlockAssembler(Chainstate& chainstate, const CTxMemPool* mempool nBlockMaxWeight = std::max<size_t>(4000, std::min<size_t>(MAX_BLOCK_WEIGHT - 4000, options.nBlockMaxWeight)); } -static BlockAssembler::Options DefaultOptions() +void ApplyArgsManOptions(const ArgsManager& gArgs, BlockAssembler::Options& options) { // Block resource limits // If -blockmaxweight is not given, limit to DEFAULT_BLOCK_MAX_WEIGHT - BlockAssembler::Options options; options.nBlockMaxWeight = gArgs.GetIntArg("-blockmaxweight", DEFAULT_BLOCK_MAX_WEIGHT); if (gArgs.IsArgSet("-blockmintxfee")) { std::optional<CAmount> parsed = ParseMoney(gArgs.GetArg("-blockmintxfee", "")); @@ -84,11 +85,16 @@ static BlockAssembler::Options DefaultOptions() } else { options.blockMinFeeRate = CFeeRate{DEFAULT_BLOCK_MIN_TX_FEE}; } +} +static BlockAssembler::Options ConfiguredOptions() +{ + BlockAssembler::Options options; + ApplyArgsManOptions(gArgs, options); return options; } BlockAssembler::BlockAssembler(Chainstate& chainstate, const CTxMemPool* mempool) - : BlockAssembler(chainstate, mempool, DefaultOptions()) {} + : BlockAssembler(chainstate, mempool, ConfiguredOptions()) {} void BlockAssembler::resetBlock() { @@ -170,7 +176,8 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc pblocktemplate->vTxSigOpsCost[0] = WITNESS_SCALE_FACTOR * GetLegacySigOpCount(*pblock->vtx[0]); BlockValidationState state; - if (!TestBlockValidity(state, chainparams, m_chainstate, *pblock, pindexPrev, GetAdjustedTime, false, false)) { + if (test_block_validity && !TestBlockValidity(state, chainparams, m_chainstate, *pblock, pindexPrev, + GetAdjustedTime, /*fCheckPOW=*/false, /*fCheckMerkleRoot=*/false)) { throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, state.ToString())); } const auto time_2{SteadyClock::now()}; diff --git a/src/node/miner.h b/src/node/miner.h index 8cc5e07237..ea9e470a64 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -16,6 +16,7 @@ #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index_container.hpp> +class ArgsManager; class ChainstateManager; class CBlockIndex; class CChainParams; @@ -135,6 +136,9 @@ private: unsigned int nBlockMaxWeight; CFeeRate blockMinFeeRate; + // Whether to call TestBlockValidity() at the end of CreateNewBlock(). + const bool test_block_validity; + // Information on the current status of the block uint64_t nBlockWeight; uint64_t nBlockTx; @@ -155,6 +159,7 @@ public: Options(); size_t nBlockMaxWeight; CFeeRate blockMinFeeRate; + bool test_block_validity; }; explicit BlockAssembler(Chainstate& chainstate, const CTxMemPool* mempool); @@ -197,6 +202,9 @@ int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParam /** Update an old GenerateCoinbaseCommitment from CreateNewBlock after the block txs have changed */ void RegenerateCommitments(CBlock& block, ChainstateManager& chainman); + +/** Apply -blockmintxfee and -blockmaxweight options from ArgsManager to BlockAssembler options. */ +void ApplyArgsManOptions(const ArgsManager& gArgs, BlockAssembler::Options& options); } // namespace node #endif // BITCOIN_NODE_MINER_H diff --git a/src/node/utxo_snapshot.cpp b/src/node/utxo_snapshot.cpp index bab1b75211..cccf95e552 100644 --- a/src/node/utxo_snapshot.cpp +++ b/src/node/utxo_snapshot.cpp @@ -7,12 +7,17 @@ #include <fs.h> #include <logging.h> #include <streams.h> +#include <sync.h> +#include <tinyformat.h> +#include <txdb.h> #include <uint256.h> #include <util/system.h> #include <validation.h> +#include <cassert> #include <cstdio> #include <optional> +#include <string> namespace node { diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index b5ed9ef9fe..c5c018c9e8 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -7,13 +7,16 @@ #define BITCOIN_NODE_UTXO_SNAPSHOT_H #include <fs.h> -#include <uint256.h> +#include <kernel/cs_main.h> #include <serialize.h> -#include <validation.h> +#include <sync.h> +#include <uint256.h> +#include <cstdint> #include <optional> +#include <string_view> -extern RecursiveMutex cs_main; +class Chainstate; namespace node { //! Metadata describing a serialized version of a UTXO set from which an diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index e4eb932e5c..d244de1bb2 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -528,7 +528,7 @@ bool CBlockPolicyEstimator::_removeTx(const uint256& hash, bool inBlock) } CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath) - : m_estimation_filepath{estimation_filepath}, nBestSeenHeight{0}, firstRecordedHeight{0}, historicalFirst{0}, historicalBest{0}, trackedTxs{0}, untrackedTxs{0} + : m_estimation_filepath{estimation_filepath} { static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero"); size_t bucketIndex = 0; diff --git a/src/policy/fees.h b/src/policy/fees.h index dd4f031180..1c24b8c7c3 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -242,16 +242,16 @@ public: private: mutable Mutex m_cs_fee_estimator; - unsigned int nBestSeenHeight GUARDED_BY(m_cs_fee_estimator); - unsigned int firstRecordedHeight GUARDED_BY(m_cs_fee_estimator); - unsigned int historicalFirst GUARDED_BY(m_cs_fee_estimator); - unsigned int historicalBest GUARDED_BY(m_cs_fee_estimator); + unsigned int nBestSeenHeight GUARDED_BY(m_cs_fee_estimator){0}; + unsigned int firstRecordedHeight GUARDED_BY(m_cs_fee_estimator){0}; + unsigned int historicalFirst GUARDED_BY(m_cs_fee_estimator){0}; + unsigned int historicalBest GUARDED_BY(m_cs_fee_estimator){0}; struct TxStatsInfo { - unsigned int blockHeight; - unsigned int bucketIndex; - TxStatsInfo() : blockHeight(0), bucketIndex(0) {} + unsigned int blockHeight{0}; + unsigned int bucketIndex{0}; + TxStatsInfo() {} }; // map of txids to information about that transaction @@ -262,8 +262,8 @@ private: std::unique_ptr<TxConfirmStats> shortStats PT_GUARDED_BY(m_cs_fee_estimator); std::unique_ptr<TxConfirmStats> longStats PT_GUARDED_BY(m_cs_fee_estimator); - unsigned int trackedTxs GUARDED_BY(m_cs_fee_estimator); - unsigned int untrackedTxs GUARDED_BY(m_cs_fee_estimator); + unsigned int trackedTxs GUARDED_BY(m_cs_fee_estimator){0}; + unsigned int untrackedTxs GUARDED_BY(m_cs_fee_estimator){0}; std::vector<double> buckets GUARDED_BY(m_cs_fee_estimator); // The upper-bound of the range for the bucket (inclusive) std::map<double, unsigned int> bucketMap GUARDED_BY(m_cs_fee_estimator); // Map of bucket upper-bound to index into all vectors by bucket diff --git a/src/psbt.h b/src/psbt.h index 40d69cd454..8fafadcea9 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -206,7 +206,7 @@ struct PSBTInput // Taproot fields std::vector<unsigned char> m_tap_key_sig; std::map<std::pair<XOnlyPubKey, uint256>, std::vector<unsigned char>> m_tap_script_sigs; - std::map<std::pair<CScript, int>, std::set<std::vector<unsigned char>, ShortestVectorFirstComparator>> m_tap_scripts; + std::map<std::pair<std::vector<unsigned char>, int>, std::set<std::vector<unsigned char>, ShortestVectorFirstComparator>> m_tap_scripts; std::map<XOnlyPubKey, std::pair<std::set<uint256>, KeyOriginInfo>> m_tap_bip32_paths; XOnlyPubKey m_tap_internal_key; uint256 m_tap_merkle_root; @@ -621,7 +621,7 @@ struct PSBTInput } uint8_t leaf_ver = script_v.back(); script_v.pop_back(); - const auto leaf_script = std::make_pair(CScript(script_v.begin(), script_v.end()), (int)leaf_ver); + const auto leaf_script = std::make_pair(script_v, (int)leaf_ver); m_tap_scripts[leaf_script].insert(std::vector<unsigned char>(key.begin() + 1, key.end())); break; } @@ -713,7 +713,7 @@ struct PSBTOutput CScript witness_script; std::map<CPubKey, KeyOriginInfo> hd_keypaths; XOnlyPubKey m_tap_internal_key; - std::vector<std::tuple<uint8_t, uint8_t, CScript>> m_tap_tree; + std::vector<std::tuple<uint8_t, uint8_t, std::vector<unsigned char>>> m_tap_tree; std::map<XOnlyPubKey, std::pair<std::set<uint256>, KeyOriginInfo>> m_tap_bip32_paths; std::map<std::vector<unsigned char>, std::vector<unsigned char>> unknown; std::set<PSBTProprietary> m_proprietary; @@ -864,7 +864,7 @@ struct PSBTOutput while (!s_tree.empty()) { uint8_t depth; uint8_t leaf_ver; - CScript script; + std::vector<unsigned char> script; s_tree >> depth; s_tree >> leaf_ver; s_tree >> script; diff --git a/src/pubkey.cpp b/src/pubkey.cpp index ff0d751a72..ae5dccfb5a 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -16,10 +16,16 @@ #include <algorithm> #include <cassert> -namespace +namespace { + +struct Secp256k1SelfTester { -/* Global secp256k1_context object used for verification. */ -secp256k1_context* secp256k1_context_verify = nullptr; + Secp256k1SelfTester() { + /* Run libsecp256k1 self-test before using the secp256k1_context_static. */ + secp256k1_selftest(); + } +} SECP256K1_SELFTESTER; + } // namespace /** This function is taken from the libsecp256k1 distribution and implements @@ -32,7 +38,7 @@ secp256k1_context* secp256k1_context_verify = nullptr; * strict DER before being passed to this module, and we know it supports all * violations present in the blockchain before that point. */ -int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_signature* sig, const unsigned char *input, size_t inputlen) { +int ecdsa_signature_parse_der_lax(secp256k1_ecdsa_signature* sig, const unsigned char *input, size_t inputlen) { size_t rpos, rlen, spos, slen; size_t pos = 0; size_t lenbyte; @@ -40,7 +46,7 @@ int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_ int overflow = 0; /* Hack to initialize sig with a correctly-parsed but invalid signature. */ - secp256k1_ecdsa_signature_parse_compact(ctx, sig, tmpsig); + secp256k1_ecdsa_signature_parse_compact(secp256k1_context_static, sig, tmpsig); /* Sequence tag byte */ if (pos == inputlen || input[pos] != 0x30) { @@ -163,13 +169,13 @@ int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_ } if (!overflow) { - overflow = !secp256k1_ecdsa_signature_parse_compact(ctx, sig, tmpsig); + overflow = !secp256k1_ecdsa_signature_parse_compact(secp256k1_context_static, sig, tmpsig); } if (overflow) { /* Overwrite the result again with a correctly-parsed but invalid signature if parsing failed. */ memset(tmpsig, 0, 64); - secp256k1_ecdsa_signature_parse_compact(ctx, sig, tmpsig); + secp256k1_ecdsa_signature_parse_compact(secp256k1_context_static, sig, tmpsig); } return 1; } @@ -200,15 +206,15 @@ std::vector<CKeyID> XOnlyPubKey::GetKeyIDs() const bool XOnlyPubKey::IsFullyValid() const { secp256k1_xonly_pubkey pubkey; - return secp256k1_xonly_pubkey_parse(secp256k1_context_verify, &pubkey, m_keydata.data()); + return secp256k1_xonly_pubkey_parse(secp256k1_context_static, &pubkey, m_keydata.data()); } bool XOnlyPubKey::VerifySchnorr(const uint256& msg, Span<const unsigned char> sigbytes) const { assert(sigbytes.size() == 64); secp256k1_xonly_pubkey pubkey; - if (!secp256k1_xonly_pubkey_parse(secp256k1_context_verify, &pubkey, m_keydata.data())) return false; - return secp256k1_schnorrsig_verify(secp256k1_context_verify, sigbytes.data(), msg.begin(), 32, &pubkey); + if (!secp256k1_xonly_pubkey_parse(secp256k1_context_static, &pubkey, m_keydata.data())) return false; + return secp256k1_schnorrsig_verify(secp256k1_context_static, sigbytes.data(), msg.begin(), 32, &pubkey); } static const HashWriter HASHER_TAPTWEAK{TaggedHash("TapTweak")}; @@ -227,23 +233,23 @@ uint256 XOnlyPubKey::ComputeTapTweakHash(const uint256* merkle_root) const bool XOnlyPubKey::CheckTapTweak(const XOnlyPubKey& internal, const uint256& merkle_root, bool parity) const { secp256k1_xonly_pubkey internal_key; - if (!secp256k1_xonly_pubkey_parse(secp256k1_context_verify, &internal_key, internal.data())) return false; + if (!secp256k1_xonly_pubkey_parse(secp256k1_context_static, &internal_key, internal.data())) return false; uint256 tweak = internal.ComputeTapTweakHash(&merkle_root); - return secp256k1_xonly_pubkey_tweak_add_check(secp256k1_context_verify, m_keydata.begin(), parity, &internal_key, tweak.begin()); + return secp256k1_xonly_pubkey_tweak_add_check(secp256k1_context_static, m_keydata.begin(), parity, &internal_key, tweak.begin()); } std::optional<std::pair<XOnlyPubKey, bool>> XOnlyPubKey::CreateTapTweak(const uint256* merkle_root) const { secp256k1_xonly_pubkey base_point; - if (!secp256k1_xonly_pubkey_parse(secp256k1_context_verify, &base_point, data())) return std::nullopt; + if (!secp256k1_xonly_pubkey_parse(secp256k1_context_static, &base_point, data())) return std::nullopt; secp256k1_pubkey out; uint256 tweak = ComputeTapTweakHash(merkle_root); - if (!secp256k1_xonly_pubkey_tweak_add(secp256k1_context_verify, &out, &base_point, tweak.data())) return std::nullopt; + if (!secp256k1_xonly_pubkey_tweak_add(secp256k1_context_static, &out, &base_point, tweak.data())) return std::nullopt; int parity = -1; std::pair<XOnlyPubKey, bool> ret; secp256k1_xonly_pubkey out_xonly; - if (!secp256k1_xonly_pubkey_from_pubkey(secp256k1_context_verify, &out_xonly, &parity, &out)) return std::nullopt; - secp256k1_xonly_pubkey_serialize(secp256k1_context_verify, ret.first.begin(), &out_xonly); + if (!secp256k1_xonly_pubkey_from_pubkey(secp256k1_context_static, &out_xonly, &parity, &out)) return std::nullopt; + secp256k1_xonly_pubkey_serialize(secp256k1_context_static, ret.first.begin(), &out_xonly); assert(parity == 0 || parity == 1); ret.second = parity; return ret; @@ -255,17 +261,16 @@ bool CPubKey::Verify(const uint256 &hash, const std::vector<unsigned char>& vchS return false; secp256k1_pubkey pubkey; secp256k1_ecdsa_signature sig; - assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); - if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { + if (!secp256k1_ec_pubkey_parse(secp256k1_context_static, &pubkey, vch, size())) { return false; } - if (!ecdsa_signature_parse_der_lax(secp256k1_context_verify, &sig, vchSig.data(), vchSig.size())) { + if (!ecdsa_signature_parse_der_lax(&sig, vchSig.data(), vchSig.size())) { return false; } /* libsecp256k1's ECDSA verification requires lower-S signatures, which have * not historically been enforced in Bitcoin, so normalize them first. */ - secp256k1_ecdsa_signature_normalize(secp256k1_context_verify, &sig, &sig); - return secp256k1_ecdsa_verify(secp256k1_context_verify, &sig, hash.begin(), &pubkey); + secp256k1_ecdsa_signature_normalize(secp256k1_context_static, &sig, &sig); + return secp256k1_ecdsa_verify(secp256k1_context_static, &sig, hash.begin(), &pubkey); } bool CPubKey::RecoverCompact(const uint256 &hash, const std::vector<unsigned char>& vchSig) { @@ -275,16 +280,15 @@ bool CPubKey::RecoverCompact(const uint256 &hash, const std::vector<unsigned cha bool fComp = ((vchSig[0] - 27) & 4) != 0; secp256k1_pubkey pubkey; secp256k1_ecdsa_recoverable_signature sig; - assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); - if (!secp256k1_ecdsa_recoverable_signature_parse_compact(secp256k1_context_verify, &sig, &vchSig[1], recid)) { + if (!secp256k1_ecdsa_recoverable_signature_parse_compact(secp256k1_context_static, &sig, &vchSig[1], recid)) { return false; } - if (!secp256k1_ecdsa_recover(secp256k1_context_verify, &pubkey, &sig, hash.begin())) { + if (!secp256k1_ecdsa_recover(secp256k1_context_static, &pubkey, &sig, hash.begin())) { return false; } unsigned char pub[SIZE]; size_t publen = SIZE; - secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, fComp ? SECP256K1_EC_COMPRESSED : SECP256K1_EC_UNCOMPRESSED); + secp256k1_ec_pubkey_serialize(secp256k1_context_static, pub, &publen, &pubkey, fComp ? SECP256K1_EC_COMPRESSED : SECP256K1_EC_UNCOMPRESSED); Set(pub, pub + publen); return true; } @@ -293,21 +297,19 @@ bool CPubKey::IsFullyValid() const { if (!IsValid()) return false; secp256k1_pubkey pubkey; - assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); - return secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size()); + return secp256k1_ec_pubkey_parse(secp256k1_context_static, &pubkey, vch, size()); } bool CPubKey::Decompress() { if (!IsValid()) return false; secp256k1_pubkey pubkey; - assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); - if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { + if (!secp256k1_ec_pubkey_parse(secp256k1_context_static, &pubkey, vch, size())) { return false; } unsigned char pub[SIZE]; size_t publen = SIZE; - secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, SECP256K1_EC_UNCOMPRESSED); + secp256k1_ec_pubkey_serialize(secp256k1_context_static, pub, &publen, &pubkey, SECP256K1_EC_UNCOMPRESSED); Set(pub, pub + publen); return true; } @@ -320,16 +322,15 @@ bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChi BIP32Hash(cc, nChild, *begin(), begin()+1, out); memcpy(ccChild.begin(), out+32, 32); secp256k1_pubkey pubkey; - assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); - if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { + if (!secp256k1_ec_pubkey_parse(secp256k1_context_static, &pubkey, vch, size())) { return false; } - if (!secp256k1_ec_pubkey_tweak_add(secp256k1_context_verify, &pubkey, out)) { + if (!secp256k1_ec_pubkey_tweak_add(secp256k1_context_static, &pubkey, out)) { return false; } unsigned char pub[COMPRESSED_SIZE]; size_t publen = COMPRESSED_SIZE; - secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, SECP256K1_EC_COMPRESSED); + secp256k1_ec_pubkey_serialize(secp256k1_context_static, pub, &publen, &pubkey, SECP256K1_EC_COMPRESSED); pubkeyChild.Set(pub, pub + publen); return true; } @@ -375,35 +376,8 @@ bool CExtPubKey::Derive(CExtPubKey &out, unsigned int _nChild) const { /* static */ bool CPubKey::CheckLowS(const std::vector<unsigned char>& vchSig) { secp256k1_ecdsa_signature sig; - assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); - if (!ecdsa_signature_parse_der_lax(secp256k1_context_verify, &sig, vchSig.data(), vchSig.size())) { + if (!ecdsa_signature_parse_der_lax(&sig, vchSig.data(), vchSig.size())) { return false; } - return (!secp256k1_ecdsa_signature_normalize(secp256k1_context_verify, nullptr, &sig)); -} - -/* static */ int ECCVerifyHandle::refcount = 0; - -ECCVerifyHandle::ECCVerifyHandle() -{ - if (refcount == 0) { - assert(secp256k1_context_verify == nullptr); - secp256k1_context_verify = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); - assert(secp256k1_context_verify != nullptr); - } - refcount++; -} - -ECCVerifyHandle::~ECCVerifyHandle() -{ - refcount--; - if (refcount == 0) { - assert(secp256k1_context_verify != nullptr); - secp256k1_context_destroy(secp256k1_context_verify); - secp256k1_context_verify = nullptr; - } -} - -const secp256k1_context* GetVerifyContext() { - return secp256k1_context_verify; + return (!secp256k1_ecdsa_signature_normalize(secp256k1_context_static, nullptr, &sig)); } diff --git a/src/pubkey.h b/src/pubkey.h index 047fcd0f6d..b3edafea7f 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -330,21 +330,4 @@ struct CExtPubKey { [[nodiscard]] bool Derive(CExtPubKey& out, unsigned int nChild) const; }; -/** Users of this module must hold an ECCVerifyHandle. The constructor and - * destructor of these are not allowed to run in parallel, though. */ -class ECCVerifyHandle -{ - static int refcount; - -public: - ECCVerifyHandle(); - ~ECCVerifyHandle(); -}; - -typedef struct secp256k1_context_struct secp256k1_context; - -/** Access to the internal secp256k1 context used for verification. Only intended to be used - * by key.cpp. */ -const secp256k1_context* GetVerifyContext(); - #endif // BITCOIN_PUBKEY_H diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index cc24b7cfcb..b888fc43e2 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -64,7 +64,6 @@ protected: AddressBookPage::AddressBookPage(const PlatformStyle *platformStyle, Mode _mode, Tabs _tab, QWidget *parent) : QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::AddressBookPage), - model(nullptr), mode(_mode), tab(_tab) { diff --git a/src/qt/addressbookpage.h b/src/qt/addressbookpage.h index 144990f419..283209d00c 100644 --- a/src/qt/addressbookpage.h +++ b/src/qt/addressbookpage.h @@ -49,7 +49,7 @@ public Q_SLOTS: private: Ui::AddressBookPage *ui; - AddressTableModel *model; + AddressTableModel* model{nullptr}; Mode mode; Tabs tab; QString returnValue; diff --git a/src/qt/askpassphrasedialog.cpp b/src/qt/askpassphrasedialog.cpp index f16700bfd9..d15aba5cdd 100644 --- a/src/qt/askpassphrasedialog.cpp +++ b/src/qt/askpassphrasedialog.cpp @@ -23,8 +23,6 @@ AskPassphraseDialog::AskPassphraseDialog(Mode _mode, QWidget *parent, SecureStri QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::AskPassphraseDialog), mode(_mode), - model(nullptr), - fCapsLock(false), m_passphrase_out(passphrase_out) { ui->setupUi(this); diff --git a/src/qt/askpassphrasedialog.h b/src/qt/askpassphrasedialog.h index 66031b08d3..370ea1de7e 100644 --- a/src/qt/askpassphrasedialog.h +++ b/src/qt/askpassphrasedialog.h @@ -38,8 +38,8 @@ public: private: Ui::AskPassphraseDialog *ui; Mode mode; - WalletModel *model; - bool fCapsLock; + WalletModel* model{nullptr}; + bool fCapsLock{false}; SecureString* m_passphrase_out; private Q_SLOTS: diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index b6720b0433..59f433749d 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -228,14 +228,8 @@ void DebugMessageHandler(QtMsgType type, const QMessageLogContext& context, cons static int qt_argc = 1; static const char* qt_argv = "bitcoin-qt"; -BitcoinApplication::BitcoinApplication(): - QApplication(qt_argc, const_cast<char **>(&qt_argv)), - optionsModel(nullptr), - clientModel(nullptr), - window(nullptr), - pollShutdownTimer(nullptr), - returnValue(0), - platformStyle(nullptr) +BitcoinApplication::BitcoinApplication() + : QApplication(qt_argc, const_cast<char**>(&qt_argv)) { // Qt runs setlocale(LC_ALL, "") on initialization. RegisterMetaTypes(); @@ -592,29 +586,30 @@ int GuiMain(int argc, char* argv[]) // Gracefully exit if the user cancels if (!Intro::showIfNeeded(did_show_intro, prune_MiB)) return EXIT_SUCCESS; - /// 6. Determine availability of data directory and parse bitcoin.conf - /// - Do not call gArgs.GetDataDirNet() before this step finishes + /// 6a. Determine availability of data directory if (!CheckDataDirOption()) { InitError(strprintf(Untranslated("Specified data directory \"%s\" does not exist.\n"), gArgs.GetArg("-datadir", ""))); QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error: Specified data directory \"%1\" does not exist.").arg(QString::fromStdString(gArgs.GetArg("-datadir", "")))); return EXIT_FAILURE; } - if (!gArgs.ReadConfigFiles(error, true)) { - InitError(strprintf(Untranslated("Error reading configuration file: %s\n"), error)); - QMessageBox::critical(nullptr, PACKAGE_NAME, - QObject::tr("Error: Cannot parse configuration file: %1.").arg(QString::fromStdString(error))); - return EXIT_FAILURE; - } + try { + /// 6b. Parse bitcoin.conf + /// - Do not call gArgs.GetDataDirNet() before this step finishes + if (!gArgs.ReadConfigFiles(error, true)) { + InitError(strprintf(Untranslated("Error reading configuration file: %s\n"), error)); + QMessageBox::critical(nullptr, PACKAGE_NAME, + QObject::tr("Error: Cannot parse configuration file: %1.").arg(QString::fromStdString(error))); + return EXIT_FAILURE; + } - /// 7. Determine network (and switch to network specific options) - // - Do not call Params() before this step - // - Do this after parsing the configuration file, as the network can be switched there - // - QSettings() will use the new application name after this, resulting in network-specific settings - // - Needs to be done before createOptionsModel + /// 7. Determine network (and switch to network specific options) + // - Do not call Params() before this step + // - Do this after parsing the configuration file, as the network can be switched there + // - QSettings() will use the new application name after this, resulting in network-specific settings + // - Needs to be done before createOptionsModel - // Check for chain settings (Params() calls are only valid after this clause) - try { + // Check for chain settings (Params() calls are only valid after this clause) SelectParams(gArgs.GetChainName()); } catch(std::exception &e) { InitError(Untranslated(strprintf("%s\n", e.what()))); diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 61a18ba6a1..9174e23de6 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -96,16 +96,16 @@ protected: private: std::optional<InitExecutor> m_executor; - OptionsModel *optionsModel; - ClientModel *clientModel; - BitcoinGUI *window; - QTimer *pollShutdownTimer; + OptionsModel* optionsModel{nullptr}; + ClientModel* clientModel{nullptr}; + BitcoinGUI* window{nullptr}; + QTimer* pollShutdownTimer{nullptr}; #ifdef ENABLE_WALLET PaymentServer* paymentServer{nullptr}; WalletController* m_wallet_controller{nullptr}; #endif - int returnValue; - const PlatformStyle *platformStyle; + int returnValue{0}; + const PlatformStyle* platformStyle{nullptr}; std::unique_ptr<QWidget> shutdownWindow; SplashScreen* m_splash = nullptr; std::unique_ptr<interfaces::Node> m_node; diff --git a/src/qt/bitcoinamountfield.cpp b/src/qt/bitcoinamountfield.cpp index 765a2a5ffc..a7e2d22488 100644 --- a/src/qt/bitcoinamountfield.cpp +++ b/src/qt/bitcoinamountfield.cpp @@ -217,9 +217,8 @@ Q_SIGNALS: #include <qt/bitcoinamountfield.moc> -BitcoinAmountField::BitcoinAmountField(QWidget *parent) : - QWidget(parent), - amount(nullptr) +BitcoinAmountField::BitcoinAmountField(QWidget* parent) + : QWidget(parent) { amount = new AmountSpinBox(this); amount->setLocale(QLocale::c()); diff --git a/src/qt/bitcoinamountfield.h b/src/qt/bitcoinamountfield.h index a40cd38332..f710915a49 100644 --- a/src/qt/bitcoinamountfield.h +++ b/src/qt/bitcoinamountfield.h @@ -74,7 +74,7 @@ protected: bool eventFilter(QObject *object, QEvent *event) override; private: - AmountSpinBox *amount; + AmountSpinBox* amount{nullptr}; QValueComboBox *unit; private Q_SLOTS: diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index a0731b337a..a7ffd367d7 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -1541,10 +1541,8 @@ bool BitcoinGUI::isPrivacyModeActivated() const return m_mask_values_action->isChecked(); } -UnitDisplayStatusBarControl::UnitDisplayStatusBarControl(const PlatformStyle *platformStyle) - : optionsModel(nullptr), - menu(nullptr), - m_platform_style{platformStyle} +UnitDisplayStatusBarControl::UnitDisplayStatusBarControl(const PlatformStyle* platformStyle) + : m_platform_style{platformStyle} { createContextMenu(); setToolTip(tr("Unit to show amounts in. Click to select another unit.")); diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 9970e87833..1a83057146 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -333,8 +333,8 @@ protected: void changeEvent(QEvent* e) override; private: - OptionsModel *optionsModel; - QMenu* menu; + OptionsModel* optionsModel{nullptr}; + QMenu* menu{nullptr}; const PlatformStyle* m_platform_style; /** Shows context menu with Display Unit options by the mouse coordinates */ diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 6f53626fd1..c0d1a0e226 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -34,8 +34,6 @@ ClientModel::ClientModel(interfaces::Node& node, OptionsModel *_optionsModel, QO QObject(parent), m_node(node), optionsModel(_optionsModel), - peerTableModel(nullptr), - banTableModel(nullptr), m_thread(new QThread(this)) { cachedBestHeaderHeight = -1; diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 49ac75452f..9ff64fe772 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -104,9 +104,9 @@ private: std::unique_ptr<interfaces::Handler> m_handler_notify_block_tip; std::unique_ptr<interfaces::Handler> m_handler_notify_header_tip; OptionsModel *optionsModel; - PeerTableModel *peerTableModel; + PeerTableModel* peerTableModel{nullptr}; PeerTableSortProxy* m_peer_table_sort_proxy{nullptr}; - BanTableModel *banTableModel; + BanTableModel* banTableModel{nullptr}; //! A thread to interact with m_node asynchronously QThread* const m_thread; diff --git a/src/qt/csvmodelwriter.cpp b/src/qt/csvmodelwriter.cpp index 656afb6e87..445ea41475 100644 --- a/src/qt/csvmodelwriter.cpp +++ b/src/qt/csvmodelwriter.cpp @@ -8,9 +8,9 @@ #include <QFile> #include <QTextStream> -CSVModelWriter::CSVModelWriter(const QString &_filename, QObject *parent) : - QObject(parent), - filename(_filename), model(nullptr) +CSVModelWriter::CSVModelWriter(const QString& _filename, QObject* parent) + : QObject(parent), + filename(_filename) { } diff --git a/src/qt/csvmodelwriter.h b/src/qt/csvmodelwriter.h index ad247b6859..a1e77826a8 100644 --- a/src/qt/csvmodelwriter.h +++ b/src/qt/csvmodelwriter.h @@ -32,7 +32,7 @@ public: private: QString filename; - const QAbstractItemModel *model; + const QAbstractItemModel* model{nullptr}; struct Column { diff --git a/src/qt/editaddressdialog.cpp b/src/qt/editaddressdialog.cpp index fa27635981..9b3319415d 100644 --- a/src/qt/editaddressdialog.cpp +++ b/src/qt/editaddressdialog.cpp @@ -12,12 +12,10 @@ #include <QMessageBox> -EditAddressDialog::EditAddressDialog(Mode _mode, QWidget *parent) : - QDialog(parent, GUIUtil::dialog_flags), - ui(new Ui::EditAddressDialog), - mapper(nullptr), - mode(_mode), - model(nullptr) +EditAddressDialog::EditAddressDialog(Mode _mode, QWidget* parent) + : QDialog(parent, GUIUtil::dialog_flags), + ui(new Ui::EditAddressDialog), + mode(_mode) { ui->setupUi(this); diff --git a/src/qt/editaddressdialog.h b/src/qt/editaddressdialog.h index f7ad80bb2d..7bfadcfbcc 100644 --- a/src/qt/editaddressdialog.h +++ b/src/qt/editaddressdialog.h @@ -49,9 +49,9 @@ private: QString getDuplicateAddressWarning() const; Ui::EditAddressDialog *ui; - QDataWidgetMapper *mapper; + QDataWidgetMapper* mapper{nullptr}; Mode mode; - AddressTableModel *model; + AddressTableModel* model{nullptr}; QString address; }; diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index ea5b1a7242..12aa02340a 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -122,8 +122,6 @@ int GetPruneTargetGB() Intro::Intro(QWidget *parent, int64_t blockchain_size_gb, int64_t chain_state_size_gb) : QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::Intro), - thread(nullptr), - signalled(false), m_blockchain_size_gb(blockchain_size_gb), m_chain_state_size_gb(chain_state_size_gb), m_prune_target_gb{GetPruneTargetGB()} diff --git a/src/qt/intro.h b/src/qt/intro.h index d9c45007e1..900c657b27 100644 --- a/src/qt/intro.h +++ b/src/qt/intro.h @@ -64,9 +64,9 @@ private Q_SLOTS: private: Ui::Intro *ui; - QThread *thread; + QThread* thread{nullptr}; QMutex mutex; - bool signalled; + bool signalled{false}; QString pathToCheck; const int64_t m_blockchain_size_gb; const int64_t m_chain_state_size_gb; diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index 585599141a..b09e230bce 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -12,13 +12,10 @@ #include <QPropertyAnimation> #include <QResizeEvent> -ModalOverlay::ModalOverlay(bool enable_wallet, QWidget *parent) : -QWidget(parent), -ui(new Ui::ModalOverlay), -bestHeaderHeight(0), -bestHeaderDate(QDateTime()), -layerIsVisible(false), -userClosed(false) +ModalOverlay::ModalOverlay(bool enable_wallet, QWidget* parent) + : QWidget(parent), + ui(new Ui::ModalOverlay), + bestHeaderDate(QDateTime()) { ui->setupUi(this); connect(ui->closeButton, &QPushButton::clicked, this, &ModalOverlay::closeClicked); diff --git a/src/qt/modaloverlay.h b/src/qt/modaloverlay.h index 8d0c8fbf9a..40e487b249 100644 --- a/src/qt/modaloverlay.h +++ b/src/qt/modaloverlay.h @@ -45,11 +45,11 @@ protected: private: Ui::ModalOverlay *ui; - int bestHeaderHeight; //best known height (based on the headers) + int bestHeaderHeight{0}; // best known height (based on the headers) QDateTime bestHeaderDate; QVector<QPair<qint64, double> > blockProcessTime; - bool layerIsVisible; - bool userClosed; + bool layerIsVisible{false}; + bool userClosed{false}; QPropertyAnimation m_animation; void UpdateHeaderSyncLabel(); void UpdateHeaderPresyncLabel(int height, const QDateTime& blockDate); diff --git a/src/qt/notificator.cpp b/src/qt/notificator.cpp index c97644ad7d..88bc33098a 100644 --- a/src/qt/notificator.cpp +++ b/src/qt/notificator.cpp @@ -32,11 +32,7 @@ Notificator::Notificator(const QString &_programName, QSystemTrayIcon *_trayIcon QObject(_parent), parent(_parent), programName(_programName), - mode(None), trayIcon(_trayIcon) -#ifdef USE_DBUS - ,interface(nullptr) -#endif { if(_trayIcon && _trayIcon->supportsMessages()) { diff --git a/src/qt/notificator.h b/src/qt/notificator.h index ac4bbe5a0c..1fd8181a22 100644 --- a/src/qt/notificator.h +++ b/src/qt/notificator.h @@ -61,10 +61,10 @@ private: UserNotificationCenter /**< Use the 10.8+ User Notification Center (Mac only) */ }; QString programName; - Mode mode; + Mode mode{None}; QSystemTrayIcon *trayIcon; #ifdef USE_DBUS - QDBusInterface *interface; + QDBusInterface* interface{nullptr}; void notifyDBus(Class cls, const QString &title, const QString &text, const QIcon &icon, int millisTimeout); #endif diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 9f53fbf429..53b0c3832b 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -31,11 +31,9 @@ #include <QSystemTrayIcon> #include <QTimer> -OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : - QDialog(parent, GUIUtil::dialog_flags), - ui(new Ui::OptionsDialog), - model(nullptr), - mapper(nullptr) +OptionsDialog::OptionsDialog(QWidget* parent, bool enableWallet) + : QDialog(parent, GUIUtil::dialog_flags), + ui(new Ui::OptionsDialog) { ui->setupUi(this); diff --git a/src/qt/optionsdialog.h b/src/qt/optionsdialog.h index 6cf0d1064f..031e4d3163 100644 --- a/src/qt/optionsdialog.h +++ b/src/qt/optionsdialog.h @@ -75,8 +75,8 @@ Q_SIGNALS: private: Ui::OptionsDialog *ui; ClientModel* m_client_model{nullptr}; - OptionsModel *model; - QDataWidgetMapper *mapper; + OptionsModel* model{nullptr}; + QDataWidgetMapper* mapper{nullptr}; }; #endif // BITCOIN_QT_OPTIONSDIALOG_H diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index cd0a1a19ee..00b7952ca4 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -31,7 +31,7 @@ const char *DEFAULT_GUI_PROXY_HOST = "127.0.0.1"; -static const QString GetDefaultProxyAddress(); +static QString GetDefaultProxyAddress(); /** Map GUI option ID to node setting name. */ static const char* SettingName(OptionsModel::OptionID option) @@ -308,7 +308,7 @@ static std::string ProxyString(bool is_set, QString ip, QString port) return is_set ? QString(ip + ":" + port).toStdString() : ""; } -static const QString GetDefaultProxyAddress() +static QString GetDefaultProxyAddress() { return QString("%1:%2").arg(DEFAULT_GUI_PROXY_HOST).arg(DEFAULT_GUI_PROXY_PORT); } diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 3a4861afd4..3cd5f4061c 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -140,8 +140,6 @@ private: OverviewPage::OverviewPage(const PlatformStyle *platformStyle, QWidget *parent) : QWidget(parent), ui(new Ui::OverviewPage), - clientModel(nullptr), - walletModel(nullptr), m_platform_style{platformStyle}, txdelegate(new TxViewDelegate(platformStyle, this)) { @@ -264,7 +262,6 @@ void OverviewPage::setWalletModel(WalletModel *model) // Set up transaction list filter.reset(new TransactionFilterProxy()); filter->setSourceModel(model->getTransactionTableModel()); - filter->setLimit(NUM_ITEMS); filter->setDynamicSortFilter(true); filter->setSortRole(Qt::EditRole); filter->setShowInactive(false); @@ -273,6 +270,10 @@ void OverviewPage::setWalletModel(WalletModel *model) ui->listTransactions->setModel(filter.get()); ui->listTransactions->setModelColumn(TransactionTableModel::ToAddress); + connect(filter.get(), &TransactionFilterProxy::rowsInserted, this, &OverviewPage::LimitTransactionRows); + connect(filter.get(), &TransactionFilterProxy::rowsRemoved, this, &OverviewPage::LimitTransactionRows); + connect(filter.get(), &TransactionFilterProxy::rowsMoved, this, &OverviewPage::LimitTransactionRows); + LimitTransactionRows(); // Keep up to date with wallet setBalance(model->getCachedBalance()); connect(model, &WalletModel::balanceChanged, this, &OverviewPage::setBalance); @@ -301,6 +302,16 @@ void OverviewPage::changeEvent(QEvent* e) QWidget::changeEvent(e); } +// Only show most recent NUM_ITEMS rows +void OverviewPage::LimitTransactionRows() +{ + if (filter && ui->listTransactions && ui->listTransactions->model() && filter.get() == ui->listTransactions->model()) { + for (int i = 0; i < filter->rowCount(); ++i) { + ui->listTransactions->setRowHidden(i, i >= NUM_ITEMS); + } + } +} + void OverviewPage::updateDisplayUnit() { if (walletModel && walletModel->getOptionsModel()) { diff --git a/src/qt/overviewpage.h b/src/qt/overviewpage.h index c03b87c616..5c487ee116 100644 --- a/src/qt/overviewpage.h +++ b/src/qt/overviewpage.h @@ -50,8 +50,8 @@ protected: private: Ui::OverviewPage *ui; - ClientModel *clientModel; - WalletModel *walletModel; + ClientModel* clientModel{nullptr}; + WalletModel* walletModel{nullptr}; bool m_privacy{false}; const PlatformStyle* m_platform_style; @@ -60,6 +60,7 @@ private: std::unique_ptr<TransactionFilterProxy> filter; private Q_SLOTS: + void LimitTransactionRows(); void updateDisplayUnit(); void handleTransactionClicked(const QModelIndex &index); void updateAlerts(const QString &warnings); diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index c6aeea065b..3f9d1b040b 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -126,11 +126,8 @@ bool PaymentServer::ipcSendCommandLine() return fResult; } -PaymentServer::PaymentServer(QObject* parent, bool startLocalServer) : - QObject(parent), - saveURIs(true), - uriServer(nullptr), - optionsModel(nullptr) +PaymentServer::PaymentServer(QObject* parent, bool startLocalServer) + : QObject(parent) { // Install global event filter to catch QFileOpenEvents // on Mac: sent when you click bitcoin: links diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index 08b83244ab..63f4faa772 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -101,9 +101,9 @@ protected: bool eventFilter(QObject *object, QEvent *event) override; private: - bool saveURIs; // true during startup - QLocalServer* uriServer; - OptionsModel *optionsModel; + bool saveURIs{true}; // true during startup + QLocalServer* uriServer{nullptr}; + OptionsModel* optionsModel{nullptr}; }; #endif // BITCOIN_QT_PAYMENTSERVER_H diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index b4be1b031a..9f47213ed9 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -14,10 +14,9 @@ #include <QList> #include <QTimer> -PeerTableModel::PeerTableModel(interfaces::Node& node, QObject* parent) : - QAbstractTableModel(parent), - m_node(node), - timer(nullptr) +PeerTableModel::PeerTableModel(interfaces::Node& node, QObject* parent) + : QAbstractTableModel(parent), + m_node(node) { // set up timer for auto refresh timer = new QTimer(this); diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index e51b152538..a0174c3af4 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -110,7 +110,7 @@ private: /*: Title of Peers Table column which contains the peer's User Agent string. */ tr("User Agent")}; - QTimer *timer; + QTimer* timer{nullptr}; }; #endif // BITCOIN_QT_PEERTABLEMODEL_H diff --git a/src/qt/qrimagewidget.cpp b/src/qt/qrimagewidget.cpp index 05a67117b6..00f928b355 100644 --- a/src/qt/qrimagewidget.cpp +++ b/src/qt/qrimagewidget.cpp @@ -23,8 +23,8 @@ #include <qrencode.h> #endif -QRImageWidget::QRImageWidget(QWidget *parent): - QLabel(parent), contextMenu(nullptr) +QRImageWidget::QRImageWidget(QWidget* parent) + : QLabel(parent) { contextMenu = new QMenu(this); contextMenu->addAction(tr("&Save Image…"), this, &QRImageWidget::saveImage); diff --git a/src/qt/qrimagewidget.h b/src/qt/qrimagewidget.h index 5dbba074a1..d9ca2b899f 100644 --- a/src/qt/qrimagewidget.h +++ b/src/qt/qrimagewidget.h @@ -41,7 +41,7 @@ protected: virtual void contextMenuEvent(QContextMenuEvent *event) override; private: - QMenu *contextMenu; + QMenu* contextMenu{nullptr}; }; #endif // BITCOIN_QT_QRIMAGEWIDGET_H diff --git a/src/qt/qvalidatedlineedit.cpp b/src/qt/qvalidatedlineedit.cpp index bdb302715b..b646332001 100644 --- a/src/qt/qvalidatedlineedit.cpp +++ b/src/qt/qvalidatedlineedit.cpp @@ -7,10 +7,8 @@ #include <qt/bitcoinaddressvalidator.h> #include <qt/guiconstants.h> -QValidatedLineEdit::QValidatedLineEdit(QWidget *parent) : - QLineEdit(parent), - valid(true), - checkValidator(nullptr) +QValidatedLineEdit::QValidatedLineEdit(QWidget* parent) + : QLineEdit(parent) { connect(this, &QValidatedLineEdit::textChanged, this, &QValidatedLineEdit::markValid); } diff --git a/src/qt/qvalidatedlineedit.h b/src/qt/qvalidatedlineedit.h index 4b1139d8f0..b1ae013957 100644 --- a/src/qt/qvalidatedlineedit.h +++ b/src/qt/qvalidatedlineedit.h @@ -25,8 +25,8 @@ protected: void focusOutEvent(QFocusEvent *evt) override; private: - bool valid; - const QValidator *checkValidator; + bool valid{true}; + const QValidator* checkValidator{nullptr}; public Q_SLOTS: void setText(const QString&); diff --git a/src/qt/qvaluecombobox.cpp b/src/qt/qvaluecombobox.cpp index f94486a2f3..c163ba56dc 100644 --- a/src/qt/qvaluecombobox.cpp +++ b/src/qt/qvaluecombobox.cpp @@ -4,8 +4,8 @@ #include <qt/qvaluecombobox.h> -QValueComboBox::QValueComboBox(QWidget *parent) : - QComboBox(parent), role(Qt::UserRole) +QValueComboBox::QValueComboBox(QWidget* parent) + : QComboBox(parent) { connect(this, qOverload<int>(&QComboBox::currentIndexChanged), this, &QValueComboBox::handleSelectionChanged); } diff --git a/src/qt/qvaluecombobox.h b/src/qt/qvaluecombobox.h index bde9c0d1a6..14379dd612 100644 --- a/src/qt/qvaluecombobox.h +++ b/src/qt/qvaluecombobox.h @@ -28,7 +28,7 @@ Q_SIGNALS: void valueChanged(); private: - int role; + int role{Qt::UserRole}; private Q_SLOTS: void handleSelectionChanged(int idx); diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index acbe179463..22eb642ecd 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -25,7 +25,6 @@ ReceiveCoinsDialog::ReceiveCoinsDialog(const PlatformStyle *_platformStyle, QWidget *parent) : QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::ReceiveCoinsDialog), - model(nullptr), platformStyle(_platformStyle) { ui->setupUi(this); diff --git a/src/qt/receivecoinsdialog.h b/src/qt/receivecoinsdialog.h index a089b8aa6a..0bb02ebcf2 100644 --- a/src/qt/receivecoinsdialog.h +++ b/src/qt/receivecoinsdialog.h @@ -51,7 +51,7 @@ public Q_SLOTS: private: Ui::ReceiveCoinsDialog *ui; - WalletModel *model; + WalletModel* model{nullptr}; QMenu *contextMenu; QAction* copyLabelAction; QAction* copyMessageAction; diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index c0135283b1..3453857f98 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -18,10 +18,9 @@ #include <config/bitcoin-config.h> /* for USE_QRCODE */ #endif -ReceiveRequestDialog::ReceiveRequestDialog(QWidget *parent) : - QDialog(parent, GUIUtil::dialog_flags), - ui(new Ui::ReceiveRequestDialog), - model(nullptr) +ReceiveRequestDialog::ReceiveRequestDialog(QWidget* parent) + : QDialog(parent, GUIUtil::dialog_flags), + ui(new Ui::ReceiveRequestDialog) { ui->setupUi(this); GUIUtil::handleCloseWindowShortcut(this); diff --git a/src/qt/receiverequestdialog.h b/src/qt/receiverequestdialog.h index c861680761..d64fee3663 100644 --- a/src/qt/receiverequestdialog.h +++ b/src/qt/receiverequestdialog.h @@ -33,7 +33,7 @@ private Q_SLOTS: private: Ui::ReceiveRequestDialog *ui; - WalletModel *model; + WalletModel* model{nullptr}; SendCoinsRecipient info; }; diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 85ade624cf..52d4e45d49 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -175,7 +175,7 @@ void RecentRequestsTableModel::addNewRequest(const SendCoinsRecipient &recipient newEntry.date = QDateTime::currentDateTime(); newEntry.recipient = recipient; - CDataStream ss(SER_DISK, CLIENT_VERSION); + DataStream ss{}; ss << newEntry; if (!walletModel->wallet().setAddressReceiveRequest(DecodeDestination(recipient.address.toStdString()), ToString(newEntry.id), ss.str())) @@ -188,7 +188,7 @@ void RecentRequestsTableModel::addNewRequest(const SendCoinsRecipient &recipient void RecentRequestsTableModel::addNewRequest(const std::string &recipient) { std::vector<uint8_t> data(recipient.begin(), recipient.end()); - CDataStream ss(data, SER_DISK, CLIENT_VERSION); + DataStream ss{data}; RecentRequestEntry entry; ss >> entry; diff --git a/src/qt/recentrequeststablemodel.h b/src/qt/recentrequeststablemodel.h index cf7cd24ce2..151f8322a8 100644 --- a/src/qt/recentrequeststablemodel.h +++ b/src/qt/recentrequeststablemodel.h @@ -18,11 +18,11 @@ class WalletModel; class RecentRequestEntry { public: - RecentRequestEntry() : nVersion(RecentRequestEntry::CURRENT_VERSION), id(0) { } + RecentRequestEntry() : nVersion(RecentRequestEntry::CURRENT_VERSION) {} static const int CURRENT_VERSION = 1; int nVersion; - int64_t id; + int64_t id{0}; QDateTime date; SendCoinsRecipient recipient; diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 843cd46d13..b46a3c039b 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -780,8 +780,8 @@ void RPCConsole::addWallet(WalletModel * const walletModel) { // use name for text and wallet model for internal data object (to allow to move to a wallet id later) ui->WalletSelector->addItem(walletModel->getDisplayName(), QVariant::fromValue(walletModel)); - if (ui->WalletSelector->count() == 2 && !isVisible()) { - // First wallet added, set to default so long as the window isn't presently visible (and potentially in use) + if (ui->WalletSelector->count() == 2) { + // First wallet added, set to default to match wallet RPC behavior ui->WalletSelector->setCurrentIndex(1); } if (ui->WalletSelector->count() > 2) { diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 249e3f2101..1604cad503 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -64,11 +64,7 @@ int getIndexForConfTarget(int target) { SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *parent) : QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::SendCoinsDialog), - clientModel(nullptr), - model(nullptr), m_coin_control(new CCoinControl), - fNewRecipientAllowed(true), - fFeeMinimized(true), platformStyle(_platformStyle) { ui->setupUi(this); diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 918e3fd19f..2fcdf5b32a 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -62,12 +62,12 @@ Q_SIGNALS: private: Ui::SendCoinsDialog *ui; - ClientModel *clientModel; - WalletModel *model; + ClientModel* clientModel{nullptr}; + WalletModel* model{nullptr}; std::unique_ptr<wallet::CCoinControl> m_coin_control; std::unique_ptr<WalletModelTransaction> m_current_transaction; - bool fNewRecipientAllowed; - bool fFeeMinimized; + bool fNewRecipientAllowed{true}; + bool fFeeMinimized{true}; const PlatformStyle *platformStyle; // Copy PSBT to clipboard and offer to save it. diff --git a/src/qt/sendcoinsentry.cpp b/src/qt/sendcoinsentry.cpp index 58d8fb69e3..0536635c84 100644 --- a/src/qt/sendcoinsentry.cpp +++ b/src/qt/sendcoinsentry.cpp @@ -22,7 +22,6 @@ SendCoinsEntry::SendCoinsEntry(const PlatformStyle *_platformStyle, QWidget *parent) : QWidget(parent), ui(new Ui::SendCoinsEntry), - model(nullptr), platformStyle(_platformStyle) { ui->setupUi(this); diff --git a/src/qt/sendcoinsentry.h b/src/qt/sendcoinsentry.h index 424809b01e..0edc0d1203 100644 --- a/src/qt/sendcoinsentry.h +++ b/src/qt/sendcoinsentry.h @@ -73,7 +73,7 @@ protected: private: SendCoinsRecipient recipient; Ui::SendCoinsEntry *ui; - WalletModel *model; + WalletModel* model{nullptr}; const PlatformStyle *platformStyle; bool updateLabel(const QString &address); diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 52213a428a..0e725acb33 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -21,7 +21,6 @@ SignVerifyMessageDialog::SignVerifyMessageDialog(const PlatformStyle *_platformStyle, QWidget *parent) : QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::SignVerifyMessageDialog), - model(nullptr), platformStyle(_platformStyle) { ui->setupUi(this); diff --git a/src/qt/signverifymessagedialog.h b/src/qt/signverifymessagedialog.h index 1a3708bf8f..4072707b61 100644 --- a/src/qt/signverifymessagedialog.h +++ b/src/qt/signverifymessagedialog.h @@ -35,7 +35,7 @@ protected: private: Ui::SignVerifyMessageDialog *ui; - WalletModel *model; + WalletModel* model{nullptr}; const PlatformStyle *platformStyle; private Q_SLOTS: diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 451a0183c0..096f8a0ded 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -28,7 +28,7 @@ SplashScreen::SplashScreen(const NetworkStyle* networkStyle) - : QWidget(), curAlignment(0) + : QWidget() { // set reference point, paddings int paddingRight = 50; diff --git a/src/qt/splashscreen.h b/src/qt/splashscreen.h index df53f625f5..2356bbacd3 100644 --- a/src/qt/splashscreen.h +++ b/src/qt/splashscreen.h @@ -57,7 +57,7 @@ private: QPixmap pixmap; QString curMessage; QColor curColor; - int curAlignment; + int curAlignment{0}; interfaces::Node* m_node = nullptr; bool m_shutdown = false; diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 59a5934890..15fe37c164 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -289,7 +289,7 @@ void TestGUI(interfaces::Node& node) std::vector<std::string> requests = walletModel.wallet().getAddressReceiveRequests(); QCOMPARE(requests.size(), size_t{1}); RecentRequestEntry entry; - CDataStream{MakeUCharSpan(requests[0]), SER_DISK, CLIENT_VERSION} >> entry; + DataStream{MakeUCharSpan(requests[0])} >> entry; QCOMPARE(entry.nVersion, int{1}); QCOMPARE(entry.id, int64_t{1}); QVERIFY(entry.date.isValid()); diff --git a/src/qt/trafficgraphwidget.cpp b/src/qt/trafficgraphwidget.cpp index 3f27eed3f8..08789c1048 100644 --- a/src/qt/trafficgraphwidget.cpp +++ b/src/qt/trafficgraphwidget.cpp @@ -19,15 +19,10 @@ #define XMARGIN 10 #define YMARGIN 10 -TrafficGraphWidget::TrafficGraphWidget(QWidget *parent) : - QWidget(parent), - timer(nullptr), - fMax(0.0f), - vSamplesIn(), - vSamplesOut(), - nLastBytesIn(0), - nLastBytesOut(0), - clientModel(nullptr) +TrafficGraphWidget::TrafficGraphWidget(QWidget* parent) + : QWidget(parent), + vSamplesIn(), + vSamplesOut() { timer = new QTimer(this); connect(timer, &QTimer::timeout, this, &TrafficGraphWidget::updateRates); diff --git a/src/qt/trafficgraphwidget.h b/src/qt/trafficgraphwidget.h index 1cc8fd4adb..5e5557ec82 100644 --- a/src/qt/trafficgraphwidget.h +++ b/src/qt/trafficgraphwidget.h @@ -37,14 +37,14 @@ public Q_SLOTS: private: void paintPath(QPainterPath &path, QQueue<float> &samples); - QTimer *timer; - float fMax; + QTimer* timer{nullptr}; + float fMax{0.0f}; std::chrono::minutes m_range{0}; QQueue<float> vSamplesIn; QQueue<float> vSamplesOut; - quint64 nLastBytesIn; - quint64 nLastBytesOut; - ClientModel *clientModel; + quint64 nLastBytesIn{0}; + quint64 nLastBytesOut{0}; + ClientModel* clientModel{nullptr}; }; #endif // BITCOIN_QT_TRAFFICGRAPHWIDGET_H diff --git a/src/qt/transactionfilterproxy.cpp b/src/qt/transactionfilterproxy.cpp index 3be7e1a969..173fd326a3 100644 --- a/src/qt/transactionfilterproxy.cpp +++ b/src/qt/transactionfilterproxy.cpp @@ -11,14 +11,10 @@ #include <cstdlib> #include <optional> -TransactionFilterProxy::TransactionFilterProxy(QObject *parent) : - QSortFilterProxyModel(parent), - m_search_string(), - typeFilter(ALL_TYPES), - watchOnlyFilter(WatchOnlyFilter_All), - minAmount(0), - limitRows(-1), - showInactive(true) +TransactionFilterProxy::TransactionFilterProxy(QObject* parent) + : QSortFilterProxyModel(parent), + m_search_string(), + typeFilter(ALL_TYPES) { } @@ -92,25 +88,8 @@ void TransactionFilterProxy::setWatchOnlyFilter(WatchOnlyFilter filter) invalidateFilter(); } -void TransactionFilterProxy::setLimit(int limit) -{ - this->limitRows = limit; -} - void TransactionFilterProxy::setShowInactive(bool _showInactive) { this->showInactive = _showInactive; invalidateFilter(); } - -int TransactionFilterProxy::rowCount(const QModelIndex &parent) const -{ - if(limitRows != -1) - { - return std::min(QSortFilterProxyModel::rowCount(parent), limitRows); - } - else - { - return QSortFilterProxyModel::rowCount(parent); - } -} diff --git a/src/qt/transactionfilterproxy.h b/src/qt/transactionfilterproxy.h index fd9be52842..73c4f21426 100644 --- a/src/qt/transactionfilterproxy.h +++ b/src/qt/transactionfilterproxy.h @@ -42,14 +42,9 @@ public: void setMinAmount(const CAmount& minimum); void setWatchOnlyFilter(WatchOnlyFilter filter); - /** Set maximum number of rows returned, -1 if unlimited. */ - void setLimit(int limit); - /** Set whether to show conflicted transactions. */ void setShowInactive(bool showInactive); - int rowCount(const QModelIndex &parent = QModelIndex()) const override; - protected: bool filterAcceptsRow(int source_row, const QModelIndex & source_parent) const override; @@ -58,10 +53,9 @@ private: std::optional<QDateTime> dateTo; QString m_search_string; quint32 typeFilter; - WatchOnlyFilter watchOnlyFilter; - CAmount minAmount; - int limitRows; - bool showInactive; + WatchOnlyFilter watchOnlyFilter{WatchOnlyFilter_All}; + CAmount minAmount{0}; + bool showInactive{true}; }; #endif // BITCOIN_QT_TRANSACTIONFILTERPROXY_H diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 339b3058f4..3b32137bd4 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -253,7 +253,6 @@ TransactionTableModel::TransactionTableModel(const PlatformStyle *_platformStyle QAbstractTableModel(parent), walletModel(parent), priv(new TransactionTablePriv(this)), - fProcessingQueuedTransactions(false), platformStyle(_platformStyle) { subscribeToCoreSignals(); diff --git a/src/qt/transactiontablemodel.h b/src/qt/transactiontablemodel.h index f8576edd59..92ba9bf894 100644 --- a/src/qt/transactiontablemodel.h +++ b/src/qt/transactiontablemodel.h @@ -89,7 +89,7 @@ private: std::unique_ptr<interfaces::Handler> m_handler_show_progress; QStringList columns; TransactionTablePriv *priv; - bool fProcessingQueuedTransactions; + bool fProcessingQueuedTransactions{false}; const PlatformStyle *platformStyle; void subscribeToCoreSignals(); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 097b2b0364..cb8491e27a 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -46,10 +46,6 @@ WalletModel::WalletModel(std::unique_ptr<interfaces::Wallet> wallet, ClientModel m_client_model(&client_model), m_node(client_model.node()), optionsModel(client_model.getOptionsModel()), - addressTableModel(nullptr), - transactionTableModel(nullptr), - recentRequestsTableModel(nullptr), - cachedEncryptionStatus(Unencrypted), timer(new QTimer(this)) { fHaveWatchOnly = m_wallet->haveWatchOnly(); diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 88a6f3a598..604a9e03c8 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -181,13 +181,13 @@ private: // (transaction fee, for example) OptionsModel *optionsModel; - AddressTableModel *addressTableModel; - TransactionTableModel *transactionTableModel; - RecentRequestsTableModel *recentRequestsTableModel; + AddressTableModel* addressTableModel{nullptr}; + TransactionTableModel* transactionTableModel{nullptr}; + RecentRequestsTableModel* recentRequestsTableModel{nullptr}; // Cache some values to be able to detect changes interfaces::WalletBalances m_cached_balances; - EncryptionStatus cachedEncryptionStatus; + EncryptionStatus cachedEncryptionStatus{Unencrypted}; QTimer* timer; // Block hash denoting when the last balance update was done. diff --git a/src/qt/walletmodeltransaction.cpp b/src/qt/walletmodeltransaction.cpp index b579dcc0f0..61ccd9dd82 100644 --- a/src/qt/walletmodeltransaction.cpp +++ b/src/qt/walletmodeltransaction.cpp @@ -10,9 +10,8 @@ #include <policy/policy.h> -WalletModelTransaction::WalletModelTransaction(const QList<SendCoinsRecipient> &_recipients) : - recipients(_recipients), - fee(0) +WalletModelTransaction::WalletModelTransaction(const QList<SendCoinsRecipient>& _recipients) + : recipients(_recipients) { } diff --git a/src/qt/walletmodeltransaction.h b/src/qt/walletmodeltransaction.h index 28fb551364..0e6ed8be49 100644 --- a/src/qt/walletmodeltransaction.h +++ b/src/qt/walletmodeltransaction.h @@ -41,7 +41,7 @@ public: private: QList<SendCoinsRecipient> recipients; CTransactionRef wtx; - CAmount fee; + CAmount fee{0}; }; #endif // BITCOIN_QT_WALLETMODELTRANSACTION_H diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index a4cd1a6caf..e62821d5bd 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -31,7 +31,6 @@ WalletView::WalletView(WalletModel* wallet_model, const PlatformStyle* _platformStyle, QWidget* parent) : QStackedWidget(parent), - clientModel(nullptr), walletModel(wallet_model), platformStyle(_platformStyle) { diff --git a/src/qt/walletview.h b/src/qt/walletview.h index 301084ffa9..ebceef9cf9 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -50,7 +50,7 @@ public: void showOutOfSyncWarning(bool fShow); private: - ClientModel *clientModel; + ClientModel* clientModel{nullptr}; //! //! The wallet model represents a bitcoin wallet, and offers access to diff --git a/src/random.cpp b/src/random.cpp index 8e04d449f3..23ea9ba6b7 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -62,7 +62,7 @@ static inline int64_t GetPerformanceCounter() noexcept __asm__ volatile ("rdtsc" : "=a"(r1), "=d"(r2)); // Constrain r1 to rax and r2 to rdx. return (r2 << 32) | r1; #else - // Fall back to using C++11 clock (usually microsecond or nanosecond precision) + // Fall back to using standard library clock (usually microsecond or nanosecond precision) return std::chrono::high_resolution_clock::now().time_since_epoch().count(); #endif } @@ -438,7 +438,7 @@ public: RNGState& GetRNGState() noexcept { - // This C++11 idiom relies on the guarantee that static variable are initialized + // This idiom relies on the guarantee that static variable are initialized // on first call, even when multiple parallel calls are permitted. static std::vector<RNGState, secure_allocator<RNGState>> g_rng(1); return g_rng[0]; diff --git a/src/random.h b/src/random.h index 6223da0377..e890e909c7 100644 --- a/src/random.h +++ b/src/random.h @@ -250,7 +250,7 @@ public: /* interval [0..0] */ Dur{0}; }; - // Compatibility with the C++11 UniformRandomBitGenerator concept + // Compatibility with the UniformRandomBitGenerator concept typedef uint64_t result_type; static constexpr uint64_t min() { return 0; } static constexpr uint64_t max() { return std::numeric_limits<uint64_t>::max(); } diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 0b97d747cb..3e4d5a587d 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -13,21 +13,21 @@ #include <compat/cpuid.h> #include <crypto/sha512.h> #include <support/cleanse.h> -#include <util/time.h> // for GetTime() -#ifdef WIN32 -#include <compat/compat.h> -#endif +#include <util/time.h> #include <algorithm> #include <atomic> +#include <cstdint> +#include <cstring> #include <chrono> #include <climits> #include <thread> #include <vector> -#include <stdint.h> -#include <string.h> -#ifndef WIN32 +#ifdef WIN32 +#include <windows.h> +#include <winreg.h> +#else #include <sys/types.h> // must go before a number of other headers #include <fcntl.h> #include <netinet/in.h> @@ -250,7 +250,7 @@ void RandAddDynamicEnv(CSHA512& hasher) gettimeofday(&tv, nullptr); hasher << tv; #endif - // Probably redundant, but also use all the clocks C++11 provides: + // Probably redundant, but also use all the standard library clocks: hasher << std::chrono::system_clock::now().time_since_epoch().count(); hasher << std::chrono::steady_clock::now().time_since_epoch().count(); hasher << std::chrono::high_resolution_clock::now().time_since_epoch().count(); diff --git a/src/rest.cpp b/src/rest.cpp index add2bb73b0..a874f4eb6d 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -236,7 +236,7 @@ static bool rest_headers(const std::any& context, switch (rf) { case RESTResponseFormat::BINARY: { - CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssHeader{}; for (const CBlockIndex *pindex : headers) { ssHeader << pindex->GetBlockHeader(); } @@ -248,7 +248,7 @@ static bool rest_headers(const std::any& context, } case RESTResponseFormat::HEX: { - CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssHeader{}; for (const CBlockIndex *pindex : headers) { ssHeader << pindex->GetBlockHeader(); } @@ -435,7 +435,7 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const switch (rf) { case RESTResponseFormat::BINARY: { - CDataStream ssHeader{SER_NETWORK, PROTOCOL_VERSION}; + DataStream ssHeader{}; for (const uint256& header : filter_headers) { ssHeader << header; } @@ -446,7 +446,7 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const return true; } case RESTResponseFormat::HEX: { - CDataStream ssHeader{SER_NETWORK, PROTOCOL_VERSION}; + DataStream ssHeader{}; for (const uint256& header : filter_headers) { ssHeader << header; } @@ -534,7 +534,7 @@ static bool rest_block_filter(const std::any& context, HTTPRequest* req, const s switch (rf) { case RESTResponseFormat::BINARY: { - CDataStream ssResp{SER_NETWORK, PROTOCOL_VERSION}; + DataStream ssResp{}; ssResp << filter; std::string binaryResp = ssResp.str(); @@ -543,7 +543,7 @@ static bool rest_block_filter(const std::any& context, HTTPRequest* req, const s return true; } case RESTResponseFormat::HEX: { - CDataStream ssResp{SER_NETWORK, PROTOCOL_VERSION}; + DataStream ssResp{}; ssResp << filter; std::string strHex = HexStr(ssResp) + "\n"; @@ -793,7 +793,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: if (fInputParsed) //don't allow sending input over URI and HTTP RAW DATA return RESTERR(req, HTTP_BAD_REQUEST, "Combination of URI scheme inputs and raw post data is not allowed"); - CDataStream oss(SER_NETWORK, PROTOCOL_VERSION); + DataStream oss{}; oss << strRequestMutable; oss >> fCheckMemPool; oss >> vOutPoints; @@ -866,7 +866,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: case RESTResponseFormat::BINARY: { // serialize data // use exact same output as mentioned in Bip64 - CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssGetUTXOResponse{}; ssGetUTXOResponse << active_height << active_hash << bitmap << outs; std::string ssGetUTXOResponseString = ssGetUTXOResponse.str(); @@ -876,7 +876,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: } case RESTResponseFormat::HEX: { - CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssGetUTXOResponse{}; ssGetUTXOResponse << active_height << active_hash << bitmap << outs; std::string strHex = HexStr(ssGetUTXOResponse) + "\n"; @@ -946,7 +946,7 @@ static bool rest_blockhash_by_height(const std::any& context, HTTPRequest* req, } switch (rf) { case RESTResponseFormat::BINARY: { - CDataStream ss_blockhash(SER_NETWORK, PROTOCOL_VERSION); + DataStream ss_blockhash{}; ss_blockhash << pblockindex->GetBlockHash(); req->WriteHeader("Content-Type", "application/octet-stream"); req->WriteReply(HTTP_OK, ss_blockhash.str()); diff --git a/src/reverse_iterator.h b/src/reverse_iterator.h index 729d8c11cc..4db001c04b 100644 --- a/src/reverse_iterator.h +++ b/src/reverse_iterator.h @@ -4,7 +4,7 @@ #define BITCOIN_REVERSE_ITERATOR_H /** - * Template used for reverse iteration in C++11 range-based for loops. + * Template used for reverse iteration in range-based for loops. * * std::vector<int> v = {1, 2, 3, 4, 5}; * for (auto x : reverse_iterate(v)) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 784fb64d36..8bee066ab8 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -25,6 +25,7 @@ #include <net_processing.h> #include <node/blockstorage.h> #include <node/context.h> +#include <node/transaction.h> #include <node/utxo_snapshot.h> #include <primitives/transaction.h> #include <rpc/server.h> @@ -444,11 +445,6 @@ static RPCHelpMan getblockfrompeer() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, { - UniValue::VSTR, // blockhash - UniValue::VNUM, // peer_id - }); - const NodeContext& node = EnsureAnyNodeContext(request.context); ChainstateManager& chainman = EnsureChainman(node); PeerManager& peerman = EnsurePeerman(node); @@ -464,7 +460,7 @@ static RPCHelpMan getblockfrompeer() // Fetching blocks before the node has syncing past their height can prevent block files from // being pruned, so we avoid it if the node is in prune mode. - if (node::fPruneMode && index->nHeight > WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip()->nHeight)) { + if (chainman.m_blockman.IsPruneMode() && index->nHeight > WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip()->nHeight)) { throw JSONRPCError(RPC_MISC_ERROR, "In prune mode, only blocks that the node has already synced previously can be fetched from a peer"); } @@ -569,7 +565,7 @@ static RPCHelpMan getblockheader() if (!fVerbose) { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssBlock{}; ssBlock << pblockindex->GetBlockHeader(); std::string strHex = HexStr(ssBlock); return strHex; @@ -654,7 +650,8 @@ static RPCHelpMan getblock() "If verbosity is 3, returns an Object with information about block <hash> and information about each transaction, including prevout information for inputs (only for unpruned blocks in the current best chain).\n", { {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash"}, - {"verbosity|verbose", RPCArg::Type::NUM, RPCArg::Default{1}, "0 for hex-encoded data, 1 for a JSON object, 2 for JSON object with transaction data, and 3 for JSON object with transaction data including prevout information for inputs"}, + {"verbosity|verbose", RPCArg::Type::NUM, RPCArg::Default{1}, "0 for hex-encoded data, 1 for a JSON object, 2 for JSON object with transaction data, and 3 for JSON object with transaction data including prevout information for inputs", + RPCArgOptions{.skip_type_check = true}}, }, { RPCResult{"for verbosity = 0", @@ -778,10 +775,11 @@ static RPCHelpMan pruneblockchain() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - if (!node::fPruneMode) + ChainstateManager& chainman = EnsureAnyChainman(request.context); + if (!chainman.m_blockman.IsPruneMode()) { throw JSONRPCError(RPC_MISC_ERROR, "Cannot prune blocks because node is not in prune mode."); + } - ChainstateManager& chainman = EnsureAnyChainman(request.context); LOCK(cs_main); Chainstate& active_chainstate = chainman.ActiveChainstate(); CChain& active_chain = active_chainstate.m_chain; @@ -872,7 +870,11 @@ static RPCHelpMan gettxoutsetinfo() "Note this call may take some time if you are not using coinstatsindex.\n", { {"hash_type", RPCArg::Type::STR, RPCArg::Default{"hash_serialized_2"}, "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'muhash', 'none'."}, - {"hash_or_height", RPCArg::Type::NUM, RPCArg::DefaultHint{"the current best block"}, "The block hash or height of the target height (only available with coinstatsindex).", RPCArgOptions{.type_str={"", "string or numeric"}}}, + {"hash_or_height", RPCArg::Type::NUM, RPCArg::DefaultHint{"the current best block"}, "The block hash or height of the target height (only available with coinstatsindex).", + RPCArgOptions{ + .skip_type_check = true, + .type_str = {"", "string or numeric"}, + }}, {"use_index", RPCArg::Type::BOOL, RPCArg::Default{true}, "Use coinstatsindex, if available."}, }, RPCResult{ @@ -1265,15 +1267,15 @@ RPCHelpMan getblockchaininfo() obj.pushKV("initialblockdownload", active_chainstate.IsInitialBlockDownload()); obj.pushKV("chainwork", tip.nChainWork.GetHex()); obj.pushKV("size_on_disk", chainman.m_blockman.CalculateCurrentUsage()); - obj.pushKV("pruned", node::fPruneMode); - if (node::fPruneMode) { + obj.pushKV("pruned", chainman.m_blockman.IsPruneMode()); + if (chainman.m_blockman.IsPruneMode()) { obj.pushKV("pruneheight", chainman.m_blockman.GetFirstStoredBlock(tip)->nHeight); // if 0, execution bypasses the whole if block. bool automatic_pruning{args.GetIntArg("-prune", 0) != 1}; obj.pushKV("automatic_pruning", automatic_pruning); if (automatic_pruning) { - obj.pushKV("prune_target_size", node::nPruneTarget); + obj.pushKV("prune_target_size", chainman.m_blockman.GetPruneTarget()); } } @@ -1742,7 +1744,11 @@ static RPCHelpMan getblockstats() "\nCompute per block statistics for a given window. All amounts are in satoshis.\n" "It won't work for some heights with pruning.\n", { - {"hash_or_height", RPCArg::Type::NUM, RPCArg::Optional::NO, "The block hash or height of the target block", RPCArgOptions{.type_str={"", "string or numeric"}}}, + {"hash_or_height", RPCArg::Type::NUM, RPCArg::Optional::NO, "The block hash or height of the target block", + RPCArgOptions{ + .skip_type_check = true, + .type_str = {"", "string or numeric"}, + }}, {"stats", RPCArg::Type::ARR, RPCArg::DefaultHint{"all values"}, "Values to plot (see result below)", { {"height", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Selected statistic"}, @@ -2144,8 +2150,6 @@ static RPCHelpMan scantxoutset() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VARR}); - UniValue result(UniValue::VOBJ); if (request.params[0].get_str() == "status") { CoinsViewScanReserver reserver; @@ -2268,17 +2272,47 @@ public: } }; +static bool CheckBlockFilterMatches(BlockManager& blockman, const CBlockIndex& blockindex, const GCSFilter::ElementSet& needles) +{ + const CBlock block{GetBlockChecked(blockman, &blockindex)}; + const CBlockUndo block_undo{GetUndoChecked(blockman, &blockindex)}; + + // Check if any of the outputs match the scriptPubKey + for (const auto& tx : block.vtx) { + if (std::any_of(tx->vout.cbegin(), tx->vout.cend(), [&](const auto& txout) { + return needles.count(std::vector<unsigned char>(txout.scriptPubKey.begin(), txout.scriptPubKey.end())) != 0; + })) { + return true; + } + } + // Check if any of the inputs match the scriptPubKey + for (const auto& txundo : block_undo.vtxundo) { + if (std::any_of(txundo.vprevout.cbegin(), txundo.vprevout.cend(), [&](const auto& coin) { + return needles.count(std::vector<unsigned char>(coin.out.scriptPubKey.begin(), coin.out.scriptPubKey.end())) != 0; + })) { + return true; + } + } + + return false; +} + static RPCHelpMan scanblocks() { return RPCHelpMan{"scanblocks", - "\nReturn relevant blockhashes for given descriptors.\n" + "\nReturn relevant blockhashes for given descriptors (requires blockfilterindex).\n" "This call may take several minutes. Make sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)", { scan_action_arg_desc, scan_objects_arg_desc, RPCArg{"start_height", RPCArg::Type::NUM, RPCArg::Default{0}, "Height to start to scan from"}, RPCArg{"stop_height", RPCArg::Type::NUM, RPCArg::DefaultHint{"chain tip"}, "Height to stop to scan"}, - RPCArg{"filtertype", RPCArg::Type::STR, RPCArg::Default{BlockFilterTypeName(BlockFilterType::BASIC)}, "The type name of the filter"} + RPCArg{"filtertype", RPCArg::Type::STR, RPCArg::Default{BlockFilterTypeName(BlockFilterType::BASIC)}, "The type name of the filter"}, + RPCArg{"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + { + {"filter_false_positives", RPCArg::Type::BOOL, RPCArg::Default{false}, "Filter false positives (slower and may fail on pruned nodes). Otherwise they may occur at a rate of 1/M"}, + }, + RPCArgOptions{.oneline_description="\"options\""}}, }, { scan_result_status_none, @@ -2338,6 +2372,9 @@ static RPCHelpMan scanblocks() throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown filtertype"); } + UniValue options{request.params[5].isNull() ? UniValue::VOBJ : request.params[5]}; + bool filter_false_positives{options.exists("filter_false_positives") ? options["filter_false_positives"].get_bool() : false}; + BlockFilterIndex* index = GetBlockFilterIndex(filtertype); if (!index) { throw JSONRPCError(RPC_MISC_ERROR, "Index is not enabled for filtertype " + filtertype_name); @@ -2408,6 +2445,15 @@ static RPCHelpMan scanblocks() for (const BlockFilter& filter : filters) { // compare the elements-set with each filter if (filter.GetFilter().MatchAny(needle_set)) { + if (filter_false_positives) { + // Double check the filter matches by scanning the block + const CBlockIndex& blockindex = *CHECK_NONFATAL(WITH_LOCK(cs_main, return chainman.m_blockman.LookupBlockIndex(filter.GetBlockHash()))); + + if (!CheckBlockFilterMatches(chainman.m_blockman, blockindex, needle_set)) { + continue; + } + } + blocks.push_back(filter.GetBlockHash().GetHex()); LogPrint(BCLog::RPC, "scanblocks: found match in %s\n", filter.GetBlockHash().GetHex()); } diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 2a802f7fee..9ccb87b78a 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -16,8 +16,6 @@ #include <stdint.h> #include <vector> -extern RecursiveMutex cs_main; - class CBlock; class CBlockIndex; class Chainstate; diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index e6ce188a56..5fe914f0a1 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -86,6 +86,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "scanblocks", 1, "scanobjects" }, { "scanblocks", 2, "start_height" }, { "scanblocks", 3, "stop_height" }, + { "scanblocks", 5, "options" }, { "scantxoutset", 1, "scanobjects" }, { "addmultisigaddress", 0, "nrequired" }, { "addmultisigaddress", 1, "keys" }, @@ -226,11 +227,16 @@ private: public: CRPCConvertTable(); - bool convert(const std::string& method, int idx) { - return (members.count(std::make_pair(method, idx)) > 0); + /** Return arg_value as UniValue, and first parse it if it is a non-string parameter */ + UniValue ArgToUniValue(const std::string& arg_value, const std::string& method, int param_idx) + { + return members.count(std::make_pair(method, param_idx)) > 0 ? ParseNonRFCJSONValue(arg_value) : arg_value; } - bool convert(const std::string& method, const std::string& name) { - return (membersByName.count(std::make_pair(method, name)) > 0); + + /** Return arg_value as UniValue, and first parse it if it is a non-string parameter */ + UniValue ArgToUniValue(const std::string& arg_value, const std::string& method, const std::string& param_name) + { + return membersByName.count(std::make_pair(method, param_name)) > 0 ? ParseNonRFCJSONValue(arg_value) : arg_value; } }; @@ -262,14 +268,7 @@ UniValue RPCConvertValues(const std::string &strMethod, const std::vector<std::s for (unsigned int idx = 0; idx < strParams.size(); idx++) { const std::string& strVal = strParams[idx]; - - if (!rpcCvtTable.convert(strMethod, idx)) { - // insert string value directly - params.push_back(strVal); - } else { - // parse string as JSON, insert bool/number/object/etc. value - params.push_back(ParseNonRFCJSONValue(strVal)); - } + params.push_back(rpcCvtTable.ArgToUniValue(strVal, strMethod, idx)); } return params; @@ -283,7 +282,7 @@ UniValue RPCConvertNamedValues(const std::string &strMethod, const std::vector<s for (const std::string &s: strParams) { size_t pos = s.find('='); if (pos == std::string::npos) { - positional_args.push_back(rpcCvtTable.convert(strMethod, positional_args.size()) ? ParseNonRFCJSONValue(s) : s); + positional_args.push_back(rpcCvtTable.ArgToUniValue(s, strMethod, positional_args.size())); continue; } @@ -293,13 +292,7 @@ UniValue RPCConvertNamedValues(const std::string &strMethod, const std::vector<s // Intentionally overwrite earlier named values with later ones as a // convenience for scripts and command line users that want to merge // options. - if (!rpcCvtTable.convert(strMethod, name)) { - // insert string value directly - params.pushKV(name, value); - } else { - // parse string as JSON, insert bool/number/object/etc. value - params.pushKV(name, ParseNonRFCJSONValue(value)); - } + params.pushKV(name, rpcCvtTable.ArgToUniValue(value, strMethod, name)); } if (!positional_args.empty()) { diff --git a/src/rpc/fees.cpp b/src/rpc/fees.cpp index cd2f6390e5..62396d4c58 100644 --- a/src/rpc/fees.cpp +++ b/src/rpc/fees.cpp @@ -63,8 +63,6 @@ static RPCHelpMan estimatesmartfee() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VSTR}); - CBlockPolicyEstimator& fee_estimator = EnsureAnyFeeEstimator(request.context); const NodeContext& node = EnsureAnyNodeContext(request.context); const CTxMemPool& mempool = EnsureMemPool(node); @@ -155,8 +153,6 @@ static RPCHelpMan estimaterawfee() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VNUM}, true); - CBlockPolicyEstimator& fee_estimator = EnsureAnyFeeEstimator(request.context); unsigned int max_target = fee_estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 44bff55f96..44f7435a26 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -61,11 +61,6 @@ static RPCHelpMan sendrawtransaction() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, { - UniValue::VSTR, - UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() - }); - CMutableTransaction mtx; if (!DecodeHexTx(mtx, request.params[0].get_str())) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input."); @@ -147,10 +142,6 @@ static RPCHelpMan testmempoolaccept() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, { - UniValue::VARR, - UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() - }); const UniValue raw_transactions = request.params[0].get_array(); if (raw_transactions.size() < 1 || raw_transactions.size() > MAX_PACKAGE_COUNT) { throw JSONRPCError(RPC_INVALID_PARAMETER, @@ -800,9 +791,6 @@ static RPCHelpMan submitpackage() if (!Params().IsMockableChain()) { throw std::runtime_error("submitpackage is for regression testing (-regtest mode) only"); } - RPCTypeCheck(request.params, { - UniValue::VARR, - }); const UniValue raw_transactions = request.params[0].get_array(); if (raw_transactions.size() < 1 || raw_transactions.size() > MAX_PACKAGE_COUNT) { throw JSONRPCError(RPC_INVALID_PARAMETER, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 764c4c675b..8753f845a5 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -441,7 +441,7 @@ static RPCHelpMan prioritisetransaction() "Accepts the transaction into mined blocks at a higher (or lower) priority\n", { {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id."}, - {"dummy", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "API-Compatibility for previous API. Must be zero or null.\n" + {"dummy", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "API-Compatibility for previous API. Must be zero or null.\n" " DEPRECATED. For forward compatibility use named arguments and omit this parameter."}, {"fee_delta", RPCArg::Type::NUM, RPCArg::Optional::NO, "The fee value (in satoshis) to add (or subtract, if negative).\n" " Note, that this value is not a fee rate. It is a value to modify absolute fee of the TX.\n" @@ -513,8 +513,8 @@ static RPCHelpMan getblocktemplate() { {"template_request", RPCArg::Type::OBJ, RPCArg::Default{UniValue::VOBJ}, "Format of the template", { - {"mode", RPCArg::Type::STR, /* treat as named arg */ RPCArg::Optional::OMITTED_NAMED_ARG, "This must be set to \"template\", \"proposal\" (see BIP 23), or omitted"}, - {"capabilities", RPCArg::Type::ARR, /* treat as named arg */ RPCArg::Optional::OMITTED_NAMED_ARG, "A list of strings", + {"mode", RPCArg::Type::STR, /* treat as named arg */ RPCArg::Optional::OMITTED, "This must be set to \"template\", \"proposal\" (see BIP 23), or omitted"}, + {"capabilities", RPCArg::Type::ARR, /* treat as named arg */ RPCArg::Optional::OMITTED, "A list of strings", { {"str", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "client side supported feature, 'longpoll', 'coinbasevalue', 'proposal', 'serverlist', 'workid'"}, }}, diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index d430087358..f0e5b90509 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -114,7 +114,7 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::STR, "SERVICE_NAME", "the service name if it is recognised"} }}, - {RPCResult::Type::BOOL, "relaytxes", /*optional=*/true, "Whether we relay transactions to this peer"}, + {RPCResult::Type::BOOL, "relaytxes", "Whether we relay transactions to this peer"}, {RPCResult::Type::NUM_TIME, "lastsend", "The " + UNIX_EPOCH_TIME + " of the last send"}, {RPCResult::Type::NUM_TIME, "lastrecv", "The " + UNIX_EPOCH_TIME + " of the last receive"}, {RPCResult::Type::NUM_TIME, "last_transaction", "The " + UNIX_EPOCH_TIME + " of the last valid transaction received from this peer"}, @@ -131,17 +131,17 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::BOOL, "inbound", "Inbound (true) or Outbound (false)"}, {RPCResult::Type::BOOL, "bip152_hb_to", "Whether we selected peer as (compact blocks) high-bandwidth peer"}, {RPCResult::Type::BOOL, "bip152_hb_from", "Whether peer selected us as (compact blocks) high-bandwidth peer"}, - {RPCResult::Type::NUM, "startingheight", /*optional=*/true, "The starting height (block) of the peer"}, - {RPCResult::Type::NUM, "presynced_headers", /*optional=*/true, "The current height of header pre-synchronization with this peer, or -1 if no low-work sync is in progress"}, - {RPCResult::Type::NUM, "synced_headers", /*optional=*/true, "The last header we have in common with this peer"}, - {RPCResult::Type::NUM, "synced_blocks", /*optional=*/true, "The last block we have in common with this peer"}, - {RPCResult::Type::ARR, "inflight", /*optional=*/true, "", + {RPCResult::Type::NUM, "startingheight", "The starting height (block) of the peer"}, + {RPCResult::Type::NUM, "presynced_headers", "The current height of header pre-synchronization with this peer, or -1 if no low-work sync is in progress"}, + {RPCResult::Type::NUM, "synced_headers", "The last header we have in common with this peer"}, + {RPCResult::Type::NUM, "synced_blocks", "The last block we have in common with this peer"}, + {RPCResult::Type::ARR, "inflight", "", { {RPCResult::Type::NUM, "n", "The heights of blocks we're currently asking from this peer"}, }}, - {RPCResult::Type::BOOL, "addr_relay_enabled", /*optional=*/true, "Whether we participate in address relay with this peer"}, - {RPCResult::Type::NUM, "addr_processed", /*optional=*/true, "The total number of addresses processed, excluding those dropped due to rate limiting"}, - {RPCResult::Type::NUM, "addr_rate_limited", /*optional=*/true, "The total number of addresses dropped due to rate limiting"}, + {RPCResult::Type::BOOL, "addr_relay_enabled", "Whether we participate in address relay with this peer"}, + {RPCResult::Type::NUM, "addr_processed", "The total number of addresses processed, excluding those dropped due to rate limiting"}, + {RPCResult::Type::NUM, "addr_rate_limited", "The total number of addresses dropped due to rate limiting"}, {RPCResult::Type::ARR, "permissions", "Any special permissions that have been granted to this peer", { {RPCResult::Type::STR, "permission_type", Join(NET_PERMISSIONS_DOC, ",\n") + ".\n"}, @@ -205,12 +205,10 @@ static RPCHelpMan getpeerinfo() if (stats.m_mapped_as != 0) { obj.pushKV("mapped_as", uint64_t(stats.m_mapped_as)); } - ServiceFlags services{fStateStats ? statestats.their_services : ServiceFlags::NODE_NONE}; + ServiceFlags services{statestats.their_services}; obj.pushKV("services", strprintf("%016x", services)); obj.pushKV("servicesnames", GetServicesNames(services)); - if (fStateStats) { - obj.pushKV("relaytxes", statestats.m_relay_txs); - } + obj.pushKV("relaytxes", statestats.m_relay_txs); obj.pushKV("lastsend", count_seconds(stats.m_last_send)); obj.pushKV("lastrecv", count_seconds(stats.m_last_recv)); obj.pushKV("last_transaction", count_seconds(stats.m_last_tx_time)); @@ -225,7 +223,7 @@ static RPCHelpMan getpeerinfo() if (stats.m_min_ping_time < std::chrono::microseconds::max()) { obj.pushKV("minping", Ticks<SecondsDouble>(stats.m_min_ping_time)); } - if (fStateStats && statestats.m_ping_wait > 0s) { + if (statestats.m_ping_wait > 0s) { obj.pushKV("pingwait", Ticks<SecondsDouble>(statestats.m_ping_wait)); } obj.pushKV("version", stats.nVersion); @@ -236,26 +234,24 @@ static RPCHelpMan getpeerinfo() obj.pushKV("inbound", stats.fInbound); obj.pushKV("bip152_hb_to", stats.m_bip152_highbandwidth_to); obj.pushKV("bip152_hb_from", stats.m_bip152_highbandwidth_from); - if (fStateStats) { - obj.pushKV("startingheight", statestats.m_starting_height); - obj.pushKV("presynced_headers", statestats.presync_height); - obj.pushKV("synced_headers", statestats.nSyncHeight); - obj.pushKV("synced_blocks", statestats.nCommonHeight); - UniValue heights(UniValue::VARR); - for (const int height : statestats.vHeightInFlight) { - heights.push_back(height); - } - obj.pushKV("inflight", heights); - obj.pushKV("addr_relay_enabled", statestats.m_addr_relay_enabled); - obj.pushKV("addr_processed", statestats.m_addr_processed); - obj.pushKV("addr_rate_limited", statestats.m_addr_rate_limited); + obj.pushKV("startingheight", statestats.m_starting_height); + obj.pushKV("presynced_headers", statestats.presync_height); + obj.pushKV("synced_headers", statestats.nSyncHeight); + obj.pushKV("synced_blocks", statestats.nCommonHeight); + UniValue heights(UniValue::VARR); + for (const int height : statestats.vHeightInFlight) { + heights.push_back(height); } + obj.pushKV("inflight", heights); + obj.pushKV("addr_relay_enabled", statestats.m_addr_relay_enabled); + obj.pushKV("addr_processed", statestats.m_addr_processed); + obj.pushKV("addr_rate_limited", statestats.m_addr_rate_limited); UniValue permissions(UniValue::VARR); for (const auto& permission : NetPermissions::ToStrings(stats.m_permission_flags)) { permissions.push_back(permission); } obj.pushKV("permissions", permissions); - obj.pushKV("minfeefilter", fStateStats ? ValueFromAmount(statestats.m_fee_filter_received) : 0); + obj.pushKV("minfeefilter", ValueFromAmount(statestats.m_fee_filter_received)); UniValue sendPerMsgType(UniValue::VOBJ); for (const auto& i : stats.mapSendBytesPerMsgType) { @@ -362,7 +358,6 @@ static RPCHelpMan addconnection() throw std::runtime_error("addconnection is for regression testing (-regtest mode) only."); } - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VSTR}); const std::string address = request.params[0].get_str(); const std::string conn_type_in{TrimString(request.params[1].get_str())}; ConnectionType conn_type{}; diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index d8f2576b6d..5918bc6e38 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -12,6 +12,7 @@ #include <interfaces/echo.h> #include <interfaces/init.h> #include <interfaces/ipc.h> +#include <kernel/cs_main.h> #include <node/context.h> #include <rpc/server.h> #include <rpc/server_util.h> @@ -52,7 +53,6 @@ static RPCHelpMan setmocktime() // ensure all call sites of GetTime() are accessing this safely. LOCK(cs_main); - RPCTypeCheck(request.params, {UniValue::VNUM}); const int64_t time{request.params[0].getInt<int64_t>()}; if (time < 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Mocktime cannot be negative: %s.", time)); @@ -106,8 +106,6 @@ static RPCHelpMan mockscheduler() throw std::runtime_error("mockscheduler is for regression testing (-regtest mode) only"); } - // check params are valid values - RPCTypeCheck(request.params, {UniValue::VNUM}); int64_t delta_seconds = request.params[0].getInt<int64_t>(); if (delta_seconds <= 0 || delta_seconds > 3600) { throw std::runtime_error("delta_time must be between 1 and 3600 seconds (1 hr)"); @@ -242,11 +240,11 @@ static RPCHelpMan logging() " - \"none\", \"0\" : even if other logging categories are specified, ignore all of them.\n" , { - {"include", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "The categories to add to debug logging", + {"include", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "The categories to add to debug logging", { {"include_category", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "the valid logging category"}, }}, - {"exclude", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "The categories to remove from debug logging", + {"exclude", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "The categories to remove from debug logging", { {"exclude_category", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "the valid logging category"}, }}, @@ -295,18 +293,18 @@ static RPCHelpMan echo(const std::string& name) "\nIt will return an internal bug report when arg9='trigger_internal_bug' is passed.\n" "\nThe difference between echo and echojson is that echojson has argument conversion enabled in the client-side table in " "bitcoin-cli and the GUI. There is no server-side difference.", - { - {"arg0", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg1", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg2", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg3", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg4", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg5", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg6", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg7", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg8", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg9", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - }, + { + {"arg0", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "", RPCArgOptions{.skip_type_check = true}}, + {"arg1", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "", RPCArgOptions{.skip_type_check = true}}, + {"arg2", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "", RPCArgOptions{.skip_type_check = true}}, + {"arg3", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "", RPCArgOptions{.skip_type_check = true}}, + {"arg4", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "", RPCArgOptions{.skip_type_check = true}}, + {"arg5", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "", RPCArgOptions{.skip_type_check = true}}, + {"arg6", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "", RPCArgOptions{.skip_type_check = true}}, + {"arg7", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "", RPCArgOptions{.skip_type_check = true}}, + {"arg8", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "", RPCArgOptions{.skip_type_check = true}}, + {"arg9", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "", RPCArgOptions{.skip_type_check = true}}, + }, RPCResult{RPCResult::Type::ANY, "", "Returns whatever was passed in"}, RPCExamples{""}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue @@ -378,7 +376,7 @@ static RPCHelpMan getindexinfo() return RPCHelpMan{"getindexinfo", "\nReturns the status of one or all available indices currently running in the node.\n", { - {"index_name", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Filter results for an index with a specific name."}, + {"index_name", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Filter results for an index with a specific name."}, }, RPCResult{ RPCResult::Type::OBJ_DYN, "", "", { diff --git a/src/rpc/output_script.cpp b/src/rpc/output_script.cpp index 2ac6d6d76f..bb04f58424 100644 --- a/src/rpc/output_script.cpp +++ b/src/rpc/output_script.cpp @@ -195,8 +195,6 @@ static RPCHelpMan getdescriptorinfo() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VSTR}); - FlatSigningProvider provider; std::string error; auto desc = Parse(request.params[0].get_str(), provider, error); @@ -232,7 +230,7 @@ static RPCHelpMan deriveaddresses() "For more information on output descriptors, see the documentation in the doc/descriptors.md file.\n"}, { {"descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The descriptor."}, - {"range", RPCArg::Type::RANGE, RPCArg::Optional::OMITTED_NAMED_ARG, "If a ranged descriptor is used, this specifies the end or the range (in [begin,end] notation) to derive."}, + {"range", RPCArg::Type::RANGE, RPCArg::Optional::OMITTED, "If a ranged descriptor is used, this specifies the end or the range (in [begin,end] notation) to derive."}, }, RPCResult{ RPCResult::Type::ARR, "", "", @@ -247,7 +245,6 @@ static RPCHelpMan deriveaddresses() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VSTR, UniValueType()}); // Range argument is checked later const std::string desc_str = request.params[0].get_str(); int64_t range_begin = 0; diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index c712536b91..7704496c10 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -54,8 +54,11 @@ using node::PSBTAnalysis; using node::ReadBlockFromDisk; using node::UndoReadFromDisk; -static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, Chainstate& active_chainstate, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_TXID) +static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, + Chainstate& active_chainstate, const CTxUndo* txundo = nullptr, + TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS) { + CHECK_NONFATAL(verbosity >= TxVerbosity::SHOW_DETAILS); // Call into TxToUniv() in bitcoin-common to decode the transaction hex. // // Blockchain contextual information (confirmations and blocktime) is not @@ -162,7 +165,7 @@ static std::vector<RPCArg> CreateTxDoc() }, }, }, - }, + RPCArgOptions{.skip_type_check = true}}, {"locktime", RPCArg::Type::NUM, RPCArg::Default{0}, "Raw locktime. Non-0 value also locktime-activates inputs"}, {"replaceable", RPCArg::Type::BOOL, RPCArg::Default{true}, "Marks this transaction as BIP125-replaceable.\n" "Allows this transaction to be replaced by a transaction with higher fees. If provided, it is an error if explicit sequence numbers are incompatible."}, @@ -185,8 +188,9 @@ static RPCHelpMan getrawtransaction() "If verbosity is 2, returns a JSON Object with information about the transaction, including fee and prevout information.", { {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, - {"verbosity|verbose", RPCArg::Type::NUM, RPCArg::Default{0}, "0 for hex-encoded data, 1 for a JSON object, and 2 for JSON object with fee and prevout"}, - {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED_NAMED_ARG, "The block in which to look for the transaction"}, + {"verbosity|verbose", RPCArg::Type::NUM, RPCArg::Default{0}, "0 for hex-encoded data, 1 for a JSON object, and 2 for JSON object with fee and prevout", + RPCArgOptions{.skip_type_check = true}}, + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "The block in which to look for the transaction"}, }, { RPCResult{"if verbosity is not set or set to 0", @@ -354,14 +358,6 @@ static RPCHelpMan createrawtransaction() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, { - UniValue::VARR, - UniValueType(), // ARR or OBJ, checked later - UniValue::VNUM, - UniValue::VBOOL - }, true - ); - std::optional<bool> rbf; if (!request.params[3].isNull()) { rbf = request.params[3].get_bool(); @@ -397,8 +393,6 @@ static RPCHelpMan decoderawtransaction() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VBOOL}); - CMutableTransaction mtx; bool try_witness = request.params[1].isNull() ? true : request.params[1].get_bool(); @@ -451,8 +445,6 @@ static RPCHelpMan decodescript() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VSTR}); - UniValue r(UniValue::VOBJ); CScript script; if (request.params[0].get_str().size() > 0){ @@ -650,7 +642,7 @@ static RPCHelpMan signrawtransactionwithkey() {"privatekey", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "private key in base58-encoding"}, }, }, - {"prevtxs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "The previous dependent transaction outputs", + {"prevtxs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "The previous dependent transaction outputs", { {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { @@ -702,8 +694,6 @@ static RPCHelpMan signrawtransactionwithkey() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VARR, UniValue::VARR, UniValue::VSTR}, true); - CMutableTransaction mtx; if (!DecodeHexTx(mtx, request.params[0].get_str())) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input."); @@ -981,8 +971,6 @@ static RPCHelpMan decodepsbt() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VSTR}); - // Unserialize the transactions PartiallySignedTransaction psbtx; std::string error; @@ -1395,8 +1383,6 @@ static RPCHelpMan combinepsbt() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VARR}, true); - // Unserialize the transactions std::vector<PartiallySignedTransaction> psbtxs; UniValue txs = request.params[0].get_array(); @@ -1450,8 +1436,6 @@ static RPCHelpMan finalizepsbt() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VBOOL}, true); - // Unserialize the transactions PartiallySignedTransaction psbtx; std::string error; @@ -1499,14 +1483,6 @@ static RPCHelpMan createpsbt() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, { - UniValue::VARR, - UniValueType(), // ARR or OBJ, checked later - UniValue::VNUM, - UniValue::VBOOL, - }, true - ); - std::optional<bool> rbf; if (!request.params[3].isNull()) { rbf = request.params[3].get_bool(); @@ -1560,8 +1536,6 @@ static RPCHelpMan converttopsbt() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VBOOL, UniValue::VBOOL}, true); - // parse hex string from parameter CMutableTransaction tx; bool permitsigdata = request.params[1].isNull() ? false : request.params[1].get_bool(); @@ -1607,7 +1581,7 @@ static RPCHelpMan utxoupdatepsbt() "\nUpdates all segwit inputs and outputs in a PSBT with data from output descriptors, the UTXO set or the mempool.\n", { {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "A base64 string of a PSBT"}, - {"descriptors", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "An array of either strings or objects", { + {"descriptors", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "An array of either strings or objects", { {"", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"}, {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with an output descriptor and extra information", { {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"}, @@ -1623,8 +1597,6 @@ static RPCHelpMan utxoupdatepsbt() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VARR}, true); - // Unserialize the transactions PartiallySignedTransaction psbtx; std::string error; @@ -1714,8 +1686,6 @@ static RPCHelpMan joinpsbts() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VARR}, true); - // Unserialize the transactions std::vector<PartiallySignedTransaction> psbtxs; UniValue txs = request.params[0].get_array(); @@ -1842,8 +1812,6 @@ static RPCHelpMan analyzepsbt() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, {UniValue::VSTR}); - // Unserialize the transaction PartiallySignedTransaction psbtx; std::string error; diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 9f57a56297..44d7e2676b 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -168,7 +168,7 @@ static RPCHelpMan stop() // to the client (intended for testing) "\nRequest a graceful shutdown of " PACKAGE_NAME ".", { - {"wait", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "how long to wait in ms", RPCArgOptions{.hidden=true}}, + {"wait", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "how long to wait in ms", RPCArgOptions{.hidden=true}}, }, RPCResult{RPCResult::Type::STR, "", "A string with the content '" + RESULT + "'"}, RPCExamples{""}, diff --git a/src/rpc/txoutproof.cpp b/src/rpc/txoutproof.cpp index 8c5468634d..24b5d04115 100644 --- a/src/rpc/txoutproof.cpp +++ b/src/rpc/txoutproof.cpp @@ -34,7 +34,7 @@ static RPCHelpMan gettxoutproof() {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A transaction hash"}, }, }, - {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED_NAMED_ARG, "If specified, looks for txid in the block with this hash"}, + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "If specified, looks for txid in the block with this hash"}, }, RPCResult{ RPCResult::Type::STR, "data", "A string that is a serialized, hex-encoded data for the proof." @@ -112,7 +112,7 @@ static RPCHelpMan gettxoutproof() throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Not all transactions found in specified or retrieved block"); } - CDataStream ssMB(SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); + DataStream ssMB{}; CMerkleBlock mb(block, setTxids); ssMB << mb; std::string strHex = HexStr(ssMB); @@ -138,7 +138,7 @@ static RPCHelpMan verifytxoutproof() RPCExamples{""}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - CDataStream ssMB(ParseHexV(request.params[0], "proof"), SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); + DataStream ssMB{ParseHexV(request.params[0], "proof")}; CMerkleBlock merkleBlock; ssMB >> merkleBlock; diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 53785b1941..a1020c3b2b 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <clientversion.h> #include <consensus/amount.h> #include <key_io.h> #include <outputtype.h> @@ -30,31 +31,6 @@ std::string GetAllOutputTypes() return Join(ret, ", "); } -void RPCTypeCheck(const UniValue& params, - const std::list<UniValueType>& typesExpected, - bool fAllowNull) -{ - unsigned int i = 0; - for (const UniValueType& t : typesExpected) { - if (params.size() <= i) - break; - - const UniValue& v = params[i]; - if (!(fAllowNull && v.isNull())) { - RPCTypeCheckArgument(v, t); - } - i++; - } -} - -void RPCTypeCheckArgument(const UniValue& value, const UniValueType& typeExpected) -{ - if (!typeExpected.typeAny && value.type() != typeExpected.type) { - throw JSONRPCError(RPC_TYPE_ERROR, - strprintf("JSON value of type %s is not of expected type %s", uvTypeName(value.type()), uvTypeName(typeExpected.type))); - } -} - void RPCTypeCheckObj(const UniValue& o, const std::map<std::string, UniValueType>& typesExpected, bool fAllowNull, @@ -405,6 +381,7 @@ struct Sections { const auto indent = std::string(current_indent, ' '); const auto indent_next = std::string(current_indent + 2, ' '); const bool push_name{outer_type == OuterType::OBJ}; // Dictionary keys must have a name + const bool is_top_level_arg{outer_type == OuterType::NONE}; // True on the first recursion switch (arg.m_type) { case RPCArg::Type::STR_HEX: @@ -413,7 +390,7 @@ struct Sections { case RPCArg::Type::AMOUNT: case RPCArg::Type::RANGE: case RPCArg::Type::BOOL: { - if (outer_type == OuterType::NONE) return; // Nothing more to do for non-recursive types on first recursion + if (is_top_level_arg) return; // Nothing more to do for non-recursive types on first recursion auto left = indent; if (arg.m_opts.type_str.size() != 0 && push_name) { left += "\"" + arg.GetName() + "\": " + arg.m_opts.type_str.at(0); @@ -421,12 +398,12 @@ struct Sections { left += push_name ? arg.ToStringObj(/*oneline=*/false) : arg.ToString(/*oneline=*/false); } left += ","; - PushSection({left, arg.ToDescriptionString()}); + PushSection({left, arg.ToDescriptionString(/*is_named_arg=*/push_name)}); break; } case RPCArg::Type::OBJ: case RPCArg::Type::OBJ_USER_KEYS: { - const auto right = outer_type == OuterType::NONE ? "" : arg.ToDescriptionString(); + const auto right = is_top_level_arg ? "" : arg.ToDescriptionString(/*is_named_arg=*/push_name); PushSection({indent + (push_name ? "\"" + arg.GetName() + "\": " : "") + "{", right}); for (const auto& arg_inner : arg.m_inner) { Push(arg_inner, current_indent + 2, OuterType::OBJ); @@ -434,20 +411,20 @@ struct Sections { if (arg.m_type != RPCArg::Type::OBJ) { PushSection({indent_next + "...", ""}); } - PushSection({indent + "}" + (outer_type != OuterType::NONE ? "," : ""), ""}); + PushSection({indent + "}" + (is_top_level_arg ? "" : ","), ""}); break; } case RPCArg::Type::ARR: { auto left = indent; left += push_name ? "\"" + arg.GetName() + "\": " : ""; left += "["; - const auto right = outer_type == OuterType::NONE ? "" : arg.ToDescriptionString(); + const auto right = is_top_level_arg ? "" : arg.ToDescriptionString(/*is_named_arg=*/push_name); PushSection({left, right}); for (const auto& arg_inner : arg.m_inner) { Push(arg_inner, current_indent + 2, OuterType::ARR); } PushSection({indent_next + "...", ""}); - PushSection({indent + "]" + (outer_type != OuterType::NONE ? "," : ""), ""}); + PushSection({indent + "]" + (is_top_level_arg ? "" : ","), ""}); break; } } // no default case, so the compiler can warn about missing cases @@ -579,9 +556,39 @@ UniValue RPCHelpMan::HandleRequest(const JSONRPCRequest& request) const if (request.mode == JSONRPCRequest::GET_HELP || !IsValidNumArgs(request.params.size())) { throw std::runtime_error(ToString()); } + UniValue arg_mismatch{UniValue::VOBJ}; + for (size_t i{0}; i < m_args.size(); ++i) { + const auto& arg{m_args.at(i)}; + UniValue match{arg.MatchesType(request.params[i])}; + if (!match.isTrue()) { + arg_mismatch.pushKV(strprintf("Position %s (%s)", i + 1, arg.m_names), std::move(match)); + } + } + if (!arg_mismatch.empty()) { + throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Wrong type passed:\n%s", arg_mismatch.write(4))); + } UniValue ret = m_fun(*this, request); if (gArgs.GetBoolArg("-rpcdoccheck", DEFAULT_RPC_DOC_CHECK)) { - CHECK_NONFATAL(std::any_of(m_results.m_results.begin(), m_results.m_results.end(), [&ret](const RPCResult& res) { return res.MatchesType(ret); })); + UniValue mismatch{UniValue::VARR}; + for (const auto& res : m_results.m_results) { + UniValue match{res.MatchesType(ret)}; + if (match.isTrue()) { + mismatch.setNull(); + break; + } + mismatch.push_back(match); + } + if (!mismatch.isNull()) { + std::string explain{ + mismatch.empty() ? "no possible results defined" : + mismatch.size() == 1 ? mismatch[0].write(4) : + mismatch.write(4)}; + throw std::runtime_error{ + strprintf("Internal bug detected: RPC call \"%s\" returned incorrect type:\n%s\n%s %s\nPlease report this issue here: %s\n", + m_name, explain, + PACKAGE_NAME, FormatFullVersion(), + PACKAGE_BUGREPORT)}; + } } return ret; } @@ -641,7 +648,7 @@ std::string RPCHelpMan::ToString() const if (i == 0) ret += "\nArguments:\n"; // Push named argument name and description - sections.m_sections.emplace_back(::ToString(i + 1) + ". " + arg.GetFirstName(), arg.ToDescriptionString()); + sections.m_sections.emplace_back(::ToString(i + 1) + ". " + arg.GetFirstName(), arg.ToDescriptionString(/*is_named_arg=*/true)); sections.m_max_pad = std::max(sections.m_max_pad, sections.m_sections.back().m_left.size()); // Recursively push nested args @@ -677,6 +684,52 @@ UniValue RPCHelpMan::GetArgMap() const return arr; } +static std::optional<UniValue::VType> ExpectedType(RPCArg::Type type) +{ + using Type = RPCArg::Type; + switch (type) { + case Type::STR_HEX: + case Type::STR: { + return UniValue::VSTR; + } + case Type::NUM: { + return UniValue::VNUM; + } + case Type::AMOUNT: { + // VNUM or VSTR, checked inside AmountFromValue() + return std::nullopt; + } + case Type::RANGE: { + // VNUM or VARR, checked inside ParseRange() + return std::nullopt; + } + case Type::BOOL: { + return UniValue::VBOOL; + } + case Type::OBJ: + case Type::OBJ_USER_KEYS: { + return UniValue::VOBJ; + } + case Type::ARR: { + return UniValue::VARR; + } + } // no default case, so the compiler can warn about missing cases + NONFATAL_UNREACHABLE(); +} + +UniValue RPCArg::MatchesType(const UniValue& request) const +{ + if (m_opts.skip_type_check) return true; + if (IsOptional() && request.isNull()) return true; + const auto exp_type{ExpectedType(m_type)}; + if (!exp_type) return true; // nothing to check + + if (*exp_type != request.getType()) { + return strprintf("JSON value of type %s is not of expected type %s", uvTypeName(request.getType()), uvTypeName(*exp_type)); + } + return true; +} + std::string RPCArg::GetFirstName() const { return m_names.substr(0, m_names.find("|")); @@ -697,7 +750,7 @@ bool RPCArg::IsOptional() const } } -std::string RPCArg::ToDescriptionString() const +std::string RPCArg::ToDescriptionString(bool is_named_arg) const { std::string ret; ret += "("; @@ -744,13 +797,10 @@ std::string RPCArg::ToDescriptionString() const } else { switch (std::get<RPCArg::Optional>(m_fallback)) { case RPCArg::Optional::OMITTED: { + if (is_named_arg) ret += ", optional"; // Default value is "null" in dicts. Otherwise, // nothing to do. Element is treated as if not present and has no default value break; } - case RPCArg::Optional::OMITTED_NAMED_ARG: { - ret += ", optional"; // Default value is "null" - break; - } case RPCArg::Optional::NO: { ret += ", required"; break; @@ -860,53 +910,77 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const NONFATAL_UNREACHABLE(); } -bool RPCResult::MatchesType(const UniValue& result) const +static std::optional<UniValue::VType> ExpectedType(RPCResult::Type type) { - if (m_skip_type_check) { - return true; - } - switch (m_type) { + using Type = RPCResult::Type; + switch (type) { case Type::ELISION: case Type::ANY: { - return true; + return std::nullopt; } case Type::NONE: { - return UniValue::VNULL == result.getType(); + return UniValue::VNULL; } case Type::STR: case Type::STR_HEX: { - return UniValue::VSTR == result.getType(); + return UniValue::VSTR; } case Type::NUM: case Type::STR_AMOUNT: case Type::NUM_TIME: { - return UniValue::VNUM == result.getType(); + return UniValue::VNUM; } case Type::BOOL: { - return UniValue::VBOOL == result.getType(); + return UniValue::VBOOL; } case Type::ARR_FIXED: case Type::ARR: { - if (UniValue::VARR != result.getType()) return false; + return UniValue::VARR; + } + case Type::OBJ_DYN: + case Type::OBJ: { + return UniValue::VOBJ; + } + } // no default case, so the compiler can warn about missing cases + NONFATAL_UNREACHABLE(); +} + +UniValue RPCResult::MatchesType(const UniValue& result) const +{ + if (m_skip_type_check) { + return true; + } + + const auto exp_type = ExpectedType(m_type); + if (!exp_type) return true; // can be any type, so nothing to check + + if (*exp_type != result.getType()) { + return strprintf("returned type is %s, but declared as %s in doc", uvTypeName(result.getType()), uvTypeName(*exp_type)); + } + + if (UniValue::VARR == result.getType()) { + UniValue errors(UniValue::VOBJ); for (size_t i{0}; i < result.get_array().size(); ++i) { // If there are more results than documented, re-use the last doc_inner. const RPCResult& doc_inner{m_inner.at(std::min(m_inner.size() - 1, i))}; - if (!doc_inner.MatchesType(result.get_array()[i])) return false; + UniValue match{doc_inner.MatchesType(result.get_array()[i])}; + if (!match.isTrue()) errors.pushKV(strprintf("%d", i), match); } - return true; // empty result array is valid + if (errors.empty()) return true; // empty result array is valid + return errors; } - case Type::OBJ_DYN: - case Type::OBJ: { - if (UniValue::VOBJ != result.getType()) return false; + + if (UniValue::VOBJ == result.getType()) { if (!m_inner.empty() && m_inner.at(0).m_type == Type::ELISION) return true; + UniValue errors(UniValue::VOBJ); if (m_type == Type::OBJ_DYN) { const RPCResult& doc_inner{m_inner.at(0)}; // Assume all types are the same, randomly pick the first for (size_t i{0}; i < result.get_obj().size(); ++i) { - if (!doc_inner.MatchesType(result.get_obj()[i])) { - return false; - } + UniValue match{doc_inner.MatchesType(result.get_obj()[i])}; + if (!match.isTrue()) errors.pushKV(result.getKeys()[i], match); } - return true; // empty result obj is valid + if (errors.empty()) return true; // empty result obj is valid + return errors; } std::set<std::string> doc_keys; for (const auto& doc_entry : m_inner) { @@ -916,7 +990,7 @@ bool RPCResult::MatchesType(const UniValue& result) const result.getObjMap(result_obj); for (const auto& result_entry : result_obj) { if (doc_keys.find(result_entry.first) == doc_keys.end()) { - return false; // missing documentation + errors.pushKV(result_entry.first, "key returned that was not in doc"); } } @@ -924,18 +998,18 @@ bool RPCResult::MatchesType(const UniValue& result) const const auto result_it{result_obj.find(doc_entry.m_key_name)}; if (result_it == result_obj.end()) { if (!doc_entry.m_optional) { - return false; // result is missing a required key + errors.pushKV(doc_entry.m_key_name, "key missing, despite not being optional in doc"); } continue; } - if (!doc_entry.MatchesType(result_it->second)) { - return false; // wrong type - } + UniValue match{doc_entry.MatchesType(result_it->second)}; + if (!match.isTrue()) errors.pushKV(doc_entry.m_key_name, match); } - return true; + if (errors.empty()) return true; + return errors; } - } // no default case, so the compiler can warn about missing cases - NONFATAL_UNREACHABLE(); + + return true; } void RPCResult::CheckInnerDoc() const diff --git a/src/rpc/util.h b/src/rpc/util.h index 387fee34d3..e3783c8f76 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -62,18 +62,6 @@ struct UniValueType { UniValue::VType type; }; -/** - * Type-check arguments; throws JSONRPCError if wrong type given. Does not check that - * the right number of arguments are passed, just that any passed are the correct type. - */ -void RPCTypeCheck(const UniValue& params, - const std::list<UniValueType>& typesExpected, bool fAllowNull=false); - -/** - * Type-check one argument; throws JSONRPCError if wrong type given. - */ -void RPCTypeCheckArgument(const UniValue& value, const UniValueType& typeExpected); - /* Check for expected keys/value types in an Object. */ @@ -138,6 +126,7 @@ enum class OuterType { }; struct RPCArgOptions { + bool skip_type_check{false}; std::string oneline_description{}; //!< Should be empty unless it is supposed to override the auto-generated summary line std::vector<std::string> type_str{}; //!< Should be empty unless it is supposed to override the auto-generated type strings. Vector length is either 0 or 2, m_opts.type_str.at(0) will override the type of the value in a key-value pair, m_opts.type_str.at(1) will override the type in the argument description. bool hidden{false}; //!< For testing only @@ -160,21 +149,20 @@ struct RPCArg { /** Required arg */ NO, /** - * Optional arg that is a named argument and has a default value of - * `null`. When possible, the default value should be specified. - */ - OMITTED_NAMED_ARG, - /** - * Optional argument with default value omitted because they are - * implicitly clear. That is, elements in an array or object may not - * exist by default. + * Optional argument for which the default value is omitted from + * help text for one of two reasons: + * - It's a named argument and has a default value of `null`. + * - Its default value is implicitly clear. That is, elements in an + * array may not exist by default. * When possible, the default value should be specified. */ OMITTED, }; + /** Hint for default value */ using DefaultHint = std::string; + /** Default constant value */ using Default = UniValue; - using Fallback = std::variant<Optional, /* hint for default value */ DefaultHint, /* default constant value */ Default>; + using Fallback = std::variant<Optional, DefaultHint, Default>; const std::string m_names; //!< The name of the arg (can be empty for inner args, can contain multiple aliases separated by | for named request arguments) const Type m_type; @@ -184,10 +172,10 @@ struct RPCArg { const RPCArgOptions m_opts; RPCArg( - const std::string name, - const Type type, - const Fallback fallback, - const std::string description, + std::string name, + Type type, + Fallback fallback, + std::string description, RPCArgOptions opts = {}) : m_names{std::move(name)}, m_type{std::move(type)}, @@ -199,11 +187,11 @@ struct RPCArg { } RPCArg( - const std::string name, - const Type type, - const Fallback fallback, - const std::string description, - const std::vector<RPCArg> inner, + std::string name, + Type type, + Fallback fallback, + std::string description, + std::vector<RPCArg> inner, RPCArgOptions opts = {}) : m_names{std::move(name)}, m_type{std::move(type)}, @@ -217,6 +205,12 @@ struct RPCArg { bool IsOptional() const; + /** + * Check whether the request JSON type matches. + * Returns true if type matches, or object describing error(s) if not. + */ + UniValue MatchesType(const UniValue& request) const; + /** Return the first of all aliases */ std::string GetFirstName() const; @@ -237,7 +231,7 @@ struct RPCArg { * Return the description string, including the argument type and whether * the argument is required. */ - std::string ToDescriptionString() const; + std::string ToDescriptionString(bool is_named_arg) const; }; struct RPCResult { @@ -266,12 +260,12 @@ struct RPCResult { const std::string m_cond; RPCResult( - const std::string cond, - const Type type, - const std::string m_key_name, - const bool optional, - const std::string description, - const std::vector<RPCResult> inner = {}) + std::string cond, + Type type, + std::string m_key_name, + bool optional, + std::string description, + std::vector<RPCResult> inner = {}) : m_type{std::move(type)}, m_key_name{std::move(m_key_name)}, m_inner{std::move(inner)}, @@ -285,19 +279,19 @@ struct RPCResult { } RPCResult( - const std::string cond, - const Type type, - const std::string m_key_name, - const std::string description, - const std::vector<RPCResult> inner = {}) - : RPCResult{cond, type, m_key_name, false, description, inner} {} + std::string cond, + Type type, + std::string m_key_name, + std::string description, + std::vector<RPCResult> inner = {}) + : RPCResult{std::move(cond), type, std::move(m_key_name), /*optional=*/false, std::move(description), std::move(inner)} {} RPCResult( - const Type type, - const std::string m_key_name, - const bool optional, - const std::string description, - const std::vector<RPCResult> inner = {}, + Type type, + std::string m_key_name, + bool optional, + std::string description, + std::vector<RPCResult> inner = {}, bool skip_type_check = false) : m_type{std::move(type)}, m_key_name{std::move(m_key_name)}, @@ -311,12 +305,12 @@ struct RPCResult { } RPCResult( - const Type type, - const std::string m_key_name, - const std::string description, - const std::vector<RPCResult> inner = {}, + Type type, + std::string m_key_name, + std::string description, + std::vector<RPCResult> inner = {}, bool skip_type_check = false) - : RPCResult{type, m_key_name, false, description, inner, skip_type_check} {} + : RPCResult{type, std::move(m_key_name), /*optional=*/false, std::move(description), std::move(inner), skip_type_check} {} /** Append the sections of the result. */ void ToSections(Sections& sections, OuterType outer_type = OuterType::NONE, const int current_indent = 0) const; @@ -324,8 +318,10 @@ struct RPCResult { std::string ToStringObj() const; /** Return the description string, including the result type. */ std::string ToDescriptionString() const; - /** Check whether the result JSON type matches. */ - bool MatchesType(const UniValue& result) const; + /** Check whether the result JSON type matches. + * Returns true if type matches, or object describing error(s) if not. + */ + UniValue MatchesType(const UniValue& result) const; private: void CheckInnerDoc() const; diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp index 6b87902daf..4fab481b39 100644 --- a/src/script/bitcoinconsensus.cpp +++ b/src/script/bitcoinconsensus.cpp @@ -62,12 +62,6 @@ inline int set_error(bitcoinconsensus_error* ret, bitcoinconsensus_error serror) return 0; } -struct ECCryptoClosure -{ - ECCVerifyHandle handle; -}; - -ECCryptoClosure instance_of_eccryptoclosure; } // namespace /** Check that all specified flags are part of the libconsensus interface. */ diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 864eb8864f..8991bda340 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -4,11 +4,13 @@ #include <script/descriptor.h> +#include <hash.h> #include <key_io.h> #include <pubkey.h> #include <script/miniscript.h> #include <script/script.h> #include <script/standard.h> +#include <uint256.h> #include <span.h> #include <util/bip32.h> @@ -1618,8 +1620,7 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo } } if (txntype == TxoutType::WITNESS_V0_SCRIPTHASH && (ctx == ParseScriptContext::TOP || ctx == ParseScriptContext::P2SH)) { - CScriptID scriptid; - CRIPEMD160().Write(data[0].data(), data[0].size()).Finalize(scriptid.begin()); + CScriptID scriptid{RIPEMD160(data[0])}; CScript subscript; if (provider.GetCScript(scriptid, subscript)) { auto sub = InferScript(subscript, ParseScriptContext::P2WSH, provider); @@ -1643,7 +1644,7 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo for (const auto& [depth, script, leaf_ver] : *tree) { std::unique_ptr<DescriptorImpl> subdesc; if (leaf_ver == TAPROOT_LEAF_TAPSCRIPT) { - subdesc = InferScript(script, ParseScriptContext::P2TR, provider); + subdesc = InferScript(CScript(script.begin(), script.end()), ParseScriptContext::P2TR, provider); } if (!subdesc) { ok = false; @@ -1832,17 +1833,17 @@ DescriptorCache DescriptorCache::MergeAndDiff(const DescriptorCache& other) return diff; } -const ExtPubKeyMap DescriptorCache::GetCachedParentExtPubKeys() const +ExtPubKeyMap DescriptorCache::GetCachedParentExtPubKeys() const { return m_parent_xpubs; } -const std::unordered_map<uint32_t, ExtPubKeyMap> DescriptorCache::GetCachedDerivedExtPubKeys() const +std::unordered_map<uint32_t, ExtPubKeyMap> DescriptorCache::GetCachedDerivedExtPubKeys() const { return m_derived_xpubs; } -const ExtPubKeyMap DescriptorCache::GetCachedLastHardenedExtPubKeys() const +ExtPubKeyMap DescriptorCache::GetCachedLastHardenedExtPubKeys() const { return m_last_hardened_xpubs; } diff --git a/src/script/descriptor.h b/src/script/descriptor.h index 16ee2f6d97..cb3b366acf 100644 --- a/src/script/descriptor.h +++ b/src/script/descriptor.h @@ -66,11 +66,11 @@ public: bool GetCachedLastHardenedExtPubKey(uint32_t key_exp_pos, CExtPubKey& xpub) const; /** Retrieve all cached parent xpubs */ - const ExtPubKeyMap GetCachedParentExtPubKeys() const; + ExtPubKeyMap GetCachedParentExtPubKeys() const; /** Retrieve all cached derived xpubs */ - const std::unordered_map<uint32_t, ExtPubKeyMap> GetCachedDerivedExtPubKeys() const; + std::unordered_map<uint32_t, ExtPubKeyMap> GetCachedDerivedExtPubKeys() const; /** Retrieve all cached last hardened xpubs */ - const ExtPubKeyMap GetCachedLastHardenedExtPubKeys() const; + ExtPubKeyMap GetCachedLastHardenedExtPubKeys() const; /** Combine another DescriptorCache into this one. * Returns a cache containing the items from the other cache unknown to current cache diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index a942ff349b..03b157a847 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1825,9 +1825,20 @@ static bool ExecuteWitnessScript(const Span<const valtype>& stack_span, const CS return true; } -uint256 ComputeTapleafHash(uint8_t leaf_version, const CScript& script) +uint256 ComputeTapleafHash(uint8_t leaf_version, Span<const unsigned char> script) { - return (HashWriter{HASHER_TAPLEAF} << leaf_version << script).GetSHA256(); + return (HashWriter{HASHER_TAPLEAF} << leaf_version << CompactSizeWriter(script.size()) << script).GetSHA256(); +} + +uint256 ComputeTapbranchHash(Span<const unsigned char> a, Span<const unsigned char> b) +{ + HashWriter ss_branch{HASHER_TAPBRANCH}; + if (std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end())) { + ss_branch << a << b; + } else { + ss_branch << b << a; + } + return ss_branch.GetSHA256(); } uint256 ComputeTaprootMerkleRoot(Span<const unsigned char> control, const uint256& tapleaf_hash) @@ -1839,14 +1850,8 @@ uint256 ComputeTaprootMerkleRoot(Span<const unsigned char> control, const uint25 const int path_len = (control.size() - TAPROOT_CONTROL_BASE_SIZE) / TAPROOT_CONTROL_NODE_SIZE; uint256 k = tapleaf_hash; for (int i = 0; i < path_len; ++i) { - HashWriter ss_branch{HASHER_TAPBRANCH}; Span node{Span{control}.subspan(TAPROOT_CONTROL_BASE_SIZE + TAPROOT_CONTROL_NODE_SIZE * i, TAPROOT_CONTROL_NODE_SIZE)}; - if (std::lexicographical_compare(k.begin(), k.end(), node.begin(), node.end())) { - ss_branch << k << node; - } else { - ss_branch << node << k; - } - k = ss_branch.GetSHA256(); + k = ComputeTapbranchHash(k, node); } return k; } @@ -1917,18 +1922,18 @@ static bool VerifyWitnessProgram(const CScriptWitness& witness, int witversion, } else { // Script path spending (stack size is >1 after removing optional annex) const valtype& control = SpanPopBack(stack); - const valtype& script_bytes = SpanPopBack(stack); - exec_script = CScript(script_bytes.begin(), script_bytes.end()); + const valtype& script = SpanPopBack(stack); if (control.size() < TAPROOT_CONTROL_BASE_SIZE || control.size() > TAPROOT_CONTROL_MAX_SIZE || ((control.size() - TAPROOT_CONTROL_BASE_SIZE) % TAPROOT_CONTROL_NODE_SIZE) != 0) { return set_error(serror, SCRIPT_ERR_TAPROOT_WRONG_CONTROL_SIZE); } - execdata.m_tapleaf_hash = ComputeTapleafHash(control[0] & TAPROOT_LEAF_MASK, exec_script); + execdata.m_tapleaf_hash = ComputeTapleafHash(control[0] & TAPROOT_LEAF_MASK, script); if (!VerifyTaprootCommitment(control, program, execdata.m_tapleaf_hash)) { return set_error(serror, SCRIPT_ERR_WITNESS_PROGRAM_MISMATCH); } execdata.m_tapleaf_hash_init = true; if ((control[0] & TAPROOT_LEAF_MASK) == TAPROOT_LEAF_TAPSCRIPT) { // Tapscript (leaf version 0xc0) + exec_script = CScript(script.begin(), script.end()); execdata.m_validation_weight_left = ::GetSerializeSize(witness.stack, PROTOCOL_VERSION) + VALIDATION_WEIGHT_OFFSET; execdata.m_validation_weight_left_init = true; return ExecuteWitnessScript(stack, exec_script, flags, SigVersion::TAPSCRIPT, checker, execdata, serror); diff --git a/src/script/interpreter.h b/src/script/interpreter.h index 42282e6e5c..ac1013302d 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -333,7 +333,10 @@ public: }; /** Compute the BIP341 tapleaf hash from leaf version & script. */ -uint256 ComputeTapleafHash(uint8_t leaf_version, const CScript& script); +uint256 ComputeTapleafHash(uint8_t leaf_version, Span<const unsigned char> script); +/** Compute the BIP341 tapbranch hash from two branches. + * Spans must be 32 bytes each. */ +uint256 ComputeTapbranchHash(Span<const unsigned char> a, Span<const unsigned char> b); /** Compute the BIP341 taproot script tree Merkle root from control block and leaf hash. * Requires control block to have valid length (33 + k*32, with k in {0,1,..,128}). */ uint256 ComputeTaprootMerkleRoot(Span<const unsigned char> control, const uint256& tapleaf_hash); diff --git a/src/script/miniscript.h b/src/script/miniscript.h index fa3b0350e9..3a3f724f03 100644 --- a/src/script/miniscript.h +++ b/src/script/miniscript.h @@ -1378,7 +1378,7 @@ inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) assert(constructed.size() == 1); assert(constructed[0]->ScriptSize() == script_size); if (in.size() > 0) return {}; - const NodeRef<Key> tl_node = std::move(constructed.front()); + NodeRef<Key> tl_node = std::move(constructed.front()); tl_node->DuplicateKeyCheck(ctx); return tl_node; } @@ -1813,7 +1813,7 @@ inline NodeRef<Key> DecodeScript(I& in, I last, const Ctx& ctx) } } if (constructed.size() != 1) return {}; - const NodeRef<Key> tl_node = std::move(constructed.front()); + NodeRef<Key> tl_node = std::move(constructed.front()); tl_node->DuplicateKeyCheck(ctx); // Note that due to how ComputeType works (only assign the type to the node if the // subs' types are valid) this would fail if any node of tree is badly typed. diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 1a8558cd9f..70df9ee62c 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -169,13 +169,14 @@ static bool CreateTaprootScriptSig(const BaseSignatureCreator& creator, Signatur return false; } -static bool SignTaprootScript(const SigningProvider& provider, const BaseSignatureCreator& creator, SignatureData& sigdata, int leaf_version, const CScript& script, std::vector<valtype>& result) +static bool SignTaprootScript(const SigningProvider& provider, const BaseSignatureCreator& creator, SignatureData& sigdata, int leaf_version, Span<const unsigned char> script_bytes, std::vector<valtype>& result) { // Only BIP342 tapscript signing is supported for now. if (leaf_version != TAPROOT_LEAF_TAPSCRIPT) return false; SigVersion sigversion = SigVersion::TAPSCRIPT; - uint256 leaf_hash = (HashWriter{HASHER_TAPLEAF} << uint8_t(leaf_version) << script).GetSHA256(); + uint256 leaf_hash = ComputeTapleafHash(leaf_version, script_bytes); + CScript script = CScript(script_bytes.begin(), script_bytes.end()); // <xonly pubkey> OP_CHECKSIG if (script.size() == 34 && script[33] == OP_CHECKSIG && script[0] == 0x20) { @@ -285,7 +286,6 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator std::vector<valtype>& ret, TxoutType& whichTypeRet, SigVersion sigversion, SignatureData& sigdata) { CScript scriptRet; - uint160 h160; ret.clear(); std::vector<unsigned char> sig; @@ -314,8 +314,8 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator ret.push_back(ToByteVector(pubkey)); return true; } - case TxoutType::SCRIPTHASH: - h160 = uint160(vSolutions[0]); + case TxoutType::SCRIPTHASH: { + uint160 h160{vSolutions[0]}; if (GetCScript(provider, sigdata, CScriptID{h160}, scriptRet)) { ret.push_back(std::vector<unsigned char>(scriptRet.begin(), scriptRet.end())); return true; @@ -323,7 +323,7 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator // Could not find redeemScript, add to missing sigdata.missing_redeem_script = h160; return false; - + } case TxoutType::MULTISIG: { size_t required = vSolutions.front()[0]; ret.push_back(valtype()); // workaround CHECKMULTISIG bug @@ -349,8 +349,7 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator return true; case TxoutType::WITNESS_V0_SCRIPTHASH: - CRIPEMD160().Write(vSolutions[0].data(), vSolutions[0].size()).Finalize(h160.begin()); - if (GetCScript(provider, sigdata, CScriptID{h160}, scriptRet)) { + if (GetCScript(provider, sigdata, CScriptID{RIPEMD160(vSolutions[0])}, scriptRet)) { ret.push_back(std::vector<unsigned char>(scriptRet.begin(), scriptRet.end())); return true; } diff --git a/src/script/sign.h b/src/script/sign.h index b32bb55dd3..263fb61fc5 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -13,6 +13,7 @@ #include <script/interpreter.h> #include <script/keyorigin.h> #include <script/standard.h> +#include <uint256.h> class CKey; class CKeyID; diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 27b9a5c741..7c4a05b6e6 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -370,12 +370,7 @@ bool IsValidDestination(const CTxDestination& dest) { leaf.merkle_branch.push_back(a.hash); ret.leaves.emplace_back(std::move(leaf)); } - /* Lexicographically sort a and b's hash, and compute parent hash. */ - if (a.hash < b.hash) { - ret.hash = (HashWriter{HASHER_TAPBRANCH} << a.hash << b.hash).GetSHA256(); - } else { - ret.hash = (HashWriter{HASHER_TAPBRANCH} << b.hash << a.hash).GetSHA256(); - } + ret.hash = ComputeTapbranchHash(a.hash, b.hash); return ret; } @@ -443,14 +438,14 @@ void TaprootBuilder::Insert(TaprootBuilder::NodeInfo&& node, int depth) return branch.size() == 0 || (branch.size() == 1 && branch[0]); } -TaprootBuilder& TaprootBuilder::Add(int depth, const CScript& script, int leaf_version, bool track) +TaprootBuilder& TaprootBuilder::Add(int depth, Span<const unsigned char> script, int leaf_version, bool track) { assert((leaf_version & ~TAPROOT_LEAF_MASK) == 0); if (!IsValid()) return *this; /* Construct NodeInfo object with leaf hash and (if track is true) also leaf information. */ NodeInfo node; - node.hash = (HashWriter{HASHER_TAPLEAF} << uint8_t(leaf_version) << script).GetSHA256(); - if (track) node.leaves.emplace_back(LeafInfo{script, leaf_version, {}}); + node.hash = ComputeTapleafHash(leaf_version, script); + if (track) node.leaves.emplace_back(LeafInfo{std::vector<unsigned char>(script.begin(), script.end()), leaf_version, {}}); /* Insert into the branch. */ Insert(std::move(node), depth); return *this; @@ -506,13 +501,13 @@ TaprootSpendData TaprootBuilder::GetSpendData() const return spd; } -std::optional<std::vector<std::tuple<int, CScript, int>>> InferTaprootTree(const TaprootSpendData& spenddata, const XOnlyPubKey& output) +std::optional<std::vector<std::tuple<int, std::vector<unsigned char>, int>>> InferTaprootTree(const TaprootSpendData& spenddata, const XOnlyPubKey& output) { // Verify that the output matches the assumed Merkle root and internal key. auto tweak = spenddata.internal_key.CreateTapTweak(spenddata.merkle_root.IsNull() ? nullptr : &spenddata.merkle_root); if (!tweak || tweak->first != output) return std::nullopt; // If the Merkle root is 0, the tree is empty, and we're done. - std::vector<std::tuple<int, CScript, int>> ret; + std::vector<std::tuple<int, std::vector<unsigned char>, int>> ret; if (spenddata.merkle_root.IsNull()) return ret; /** Data structure to represent the nodes of the tree we're going to build. */ @@ -523,7 +518,7 @@ std::optional<std::vector<std::tuple<int, CScript, int>>> InferTaprootTree(const std::unique_ptr<TreeNode> sub[2]; /** If this is known to be a leaf node, a pointer to the (script, leaf_ver) pair. * nullptr otherwise. */ - const std::pair<CScript, int>* leaf = nullptr; + const std::pair<std::vector<unsigned char>, int>* leaf = nullptr; /** Whether or not this node has been explored (is known to be a leaf, or known to have children). */ bool explored = false; /** Whether or not this node is an inner node (unknown until explored = true). */ @@ -607,7 +602,7 @@ std::optional<std::vector<std::tuple<int, CScript, int>>> InferTaprootTree(const node.done = true; stack.pop_back(); } else if (node.sub[0]->done && !node.sub[1]->done && !node.sub[1]->explored && !node.sub[1]->hash.IsNull() && - (HashWriter{HASHER_TAPBRANCH} << node.sub[1]->hash << node.sub[1]->hash).GetSHA256() == node.hash) { + ComputeTapbranchHash(node.sub[1]->hash, node.sub[1]->hash) == node.hash) { // Whenever there are nodes with two identical subtrees under it, we run into a problem: // the control blocks for the leaves underneath those will be identical as well, and thus // they will all be matched to the same path in the tree. The result is that at the location @@ -641,10 +636,10 @@ std::optional<std::vector<std::tuple<int, CScript, int>>> InferTaprootTree(const return ret; } -std::vector<std::tuple<uint8_t, uint8_t, CScript>> TaprootBuilder::GetTreeTuples() const +std::vector<std::tuple<uint8_t, uint8_t, std::vector<unsigned char>>> TaprootBuilder::GetTreeTuples() const { assert(IsComplete()); - std::vector<std::tuple<uint8_t, uint8_t, CScript>> tuples; + std::vector<std::tuple<uint8_t, uint8_t, std::vector<unsigned char>>> tuples; if (m_branch.size()) { const auto& leaves = m_branch[0]->leaves; for (const auto& leaf : leaves) { diff --git a/src/script/standard.h b/src/script/standard.h index f08258af4f..18cf5c8c88 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -217,7 +217,7 @@ struct TaprootSpendData * inference can reconstruct the full tree. Within each set, the control * blocks are sorted by size, so that the signing logic can easily * prefer the cheapest one. */ - std::map<std::pair<CScript, int>, std::set<std::vector<unsigned char>, ShortestVectorFirstComparator>> scripts; + std::map<std::pair<std::vector<unsigned char>, int>, std::set<std::vector<unsigned char>, ShortestVectorFirstComparator>> scripts; /** Merge other TaprootSpendData (for the same scriptPubKey) into this. */ void Merge(TaprootSpendData other); }; @@ -229,7 +229,7 @@ private: /** Information about a tracked leaf in the Merkle tree. */ struct LeafInfo { - CScript script; //!< The script. + std::vector<unsigned char> script; //!< The script. int leaf_version; //!< The leaf version for that script. std::vector<uint256> merkle_branch; //!< The hashing partners above this leaf. }; @@ -296,7 +296,7 @@ public: /** Add a new script at a certain depth in the tree. Add() operations must be called * in depth-first traversal order of binary tree. If track is true, it will be included in * the GetSpendData() output. */ - TaprootBuilder& Add(int depth, const CScript& script, int leaf_version, bool track = true); + TaprootBuilder& Add(int depth, Span<const unsigned char> script, int leaf_version, bool track = true); /** Like Add(), but for a Merkle node with a given hash to the tree. */ TaprootBuilder& AddOmitted(int depth, const uint256& hash); /** Finalize the construction. Can only be called when IsComplete() is true. @@ -314,7 +314,7 @@ public: /** Compute spending data (after Finalize()). */ TaprootSpendData GetSpendData() const; /** Returns a vector of tuples representing the depth, leaf version, and script */ - std::vector<std::tuple<uint8_t, uint8_t, CScript>> GetTreeTuples() const; + std::vector<std::tuple<uint8_t, uint8_t, std::vector<unsigned char>>> GetTreeTuples() const; /** Returns true if there are any tapscripts */ bool HasScripts() const { return !m_branch.empty(); } }; @@ -325,6 +325,6 @@ public: * std::nullopt is returned. Otherwise, a vector of (depth, script, leaf_ver) tuples is * returned, corresponding to a depth-first traversal of the script tree. */ -std::optional<std::vector<std::tuple<int, CScript, int>>> InferTaprootTree(const TaprootSpendData& spenddata, const XOnlyPubKey& output); +std::optional<std::vector<std::tuple<int, std::vector<unsigned char>, int>>> InferTaprootTree(const TaprootSpendData& spenddata, const XOnlyPubKey& output); #endif // BITCOIN_SCRIPT_STANDARD_H diff --git a/src/secp256k1/.cirrus.yml b/src/secp256k1/.cirrus.yml index a2e7f36d1f..51e3bc9484 100644 --- a/src/secp256k1/.cirrus.yml +++ b/src/secp256k1/.cirrus.yml @@ -26,6 +26,11 @@ env: # Compile and run the tests EXAMPLES: yes +# https://cirrus-ci.org/pricing/#compute-credits +credits_snippet: &CREDITS + # Don't use any credits for now. + use_compute_credits: false + cat_logs_snippet: &CAT_LOGS always: cat_tests_log_script: @@ -36,7 +41,6 @@ cat_logs_snippet: &CAT_LOGS - cat valgrind_ctime_test.log || true cat_bench_log_script: - cat bench.log || true - on_failure: cat_config_log_script: - cat config.log || true cat_test_env_script: @@ -69,6 +73,7 @@ task: - env: {WIDEMUL: int64, RECOVERY: yes} - env: {WIDEMUL: int64, ECDH: yes, SCHNORRSIG: yes} - env: {WIDEMUL: int128} + - env: {WIDEMUL: int128_struct} - env: {WIDEMUL: int128, RECOVERY: yes, SCHNORRSIG: yes} - env: {WIDEMUL: int128, ECDH: yes, SCHNORRSIG: yes} - env: {WIDEMUL: int128, ASM: x86_64} @@ -107,65 +112,32 @@ task: << : *CAT_LOGS task: - name: "x86_64: macOS Catalina" + name: "arm64: macOS Ventura" macos_instance: - image: catalina-base + image: ghcr.io/cirruslabs/macos-ventura-base:latest env: HOMEBREW_NO_AUTO_UPDATE: 1 HOMEBREW_NO_INSTALL_CLEANUP: 1 - # Cirrus gives us a fixed number of 12 virtual CPUs. Not that we even have that many jobs at the moment... - MAKEFLAGS: -j13 + # Cirrus gives us a fixed number of 4 virtual CPUs. Not that we even have that many jobs at the moment... + MAKEFLAGS: -j5 matrix: << : *ENV_MATRIX + env: + ASM: no + WITH_VALGRIND: no + CTIMETEST: no matrix: - env: - CC: gcc-9 + CC: gcc - env: CC: clang - # Update Command Line Tools - # Uncomment this if the Command Line Tools on the CirrusCI macOS image are too old to brew valgrind. - # See https://apple.stackexchange.com/a/195963 for the implementation. - ## update_clt_script: - ## - system_profiler SPSoftwareDataType - ## - touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress - ## - |- - ## PROD=$(softwareupdate -l | grep "*.*Command Line" | tail -n 1 | awk -F"*" '{print $2}' | sed -e 's/^ *//' | sed 's/Label: //g' | tr -d '\n') - ## # For debugging - ## - softwareupdate -l && echo "PROD: $PROD" - ## - softwareupdate -i "$PROD" --verbose - ## - rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress - ## - brew_valgrind_pre_script: - # Retry a few times because this tends to fail randomly. - - for i in {1..5}; do brew update && break || sleep 15; done - - brew config - - brew tap LouisBrunner/valgrind - # Fetch valgrind source but don't build it yet. - - brew fetch --HEAD LouisBrunner/valgrind/valgrind - brew_valgrind_cache: - # This is $(brew --cellar valgrind) but command substition does not work here. - folder: /usr/local/Cellar/valgrind - # Rebuild cache if ... - fingerprint_script: - # ... macOS version changes: - - sw_vers - # ... brew changes: - - brew config - # ... valgrind changes: - - git -C "$(brew --cache)/valgrind--git" rev-parse HEAD - populate_script: - # If there's no hit in the cache, build and install valgrind. - - brew install --HEAD LouisBrunner/valgrind/valgrind - brew_valgrind_post_script: - # If we have restored valgrind from the cache, tell brew to create symlink to the PATH. - # If we haven't restored from cached (and just run brew install), this is a no-op. - - brew link valgrind brew_script: - - brew install automake libtool gcc@9 + - brew install automake libtool gcc << : *MERGE_BASE test_script: - ./ci/cirrus.sh << : *CAT_LOGS + << : *CREDITS task: name: "s390x (big-endian): Linux (Debian stable, QEMU)" @@ -241,17 +213,63 @@ task: << : *CAT_LOGS task: - name: "x86_64 (mingw32-w64): Windows (Debian stable, Wine)" << : *LINUX_CONTAINER env: - WRAPPER_CMD: wine64-stable - SECP256K1_TEST_ITERS: 16 - HOST: x86_64-w64-mingw32 + WRAPPER_CMD: wine WITH_VALGRIND: no ECDH: yes RECOVERY: yes SCHNORRSIG: yes CTIMETEST: no + matrix: + - name: "x86_64 (mingw32-w64): Windows (Debian stable, Wine)" + env: + HOST: x86_64-w64-mingw32 + - name: "i686 (mingw32-w64): Windows (Debian stable, Wine)" + env: + HOST: i686-w64-mingw32 + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + << : *LINUX_CONTAINER + env: + WRAPPER_CMD: wine + WERROR_CFLAGS: -WX + WITH_VALGRIND: no + ECDH: yes + RECOVERY: yes + EXPERIMENTAL: yes + SCHNORRSIG: yes + CTIMETEST: no + # Use a MinGW-w64 host to tell ./configure we're building for Windows. + # This will detect some MinGW-w64 tools but then make will need only + # the MSVC tools CC, AR and NM as specified below. + HOST: x86_64-w64-mingw32 + CC: /opt/msvc/bin/x64/cl + AR: /opt/msvc/bin/x64/lib + NM: /opt/msvc/bin/x64/dumpbin -symbols -headers + # Set non-essential options that affect the CLI messages here. + # (They depend on the user's taste, so we don't want to set them automatically in configure.ac.) + CFLAGS: -nologo -diagnostics:caret + LDFLAGS: -XCClinker -nologo -XCClinker -diagnostics:caret + matrix: + - name: "x86_64 (MSVC): Windows (Debian stable, Wine)" + - name: "x86_64 (MSVC): Windows (Debian stable, Wine, int128_struct)" + env: + WIDEMUL: int128_struct + - name: "x86_64 (MSVC): Windows (Debian stable, Wine, int128_struct with __(u)mulh)" + env: + WIDEMUL: int128_struct + CPPFLAGS: -DSECP256K1_MSVC_MULH_TEST_OVERRIDE + - name: "i686 (MSVC): Windows (Debian stable, Wine)" + env: + HOST: i686-w64-mingw32 + CC: /opt/msvc/bin/x86/cl + AR: /opt/msvc/bin/x86/lib + NM: /opt/msvc/bin/x86/dumpbin -symbols -headers << : *MERGE_BASE test_script: - ./ci/cirrus.sh @@ -301,14 +319,39 @@ task: - ./ci/cirrus.sh << : *CAT_LOGS +# Memory sanitizers task: - name: "C++ -fpermissive" << : *LINUX_CONTAINER + name: "MSan" env: - # ./configure correctly errors out when given CC=g++. - # We hack around this by passing CC=g++ only to make. - CC: gcc - MAKEFLAGS: -j4 CC=g++ CFLAGS=-fpermissive\ -g + ECDH: yes + RECOVERY: yes + SCHNORRSIG: yes + CTIMETEST: no + CC: clang + SECP256K1_TEST_ITERS: 32 + ASM: no + container: + memory: 2G + matrix: + - env: + CFLAGS: "-fsanitize=memory -g" + - env: + ECMULTGENPRECISION: 2 + ECMULTWINDOW: 2 + CFLAGS: "-fsanitize=memory -g -O3" + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + name: "C++ -fpermissive (entire project)" + << : *LINUX_CONTAINER + env: + CC: g++ + CFLAGS: -fpermissive -g + CPPFLAGS: -DSECP256K1_CPLUSPLUS_TEST_OVERRIDE WERROR_CFLAGS: ECDH: yes RECOVERY: yes @@ -319,6 +362,14 @@ task: << : *CAT_LOGS task: + name: "C++ (public headers)" + << : *LINUX_CONTAINER + test_script: + - g++ -Werror include/*.h + - clang -Werror -x c++-header include/*.h + - /opt/msvc/bin/x64/cl.exe -c -WX -TP include/*.h + +task: name: "sage prover" << : *LINUX_CONTAINER test_script: diff --git a/src/secp256k1/.gitignore b/src/secp256k1/.gitignore index d88627d72e..80c646b771 100644 --- a/src/secp256k1/.gitignore +++ b/src/secp256k1/.gitignore @@ -13,9 +13,9 @@ schnorr_example *.so *.a *.csv -!.gitignore *.log *.trs +*.sage.py Makefile configure @@ -34,8 +34,6 @@ libtool *.lo *.o *~ -*.log -*.trs coverage/ coverage.html diff --git a/src/secp256k1/CHANGELOG.md b/src/secp256k1/CHANGELOG.md new file mode 100644 index 0000000000..7443483423 --- /dev/null +++ b/src/secp256k1/CHANGELOG.md @@ -0,0 +1,28 @@ +# Changelog + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +## [Unreleased] + +## [0.2.0] - 2022-12-12 + +### Added + - Added `secp256k1_selftest`, to be used in conjunction with `secp256k1_context_static`. + +### Changed + - Enabled modules schnorrsig, extrakeys and ECDH by default in `./configure`. + +### Deprecated + - Deprecated context flags `SECP256K1_CONTEXT_VERIFY` and `SECP256K1_CONTEXT_SIGN`. Use `SECP256K1_CONTEXT_NONE` instead. + - Renamed `secp256k1_context_no_precomp` to `secp256k1_context_static`. + +### ABI Compatibility + +Since this is the first release, we do not compare application binary interfaces. +However, there are unreleased versions of libsecp256k1 that are *not* ABI compatible with this version. + +## [0.1.0] - 2013-03-05 to 2021-12-25 + +This version was in fact never released. +The number was given by the build system since the introduction of autotools in Jan 2014 (ea0fe5a5bf0c04f9cc955b2966b614f5f378c6f6). +Therefore, this version number does not uniquely identify a set of source files. diff --git a/src/secp256k1/Makefile.am b/src/secp256k1/Makefile.am index 51c5960301..ad50504f7e 100644 --- a/src/secp256k1/Makefile.am +++ b/src/secp256k1/Makefile.am @@ -48,6 +48,12 @@ noinst_HEADERS += src/precomputed_ecmult.h noinst_HEADERS += src/precomputed_ecmult_gen.h noinst_HEADERS += src/assumptions.h noinst_HEADERS += src/util.h +noinst_HEADERS += src/int128.h +noinst_HEADERS += src/int128_impl.h +noinst_HEADERS += src/int128_native.h +noinst_HEADERS += src/int128_native_impl.h +noinst_HEADERS += src/int128_struct.h +noinst_HEADERS += src/int128_struct_impl.h noinst_HEADERS += src/scratch.h noinst_HEADERS += src/scratch_impl.h noinst_HEADERS += src/selftest.h @@ -58,7 +64,6 @@ noinst_HEADERS += src/hash_impl.h noinst_HEADERS += src/field.h noinst_HEADERS += src/field_impl.h noinst_HEADERS += src/bench.h -noinst_HEADERS += src/basic-config.h noinst_HEADERS += contrib/lax_der_parsing.h noinst_HEADERS += contrib/lax_der_parsing.c noinst_HEADERS += contrib/lax_der_privatekey_parsing.h @@ -87,7 +92,7 @@ endif endif libsecp256k1_la_SOURCES = src/secp256k1.c -libsecp256k1_la_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/src $(SECP_INCLUDES) +libsecp256k1_la_CPPFLAGS = $(SECP_INCLUDES) libsecp256k1_la_LIBADD = $(SECP_LIBS) $(COMMON_LIB) $(PRECOMPUTED_LIB) libsecp256k1_la_LDFLAGS = -no-undefined -version-info $(LIB_VERSION_CURRENT):$(LIB_VERSION_REVISION):$(LIB_VERSION_AGE) @@ -112,7 +117,7 @@ TESTS = if USE_TESTS noinst_PROGRAMS += tests tests_SOURCES = src/tests.c -tests_CPPFLAGS = -I$(top_srcdir)/src -I$(top_srcdir)/include $(SECP_INCLUDES) $(SECP_TEST_INCLUDES) +tests_CPPFLAGS = $(SECP_INCLUDES) $(SECP_TEST_INCLUDES) if VALGRIND_ENABLED tests_CPPFLAGS += -DVALGRIND noinst_PROGRAMS += valgrind_ctime_test @@ -211,7 +216,15 @@ maintainer-clean-local: clean-precomp clean-precomp: rm -f $(PRECOMP) -EXTRA_DIST = autogen.sh SECURITY.md +EXTRA_DIST = autogen.sh CHANGELOG.md SECURITY.md +EXTRA_DIST += doc/release-process.md doc/safegcd_implementation.md +EXTRA_DIST += examples/EXAMPLES_COPYING +EXTRA_DIST += sage/gen_exhaustive_groups.sage +EXTRA_DIST += sage/gen_split_lambda_constants.sage +EXTRA_DIST += sage/group_prover.sage +EXTRA_DIST += sage/prove_group_implementations.sage +EXTRA_DIST += sage/secp256k1_params.sage +EXTRA_DIST += sage/weierstrass_prover.sage if ENABLE_MODULE_ECDH include src/modules/ecdh/Makefile.am.include diff --git a/src/secp256k1/README.md b/src/secp256k1/README.md index f5db915e83..ffdc9aeaee 100644 --- a/src/secp256k1/README.md +++ b/src/secp256k1/README.md @@ -2,6 +2,8 @@ libsecp256k1 ============ [](https://cirrus-ci.com/github/bitcoin-core/secp256k1) + +[](https://web.libera.chat/#secp256k1) Optimized C library for ECDSA signatures and secret/public key operations on curve secp256k1. @@ -15,6 +17,7 @@ Features: * Derandomized ECDSA (via RFC6979 or with a caller provided function.) * Very efficient implementation. * Suitable for embedded systems. +* No runtime dependencies. * Optional module for public key recovery. * Optional module for ECDH key exchange. * Optional module for Schnorr signatures according to [BIP-340](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki). @@ -72,11 +75,12 @@ To compile optional modules (such as Schnorr signatures), you need to run `./con Usage examples ----------- - Usage examples can be found in the [examples](examples) directory. To compile them you need to configure with `--enable-examples`. +Usage examples can be found in the [examples](examples) directory. To compile them you need to configure with `--enable-examples`. * [ECDSA example](examples/ecdsa.c) * [Schnorr signatures example](examples/schnorr.c) * [Deriving a shared secret (ECDH) example](examples/ecdh.c) - To compile the Schnorr signature and ECDH examples, you also need to configure with `--enable-module-schnorrsig` and `--enable-module-ecdh`. + +To compile the Schnorr signature and ECDH examples, you also need to configure with `--enable-module-schnorrsig` and `--enable-module-ecdh`. Test coverage ----------- diff --git a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 index 9cb54de098..98be915b67 100644 --- a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 +++ b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 @@ -10,6 +10,7 @@ AC_MSG_RESULT([$has_64bit_asm]) ]) AC_DEFUN([SECP_VALGRIND_CHECK],[ +AC_MSG_CHECKING([for valgrind support]) if test x"$has_valgrind" != x"yes"; then CPPFLAGS_TEMP="$CPPFLAGS" CPPFLAGS="$VALGRIND_CPPFLAGS $CPPFLAGS" @@ -21,6 +22,7 @@ if test x"$has_valgrind" != x"yes"; then #endif ]])], [has_valgrind=yes; AC_DEFINE(HAVE_VALGRIND,1,[Define this symbol if valgrind is installed, and it supports the host platform])]) fi +AC_MSG_RESULT($has_valgrind) ]) dnl SECP_TRY_APPEND_CFLAGS(flags, VAR) diff --git a/src/secp256k1/ci/cirrus.sh b/src/secp256k1/ci/cirrus.sh index b85f012d3f..fb5854a777 100755 --- a/src/secp256k1/ci/cirrus.sh +++ b/src/secp256k1/ci/cirrus.sh @@ -5,10 +5,47 @@ set -x export LC_ALL=C +# Print relevant CI environment to allow reproducing the job outside of CI. +print_environment() { + # Turn off -x because it messes up the output + set +x + # There are many ways to print variable names and their content. This one + # does not rely on bash. + for i in WERROR_CFLAGS MAKEFLAGS BUILD \ + ECMULTWINDOW ECMULTGENPRECISION ASM WIDEMUL WITH_VALGRIND EXTRAFLAGS \ + EXPERIMENTAL ECDH RECOVERY SCHNORRSIG \ + SECP256K1_TEST_ITERS BENCH SECP256K1_BENCH_ITERS CTIMETEST\ + EXAMPLES \ + WRAPPER_CMD CC AR NM HOST + do + eval 'printf "%s %s " "$i=\"${'"$i"'}\""' + done + echo "$0" + set -x +} +print_environment + +# Start persistent wineserver if necessary. +# This speeds up jobs with many invocations of wine (e.g., ./configure with MSVC) tremendously. +case "$WRAPPER_CMD" in + *wine*) + # This is apparently only reliable when we run a dummy command such as "hh.exe" afterwards. + wineserver -p && wine hh.exe + ;; +esac + env >> test_env.log -$CC -v || true -valgrind --version || true +if [ -n "$CC" ]; then + # The MSVC compiler "cl" doesn't understand "-v" + $CC -v || true +fi +if [ "$WITH_VALGRIND" = "yes" ]; then + valgrind --version +fi +if [ -n "$WRAPPER_CMD" ]; then + $WRAPPER_CMD --version +fi ./autogen.sh @@ -63,6 +100,9 @@ then make precomp fi +# Shutdown wineserver again +wineserver -k || true + # Check that no repo files have been modified by the build. # (This fails for example if the precomp files need to be updated in the repo.) git diff --exit-code diff --git a/src/secp256k1/ci/linux-debian.Dockerfile b/src/secp256k1/ci/linux-debian.Dockerfile index 5cccbb5565..a83a4e36db 100644 --- a/src/secp256k1/ci/linux-debian.Dockerfile +++ b/src/secp256k1/ci/linux-debian.Dockerfile @@ -1,15 +1,14 @@ FROM debian:stable -RUN dpkg --add-architecture i386 -RUN dpkg --add-architecture s390x -RUN dpkg --add-architecture armhf -RUN dpkg --add-architecture arm64 -RUN dpkg --add-architecture ppc64el -RUN apt-get update +RUN dpkg --add-architecture i386 && \ + dpkg --add-architecture s390x && \ + dpkg --add-architecture armhf && \ + dpkg --add-architecture arm64 && \ + dpkg --add-architecture ppc64el # dkpg-dev: to make pkg-config work in cross-builds # llvm: for llvm-symbolizer, which is used by clang's UBSan for symbolized stack traces -RUN apt-get install --no-install-recommends --no-upgrade -y \ +RUN apt-get update && apt-get install --no-install-recommends -y \ git ca-certificates \ make automake libtool pkg-config dpkg-dev valgrind qemu-user \ gcc clang llvm libc6-dbg \ @@ -19,8 +18,20 @@ RUN apt-get install --no-install-recommends --no-upgrade -y \ gcc-arm-linux-gnueabihf libc6-dev-armhf-cross libc6-dbg:armhf \ gcc-aarch64-linux-gnu libc6-dev-arm64-cross libc6-dbg:arm64 \ gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross libc6-dbg:ppc64el \ - wine gcc-mingw-w64-x86-64 \ + gcc-mingw-w64-x86-64-win32 wine64 wine \ + gcc-mingw-w64-i686-win32 wine32 \ sagemath -# Run a dummy command in wine to make it set up configuration -RUN wine64-stable xcopy || true +WORKDIR /root +# The "wine" package provides a convience wrapper that we need +RUN apt-get update && apt-get install --no-install-recommends -y \ + git ca-certificates wine64 wine python3-simplejson python3-six msitools winbind procps && \ + git clone https://github.com/mstorsjo/msvc-wine && \ + mkdir /opt/msvc && \ + python3 msvc-wine/vsdownload.py --accept-license --dest /opt/msvc Microsoft.VisualStudio.Workload.VCTools && \ + msvc-wine/install.sh /opt/msvc + +# Initialize the wine environment. Wait until the wineserver process has +# exited before closing the session, to avoid corrupting the wine prefix. +RUN wine64 wineboot --init && \ + while (ps -A | grep wineserver) > /dev/null; do sleep 1; done diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index 2db59a8ff3..68f279b17b 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -4,20 +4,20 @@ AC_PREREQ([2.60]) # the API. All changes in experimental modules are treated as # backwards-compatible and therefore at most increase the minor version. define(_PKG_VERSION_MAJOR, 0) -define(_PKG_VERSION_MINOR, 1) -define(_PKG_VERSION_BUILD, 0) -define(_PKG_VERSION_IS_RELEASE, false) +define(_PKG_VERSION_MINOR, 2) +define(_PKG_VERSION_PATCH, 0) +define(_PKG_VERSION_IS_RELEASE, true) # The library version is based on libtool versioning of the ABI. The set of # rules for updating the version can be found here: # https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html # All changes in experimental modules are treated as if they don't affect the # interface and therefore only increase the revision. -define(_LIB_VERSION_CURRENT, 0) +define(_LIB_VERSION_CURRENT, 1) define(_LIB_VERSION_REVISION, 0) define(_LIB_VERSION_AGE, 0) -AC_INIT([libsecp256k1],m4_join([.], _PKG_VERSION_MAJOR, _PKG_VERSION_MINOR, _PKG_VERSION_BUILD)m4_if(_PKG_VERSION_IS_RELEASE, [true], [], [-pre]),[https://github.com/bitcoin-core/secp256k1/issues],[libsecp256k1],[https://github.com/bitcoin-core/secp256k1]) +AC_INIT([libsecp256k1],m4_join([.], _PKG_VERSION_MAJOR, _PKG_VERSION_MINOR, _PKG_VERSION_PATCH)m4_if(_PKG_VERSION_IS_RELEASE, [true], [], [-dev]),[https://github.com/bitcoin-core/secp256k1/issues],[libsecp256k1],[https://github.com/bitcoin-core/secp256k1]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) @@ -33,12 +33,14 @@ AM_INIT_AUTOMAKE([1.11.2 foreign subdir-objects]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AC_PROG_CC -if test x"$ac_cv_prog_cc_c89" = x"no"; then - AC_MSG_ERROR([c89 compiler support required]) -fi AM_PROG_AS AM_PROG_AR +# Clear some cache variables as a workaround for a bug that appears due to a bad +# interaction between AM_PROG_AR and LT_INIT when combining MSVC's archiver lib.exe. +# https://debbugs.gnu.org/cgi/bugreport.cgi?bug=54421 +AS_UNSET(ac_cv_prog_AR) +AS_UNSET(ac_cv_prog_ac_ct_AR) LT_INIT([win32-dll]) build_windows=no @@ -87,23 +89,35 @@ esac # # TODO We should analogously not touch CPPFLAGS and LDFLAGS but currently there are no issues. AC_DEFUN([SECP_TRY_APPEND_DEFAULT_CFLAGS], [ - # Try to append -Werror=unknown-warning-option to CFLAGS temporarily. Otherwise clang will - # not error out if it gets unknown warning flags and the checks here will always succeed - # no matter if clang knows the flag or not. - SECP_TRY_APPEND_DEFAULT_CFLAGS_saved_CFLAGS="$CFLAGS" - SECP_TRY_APPEND_CFLAGS([-Werror=unknown-warning-option], CFLAGS) - - SECP_TRY_APPEND_CFLAGS([-std=c89 -pedantic -Wno-long-long -Wnested-externs -Wshadow -Wstrict-prototypes -Wundef], $1) # GCC >= 3.0, -Wlong-long is implied by -pedantic. - SECP_TRY_APPEND_CFLAGS([-Wno-overlength-strings], $1) # GCC >= 4.2, -Woverlength-strings is implied by -pedantic. - SECP_TRY_APPEND_CFLAGS([-Wall], $1) # GCC >= 2.95 and probably many other compilers - SECP_TRY_APPEND_CFLAGS([-Wno-unused-function], $1) # GCC >= 3.0, -Wunused-function is implied by -Wall. - SECP_TRY_APPEND_CFLAGS([-Wextra], $1) # GCC >= 3.4, this is the newer name of -W, which we don't use because older GCCs will warn about unused functions. - SECP_TRY_APPEND_CFLAGS([-Wcast-align], $1) # GCC >= 2.95 - SECP_TRY_APPEND_CFLAGS([-Wcast-align=strict], $1) # GCC >= 8.0 - SECP_TRY_APPEND_CFLAGS([-Wconditional-uninitialized], $1) # Clang >= 3.0 only - SECP_TRY_APPEND_CFLAGS([-fvisibility=hidden], $1) # GCC >= 4.0 - - CFLAGS="$SECP_TRY_APPEND_DEFAULT_CFLAGS_saved_CFLAGS" + # GCC and compatible (incl. clang) + if test "x$GCC" = "xyes"; then + # Try to append -Werror=unknown-warning-option to CFLAGS temporarily. Otherwise clang will + # not error out if it gets unknown warning flags and the checks here will always succeed + # no matter if clang knows the flag or not. + SECP_TRY_APPEND_DEFAULT_CFLAGS_saved_CFLAGS="$CFLAGS" + SECP_TRY_APPEND_CFLAGS([-Werror=unknown-warning-option], CFLAGS) + + SECP_TRY_APPEND_CFLAGS([-std=c89 -pedantic -Wno-long-long -Wnested-externs -Wshadow -Wstrict-prototypes -Wundef], $1) # GCC >= 3.0, -Wlong-long is implied by -pedantic. + SECP_TRY_APPEND_CFLAGS([-Wno-overlength-strings], $1) # GCC >= 4.2, -Woverlength-strings is implied by -pedantic. + SECP_TRY_APPEND_CFLAGS([-Wall], $1) # GCC >= 2.95 and probably many other compilers + SECP_TRY_APPEND_CFLAGS([-Wno-unused-function], $1) # GCC >= 3.0, -Wunused-function is implied by -Wall. + SECP_TRY_APPEND_CFLAGS([-Wextra], $1) # GCC >= 3.4, this is the newer name of -W, which we don't use because older GCCs will warn about unused functions. + SECP_TRY_APPEND_CFLAGS([-Wcast-align], $1) # GCC >= 2.95 + SECP_TRY_APPEND_CFLAGS([-Wcast-align=strict], $1) # GCC >= 8.0 + SECP_TRY_APPEND_CFLAGS([-Wconditional-uninitialized], $1) # Clang >= 3.0 only + SECP_TRY_APPEND_CFLAGS([-fvisibility=hidden], $1) # GCC >= 4.0 + + CFLAGS="$SECP_TRY_APPEND_DEFAULT_CFLAGS_saved_CFLAGS" + fi + + # MSVC + # Assume MSVC if we're building for Windows but not with GCC or compatible; + # libtool makes the same assumption internally. + # Note that "/opt" and "-opt" are equivalent for MSVC; we use "-opt" because "/opt" looks like a path. + if test x"$GCC" != x"yes" && test x"$build_windows" = x"yes"; then + SECP_TRY_APPEND_CFLAGS([-W2 -wd4146], $1) # Moderate warning level, disable warning C4146 "unary minus operator applied to unsigned type, result still unsigned" + SECP_TRY_APPEND_CFLAGS([-external:anglebrackets -external:W0], $1) # Suppress warnings from #include <...> files + fi ]) SECP_TRY_APPEND_DEFAULT_CFLAGS(SECP_CFLAGS) @@ -141,27 +155,31 @@ AC_ARG_ENABLE(examples, [SECP_SET_DEFAULT([enable_examples], [no], [yes])]) AC_ARG_ENABLE(module_ecdh, - AS_HELP_STRING([--enable-module-ecdh],[enable ECDH module [default=no]]), [], - [SECP_SET_DEFAULT([enable_module_ecdh], [no], [yes])]) + AS_HELP_STRING([--enable-module-ecdh],[enable ECDH module [default=yes]]), [], + [SECP_SET_DEFAULT([enable_module_ecdh], [yes], [yes])]) AC_ARG_ENABLE(module_recovery, AS_HELP_STRING([--enable-module-recovery],[enable ECDSA pubkey recovery module [default=no]]), [], [SECP_SET_DEFAULT([enable_module_recovery], [no], [yes])]) AC_ARG_ENABLE(module_extrakeys, - AS_HELP_STRING([--enable-module-extrakeys],[enable extrakeys module [default=no]]), [], - [SECP_SET_DEFAULT([enable_module_extrakeys], [no], [yes])]) + AS_HELP_STRING([--enable-module-extrakeys],[enable extrakeys module [default=yes]]), [], + [SECP_SET_DEFAULT([enable_module_extrakeys], [yes], [yes])]) AC_ARG_ENABLE(module_schnorrsig, - AS_HELP_STRING([--enable-module-schnorrsig],[enable schnorrsig module [default=no]]), [], - [SECP_SET_DEFAULT([enable_module_schnorrsig], [no], [yes])]) + AS_HELP_STRING([--enable-module-schnorrsig],[enable schnorrsig module [default=yes]]), [], + [SECP_SET_DEFAULT([enable_module_schnorrsig], [yes], [yes])]) AC_ARG_ENABLE(external_default_callbacks, AS_HELP_STRING([--enable-external-default-callbacks],[enable external default callback functions [default=no]]), [], [SECP_SET_DEFAULT([enable_external_default_callbacks], [no], [no])]) # Test-only override of the (autodetected by the C code) "widemul" setting. -# Legal values are int64 (for [u]int64_t), int128 (for [unsigned] __int128), and auto (the default). +# Legal values are: +# * int64 (for [u]int64_t), +# * int128 (for [unsigned] __int128), +# * int128_struct (for int128 implemented as a structure), +# * and auto (the default). AC_ARG_WITH([test-override-wide-multiply], [] ,[set_widemul=$withval], [set_widemul=auto]) AC_ARG_WITH([asm], [AS_HELP_STRING([--with-asm=x86_64|arm|no|auto], @@ -271,6 +289,9 @@ fi # Select wide multiplication implementation case $set_widemul in +int128_struct) + AC_DEFINE(USE_FORCE_WIDEMUL_INT128_STRUCT, 1, [Define this symbol to force the use of the structure for simulating (unsigned) int128 based wide multiplication]) + ;; int128) AC_DEFINE(USE_FORCE_WIDEMUL_INT128, 1, [Define this symbol to force the use of the (unsigned) __int128 based wide multiplication implementation]) ;; @@ -326,7 +347,9 @@ if test x"$enable_valgrind" = x"yes"; then SECP_INCLUDES="$SECP_INCLUDES $VALGRIND_CPPFLAGS" fi -# Add -Werror and similar flags passed from the outside (for testing, e.g., in CI) +# Add -Werror and similar flags passed from the outside (for testing, e.g., in CI). +# We don't want to set the user variable CFLAGS in CI because this would disable +# autoconf's logic for setting default CFLAGS, which we would like to test in CI. SECP_CFLAGS="$SECP_CFLAGS $WERROR_CFLAGS" ### diff --git a/src/secp256k1/contrib/lax_der_privatekey_parsing.h b/src/secp256k1/contrib/lax_der_privatekey_parsing.h index 1a8ad8ae0c..3749e418fe 100644 --- a/src/secp256k1/contrib/lax_der_privatekey_parsing.h +++ b/src/secp256k1/contrib/lax_der_privatekey_parsing.h @@ -43,8 +43,7 @@ extern "C" { /** Export a private key in DER format. * * Returns: 1 if the private key was valid. - * Args: ctx: pointer to a context object, initialized for signing (cannot - * be NULL) + * Args: ctx: pointer to a context object (not secp256k1_context_static). * Out: privkey: pointer to an array for storing the private key in BER. * Should have space for 279 bytes, and cannot be NULL. * privkeylen: Pointer to an int where the length of the private key in diff --git a/src/secp256k1/doc/CHANGELOG.md b/src/secp256k1/doc/CHANGELOG.md deleted file mode 100644 index 3c4c2e4583..0000000000 --- a/src/secp256k1/doc/CHANGELOG.md +++ /dev/null @@ -1,12 +0,0 @@ -# Changelog - -This file is currently only a template for future use. - -Each change falls into one of the following categories: Added, Changed, Deprecated, Removed, Fixed or Security. - -## [Unreleased] - -## [MAJOR.MINOR.PATCH] - YYYY-MM-DD - -### Added/Changed/Deprecated/Removed/Fixed/Security -- [Title with link to Pull Request](https://link-to-pr) diff --git a/src/secp256k1/doc/release-process.md b/src/secp256k1/doc/release-process.md index a35b8a9db3..91e3616915 100644 --- a/src/secp256k1/doc/release-process.md +++ b/src/secp256k1/doc/release-process.md @@ -1,14 +1,52 @@ # Release Process -1. Open PR to master that - 1. adds release notes to `doc/CHANGELOG.md` and - 2. if this is **not** a patch release, updates `_PKG_VERSION_{MAJOR,MINOR}` and `_LIB_VERSIONS_*` in `configure.ac` -2. After the PR is merged, - * if this is **not** a patch release, create a release branch with name `MAJOR.MINOR`. - Make sure that the branch contains the right commits. - Create commit on the release branch that sets `_PKG_VERSION_IS_RELEASE` in `configure.ac` to `true`. - * if this **is** a patch release, open a pull request with the bugfixes to the `MAJOR.MINOR` branch. - Also include the release note commit bump `_PKG_VERSION_BUILD` and `_LIB_VERSIONS_*` in `configure.ac`. -4. Tag the commit with `git tag -s vMAJOR.MINOR.PATCH`. -5. Push branch and tag with `git push origin --tags`. -6. Create a new GitHub release with a link to the corresponding entry in `doc/CHANGELOG.md`. +This document outlines the process for releasing versions of the form `$MAJOR.$MINOR.$PATCH`. + +We distinguish between two types of releases: *regular* and *maintenance* releases. +Regular releases are releases of a new major or minor version as well as patches of the most recent release. +Maintenance releases, on the other hand, are required for patches of older releases. + +You should coordinate with the other maintainers on the release date, if possible. +This date will be part of the release entry in [CHANGELOG.md](../CHANGELOG.md) and it should match the dates of the remaining steps in the release process (including the date of the tag and the GitHub release). +It is best if the maintainers are present during the release, so they can help ensure that the process is followed correctly and, in the case of a regular release, they are aware that they should not modify the master branch between merging the PR in step 1 and the PR in step 3. + +This process also assumes that there will be no minor releases for old major releases. + +## Regular release + +1. Open a PR to the master branch with a commit (using message `"release: prepare for $MAJOR.$MINOR.$PATCH"`, for example) that + * finalizes the release notes in [CHANGELOG.md](../CHANGELOG.md) (make sure to include an entry for `### ABI Compatibility`) and + * updates `_PKG_VERSION_*`, `_LIB_VERSION_*`, and sets `_PKG_VERSION_IS_RELEASE` to `true` in `configure.ac`. +2. After the PR is merged, tag the commit and push it: + ``` + RELEASE_COMMIT=<merge commit of step 1> + git tag -s v$MAJOR.$MINOR.$PATCH -m "libsecp256k1 $MAJOR.$MINOR.$PATCH" $RELEASE_COMMIT + git push git@github.com:bitcoin-core/secp256k1.git v$MAJOR.$MINOR.$PATCH + ``` +3. Open a PR to the master branch with a commit (using message `"release: bump version after $MAJOR.$MINOR.$PATCH"`, for example) that sets `_PKG_VERSION_IS_RELEASE` to `false` and `_PKG_VERSION_PATCH` to `$PATCH + 1` and increases `_LIB_VERSION_REVISION`. If other maintainers are not present to approve the PR, it can be merged without ACKs. +4. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md). + +## Maintenance release + +Note that bugfixes only need to be backported to releases for which no compatible release without the bug exists. + +1. If `$PATCH = 1`, create maintenance branch `$MAJOR.$MINOR`: + ``` + git checkout -b $MAJOR.$MINOR v$MAJOR.$MINOR.0 + git push git@github.com:bitcoin-core/secp256k1.git $MAJOR.$MINOR + ``` +2. Open a pull request to the `$MAJOR.$MINOR` branch that + * includes the bugfixes, + * finalizes the release notes, + * bumps `_PKG_VERSION_PATCH` and `_LIB_VERSION_REVISION` in `configure.ac` (with commit message `"release: update PKG_ and LIB_VERSION for $MAJOR.$MINOR.$PATCH"`, for example). +3. After the PRs are merged, update the release branch and tag the commit: + ``` + git checkout $MAJOR.$MINOR && git pull + git tag -s v$MAJOR.$MINOR.$PATCH -m "libsecp256k1 $MAJOR.$MINOR.$PATCH" + ``` +4. Push tag: + ``` + git push git@github.com:bitcoin-core/secp256k1.git v$MAJOR.$MINOR.$PATCH + ``` +5. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md). +6. Open PR to the master branch that includes a commit (with commit message `"release notes: add $MAJOR.$MINOR.$PATCH"`, for example) that adds release notes to [CHANGELOG.md](../CHANGELOG.md). diff --git a/src/secp256k1/examples/ecdh.c b/src/secp256k1/examples/ecdh.c index d7e8add361..027d52fd5f 100644 --- a/src/secp256k1/examples/ecdh.c +++ b/src/secp256k1/examples/ecdh.c @@ -30,12 +30,8 @@ int main(void) { secp256k1_pubkey pubkey1; secp256k1_pubkey pubkey2; - /* The specification in secp256k1.h states that `secp256k1_ec_pubkey_create` - * needs a context object initialized for signing, which is why we create - * a context with the SECP256K1_CONTEXT_SIGN flag. - * (The docs for `secp256k1_ecdh` don't require any special context, just - * some initialized context) */ - secp256k1_context* ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + /* Before we can call actual API functions, we need to create a "context". */ + secp256k1_context* ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); if (!fill_random(randomize, sizeof(randomize))) { printf("Failed to generate randomness\n"); return 1; diff --git a/src/secp256k1/examples/ecdsa.c b/src/secp256k1/examples/ecdsa.c index 434c856ba0..7e4f1b13ac 100644 --- a/src/secp256k1/examples/ecdsa.c +++ b/src/secp256k1/examples/ecdsa.c @@ -38,12 +38,8 @@ int main(void) { int return_val; secp256k1_pubkey pubkey; secp256k1_ecdsa_signature sig; - /* The specification in secp256k1.h states that `secp256k1_ec_pubkey_create` needs - * a context object initialized for signing and `secp256k1_ecdsa_verify` needs - * a context initialized for verification, which is why we create a context - * for both signing and verification with the SECP256K1_CONTEXT_SIGN and - * SECP256K1_CONTEXT_VERIFY flags. */ - secp256k1_context* ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + /* Before we can call actual API functions, we need to create a "context". */ + secp256k1_context* ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); if (!fill_random(randomize, sizeof(randomize))) { printf("Failed to generate randomness\n"); return 1; diff --git a/src/secp256k1/examples/schnorr.c b/src/secp256k1/examples/schnorr.c index 82eb07d5d7..207c45c422 100644 --- a/src/secp256k1/examples/schnorr.c +++ b/src/secp256k1/examples/schnorr.c @@ -30,12 +30,8 @@ int main(void) { int return_val; secp256k1_xonly_pubkey pubkey; secp256k1_keypair keypair; - /* The specification in secp256k1_extrakeys.h states that `secp256k1_keypair_create` - * needs a context object initialized for signing. And in secp256k1_schnorrsig.h - * they state that `secp256k1_schnorrsig_verify` needs a context initialized for - * verification, which is why we create a context for both signing and verification - * with the SECP256K1_CONTEXT_SIGN and SECP256K1_CONTEXT_VERIFY flags. */ - secp256k1_context* ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + /* Before we can call actual API functions, we need to create a "context". */ + secp256k1_context* ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); if (!fill_random(randomize, sizeof(randomize))) { printf("Failed to generate randomness\n"); return 1; diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index dddab346ae..826ab75850 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -7,7 +7,7 @@ extern "C" { #include <stddef.h> -/* Unless explicitly stated all pointer arguments must not be NULL. +/** Unless explicitly stated all pointer arguments must not be NULL. * * The following rules specify the order of arguments in API calls: * @@ -24,15 +24,19 @@ extern "C" { * 5. Opaque data pointers follow the function pointer they are to be passed to. */ -/** Opaque data structure that holds context information (precomputed tables etc.). +/** Opaque data structure that holds context information * - * The purpose of context structures is to cache large precomputed data tables - * that are expensive to construct, and also to maintain the randomization data - * for blinding. + * The primary purpose of context objects is to store randomization data for + * enhanced protection against side-channel leakage. This protection is only + * effective if the context is randomized after its creation. See + * secp256k1_context_create for creation of contexts and + * secp256k1_context_randomize for randomization. * - * Do not create a new context object for each operation, as construction is - * far slower than all other API calls (~100 times slower than an ECDSA - * verification). + * A secondary purpose of context objects is to store pointers to callback + * functions that the library will call when certain error states arise. See + * secp256k1_context_set_error_callback as well as + * secp256k1_context_set_illegal_callback for details. Future library versions + * may use context objects for additional purposes. * * A constructed context can safely be used from multiple threads * simultaneously, but API calls that take a non-const pointer to a context @@ -45,7 +49,7 @@ extern "C" { */ typedef struct secp256k1_context_struct secp256k1_context; -/** Opaque data structure that holds rewriteable "scratch space" +/** Opaque data structure that holds rewritable "scratch space" * * The purpose of this structure is to replace dynamic memory allocations, * because we target architectures where this may not be available. It is @@ -130,7 +134,7 @@ typedef int (*secp256k1_nonce_function)( # define SECP256K1_INLINE inline # endif -/** When this header is used at build-time the SECP256K1_BUILD define needs to be set +/* When this header is used at build-time the SECP256K1_BUILD define needs to be set * to correctly setup export attributes and nullness checks. This is normally done * by secp256k1.c but to guard against this header being included before secp256k1.c * has had a chance to set the define (e.g. via test harnesses that just includes @@ -159,9 +163,9 @@ typedef int (*secp256k1_nonce_function)( # endif #endif -/**Warning attributes - * NONNULL is not used if SECP256K1_BUILD is set to avoid the compiler optimizing out - * some paranoid null checks. */ +/* Warning attributes + * NONNULL is not used if SECP256K1_BUILD is set to avoid the compiler optimizing out + * some paranoid null checks. */ # if defined(__GNUC__) && SECP256K1_GNUC_PREREQ(3, 4) # define SECP256K1_WARN_UNUSED_RESULT __attribute__ ((__warn_unused_result__)) # else @@ -173,7 +177,7 @@ typedef int (*secp256k1_nonce_function)( # define SECP256K1_ARG_NONNULL(_x) # endif -/** Attribute for marking functions, types, and variables as deprecated */ +/* Attribute for marking functions, types, and variables as deprecated */ #if !defined(SECP256K1_BUILD) && defined(__has_attribute) # if __has_attribute(__deprecated__) # define SECP256K1_DEPRECATED(_msg) __attribute__ ((__deprecated__(_msg))) @@ -184,22 +188,26 @@ typedef int (*secp256k1_nonce_function)( # define SECP256K1_DEPRECATED(_msg) #endif -/** All flags' lower 8 bits indicate what they're for. Do not use directly. */ +/* All flags' lower 8 bits indicate what they're for. Do not use directly. */ #define SECP256K1_FLAGS_TYPE_MASK ((1 << 8) - 1) #define SECP256K1_FLAGS_TYPE_CONTEXT (1 << 0) #define SECP256K1_FLAGS_TYPE_COMPRESSION (1 << 1) -/** The higher bits contain the actual data. Do not use directly. */ +/* The higher bits contain the actual data. Do not use directly. */ #define SECP256K1_FLAGS_BIT_CONTEXT_VERIFY (1 << 8) #define SECP256K1_FLAGS_BIT_CONTEXT_SIGN (1 << 9) #define SECP256K1_FLAGS_BIT_CONTEXT_DECLASSIFY (1 << 10) #define SECP256K1_FLAGS_BIT_COMPRESSION (1 << 8) -/** Flags to pass to secp256k1_context_create, secp256k1_context_preallocated_size, and +/** Context flags to pass to secp256k1_context_create, secp256k1_context_preallocated_size, and * secp256k1_context_preallocated_create. */ +#define SECP256K1_CONTEXT_NONE (SECP256K1_FLAGS_TYPE_CONTEXT) + +/** Deprecated context flags. These flags are treated equivalent to SECP256K1_CONTEXT_NONE. */ #define SECP256K1_CONTEXT_VERIFY (SECP256K1_FLAGS_TYPE_CONTEXT | SECP256K1_FLAGS_BIT_CONTEXT_VERIFY) #define SECP256K1_CONTEXT_SIGN (SECP256K1_FLAGS_TYPE_CONTEXT | SECP256K1_FLAGS_BIT_CONTEXT_SIGN) + +/* Testing flag. Do not use. */ #define SECP256K1_CONTEXT_DECLASSIFY (SECP256K1_FLAGS_TYPE_CONTEXT | SECP256K1_FLAGS_BIT_CONTEXT_DECLASSIFY) -#define SECP256K1_CONTEXT_NONE (SECP256K1_FLAGS_TYPE_CONTEXT) /** Flag to pass to secp256k1_ec_pubkey_serialize. */ #define SECP256K1_EC_COMPRESSED (SECP256K1_FLAGS_TYPE_COMPRESSION | SECP256K1_FLAGS_BIT_COMPRESSION) @@ -212,23 +220,66 @@ typedef int (*secp256k1_nonce_function)( #define SECP256K1_TAG_PUBKEY_HYBRID_EVEN 0x06 #define SECP256K1_TAG_PUBKEY_HYBRID_ODD 0x07 -/** A simple secp256k1 context object with no precomputed tables. These are useful for - * type serialization/parsing functions which require a context object to maintain - * API consistency, but currently do not require expensive precomputations or dynamic - * allocations. +/** A built-in constant secp256k1 context object with static storage duration, to be + * used in conjunction with secp256k1_selftest. + * + * This context object offers *only limited functionality* , i.e., it cannot be used + * for API functions that perform computations involving secret keys, e.g., signing + * and public key generation. If this restriction applies to a specific API function, + * it is mentioned in its documentation. See secp256k1_context_create if you need a + * full context object that supports all functionality offered by the library. + * + * It is highly recommended to call secp256k1_selftest before using this context. + */ +SECP256K1_API extern const secp256k1_context *secp256k1_context_static; + +/** Deprecated alias for secp256k1_context_static. */ +SECP256K1_API extern const secp256k1_context *secp256k1_context_no_precomp +SECP256K1_DEPRECATED("Use secp256k1_context_static instead"); + +/** Perform basic self tests (to be used in conjunction with secp256k1_context_static) + * + * This function performs self tests that detect some serious usage errors and + * similar conditions, e.g., when the library is compiled for the wrong endianness. + * This is a last resort measure to be used in production. The performed tests are + * very rudimentary and are not intended as a replacement for running the test + * binaries. + * + * It is highly recommended to call this before using secp256k1_context_static. + * It is not necessary to call this function before using a context created with + * secp256k1_context_create (or secp256k1_context_preallocated_create), which will + * take care of performing the self tests. + * + * If the tests fail, this function will call the default error handler to abort the + * program (see secp256k1_context_set_error_callback). */ -SECP256K1_API extern const secp256k1_context *secp256k1_context_no_precomp; +SECP256K1_API void secp256k1_selftest(void); + /** Create a secp256k1 context object (in dynamically allocated memory). * * This function uses malloc to allocate memory. It is guaranteed that malloc is * called at most once for every call of this function. If you need to avoid dynamic - * memory allocation entirely, see the functions in secp256k1_preallocated.h. + * memory allocation entirely, see secp256k1_context_static and the functions in + * secp256k1_preallocated.h. * * Returns: a newly created context object. - * In: flags: which parts of the context to initialize. + * In: flags: Always set to SECP256K1_CONTEXT_NONE (see below). + * + * The only valid non-deprecated flag in recent library versions is + * SECP256K1_CONTEXT_NONE, which will create a context sufficient for all functionality + * offered by the library. All other (deprecated) flags will be treated as equivalent + * to the SECP256K1_CONTEXT_NONE flag. Though the flags parameter primarily exists for + * historical reasons, future versions of the library may introduce new flags. * - * See also secp256k1_context_randomize. + * If the context is intended to be used for API functions that perform computations + * involving secret keys, e.g., signing and public key generation, then it is highly + * recommended to call secp256k1_context_randomize on the context before calling + * those API functions. This will provide enhanced protection against side-channel + * leakage, see secp256k1_context_randomize for details. + * + * Do not create a new context object for each operation, as construction and + * randomization can take non-negligible time. */ SECP256K1_API secp256k1_context* secp256k1_context_create( unsigned int flags @@ -308,7 +359,10 @@ SECP256K1_API void secp256k1_context_set_illegal_callback( ) SECP256K1_ARG_NONNULL(1); /** Set a callback function to be called when an internal consistency check - * fails. The default is crashing. + * fails. + * + * The default callback writes an error message to stderr and calls abort + * to abort the program. * * This can only trigger in case of a hardware failure, miscompilation, * memory corruption, serious bug in the library, or other error would can @@ -426,8 +480,8 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_cmp( * encoding is invalid. R and S with value 0 are allowed in the encoding. * * After the call, sig will always be initialized. If parsing failed or R or - * S are zero, the resulting sig value is guaranteed to fail validation for any - * message and public key. + * S are zero, the resulting sig value is guaranteed to fail verification for + * any message and public key. */ SECP256K1_API int secp256k1_ecdsa_signature_parse_compact( const secp256k1_context* ctx, @@ -447,7 +501,7 @@ SECP256K1_API int secp256k1_ecdsa_signature_parse_compact( * encoded numbers are out of range. * * After the call, sig will always be initialized. If parsing failed or the - * encoded numbers are out of range, signature validation with it is + * encoded numbers are out of range, signature verification with it is * guaranteed to fail for every message and public key. */ SECP256K1_API int secp256k1_ecdsa_signature_parse_der( @@ -494,7 +548,7 @@ SECP256K1_API int secp256k1_ecdsa_signature_serialize_compact( * * Returns: 1: correct signature * 0: incorrect or unparseable signature - * Args: ctx: a secp256k1 context object, initialized for verification. + * Args: ctx: a secp256k1 context object. * In: sig: the signature being verified. * msghash32: the 32-byte message hash being verified. * The verifier must make sure to apply a cryptographic @@ -511,7 +565,7 @@ SECP256K1_API int secp256k1_ecdsa_signature_serialize_compact( * * If you need to accept ECDSA signatures from sources that do not obey this * rule, apply secp256k1_ecdsa_signature_normalize to the signature prior to - * validation, but be aware that doing so results in malleable signatures. + * verification, but be aware that doing so results in malleable signatures. * * For details, see the comments for that function. */ @@ -582,7 +636,7 @@ SECP256K1_API extern const secp256k1_nonce_function secp256k1_nonce_function_def * * Returns: 1: signature created * 0: the nonce generation function failed, or the secret key was invalid. - * Args: ctx: pointer to a context object, initialized for signing. + * Args: ctx: pointer to a context object (not secp256k1_context_static). * Out: sig: pointer to an array where the signature will be placed. * In: msghash32: the 32-byte message hash being signed. * seckey: pointer to a 32-byte secret key. @@ -626,7 +680,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_verify( * * Returns: 1: secret was valid, public key stores. * 0: secret was invalid, try again. - * Args: ctx: pointer to a context object, initialized for signing. + * Args: ctx: pointer to a context object (not secp256k1_context_static). * Out: pubkey: pointer to the created public key. * In: seckey: pointer to a 32-byte secret key. */ @@ -705,7 +759,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_add( * Returns: 0 if the arguments are invalid or the resulting public key would be * invalid (only when the tweak is the negation of the corresponding * secret key). 1 otherwise. - * Args: ctx: pointer to a context object initialized for validation. + * Args: ctx: pointer to a context object. * In/Out: pubkey: pointer to a public key object. pubkey will be set to an * invalid value if this function returns 0. * In: tweak32: pointer to a 32-byte tweak. If the tweak is invalid according to @@ -750,7 +804,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_mul( /** Tweak a public key by multiplying it by a tweak value. * * Returns: 0 if the arguments are invalid. 1 otherwise. - * Args: ctx: pointer to a context object initialized for validation. + * Args: ctx: pointer to a context object. * In/Out: pubkey: pointer to a public key object. pubkey will be set to an * invalid value if this function returns 0. * In: tweak32: pointer to a 32-byte tweak. If the tweak is invalid according to @@ -764,30 +818,41 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_mul( const unsigned char *tweak32 ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); -/** Updates the context randomization to protect against side-channel leakage. - * Returns: 1: randomization successfully updated or nothing to randomize +/** Randomizes the context to provide enhanced protection against side-channel leakage. + * + * Returns: 1: randomization successful (or called on copy of secp256k1_context_static) * 0: error * Args: ctx: pointer to a context object. * In: seed32: pointer to a 32-byte random seed (NULL resets to initial state) * - * While secp256k1 code is written to be constant-time no matter what secret - * values are, it's possible that a future compiler may output code which isn't, + * While secp256k1 code is written and tested to be constant-time no matter what + * secret values are, it is possible that a compiler may output code which is not, * and also that the CPU may not emit the same radio frequencies or draw the same - * amount power for all values. - * - * This function provides a seed which is combined into the blinding value: that - * blinding value is added before each multiplication (and removed afterwards) so - * that it does not affect function results, but shields against attacks which - * rely on any input-dependent behaviour. - * - * This function has currently an effect only on contexts initialized for signing - * because randomization is currently used only for signing. However, this is not - * guaranteed and may change in the future. It is safe to call this function on - * contexts not initialized for signing; then it will have no effect and return 1. - * - * You should call this after secp256k1_context_create or - * secp256k1_context_clone (and secp256k1_context_preallocated_create or - * secp256k1_context_clone, resp.), and you may call this repeatedly afterwards. + * amount of power for all values. Randomization of the context shields against + * side-channel observations which aim to exploit secret-dependent behaviour in + * certain computations which involve secret keys. + * + * It is highly recommended to call this function on contexts returned from + * secp256k1_context_create or secp256k1_context_clone (or from the corresponding + * functions in secp256k1_preallocated.h) before using these contexts to call API + * functions that perform computations involving secret keys, e.g., signing and + * public key generation. It is possible to call this function more than once on + * the same context, and doing so before every few computations involving secret + * keys is recommended as a defense-in-depth measure. + * + * Currently, the random seed is mainly used for blinding multiplications of a + * secret scalar with the elliptic curve base point. Multiplications of this + * kind are performed by exactly those API functions which are documented to + * require a context that is not the secp256k1_context_static. As a rule of thumb, + * these are all functions which take a secret key (or a keypair) as an input. + * A notable exception to that rule is the ECDH module, which relies on a different + * kind of elliptic curve point multiplication and thus does not benefit from + * enhanced protection against side-channel leakage currently. + * + * It is safe call this function on a copy of secp256k1_context_static in writable + * memory (e.g., obtained via secp256k1_context_clone). In that case, this + * function is guaranteed to return 1, but the call will have no effect because + * the static context (or a copy thereof) is not meant to be randomized. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_context_randomize( secp256k1_context* ctx, diff --git a/src/secp256k1/include/secp256k1_extrakeys.h b/src/secp256k1/include/secp256k1_extrakeys.h index 09cbeaaa80..3591bc0012 100644 --- a/src/secp256k1/include/secp256k1_extrakeys.h +++ b/src/secp256k1/include/secp256k1_extrakeys.h @@ -108,7 +108,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_from_pubke * invalid (only when the tweak is the negation of the corresponding * secret key). 1 otherwise. * - * Args: ctx: pointer to a context object initialized for verification. + * Args: ctx: pointer to a context object. * Out: output_pubkey: pointer to a public key to store the result. Will be set * to an invalid value if this function returns 0. * In: internal_pubkey: pointer to an x-only pubkey to apply the tweak to. @@ -137,7 +137,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_tweak_add( * * Returns: 0 if the arguments are invalid or the tweaked pubkey is not the * result of tweaking the internal_pubkey with tweak32. 1 otherwise. - * Args: ctx: pointer to a context object initialized for verification. + * Args: ctx: pointer to a context object. * In: tweaked_pubkey32: pointer to a serialized xonly_pubkey. * tweaked_pk_parity: the parity of the tweaked pubkey (whose serialization * is passed in as tweaked_pubkey32). This must match the @@ -159,7 +159,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_tweak_add_ * * Returns: 1: secret was valid, keypair is ready to use * 0: secret was invalid, try again with a different secret - * Args: ctx: pointer to a context object, initialized for signing. + * Args: ctx: pointer to a context object (not secp256k1_context_static). * Out: keypair: pointer to the created keypair. * In: seckey: pointer to a 32-byte secret key. */ @@ -228,7 +228,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_xonly_pub( * invalid (only when the tweak is the negation of the keypair's * secret key). 1 otherwise. * - * Args: ctx: pointer to a context object initialized for verification. + * Args: ctx: pointer to a context object. * In/Out: keypair: pointer to a keypair to apply the tweak to. Will be set to * an invalid value if this function returns 0. * In: tweak32: pointer to a 32-byte tweak. If the tweak is invalid according diff --git a/src/secp256k1/include/secp256k1_preallocated.h b/src/secp256k1/include/secp256k1_preallocated.h index d2d9014f02..ed846f75f9 100644 --- a/src/secp256k1/include/secp256k1_preallocated.h +++ b/src/secp256k1/include/secp256k1_preallocated.h @@ -58,6 +58,8 @@ SECP256K1_API size_t secp256k1_context_preallocated_size( * bytes, as detailed above. * flags: which parts of the context to initialize. * + * See secp256k1_context_create (in secp256k1.h) for further details. + * * See also secp256k1_context_randomize (in secp256k1.h) * and secp256k1_context_preallocated_destroy. */ diff --git a/src/secp256k1/include/secp256k1_recovery.h b/src/secp256k1/include/secp256k1_recovery.h index 0e2847db96..824c604025 100644 --- a/src/secp256k1/include/secp256k1_recovery.h +++ b/src/secp256k1/include/secp256k1_recovery.h @@ -72,7 +72,7 @@ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_serialize_compact( * * Returns: 1: signature created * 0: the nonce generation function failed, or the secret key was invalid. - * Args: ctx: pointer to a context object, initialized for signing. + * Args: ctx: pointer to a context object (not secp256k1_context_static). * Out: sig: pointer to an array where the signature will be placed. * In: msghash32: the 32-byte message hash being signed. * seckey: pointer to a 32-byte secret key. @@ -94,7 +94,7 @@ SECP256K1_API int secp256k1_ecdsa_sign_recoverable( * * Returns: 1: public key successfully recovered (which guarantees a correct signature). * 0: otherwise. - * Args: ctx: pointer to a context object, initialized for verification. + * Args: ctx: pointer to a context object. * Out: pubkey: pointer to the recovered public key. * In: sig: pointer to initialized signature that supports pubkey recovery. * msghash32: the 32-byte message hash assumed to be signed. diff --git a/src/secp256k1/include/secp256k1_schnorrsig.h b/src/secp256k1/include/secp256k1_schnorrsig.h index 5fedcb07b0..e579e1b1d8 100644 --- a/src/secp256k1/include/secp256k1_schnorrsig.h +++ b/src/secp256k1/include/secp256k1_schnorrsig.h @@ -106,7 +106,7 @@ typedef struct { * signatures from being valid in multiple contexts by accident. * * Returns 1 on success, 0 on failure. - * Args: ctx: pointer to a context object, initialized for signing. + * Args: ctx: pointer to a context object (not secp256k1_context_static). * Out: sig64: pointer to a 64-byte array to store the serialized signature. * In: msg32: the 32-byte message being signed. * keypair: pointer to an initialized keypair. @@ -161,7 +161,7 @@ SECP256K1_API int secp256k1_schnorrsig_sign_custom( * * Returns: 1: correct signature * 0: incorrect signature - * Args: ctx: a secp256k1 context object, initialized for verification. + * Args: ctx: a secp256k1 context object. * In: sig64: pointer to the 64-byte signature to verify. * msg: the message being verified. Can only be NULL if msglen is 0. * msglen: length of the message diff --git a/src/secp256k1/src/assumptions.h b/src/secp256k1/src/assumptions.h index 6dc527b288..8ed04209e9 100644 --- a/src/secp256k1/src/assumptions.h +++ b/src/secp256k1/src/assumptions.h @@ -10,6 +10,9 @@ #include <limits.h> #include "util.h" +#if defined(SECP256K1_INT128_NATIVE) +#include "int128_native.h" +#endif /* This library, like most software, relies on a number of compiler implementation defined (but not undefined) behaviours. Although the behaviours we require are essentially universal we test them specifically here to @@ -55,7 +58,7 @@ struct secp256k1_assumption_checker { /* To int64_t. */ ((int64_t)(uint64_t)0xB123C456D789E012ULL == (int64_t)-(int64_t)0x4EDC3BA928761FEEULL) && -#if defined(SECP256K1_WIDEMUL_INT128) +#if defined(SECP256K1_INT128_NATIVE) ((int64_t)(((uint128_t)0xA1234567B8901234ULL << 64) + 0xC5678901D2345678ULL) == (int64_t)-(int64_t)0x3A9876FE2DCBA988ULL) && (((int64_t)(int128_t)(((uint128_t)0xB1C2D3E4F5A6B7C8ULL << 64) + 0xD9E0F1A2B3C4D5E6ULL)) == (int64_t)(uint64_t)0xD9E0F1A2B3C4D5E6ULL) && (((int64_t)(int128_t)(((uint128_t)0xABCDEF0123456789ULL << 64) + 0x0123456789ABCDEFULL)) == (int64_t)(uint64_t)0x0123456789ABCDEFULL) && @@ -71,7 +74,7 @@ struct secp256k1_assumption_checker { ((((int16_t)0xE9AC) >> 4) == (int16_t)(uint16_t)0xFE9A) && ((((int32_t)0x937C918A) >> 9) == (int32_t)(uint32_t)0xFFC9BE48) && ((((int64_t)0xA8B72231DF9CF4B9ULL) >> 19) == (int64_t)(uint64_t)0xFFFFF516E4463BF3ULL) && -#if defined(SECP256K1_WIDEMUL_INT128) +#if defined(SECP256K1_INT128_NATIVE) ((((int128_t)(((uint128_t)0xCD833A65684A0DBCULL << 64) + 0xB349312F71EA7637ULL)) >> 39) == (int128_t)(((uint128_t)0xFFFFFFFFFF9B0674ULL << 64) + 0xCAD0941B79669262ULL)) && #endif 1) * 2 - 1]; diff --git a/src/secp256k1/src/basic-config.h b/src/secp256k1/src/basic-config.h deleted file mode 100644 index 6f7693cb8f..0000000000 --- a/src/secp256k1/src/basic-config.h +++ /dev/null @@ -1,17 +0,0 @@ -/*********************************************************************** - * Copyright (c) 2013, 2014 Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or https://www.opensource.org/licenses/mit-license.php.* - ***********************************************************************/ - -#ifndef SECP256K1_BASIC_CONFIG_H -#define SECP256K1_BASIC_CONFIG_H - -#ifdef USE_BASIC_CONFIG - -#define ECMULT_WINDOW_SIZE 15 -#define ECMULT_GEN_PREC_BITS 4 - -#endif /* USE_BASIC_CONFIG */ - -#endif /* SECP256K1_BASIC_CONFIG_H */ diff --git a/src/secp256k1/src/bench.c b/src/secp256k1/src/bench.c index d5937b763f..e68021aa28 100644 --- a/src/secp256k1/src/bench.c +++ b/src/secp256k1/src/bench.c @@ -164,7 +164,7 @@ int main(int argc, char** argv) { /* Check if the user tries to benchmark optional module without building it */ #ifndef ENABLE_MODULE_ECDH - if (have_flag(argc, argv, "ecdh")) { + if (have_flag(argc, argv, "ecdh")) { fprintf(stderr, "./bench: ECDH module not enabled.\n"); fprintf(stderr, "Use ./configure --enable-module-ecdh.\n\n"); return 1; @@ -172,7 +172,7 @@ int main(int argc, char** argv) { #endif #ifndef ENABLE_MODULE_RECOVERY - if (have_flag(argc, argv, "recover") || have_flag(argc, argv, "ecdsa_recover")) { + if (have_flag(argc, argv, "recover") || have_flag(argc, argv, "ecdsa_recover")) { fprintf(stderr, "./bench: Public key recovery module not enabled.\n"); fprintf(stderr, "Use ./configure --enable-module-recovery.\n\n"); return 1; @@ -180,15 +180,15 @@ int main(int argc, char** argv) { #endif #ifndef ENABLE_MODULE_SCHNORRSIG - if (have_flag(argc, argv, "schnorrsig") || have_flag(argc, argv, "schnorrsig_sign") || have_flag(argc, argv, "schnorrsig_verify")) { + if (have_flag(argc, argv, "schnorrsig") || have_flag(argc, argv, "schnorrsig_sign") || have_flag(argc, argv, "schnorrsig_verify")) { fprintf(stderr, "./bench: Schnorr signatures module not enabled.\n"); fprintf(stderr, "Use ./configure --enable-module-schnorrsig.\n\n"); return 1; } #endif - /* ECDSA verification benchmark */ - data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + /* ECDSA benchmark */ + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); for (i = 0; i < 32; i++) { data.msg[i] = 1 + i; @@ -206,11 +206,6 @@ int main(int argc, char** argv) { print_output_table_header_row(); if (d || have_flag(argc, argv, "ecdsa") || have_flag(argc, argv, "verify") || have_flag(argc, argv, "ecdsa_verify")) run_benchmark("ecdsa_verify", bench_verify, NULL, NULL, &data, 10, iters); - secp256k1_context_destroy(data.ctx); - - /* ECDSA signing benchmark */ - data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); - if (d || have_flag(argc, argv, "ecdsa") || have_flag(argc, argv, "sign") || have_flag(argc, argv, "ecdsa_sign")) run_benchmark("ecdsa_sign", bench_sign_run, bench_sign_setup, NULL, &data, 10, iters); secp256k1_context_destroy(data.ctx); diff --git a/src/secp256k1/src/bench.h b/src/secp256k1/src/bench.h index aa275fe919..611ba11f04 100644 --- a/src/secp256k1/src/bench.h +++ b/src/secp256k1/src/bench.h @@ -7,15 +7,31 @@ #ifndef SECP256K1_BENCH_H #define SECP256K1_BENCH_H +#include <stdlib.h> #include <stdint.h> #include <stdio.h> #include <string.h> -#include "sys/time.h" + +#if (defined(_MSC_VER) && _MSC_VER >= 1900) +# include <time.h> +#else +# include "sys/time.h" +#endif static int64_t gettime_i64(void) { +#if (defined(_MSC_VER) && _MSC_VER >= 1900) + /* C11 way to get wallclock time */ + struct timespec tv; + if (!timespec_get(&tv, TIME_UTC)) { + fputs("timespec_get failed!", stderr); + exit(1); + } + return (int64_t)tv.tv_nsec / 1000 + (int64_t)tv.tv_sec * 1000000LL; +#else struct timeval tv; gettimeofday(&tv, NULL); return (int64_t)tv.tv_usec + (int64_t)tv.tv_sec * 1000000LL; +#endif } #define FP_EXP (6) diff --git a/src/secp256k1/src/bench_ecmult.c b/src/secp256k1/src/bench_ecmult.c index 4030e0263f..9d0db340e1 100644 --- a/src/secp256k1/src/bench_ecmult.c +++ b/src/secp256k1/src/bench_ecmult.c @@ -84,9 +84,7 @@ static void bench_ecmult_teardown_helper(bench_data* data, size_t* seckey_offset } } secp256k1_ecmult_gen(&data->ctx->ecmult_gen_ctx, &tmp, &sum_scalars); - secp256k1_gej_neg(&tmp, &tmp); - secp256k1_gej_add_var(&tmp, &tmp, &sum_output, NULL); - CHECK(secp256k1_gej_is_infinity(&tmp)); + CHECK(secp256k1_gej_eq_var(&tmp, &sum_output)); } static void bench_ecmult_setup(void* arg) { @@ -308,7 +306,7 @@ int main(int argc, char **argv) { } } - data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); scratch_size = secp256k1_strauss_scratch_size(POINTS) + STRAUSS_SCRATCH_OBJECTS*16; if (!have_flag(argc, argv, "simple")) { data.scratch = secp256k1_scratch_space_create(data.ctx, scratch_size); diff --git a/src/secp256k1/src/bench_internal.c b/src/secp256k1/src/bench_internal.c index 7eb3af28d7..2224058f64 100644 --- a/src/secp256k1/src/bench_internal.c +++ b/src/secp256k1/src/bench_internal.c @@ -343,19 +343,11 @@ void bench_rfc6979_hmac_sha256(void* arg, int iters) { } } -void bench_context_verify(void* arg, int iters) { +void bench_context(void* arg, int iters) { int i; (void)arg; for (i = 0; i < iters; i++) { - secp256k1_context_destroy(secp256k1_context_create(SECP256K1_CONTEXT_VERIFY)); - } -} - -void bench_context_sign(void* arg, int iters) { - int i; - (void)arg; - for (i = 0; i < iters; i++) { - secp256k1_context_destroy(secp256k1_context_create(SECP256K1_CONTEXT_SIGN)); + secp256k1_context_destroy(secp256k1_context_create(SECP256K1_CONTEXT_NONE)); } } @@ -395,8 +387,7 @@ int main(int argc, char **argv) { if (d || have_flag(argc, argv, "hash") || have_flag(argc, argv, "hmac")) run_benchmark("hash_hmac_sha256", bench_hmac_sha256, bench_setup, NULL, &data, 10, iters); if (d || have_flag(argc, argv, "hash") || have_flag(argc, argv, "rng6979")) run_benchmark("hash_rfc6979_hmac_sha256", bench_rfc6979_hmac_sha256, bench_setup, NULL, &data, 10, iters); - if (d || have_flag(argc, argv, "context") || have_flag(argc, argv, "verify")) run_benchmark("context_verify", bench_context_verify, bench_setup, NULL, &data, 10, 1 + iters/1000); - if (d || have_flag(argc, argv, "context") || have_flag(argc, argv, "sign")) run_benchmark("context_sign", bench_context_sign, bench_setup, NULL, &data, 10, 1 + iters/100); + if (d || have_flag(argc, argv, "context")) run_benchmark("context_create", bench_context, bench_setup, NULL, &data, 10, iters); return 0; } diff --git a/src/secp256k1/src/ecmult.h b/src/secp256k1/src/ecmult.h index b47d8f494a..e28c602506 100644 --- a/src/secp256k1/src/ecmult.h +++ b/src/secp256k1/src/ecmult.h @@ -11,6 +11,17 @@ #include "scalar.h" #include "scratch.h" +#ifndef ECMULT_WINDOW_SIZE +# define ECMULT_WINDOW_SIZE 15 +# ifdef DEBUG_CONFIG +# pragma message DEBUG_CONFIG_MSG("ECMULT_WINDOW_SIZE undefined, assuming default value") +# endif +#endif + +#ifdef DEBUG_CONFIG +# pragma message DEBUG_CONFIG_DEF(ECMULT_WINDOW_SIZE) +#endif + /* Noone will ever need more than a window size of 24. The code might * be correct for larger values of ECMULT_WINDOW_SIZE but this is not * tested. diff --git a/src/secp256k1/src/ecmult_gen.h b/src/secp256k1/src/ecmult_gen.h index f48f266461..a430e8d5d9 100644 --- a/src/secp256k1/src/ecmult_gen.h +++ b/src/secp256k1/src/ecmult_gen.h @@ -10,9 +10,21 @@ #include "scalar.h" #include "group.h" +#ifndef ECMULT_GEN_PREC_BITS +# define ECMULT_GEN_PREC_BITS 4 +# ifdef DEBUG_CONFIG +# pragma message DEBUG_CONFIG_MSG("ECMULT_GEN_PREC_BITS undefined, assuming default value") +# endif +#endif + +#ifdef DEBUG_CONFIG +# pragma message DEBUG_CONFIG_DEF(ECMULT_GEN_PREC_BITS) +#endif + #if ECMULT_GEN_PREC_BITS != 2 && ECMULT_GEN_PREC_BITS != 4 && ECMULT_GEN_PREC_BITS != 8 # error "Set ECMULT_GEN_PREC_BITS to 2, 4 or 8." #endif + #define ECMULT_GEN_PREC_G(bits) (1 << bits) #define ECMULT_GEN_PREC_N(bits) (256 / bits) diff --git a/src/secp256k1/src/ecmult_gen_impl.h b/src/secp256k1/src/ecmult_gen_impl.h index 2c8a503acc..4f5ea9f3c0 100644 --- a/src/secp256k1/src/ecmult_gen_impl.h +++ b/src/secp256k1/src/ecmult_gen_impl.h @@ -88,31 +88,31 @@ static void secp256k1_ecmult_gen_blind(secp256k1_ecmult_gen_context *ctx, const unsigned char nonce32[32]; secp256k1_rfc6979_hmac_sha256 rng; int overflow; - unsigned char keydata[64] = {0}; + unsigned char keydata[64]; if (seed32 == NULL) { /* When seed is NULL, reset the initial point and blinding value. */ secp256k1_gej_set_ge(&ctx->initial, &secp256k1_ge_const_g); secp256k1_gej_neg(&ctx->initial, &ctx->initial); secp256k1_scalar_set_int(&ctx->blind, 1); + return; } /* The prior blinding value (if not reset) is chained forward by including it in the hash. */ - secp256k1_scalar_get_b32(nonce32, &ctx->blind); + secp256k1_scalar_get_b32(keydata, &ctx->blind); /** Using a CSPRNG allows a failure free interface, avoids needing large amounts of random data, * and guards against weak or adversarial seeds. This is a simpler and safer interface than * asking the caller for blinding values directly and expecting them to retry on failure. */ - memcpy(keydata, nonce32, 32); - if (seed32 != NULL) { - memcpy(keydata + 32, seed32, 32); - } - secp256k1_rfc6979_hmac_sha256_initialize(&rng, keydata, seed32 ? 64 : 32); + VERIFY_CHECK(seed32 != NULL); + memcpy(keydata + 32, seed32, 32); + secp256k1_rfc6979_hmac_sha256_initialize(&rng, keydata, 64); memset(keydata, 0, sizeof(keydata)); /* Accept unobservably small non-uniformity. */ secp256k1_rfc6979_hmac_sha256_generate(&rng, nonce32, 32); overflow = !secp256k1_fe_set_b32(&s, nonce32); overflow |= secp256k1_fe_is_zero(&s); secp256k1_fe_cmov(&s, &secp256k1_fe_one, overflow); - /* Randomize the projection to defend against multiplier sidechannels. */ + /* Randomize the projection to defend against multiplier sidechannels. + Do this before our own call to secp256k1_ecmult_gen below. */ secp256k1_gej_rescale(&ctx->initial, &s); secp256k1_fe_clear(&s); secp256k1_rfc6979_hmac_sha256_generate(&rng, nonce32, 32); @@ -121,6 +121,7 @@ static void secp256k1_ecmult_gen_blind(secp256k1_ecmult_gen_context *ctx, const secp256k1_scalar_cmov(&b, &secp256k1_scalar_one, secp256k1_scalar_is_zero(&b)); secp256k1_rfc6979_hmac_sha256_finalize(&rng); memset(nonce32, 0, 32); + /* The random projection in ctx->initial ensures that gb will have a random projection. */ secp256k1_ecmult_gen(ctx, &gb, &b); secp256k1_scalar_negate(&b, &b); ctx->blind = b; diff --git a/src/secp256k1/src/ecmult_impl.h b/src/secp256k1/src/ecmult_impl.h index bbc820c77c..3776fe73fc 100644 --- a/src/secp256k1/src/ecmult_impl.h +++ b/src/secp256k1/src/ecmult_impl.h @@ -200,9 +200,15 @@ static int secp256k1_ecmult_wnaf(int *wnaf, int len, const secp256k1_scalar *a, bit += now; } #ifdef VERIFY - CHECK(carry == 0); - while (bit < 256) { - CHECK(secp256k1_scalar_get_bits(&s, bit++, 1) == 0); + { + int verify_bit = bit; + + VERIFY_CHECK(carry == 0); + + while (verify_bit < 256) { + VERIFY_CHECK(secp256k1_scalar_get_bits(&s, verify_bit, 1) == 0); + verify_bit++; + } } #endif return last_set_bit + 1; diff --git a/src/secp256k1/src/field_5x52_int128_impl.h b/src/secp256k1/src/field_5x52_int128_impl.h index 0ed6118cc9..18567b95f3 100644 --- a/src/secp256k1/src/field_5x52_int128_impl.h +++ b/src/secp256k1/src/field_5x52_int128_impl.h @@ -9,14 +9,18 @@ #include <stdint.h> +#include "int128.h" + #ifdef VERIFY #define VERIFY_BITS(x, n) VERIFY_CHECK(((x) >> (n)) == 0) +#define VERIFY_BITS_128(x, n) VERIFY_CHECK(secp256k1_u128_check_bits((x), (n))) #else #define VERIFY_BITS(x, n) do { } while(0) +#define VERIFY_BITS_128(x, n) do { } while(0) #endif SECP256K1_INLINE static void secp256k1_fe_mul_inner(uint64_t *r, const uint64_t *a, const uint64_t * SECP256K1_RESTRICT b) { - uint128_t c, d; + secp256k1_uint128 c, d; uint64_t t3, t4, tx, u0; uint64_t a0 = a[0], a1 = a[1], a2 = a[2], a3 = a[3], a4 = a[4]; const uint64_t M = 0xFFFFFFFFFFFFFULL, R = 0x1000003D10ULL; @@ -40,121 +44,119 @@ SECP256K1_INLINE static void secp256k1_fe_mul_inner(uint64_t *r, const uint64_t * Note that [x 0 0 0 0 0] = [x*R]. */ - d = (uint128_t)a0 * b[3] - + (uint128_t)a1 * b[2] - + (uint128_t)a2 * b[1] - + (uint128_t)a3 * b[0]; - VERIFY_BITS(d, 114); + secp256k1_u128_mul(&d, a0, b[3]); + secp256k1_u128_accum_mul(&d, a1, b[2]); + secp256k1_u128_accum_mul(&d, a2, b[1]); + secp256k1_u128_accum_mul(&d, a3, b[0]); + VERIFY_BITS_128(&d, 114); /* [d 0 0 0] = [p3 0 0 0] */ - c = (uint128_t)a4 * b[4]; - VERIFY_BITS(c, 112); + secp256k1_u128_mul(&c, a4, b[4]); + VERIFY_BITS_128(&c, 112); /* [c 0 0 0 0 d 0 0 0] = [p8 0 0 0 0 p3 0 0 0] */ - d += (uint128_t)R * (uint64_t)c; c >>= 64; - VERIFY_BITS(d, 115); - VERIFY_BITS(c, 48); + secp256k1_u128_accum_mul(&d, R, secp256k1_u128_to_u64(&c)); secp256k1_u128_rshift(&c, 64); + VERIFY_BITS_128(&d, 115); + VERIFY_BITS_128(&c, 48); /* [(c<<12) 0 0 0 0 0 d 0 0 0] = [p8 0 0 0 0 p3 0 0 0] */ - t3 = d & M; d >>= 52; + t3 = secp256k1_u128_to_u64(&d) & M; secp256k1_u128_rshift(&d, 52); VERIFY_BITS(t3, 52); - VERIFY_BITS(d, 63); + VERIFY_BITS_128(&d, 63); /* [(c<<12) 0 0 0 0 d t3 0 0 0] = [p8 0 0 0 0 p3 0 0 0] */ - d += (uint128_t)a0 * b[4] - + (uint128_t)a1 * b[3] - + (uint128_t)a2 * b[2] - + (uint128_t)a3 * b[1] - + (uint128_t)a4 * b[0]; - VERIFY_BITS(d, 115); + secp256k1_u128_accum_mul(&d, a0, b[4]); + secp256k1_u128_accum_mul(&d, a1, b[3]); + secp256k1_u128_accum_mul(&d, a2, b[2]); + secp256k1_u128_accum_mul(&d, a3, b[1]); + secp256k1_u128_accum_mul(&d, a4, b[0]); + VERIFY_BITS_128(&d, 115); /* [(c<<12) 0 0 0 0 d t3 0 0 0] = [p8 0 0 0 p4 p3 0 0 0] */ - d += (uint128_t)(R << 12) * (uint64_t)c; - VERIFY_BITS(d, 116); + secp256k1_u128_accum_mul(&d, R << 12, secp256k1_u128_to_u64(&c)); + VERIFY_BITS_128(&d, 116); /* [d t3 0 0 0] = [p8 0 0 0 p4 p3 0 0 0] */ - t4 = d & M; d >>= 52; + t4 = secp256k1_u128_to_u64(&d) & M; secp256k1_u128_rshift(&d, 52); VERIFY_BITS(t4, 52); - VERIFY_BITS(d, 64); + VERIFY_BITS_128(&d, 64); /* [d t4 t3 0 0 0] = [p8 0 0 0 p4 p3 0 0 0] */ tx = (t4 >> 48); t4 &= (M >> 4); VERIFY_BITS(tx, 4); VERIFY_BITS(t4, 48); /* [d t4+(tx<<48) t3 0 0 0] = [p8 0 0 0 p4 p3 0 0 0] */ - c = (uint128_t)a0 * b[0]; - VERIFY_BITS(c, 112); + secp256k1_u128_mul(&c, a0, b[0]); + VERIFY_BITS_128(&c, 112); /* [d t4+(tx<<48) t3 0 0 c] = [p8 0 0 0 p4 p3 0 0 p0] */ - d += (uint128_t)a1 * b[4] - + (uint128_t)a2 * b[3] - + (uint128_t)a3 * b[2] - + (uint128_t)a4 * b[1]; - VERIFY_BITS(d, 115); + secp256k1_u128_accum_mul(&d, a1, b[4]); + secp256k1_u128_accum_mul(&d, a2, b[3]); + secp256k1_u128_accum_mul(&d, a3, b[2]); + secp256k1_u128_accum_mul(&d, a4, b[1]); + VERIFY_BITS_128(&d, 115); /* [d t4+(tx<<48) t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ - u0 = d & M; d >>= 52; + u0 = secp256k1_u128_to_u64(&d) & M; secp256k1_u128_rshift(&d, 52); VERIFY_BITS(u0, 52); - VERIFY_BITS(d, 63); + VERIFY_BITS_128(&d, 63); /* [d u0 t4+(tx<<48) t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ /* [d 0 t4+(tx<<48)+(u0<<52) t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ u0 = (u0 << 4) | tx; VERIFY_BITS(u0, 56); /* [d 0 t4+(u0<<48) t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ - c += (uint128_t)u0 * (R >> 4); - VERIFY_BITS(c, 115); + secp256k1_u128_accum_mul(&c, u0, R >> 4); + VERIFY_BITS_128(&c, 115); /* [d 0 t4 t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ - r[0] = c & M; c >>= 52; + r[0] = secp256k1_u128_to_u64(&c) & M; secp256k1_u128_rshift(&c, 52); VERIFY_BITS(r[0], 52); - VERIFY_BITS(c, 61); + VERIFY_BITS_128(&c, 61); /* [d 0 t4 t3 0 c r0] = [p8 0 0 p5 p4 p3 0 0 p0] */ - c += (uint128_t)a0 * b[1] - + (uint128_t)a1 * b[0]; - VERIFY_BITS(c, 114); + secp256k1_u128_accum_mul(&c, a0, b[1]); + secp256k1_u128_accum_mul(&c, a1, b[0]); + VERIFY_BITS_128(&c, 114); /* [d 0 t4 t3 0 c r0] = [p8 0 0 p5 p4 p3 0 p1 p0] */ - d += (uint128_t)a2 * b[4] - + (uint128_t)a3 * b[3] - + (uint128_t)a4 * b[2]; - VERIFY_BITS(d, 114); + secp256k1_u128_accum_mul(&d, a2, b[4]); + secp256k1_u128_accum_mul(&d, a3, b[3]); + secp256k1_u128_accum_mul(&d, a4, b[2]); + VERIFY_BITS_128(&d, 114); /* [d 0 t4 t3 0 c r0] = [p8 0 p6 p5 p4 p3 0 p1 p0] */ - c += (d & M) * R; d >>= 52; - VERIFY_BITS(c, 115); - VERIFY_BITS(d, 62); + secp256k1_u128_accum_mul(&c, secp256k1_u128_to_u64(&d) & M, R); secp256k1_u128_rshift(&d, 52); + VERIFY_BITS_128(&c, 115); + VERIFY_BITS_128(&d, 62); /* [d 0 0 t4 t3 0 c r0] = [p8 0 p6 p5 p4 p3 0 p1 p0] */ - r[1] = c & M; c >>= 52; + r[1] = secp256k1_u128_to_u64(&c) & M; secp256k1_u128_rshift(&c, 52); VERIFY_BITS(r[1], 52); - VERIFY_BITS(c, 63); + VERIFY_BITS_128(&c, 63); /* [d 0 0 t4 t3 c r1 r0] = [p8 0 p6 p5 p4 p3 0 p1 p0] */ - c += (uint128_t)a0 * b[2] - + (uint128_t)a1 * b[1] - + (uint128_t)a2 * b[0]; - VERIFY_BITS(c, 114); + secp256k1_u128_accum_mul(&c, a0, b[2]); + secp256k1_u128_accum_mul(&c, a1, b[1]); + secp256k1_u128_accum_mul(&c, a2, b[0]); + VERIFY_BITS_128(&c, 114); /* [d 0 0 t4 t3 c r1 r0] = [p8 0 p6 p5 p4 p3 p2 p1 p0] */ - d += (uint128_t)a3 * b[4] - + (uint128_t)a4 * b[3]; - VERIFY_BITS(d, 114); + secp256k1_u128_accum_mul(&d, a3, b[4]); + secp256k1_u128_accum_mul(&d, a4, b[3]); + VERIFY_BITS_128(&d, 114); /* [d 0 0 t4 t3 c t1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ - c += (uint128_t)R * (uint64_t)d; d >>= 64; - VERIFY_BITS(c, 115); - VERIFY_BITS(d, 50); + secp256k1_u128_accum_mul(&c, R, secp256k1_u128_to_u64(&d)); secp256k1_u128_rshift(&d, 64); + VERIFY_BITS_128(&c, 115); + VERIFY_BITS_128(&d, 50); /* [(d<<12) 0 0 0 t4 t3 c r1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ - r[2] = c & M; c >>= 52; + r[2] = secp256k1_u128_to_u64(&c) & M; secp256k1_u128_rshift(&c, 52); VERIFY_BITS(r[2], 52); - VERIFY_BITS(c, 63); + VERIFY_BITS_128(&c, 63); /* [(d<<12) 0 0 0 t4 t3+c r2 r1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ - c += (uint128_t)(R << 12) * (uint64_t)d + t3; - VERIFY_BITS(c, 100); + secp256k1_u128_accum_mul(&c, R << 12, secp256k1_u128_to_u64(&d)); + secp256k1_u128_accum_u64(&c, t3); + VERIFY_BITS_128(&c, 100); /* [t4 c r2 r1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ - r[3] = c & M; c >>= 52; + r[3] = secp256k1_u128_to_u64(&c) & M; secp256k1_u128_rshift(&c, 52); VERIFY_BITS(r[3], 52); - VERIFY_BITS(c, 48); + VERIFY_BITS_128(&c, 48); /* [t4+c r3 r2 r1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ - c += t4; - VERIFY_BITS(c, 49); - /* [c r3 r2 r1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ - r[4] = c; + r[4] = secp256k1_u128_to_u64(&c) + t4; VERIFY_BITS(r[4], 49); /* [r4 r3 r2 r1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ } SECP256K1_INLINE static void secp256k1_fe_sqr_inner(uint64_t *r, const uint64_t *a) { - uint128_t c, d; + secp256k1_uint128 c, d; uint64_t a0 = a[0], a1 = a[1], a2 = a[2], a3 = a[3], a4 = a[4]; int64_t t3, t4, tx, u0; const uint64_t M = 0xFFFFFFFFFFFFFULL, R = 0x1000003D10ULL; @@ -170,107 +172,105 @@ SECP256K1_INLINE static void secp256k1_fe_sqr_inner(uint64_t *r, const uint64_t * Note that [x 0 0 0 0 0] = [x*R]. */ - d = (uint128_t)(a0*2) * a3 - + (uint128_t)(a1*2) * a2; - VERIFY_BITS(d, 114); + secp256k1_u128_mul(&d, a0*2, a3); + secp256k1_u128_accum_mul(&d, a1*2, a2); + VERIFY_BITS_128(&d, 114); /* [d 0 0 0] = [p3 0 0 0] */ - c = (uint128_t)a4 * a4; - VERIFY_BITS(c, 112); + secp256k1_u128_mul(&c, a4, a4); + VERIFY_BITS_128(&c, 112); /* [c 0 0 0 0 d 0 0 0] = [p8 0 0 0 0 p3 0 0 0] */ - d += (uint128_t)R * (uint64_t)c; c >>= 64; - VERIFY_BITS(d, 115); - VERIFY_BITS(c, 48); + secp256k1_u128_accum_mul(&d, R, secp256k1_u128_to_u64(&c)); secp256k1_u128_rshift(&c, 64); + VERIFY_BITS_128(&d, 115); + VERIFY_BITS_128(&c, 48); /* [(c<<12) 0 0 0 0 0 d 0 0 0] = [p8 0 0 0 0 p3 0 0 0] */ - t3 = d & M; d >>= 52; + t3 = secp256k1_u128_to_u64(&d) & M; secp256k1_u128_rshift(&d, 52); VERIFY_BITS(t3, 52); - VERIFY_BITS(d, 63); + VERIFY_BITS_128(&d, 63); /* [(c<<12) 0 0 0 0 d t3 0 0 0] = [p8 0 0 0 0 p3 0 0 0] */ a4 *= 2; - d += (uint128_t)a0 * a4 - + (uint128_t)(a1*2) * a3 - + (uint128_t)a2 * a2; - VERIFY_BITS(d, 115); + secp256k1_u128_accum_mul(&d, a0, a4); + secp256k1_u128_accum_mul(&d, a1*2, a3); + secp256k1_u128_accum_mul(&d, a2, a2); + VERIFY_BITS_128(&d, 115); /* [(c<<12) 0 0 0 0 d t3 0 0 0] = [p8 0 0 0 p4 p3 0 0 0] */ - d += (uint128_t)(R << 12) * (uint64_t)c; - VERIFY_BITS(d, 116); + secp256k1_u128_accum_mul(&d, R << 12, secp256k1_u128_to_u64(&c)); + VERIFY_BITS_128(&d, 116); /* [d t3 0 0 0] = [p8 0 0 0 p4 p3 0 0 0] */ - t4 = d & M; d >>= 52; + t4 = secp256k1_u128_to_u64(&d) & M; secp256k1_u128_rshift(&d, 52); VERIFY_BITS(t4, 52); - VERIFY_BITS(d, 64); + VERIFY_BITS_128(&d, 64); /* [d t4 t3 0 0 0] = [p8 0 0 0 p4 p3 0 0 0] */ tx = (t4 >> 48); t4 &= (M >> 4); VERIFY_BITS(tx, 4); VERIFY_BITS(t4, 48); /* [d t4+(tx<<48) t3 0 0 0] = [p8 0 0 0 p4 p3 0 0 0] */ - c = (uint128_t)a0 * a0; - VERIFY_BITS(c, 112); + secp256k1_u128_mul(&c, a0, a0); + VERIFY_BITS_128(&c, 112); /* [d t4+(tx<<48) t3 0 0 c] = [p8 0 0 0 p4 p3 0 0 p0] */ - d += (uint128_t)a1 * a4 - + (uint128_t)(a2*2) * a3; - VERIFY_BITS(d, 114); + secp256k1_u128_accum_mul(&d, a1, a4); + secp256k1_u128_accum_mul(&d, a2*2, a3); + VERIFY_BITS_128(&d, 114); /* [d t4+(tx<<48) t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ - u0 = d & M; d >>= 52; + u0 = secp256k1_u128_to_u64(&d) & M; secp256k1_u128_rshift(&d, 52); VERIFY_BITS(u0, 52); - VERIFY_BITS(d, 62); + VERIFY_BITS_128(&d, 62); /* [d u0 t4+(tx<<48) t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ /* [d 0 t4+(tx<<48)+(u0<<52) t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ u0 = (u0 << 4) | tx; VERIFY_BITS(u0, 56); /* [d 0 t4+(u0<<48) t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ - c += (uint128_t)u0 * (R >> 4); - VERIFY_BITS(c, 113); + secp256k1_u128_accum_mul(&c, u0, R >> 4); + VERIFY_BITS_128(&c, 113); /* [d 0 t4 t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ - r[0] = c & M; c >>= 52; + r[0] = secp256k1_u128_to_u64(&c) & M; secp256k1_u128_rshift(&c, 52); VERIFY_BITS(r[0], 52); - VERIFY_BITS(c, 61); + VERIFY_BITS_128(&c, 61); /* [d 0 t4 t3 0 c r0] = [p8 0 0 p5 p4 p3 0 0 p0] */ a0 *= 2; - c += (uint128_t)a0 * a1; - VERIFY_BITS(c, 114); + secp256k1_u128_accum_mul(&c, a0, a1); + VERIFY_BITS_128(&c, 114); /* [d 0 t4 t3 0 c r0] = [p8 0 0 p5 p4 p3 0 p1 p0] */ - d += (uint128_t)a2 * a4 - + (uint128_t)a3 * a3; - VERIFY_BITS(d, 114); + secp256k1_u128_accum_mul(&d, a2, a4); + secp256k1_u128_accum_mul(&d, a3, a3); + VERIFY_BITS_128(&d, 114); /* [d 0 t4 t3 0 c r0] = [p8 0 p6 p5 p4 p3 0 p1 p0] */ - c += (d & M) * R; d >>= 52; - VERIFY_BITS(c, 115); - VERIFY_BITS(d, 62); + secp256k1_u128_accum_mul(&c, secp256k1_u128_to_u64(&d) & M, R); secp256k1_u128_rshift(&d, 52); + VERIFY_BITS_128(&c, 115); + VERIFY_BITS_128(&d, 62); /* [d 0 0 t4 t3 0 c r0] = [p8 0 p6 p5 p4 p3 0 p1 p0] */ - r[1] = c & M; c >>= 52; + r[1] = secp256k1_u128_to_u64(&c) & M; secp256k1_u128_rshift(&c, 52); VERIFY_BITS(r[1], 52); - VERIFY_BITS(c, 63); + VERIFY_BITS_128(&c, 63); /* [d 0 0 t4 t3 c r1 r0] = [p8 0 p6 p5 p4 p3 0 p1 p0] */ - c += (uint128_t)a0 * a2 - + (uint128_t)a1 * a1; - VERIFY_BITS(c, 114); + secp256k1_u128_accum_mul(&c, a0, a2); + secp256k1_u128_accum_mul(&c, a1, a1); + VERIFY_BITS_128(&c, 114); /* [d 0 0 t4 t3 c r1 r0] = [p8 0 p6 p5 p4 p3 p2 p1 p0] */ - d += (uint128_t)a3 * a4; - VERIFY_BITS(d, 114); + secp256k1_u128_accum_mul(&d, a3, a4); + VERIFY_BITS_128(&d, 114); /* [d 0 0 t4 t3 c r1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ - c += (uint128_t)R * (uint64_t)d; d >>= 64; - VERIFY_BITS(c, 115); - VERIFY_BITS(d, 50); + secp256k1_u128_accum_mul(&c, R, secp256k1_u128_to_u64(&d)); secp256k1_u128_rshift(&d, 64); + VERIFY_BITS_128(&c, 115); + VERIFY_BITS_128(&d, 50); /* [(d<<12) 0 0 0 t4 t3 c r1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ - r[2] = c & M; c >>= 52; + r[2] = secp256k1_u128_to_u64(&c) & M; secp256k1_u128_rshift(&c, 52); VERIFY_BITS(r[2], 52); - VERIFY_BITS(c, 63); + VERIFY_BITS_128(&c, 63); /* [(d<<12) 0 0 0 t4 t3+c r2 r1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ - c += (uint128_t)(R << 12) * (uint64_t)d + t3; - VERIFY_BITS(c, 100); + secp256k1_u128_accum_mul(&c, R << 12, secp256k1_u128_to_u64(&d)); + secp256k1_u128_accum_u64(&c, t3); + VERIFY_BITS_128(&c, 100); /* [t4 c r2 r1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ - r[3] = c & M; c >>= 52; + r[3] = secp256k1_u128_to_u64(&c) & M; secp256k1_u128_rshift(&c, 52); VERIFY_BITS(r[3], 52); - VERIFY_BITS(c, 48); + VERIFY_BITS_128(&c, 48); /* [t4+c r3 r2 r1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ - c += t4; - VERIFY_BITS(c, 49); - /* [c r3 r2 r1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ - r[4] = c; + r[4] = secp256k1_u128_to_u64(&c) + t4; VERIFY_BITS(r[4], 49); /* [r4 r3 r2 r1 r0] = [p8 p7 p6 p5 p4 p3 p2 p1 p0] */ } diff --git a/src/secp256k1/src/group.h b/src/secp256k1/src/group.h index bb7dae1cf7..b79ba597db 100644 --- a/src/secp256k1/src/group.h +++ b/src/secp256k1/src/group.h @@ -23,7 +23,7 @@ typedef struct { #define SECP256K1_GE_CONST_INFINITY {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), 1} /** A group element of the secp256k1 curve, in jacobian coordinates. - * Note: For exhastive test mode, sepc256k1 is replaced by a small subgroup of a different curve. + * Note: For exhastive test mode, secp256k1 is replaced by a small subgroup of a different curve. */ typedef struct { secp256k1_fe x; /* actual X: x/z^2 */ @@ -97,6 +97,9 @@ static void secp256k1_gej_set_infinity(secp256k1_gej *r); /** Set a group element (jacobian) equal to another which is given in affine coordinates. */ static void secp256k1_gej_set_ge(secp256k1_gej *r, const secp256k1_ge *a); +/** Check two group elements (jacobian) for equality in variable time. */ +static int secp256k1_gej_eq_var(const secp256k1_gej *a, const secp256k1_gej *b); + /** Compare the X coordinate of a group element (jacobian). */ static int secp256k1_gej_eq_x_var(const secp256k1_fe *x, const secp256k1_gej *a); diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index 63735ab682..dfe6e32c7f 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -236,6 +236,13 @@ static void secp256k1_gej_set_ge(secp256k1_gej *r, const secp256k1_ge *a) { secp256k1_fe_set_int(&r->z, 1); } +static int secp256k1_gej_eq_var(const secp256k1_gej *a, const secp256k1_gej *b) { + secp256k1_gej tmp; + secp256k1_gej_neg(&tmp, a); + secp256k1_gej_add_var(&tmp, &tmp, b, NULL); + return secp256k1_gej_is_infinity(&tmp); +} + static int secp256k1_gej_eq_x_var(const secp256k1_fe *x, const secp256k1_gej *a) { secp256k1_fe r, r2; VERIFY_CHECK(!a->infinity); diff --git a/src/secp256k1/src/int128.h b/src/secp256k1/src/int128.h new file mode 100644 index 0000000000..84d969a236 --- /dev/null +++ b/src/secp256k1/src/int128.h @@ -0,0 +1,85 @@ +#ifndef SECP256K1_INT128_H +#define SECP256K1_INT128_H + +#include "util.h" + +#if defined(SECP256K1_WIDEMUL_INT128) +# if defined(SECP256K1_INT128_NATIVE) +# include "int128_native.h" +# elif defined(SECP256K1_INT128_STRUCT) +# include "int128_struct.h" +# else +# error "Please select int128 implementation" +# endif + +/* Construct an unsigned 128-bit value from a high and a low 64-bit value. */ +static SECP256K1_INLINE void secp256k1_u128_load(secp256k1_uint128 *r, uint64_t hi, uint64_t lo); + +/* Multiply two unsigned 64-bit values a and b and write the result to r. */ +static SECP256K1_INLINE void secp256k1_u128_mul(secp256k1_uint128 *r, uint64_t a, uint64_t b); + +/* Multiply two unsigned 64-bit values a and b and add the result to r. + * The final result is taken modulo 2^128. + */ +static SECP256K1_INLINE void secp256k1_u128_accum_mul(secp256k1_uint128 *r, uint64_t a, uint64_t b); + +/* Add an unsigned 64-bit value a to r. + * The final result is taken modulo 2^128. + */ +static SECP256K1_INLINE void secp256k1_u128_accum_u64(secp256k1_uint128 *r, uint64_t a); + +/* Unsigned (logical) right shift. + * Non-constant time in n. + */ +static SECP256K1_INLINE void secp256k1_u128_rshift(secp256k1_uint128 *r, unsigned int n); + +/* Return the low 64-bits of a 128-bit value as an unsigned 64-bit value. */ +static SECP256K1_INLINE uint64_t secp256k1_u128_to_u64(const secp256k1_uint128 *a); + +/* Return the high 64-bits of a 128-bit value as an unsigned 64-bit value. */ +static SECP256K1_INLINE uint64_t secp256k1_u128_hi_u64(const secp256k1_uint128 *a); + +/* Write an unsigned 64-bit value to r. */ +static SECP256K1_INLINE void secp256k1_u128_from_u64(secp256k1_uint128 *r, uint64_t a); + +/* Tests if r is strictly less than to 2^n. + * n must be strictly less than 128. + */ +static SECP256K1_INLINE int secp256k1_u128_check_bits(const secp256k1_uint128 *r, unsigned int n); + +/* Construct an signed 128-bit value from a high and a low 64-bit value. */ +static SECP256K1_INLINE void secp256k1_i128_load(secp256k1_int128 *r, int64_t hi, uint64_t lo); + +/* Multiply two signed 64-bit values a and b and write the result to r. */ +static SECP256K1_INLINE void secp256k1_i128_mul(secp256k1_int128 *r, int64_t a, int64_t b); + +/* Multiply two signed 64-bit values a and b and add the result to r. + * Overflow or underflow from the addition is undefined behaviour. + */ +static SECP256K1_INLINE void secp256k1_i128_accum_mul(secp256k1_int128 *r, int64_t a, int64_t b); + +/* Compute a*d - b*c from signed 64-bit values and write the result to r. */ +static SECP256K1_INLINE void secp256k1_i128_det(secp256k1_int128 *r, int64_t a, int64_t b, int64_t c, int64_t d); + +/* Signed (arithmetic) right shift. + * Non-constant time in b. + */ +static SECP256K1_INLINE void secp256k1_i128_rshift(secp256k1_int128 *r, unsigned int b); + +/* Return the low 64-bits of a 128-bit value interpreted as an signed 64-bit value. */ +static SECP256K1_INLINE int64_t secp256k1_i128_to_i64(const secp256k1_int128 *a); + +/* Write a signed 64-bit value to r. */ +static SECP256K1_INLINE void secp256k1_i128_from_i64(secp256k1_int128 *r, int64_t a); + +/* Compare two 128-bit values for equality. */ +static SECP256K1_INLINE int secp256k1_i128_eq_var(const secp256k1_int128 *a, const secp256k1_int128 *b); + +/* Tests if r is equal to 2^n. + * n must be strictly less than 127. + */ +static SECP256K1_INLINE int secp256k1_i128_check_pow2(const secp256k1_int128 *r, unsigned int n); + +#endif + +#endif diff --git a/src/secp256k1/src/int128_impl.h b/src/secp256k1/src/int128_impl.h new file mode 100644 index 0000000000..cfc573408a --- /dev/null +++ b/src/secp256k1/src/int128_impl.h @@ -0,0 +1,18 @@ +#ifndef SECP256K1_INT128_IMPL_H +#define SECP256K1_INT128_IMPL_H + +#include "util.h" + +#include "int128.h" + +#if defined(SECP256K1_WIDEMUL_INT128) +# if defined(SECP256K1_INT128_NATIVE) +# include "int128_native_impl.h" +# elif defined(SECP256K1_INT128_STRUCT) +# include "int128_struct_impl.h" +# else +# error "Please select int128 implementation" +# endif +#endif + +#endif diff --git a/src/secp256k1/src/int128_native.h b/src/secp256k1/src/int128_native.h new file mode 100644 index 0000000000..7c97aafc74 --- /dev/null +++ b/src/secp256k1/src/int128_native.h @@ -0,0 +1,19 @@ +#ifndef SECP256K1_INT128_NATIVE_H +#define SECP256K1_INT128_NATIVE_H + +#include <stdint.h> +#include "util.h" + +#if !defined(UINT128_MAX) && defined(__SIZEOF_INT128__) +SECP256K1_GNUC_EXT typedef unsigned __int128 uint128_t; +SECP256K1_GNUC_EXT typedef __int128 int128_t; +# define UINT128_MAX ((uint128_t)(-1)) +# define INT128_MAX ((int128_t)(UINT128_MAX >> 1)) +# define INT128_MIN (-INT128_MAX - 1) +/* No (U)INT128_C macros because compilers providing __int128 do not support 128-bit literals. */ +#endif + +typedef uint128_t secp256k1_uint128; +typedef int128_t secp256k1_int128; + +#endif diff --git a/src/secp256k1/src/int128_native_impl.h b/src/secp256k1/src/int128_native_impl.h new file mode 100644 index 0000000000..e4b7f4106c --- /dev/null +++ b/src/secp256k1/src/int128_native_impl.h @@ -0,0 +1,87 @@ +#ifndef SECP256K1_INT128_NATIVE_IMPL_H +#define SECP256K1_INT128_NATIVE_IMPL_H + +#include "int128.h" + +static SECP256K1_INLINE void secp256k1_u128_load(secp256k1_uint128 *r, uint64_t hi, uint64_t lo) { + *r = (((uint128_t)hi) << 64) + lo; +} + +static SECP256K1_INLINE void secp256k1_u128_mul(secp256k1_uint128 *r, uint64_t a, uint64_t b) { + *r = (uint128_t)a * b; +} + +static SECP256K1_INLINE void secp256k1_u128_accum_mul(secp256k1_uint128 *r, uint64_t a, uint64_t b) { + *r += (uint128_t)a * b; +} + +static SECP256K1_INLINE void secp256k1_u128_accum_u64(secp256k1_uint128 *r, uint64_t a) { + *r += a; +} + +static SECP256K1_INLINE void secp256k1_u128_rshift(secp256k1_uint128 *r, unsigned int n) { + VERIFY_CHECK(n < 128); + *r >>= n; +} + +static SECP256K1_INLINE uint64_t secp256k1_u128_to_u64(const secp256k1_uint128 *a) { + return (uint64_t)(*a); +} + +static SECP256K1_INLINE uint64_t secp256k1_u128_hi_u64(const secp256k1_uint128 *a) { + return (uint64_t)(*a >> 64); +} + +static SECP256K1_INLINE void secp256k1_u128_from_u64(secp256k1_uint128 *r, uint64_t a) { + *r = a; +} + +static SECP256K1_INLINE int secp256k1_u128_check_bits(const secp256k1_uint128 *r, unsigned int n) { + VERIFY_CHECK(n < 128); + return (*r >> n == 0); +} + +static SECP256K1_INLINE void secp256k1_i128_load(secp256k1_int128 *r, int64_t hi, uint64_t lo) { + *r = (((uint128_t)(uint64_t)hi) << 64) + lo; +} + +static SECP256K1_INLINE void secp256k1_i128_mul(secp256k1_int128 *r, int64_t a, int64_t b) { + *r = (int128_t)a * b; +} + +static SECP256K1_INLINE void secp256k1_i128_accum_mul(secp256k1_int128 *r, int64_t a, int64_t b) { + int128_t ab = (int128_t)a * b; + VERIFY_CHECK(0 <= ab ? *r <= INT128_MAX - ab : INT128_MIN - ab <= *r); + *r += ab; +} + +static SECP256K1_INLINE void secp256k1_i128_det(secp256k1_int128 *r, int64_t a, int64_t b, int64_t c, int64_t d) { + int128_t ad = (int128_t)a * d; + int128_t bc = (int128_t)b * c; + VERIFY_CHECK(0 <= bc ? INT128_MIN + bc <= ad : ad <= INT128_MAX + bc); + *r = ad - bc; +} + +static SECP256K1_INLINE void secp256k1_i128_rshift(secp256k1_int128 *r, unsigned int n) { + VERIFY_CHECK(n < 128); + *r >>= n; +} + +static SECP256K1_INLINE int64_t secp256k1_i128_to_i64(const secp256k1_int128 *a) { + return *a; +} + +static SECP256K1_INLINE void secp256k1_i128_from_i64(secp256k1_int128 *r, int64_t a) { + *r = a; +} + +static SECP256K1_INLINE int secp256k1_i128_eq_var(const secp256k1_int128 *a, const secp256k1_int128 *b) { + return *a == *b; +} + +static SECP256K1_INLINE int secp256k1_i128_check_pow2(const secp256k1_int128 *r, unsigned int n) { + VERIFY_CHECK(n < 127); + return (*r == (int128_t)1 << n); +} + +#endif diff --git a/src/secp256k1/src/int128_struct.h b/src/secp256k1/src/int128_struct.h new file mode 100644 index 0000000000..6156f82cc2 --- /dev/null +++ b/src/secp256k1/src/int128_struct.h @@ -0,0 +1,14 @@ +#ifndef SECP256K1_INT128_STRUCT_H +#define SECP256K1_INT128_STRUCT_H + +#include <stdint.h> +#include "util.h" + +typedef struct { + uint64_t lo; + uint64_t hi; +} secp256k1_uint128; + +typedef secp256k1_uint128 secp256k1_int128; + +#endif diff --git a/src/secp256k1/src/int128_struct_impl.h b/src/secp256k1/src/int128_struct_impl.h new file mode 100644 index 0000000000..b5f8fb7b65 --- /dev/null +++ b/src/secp256k1/src/int128_struct_impl.h @@ -0,0 +1,192 @@ +#ifndef SECP256K1_INT128_STRUCT_IMPL_H +#define SECP256K1_INT128_STRUCT_IMPL_H + +#include "int128.h" + +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) /* MSVC */ +# include <intrin.h> +# if defined(_M_ARM64) || defined(SECP256K1_MSVC_MULH_TEST_OVERRIDE) +/* On ARM64 MSVC, use __(u)mulh for the upper half of 64x64 multiplications. + (Define SECP256K1_MSVC_MULH_TEST_OVERRIDE to test this code path on X64, + which supports both __(u)mulh and _umul128.) */ +# if defined(SECP256K1_MSVC_MULH_TEST_OVERRIDE) +# pragma message(__FILE__ ": SECP256K1_MSVC_MULH_TEST_OVERRIDE is defined, forcing use of __(u)mulh.") +# endif +static SECP256K1_INLINE uint64_t secp256k1_umul128(uint64_t a, uint64_t b, uint64_t* hi) { + *hi = __umulh(a, b); + return a * b; +} + +static SECP256K1_INLINE int64_t secp256k1_mul128(int64_t a, int64_t b, int64_t* hi) { + *hi = __mulh(a, b); + return (uint64_t)a * (uint64_t)b; +} +# else +/* On x84_64 MSVC, use native _(u)mul128 for 64x64->128 multiplications. */ +# define secp256k1_umul128 _umul128 +# define secp256k1_mul128 _mul128 +# endif +#else +/* On other systems, emulate 64x64->128 multiplications using 32x32->64 multiplications. */ +static SECP256K1_INLINE uint64_t secp256k1_umul128(uint64_t a, uint64_t b, uint64_t* hi) { + uint64_t ll = (uint64_t)(uint32_t)a * (uint32_t)b; + uint64_t lh = (uint32_t)a * (b >> 32); + uint64_t hl = (a >> 32) * (uint32_t)b; + uint64_t hh = (a >> 32) * (b >> 32); + uint64_t mid34 = (ll >> 32) + (uint32_t)lh + (uint32_t)hl; + *hi = hh + (lh >> 32) + (hl >> 32) + (mid34 >> 32); + return (mid34 << 32) + (uint32_t)ll; +} + +static SECP256K1_INLINE int64_t secp256k1_mul128(int64_t a, int64_t b, int64_t* hi) { + uint64_t ll = (uint64_t)(uint32_t)a * (uint32_t)b; + int64_t lh = (uint32_t)a * (b >> 32); + int64_t hl = (a >> 32) * (uint32_t)b; + int64_t hh = (a >> 32) * (b >> 32); + uint64_t mid34 = (ll >> 32) + (uint32_t)lh + (uint32_t)hl; + *hi = hh + (lh >> 32) + (hl >> 32) + (mid34 >> 32); + return (mid34 << 32) + (uint32_t)ll; +} +#endif + +static SECP256K1_INLINE void secp256k1_u128_load(secp256k1_uint128 *r, uint64_t hi, uint64_t lo) { + r->hi = hi; + r->lo = lo; +} + +static SECP256K1_INLINE void secp256k1_u128_mul(secp256k1_uint128 *r, uint64_t a, uint64_t b) { + r->lo = secp256k1_umul128(a, b, &r->hi); +} + +static SECP256K1_INLINE void secp256k1_u128_accum_mul(secp256k1_uint128 *r, uint64_t a, uint64_t b) { + uint64_t lo, hi; + lo = secp256k1_umul128(a, b, &hi); + r->lo += lo; + r->hi += hi + (r->lo < lo); +} + +static SECP256K1_INLINE void secp256k1_u128_accum_u64(secp256k1_uint128 *r, uint64_t a) { + r->lo += a; + r->hi += r->lo < a; +} + +/* Unsigned (logical) right shift. + * Non-constant time in n. + */ +static SECP256K1_INLINE void secp256k1_u128_rshift(secp256k1_uint128 *r, unsigned int n) { + VERIFY_CHECK(n < 128); + if (n >= 64) { + r->lo = r->hi >> (n-64); + r->hi = 0; + } else if (n > 0) { + r->lo = ((1U * r->hi) << (64-n)) | r->lo >> n; + r->hi >>= n; + } +} + +static SECP256K1_INLINE uint64_t secp256k1_u128_to_u64(const secp256k1_uint128 *a) { + return a->lo; +} + +static SECP256K1_INLINE uint64_t secp256k1_u128_hi_u64(const secp256k1_uint128 *a) { + return a->hi; +} + +static SECP256K1_INLINE void secp256k1_u128_from_u64(secp256k1_uint128 *r, uint64_t a) { + r->hi = 0; + r->lo = a; +} + +static SECP256K1_INLINE int secp256k1_u128_check_bits(const secp256k1_uint128 *r, unsigned int n) { + VERIFY_CHECK(n < 128); + return n >= 64 ? r->hi >> (n - 64) == 0 + : r->hi == 0 && r->lo >> n == 0; +} + +static SECP256K1_INLINE void secp256k1_i128_load(secp256k1_int128 *r, int64_t hi, uint64_t lo) { + r->hi = hi; + r->lo = lo; +} + +static SECP256K1_INLINE void secp256k1_i128_mul(secp256k1_int128 *r, int64_t a, int64_t b) { + int64_t hi; + r->lo = (uint64_t)secp256k1_mul128(a, b, &hi); + r->hi = (uint64_t)hi; +} + +static SECP256K1_INLINE void secp256k1_i128_accum_mul(secp256k1_int128 *r, int64_t a, int64_t b) { + int64_t hi; + uint64_t lo = (uint64_t)secp256k1_mul128(a, b, &hi); + r->lo += lo; + hi += r->lo < lo; + /* Verify no overflow. + * If r represents a positive value (the sign bit is not set) and the value we are adding is a positive value (the sign bit is not set), + * then we require that the resulting value also be positive (the sign bit is not set). + * Note that (X <= Y) means (X implies Y) when X and Y are boolean values (i.e. 0 or 1). + */ + VERIFY_CHECK((r->hi <= 0x7fffffffffffffffu && (uint64_t)hi <= 0x7fffffffffffffffu) <= (r->hi + (uint64_t)hi <= 0x7fffffffffffffffu)); + /* Verify no underflow. + * If r represents a negative value (the sign bit is set) and the value we are adding is a negative value (the sign bit is set), + * then we require that the resulting value also be negative (the sign bit is set). + */ + VERIFY_CHECK((r->hi > 0x7fffffffffffffffu && (uint64_t)hi > 0x7fffffffffffffffu) <= (r->hi + (uint64_t)hi > 0x7fffffffffffffffu)); + r->hi += hi; +} + +static SECP256K1_INLINE void secp256k1_i128_dissip_mul(secp256k1_int128 *r, int64_t a, int64_t b) { + int64_t hi; + uint64_t lo = (uint64_t)secp256k1_mul128(a, b, &hi); + hi += r->lo < lo; + /* Verify no overflow. + * If r represents a positive value (the sign bit is not set) and the value we are subtracting is a negative value (the sign bit is set), + * then we require that the resulting value also be positive (the sign bit is not set). + */ + VERIFY_CHECK((r->hi <= 0x7fffffffffffffffu && (uint64_t)hi > 0x7fffffffffffffffu) <= (r->hi - (uint64_t)hi <= 0x7fffffffffffffffu)); + /* Verify no underflow. + * If r represents a negative value (the sign bit is set) and the value we are subtracting is a positive value (the sign sign bit is not set), + * then we require that the resulting value also be negative (the sign bit is set). + */ + VERIFY_CHECK((r->hi > 0x7fffffffffffffffu && (uint64_t)hi <= 0x7fffffffffffffffu) <= (r->hi - (uint64_t)hi > 0x7fffffffffffffffu)); + r->hi -= hi; + r->lo -= lo; +} + +static SECP256K1_INLINE void secp256k1_i128_det(secp256k1_int128 *r, int64_t a, int64_t b, int64_t c, int64_t d) { + secp256k1_i128_mul(r, a, d); + secp256k1_i128_dissip_mul(r, b, c); +} + +/* Signed (arithmetic) right shift. + * Non-constant time in n. + */ +static SECP256K1_INLINE void secp256k1_i128_rshift(secp256k1_int128 *r, unsigned int n) { + VERIFY_CHECK(n < 128); + if (n >= 64) { + r->lo = (uint64_t)((int64_t)(r->hi) >> (n-64)); + r->hi = (uint64_t)((int64_t)(r->hi) >> 63); + } else if (n > 0) { + r->lo = ((1U * r->hi) << (64-n)) | r->lo >> n; + r->hi = (uint64_t)((int64_t)(r->hi) >> n); + } +} + +static SECP256K1_INLINE int64_t secp256k1_i128_to_i64(const secp256k1_int128 *a) { + return (int64_t)a->lo; +} + +static SECP256K1_INLINE void secp256k1_i128_from_i64(secp256k1_int128 *r, int64_t a) { + r->hi = (uint64_t)(a >> 63); + r->lo = (uint64_t)a; +} + +static SECP256K1_INLINE int secp256k1_i128_eq_var(const secp256k1_int128 *a, const secp256k1_int128 *b) { + return a->hi == b->hi && a->lo == b->lo; +} + +static SECP256K1_INLINE int secp256k1_i128_check_pow2(const secp256k1_int128 *r, unsigned int n) { + VERIFY_CHECK(n < 127); + return n >= 64 ? r->hi == (uint64_t)1 << (n - 64) && r->lo == 0 + : r->hi == 0 && r->lo == (uint64_t)1 << n; +} + +#endif diff --git a/src/secp256k1/src/modinv64_impl.h b/src/secp256k1/src/modinv64_impl.h index 0743a9c821..50be2e5e78 100644 --- a/src/secp256k1/src/modinv64_impl.h +++ b/src/secp256k1/src/modinv64_impl.h @@ -7,10 +7,9 @@ #ifndef SECP256K1_MODINV64_IMPL_H #define SECP256K1_MODINV64_IMPL_H +#include "int128.h" #include "modinv64.h" -#include "util.h" - /* This file implements modular inversion based on the paper "Fast constant-time gcd computation and * modular inversion" by Daniel J. Bernstein and Bo-Yin Yang. * @@ -18,6 +17,15 @@ * implementation for N=62, using 62-bit signed limbs represented as int64_t. */ +/* Data type for transition matrices (see section 3 of explanation). + * + * t = [ u v ] + * [ q r ] + */ +typedef struct { + int64_t u, v, q, r; +} secp256k1_modinv64_trans2x2; + #ifdef VERIFY /* Helper function to compute the absolute value of an int64_t. * (we don't use abs/labs/llabs as it depends on the int sizes). */ @@ -32,15 +40,17 @@ static const secp256k1_modinv64_signed62 SECP256K1_SIGNED62_ONE = {{1}}; /* Compute a*factor and put it in r. All but the top limb in r will be in range [0,2^62). */ static void secp256k1_modinv64_mul_62(secp256k1_modinv64_signed62 *r, const secp256k1_modinv64_signed62 *a, int alen, int64_t factor) { const int64_t M62 = (int64_t)(UINT64_MAX >> 2); - int128_t c = 0; + secp256k1_int128 c, d; int i; + secp256k1_i128_from_i64(&c, 0); for (i = 0; i < 4; ++i) { - if (i < alen) c += (int128_t)a->v[i] * factor; - r->v[i] = (int64_t)c & M62; c >>= 62; + if (i < alen) secp256k1_i128_accum_mul(&c, a->v[i], factor); + r->v[i] = secp256k1_i128_to_i64(&c) & M62; secp256k1_i128_rshift(&c, 62); } - if (4 < alen) c += (int128_t)a->v[4] * factor; - VERIFY_CHECK(c == (int64_t)c); - r->v[4] = (int64_t)c; + if (4 < alen) secp256k1_i128_accum_mul(&c, a->v[4], factor); + secp256k1_i128_from_i64(&d, secp256k1_i128_to_i64(&c)); + VERIFY_CHECK(secp256k1_i128_eq_var(&c, &d)); + r->v[4] = secp256k1_i128_to_i64(&c); } /* Return -1 for a<b*factor, 0 for a==b*factor, 1 for a>b*factor. A has alen limbs; b has 5. */ @@ -60,6 +70,13 @@ static int secp256k1_modinv64_mul_cmp_62(const secp256k1_modinv64_signed62 *a, i } return 0; } + +/* Check if the determinant of t is equal to 1 << n. */ +static int secp256k1_modinv64_det_check_pow2(const secp256k1_modinv64_trans2x2 *t, unsigned int n) { + secp256k1_int128 a; + secp256k1_i128_det(&a, t->u, t->v, t->q, t->r); + return secp256k1_i128_check_pow2(&a, n); +} #endif /* Take as input a signed62 number in range (-2*modulus,modulus), and add a multiple of the modulus @@ -136,15 +153,6 @@ static void secp256k1_modinv64_normalize_62(secp256k1_modinv64_signed62 *r, int6 #endif } -/* Data type for transition matrices (see section 3 of explanation). - * - * t = [ u v ] - * [ q r ] - */ -typedef struct { - int64_t u, v, q, r; -} secp256k1_modinv64_trans2x2; - /* Compute the transition matrix and eta for 59 divsteps (where zeta=-(delta+1/2)). * Note that the transformation matrix is scaled by 2^62 and not 2^59. * @@ -203,13 +211,15 @@ static int64_t secp256k1_modinv64_divsteps_59(int64_t zeta, uint64_t f0, uint64_ t->v = (int64_t)v; t->q = (int64_t)q; t->r = (int64_t)r; +#ifdef VERIFY /* The determinant of t must be a power of two. This guarantees that multiplication with t * does not change the gcd of f and g, apart from adding a power-of-2 factor to it (which * will be divided out again). As each divstep's individual matrix has determinant 2, the * aggregate of 59 of them will have determinant 2^59. Multiplying with the initial * 8*identity (which has determinant 2^6) means the overall outputs has determinant * 2^65. */ - VERIFY_CHECK((int128_t)t->u * t->r - (int128_t)t->v * t->q == ((int128_t)1) << 65); + VERIFY_CHECK(secp256k1_modinv64_det_check_pow2(t, 65)); +#endif return zeta; } @@ -286,11 +296,13 @@ static int64_t secp256k1_modinv64_divsteps_62_var(int64_t eta, uint64_t f0, uint t->v = (int64_t)v; t->q = (int64_t)q; t->r = (int64_t)r; +#ifdef VERIFY /* The determinant of t must be a power of two. This guarantees that multiplication with t * does not change the gcd of f and g, apart from adding a power-of-2 factor to it (which * will be divided out again). As each divstep's individual matrix has determinant 2, the * aggregate of 62 of them will have determinant 2^62. */ - VERIFY_CHECK((int128_t)t->u * t->r - (int128_t)t->v * t->q == ((int128_t)1) << 62); + VERIFY_CHECK(secp256k1_modinv64_det_check_pow2(t, 62)); +#endif return eta; } @@ -307,7 +319,7 @@ static void secp256k1_modinv64_update_de_62(secp256k1_modinv64_signed62 *d, secp const int64_t e0 = e->v[0], e1 = e->v[1], e2 = e->v[2], e3 = e->v[3], e4 = e->v[4]; const int64_t u = t->u, v = t->v, q = t->q, r = t->r; int64_t md, me, sd, se; - int128_t cd, ce; + secp256k1_int128 cd, ce; #ifdef VERIFY VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(d, 5, &modinfo->modulus, -2) > 0); /* d > -2*modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(d, 5, &modinfo->modulus, 1) < 0); /* d < modulus */ @@ -324,54 +336,64 @@ static void secp256k1_modinv64_update_de_62(secp256k1_modinv64_signed62 *d, secp md = (u & sd) + (v & se); me = (q & sd) + (r & se); /* Begin computing t*[d,e]. */ - cd = (int128_t)u * d0 + (int128_t)v * e0; - ce = (int128_t)q * d0 + (int128_t)r * e0; + secp256k1_i128_mul(&cd, u, d0); + secp256k1_i128_accum_mul(&cd, v, e0); + secp256k1_i128_mul(&ce, q, d0); + secp256k1_i128_accum_mul(&ce, r, e0); /* Correct md,me so that t*[d,e]+modulus*[md,me] has 62 zero bottom bits. */ - md -= (modinfo->modulus_inv62 * (uint64_t)cd + md) & M62; - me -= (modinfo->modulus_inv62 * (uint64_t)ce + me) & M62; + md -= (modinfo->modulus_inv62 * (uint64_t)secp256k1_i128_to_i64(&cd) + md) & M62; + me -= (modinfo->modulus_inv62 * (uint64_t)secp256k1_i128_to_i64(&ce) + me) & M62; /* Update the beginning of computation for t*[d,e]+modulus*[md,me] now md,me are known. */ - cd += (int128_t)modinfo->modulus.v[0] * md; - ce += (int128_t)modinfo->modulus.v[0] * me; + secp256k1_i128_accum_mul(&cd, modinfo->modulus.v[0], md); + secp256k1_i128_accum_mul(&ce, modinfo->modulus.v[0], me); /* Verify that the low 62 bits of the computation are indeed zero, and then throw them away. */ - VERIFY_CHECK(((int64_t)cd & M62) == 0); cd >>= 62; - VERIFY_CHECK(((int64_t)ce & M62) == 0); ce >>= 62; + VERIFY_CHECK((secp256k1_i128_to_i64(&cd) & M62) == 0); secp256k1_i128_rshift(&cd, 62); + VERIFY_CHECK((secp256k1_i128_to_i64(&ce) & M62) == 0); secp256k1_i128_rshift(&ce, 62); /* Compute limb 1 of t*[d,e]+modulus*[md,me], and store it as output limb 0 (= down shift). */ - cd += (int128_t)u * d1 + (int128_t)v * e1; - ce += (int128_t)q * d1 + (int128_t)r * e1; + secp256k1_i128_accum_mul(&cd, u, d1); + secp256k1_i128_accum_mul(&cd, v, e1); + secp256k1_i128_accum_mul(&ce, q, d1); + secp256k1_i128_accum_mul(&ce, r, e1); if (modinfo->modulus.v[1]) { /* Optimize for the case where limb of modulus is zero. */ - cd += (int128_t)modinfo->modulus.v[1] * md; - ce += (int128_t)modinfo->modulus.v[1] * me; + secp256k1_i128_accum_mul(&cd, modinfo->modulus.v[1], md); + secp256k1_i128_accum_mul(&ce, modinfo->modulus.v[1], me); } - d->v[0] = (int64_t)cd & M62; cd >>= 62; - e->v[0] = (int64_t)ce & M62; ce >>= 62; + d->v[0] = secp256k1_i128_to_i64(&cd) & M62; secp256k1_i128_rshift(&cd, 62); + e->v[0] = secp256k1_i128_to_i64(&ce) & M62; secp256k1_i128_rshift(&ce, 62); /* Compute limb 2 of t*[d,e]+modulus*[md,me], and store it as output limb 1. */ - cd += (int128_t)u * d2 + (int128_t)v * e2; - ce += (int128_t)q * d2 + (int128_t)r * e2; + secp256k1_i128_accum_mul(&cd, u, d2); + secp256k1_i128_accum_mul(&cd, v, e2); + secp256k1_i128_accum_mul(&ce, q, d2); + secp256k1_i128_accum_mul(&ce, r, e2); if (modinfo->modulus.v[2]) { /* Optimize for the case where limb of modulus is zero. */ - cd += (int128_t)modinfo->modulus.v[2] * md; - ce += (int128_t)modinfo->modulus.v[2] * me; + secp256k1_i128_accum_mul(&cd, modinfo->modulus.v[2], md); + secp256k1_i128_accum_mul(&ce, modinfo->modulus.v[2], me); } - d->v[1] = (int64_t)cd & M62; cd >>= 62; - e->v[1] = (int64_t)ce & M62; ce >>= 62; + d->v[1] = secp256k1_i128_to_i64(&cd) & M62; secp256k1_i128_rshift(&cd, 62); + e->v[1] = secp256k1_i128_to_i64(&ce) & M62; secp256k1_i128_rshift(&ce, 62); /* Compute limb 3 of t*[d,e]+modulus*[md,me], and store it as output limb 2. */ - cd += (int128_t)u * d3 + (int128_t)v * e3; - ce += (int128_t)q * d3 + (int128_t)r * e3; + secp256k1_i128_accum_mul(&cd, u, d3); + secp256k1_i128_accum_mul(&cd, v, e3); + secp256k1_i128_accum_mul(&ce, q, d3); + secp256k1_i128_accum_mul(&ce, r, e3); if (modinfo->modulus.v[3]) { /* Optimize for the case where limb of modulus is zero. */ - cd += (int128_t)modinfo->modulus.v[3] * md; - ce += (int128_t)modinfo->modulus.v[3] * me; + secp256k1_i128_accum_mul(&cd, modinfo->modulus.v[3], md); + secp256k1_i128_accum_mul(&ce, modinfo->modulus.v[3], me); } - d->v[2] = (int64_t)cd & M62; cd >>= 62; - e->v[2] = (int64_t)ce & M62; ce >>= 62; + d->v[2] = secp256k1_i128_to_i64(&cd) & M62; secp256k1_i128_rshift(&cd, 62); + e->v[2] = secp256k1_i128_to_i64(&ce) & M62; secp256k1_i128_rshift(&ce, 62); /* Compute limb 4 of t*[d,e]+modulus*[md,me], and store it as output limb 3. */ - cd += (int128_t)u * d4 + (int128_t)v * e4; - ce += (int128_t)q * d4 + (int128_t)r * e4; - cd += (int128_t)modinfo->modulus.v[4] * md; - ce += (int128_t)modinfo->modulus.v[4] * me; - d->v[3] = (int64_t)cd & M62; cd >>= 62; - e->v[3] = (int64_t)ce & M62; ce >>= 62; + secp256k1_i128_accum_mul(&cd, u, d4); + secp256k1_i128_accum_mul(&cd, v, e4); + secp256k1_i128_accum_mul(&ce, q, d4); + secp256k1_i128_accum_mul(&ce, r, e4); + secp256k1_i128_accum_mul(&cd, modinfo->modulus.v[4], md); + secp256k1_i128_accum_mul(&ce, modinfo->modulus.v[4], me); + d->v[3] = secp256k1_i128_to_i64(&cd) & M62; secp256k1_i128_rshift(&cd, 62); + e->v[3] = secp256k1_i128_to_i64(&ce) & M62; secp256k1_i128_rshift(&ce, 62); /* What remains is limb 5 of t*[d,e]+modulus*[md,me]; store it as output limb 4. */ - d->v[4] = (int64_t)cd; - e->v[4] = (int64_t)ce; + d->v[4] = secp256k1_i128_to_i64(&cd); + e->v[4] = secp256k1_i128_to_i64(&ce); #ifdef VERIFY VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(d, 5, &modinfo->modulus, -2) > 0); /* d > -2*modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(d, 5, &modinfo->modulus, 1) < 0); /* d < modulus */ @@ -389,36 +411,46 @@ static void secp256k1_modinv64_update_fg_62(secp256k1_modinv64_signed62 *f, secp const int64_t f0 = f->v[0], f1 = f->v[1], f2 = f->v[2], f3 = f->v[3], f4 = f->v[4]; const int64_t g0 = g->v[0], g1 = g->v[1], g2 = g->v[2], g3 = g->v[3], g4 = g->v[4]; const int64_t u = t->u, v = t->v, q = t->q, r = t->r; - int128_t cf, cg; + secp256k1_int128 cf, cg; /* Start computing t*[f,g]. */ - cf = (int128_t)u * f0 + (int128_t)v * g0; - cg = (int128_t)q * f0 + (int128_t)r * g0; + secp256k1_i128_mul(&cf, u, f0); + secp256k1_i128_accum_mul(&cf, v, g0); + secp256k1_i128_mul(&cg, q, f0); + secp256k1_i128_accum_mul(&cg, r, g0); /* Verify that the bottom 62 bits of the result are zero, and then throw them away. */ - VERIFY_CHECK(((int64_t)cf & M62) == 0); cf >>= 62; - VERIFY_CHECK(((int64_t)cg & M62) == 0); cg >>= 62; + VERIFY_CHECK((secp256k1_i128_to_i64(&cf) & M62) == 0); secp256k1_i128_rshift(&cf, 62); + VERIFY_CHECK((secp256k1_i128_to_i64(&cg) & M62) == 0); secp256k1_i128_rshift(&cg, 62); /* Compute limb 1 of t*[f,g], and store it as output limb 0 (= down shift). */ - cf += (int128_t)u * f1 + (int128_t)v * g1; - cg += (int128_t)q * f1 + (int128_t)r * g1; - f->v[0] = (int64_t)cf & M62; cf >>= 62; - g->v[0] = (int64_t)cg & M62; cg >>= 62; + secp256k1_i128_accum_mul(&cf, u, f1); + secp256k1_i128_accum_mul(&cf, v, g1); + secp256k1_i128_accum_mul(&cg, q, f1); + secp256k1_i128_accum_mul(&cg, r, g1); + f->v[0] = secp256k1_i128_to_i64(&cf) & M62; secp256k1_i128_rshift(&cf, 62); + g->v[0] = secp256k1_i128_to_i64(&cg) & M62; secp256k1_i128_rshift(&cg, 62); /* Compute limb 2 of t*[f,g], and store it as output limb 1. */ - cf += (int128_t)u * f2 + (int128_t)v * g2; - cg += (int128_t)q * f2 + (int128_t)r * g2; - f->v[1] = (int64_t)cf & M62; cf >>= 62; - g->v[1] = (int64_t)cg & M62; cg >>= 62; + secp256k1_i128_accum_mul(&cf, u, f2); + secp256k1_i128_accum_mul(&cf, v, g2); + secp256k1_i128_accum_mul(&cg, q, f2); + secp256k1_i128_accum_mul(&cg, r, g2); + f->v[1] = secp256k1_i128_to_i64(&cf) & M62; secp256k1_i128_rshift(&cf, 62); + g->v[1] = secp256k1_i128_to_i64(&cg) & M62; secp256k1_i128_rshift(&cg, 62); /* Compute limb 3 of t*[f,g], and store it as output limb 2. */ - cf += (int128_t)u * f3 + (int128_t)v * g3; - cg += (int128_t)q * f3 + (int128_t)r * g3; - f->v[2] = (int64_t)cf & M62; cf >>= 62; - g->v[2] = (int64_t)cg & M62; cg >>= 62; + secp256k1_i128_accum_mul(&cf, u, f3); + secp256k1_i128_accum_mul(&cf, v, g3); + secp256k1_i128_accum_mul(&cg, q, f3); + secp256k1_i128_accum_mul(&cg, r, g3); + f->v[2] = secp256k1_i128_to_i64(&cf) & M62; secp256k1_i128_rshift(&cf, 62); + g->v[2] = secp256k1_i128_to_i64(&cg) & M62; secp256k1_i128_rshift(&cg, 62); /* Compute limb 4 of t*[f,g], and store it as output limb 3. */ - cf += (int128_t)u * f4 + (int128_t)v * g4; - cg += (int128_t)q * f4 + (int128_t)r * g4; - f->v[3] = (int64_t)cf & M62; cf >>= 62; - g->v[3] = (int64_t)cg & M62; cg >>= 62; + secp256k1_i128_accum_mul(&cf, u, f4); + secp256k1_i128_accum_mul(&cf, v, g4); + secp256k1_i128_accum_mul(&cg, q, f4); + secp256k1_i128_accum_mul(&cg, r, g4); + f->v[3] = secp256k1_i128_to_i64(&cf) & M62; secp256k1_i128_rshift(&cf, 62); + g->v[3] = secp256k1_i128_to_i64(&cg) & M62; secp256k1_i128_rshift(&cg, 62); /* What remains is limb 5 of t*[f,g]; store it as output limb 4. */ - f->v[4] = (int64_t)cf; - g->v[4] = (int64_t)cg; + f->v[4] = secp256k1_i128_to_i64(&cf); + g->v[4] = secp256k1_i128_to_i64(&cg); } /* Compute (t/2^62) * [f, g], where t is a transition matrix for 62 divsteps. @@ -431,30 +463,34 @@ static void secp256k1_modinv64_update_fg_62_var(int len, secp256k1_modinv64_sign const int64_t M62 = (int64_t)(UINT64_MAX >> 2); const int64_t u = t->u, v = t->v, q = t->q, r = t->r; int64_t fi, gi; - int128_t cf, cg; + secp256k1_int128 cf, cg; int i; VERIFY_CHECK(len > 0); /* Start computing t*[f,g]. */ fi = f->v[0]; gi = g->v[0]; - cf = (int128_t)u * fi + (int128_t)v * gi; - cg = (int128_t)q * fi + (int128_t)r * gi; + secp256k1_i128_mul(&cf, u, fi); + secp256k1_i128_accum_mul(&cf, v, gi); + secp256k1_i128_mul(&cg, q, fi); + secp256k1_i128_accum_mul(&cg, r, gi); /* Verify that the bottom 62 bits of the result are zero, and then throw them away. */ - VERIFY_CHECK(((int64_t)cf & M62) == 0); cf >>= 62; - VERIFY_CHECK(((int64_t)cg & M62) == 0); cg >>= 62; + VERIFY_CHECK((secp256k1_i128_to_i64(&cf) & M62) == 0); secp256k1_i128_rshift(&cf, 62); + VERIFY_CHECK((secp256k1_i128_to_i64(&cg) & M62) == 0); secp256k1_i128_rshift(&cg, 62); /* Now iteratively compute limb i=1..len of t*[f,g], and store them in output limb i-1 (shifting * down by 62 bits). */ for (i = 1; i < len; ++i) { fi = f->v[i]; gi = g->v[i]; - cf += (int128_t)u * fi + (int128_t)v * gi; - cg += (int128_t)q * fi + (int128_t)r * gi; - f->v[i - 1] = (int64_t)cf & M62; cf >>= 62; - g->v[i - 1] = (int64_t)cg & M62; cg >>= 62; + secp256k1_i128_accum_mul(&cf, u, fi); + secp256k1_i128_accum_mul(&cf, v, gi); + secp256k1_i128_accum_mul(&cg, q, fi); + secp256k1_i128_accum_mul(&cg, r, gi); + f->v[i - 1] = secp256k1_i128_to_i64(&cf) & M62; secp256k1_i128_rshift(&cf, 62); + g->v[i - 1] = secp256k1_i128_to_i64(&cg) & M62; secp256k1_i128_rshift(&cg, 62); } /* What remains is limb (len) of t*[f,g]; store it as output limb (len-1). */ - f->v[len - 1] = (int64_t)cf; - g->v[len - 1] = (int64_t)cg; + f->v[len - 1] = secp256k1_i128_to_i64(&cf); + g->v[len - 1] = secp256k1_i128_to_i64(&cg); } /* Compute the inverse of x modulo modinfo->modulus, and replace x with it (constant time in x). */ diff --git a/src/secp256k1/src/modules/ecdh/bench_impl.h b/src/secp256k1/src/modules/ecdh/bench_impl.h index 94d833462f..8df15bcf43 100644 --- a/src/secp256k1/src/modules/ecdh/bench_impl.h +++ b/src/secp256k1/src/modules/ecdh/bench_impl.h @@ -7,7 +7,7 @@ #ifndef SECP256K1_MODULE_ECDH_BENCH_H #define SECP256K1_MODULE_ECDH_BENCH_H -#include "../include/secp256k1_ecdh.h" +#include "../../../include/secp256k1_ecdh.h" typedef struct { secp256k1_context *ctx; diff --git a/src/secp256k1/src/modules/ecdh/tests_impl.h b/src/secp256k1/src/modules/ecdh/tests_impl.h index 10b7075c38..ce644d572a 100644 --- a/src/secp256k1/src/modules/ecdh/tests_impl.h +++ b/src/secp256k1/src/modules/ecdh/tests_impl.h @@ -26,7 +26,7 @@ int ecdh_hash_function_custom(unsigned char *output, const unsigned char *x, con void test_ecdh_api(void) { /* Setup context that just counts errors */ - secp256k1_context *tctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + secp256k1_context *tctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); secp256k1_pubkey point; unsigned char res[32]; unsigned char s_one[32] = { 0 }; diff --git a/src/secp256k1/src/modules/extrakeys/tests_exhaustive_impl.h b/src/secp256k1/src/modules/extrakeys/tests_exhaustive_impl.h index d4a2f5bdf4..5ecc90d50f 100644 --- a/src/secp256k1/src/modules/extrakeys/tests_exhaustive_impl.h +++ b/src/secp256k1/src/modules/extrakeys/tests_exhaustive_impl.h @@ -7,8 +7,8 @@ #ifndef SECP256K1_MODULE_EXTRAKEYS_TESTS_EXHAUSTIVE_H #define SECP256K1_MODULE_EXTRAKEYS_TESTS_EXHAUSTIVE_H -#include "src/modules/extrakeys/main_impl.h" #include "../../../include/secp256k1_extrakeys.h" +#include "main_impl.h" static void test_exhaustive_extrakeys(const secp256k1_context *ctx, const secp256k1_ge* group) { secp256k1_keypair keypair[EXHAUSTIVE_TEST_ORDER - 1]; diff --git a/src/secp256k1/src/modules/extrakeys/tests_impl.h b/src/secp256k1/src/modules/extrakeys/tests_impl.h index c8a99f4466..8030aedad6 100644 --- a/src/secp256k1/src/modules/extrakeys/tests_impl.h +++ b/src/secp256k1/src/modules/extrakeys/tests_impl.h @@ -9,11 +9,9 @@ #include "../../../include/secp256k1_extrakeys.h" -static secp256k1_context* api_test_context(int flags, int *ecount) { - secp256k1_context *ctx0 = secp256k1_context_create(flags); +static void set_counting_callbacks(secp256k1_context *ctx0, int *ecount) { secp256k1_context_set_error_callback(ctx0, counting_illegal_callback_fn, ecount); secp256k1_context_set_illegal_callback(ctx0, counting_illegal_callback_fn, ecount); - return ctx0; } void test_xonly_pubkey(void) { @@ -31,28 +29,25 @@ void test_xonly_pubkey(void) { int i; int ecount; - secp256k1_context *none = api_test_context(SECP256K1_CONTEXT_NONE, &ecount); - secp256k1_context *sign = api_test_context(SECP256K1_CONTEXT_SIGN, &ecount); - secp256k1_context *verify = api_test_context(SECP256K1_CONTEXT_VERIFY, &ecount); + + set_counting_callbacks(ctx, &ecount); secp256k1_testrand256(sk); memset(ones32, 0xFF, 32); secp256k1_testrand256(xy_sk); - CHECK(secp256k1_ec_pubkey_create(sign, &pk, sk) == 1); - CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, &pk_parity, &pk) == 1); + CHECK(secp256k1_ec_pubkey_create(ctx, &pk, sk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pk, &pk_parity, &pk) == 1); /* Test xonly_pubkey_from_pubkey */ ecount = 0; - CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, &pk_parity, &pk) == 1); - CHECK(secp256k1_xonly_pubkey_from_pubkey(sign, &xonly_pk, &pk_parity, &pk) == 1); - CHECK(secp256k1_xonly_pubkey_from_pubkey(verify, &xonly_pk, &pk_parity, &pk) == 1); - CHECK(secp256k1_xonly_pubkey_from_pubkey(none, NULL, &pk_parity, &pk) == 0); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pk, &pk_parity, &pk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, NULL, &pk_parity, &pk) == 0); CHECK(ecount == 1); - CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, NULL, &pk) == 1); - CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, &pk_parity, NULL) == 0); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pk, NULL, &pk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pk, &pk_parity, NULL) == 0); CHECK(ecount == 2); memset(&pk, 0, sizeof(pk)); - CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, &pk_parity, &pk) == 0); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pk, &pk_parity, &pk) == 0); CHECK(ecount == 3); /* Choose a secret key such that the resulting pubkey and xonly_pubkey match. */ @@ -78,9 +73,9 @@ void test_xonly_pubkey(void) { /* Test xonly_pubkey_serialize and xonly_pubkey_parse */ ecount = 0; - CHECK(secp256k1_xonly_pubkey_serialize(none, NULL, &xonly_pk) == 0); + CHECK(secp256k1_xonly_pubkey_serialize(ctx, NULL, &xonly_pk) == 0); CHECK(ecount == 1); - CHECK(secp256k1_xonly_pubkey_serialize(none, buf32, NULL) == 0); + CHECK(secp256k1_xonly_pubkey_serialize(ctx, buf32, NULL) == 0); CHECK(secp256k1_memcmp_var(buf32, zeros64, 32) == 0); CHECK(ecount == 2); { @@ -88,20 +83,20 @@ void test_xonly_pubkey(void) { * special casing. */ secp256k1_xonly_pubkey pk_tmp; memset(&pk_tmp, 0, sizeof(pk_tmp)); - CHECK(secp256k1_xonly_pubkey_serialize(none, buf32, &pk_tmp) == 0); + CHECK(secp256k1_xonly_pubkey_serialize(ctx, buf32, &pk_tmp) == 0); } /* pubkey_load called illegal callback */ CHECK(ecount == 3); - CHECK(secp256k1_xonly_pubkey_serialize(none, buf32, &xonly_pk) == 1); + CHECK(secp256k1_xonly_pubkey_serialize(ctx, buf32, &xonly_pk) == 1); ecount = 0; - CHECK(secp256k1_xonly_pubkey_parse(none, NULL, buf32) == 0); + CHECK(secp256k1_xonly_pubkey_parse(ctx, NULL, buf32) == 0); CHECK(ecount == 1); - CHECK(secp256k1_xonly_pubkey_parse(none, &xonly_pk, NULL) == 0); + CHECK(secp256k1_xonly_pubkey_parse(ctx, &xonly_pk, NULL) == 0); CHECK(ecount == 2); /* Serialization and parse roundtrip */ - CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, NULL, &pk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pk, NULL, &pk) == 1); CHECK(secp256k1_xonly_pubkey_serialize(ctx, buf32, &xonly_pk) == 1); CHECK(secp256k1_xonly_pubkey_parse(ctx, &xonly_pk_tmp, buf32) == 1); CHECK(secp256k1_memcmp_var(&xonly_pk, &xonly_pk_tmp, sizeof(xonly_pk)) == 0); @@ -109,11 +104,11 @@ void test_xonly_pubkey(void) { /* Test parsing invalid field elements */ memset(&xonly_pk, 1, sizeof(xonly_pk)); /* Overflowing field element */ - CHECK(secp256k1_xonly_pubkey_parse(none, &xonly_pk, ones32) == 0); + CHECK(secp256k1_xonly_pubkey_parse(ctx, &xonly_pk, ones32) == 0); CHECK(secp256k1_memcmp_var(&xonly_pk, zeros64, sizeof(xonly_pk)) == 0); memset(&xonly_pk, 1, sizeof(xonly_pk)); /* There's no point with x-coordinate 0 on secp256k1 */ - CHECK(secp256k1_xonly_pubkey_parse(none, &xonly_pk, zeros64) == 0); + CHECK(secp256k1_xonly_pubkey_parse(ctx, &xonly_pk, zeros64) == 0); CHECK(secp256k1_memcmp_var(&xonly_pk, zeros64, sizeof(xonly_pk)) == 0); /* If a random 32-byte string can not be parsed with ec_pubkey_parse * (because interpreted as X coordinate it does not correspond to a point on @@ -131,10 +126,6 @@ void test_xonly_pubkey(void) { } } CHECK(ecount == 2); - - secp256k1_context_destroy(none); - secp256k1_context_destroy(sign); - secp256k1_context_destroy(verify); } void test_xonly_pubkey_comparison(void) { @@ -149,29 +140,28 @@ void test_xonly_pubkey_comparison(void) { secp256k1_xonly_pubkey pk1; secp256k1_xonly_pubkey pk2; int ecount = 0; - secp256k1_context *none = api_test_context(SECP256K1_CONTEXT_NONE, &ecount); - CHECK(secp256k1_xonly_pubkey_parse(none, &pk1, pk1_ser) == 1); - CHECK(secp256k1_xonly_pubkey_parse(none, &pk2, pk2_ser) == 1); + set_counting_callbacks(ctx, &ecount); - CHECK(secp256k1_xonly_pubkey_cmp(none, NULL, &pk2) < 0); + CHECK(secp256k1_xonly_pubkey_parse(ctx, &pk1, pk1_ser) == 1); + CHECK(secp256k1_xonly_pubkey_parse(ctx, &pk2, pk2_ser) == 1); + + CHECK(secp256k1_xonly_pubkey_cmp(ctx, NULL, &pk2) < 0); CHECK(ecount == 1); - CHECK(secp256k1_xonly_pubkey_cmp(none, &pk1, NULL) > 0); + CHECK(secp256k1_xonly_pubkey_cmp(ctx, &pk1, NULL) > 0); CHECK(ecount == 2); - CHECK(secp256k1_xonly_pubkey_cmp(none, &pk1, &pk2) < 0); - CHECK(secp256k1_xonly_pubkey_cmp(none, &pk2, &pk1) > 0); - CHECK(secp256k1_xonly_pubkey_cmp(none, &pk1, &pk1) == 0); - CHECK(secp256k1_xonly_pubkey_cmp(none, &pk2, &pk2) == 0); + CHECK(secp256k1_xonly_pubkey_cmp(ctx, &pk1, &pk2) < 0); + CHECK(secp256k1_xonly_pubkey_cmp(ctx, &pk2, &pk1) > 0); + CHECK(secp256k1_xonly_pubkey_cmp(ctx, &pk1, &pk1) == 0); + CHECK(secp256k1_xonly_pubkey_cmp(ctx, &pk2, &pk2) == 0); CHECK(ecount == 2); memset(&pk1, 0, sizeof(pk1)); /* illegal pubkey */ - CHECK(secp256k1_xonly_pubkey_cmp(none, &pk1, &pk2) < 0); + CHECK(secp256k1_xonly_pubkey_cmp(ctx, &pk1, &pk2) < 0); CHECK(ecount == 3); - CHECK(secp256k1_xonly_pubkey_cmp(none, &pk1, &pk1) == 0); + CHECK(secp256k1_xonly_pubkey_cmp(ctx, &pk1, &pk1) == 0); CHECK(ecount == 5); - CHECK(secp256k1_xonly_pubkey_cmp(none, &pk2, &pk1) > 0); + CHECK(secp256k1_xonly_pubkey_cmp(ctx, &pk2, &pk1) > 0); CHECK(ecount == 6); - - secp256k1_context_destroy(none); } void test_xonly_pubkey_tweak(void) { @@ -186,39 +176,38 @@ void test_xonly_pubkey_tweak(void) { int i; int ecount; - secp256k1_context *none = api_test_context(SECP256K1_CONTEXT_NONE, &ecount); - secp256k1_context *sign = api_test_context(SECP256K1_CONTEXT_SIGN, &ecount); - secp256k1_context *verify = api_test_context(SECP256K1_CONTEXT_VERIFY, &ecount); + + set_counting_callbacks(ctx, &ecount); memset(overflows, 0xff, sizeof(overflows)); secp256k1_testrand256(tweak); secp256k1_testrand256(sk); CHECK(secp256k1_ec_pubkey_create(ctx, &internal_pk, sk) == 1); - CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &internal_xonly_pk, &pk_parity, &internal_pk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &internal_xonly_pk, &pk_parity, &internal_pk) == 1); ecount = 0; - CHECK(secp256k1_xonly_pubkey_tweak_add(none, &output_pk, &internal_xonly_pk, tweak) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, &internal_xonly_pk, tweak) == 1); CHECK(ecount == 0); - CHECK(secp256k1_xonly_pubkey_tweak_add(sign, &output_pk, &internal_xonly_pk, tweak) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, &internal_xonly_pk, tweak) == 1); CHECK(ecount == 0); - CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, tweak) == 1); - CHECK(secp256k1_xonly_pubkey_tweak_add(verify, NULL, &internal_xonly_pk, tweak) == 0); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, &internal_xonly_pk, tweak) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, NULL, &internal_xonly_pk, tweak) == 0); CHECK(ecount == 1); - CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, NULL, tweak) == 0); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, NULL, tweak) == 0); CHECK(ecount == 2); /* NULL internal_xonly_pk zeroes the output_pk */ CHECK(secp256k1_memcmp_var(&output_pk, zeros64, sizeof(output_pk)) == 0); - CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, NULL) == 0); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, &internal_xonly_pk, NULL) == 0); CHECK(ecount == 3); /* NULL tweak zeroes the output_pk */ CHECK(secp256k1_memcmp_var(&output_pk, zeros64, sizeof(output_pk)) == 0); /* Invalid tweak zeroes the output_pk */ - CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, overflows) == 0); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, &internal_xonly_pk, overflows) == 0); CHECK(secp256k1_memcmp_var(&output_pk, zeros64, sizeof(output_pk)) == 0); /* A zero tweak is fine */ - CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, zeros64) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, &internal_xonly_pk, zeros64) == 1); /* Fails if the resulting key was infinity */ for (i = 0; i < count; i++) { @@ -228,8 +217,8 @@ void test_xonly_pubkey_tweak(void) { secp256k1_scalar_set_b32(&scalar_tweak, sk, NULL); secp256k1_scalar_negate(&scalar_tweak, &scalar_tweak); secp256k1_scalar_get_b32(tweak, &scalar_tweak); - CHECK((secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, sk) == 0) - || (secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, tweak) == 0)); + CHECK((secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, &internal_xonly_pk, sk) == 0) + || (secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, &internal_xonly_pk, tweak) == 0)); CHECK(secp256k1_memcmp_var(&output_pk, zeros64, sizeof(output_pk)) == 0); } @@ -237,13 +226,9 @@ void test_xonly_pubkey_tweak(void) { memset(&internal_xonly_pk, 0, sizeof(internal_xonly_pk)); secp256k1_testrand256(tweak); ecount = 0; - CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, tweak) == 0); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, &internal_xonly_pk, tweak) == 0); CHECK(ecount == 1); CHECK(secp256k1_memcmp_var(&output_pk, zeros64, sizeof(output_pk)) == 0); - - secp256k1_context_destroy(none); - secp256k1_context_destroy(sign); - secp256k1_context_destroy(verify); } void test_xonly_pubkey_tweak_check(void) { @@ -260,33 +245,32 @@ void test_xonly_pubkey_tweak_check(void) { unsigned char tweak[32]; int ecount; - secp256k1_context *none = api_test_context(SECP256K1_CONTEXT_NONE, &ecount); - secp256k1_context *sign = api_test_context(SECP256K1_CONTEXT_SIGN, &ecount); - secp256k1_context *verify = api_test_context(SECP256K1_CONTEXT_VERIFY, &ecount); + + set_counting_callbacks(ctx, &ecount); memset(overflows, 0xff, sizeof(overflows)); secp256k1_testrand256(tweak); secp256k1_testrand256(sk); CHECK(secp256k1_ec_pubkey_create(ctx, &internal_pk, sk) == 1); - CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &internal_xonly_pk, &pk_parity, &internal_pk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &internal_xonly_pk, &pk_parity, &internal_pk) == 1); ecount = 0; - CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, tweak) == 1); - CHECK(secp256k1_xonly_pubkey_from_pubkey(verify, &output_xonly_pk, &pk_parity, &output_pk) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, &internal_xonly_pk, tweak) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &output_xonly_pk, &pk_parity, &output_pk) == 1); CHECK(secp256k1_xonly_pubkey_serialize(ctx, buf32, &output_xonly_pk) == 1); - CHECK(secp256k1_xonly_pubkey_tweak_add_check(none, buf32, pk_parity, &internal_xonly_pk, tweak) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, buf32, pk_parity, &internal_xonly_pk, tweak) == 1); CHECK(ecount == 0); - CHECK(secp256k1_xonly_pubkey_tweak_add_check(sign, buf32, pk_parity, &internal_xonly_pk, tweak) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, buf32, pk_parity, &internal_xonly_pk, tweak) == 1); CHECK(ecount == 0); - CHECK(secp256k1_xonly_pubkey_tweak_add_check(verify, buf32, pk_parity, &internal_xonly_pk, tweak) == 1); - CHECK(secp256k1_xonly_pubkey_tweak_add_check(verify, NULL, pk_parity, &internal_xonly_pk, tweak) == 0); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, buf32, pk_parity, &internal_xonly_pk, tweak) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, NULL, pk_parity, &internal_xonly_pk, tweak) == 0); CHECK(ecount == 1); /* invalid pk_parity value */ - CHECK(secp256k1_xonly_pubkey_tweak_add_check(verify, buf32, 2, &internal_xonly_pk, tweak) == 0); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, buf32, 2, &internal_xonly_pk, tweak) == 0); CHECK(ecount == 1); - CHECK(secp256k1_xonly_pubkey_tweak_add_check(verify, buf32, pk_parity, NULL, tweak) == 0); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, buf32, pk_parity, NULL, tweak) == 0); CHECK(ecount == 2); - CHECK(secp256k1_xonly_pubkey_tweak_add_check(verify, buf32, pk_parity, &internal_xonly_pk, NULL) == 0); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, buf32, pk_parity, &internal_xonly_pk, NULL) == 0); CHECK(ecount == 3); memset(tweak, 1, sizeof(tweak)); @@ -307,10 +291,6 @@ void test_xonly_pubkey_tweak_check(void) { CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, &internal_xonly_pk, overflows) == 0); CHECK(secp256k1_memcmp_var(&output_pk, zeros64, sizeof(output_pk)) == 0); CHECK(ecount == 3); - - secp256k1_context_destroy(none); - secp256k1_context_destroy(sign); - secp256k1_context_destroy(verify); } /* Starts with an initial pubkey and recursively creates N_PUBKEYS - 1 @@ -356,12 +336,10 @@ void test_keypair(void) { secp256k1_xonly_pubkey xonly_pk, xonly_pk_tmp; int pk_parity, pk_parity_tmp; int ecount; - secp256k1_context *none = api_test_context(SECP256K1_CONTEXT_NONE, &ecount); - secp256k1_context *sign = api_test_context(SECP256K1_CONTEXT_SIGN, &ecount); - secp256k1_context *verify = api_test_context(SECP256K1_CONTEXT_VERIFY, &ecount); - secp256k1_context *sttc = secp256k1_context_clone(secp256k1_context_no_precomp); - secp256k1_context_set_error_callback(sttc, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(sttc, counting_illegal_callback_fn, &ecount); + secp256k1_context *sttc = secp256k1_context_clone(secp256k1_context_static); + + set_counting_callbacks(ctx, &ecount); + set_counting_callbacks(sttc, &ecount); CHECK(sizeof(zeros96) == sizeof(keypair)); memset(overflows, 0xFF, sizeof(overflows)); @@ -369,75 +347,75 @@ void test_keypair(void) { /* Test keypair_create */ ecount = 0; secp256k1_testrand256(sk); - CHECK(secp256k1_keypair_create(none, &keypair, sk) == 1); + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); CHECK(secp256k1_memcmp_var(zeros96, &keypair, sizeof(keypair)) != 0); CHECK(ecount == 0); - CHECK(secp256k1_keypair_create(verify, &keypair, sk) == 1); + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); CHECK(secp256k1_memcmp_var(zeros96, &keypair, sizeof(keypair)) != 0); CHECK(ecount == 0); - CHECK(secp256k1_keypair_create(sign, NULL, sk) == 0); + CHECK(secp256k1_keypair_create(ctx, NULL, sk) == 0); CHECK(ecount == 1); - CHECK(secp256k1_keypair_create(sign, &keypair, NULL) == 0); + CHECK(secp256k1_keypair_create(ctx, &keypair, NULL) == 0); CHECK(secp256k1_memcmp_var(zeros96, &keypair, sizeof(keypair)) == 0); CHECK(ecount == 2); - CHECK(secp256k1_keypair_create(sign, &keypair, sk) == 1); + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); CHECK(ecount == 2); CHECK(secp256k1_keypair_create(sttc, &keypair, sk) == 0); CHECK(secp256k1_memcmp_var(zeros96, &keypair, sizeof(keypair)) == 0); CHECK(ecount == 3); /* Invalid secret key */ - CHECK(secp256k1_keypair_create(sign, &keypair, zeros96) == 0); + CHECK(secp256k1_keypair_create(ctx, &keypair, zeros96) == 0); CHECK(secp256k1_memcmp_var(zeros96, &keypair, sizeof(keypair)) == 0); - CHECK(secp256k1_keypair_create(sign, &keypair, overflows) == 0); + CHECK(secp256k1_keypair_create(ctx, &keypair, overflows) == 0); CHECK(secp256k1_memcmp_var(zeros96, &keypair, sizeof(keypair)) == 0); /* Test keypair_pub */ ecount = 0; secp256k1_testrand256(sk); CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); - CHECK(secp256k1_keypair_pub(none, &pk, &keypair) == 1); - CHECK(secp256k1_keypair_pub(none, NULL, &keypair) == 0); + CHECK(secp256k1_keypair_pub(ctx, &pk, &keypair) == 1); + CHECK(secp256k1_keypair_pub(ctx, NULL, &keypair) == 0); CHECK(ecount == 1); - CHECK(secp256k1_keypair_pub(none, &pk, NULL) == 0); + CHECK(secp256k1_keypair_pub(ctx, &pk, NULL) == 0); CHECK(ecount == 2); CHECK(secp256k1_memcmp_var(zeros96, &pk, sizeof(pk)) == 0); /* Using an invalid keypair is fine for keypair_pub */ memset(&keypair, 0, sizeof(keypair)); - CHECK(secp256k1_keypair_pub(none, &pk, &keypair) == 1); + CHECK(secp256k1_keypair_pub(ctx, &pk, &keypair) == 1); CHECK(secp256k1_memcmp_var(zeros96, &pk, sizeof(pk)) == 0); /* keypair holds the same pubkey as pubkey_create */ - CHECK(secp256k1_ec_pubkey_create(sign, &pk, sk) == 1); - CHECK(secp256k1_keypair_create(sign, &keypair, sk) == 1); - CHECK(secp256k1_keypair_pub(none, &pk_tmp, &keypair) == 1); + CHECK(secp256k1_ec_pubkey_create(ctx, &pk, sk) == 1); + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); + CHECK(secp256k1_keypair_pub(ctx, &pk_tmp, &keypair) == 1); CHECK(secp256k1_memcmp_var(&pk, &pk_tmp, sizeof(pk)) == 0); /** Test keypair_xonly_pub **/ ecount = 0; secp256k1_testrand256(sk); CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); - CHECK(secp256k1_keypair_xonly_pub(none, &xonly_pk, &pk_parity, &keypair) == 1); - CHECK(secp256k1_keypair_xonly_pub(none, NULL, &pk_parity, &keypair) == 0); + CHECK(secp256k1_keypair_xonly_pub(ctx, &xonly_pk, &pk_parity, &keypair) == 1); + CHECK(secp256k1_keypair_xonly_pub(ctx, NULL, &pk_parity, &keypair) == 0); CHECK(ecount == 1); - CHECK(secp256k1_keypair_xonly_pub(none, &xonly_pk, NULL, &keypair) == 1); - CHECK(secp256k1_keypair_xonly_pub(none, &xonly_pk, &pk_parity, NULL) == 0); + CHECK(secp256k1_keypair_xonly_pub(ctx, &xonly_pk, NULL, &keypair) == 1); + CHECK(secp256k1_keypair_xonly_pub(ctx, &xonly_pk, &pk_parity, NULL) == 0); CHECK(ecount == 2); CHECK(secp256k1_memcmp_var(zeros96, &xonly_pk, sizeof(xonly_pk)) == 0); /* Using an invalid keypair will set the xonly_pk to 0 (first reset * xonly_pk). */ - CHECK(secp256k1_keypair_xonly_pub(none, &xonly_pk, &pk_parity, &keypair) == 1); + CHECK(secp256k1_keypair_xonly_pub(ctx, &xonly_pk, &pk_parity, &keypair) == 1); memset(&keypair, 0, sizeof(keypair)); - CHECK(secp256k1_keypair_xonly_pub(none, &xonly_pk, &pk_parity, &keypair) == 0); + CHECK(secp256k1_keypair_xonly_pub(ctx, &xonly_pk, &pk_parity, &keypair) == 0); CHECK(secp256k1_memcmp_var(zeros96, &xonly_pk, sizeof(xonly_pk)) == 0); CHECK(ecount == 3); /** keypair holds the same xonly pubkey as pubkey_create **/ - CHECK(secp256k1_ec_pubkey_create(sign, &pk, sk) == 1); - CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, &pk_parity, &pk) == 1); - CHECK(secp256k1_keypair_create(sign, &keypair, sk) == 1); - CHECK(secp256k1_keypair_xonly_pub(none, &xonly_pk_tmp, &pk_parity_tmp, &keypair) == 1); + CHECK(secp256k1_ec_pubkey_create(ctx, &pk, sk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pk, &pk_parity, &pk) == 1); + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); + CHECK(secp256k1_keypair_xonly_pub(ctx, &xonly_pk_tmp, &pk_parity_tmp, &keypair) == 1); CHECK(secp256k1_memcmp_var(&xonly_pk, &xonly_pk_tmp, sizeof(pk)) == 0); CHECK(pk_parity == pk_parity_tmp); @@ -445,27 +423,23 @@ void test_keypair(void) { ecount = 0; secp256k1_testrand256(sk); CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); - CHECK(secp256k1_keypair_sec(none, sk_tmp, &keypair) == 1); - CHECK(secp256k1_keypair_sec(none, NULL, &keypair) == 0); + CHECK(secp256k1_keypair_sec(ctx, sk_tmp, &keypair) == 1); + CHECK(secp256k1_keypair_sec(ctx, NULL, &keypair) == 0); CHECK(ecount == 1); - CHECK(secp256k1_keypair_sec(none, sk_tmp, NULL) == 0); + CHECK(secp256k1_keypair_sec(ctx, sk_tmp, NULL) == 0); CHECK(ecount == 2); CHECK(secp256k1_memcmp_var(zeros96, sk_tmp, sizeof(sk_tmp)) == 0); /* keypair returns the same seckey it got */ - CHECK(secp256k1_keypair_create(sign, &keypair, sk) == 1); - CHECK(secp256k1_keypair_sec(none, sk_tmp, &keypair) == 1); + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); + CHECK(secp256k1_keypair_sec(ctx, sk_tmp, &keypair) == 1); CHECK(secp256k1_memcmp_var(sk, sk_tmp, sizeof(sk_tmp)) == 0); /* Using an invalid keypair is fine for keypair_seckey */ memset(&keypair, 0, sizeof(keypair)); - CHECK(secp256k1_keypair_sec(none, sk_tmp, &keypair) == 1); + CHECK(secp256k1_keypair_sec(ctx, sk_tmp, &keypair) == 1); CHECK(secp256k1_memcmp_var(zeros96, sk_tmp, sizeof(sk_tmp)) == 0); - - secp256k1_context_destroy(none); - secp256k1_context_destroy(sign); - secp256k1_context_destroy(verify); secp256k1_context_destroy(sttc); } @@ -477,9 +451,8 @@ void test_keypair_add(void) { unsigned char tweak[32]; int i; int ecount = 0; - secp256k1_context *none = api_test_context(SECP256K1_CONTEXT_NONE, &ecount); - secp256k1_context *sign = api_test_context(SECP256K1_CONTEXT_SIGN, &ecount); - secp256k1_context *verify = api_test_context(SECP256K1_CONTEXT_VERIFY, &ecount); + + set_counting_callbacks(ctx, &ecount); CHECK(sizeof(zeros96) == sizeof(keypair)); secp256k1_testrand256(sk); @@ -487,14 +460,14 @@ void test_keypair_add(void) { memset(overflows, 0xFF, 32); CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); - CHECK(secp256k1_keypair_xonly_tweak_add(none, &keypair, tweak) == 1); + CHECK(secp256k1_keypair_xonly_tweak_add(ctx, &keypair, tweak) == 1); CHECK(ecount == 0); - CHECK(secp256k1_keypair_xonly_tweak_add(sign, &keypair, tweak) == 1); + CHECK(secp256k1_keypair_xonly_tweak_add(ctx, &keypair, tweak) == 1); CHECK(ecount == 0); - CHECK(secp256k1_keypair_xonly_tweak_add(verify, &keypair, tweak) == 1); - CHECK(secp256k1_keypair_xonly_tweak_add(verify, NULL, tweak) == 0); + CHECK(secp256k1_keypair_xonly_tweak_add(ctx, &keypair, tweak) == 1); + CHECK(secp256k1_keypair_xonly_tweak_add(ctx, NULL, tweak) == 0); CHECK(ecount == 1); - CHECK(secp256k1_keypair_xonly_tweak_add(verify, &keypair, NULL) == 0); + CHECK(secp256k1_keypair_xonly_tweak_add(ctx, &keypair, NULL) == 0); CHECK(ecount == 2); /* This does not set the keypair to zeroes */ CHECK(secp256k1_memcmp_var(&keypair, zeros96, sizeof(keypair)) != 0); @@ -530,18 +503,18 @@ void test_keypair_add(void) { memset(&keypair, 0, sizeof(keypair)); secp256k1_testrand256(tweak); ecount = 0; - CHECK(secp256k1_keypair_xonly_tweak_add(verify, &keypair, tweak) == 0); + CHECK(secp256k1_keypair_xonly_tweak_add(ctx, &keypair, tweak) == 0); CHECK(ecount == 1); CHECK(secp256k1_memcmp_var(&keypair, zeros96, sizeof(keypair)) == 0); /* Only seckey part of keypair invalid */ CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); memset(&keypair, 0, 32); - CHECK(secp256k1_keypair_xonly_tweak_add(verify, &keypair, tweak) == 0); + CHECK(secp256k1_keypair_xonly_tweak_add(ctx, &keypair, tweak) == 0); CHECK(ecount == 2); /* Only pubkey part of keypair invalid */ CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); memset(&keypair.data[32], 0, 64); - CHECK(secp256k1_keypair_xonly_tweak_add(verify, &keypair, tweak) == 0); + CHECK(secp256k1_keypair_xonly_tweak_add(ctx, &keypair, tweak) == 0); CHECK(ecount == 3); /* Check that the keypair_tweak_add implementation is correct */ @@ -570,13 +543,10 @@ void test_keypair_add(void) { CHECK(secp256k1_memcmp_var(&output_pk_xy, &output_pk_expected, sizeof(output_pk_xy)) == 0); /* Check that the secret key in the keypair is tweaked correctly */ - CHECK(secp256k1_keypair_sec(none, sk32, &keypair) == 1); + CHECK(secp256k1_keypair_sec(ctx, sk32, &keypair) == 1); CHECK(secp256k1_ec_pubkey_create(ctx, &output_pk_expected, sk32) == 1); CHECK(secp256k1_memcmp_var(&output_pk_xy, &output_pk_expected, sizeof(output_pk_xy)) == 0); } - secp256k1_context_destroy(none); - secp256k1_context_destroy(sign); - secp256k1_context_destroy(verify); } void run_extrakeys_tests(void) { diff --git a/src/secp256k1/src/modules/recovery/bench_impl.h b/src/secp256k1/src/modules/recovery/bench_impl.h index 4a9e886910..ffa00df479 100644 --- a/src/secp256k1/src/modules/recovery/bench_impl.h +++ b/src/secp256k1/src/modules/recovery/bench_impl.h @@ -7,7 +7,7 @@ #ifndef SECP256K1_MODULE_RECOVERY_BENCH_H #define SECP256K1_MODULE_RECOVERY_BENCH_H -#include "../include/secp256k1_recovery.h" +#include "../../../include/secp256k1_recovery.h" typedef struct { secp256k1_context *ctx; @@ -52,7 +52,7 @@ void run_recovery_bench(int iters, int argc, char** argv) { bench_recover_data data; int d = argc == 1; - data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); if (d || have_flag(argc, argv, "ecdsa") || have_flag(argc, argv, "recover") || have_flag(argc, argv, "ecdsa_recover")) run_benchmark("ecdsa_recover", bench_recover, bench_recover_setup, NULL, &data, 10, iters); diff --git a/src/secp256k1/src/modules/recovery/tests_exhaustive_impl.h b/src/secp256k1/src/modules/recovery/tests_exhaustive_impl.h index 590a972ed3..ed9386b6f8 100644 --- a/src/secp256k1/src/modules/recovery/tests_exhaustive_impl.h +++ b/src/secp256k1/src/modules/recovery/tests_exhaustive_impl.h @@ -7,7 +7,7 @@ #ifndef SECP256K1_MODULE_RECOVERY_EXHAUSTIVE_TESTS_H #define SECP256K1_MODULE_RECOVERY_EXHAUSTIVE_TESTS_H -#include "src/modules/recovery/main_impl.h" +#include "main_impl.h" #include "../../../include/secp256k1_recovery.h" void test_exhaustive_recovery_sign(const secp256k1_context *ctx, const secp256k1_ge *group) { diff --git a/src/secp256k1/src/modules/recovery/tests_impl.h b/src/secp256k1/src/modules/recovery/tests_impl.h index abf62f7f3a..0ff9294e38 100644 --- a/src/secp256k1/src/modules/recovery/tests_impl.h +++ b/src/secp256k1/src/modules/recovery/tests_impl.h @@ -30,11 +30,7 @@ static int recovery_test_nonce_function(unsigned char *nonce32, const unsigned c void test_ecdsa_recovery_api(void) { /* Setup contexts that just count errors */ - secp256k1_context *none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); - secp256k1_context *sign = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); - secp256k1_context *vrfy = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); - secp256k1_context *both = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); - secp256k1_context *sttc = secp256k1_context_clone(secp256k1_context_no_precomp); + secp256k1_context *sttc = secp256k1_context_clone(secp256k1_context_static); secp256k1_pubkey pubkey; secp256k1_pubkey recpubkey; secp256k1_ecdsa_signature normal_sig; @@ -50,15 +46,9 @@ void test_ecdsa_recovery_api(void) { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; - secp256k1_context_set_error_callback(none, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_error_callback(sign, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_error_callback(vrfy, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_error_callback(both, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(ctx, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(ctx, counting_illegal_callback_fn, &ecount); secp256k1_context_set_error_callback(sttc, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(none, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(sign, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(vrfy, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(both, counting_illegal_callback_fn, &ecount); secp256k1_context_set_illegal_callback(sttc, counting_illegal_callback_fn, &ecount); /* Construct and verify corresponding public key. */ @@ -67,89 +57,73 @@ void test_ecdsa_recovery_api(void) { /* Check bad contexts and NULLs for signing */ ecount = 0; - CHECK(secp256k1_ecdsa_sign_recoverable(none, &recsig, message, privkey, NULL, NULL) == 1); + CHECK(secp256k1_ecdsa_sign_recoverable(ctx, &recsig, message, privkey, NULL, NULL) == 1); CHECK(ecount == 0); - CHECK(secp256k1_ecdsa_sign_recoverable(sign, &recsig, message, privkey, NULL, NULL) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_ecdsa_sign_recoverable(vrfy, &recsig, message, privkey, NULL, NULL) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, NULL, NULL) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_ecdsa_sign_recoverable(both, NULL, message, privkey, NULL, NULL) == 0); + CHECK(secp256k1_ecdsa_sign_recoverable(ctx, NULL, message, privkey, NULL, NULL) == 0); CHECK(ecount == 1); - CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, NULL, privkey, NULL, NULL) == 0); + CHECK(secp256k1_ecdsa_sign_recoverable(ctx, &recsig, NULL, privkey, NULL, NULL) == 0); CHECK(ecount == 2); - CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, NULL, NULL, NULL) == 0); + CHECK(secp256k1_ecdsa_sign_recoverable(ctx, &recsig, message, NULL, NULL, NULL) == 0); CHECK(ecount == 3); CHECK(secp256k1_ecdsa_sign_recoverable(sttc, &recsig, message, privkey, NULL, NULL) == 0); CHECK(ecount == 4); /* This will fail or succeed randomly, and in either case will not ARG_CHECK failure */ - secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, recovery_test_nonce_function, NULL); + secp256k1_ecdsa_sign_recoverable(ctx, &recsig, message, privkey, recovery_test_nonce_function, NULL); CHECK(ecount == 4); /* These will all fail, but not in ARG_CHECK way */ - CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, zero_privkey, NULL, NULL) == 0); - CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, over_privkey, NULL, NULL) == 0); + CHECK(secp256k1_ecdsa_sign_recoverable(ctx, &recsig, message, zero_privkey, NULL, NULL) == 0); + CHECK(secp256k1_ecdsa_sign_recoverable(ctx, &recsig, message, over_privkey, NULL, NULL) == 0); /* This one will succeed. */ - CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, NULL, NULL) == 1); + CHECK(secp256k1_ecdsa_sign_recoverable(ctx, &recsig, message, privkey, NULL, NULL) == 1); CHECK(ecount == 4); /* Check signing with a goofy nonce function */ /* Check bad contexts and NULLs for recovery */ ecount = 0; - CHECK(secp256k1_ecdsa_recover(none, &recpubkey, &recsig, message) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_ecdsa_recover(sign, &recpubkey, &recsig, message) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_ecdsa_recover(vrfy, &recpubkey, &recsig, message) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_ecdsa_recover(both, &recpubkey, &recsig, message) == 1); + CHECK(secp256k1_ecdsa_recover(ctx, &recpubkey, &recsig, message) == 1); CHECK(ecount == 0); - CHECK(secp256k1_ecdsa_recover(both, NULL, &recsig, message) == 0); + CHECK(secp256k1_ecdsa_recover(ctx, NULL, &recsig, message) == 0); CHECK(ecount == 1); - CHECK(secp256k1_ecdsa_recover(both, &recpubkey, NULL, message) == 0); + CHECK(secp256k1_ecdsa_recover(ctx, &recpubkey, NULL, message) == 0); CHECK(ecount == 2); - CHECK(secp256k1_ecdsa_recover(both, &recpubkey, &recsig, NULL) == 0); + CHECK(secp256k1_ecdsa_recover(ctx, &recpubkey, &recsig, NULL) == 0); CHECK(ecount == 3); /* Check NULLs for conversion */ - CHECK(secp256k1_ecdsa_sign(both, &normal_sig, message, privkey, NULL, NULL) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, &normal_sig, message, privkey, NULL, NULL) == 1); ecount = 0; - CHECK(secp256k1_ecdsa_recoverable_signature_convert(both, NULL, &recsig) == 0); + CHECK(secp256k1_ecdsa_recoverable_signature_convert(ctx, NULL, &recsig) == 0); CHECK(ecount == 1); - CHECK(secp256k1_ecdsa_recoverable_signature_convert(both, &normal_sig, NULL) == 0); + CHECK(secp256k1_ecdsa_recoverable_signature_convert(ctx, &normal_sig, NULL) == 0); CHECK(ecount == 2); - CHECK(secp256k1_ecdsa_recoverable_signature_convert(both, &normal_sig, &recsig) == 1); + CHECK(secp256k1_ecdsa_recoverable_signature_convert(ctx, &normal_sig, &recsig) == 1); /* Check NULLs for de/serialization */ - CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, NULL, NULL) == 1); + CHECK(secp256k1_ecdsa_sign_recoverable(ctx, &recsig, message, privkey, NULL, NULL) == 1); ecount = 0; - CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, NULL, &recid, &recsig) == 0); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(ctx, NULL, &recid, &recsig) == 0); CHECK(ecount == 1); - CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, sig, NULL, &recsig) == 0); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(ctx, sig, NULL, &recsig) == 0); CHECK(ecount == 2); - CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, sig, &recid, NULL) == 0); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(ctx, sig, &recid, NULL) == 0); CHECK(ecount == 3); - CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, sig, &recid, &recsig) == 1); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(ctx, sig, &recid, &recsig) == 1); - CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, NULL, sig, recid) == 0); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(ctx, NULL, sig, recid) == 0); CHECK(ecount == 4); - CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, NULL, recid) == 0); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(ctx, &recsig, NULL, recid) == 0); CHECK(ecount == 5); - CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, sig, -1) == 0); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(ctx, &recsig, sig, -1) == 0); CHECK(ecount == 6); - CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, sig, 5) == 0); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(ctx, &recsig, sig, 5) == 0); CHECK(ecount == 7); /* overflow in signature will fail but not affect ecount */ memcpy(sig, over_privkey, 32); - CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, sig, recid) == 0); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(ctx, &recsig, sig, recid) == 0); CHECK(ecount == 7); /* cleanup */ - secp256k1_context_destroy(none); - secp256k1_context_destroy(sign); - secp256k1_context_destroy(vrfy); - secp256k1_context_destroy(both); secp256k1_context_destroy(sttc); } diff --git a/src/secp256k1/src/modules/schnorrsig/bench_impl.h b/src/secp256k1/src/modules/schnorrsig/bench_impl.h index 41f393c84d..f0b0d3de75 100644 --- a/src/secp256k1/src/modules/schnorrsig/bench_impl.h +++ b/src/secp256k1/src/modules/schnorrsig/bench_impl.h @@ -50,7 +50,7 @@ void run_schnorrsig_bench(int iters, int argc, char** argv) { bench_schnorrsig_data data; int d = argc == 1; - data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY | SECP256K1_CONTEXT_SIGN); + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); data.keypairs = (const secp256k1_keypair **)malloc(iters * sizeof(secp256k1_keypair *)); data.pk = (const unsigned char **)malloc(iters * sizeof(unsigned char *)); data.msgs = (const unsigned char **)malloc(iters * sizeof(unsigned char *)); @@ -91,10 +91,12 @@ void run_schnorrsig_bench(int iters, int argc, char** argv) { free((void *)data.msgs[i]); free((void *)data.sigs[i]); } - free(data.keypairs); - free(data.pk); - free(data.msgs); - free(data.sigs); + + /* Casting to (void *) avoids a stupid warning in MSVC. */ + free((void *)data.keypairs); + free((void *)data.pk); + free((void *)data.msgs); + free((void *)data.sigs); secp256k1_context_destroy(data.ctx); } diff --git a/src/secp256k1/src/modules/schnorrsig/tests_exhaustive_impl.h b/src/secp256k1/src/modules/schnorrsig/tests_exhaustive_impl.h index d8df9dd2df..55f9028a63 100644 --- a/src/secp256k1/src/modules/schnorrsig/tests_exhaustive_impl.h +++ b/src/secp256k1/src/modules/schnorrsig/tests_exhaustive_impl.h @@ -8,7 +8,7 @@ #define SECP256K1_MODULE_SCHNORRSIG_TESTS_EXHAUSTIVE_H #include "../../../include/secp256k1_schnorrsig.h" -#include "src/modules/schnorrsig/main_impl.h" +#include "main_impl.h" static const unsigned char invalid_pubkey_bytes[][32] = { /* 0 */ diff --git a/src/secp256k1/src/modules/schnorrsig/tests_impl.h b/src/secp256k1/src/modules/schnorrsig/tests_impl.h index 25840b8fa7..06cc097cc1 100644 --- a/src/secp256k1/src/modules/schnorrsig/tests_impl.h +++ b/src/secp256k1/src/modules/schnorrsig/tests_impl.h @@ -128,22 +128,12 @@ void test_schnorrsig_api(void) { secp256k1_schnorrsig_extraparams invalid_extraparams = {{ 0 }, NULL, NULL}; /** setup **/ - secp256k1_context *none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); - secp256k1_context *sign = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); - secp256k1_context *vrfy = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); - secp256k1_context *both = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); - secp256k1_context *sttc = secp256k1_context_clone(secp256k1_context_no_precomp); + secp256k1_context *sttc = secp256k1_context_clone(secp256k1_context_static); int ecount; - secp256k1_context_set_error_callback(none, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_error_callback(sign, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_error_callback(vrfy, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_error_callback(both, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(ctx, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(ctx, counting_illegal_callback_fn, &ecount); secp256k1_context_set_error_callback(sttc, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(none, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(sign, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(vrfy, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(both, counting_illegal_callback_fn, &ecount); secp256k1_context_set_illegal_callback(sttc, counting_illegal_callback_fn, &ecount); secp256k1_testrand256(sk1); @@ -160,70 +150,54 @@ void test_schnorrsig_api(void) { /** main test body **/ ecount = 0; - CHECK(secp256k1_schnorrsig_sign32(none, sig, msg, &keypairs[0], NULL) == 1); + CHECK(secp256k1_schnorrsig_sign32(ctx, sig, msg, &keypairs[0], NULL) == 1); CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_sign32(vrfy, sig, msg, &keypairs[0], NULL) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_sign32(sign, sig, msg, &keypairs[0], NULL) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_sign32(sign, NULL, msg, &keypairs[0], NULL) == 0); + CHECK(secp256k1_schnorrsig_sign32(ctx, NULL, msg, &keypairs[0], NULL) == 0); CHECK(ecount == 1); - CHECK(secp256k1_schnorrsig_sign32(sign, sig, NULL, &keypairs[0], NULL) == 0); + CHECK(secp256k1_schnorrsig_sign32(ctx, sig, NULL, &keypairs[0], NULL) == 0); CHECK(ecount == 2); - CHECK(secp256k1_schnorrsig_sign32(sign, sig, msg, NULL, NULL) == 0); + CHECK(secp256k1_schnorrsig_sign32(ctx, sig, msg, NULL, NULL) == 0); CHECK(ecount == 3); - CHECK(secp256k1_schnorrsig_sign32(sign, sig, msg, &invalid_keypair, NULL) == 0); + CHECK(secp256k1_schnorrsig_sign32(ctx, sig, msg, &invalid_keypair, NULL) == 0); CHECK(ecount == 4); CHECK(secp256k1_schnorrsig_sign32(sttc, sig, msg, &keypairs[0], NULL) == 0); CHECK(ecount == 5); ecount = 0; - CHECK(secp256k1_schnorrsig_sign_custom(none, sig, msg, sizeof(msg), &keypairs[0], &extraparams) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_sign_custom(vrfy, sig, msg, sizeof(msg), &keypairs[0], &extraparams) == 1); + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, msg, sizeof(msg), &keypairs[0], &extraparams) == 1); CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, msg, sizeof(msg), &keypairs[0], &extraparams) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_sign_custom(sign, NULL, msg, sizeof(msg), &keypairs[0], &extraparams) == 0); + CHECK(secp256k1_schnorrsig_sign_custom(ctx, NULL, msg, sizeof(msg), &keypairs[0], &extraparams) == 0); CHECK(ecount == 1); - CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, NULL, sizeof(msg), &keypairs[0], &extraparams) == 0); + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, NULL, sizeof(msg), &keypairs[0], &extraparams) == 0); CHECK(ecount == 2); - CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, NULL, 0, &keypairs[0], &extraparams) == 1); + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, NULL, 0, &keypairs[0], &extraparams) == 1); CHECK(ecount == 2); - CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, msg, sizeof(msg), NULL, &extraparams) == 0); + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, msg, sizeof(msg), NULL, &extraparams) == 0); CHECK(ecount == 3); - CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, msg, sizeof(msg), &invalid_keypair, &extraparams) == 0); + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, msg, sizeof(msg), &invalid_keypair, &extraparams) == 0); CHECK(ecount == 4); - CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, msg, sizeof(msg), &keypairs[0], NULL) == 1); + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, msg, sizeof(msg), &keypairs[0], NULL) == 1); CHECK(ecount == 4); - CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, msg, sizeof(msg), &keypairs[0], &invalid_extraparams) == 0); + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, msg, sizeof(msg), &keypairs[0], &invalid_extraparams) == 0); CHECK(ecount == 5); CHECK(secp256k1_schnorrsig_sign_custom(sttc, sig, msg, sizeof(msg), &keypairs[0], &extraparams) == 0); CHECK(ecount == 6); ecount = 0; - CHECK(secp256k1_schnorrsig_sign32(sign, sig, msg, &keypairs[0], NULL) == 1); - CHECK(secp256k1_schnorrsig_verify(none, sig, msg, sizeof(msg), &pk[0]) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_verify(sign, sig, msg, sizeof(msg), &pk[0]) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_verify(vrfy, sig, msg, sizeof(msg), &pk[0]) == 1); + CHECK(secp256k1_schnorrsig_sign32(ctx, sig, msg, &keypairs[0], NULL) == 1); + CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg, sizeof(msg), &pk[0]) == 1); CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_verify(vrfy, NULL, msg, sizeof(msg), &pk[0]) == 0); + CHECK(secp256k1_schnorrsig_verify(ctx, NULL, msg, sizeof(msg), &pk[0]) == 0); CHECK(ecount == 1); - CHECK(secp256k1_schnorrsig_verify(vrfy, sig, NULL, sizeof(msg), &pk[0]) == 0); + CHECK(secp256k1_schnorrsig_verify(ctx, sig, NULL, sizeof(msg), &pk[0]) == 0); CHECK(ecount == 2); - CHECK(secp256k1_schnorrsig_verify(vrfy, sig, NULL, 0, &pk[0]) == 0); + CHECK(secp256k1_schnorrsig_verify(ctx, sig, NULL, 0, &pk[0]) == 0); CHECK(ecount == 2); - CHECK(secp256k1_schnorrsig_verify(vrfy, sig, msg, sizeof(msg), NULL) == 0); + CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg, sizeof(msg), NULL) == 0); CHECK(ecount == 3); - CHECK(secp256k1_schnorrsig_verify(vrfy, sig, msg, sizeof(msg), &zero_pk) == 0); + CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg, sizeof(msg), &zero_pk) == 0); CHECK(ecount == 4); - secp256k1_context_destroy(none); - secp256k1_context_destroy(sign); - secp256k1_context_destroy(vrfy); - secp256k1_context_destroy(both); secp256k1_context_destroy(sttc); } diff --git a/src/secp256k1/src/precompute_ecmult.c b/src/secp256k1/src/precompute_ecmult.c index 5ccbcb3c57..2aa37b8fe3 100644 --- a/src/secp256k1/src/precompute_ecmult.c +++ b/src/secp256k1/src/precompute_ecmult.c @@ -14,10 +14,13 @@ #endif #include "../include/secp256k1.h" + #include "assumptions.h" #include "util.h" + #include "field_impl.h" #include "group_impl.h" +#include "int128_impl.h" #include "ecmult.h" #include "ecmult_compute_table_impl.h" diff --git a/src/secp256k1/src/precompute_ecmult_gen.c b/src/secp256k1/src/precompute_ecmult_gen.c index 7c6359c402..a4ec8e0dc6 100644 --- a/src/secp256k1/src/precompute_ecmult_gen.c +++ b/src/secp256k1/src/precompute_ecmult_gen.c @@ -8,9 +8,12 @@ #include <stdio.h> #include "../include/secp256k1.h" + #include "assumptions.h" #include "util.h" + #include "group.h" +#include "int128_impl.h" #include "ecmult_gen.h" #include "ecmult_gen_compute_table_impl.h" diff --git a/src/secp256k1/src/scalar_4x64_impl.h b/src/secp256k1/src/scalar_4x64_impl.h index a1def26fca..4588219d3a 100644 --- a/src/secp256k1/src/scalar_4x64_impl.h +++ b/src/secp256k1/src/scalar_4x64_impl.h @@ -7,6 +7,7 @@ #ifndef SECP256K1_SCALAR_REPR_IMPL_H #define SECP256K1_SCALAR_REPR_IMPL_H +#include "int128.h" #include "modinv64_impl.h" /* Limbs of the secp256k1 order. */ @@ -69,50 +70,61 @@ SECP256K1_INLINE static int secp256k1_scalar_check_overflow(const secp256k1_scal } SECP256K1_INLINE static int secp256k1_scalar_reduce(secp256k1_scalar *r, unsigned int overflow) { - uint128_t t; + secp256k1_uint128 t; VERIFY_CHECK(overflow <= 1); - t = (uint128_t)r->d[0] + overflow * SECP256K1_N_C_0; - r->d[0] = t & 0xFFFFFFFFFFFFFFFFULL; t >>= 64; - t += (uint128_t)r->d[1] + overflow * SECP256K1_N_C_1; - r->d[1] = t & 0xFFFFFFFFFFFFFFFFULL; t >>= 64; - t += (uint128_t)r->d[2] + overflow * SECP256K1_N_C_2; - r->d[2] = t & 0xFFFFFFFFFFFFFFFFULL; t >>= 64; - t += (uint64_t)r->d[3]; - r->d[3] = t & 0xFFFFFFFFFFFFFFFFULL; + secp256k1_u128_from_u64(&t, r->d[0]); + secp256k1_u128_accum_u64(&t, overflow * SECP256K1_N_C_0); + r->d[0] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, r->d[1]); + secp256k1_u128_accum_u64(&t, overflow * SECP256K1_N_C_1); + r->d[1] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, r->d[2]); + secp256k1_u128_accum_u64(&t, overflow * SECP256K1_N_C_2); + r->d[2] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, r->d[3]); + r->d[3] = secp256k1_u128_to_u64(&t); return overflow; } static int secp256k1_scalar_add(secp256k1_scalar *r, const secp256k1_scalar *a, const secp256k1_scalar *b) { int overflow; - uint128_t t = (uint128_t)a->d[0] + b->d[0]; - r->d[0] = t & 0xFFFFFFFFFFFFFFFFULL; t >>= 64; - t += (uint128_t)a->d[1] + b->d[1]; - r->d[1] = t & 0xFFFFFFFFFFFFFFFFULL; t >>= 64; - t += (uint128_t)a->d[2] + b->d[2]; - r->d[2] = t & 0xFFFFFFFFFFFFFFFFULL; t >>= 64; - t += (uint128_t)a->d[3] + b->d[3]; - r->d[3] = t & 0xFFFFFFFFFFFFFFFFULL; t >>= 64; - overflow = t + secp256k1_scalar_check_overflow(r); + secp256k1_uint128 t; + secp256k1_u128_from_u64(&t, a->d[0]); + secp256k1_u128_accum_u64(&t, b->d[0]); + r->d[0] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, a->d[1]); + secp256k1_u128_accum_u64(&t, b->d[1]); + r->d[1] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, a->d[2]); + secp256k1_u128_accum_u64(&t, b->d[2]); + r->d[2] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, a->d[3]); + secp256k1_u128_accum_u64(&t, b->d[3]); + r->d[3] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); + overflow = secp256k1_u128_to_u64(&t) + secp256k1_scalar_check_overflow(r); VERIFY_CHECK(overflow == 0 || overflow == 1); secp256k1_scalar_reduce(r, overflow); return overflow; } static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int flag) { - uint128_t t; + secp256k1_uint128 t; VERIFY_CHECK(bit < 256); bit += ((uint32_t) flag - 1) & 0x100; /* forcing (bit >> 6) > 3 makes this a noop */ - t = (uint128_t)r->d[0] + (((uint64_t)((bit >> 6) == 0)) << (bit & 0x3F)); - r->d[0] = t & 0xFFFFFFFFFFFFFFFFULL; t >>= 64; - t += (uint128_t)r->d[1] + (((uint64_t)((bit >> 6) == 1)) << (bit & 0x3F)); - r->d[1] = t & 0xFFFFFFFFFFFFFFFFULL; t >>= 64; - t += (uint128_t)r->d[2] + (((uint64_t)((bit >> 6) == 2)) << (bit & 0x3F)); - r->d[2] = t & 0xFFFFFFFFFFFFFFFFULL; t >>= 64; - t += (uint128_t)r->d[3] + (((uint64_t)((bit >> 6) == 3)) << (bit & 0x3F)); - r->d[3] = t & 0xFFFFFFFFFFFFFFFFULL; + secp256k1_u128_from_u64(&t, r->d[0]); + secp256k1_u128_accum_u64(&t, ((uint64_t)((bit >> 6) == 0)) << (bit & 0x3F)); + r->d[0] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, r->d[1]); + secp256k1_u128_accum_u64(&t, ((uint64_t)((bit >> 6) == 1)) << (bit & 0x3F)); + r->d[1] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, r->d[2]); + secp256k1_u128_accum_u64(&t, ((uint64_t)((bit >> 6) == 2)) << (bit & 0x3F)); + r->d[2] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, r->d[3]); + secp256k1_u128_accum_u64(&t, ((uint64_t)((bit >> 6) == 3)) << (bit & 0x3F)); + r->d[3] = secp256k1_u128_to_u64(&t); #ifdef VERIFY - VERIFY_CHECK((t >> 64) == 0); - VERIFY_CHECK(secp256k1_scalar_check_overflow(r) == 0); + VERIFY_CHECK(secp256k1_u128_hi_u64(&t) == 0); #endif } @@ -141,14 +153,19 @@ SECP256K1_INLINE static int secp256k1_scalar_is_zero(const secp256k1_scalar *a) static void secp256k1_scalar_negate(secp256k1_scalar *r, const secp256k1_scalar *a) { uint64_t nonzero = 0xFFFFFFFFFFFFFFFFULL * (secp256k1_scalar_is_zero(a) == 0); - uint128_t t = (uint128_t)(~a->d[0]) + SECP256K1_N_0 + 1; - r->d[0] = t & nonzero; t >>= 64; - t += (uint128_t)(~a->d[1]) + SECP256K1_N_1; - r->d[1] = t & nonzero; t >>= 64; - t += (uint128_t)(~a->d[2]) + SECP256K1_N_2; - r->d[2] = t & nonzero; t >>= 64; - t += (uint128_t)(~a->d[3]) + SECP256K1_N_3; - r->d[3] = t & nonzero; + secp256k1_uint128 t; + secp256k1_u128_from_u64(&t, ~a->d[0]); + secp256k1_u128_accum_u64(&t, SECP256K1_N_0 + 1); + r->d[0] = secp256k1_u128_to_u64(&t) & nonzero; secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, ~a->d[1]); + secp256k1_u128_accum_u64(&t, SECP256K1_N_1); + r->d[1] = secp256k1_u128_to_u64(&t) & nonzero; secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, ~a->d[2]); + secp256k1_u128_accum_u64(&t, SECP256K1_N_2); + r->d[2] = secp256k1_u128_to_u64(&t) & nonzero; secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, ~a->d[3]); + secp256k1_u128_accum_u64(&t, SECP256K1_N_3); + r->d[3] = secp256k1_u128_to_u64(&t) & nonzero; } SECP256K1_INLINE static int secp256k1_scalar_is_one(const secp256k1_scalar *a) { @@ -172,14 +189,19 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { * if we are flag = 1, mask = 11...11 and this is identical to secp256k1_scalar_negate */ uint64_t mask = !flag - 1; uint64_t nonzero = (secp256k1_scalar_is_zero(r) != 0) - 1; - uint128_t t = (uint128_t)(r->d[0] ^ mask) + ((SECP256K1_N_0 + 1) & mask); - r->d[0] = t & nonzero; t >>= 64; - t += (uint128_t)(r->d[1] ^ mask) + (SECP256K1_N_1 & mask); - r->d[1] = t & nonzero; t >>= 64; - t += (uint128_t)(r->d[2] ^ mask) + (SECP256K1_N_2 & mask); - r->d[2] = t & nonzero; t >>= 64; - t += (uint128_t)(r->d[3] ^ mask) + (SECP256K1_N_3 & mask); - r->d[3] = t & nonzero; + secp256k1_uint128 t; + secp256k1_u128_from_u64(&t, r->d[0] ^ mask); + secp256k1_u128_accum_u64(&t, (SECP256K1_N_0 + 1) & mask); + r->d[0] = secp256k1_u128_to_u64(&t) & nonzero; secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, r->d[1] ^ mask); + secp256k1_u128_accum_u64(&t, SECP256K1_N_1 & mask); + r->d[1] = secp256k1_u128_to_u64(&t) & nonzero; secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, r->d[2] ^ mask); + secp256k1_u128_accum_u64(&t, SECP256K1_N_2 & mask); + r->d[2] = secp256k1_u128_to_u64(&t) & nonzero; secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, r->d[3] ^ mask); + secp256k1_u128_accum_u64(&t, SECP256K1_N_3 & mask); + r->d[3] = secp256k1_u128_to_u64(&t) & nonzero; return 2 * (mask == 0) - 1; } @@ -189,9 +211,10 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { #define muladd(a,b) { \ uint64_t tl, th; \ { \ - uint128_t t = (uint128_t)a * b; \ - th = t >> 64; /* at most 0xFFFFFFFFFFFFFFFE */ \ - tl = t; \ + secp256k1_uint128 t; \ + secp256k1_u128_mul(&t, a, b); \ + th = secp256k1_u128_hi_u64(&t); /* at most 0xFFFFFFFFFFFFFFFE */ \ + tl = secp256k1_u128_to_u64(&t); \ } \ c0 += tl; /* overflow is handled on the next line */ \ th += (c0 < tl); /* at most 0xFFFFFFFFFFFFFFFF */ \ @@ -204,9 +227,10 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { #define muladd_fast(a,b) { \ uint64_t tl, th; \ { \ - uint128_t t = (uint128_t)a * b; \ - th = t >> 64; /* at most 0xFFFFFFFFFFFFFFFE */ \ - tl = t; \ + secp256k1_uint128 t; \ + secp256k1_u128_mul(&t, a, b); \ + th = secp256k1_u128_hi_u64(&t); /* at most 0xFFFFFFFFFFFFFFFE */ \ + tl = secp256k1_u128_to_u64(&t); \ } \ c0 += tl; /* overflow is handled on the next line */ \ th += (c0 < tl); /* at most 0xFFFFFFFFFFFFFFFF */ \ @@ -484,8 +508,8 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) : "g"(p0), "g"(p1), "g"(p2), "g"(p3), "g"(p4), "D"(r), "i"(SECP256K1_N_C_0), "i"(SECP256K1_N_C_1) : "rax", "rdx", "r8", "r9", "r10", "cc", "memory"); #else - uint128_t c; - uint64_t c0, c1, c2; + secp256k1_uint128 c128; + uint64_t c, c0, c1, c2; uint64_t n0 = l[4], n1 = l[5], n2 = l[6], n3 = l[7]; uint64_t m0, m1, m2, m3, m4, m5; uint32_t m6; @@ -542,14 +566,18 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) /* Reduce 258 bits into 256. */ /* r[0..3] = p[0..3] + p[4] * SECP256K1_N_C. */ - c = p0 + (uint128_t)SECP256K1_N_C_0 * p4; - r->d[0] = c & 0xFFFFFFFFFFFFFFFFULL; c >>= 64; - c += p1 + (uint128_t)SECP256K1_N_C_1 * p4; - r->d[1] = c & 0xFFFFFFFFFFFFFFFFULL; c >>= 64; - c += p2 + (uint128_t)p4; - r->d[2] = c & 0xFFFFFFFFFFFFFFFFULL; c >>= 64; - c += p3; - r->d[3] = c & 0xFFFFFFFFFFFFFFFFULL; c >>= 64; + secp256k1_u128_from_u64(&c128, p0); + secp256k1_u128_accum_mul(&c128, SECP256K1_N_C_0, p4); + r->d[0] = secp256k1_u128_to_u64(&c128); secp256k1_u128_rshift(&c128, 64); + secp256k1_u128_accum_u64(&c128, p1); + secp256k1_u128_accum_mul(&c128, SECP256K1_N_C_1, p4); + r->d[1] = secp256k1_u128_to_u64(&c128); secp256k1_u128_rshift(&c128, 64); + secp256k1_u128_accum_u64(&c128, p2); + secp256k1_u128_accum_u64(&c128, p4); + r->d[2] = secp256k1_u128_to_u64(&c128); secp256k1_u128_rshift(&c128, 64); + secp256k1_u128_accum_u64(&c128, p3); + r->d[3] = secp256k1_u128_to_u64(&c128); + c = secp256k1_u128_hi_u64(&c128); #endif /* Final reduction of r. */ diff --git a/src/secp256k1/src/scratch_impl.h b/src/secp256k1/src/scratch_impl.h index 688e18eb66..f71a20b963 100644 --- a/src/secp256k1/src/scratch_impl.h +++ b/src/secp256k1/src/scratch_impl.h @@ -25,11 +25,11 @@ static secp256k1_scratch* secp256k1_scratch_create(const secp256k1_callback* err static void secp256k1_scratch_destroy(const secp256k1_callback* error_callback, secp256k1_scratch* scratch) { if (scratch != NULL) { - VERIFY_CHECK(scratch->alloc_size == 0); /* all checkpoints should be applied */ if (secp256k1_memcmp_var(scratch->magic, "scratch", 8) != 0) { secp256k1_callback_call(error_callback, "invalid scratch space"); return; } + VERIFY_CHECK(scratch->alloc_size == 0); /* all checkpoints should be applied */ memset(scratch->magic, 0, sizeof(scratch->magic)); free(scratch); } diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index 8f34c35283..5ed3824161 100644 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -4,6 +4,17 @@ * file COPYING or https://www.opensource.org/licenses/mit-license.php.* ***********************************************************************/ +/* This is a C project. It should not be compiled with a C++ compiler, + * and we error out if we detect one. + * + * We still want to be able to test the project with a C++ compiler + * because it is still good to know if this will lead to real trouble, so + * there is a possibility to override the check. But be warned that + * compiling with a C++ compiler is not supported. */ +#if defined(__cplusplus) && !defined(SECP256K1_CPLUSPLUS_TEST_OVERRIDE) +#error Trying to compile a C project with a C++ compiler. +#endif + #define SECP256K1_BUILD #include "../include/secp256k1.h" @@ -11,6 +22,7 @@ #include "assumptions.h" #include "util.h" + #include "field_impl.h" #include "scalar_impl.h" #include "group_impl.h" @@ -20,6 +32,7 @@ #include "ecdsa_impl.h" #include "eckey_impl.h" #include "hash_impl.h" +#include "int128_impl.h" #include "scratch_impl.h" #include "selftest.h" @@ -44,6 +57,8 @@ } \ } while(0) +/* Note that whenever you change the context struct, you must also change the + * context_eq function. */ struct secp256k1_context_struct { secp256k1_ecmult_gen_context ecmult_gen_ctx; secp256k1_callback illegal_callback; @@ -51,13 +66,20 @@ struct secp256k1_context_struct { int declassify; }; -static const secp256k1_context secp256k1_context_no_precomp_ = { +static const secp256k1_context secp256k1_context_static_ = { { 0 }, { secp256k1_default_illegal_callback_fn, 0 }, { secp256k1_default_error_callback_fn, 0 }, 0 }; -const secp256k1_context *secp256k1_context_no_precomp = &secp256k1_context_no_precomp_; +const secp256k1_context *secp256k1_context_static = &secp256k1_context_static_; +const secp256k1_context *secp256k1_context_no_precomp = &secp256k1_context_static_; + +void secp256k1_selftest(void) { + if (!secp256k1_selftest_passes()) { + secp256k1_callback_call(&default_error_callback, "self test failed"); + } +} size_t secp256k1_context_preallocated_size(unsigned int flags) { size_t ret = sizeof(secp256k1_context); @@ -83,9 +105,7 @@ secp256k1_context* secp256k1_context_preallocated_create(void* prealloc, unsigne size_t prealloc_size; secp256k1_context* ret; - if (!secp256k1_selftest()) { - secp256k1_callback_call(&default_error_callback, "self test failed"); - } + secp256k1_selftest(); prealloc_size = secp256k1_context_preallocated_size(flags); if (prealloc_size == 0) { @@ -137,7 +157,7 @@ secp256k1_context* secp256k1_context_clone(const secp256k1_context* ctx) { } void secp256k1_context_preallocated_destroy(secp256k1_context* ctx) { - ARG_CHECK_NO_RETURN(ctx != secp256k1_context_no_precomp); + ARG_CHECK_NO_RETURN(ctx != secp256k1_context_static); if (ctx != NULL) { secp256k1_ecmult_gen_context_clear(&ctx->ecmult_gen_ctx); } @@ -151,7 +171,7 @@ void secp256k1_context_destroy(secp256k1_context* ctx) { } void secp256k1_context_set_illegal_callback(secp256k1_context* ctx, void (*fun)(const char* message, void* data), const void* data) { - ARG_CHECK_NO_RETURN(ctx != secp256k1_context_no_precomp); + ARG_CHECK_NO_RETURN(ctx != secp256k1_context_static); if (fun == NULL) { fun = secp256k1_default_illegal_callback_fn; } @@ -160,7 +180,7 @@ void secp256k1_context_set_illegal_callback(secp256k1_context* ctx, void (*fun)( } void secp256k1_context_set_error_callback(secp256k1_context* ctx, void (*fun)(const char* message, void* data), const void* data) { - ARG_CHECK_NO_RETURN(ctx != secp256k1_context_no_precomp); + ARG_CHECK_NO_RETURN(ctx != secp256k1_context_static); if (fun == NULL) { fun = secp256k1_default_error_callback_fn; } diff --git a/src/secp256k1/src/selftest.h b/src/secp256k1/src/selftest.h index 52f1b8442e..d083ac9524 100644 --- a/src/secp256k1/src/selftest.h +++ b/src/secp256k1/src/selftest.h @@ -25,7 +25,7 @@ static int secp256k1_selftest_sha256(void) { return secp256k1_memcmp_var(out, output32, 32) == 0; } -static int secp256k1_selftest(void) { +static int secp256k1_selftest_passes(void) { return secp256k1_selftest_sha256(); } diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index dd53173930..53613f420a 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -26,6 +26,7 @@ #include "modinv32_impl.h" #ifdef SECP256K1_WIDEMUL_INT128 #include "modinv64_impl.h" +#include "int128_impl.h" #endif #define CONDITIONAL_TEST(cnt, nam) if (count < (cnt)) { printf("Skipping %s (iteration count too low)\n", nam); } else @@ -140,6 +141,43 @@ void random_scalar_order_b32(unsigned char *b32) { secp256k1_scalar_get_b32(b32, &num); } +void run_selftest_tests(void) { + /* Test public API */ + secp256k1_selftest(); +} + +int ecmult_gen_context_eq(const secp256k1_ecmult_gen_context *a, const secp256k1_ecmult_gen_context *b) { + return a->built == b->built + && secp256k1_scalar_eq(&a->blind, &b->blind) + && secp256k1_gej_eq_var(&a->initial, &b->initial); +} + +int context_eq(const secp256k1_context *a, const secp256k1_context *b) { + return a->declassify == b->declassify + && ecmult_gen_context_eq(&a->ecmult_gen_ctx, &b->ecmult_gen_ctx) + && a->illegal_callback.fn == b->illegal_callback.fn + && a->illegal_callback.data == b->illegal_callback. +data + && a->error_callback.fn == b->error_callback.fn + && a->error_callback.data == b->error_callback.data; +} + +void test_deprecated_flags(void) { + unsigned int flags[] = { SECP256K1_CONTEXT_SIGN, + SECP256K1_CONTEXT_VERIFY, + SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY }; + int i; + /* Check that a context created with any of the flags in the flags array is + * identical to the NONE context. */ + for (i = 0; i < (int)(sizeof(flags)/sizeof(flags[0])); i++) { + secp256k1_context *tmp_ctx; + CHECK(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_NONE) == secp256k1_context_preallocated_size(flags[i])); + tmp_ctx = secp256k1_context_create(flags[i]); + CHECK(context_eq(ctx, tmp_ctx)); + secp256k1_context_destroy(tmp_ctx); + } +} + void run_context_tests(int use_prealloc) { secp256k1_pubkey pubkey; secp256k1_pubkey zero_pubkey; @@ -147,15 +185,8 @@ void run_context_tests(int use_prealloc) { unsigned char ctmp[32]; int32_t ecount; int32_t ecount2; - secp256k1_context *none; - secp256k1_context *sign; - secp256k1_context *vrfy; - secp256k1_context *both; secp256k1_context *sttc; - void *none_prealloc = NULL; - void *sign_prealloc = NULL; - void *vrfy_prealloc = NULL; - void *both_prealloc = NULL; + void *ctx_prealloc = NULL; void *sttc_prealloc = NULL; secp256k1_gej pubj; @@ -163,46 +194,36 @@ void run_context_tests(int use_prealloc) { secp256k1_scalar msg, key, nonce; secp256k1_scalar sigr, sigs; + /* Check that deprecated secp256k1_context_no_precomp is an alias to secp256k1_context_static. */ + CHECK(secp256k1_context_no_precomp == secp256k1_context_static); + if (use_prealloc) { - none_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_NONE)); - sign_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN)); - vrfy_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_VERIFY)); - both_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY)); - sttc_prealloc = malloc(secp256k1_context_preallocated_clone_size(secp256k1_context_no_precomp)); - CHECK(none_prealloc != NULL); - CHECK(sign_prealloc != NULL); - CHECK(vrfy_prealloc != NULL); - CHECK(both_prealloc != NULL); + ctx_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_NONE)); + CHECK(ctx_prealloc != NULL); + ctx = secp256k1_context_preallocated_create(ctx_prealloc, SECP256K1_CONTEXT_NONE); + sttc_prealloc = malloc(secp256k1_context_preallocated_clone_size(secp256k1_context_static)); CHECK(sttc_prealloc != NULL); - none = secp256k1_context_preallocated_create(none_prealloc, SECP256K1_CONTEXT_NONE); - sign = secp256k1_context_preallocated_create(sign_prealloc, SECP256K1_CONTEXT_SIGN); - vrfy = secp256k1_context_preallocated_create(vrfy_prealloc, SECP256K1_CONTEXT_VERIFY); - both = secp256k1_context_preallocated_create(both_prealloc, SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); - sttc = secp256k1_context_preallocated_clone(secp256k1_context_no_precomp, sttc_prealloc); + sttc = secp256k1_context_preallocated_clone(secp256k1_context_static, sttc_prealloc); } else { - none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); - sign = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); - vrfy = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); - both = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); - sttc = secp256k1_context_clone(secp256k1_context_no_precomp); + sttc = secp256k1_context_clone(secp256k1_context_static); + ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); } + test_deprecated_flags(); + memset(&zero_pubkey, 0, sizeof(zero_pubkey)); ecount = 0; ecount2 = 10; secp256k1_context_set_illegal_callback(sttc, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(sign, counting_illegal_callback_fn, &ecount2); + secp256k1_context_set_illegal_callback(ctx, counting_illegal_callback_fn, &ecount2); /* set error callback (to a function that still aborts in case malloc() fails in secp256k1_context_clone() below) */ - secp256k1_context_set_error_callback(sign, secp256k1_default_illegal_callback_fn, NULL); - CHECK(sign->error_callback.fn != vrfy->error_callback.fn); - CHECK(sign->error_callback.fn == secp256k1_default_illegal_callback_fn); + secp256k1_context_set_error_callback(ctx, secp256k1_default_illegal_callback_fn, NULL); + CHECK(ctx->error_callback.fn != sttc->error_callback.fn); + CHECK(ctx->error_callback.fn == secp256k1_default_illegal_callback_fn); /* check if sizes for cloning are consistent */ - CHECK(secp256k1_context_preallocated_clone_size(none) == secp256k1_context_preallocated_size(SECP256K1_CONTEXT_NONE)); - CHECK(secp256k1_context_preallocated_clone_size(sign) == secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN)); - CHECK(secp256k1_context_preallocated_clone_size(vrfy) == secp256k1_context_preallocated_size(SECP256K1_CONTEXT_VERIFY)); - CHECK(secp256k1_context_preallocated_clone_size(both) == secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY)); + CHECK(secp256k1_context_preallocated_clone_size(ctx) == secp256k1_context_preallocated_size(SECP256K1_CONTEXT_NONE)); CHECK(secp256k1_context_preallocated_clone_size(sttc) >= sizeof(secp256k1_context)); /*** clone and destroy all of them to make sure cloning was complete ***/ @@ -211,58 +232,31 @@ void run_context_tests(int use_prealloc) { if (use_prealloc) { /* clone into a non-preallocated context and then again into a new preallocated one. */ - ctx_tmp = none; none = secp256k1_context_clone(none); secp256k1_context_preallocated_destroy(ctx_tmp); - free(none_prealloc); none_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_NONE)); CHECK(none_prealloc != NULL); - ctx_tmp = none; none = secp256k1_context_preallocated_clone(none, none_prealloc); secp256k1_context_destroy(ctx_tmp); - - ctx_tmp = sign; sign = secp256k1_context_clone(sign); secp256k1_context_preallocated_destroy(ctx_tmp); - free(sign_prealloc); sign_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN)); CHECK(sign_prealloc != NULL); - ctx_tmp = sign; sign = secp256k1_context_preallocated_clone(sign, sign_prealloc); secp256k1_context_destroy(ctx_tmp); - - ctx_tmp = vrfy; vrfy = secp256k1_context_clone(vrfy); secp256k1_context_preallocated_destroy(ctx_tmp); - free(vrfy_prealloc); vrfy_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_VERIFY)); CHECK(vrfy_prealloc != NULL); - ctx_tmp = vrfy; vrfy = secp256k1_context_preallocated_clone(vrfy, vrfy_prealloc); secp256k1_context_destroy(ctx_tmp); - - ctx_tmp = both; both = secp256k1_context_clone(both); secp256k1_context_preallocated_destroy(ctx_tmp); - free(both_prealloc); both_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY)); CHECK(both_prealloc != NULL); - ctx_tmp = both; both = secp256k1_context_preallocated_clone(both, both_prealloc); secp256k1_context_destroy(ctx_tmp); + ctx_tmp = ctx; ctx = secp256k1_context_clone(ctx); secp256k1_context_preallocated_destroy(ctx_tmp); + free(ctx_prealloc); ctx_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_NONE)); CHECK(ctx_prealloc != NULL); + ctx_tmp = ctx; ctx = secp256k1_context_preallocated_clone(ctx, ctx_prealloc); secp256k1_context_destroy(ctx_tmp); } else { /* clone into a preallocated context and then again into a new non-preallocated one. */ void *prealloc_tmp; prealloc_tmp = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_NONE)); CHECK(prealloc_tmp != NULL); - ctx_tmp = none; none = secp256k1_context_preallocated_clone(none, prealloc_tmp); secp256k1_context_destroy(ctx_tmp); - ctx_tmp = none; none = secp256k1_context_clone(none); secp256k1_context_preallocated_destroy(ctx_tmp); - free(prealloc_tmp); - - prealloc_tmp = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN)); CHECK(prealloc_tmp != NULL); - ctx_tmp = sign; sign = secp256k1_context_preallocated_clone(sign, prealloc_tmp); secp256k1_context_destroy(ctx_tmp); - ctx_tmp = sign; sign = secp256k1_context_clone(sign); secp256k1_context_preallocated_destroy(ctx_tmp); - free(prealloc_tmp); - - prealloc_tmp = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_VERIFY)); CHECK(prealloc_tmp != NULL); - ctx_tmp = vrfy; vrfy = secp256k1_context_preallocated_clone(vrfy, prealloc_tmp); secp256k1_context_destroy(ctx_tmp); - ctx_tmp = vrfy; vrfy = secp256k1_context_clone(vrfy); secp256k1_context_preallocated_destroy(ctx_tmp); - free(prealloc_tmp); - - prealloc_tmp = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY)); CHECK(prealloc_tmp != NULL); - ctx_tmp = both; both = secp256k1_context_preallocated_clone(both, prealloc_tmp); secp256k1_context_destroy(ctx_tmp); - ctx_tmp = both; both = secp256k1_context_clone(both); secp256k1_context_preallocated_destroy(ctx_tmp); + ctx_tmp = ctx; ctx = secp256k1_context_preallocated_clone(ctx, prealloc_tmp); secp256k1_context_destroy(ctx_tmp); + ctx_tmp = ctx; ctx = secp256k1_context_clone(ctx); secp256k1_context_preallocated_destroy(ctx_tmp); free(prealloc_tmp); } } /* Verify that the error callback makes it across the clone. */ - CHECK(sign->error_callback.fn != vrfy->error_callback.fn); - CHECK(sign->error_callback.fn == secp256k1_default_illegal_callback_fn); + CHECK(ctx->error_callback.fn != sttc->error_callback.fn); + CHECK(ctx->error_callback.fn == secp256k1_default_illegal_callback_fn); /* And that it resets back to default. */ - secp256k1_context_set_error_callback(sign, NULL, NULL); - CHECK(vrfy->error_callback.fn == sign->error_callback.fn); + secp256k1_context_set_error_callback(ctx, NULL, NULL); + CHECK(ctx->error_callback.fn == sttc->error_callback.fn); /*** attempt to use them ***/ random_scalar_order_test(&msg); random_scalar_order_test(&key); - secp256k1_ecmult_gen(&both->ecmult_gen_ctx, &pubj, &key); + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &pubj, &key); secp256k1_ge_set_gej(&pub, &pubj); /* Verify context-type checking illegal-argument errors. */ @@ -270,29 +264,29 @@ void run_context_tests(int use_prealloc) { CHECK(secp256k1_ec_pubkey_create(sttc, &pubkey, ctmp) == 0); CHECK(ecount == 1); VG_UNDEF(&pubkey, sizeof(pubkey)); - CHECK(secp256k1_ec_pubkey_create(sign, &pubkey, ctmp) == 1); + CHECK(secp256k1_ec_pubkey_create(ctx, &pubkey, ctmp) == 1); VG_CHECK(&pubkey, sizeof(pubkey)); CHECK(secp256k1_ecdsa_sign(sttc, &sig, ctmp, ctmp, NULL, NULL) == 0); CHECK(ecount == 2); VG_UNDEF(&sig, sizeof(sig)); - CHECK(secp256k1_ecdsa_sign(sign, &sig, ctmp, ctmp, NULL, NULL) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, &sig, ctmp, ctmp, NULL, NULL) == 1); VG_CHECK(&sig, sizeof(sig)); CHECK(ecount2 == 10); - CHECK(secp256k1_ecdsa_verify(sign, &sig, ctmp, &pubkey) == 1); + CHECK(secp256k1_ecdsa_verify(ctx, &sig, ctmp, &pubkey) == 1); CHECK(ecount2 == 10); CHECK(secp256k1_ecdsa_verify(sttc, &sig, ctmp, &pubkey) == 1); CHECK(ecount == 2); - CHECK(secp256k1_ec_pubkey_tweak_add(sign, &pubkey, ctmp) == 1); + CHECK(secp256k1_ec_pubkey_tweak_add(ctx, &pubkey, ctmp) == 1); CHECK(ecount2 == 10); CHECK(secp256k1_ec_pubkey_tweak_add(sttc, &pubkey, ctmp) == 1); CHECK(ecount == 2); - CHECK(secp256k1_ec_pubkey_tweak_mul(sign, &pubkey, ctmp) == 1); + CHECK(secp256k1_ec_pubkey_tweak_mul(ctx, &pubkey, ctmp) == 1); CHECK(ecount2 == 10); CHECK(secp256k1_ec_pubkey_negate(sttc, &pubkey) == 1); CHECK(ecount == 2); - CHECK(secp256k1_ec_pubkey_negate(sign, &pubkey) == 1); + CHECK(secp256k1_ec_pubkey_negate(ctx, &pubkey) == 1); CHECK(ecount == 2); - CHECK(secp256k1_ec_pubkey_negate(sign, NULL) == 0); + CHECK(secp256k1_ec_pubkey_negate(ctx, NULL) == 0); CHECK(ecount2 == 11); CHECK(secp256k1_ec_pubkey_negate(sttc, &zero_pubkey) == 0); CHECK(ecount == 3); @@ -302,49 +296,37 @@ void run_context_tests(int use_prealloc) { CHECK(ecount == 3); CHECK(secp256k1_context_randomize(sttc, NULL) == 1); CHECK(ecount == 3); - CHECK(secp256k1_context_randomize(sign, ctmp) == 1); + CHECK(secp256k1_context_randomize(ctx, ctmp) == 1); CHECK(ecount2 == 11); - CHECK(secp256k1_context_randomize(sign, NULL) == 1); + CHECK(secp256k1_context_randomize(ctx, NULL) == 1); CHECK(ecount2 == 11); secp256k1_context_set_illegal_callback(sttc, NULL, NULL); - secp256k1_context_set_illegal_callback(sign, NULL, NULL); + secp256k1_context_set_illegal_callback(ctx, NULL, NULL); /* obtain a working nonce */ do { random_scalar_order_test(&nonce); - } while(!secp256k1_ecdsa_sig_sign(&both->ecmult_gen_ctx, &sigr, &sigs, &key, &msg, &nonce, NULL)); + } while(!secp256k1_ecdsa_sig_sign(&ctx->ecmult_gen_ctx, &sigr, &sigs, &key, &msg, &nonce, NULL)); /* try signing */ - CHECK(secp256k1_ecdsa_sig_sign(&sign->ecmult_gen_ctx, &sigr, &sigs, &key, &msg, &nonce, NULL)); - CHECK(secp256k1_ecdsa_sig_sign(&both->ecmult_gen_ctx, &sigr, &sigs, &key, &msg, &nonce, NULL)); + CHECK(secp256k1_ecdsa_sig_sign(&ctx->ecmult_gen_ctx, &sigr, &sigs, &key, &msg, &nonce, NULL)); /* try verifying */ CHECK(secp256k1_ecdsa_sig_verify(&sigr, &sigs, &pub, &msg)); - CHECK(secp256k1_ecdsa_sig_verify(&sigr, &sigs, &pub, &msg)); /* cleanup */ if (use_prealloc) { - secp256k1_context_preallocated_destroy(none); - secp256k1_context_preallocated_destroy(sign); - secp256k1_context_preallocated_destroy(vrfy); - secp256k1_context_preallocated_destroy(both); + secp256k1_context_preallocated_destroy(ctx); secp256k1_context_preallocated_destroy(sttc); - free(none_prealloc); - free(sign_prealloc); - free(vrfy_prealloc); - free(both_prealloc); + free(ctx_prealloc); free(sttc_prealloc); } else { - secp256k1_context_destroy(none); - secp256k1_context_destroy(sign); - secp256k1_context_destroy(vrfy); - secp256k1_context_destroy(both); + secp256k1_context_destroy(ctx); secp256k1_context_destroy(sttc); } /* Defined as no-op. */ secp256k1_context_destroy(NULL); secp256k1_context_preallocated_destroy(NULL); - } void run_scratch_tests(void) { @@ -353,83 +335,85 @@ void run_scratch_tests(void) { int32_t ecount = 0; size_t checkpoint; size_t checkpoint_2; - secp256k1_context *none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); secp256k1_scratch_space *scratch; secp256k1_scratch_space local_scratch; + ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); + /* Test public API */ - secp256k1_context_set_illegal_callback(none, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_error_callback(none, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(ctx, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(ctx, counting_illegal_callback_fn, &ecount); - scratch = secp256k1_scratch_space_create(none, 1000); + scratch = secp256k1_scratch_space_create(ctx, 1000); CHECK(scratch != NULL); CHECK(ecount == 0); /* Test internal API */ - CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 0) == 1000); - CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 1) == 1000 - (ALIGNMENT - 1)); + CHECK(secp256k1_scratch_max_allocation(&ctx->error_callback, scratch, 0) == 1000); + CHECK(secp256k1_scratch_max_allocation(&ctx->error_callback, scratch, 1) == 1000 - (ALIGNMENT - 1)); CHECK(scratch->alloc_size == 0); CHECK(scratch->alloc_size % ALIGNMENT == 0); /* Allocating 500 bytes succeeds */ - checkpoint = secp256k1_scratch_checkpoint(&none->error_callback, scratch); - CHECK(secp256k1_scratch_alloc(&none->error_callback, scratch, 500) != NULL); - CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 0) == 1000 - adj_alloc); - CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 1) == 1000 - adj_alloc - (ALIGNMENT - 1)); + checkpoint = secp256k1_scratch_checkpoint(&ctx->error_callback, scratch); + CHECK(secp256k1_scratch_alloc(&ctx->error_callback, scratch, 500) != NULL); + CHECK(secp256k1_scratch_max_allocation(&ctx->error_callback, scratch, 0) == 1000 - adj_alloc); + CHECK(secp256k1_scratch_max_allocation(&ctx->error_callback, scratch, 1) == 1000 - adj_alloc - (ALIGNMENT - 1)); CHECK(scratch->alloc_size != 0); CHECK(scratch->alloc_size % ALIGNMENT == 0); /* Allocating another 501 bytes fails */ - CHECK(secp256k1_scratch_alloc(&none->error_callback, scratch, 501) == NULL); - CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 0) == 1000 - adj_alloc); - CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 1) == 1000 - adj_alloc - (ALIGNMENT - 1)); + CHECK(secp256k1_scratch_alloc(&ctx->error_callback, scratch, 501) == NULL); + CHECK(secp256k1_scratch_max_allocation(&ctx->error_callback, scratch, 0) == 1000 - adj_alloc); + CHECK(secp256k1_scratch_max_allocation(&ctx->error_callback, scratch, 1) == 1000 - adj_alloc - (ALIGNMENT - 1)); CHECK(scratch->alloc_size != 0); CHECK(scratch->alloc_size % ALIGNMENT == 0); /* ...but it succeeds once we apply the checkpoint to undo it */ - secp256k1_scratch_apply_checkpoint(&none->error_callback, scratch, checkpoint); + secp256k1_scratch_apply_checkpoint(&ctx->error_callback, scratch, checkpoint); CHECK(scratch->alloc_size == 0); - CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 0) == 1000); - CHECK(secp256k1_scratch_alloc(&none->error_callback, scratch, 500) != NULL); + CHECK(secp256k1_scratch_max_allocation(&ctx->error_callback, scratch, 0) == 1000); + CHECK(secp256k1_scratch_alloc(&ctx->error_callback, scratch, 500) != NULL); CHECK(scratch->alloc_size != 0); /* try to apply a bad checkpoint */ - checkpoint_2 = secp256k1_scratch_checkpoint(&none->error_callback, scratch); - secp256k1_scratch_apply_checkpoint(&none->error_callback, scratch, checkpoint); + checkpoint_2 = secp256k1_scratch_checkpoint(&ctx->error_callback, scratch); + secp256k1_scratch_apply_checkpoint(&ctx->error_callback, scratch, checkpoint); CHECK(ecount == 0); - secp256k1_scratch_apply_checkpoint(&none->error_callback, scratch, checkpoint_2); /* checkpoint_2 is after checkpoint */ + secp256k1_scratch_apply_checkpoint(&ctx->error_callback, scratch, checkpoint_2); /* checkpoint_2 is after checkpoint */ CHECK(ecount == 1); - secp256k1_scratch_apply_checkpoint(&none->error_callback, scratch, (size_t) -1); /* this is just wildly invalid */ + secp256k1_scratch_apply_checkpoint(&ctx->error_callback, scratch, (size_t) -1); /* this is just wildly invalid */ CHECK(ecount == 2); /* try to use badly initialized scratch space */ - secp256k1_scratch_space_destroy(none, scratch); + secp256k1_scratch_space_destroy(ctx, scratch); memset(&local_scratch, 0, sizeof(local_scratch)); scratch = &local_scratch; - CHECK(!secp256k1_scratch_max_allocation(&none->error_callback, scratch, 0)); + CHECK(!secp256k1_scratch_max_allocation(&ctx->error_callback, scratch, 0)); CHECK(ecount == 3); - CHECK(secp256k1_scratch_alloc(&none->error_callback, scratch, 500) == NULL); + CHECK(secp256k1_scratch_alloc(&ctx->error_callback, scratch, 500) == NULL); CHECK(ecount == 4); - secp256k1_scratch_space_destroy(none, scratch); + secp256k1_scratch_space_destroy(ctx, scratch); CHECK(ecount == 5); /* Test that large integers do not wrap around in a bad way */ - scratch = secp256k1_scratch_space_create(none, 1000); + scratch = secp256k1_scratch_space_create(ctx, 1000); /* Try max allocation with a large number of objects. Only makes sense if * ALIGNMENT is greater than 1 because otherwise the objects take no extra * space. */ - CHECK(ALIGNMENT <= 1 || !secp256k1_scratch_max_allocation(&none->error_callback, scratch, (SIZE_MAX / (ALIGNMENT - 1)) + 1)); + CHECK(ALIGNMENT <= 1 || !secp256k1_scratch_max_allocation(&ctx->error_callback, scratch, (SIZE_MAX / (ALIGNMENT - 1)) + 1)); /* Try allocating SIZE_MAX to test wrap around which only happens if * ALIGNMENT > 1, otherwise it returns NULL anyway because the scratch * space is too small. */ - CHECK(secp256k1_scratch_alloc(&none->error_callback, scratch, SIZE_MAX) == NULL); - secp256k1_scratch_space_destroy(none, scratch); + CHECK(secp256k1_scratch_alloc(&ctx->error_callback, scratch, SIZE_MAX) == NULL); + secp256k1_scratch_space_destroy(ctx, scratch); /* cleanup */ - secp256k1_scratch_space_destroy(none, NULL); /* no-op */ - secp256k1_context_destroy(none); + secp256k1_scratch_space_destroy(ctx, NULL); /* no-op */ + secp256k1_context_destroy(ctx); } + void run_ctz_tests(void) { static const uint32_t b32[] = {1, 0xffffffff, 0x5e56968f, 0xe0d63129}; static const uint64_t b64[] = {1, 0xffffffffffffffff, 0xbcd02462139b3fc3, 0x98b5f80c769693ef}; @@ -697,7 +681,6 @@ void run_rfc6979_hmac_sha256_tests(void) { void run_tagged_sha256_tests(void) { int ecount = 0; - secp256k1_context *none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); unsigned char tag[32] = { 0 }; unsigned char msg[32] = { 0 }; unsigned char hash32[32]; @@ -708,23 +691,22 @@ void run_tagged_sha256_tests(void) { 0xE2, 0x76, 0x55, 0x9A, 0x3B, 0xDE, 0x55, 0xB3 }; - secp256k1_context_set_illegal_callback(none, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(ctx, counting_illegal_callback_fn, &ecount); /* API test */ - CHECK(secp256k1_tagged_sha256(none, hash32, tag, sizeof(tag), msg, sizeof(msg)) == 1); - CHECK(secp256k1_tagged_sha256(none, NULL, tag, sizeof(tag), msg, sizeof(msg)) == 0); + CHECK(secp256k1_tagged_sha256(ctx, hash32, tag, sizeof(tag), msg, sizeof(msg)) == 1); + CHECK(secp256k1_tagged_sha256(ctx, NULL, tag, sizeof(tag), msg, sizeof(msg)) == 0); CHECK(ecount == 1); - CHECK(secp256k1_tagged_sha256(none, hash32, NULL, 0, msg, sizeof(msg)) == 0); + CHECK(secp256k1_tagged_sha256(ctx, hash32, NULL, 0, msg, sizeof(msg)) == 0); CHECK(ecount == 2); - CHECK(secp256k1_tagged_sha256(none, hash32, tag, sizeof(tag), NULL, 0) == 0); + CHECK(secp256k1_tagged_sha256(ctx, hash32, tag, sizeof(tag), NULL, 0) == 0); CHECK(ecount == 3); /* Static test vector */ memcpy(tag, "tag", 3); memcpy(msg, "msg", 3); - CHECK(secp256k1_tagged_sha256(none, hash32, tag, 3, msg, 3) == 1); + CHECK(secp256k1_tagged_sha256(ctx, hash32, tag, 3, msg, 3) == 1); CHECK(secp256k1_memcmp_var(hash32, hash_expected, sizeof(hash32)) == 0); - secp256k1_context_destroy(none); } /***** RANDOM TESTS *****/ @@ -814,7 +796,8 @@ uint64_t modinv2p64(uint64_t x) { return w; } -/* compute out = (a*b) mod m; if b=NULL, treat b=1. + +/* compute out = (a*b) mod m; if b=NULL, treat b=1; if m=NULL, treat m=infinity. * * Out is a 512-bit number (represented as 32 uint16_t's in LE order). The other * arguments are 256-bit numbers (represented as 16 uint16_t's in LE order). */ @@ -856,46 +839,48 @@ void mulmod256(uint16_t* out, const uint16_t* a, const uint16_t* b, const uint16 } } - /* Compute the highest set bit in m. */ - for (i = 255; i >= 0; --i) { - if ((m[i >> 4] >> (i & 15)) & 1) { - m_bitlen = i; - break; + if (m) { + /* Compute the highest set bit in m. */ + for (i = 255; i >= 0; --i) { + if ((m[i >> 4] >> (i & 15)) & 1) { + m_bitlen = i; + break; + } } - } - /* Try do mul -= m<<i, for i going down to 0, whenever the result is not negative */ - for (i = mul_bitlen - m_bitlen; i >= 0; --i) { - uint16_t mul2[32]; - int64_t cs; - - /* Compute mul2 = mul - m<<i. */ - cs = 0; /* accumulator */ - for (j = 0; j < 32; ++j) { /* j loops over the output limbs in mul2. */ - /* Compute sub: the 16 bits in m that will be subtracted from mul2[j]. */ - uint16_t sub = 0; - int p; - for (p = 0; p < 16; ++p) { /* p loops over the bit positions in mul2[j]. */ - int bitpos = j * 16 - i + p; /* bitpos is the correspond bit position in m. */ - if (bitpos >= 0 && bitpos < 256) { - sub |= ((m[bitpos >> 4] >> (bitpos & 15)) & 1) << p; + /* Try do mul -= m<<i, for i going down to 0, whenever the result is not negative */ + for (i = mul_bitlen - m_bitlen; i >= 0; --i) { + uint16_t mul2[32]; + int64_t cs; + + /* Compute mul2 = mul - m<<i. */ + cs = 0; /* accumulator */ + for (j = 0; j < 32; ++j) { /* j loops over the output limbs in mul2. */ + /* Compute sub: the 16 bits in m that will be subtracted from mul2[j]. */ + uint16_t sub = 0; + int p; + for (p = 0; p < 16; ++p) { /* p loops over the bit positions in mul2[j]. */ + int bitpos = j * 16 - i + p; /* bitpos is the correspond bit position in m. */ + if (bitpos >= 0 && bitpos < 256) { + sub |= ((m[bitpos >> 4] >> (bitpos & 15)) & 1) << p; + } } + /* Add mul[j]-sub to accumulator, and shift bottom 16 bits out to mul2[j]. */ + cs += mul[j]; + cs -= sub; + mul2[j] = (cs & 0xFFFF); + cs >>= 16; + } + /* If remainder of subtraction is 0, set mul = mul2. */ + if (cs == 0) { + memcpy(mul, mul2, sizeof(mul)); } - /* Add mul[j]-sub to accumulator, and shift bottom 16 bits out to mul2[j]. */ - cs += mul[j]; - cs -= sub; - mul2[j] = (cs & 0xFFFF); - cs >>= 16; } - /* If remainder of subtraction is 0, set mul = mul2. */ - if (cs == 0) { - memcpy(mul, mul2, sizeof(mul)); + /* Sanity check: test that all limbs higher than m's highest are zero */ + for (i = (m_bitlen >> 4) + 1; i < 32; ++i) { + CHECK(mul[i] == 0); } } - /* Sanity check: test that all limbs higher than m's highest are zero */ - for (i = (m_bitlen >> 4) + 1; i < 32; ++i) { - CHECK(mul[i] == 0); - } memcpy(out, mul, 32); } @@ -1710,8 +1695,305 @@ void run_modinv_tests(void) { } } -/***** SCALAR TESTS *****/ +/***** INT128 TESTS *****/ + +#ifdef SECP256K1_WIDEMUL_INT128 +/* Add two 256-bit numbers (represented as 16 uint16_t's in LE order) together mod 2^256. */ +void add256(uint16_t* out, const uint16_t* a, const uint16_t* b) { + int i; + uint32_t carry = 0; + for (i = 0; i < 16; ++i) { + carry += a[i]; + carry += b[i]; + out[i] = carry; + carry >>= 16; + } +} + +/* Negate a 256-bit number (represented as 16 uint16_t's in LE order) mod 2^256. */ +void neg256(uint16_t* out, const uint16_t* a) { + int i; + uint32_t carry = 1; + for (i = 0; i < 16; ++i) { + carry += (uint16_t)~a[i]; + out[i] = carry; + carry >>= 16; + } +} + +/* Right-shift a 256-bit number (represented as 16 uint16_t's in LE order). */ +void rshift256(uint16_t* out, const uint16_t* a, int n, int sign_extend) { + uint16_t sign = sign_extend && (a[15] >> 15); + int i, j; + for (i = 15; i >= 0; --i) { + uint16_t v = 0; + for (j = 0; j < 16; ++j) { + int frompos = i*16 + j + n; + if (frompos >= 256) { + v |= sign << j; + } else { + v |= ((uint16_t)((a[frompos >> 4] >> (frompos & 15)) & 1)) << j; + } + } + out[i] = v; + } +} + +/* Load a 64-bit unsigned integer into an array of 16 uint16_t's in LE order representing a 256-bit value. */ +void load256u64(uint16_t* out, uint64_t v, int is_signed) { + int i; + uint64_t sign = is_signed && (v >> 63) ? UINT64_MAX : 0; + for (i = 0; i < 4; ++i) { + out[i] = v >> (16 * i); + } + for (i = 4; i < 16; ++i) { + out[i] = sign; + } +} + +/* Load a 128-bit unsigned integer into an array of 16 uint16_t's in LE order representing a 256-bit value. */ +void load256two64(uint16_t* out, uint64_t hi, uint64_t lo, int is_signed) { + int i; + uint64_t sign = is_signed && (hi >> 63) ? UINT64_MAX : 0; + for (i = 0; i < 4; ++i) { + out[i] = lo >> (16 * i); + } + for (i = 4; i < 8; ++i) { + out[i] = hi >> (16 * (i - 4)); + } + for (i = 8; i < 16; ++i) { + out[i] = sign; + } +} + +/* Check whether the 256-bit value represented by array of 16-bit values is in range -2^127 < v < 2^127. */ +int int256is127(const uint16_t* v) { + int all_0 = ((v[7] & 0x8000) == 0), all_1 = ((v[7] & 0x8000) == 0x8000); + int i; + for (i = 8; i < 16; ++i) { + if (v[i] != 0) all_0 = 0; + if (v[i] != 0xffff) all_1 = 0; + } + return all_0 || all_1; +} +void load256u128(uint16_t* out, const secp256k1_uint128* v) { + uint64_t lo = secp256k1_u128_to_u64(v), hi = secp256k1_u128_hi_u64(v); + load256two64(out, hi, lo, 0); +} + +void load256i128(uint16_t* out, const secp256k1_int128* v) { + uint64_t lo; + int64_t hi; + secp256k1_int128 c = *v; + lo = secp256k1_i128_to_i64(&c); + secp256k1_i128_rshift(&c, 64); + hi = secp256k1_i128_to_i64(&c); + load256two64(out, hi, lo, 1); +} + +void run_int128_test_case(void) { + unsigned char buf[32]; + uint64_t v[4]; + secp256k1_int128 swa, swz; + secp256k1_uint128 uwa, uwz; + uint64_t ub, uc; + int64_t sb, sc; + uint16_t rswa[16], rswz[32], rswr[32], ruwa[16], ruwz[32], ruwr[32]; + uint16_t rub[16], ruc[16], rsb[16], rsc[16]; + int i; + + /* Generate 32-byte random value. */ + secp256k1_testrand256_test(buf); + /* Convert into 4 64-bit integers. */ + for (i = 0; i < 4; ++i) { + uint64_t vi = 0; + int j; + for (j = 0; j < 8; ++j) vi = (vi << 8) + buf[8*i + j]; + v[i] = vi; + } + /* Convert those into a 128-bit value and two 64-bit values (signed and unsigned). */ + secp256k1_u128_load(&uwa, v[1], v[0]); + secp256k1_i128_load(&swa, v[1], v[0]); + ub = v[2]; + sb = v[2]; + uc = v[3]; + sc = v[3]; + /* Load those also into 16-bit array representations. */ + load256u128(ruwa, &uwa); + load256i128(rswa, &swa); + load256u64(rub, ub, 0); + load256u64(rsb, sb, 1); + load256u64(ruc, uc, 0); + load256u64(rsc, sc, 1); + /* test secp256k1_u128_mul */ + mulmod256(ruwr, rub, ruc, NULL); + secp256k1_u128_mul(&uwz, ub, uc); + load256u128(ruwz, &uwz); + CHECK(secp256k1_memcmp_var(ruwr, ruwz, 16) == 0); + /* test secp256k1_u128_accum_mul */ + mulmod256(ruwr, rub, ruc, NULL); + add256(ruwr, ruwr, ruwa); + uwz = uwa; + secp256k1_u128_accum_mul(&uwz, ub, uc); + load256u128(ruwz, &uwz); + CHECK(secp256k1_memcmp_var(ruwr, ruwz, 16) == 0); + /* test secp256k1_u128_accum_u64 */ + add256(ruwr, rub, ruwa); + uwz = uwa; + secp256k1_u128_accum_u64(&uwz, ub); + load256u128(ruwz, &uwz); + CHECK(secp256k1_memcmp_var(ruwr, ruwz, 16) == 0); + /* test secp256k1_u128_rshift */ + rshift256(ruwr, ruwa, uc % 128, 0); + uwz = uwa; + secp256k1_u128_rshift(&uwz, uc % 128); + load256u128(ruwz, &uwz); + CHECK(secp256k1_memcmp_var(ruwr, ruwz, 16) == 0); + /* test secp256k1_u128_to_u64 */ + CHECK(secp256k1_u128_to_u64(&uwa) == v[0]); + /* test secp256k1_u128_hi_u64 */ + CHECK(secp256k1_u128_hi_u64(&uwa) == v[1]); + /* test secp256k1_u128_from_u64 */ + secp256k1_u128_from_u64(&uwz, ub); + load256u128(ruwz, &uwz); + CHECK(secp256k1_memcmp_var(rub, ruwz, 16) == 0); + /* test secp256k1_u128_check_bits */ + { + int uwa_bits = 0; + int j; + for (j = 0; j < 128; ++j) { + if (ruwa[j / 16] >> (j % 16)) uwa_bits = 1 + j; + } + for (j = 0; j < 128; ++j) { + CHECK(secp256k1_u128_check_bits(&uwa, j) == (uwa_bits <= j)); + } + } + /* test secp256k1_i128_mul */ + mulmod256(rswr, rsb, rsc, NULL); + secp256k1_i128_mul(&swz, sb, sc); + load256i128(rswz, &swz); + CHECK(secp256k1_memcmp_var(rswr, rswz, 16) == 0); + /* test secp256k1_i128_accum_mul */ + mulmod256(rswr, rsb, rsc, NULL); + add256(rswr, rswr, rswa); + if (int256is127(rswr)) { + swz = swa; + secp256k1_i128_accum_mul(&swz, sb, sc); + load256i128(rswz, &swz); + CHECK(secp256k1_memcmp_var(rswr, rswz, 16) == 0); + } + /* test secp256k1_i128_det */ + { + uint16_t rsd[16], rse[16], rst[32]; + int64_t sd = v[0], se = v[1]; + load256u64(rsd, sd, 1); + load256u64(rse, se, 1); + mulmod256(rst, rsc, rsd, NULL); + neg256(rst, rst); + mulmod256(rswr, rsb, rse, NULL); + add256(rswr, rswr, rst); + secp256k1_i128_det(&swz, sb, sc, sd, se); + load256i128(rswz, &swz); + CHECK(secp256k1_memcmp_var(rswr, rswz, 16) == 0); + } + /* test secp256k1_i128_rshift */ + rshift256(rswr, rswa, uc % 127, 1); + swz = swa; + secp256k1_i128_rshift(&swz, uc % 127); + load256i128(rswz, &swz); + CHECK(secp256k1_memcmp_var(rswr, rswz, 16) == 0); + /* test secp256k1_i128_to_i64 */ + CHECK((uint64_t)secp256k1_i128_to_i64(&swa) == v[0]); + /* test secp256k1_i128_from_i64 */ + secp256k1_i128_from_i64(&swz, sb); + load256i128(rswz, &swz); + CHECK(secp256k1_memcmp_var(rsb, rswz, 16) == 0); + /* test secp256k1_i128_eq_var */ + { + int expect = (uc & 1); + swz = swa; + if (!expect) { + /* Make sure swz != swa */ + uint64_t v0c = v[0], v1c = v[1]; + if (ub & 64) { + v1c ^= (((uint64_t)1) << (ub & 63)); + } else { + v0c ^= (((uint64_t)1) << (ub & 63)); + } + secp256k1_i128_load(&swz, v1c, v0c); + } + CHECK(secp256k1_i128_eq_var(&swa, &swz) == expect); + } + /* test secp256k1_i128_check_pow2 */ + { + int expect = (uc & 1); + int pos = ub % 127; + if (expect) { + /* If expect==1, set swz to exactly (2 << pos). */ + uint64_t hi = 0; + uint64_t lo = 0; + if (pos & 64) { + hi = (((uint64_t)1) << (pos & 63)); + } else { + lo = (((uint64_t)1) << (pos & 63)); + } + secp256k1_i128_load(&swz, hi, lo); + } else { + /* If expect==0, set swz = swa, but update expect=1 if swa happens to equal (2 << pos). */ + if (pos & 64) { + if ((v[1] == (((uint64_t)1) << (pos & 63))) && v[0] == 0) expect = 1; + } else { + if ((v[0] == (((uint64_t)1) << (pos & 63))) && v[1] == 0) expect = 1; + } + swz = swa; + } + CHECK(secp256k1_i128_check_pow2(&swz, pos) == expect); + } +} + +void run_int128_tests(void) { + { /* secp256k1_u128_accum_mul */ + secp256k1_uint128 res; + + /* Check secp256k1_u128_accum_mul overflow */ + secp256k1_u128_mul(&res, UINT64_MAX, UINT64_MAX); + secp256k1_u128_accum_mul(&res, UINT64_MAX, UINT64_MAX); + CHECK(secp256k1_u128_to_u64(&res) == 2); + CHECK(secp256k1_u128_hi_u64(&res) == 18446744073709551612U); + } + { /* secp256k1_u128_accum_mul */ + secp256k1_int128 res; + + /* Compute INT128_MAX = 2^127 - 1 with secp256k1_i128_accum_mul */ + secp256k1_i128_mul(&res, INT64_MAX, INT64_MAX); + secp256k1_i128_accum_mul(&res, INT64_MAX, INT64_MAX); + CHECK(secp256k1_i128_to_i64(&res) == 2); + secp256k1_i128_accum_mul(&res, 4, 9223372036854775807); + secp256k1_i128_accum_mul(&res, 1, 1); + CHECK((uint64_t)secp256k1_i128_to_i64(&res) == UINT64_MAX); + secp256k1_i128_rshift(&res, 64); + CHECK(secp256k1_i128_to_i64(&res) == INT64_MAX); + + /* Compute INT128_MIN = - 2^127 with secp256k1_i128_accum_mul */ + secp256k1_i128_mul(&res, INT64_MAX, INT64_MIN); + CHECK(secp256k1_i128_to_i64(&res) == INT64_MIN); + secp256k1_i128_accum_mul(&res, INT64_MAX, INT64_MIN); + CHECK(secp256k1_i128_to_i64(&res) == 0); + secp256k1_i128_accum_mul(&res, 2, INT64_MIN); + CHECK(secp256k1_i128_to_i64(&res) == 0); + secp256k1_i128_rshift(&res, 64); + CHECK(secp256k1_i128_to_i64(&res) == INT64_MIN); + } + { + /* Randomized tests. */ + int i; + for (i = 0; i < 256 * count; ++i) run_int128_test_case(); + } +} +#endif + +/***** SCALAR TESTS *****/ void scalar_test(void) { secp256k1_scalar s; @@ -3562,6 +3844,22 @@ void run_gej(void) { test_gej_cmov(&a, &b); test_gej_cmov(&b, &a); } + + /* Tests for secp256k1_gej_eq_var */ + for (i = 0; i < count; i++) { + secp256k1_fe fe; + random_gej_test(&a); + random_gej_test(&b); + CHECK(!secp256k1_gej_eq_var(&a, &b)); + + b = a; + random_field_element_test(&fe); + if (secp256k1_fe_is_zero(&fe)) { + continue; + } + secp256k1_gej_rescale(&a, &fe); + CHECK(secp256k1_gej_eq_var(&a, &b)); + } } void test_ec_combine(void) { @@ -3767,17 +4065,12 @@ void run_ecmult_chain(void) { 0xB95CBCA2, 0xC77DA786, 0x539BE8FD, 0x53354D2D, 0x3B4F566A, 0xE6580454, 0x07ED6015, 0xEE1B2A88 ); - - secp256k1_gej_neg(&rp, &rp); - secp256k1_gej_add_var(&rp, &rp, &x, NULL); - CHECK(secp256k1_gej_is_infinity(&rp)); + CHECK(secp256k1_gej_eq_var(&rp, &x)); } } /* redo the computation, but directly with the resulting ae and ge coefficients: */ secp256k1_ecmult(&x2, &a, &ae, &ge); - secp256k1_gej_neg(&x2, &x2); - secp256k1_gej_add_var(&x2, &x2, &x, NULL); - CHECK(secp256k1_gej_is_infinity(&x2)); + CHECK(secp256k1_gej_eq_var(&x, &x2)); } void test_point_times_order(const secp256k1_gej *point) { @@ -4070,16 +4363,12 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e /* only G scalar */ secp256k1_ecmult(&r2, &ptgj, &szero, &sc[0]); CHECK(ecmult_multi(&ctx->error_callback, scratch, &r, &sc[0], ecmult_multi_callback, &data, 0)); - secp256k1_gej_neg(&r2, &r2); - secp256k1_gej_add_var(&r, &r, &r2, NULL); - CHECK(secp256k1_gej_is_infinity(&r)); + CHECK(secp256k1_gej_eq_var(&r, &r2)); /* 1-point */ secp256k1_ecmult(&r2, &ptgj, &sc[0], &szero); CHECK(ecmult_multi(&ctx->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, 1)); - secp256k1_gej_neg(&r2, &r2); - secp256k1_gej_add_var(&r, &r, &r2, NULL); - CHECK(secp256k1_gej_is_infinity(&r)); + CHECK(secp256k1_gej_eq_var(&r, &r2)); /* Try to multiply 1 point, but callback returns false */ CHECK(!ecmult_multi(&ctx->error_callback, scratch, &r, &szero, ecmult_multi_false_callback, &data, 1)); @@ -4087,16 +4376,12 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e /* 2-point */ secp256k1_ecmult(&r2, &ptgj, &sc[0], &sc[1]); CHECK(ecmult_multi(&ctx->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, 2)); - secp256k1_gej_neg(&r2, &r2); - secp256k1_gej_add_var(&r, &r, &r2, NULL); - CHECK(secp256k1_gej_is_infinity(&r)); + CHECK(secp256k1_gej_eq_var(&r, &r2)); /* 2-point with G scalar */ secp256k1_ecmult(&r2, &ptgj, &sc[0], &sc[1]); CHECK(ecmult_multi(&ctx->error_callback, scratch, &r, &sc[1], ecmult_multi_callback, &data, 1)); - secp256k1_gej_neg(&r2, &r2); - secp256k1_gej_add_var(&r, &r, &r2, NULL); - CHECK(secp256k1_gej_is_infinity(&r)); + CHECK(secp256k1_gej_eq_var(&r, &r2)); } /* Check infinite outputs of various forms */ @@ -4181,9 +4466,7 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e secp256k1_ecmult(&r2, &r, &sc[0], &szero); CHECK(ecmult_multi(&ctx->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, 20)); - secp256k1_gej_neg(&r2, &r2); - secp256k1_gej_add_var(&r, &r, &r2, NULL); - CHECK(secp256k1_gej_is_infinity(&r)); + CHECK(secp256k1_gej_eq_var(&r, &r2)); } /* Check random scalars, constant point */ @@ -4204,9 +4487,7 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e secp256k1_gej_set_ge(&p0j, &pt[0]); secp256k1_ecmult(&r2, &p0j, &rs, &szero); CHECK(ecmult_multi(&ctx->error_callback, scratch, &r, &szero, ecmult_multi_callback, &data, 20)); - secp256k1_gej_neg(&r2, &r2); - secp256k1_gej_add_var(&r, &r, &r2, NULL); - CHECK(secp256k1_gej_is_infinity(&r)); + CHECK(secp256k1_gej_eq_var(&r, &r2)); } /* Sanity check that zero scalars don't cause problems */ @@ -4268,9 +4549,7 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e secp256k1_ecmult(&expected, &ptgj, &tmp1, &szero); CHECK(ecmult_multi(&ctx->error_callback, scratch, &actual, &szero, ecmult_multi_callback, &data, 2)); - secp256k1_gej_neg(&expected, &expected); - secp256k1_gej_add_var(&actual, &actual, &expected, NULL); - CHECK(secp256k1_gej_is_infinity(&actual)); + CHECK(secp256k1_gej_eq_var(&actual, &expected)); } } } @@ -4440,9 +4719,7 @@ int test_ecmult_multi_random(secp256k1_scratch *scratch) { CHECK(ecmult_multi(&ctx->error_callback, scratch, &computed, g_scalar_ptr, ecmult_multi_callback, &data, filled)); mults += num_nonzero + g_nonzero; /* Compare with expected result. */ - secp256k1_gej_neg(&computed, &computed); - secp256k1_gej_add_var(&computed, &computed, &expected, NULL); - CHECK(secp256k1_gej_is_infinity(&computed)); + CHECK(secp256k1_gej_eq_var(&computed, &expected)); return mults; } @@ -5497,7 +5774,7 @@ void run_ec_pubkey_parse_test(void) { ecount = 0; VG_UNDEF(&pubkey, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_parse(ctx, &pubkey, pubkeyc, 65) == 1); - CHECK(secp256k1_ec_pubkey_parse(secp256k1_context_no_precomp, &pubkey, pubkeyc, 65) == 1); + CHECK(secp256k1_ec_pubkey_parse(secp256k1_context_static, &pubkey, pubkeyc, 65) == 1); VG_CHECK(&pubkey, sizeof(pubkey)); CHECK(ecount == 0); VG_UNDEF(&ge, sizeof(ge)); @@ -7083,19 +7360,27 @@ int main(int argc, char **argv) { secp256k1_testrand_init(argc > 2 ? argv[2] : NULL); /* initialize */ + run_selftest_tests(); run_context_tests(0); run_context_tests(1); run_scratch_tests(); - ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); - if (secp256k1_testrand_bits(1)) { + + ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); + /* Randomize the context only with probability 15/16 + to make sure we test without context randomization from time to time. + TODO Reconsider this when recalibrating the tests. */ + if (secp256k1_testrand_bits(4)) { unsigned char rand32[32]; secp256k1_testrand256(rand32); - CHECK(secp256k1_context_randomize(ctx, secp256k1_testrand_bits(1) ? rand32 : NULL)); + CHECK(secp256k1_context_randomize(ctx, rand32)); } run_rand_bits(); run_rand_int(); +#ifdef SECP256K1_WIDEMUL_INT128 + run_int128_tests(); +#endif run_ctz_tests(); run_modinv_tests(); run_inverse_tests(); diff --git a/src/secp256k1/src/tests_exhaustive.c b/src/secp256k1/src/tests_exhaustive.c index 6a4e2340f2..c001dcb80b 100644 --- a/src/secp256k1/src/tests_exhaustive.c +++ b/src/secp256k1/src/tests_exhaustive.c @@ -342,15 +342,15 @@ void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *grou } #ifdef ENABLE_MODULE_RECOVERY -#include "src/modules/recovery/tests_exhaustive_impl.h" +#include "modules/recovery/tests_exhaustive_impl.h" #endif #ifdef ENABLE_MODULE_EXTRAKEYS -#include "src/modules/extrakeys/tests_exhaustive_impl.h" +#include "modules/extrakeys/tests_exhaustive_impl.h" #endif #ifdef ENABLE_MODULE_SCHNORRSIG -#include "src/modules/schnorrsig/tests_exhaustive_impl.h" +#include "modules/schnorrsig/tests_exhaustive_impl.h" #endif int main(int argc, char** argv) { @@ -396,7 +396,7 @@ int main(int argc, char** argv) { while (count--) { /* Build context */ - ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + ctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); secp256k1_testrand256(rand32); CHECK(secp256k1_context_randomize(ctx, rand32)); diff --git a/src/secp256k1/src/util.h b/src/secp256k1/src/util.h index dac86bd77f..864baaee4d 100644 --- a/src/secp256k1/src/util.h +++ b/src/secp256k1/src/util.h @@ -16,6 +16,11 @@ #include <stdio.h> #include <limits.h> +#define STR_(x) #x +#define STR(x) STR_(x) +#define DEBUG_CONFIG_MSG(x) "DEBUG_CONFIG: " x +#define DEBUG_CONFIG_DEF(x) DEBUG_CONFIG_MSG(#x "=" STR(x)) + typedef struct { void (*fn)(const char *text, void* data); const void* data; @@ -225,28 +230,36 @@ static SECP256K1_INLINE void secp256k1_int_cmov(int *r, const int *a, int flag) *r = (int)(r_masked | a_masked); } -/* If USE_FORCE_WIDEMUL_{INT128,INT64} is set, use that wide multiplication implementation. - * Otherwise use the presence of __SIZEOF_INT128__ to decide. - */ -#if defined(USE_FORCE_WIDEMUL_INT128) +#if defined(USE_FORCE_WIDEMUL_INT128_STRUCT) +/* If USE_FORCE_WIDEMUL_INT128_STRUCT is set, use int128_struct. */ # define SECP256K1_WIDEMUL_INT128 1 +# define SECP256K1_INT128_STRUCT 1 +#elif defined(USE_FORCE_WIDEMUL_INT128) +/* If USE_FORCE_WIDEMUL_INT128 is set, use int128. */ +# define SECP256K1_WIDEMUL_INT128 1 +# define SECP256K1_INT128_NATIVE 1 #elif defined(USE_FORCE_WIDEMUL_INT64) +/* If USE_FORCE_WIDEMUL_INT64 is set, use int64. */ # define SECP256K1_WIDEMUL_INT64 1 #elif defined(UINT128_MAX) || defined(__SIZEOF_INT128__) +/* If a native 128-bit integer type exists, use int128. */ +# define SECP256K1_WIDEMUL_INT128 1 +# define SECP256K1_INT128_NATIVE 1 +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) +/* On 64-bit MSVC targets (x86_64 and arm64), use int128_struct + * (which has special logic to implement using intrinsics on those systems). */ # define SECP256K1_WIDEMUL_INT128 1 +# define SECP256K1_INT128_STRUCT 1 +#elif SIZE_MAX > 0xffffffff +/* Systems with 64-bit pointers (and thus registers) very likely benefit from + * using 64-bit based arithmetic (even if we need to fall back to 32x32->64 based + * multiplication logic). */ +# define SECP256K1_WIDEMUL_INT128 1 +# define SECP256K1_INT128_STRUCT 1 #else +/* Lastly, fall back to int64 based arithmetic. */ # define SECP256K1_WIDEMUL_INT64 1 #endif -#if defined(SECP256K1_WIDEMUL_INT128) -# if !defined(UINT128_MAX) && defined(__SIZEOF_INT128__) -SECP256K1_GNUC_EXT typedef unsigned __int128 uint128_t; -SECP256K1_GNUC_EXT typedef __int128 int128_t; -#define UINT128_MAX ((uint128_t)(-1)) -#define INT128_MAX ((int128_t)(UINT128_MAX >> 1)) -#define INT128_MIN (-INT128_MAX - 1) -/* No (U)INT128_C macros because compilers providing __int128 do not support 128-bit literals. */ -# endif -#endif #ifndef __has_builtin #define __has_builtin(x) 0 diff --git a/src/secp256k1/src/valgrind_ctime_test.c b/src/secp256k1/src/valgrind_ctime_test.c index 6ff0085d34..a0f888b00f 100644 --- a/src/secp256k1/src/valgrind_ctime_test.c +++ b/src/secp256k1/src/valgrind_ctime_test.c @@ -39,9 +39,7 @@ int main(void) { fprintf(stderr, "Usage: libtool --mode=execute valgrind ./valgrind_ctime_test\n"); return 1; } - ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN - | SECP256K1_CONTEXT_VERIFY - | SECP256K1_CONTEXT_DECLASSIFY); + ctx = secp256k1_context_create(SECP256K1_CONTEXT_DECLASSIFY); /** In theory, testing with a single secret input should be sufficient: * If control flow depended on secrets the tool would generate an error. */ diff --git a/src/serialize.h b/src/serialize.h index f1edc54031..7bc7b10779 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -1005,11 +1005,11 @@ struct CSerActionUnserialize class CSizeComputer { protected: - size_t nSize; + size_t nSize{0}; const int nVersion; public: - explicit CSizeComputer(int nVersionIn) : nSize(0), nVersion(nVersionIn) {} + explicit CSizeComputer(int nVersionIn) : nVersion(nVersionIn) {} void write(Span<const std::byte> src) { diff --git a/src/span.h b/src/span.h index 4d00bbc244..4692eca7fb 100644 --- a/src/span.h +++ b/src/span.h @@ -96,7 +96,7 @@ template<typename C> class Span { C* m_data; - std::size_t m_size; + std::size_t m_size{0}; template <class T> struct is_Span_int : public std::false_type {}; @@ -107,7 +107,7 @@ class Span public: - constexpr Span() noexcept : m_data(nullptr), m_size(0) {} + constexpr Span() noexcept : m_data(nullptr) {} /** Construct a span from a begin pointer and a size. * diff --git a/src/streams.h b/src/streams.h index 4f2c3ffe76..8788343809 100644 --- a/src/streams.h +++ b/src/streams.h @@ -182,16 +182,13 @@ public: * >> and << read and write unformatted data using the above serialization templates. * Fills with data in linear time; some stringstream implementations take N^2 time. */ -class CDataStream +class DataStream { protected: using vector_type = SerializeData; vector_type vch; vector_type::size_type m_read_pos{0}; - int nType; - int nVersion; - public: typedef vector_type::allocator_type allocator_type; typedef vector_type::size_type size_type; @@ -203,23 +200,9 @@ public: typedef vector_type::const_iterator const_iterator; typedef vector_type::reverse_iterator reverse_iterator; - explicit CDataStream(int nTypeIn, int nVersionIn) - : nType{nTypeIn}, - nVersion{nVersionIn} {} - - explicit CDataStream(Span<const uint8_t> sp, int type, int version) : CDataStream{AsBytes(sp), type, version} {} - explicit CDataStream(Span<const value_type> sp, int nTypeIn, int nVersionIn) - : vch(sp.data(), sp.data() + sp.size()), - nType{nTypeIn}, - nVersion{nVersionIn} {} - - template <typename... Args> - CDataStream(int nTypeIn, int nVersionIn, Args&&... args) - : nType{nTypeIn}, - nVersion{nVersionIn} - { - ::SerializeMany(*this, std::forward<Args>(args)...); - } + explicit DataStream() {} + explicit DataStream(Span<const uint8_t> sp) : DataStream{AsBytes(sp)} {} + explicit DataStream(Span<const value_type> sp) : vch(sp.data(), sp.data() + sp.size()) {} std::string str() const { @@ -271,11 +254,6 @@ public: bool eof() const { return size() == 0; } int in_avail() const { return size(); } - void SetType(int n) { nType = n; } - int GetType() const { return nType; } - void SetVersion(int n) { nVersion = n; } - int GetVersion() const { return nVersion; } - void read(Span<value_type> dst) { if (dst.size() == 0) return; @@ -283,7 +261,7 @@ public: // Read from the beginning of the buffer auto next_read_pos{CheckedAdd(m_read_pos, dst.size())}; if (!next_read_pos.has_value() || next_read_pos.value() > vch.size()) { - throw std::ios_base::failure("CDataStream::read(): end of data"); + throw std::ios_base::failure("DataStream::read(): end of data"); } memcpy(dst.data(), &vch[m_read_pos], dst.size()); if (next_read_pos.value() == vch.size()) { @@ -299,7 +277,7 @@ public: // Ignore from the beginning of the buffer auto next_read_pos{CheckedAdd(m_read_pos, num_ignore)}; if (!next_read_pos.has_value() || next_read_pos.value() > vch.size()) { - throw std::ios_base::failure("CDataStream::ignore(): end of data"); + throw std::ios_base::failure("DataStream::ignore(): end of data"); } if (next_read_pos.value() == vch.size()) { m_read_pos = 0; @@ -324,7 +302,7 @@ public: } template<typename T> - CDataStream& operator<<(const T& obj) + DataStream& operator<<(const T& obj) { // Serialize to this stream ::Serialize(*this, obj); @@ -332,7 +310,7 @@ public: } template<typename T> - CDataStream& operator>>(T&& obj) + DataStream& operator>>(T&& obj) { // Unserialize from this stream ::Unserialize(*this, obj); @@ -363,6 +341,42 @@ public: } }; +class CDataStream : public DataStream +{ +private: + int nType; + int nVersion; + +public: + explicit CDataStream(int nTypeIn, int nVersionIn) + : nType{nTypeIn}, + nVersion{nVersionIn} {} + + explicit CDataStream(Span<const uint8_t> sp, int type, int version) : CDataStream{AsBytes(sp), type, version} {} + explicit CDataStream(Span<const value_type> sp, int nTypeIn, int nVersionIn) + : DataStream{sp}, + nType{nTypeIn}, + nVersion{nVersionIn} {} + + int GetType() const { return nType; } + void SetVersion(int n) { nVersion = n; } + int GetVersion() const { return nVersion; } + + template <typename T> + CDataStream& operator<<(const T& obj) + { + ::Serialize(*this, obj); + return *this; + } + + template <typename T> + CDataStream& operator>>(T&& obj) + { + ::Unserialize(*this, obj); + return *this; + } +}; + template <typename IStream> class BitStreamReader { @@ -606,8 +620,8 @@ private: const int nVersion; FILE *src; //!< source file - uint64_t nSrcPos; //!< how many bytes have been read from source - uint64_t m_read_pos; //!< how many bytes have been read from this + uint64_t nSrcPos{0}; //!< how many bytes have been read from source + uint64_t m_read_pos{0}; //!< how many bytes have been read from this uint64_t nReadLimit; //!< up to which position we're allowed to read uint64_t nRewind; //!< how many bytes we guarantee to rewind std::vector<std::byte> vchBuf; //!< the buffer @@ -653,7 +667,7 @@ private: public: CBufferedFile(FILE* fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) - : nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), m_read_pos(0), nReadLimit(std::numeric_limits<uint64_t>::max()), nRewind(nRewindIn), vchBuf(nBufSize, std::byte{0}) + : nType(nTypeIn), nVersion(nVersionIn), nReadLimit(std::numeric_limits<uint64_t>::max()), nRewind(nRewindIn), vchBuf(nBufSize, std::byte{0}) { if (nRewindIn >= nBufSize) throw std::ios_base::failure("Rewind limit must be less than buffer size"); diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index fb59324f7a..24ae4bdd1e 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -19,6 +19,9 @@ #endif #include <algorithm> +#include <limits> +#include <stdexcept> +#include <utility> #ifdef ARENA_DEBUG #include <iomanip> #include <iostream> @@ -277,8 +280,8 @@ size_t PosixLockedPageAllocator::GetLimit() /*******************************************************************************/ // Implementation: LockedPool -LockedPool::LockedPool(std::unique_ptr<LockedPageAllocator> allocator_in, LockingFailed_Callback lf_cb_in): - allocator(std::move(allocator_in)), lf_cb(lf_cb_in), cumulative_bytes_locked(0) +LockedPool::LockedPool(std::unique_ptr<LockedPageAllocator> allocator_in, LockingFailed_Callback lf_cb_in) + : allocator(std::move(allocator_in)), lf_cb(lf_cb_in) { } diff --git a/src/support/lockedpool.h b/src/support/lockedpool.h index 03e4e371a3..1bba459377 100644 --- a/src/support/lockedpool.h +++ b/src/support/lockedpool.h @@ -5,11 +5,11 @@ #ifndef BITCOIN_SUPPORT_LOCKEDPOOL_H #define BITCOIN_SUPPORT_LOCKEDPOOL_H -#include <stdint.h> +#include <cstddef> #include <list> #include <map> -#include <mutex> #include <memory> +#include <mutex> #include <unordered_map> /** @@ -198,7 +198,7 @@ private: std::list<LockedPageArena> arenas; LockingFailed_Callback lf_cb; - size_t cumulative_bytes_locked; + size_t cumulative_bytes_locked{0}; /** Mutex protects access to this pool's data structures, including arenas. */ mutable std::mutex mutex; diff --git a/src/sync.h b/src/sync.h index 8ce2e7b124..7242a793ab 100644 --- a/src/sync.h +++ b/src/sync.h @@ -11,7 +11,7 @@ #include <logging/timer.h> #endif -#include <threadsafety.h> +#include <threadsafety.h> // IWYU pragma: export #include <util/macros.h> #include <condition_variable> diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index b15df43e8c..586cec4081 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -67,14 +67,14 @@ BOOST_AUTO_TEST_CASE(addrman_simple) CNetAddr source = ResolveIP("252.2.2.2"); // Test: Does Addrman respond correctly when empty. - BOOST_CHECK_EQUAL(addrman->size(), 0U); + BOOST_CHECK_EQUAL(addrman->Size(), 0U); auto addr_null = addrman->Select().first; BOOST_CHECK_EQUAL(addr_null.ToString(), "[::]:0"); // Test: Does Addrman::Add work as expected. CService addr1 = ResolveService("250.1.1.1", 8333); BOOST_CHECK(addrman->Add({CAddress(addr1, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman->size(), 1U); + BOOST_CHECK_EQUAL(addrman->Size(), 1U); auto addr_ret1 = addrman->Select().first; BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); @@ -82,7 +82,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) // Expected dup IP should not be added. CService addr1_dup = ResolveService("250.1.1.1", 8333); BOOST_CHECK(!addrman->Add({CAddress(addr1_dup, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman->size(), 1U); + BOOST_CHECK_EQUAL(addrman->Size(), 1U); // Test: New table has one addr and we add a diff addr we should @@ -93,7 +93,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) CService addr2 = ResolveService("250.1.1.2", 8333); BOOST_CHECK(addrman->Add({CAddress(addr2, NODE_NONE)}, source)); - BOOST_CHECK(addrman->size() >= 1); + BOOST_CHECK(addrman->Size() >= 1); // Test: reset addrman and test AddrMan::Add multiple addresses works as expected addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); @@ -101,7 +101,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) vAddr.push_back(CAddress(ResolveService("250.1.1.3", 8333), NODE_NONE)); vAddr.push_back(CAddress(ResolveService("250.1.1.4", 8333), NODE_NONE)); BOOST_CHECK(addrman->Add(vAddr, source)); - BOOST_CHECK(addrman->size() >= 1); + BOOST_CHECK(addrman->Size() >= 1); } BOOST_AUTO_TEST_CASE(addrman_ports) @@ -110,23 +110,23 @@ BOOST_AUTO_TEST_CASE(addrman_ports) CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK_EQUAL(addrman->size(), 0U); + BOOST_CHECK_EQUAL(addrman->Size(), 0U); // Test 7; Addr with same IP but diff port does not replace existing addr. CService addr1 = ResolveService("250.1.1.1", 8333); BOOST_CHECK(addrman->Add({CAddress(addr1, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman->size(), 1U); + BOOST_CHECK_EQUAL(addrman->Size(), 1U); CService addr1_port = ResolveService("250.1.1.1", 8334); BOOST_CHECK(addrman->Add({CAddress(addr1_port, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman->size(), 2U); + BOOST_CHECK_EQUAL(addrman->Size(), 2U); auto addr_ret2 = addrman->Select().first; BOOST_CHECK(addr_ret2.ToString() == "250.1.1.1:8333" || addr_ret2.ToString() == "250.1.1.1:8334"); // Test: Add same IP but diff port to tried table; this converts the entry with // the specified port to tried, but not the other. addrman->Good(CAddress(addr1_port, NODE_NONE)); - BOOST_CHECK_EQUAL(addrman->size(), 2U); + BOOST_CHECK_EQUAL(addrman->Size(), 2U); bool newOnly = true; auto addr_ret3 = addrman->Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); @@ -142,7 +142,7 @@ BOOST_AUTO_TEST_CASE(addrman_select) // Test: Select from new with 1 addr in new. CService addr1 = ResolveService("250.1.1.1", 8333); BOOST_CHECK(addrman->Add({CAddress(addr1, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman->size(), 1U); + BOOST_CHECK_EQUAL(addrman->Size(), 1U); bool newOnly = true; auto addr_ret1 = addrman->Select(newOnly).first; @@ -150,14 +150,14 @@ BOOST_AUTO_TEST_CASE(addrman_select) // Test: move addr to tried, select from new expected nothing returned. BOOST_CHECK(addrman->Good(CAddress(addr1, NODE_NONE))); - BOOST_CHECK_EQUAL(addrman->size(), 1U); + BOOST_CHECK_EQUAL(addrman->Size(), 1U); auto addr_ret2 = addrman->Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret2.ToString(), "[::]:0"); auto addr_ret3 = addrman->Select().first; BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); - BOOST_CHECK_EQUAL(addrman->size(), 1U); + BOOST_CHECK_EQUAL(addrman->Size(), 1U); // Add three addresses to new table. @@ -182,7 +182,7 @@ BOOST_AUTO_TEST_CASE(addrman_select) BOOST_CHECK(addrman->Good(CAddress(addr7, NODE_NONE))); // Test: 6 addrs + 1 addr from last test = 7. - BOOST_CHECK_EQUAL(addrman->size(), 7U); + BOOST_CHECK_EQUAL(addrman->Size(), 7U); // Test: Select pulls from new and tried regardless of port number. std::set<uint16_t> ports; @@ -200,25 +200,25 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) uint32_t num_addrs{0}; - BOOST_CHECK_EQUAL(addrman->size(), num_addrs); + BOOST_CHECK_EQUAL(addrman->Size(), num_addrs); while (num_addrs < 22) { // Magic number! 250.1.1.1 - 250.1.1.22 do not collide with deterministic key = 1 CService addr = ResolveService("250.1.1." + ToString(++num_addrs)); BOOST_CHECK(addrman->Add({CAddress(addr, NODE_NONE)}, source)); // Test: No collision in new table yet. - BOOST_CHECK_EQUAL(addrman->size(), num_addrs); + BOOST_CHECK_EQUAL(addrman->Size(), num_addrs); } // Test: new table collision! CService addr1 = ResolveService("250.1.1." + ToString(++num_addrs)); uint32_t collisions{1}; BOOST_CHECK(addrman->Add({CAddress(addr1, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman->size(), num_addrs - collisions); + BOOST_CHECK_EQUAL(addrman->Size(), num_addrs - collisions); CService addr2 = ResolveService("250.1.1." + ToString(++num_addrs)); BOOST_CHECK(addrman->Add({CAddress(addr2, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman->size(), num_addrs - collisions); + BOOST_CHECK_EQUAL(addrman->Size(), num_addrs - collisions); } BOOST_AUTO_TEST_CASE(addrman_new_multiplicity) @@ -236,7 +236,7 @@ BOOST_AUTO_TEST_CASE(addrman_new_multiplicity) } AddressPosition addr_pos = addrman->FindAddressEntry(addr).value(); BOOST_CHECK_EQUAL(addr_pos.multiplicity, 1U); - BOOST_CHECK_EQUAL(addrman->size(), 1U); + BOOST_CHECK_EQUAL(addrman->Size(), 1U); // if nTime increases, an addr can occur in up to 8 buckets // The acceptance probability decreases exponentially with existing multiplicity - @@ -250,7 +250,7 @@ BOOST_AUTO_TEST_CASE(addrman_new_multiplicity) AddressPosition addr_pos_multi = addrman->FindAddressEntry(addr).value(); BOOST_CHECK_EQUAL(addr_pos_multi.multiplicity, 8U); // multiplicity doesn't affect size - BOOST_CHECK_EQUAL(addrman->size(), 1U); + BOOST_CHECK_EQUAL(addrman->Size(), 1U); } BOOST_AUTO_TEST_CASE(addrman_tried_collisions) @@ -261,7 +261,7 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) uint32_t num_addrs{0}; - BOOST_CHECK_EQUAL(addrman->size(), num_addrs); + BOOST_CHECK_EQUAL(addrman->Size(), num_addrs); while (num_addrs < 35) { // Magic number! 250.1.1.1 - 250.1.1.35 do not collide in tried with deterministic key = 1 CService addr = ResolveService("250.1.1." + ToString(++num_addrs)); @@ -290,7 +290,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) // Test: Sanity check, GetAddr should never return anything if addrman // is empty. - BOOST_CHECK_EQUAL(addrman->size(), 0U); + BOOST_CHECK_EQUAL(addrman->Size(), 0U); std::vector<CAddress> vAddr1 = addrman->GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt); BOOST_CHECK_EQUAL(vAddr1.size(), 0U); @@ -336,11 +336,11 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) } std::vector<CAddress> vAddr = addrman->GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt); - size_t percent23 = (addrman->size() * 23) / 100; + size_t percent23 = (addrman->Size() * 23) / 100; BOOST_CHECK_EQUAL(vAddr.size(), percent23); BOOST_CHECK_EQUAL(vAddr.size(), 461U); - // (Addrman.size() < number of addresses added) due to address collisions. - BOOST_CHECK_EQUAL(addrman->size(), 2006U); + // (addrman.Size() < number of addresses added) due to address collisions. + BOOST_CHECK_EQUAL(addrman->Size(), 2006U); } @@ -681,7 +681,7 @@ BOOST_AUTO_TEST_CASE(remove_invalid) addrman->Add({new1, tried1, new2, tried2}, CNetAddr{}); addrman->Good(tried1); addrman->Good(tried2); - BOOST_REQUIRE_EQUAL(addrman->size(), 4); + BOOST_REQUIRE_EQUAL(addrman->Size(), 4); stream << *addrman; @@ -704,14 +704,14 @@ BOOST_AUTO_TEST_CASE(remove_invalid) addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); stream >> *addrman; - BOOST_CHECK_EQUAL(addrman->size(), 2); + BOOST_CHECK_EQUAL(addrman->Size(), 2); } BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) { auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); - BOOST_CHECK(addrman->size() == 0); + BOOST_CHECK(addrman->Size() == 0); // Empty addrman should return blank addrman info. BOOST_CHECK(addrman->SelectTriedCollision().first.ToString() == "[::]:0"); @@ -796,7 +796,7 @@ BOOST_AUTO_TEST_CASE(addrman_evictionworks) { auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); - BOOST_CHECK(addrman->size() == 0); + BOOST_CHECK(addrman->Size() == 0); // Empty addrman should return blank addrman info. BOOST_CHECK(addrman->SelectTriedCollision().first.ToString() == "[::]:0"); @@ -878,14 +878,14 @@ BOOST_AUTO_TEST_CASE(load_addrman) BOOST_CHECK(Lookup("252.5.1.1", source, 8333, false)); std::vector<CAddress> addresses{CAddress(addr1, NODE_NONE), CAddress(addr2, NODE_NONE), CAddress(addr3, NODE_NONE)}; BOOST_CHECK(addrman.Add(addresses, source)); - BOOST_CHECK(addrman.size() == 3); + BOOST_CHECK(addrman.Size() == 3); // Test that the de-serialization does not throw an exception. CDataStream ssPeers1 = AddrmanToStream(addrman); bool exceptionThrown = false; AddrMan addrman1{EMPTY_NETGROUPMAN, !DETERMINISTIC, GetCheckRatio(m_node)}; - BOOST_CHECK(addrman1.size() == 0); + BOOST_CHECK(addrman1.Size() == 0); try { unsigned char pchMsgTmp[4]; ssPeers1 >> pchMsgTmp; @@ -894,16 +894,16 @@ BOOST_AUTO_TEST_CASE(load_addrman) exceptionThrown = true; } - BOOST_CHECK(addrman1.size() == 3); + BOOST_CHECK(addrman1.Size() == 3); BOOST_CHECK(exceptionThrown == false); // Test that ReadFromStream creates an addrman with the correct number of addrs. CDataStream ssPeers2 = AddrmanToStream(addrman); AddrMan addrman2{EMPTY_NETGROUPMAN, !DETERMINISTIC, GetCheckRatio(m_node)}; - BOOST_CHECK(addrman2.size() == 0); + BOOST_CHECK(addrman2.Size() == 0); ReadFromStream(addrman2, ssPeers2); - BOOST_CHECK(addrman2.size() == 3); + BOOST_CHECK(addrman2.Size() == 3); } // Produce a corrupt peers.dat that claims 20 addrs when it only has one addr. @@ -939,7 +939,7 @@ BOOST_AUTO_TEST_CASE(load_addrman_corrupted) CDataStream ssPeers1 = MakeCorruptPeersDat(); bool exceptionThrown = false; AddrMan addrman1{EMPTY_NETGROUPMAN, !DETERMINISTIC, GetCheckRatio(m_node)}; - BOOST_CHECK(addrman1.size() == 0); + BOOST_CHECK(addrman1.Size() == 0); try { unsigned char pchMsgTmp[4]; ssPeers1 >> pchMsgTmp; @@ -947,15 +947,13 @@ BOOST_AUTO_TEST_CASE(load_addrman_corrupted) } catch (const std::exception&) { exceptionThrown = true; } - // Even though de-serialization failed addrman is not left in a clean state. - BOOST_CHECK(addrman1.size() == 1); BOOST_CHECK(exceptionThrown); // Test that ReadFromStream fails if peers.dat is corrupt CDataStream ssPeers2 = MakeCorruptPeersDat(); AddrMan addrman2{EMPTY_NETGROUPMAN, !DETERMINISTIC, GetCheckRatio(m_node)}; - BOOST_CHECK(addrman2.size() == 0); + BOOST_CHECK(addrman2.Size() == 0); BOOST_CHECK_THROW(ReadFromStream(addrman2, ssPeers2), std::ios_base::failure); } @@ -969,7 +967,7 @@ BOOST_AUTO_TEST_CASE(addrman_update_address) const auto start_time{Now<NodeSeconds>() - 10000s}; addr.nTime = start_time; BOOST_CHECK(addrman->Add({addr}, source)); - BOOST_CHECK_EQUAL(addrman->size(), 1U); + BOOST_CHECK_EQUAL(addrman->Size(), 1U); // Updating an addrman entry with a different port doesn't change it CAddress addr_diff_port{CAddress(ResolveService("250.1.1.1", 8334), NODE_NONE)}; @@ -990,4 +988,42 @@ BOOST_AUTO_TEST_CASE(addrman_update_address) BOOST_CHECK_EQUAL(vAddr2.at(0).nServices, NODE_NETWORK_LIMITED); } +BOOST_AUTO_TEST_CASE(addrman_size) +{ + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); + const CNetAddr source = ResolveIP("252.2.2.2"); + + // empty addrman + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/std::nullopt, /*in_new=*/std::nullopt), 0U); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/NET_IPV4, /*in_new=*/std::nullopt), 0U); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/std::nullopt, /*in_new=*/true), 0U); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/NET_IPV4, /*in_new=*/false), 0U); + + // add two ipv4 addresses, one to tried and new + const CAddress addr1{ResolveService("250.1.1.1", 8333), NODE_NONE}; + BOOST_CHECK(addrman->Add({addr1}, source)); + BOOST_CHECK(addrman->Good(addr1)); + const CAddress addr2{ResolveService("250.1.1.2", 8333), NODE_NONE}; + BOOST_CHECK(addrman->Add({addr2}, source)); + + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/std::nullopt, /*in_new=*/std::nullopt), 2U); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/NET_IPV4, /*in_new=*/std::nullopt), 2U); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/std::nullopt, /*in_new=*/true), 1U); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/std::nullopt, /*in_new=*/false), 1U); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/NET_IPV4, /*in_new=*/true), 1U); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/NET_IPV4, /*in_new=*/false), 1U); + + // add one i2p address to new + CService i2p_addr; + i2p_addr.SetSpecial("UDHDrtrcetjm5sxzskjyr5ztpeszydbh4dpl3pl4utgqqw2v4jna.b32.I2P"); + const CAddress addr3{i2p_addr, NODE_NONE}; + BOOST_CHECK(addrman->Add({addr3}, source)); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/std::nullopt, /*in_new=*/std::nullopt), 3U); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/NET_IPV4, /*in_new=*/std::nullopt), 2U); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/NET_I2P, /*in_new=*/std::nullopt), 1U); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/NET_I2P, /*in_new=*/true), 1U); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/std::nullopt, /*in_new=*/true), 2U); + BOOST_CHECK_EQUAL(addrman->Size(/*net=*/std::nullopt, /*in_new=*/false), 1U); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index 0101bcc372..601caf8102 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -5,6 +5,7 @@ #include <test/data/base58_encode_decode.json.h> #include <base58.h> +#include <test/util/json.h> #include <test/util/setup_common.h> #include <util/strencodings.h> #include <util/vector.h> @@ -16,8 +17,6 @@ using namespace std::literals; -UniValue read_json(const std::string& jsondata); - BOOST_FIXTURE_TEST_SUITE(base58_tests, BasicTestingSetup) // Goal: test low-level base58 encoding functionality diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index e1dafc6bac..e23b7228e7 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -310,7 +310,7 @@ BOOST_AUTO_TEST_CASE(TransactionsRequestSerializationTest) { req1.indexes[2] = 3; req1.indexes[3] = 4; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << req1; BlockTransactionsRequest req2; @@ -330,7 +330,7 @@ BOOST_AUTO_TEST_CASE(TransactionsRequestDeserializationMaxTest) { req0.blockhash = InsecureRand256(); req0.indexes.resize(1); req0.indexes[0] = 0xffff; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << req0; BlockTransactionsRequest req1; @@ -350,7 +350,7 @@ BOOST_AUTO_TEST_CASE(TransactionsRequestDeserializationOverflowTest) { req0.indexes[0] = 0x7000; req0.indexes[1] = 0x10000 - 0x7000 - 2; req0.indexes[2] = 0; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << req0.blockhash; WriteCompactSize(stream, req0.indexes.size()); WriteCompactSize(stream, req0.indexes[0]); diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp index 43dca57217..9388b4c96a 100644 --- a/src/test/blockfilter_tests.cpp +++ b/src/test/blockfilter_tests.cpp @@ -110,7 +110,7 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test) // Test serialization/unserialization. BlockFilter block_filter2; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << block_filter; stream >> block_filter2; diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 3d6e103c9f..4888041204 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -39,7 +39,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) filter.insert(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")); BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")), "Bloom filter doesn't contain just-inserted object (3)!"); - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << filter; std::vector<uint8_t> expected = ParseHex("03614e9b050000000000000001"); @@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize_with_tweak) filter.insert(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")); BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")), "Bloom filter doesn't contain just-inserted object (3)!"); - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << filter; std::vector<uint8_t> expected = ParseHex("03ce4299050000000100008001"); @@ -87,7 +87,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_key) uint160 hash = pubkey.GetID(); filter.insert(hash); - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << filter; std::vector<unsigned char> expected = ParseHex("038fc16b080000000000000001"); @@ -340,7 +340,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_3_and_serialize) for (unsigned int i = 0; i < vMatched.size(); i++) BOOST_CHECK(vMatched[i] == merkleBlock.vMatchedTxn[i].second); - CDataStream merkleStream(SER_NETWORK, PROTOCOL_VERSION); + DataStream merkleStream{}; merkleStream << merkleBlock; std::vector<uint8_t> expected = ParseHex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d630100000001b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f19630101"); diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index b5f961a239..55ecd41af1 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -53,9 +53,9 @@ public: uint256 GetBestBlock() const override { return hashBestBlock_; } - bool BatchWrite(CCoinsMap& mapCoins, const uint256& hashBlock) override + bool BatchWrite(CCoinsMap& mapCoins, const uint256& hashBlock, bool erase = true) override { - for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); ) { + for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); it = erase ? mapCoins.erase(it) : std::next(it)) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Same optimization used in CCoinsViewDB is to only write dirty entries. map_[it->first] = it->second.coin; @@ -64,7 +64,6 @@ public: map_.erase(it->first); } } - mapCoins.erase(it++); } if (!hashBlock.IsNull()) hashBestBlock_ = hashBlock; @@ -126,13 +125,14 @@ void SimulationTest(CCoinsView* base, bool fake_best_block) bool found_an_entry = false; bool missed_an_entry = false; bool uncached_an_entry = false; + bool flushed_without_erase = false; // A simple map to track what we expect the cache stack to represent. std::map<COutPoint, Coin> result; // The cache stack. - std::vector<CCoinsViewCacheTest*> stack; // A stack of CCoinsViewCaches on top. - stack.push_back(new CCoinsViewCacheTest(base)); // Start with one cache. + std::vector<std::unique_ptr<CCoinsViewCacheTest>> stack; // A stack of CCoinsViewCaches on top. + stack.push_back(std::make_unique<CCoinsViewCacheTest>(base)); // Start with one cache. // Use a limited set of random transaction ids, so we do test overwriting entries. std::vector<uint256> txids; @@ -154,9 +154,16 @@ void SimulationTest(CCoinsView* base, bool fake_best_block) bool test_havecoin_after = InsecureRandBits(2) == 0; bool result_havecoin = test_havecoin_before ? stack.back()->HaveCoin(COutPoint(txid, 0)) : false; - const Coin& entry = (InsecureRandRange(500) == 0) ? AccessByTxid(*stack.back(), txid) : stack.back()->AccessCoin(COutPoint(txid, 0)); + + // Infrequently, test usage of AccessByTxid instead of AccessCoin - the + // former just delegates to the latter and returns the first unspent in a txn. + const Coin& entry = (InsecureRandRange(500) == 0) ? + AccessByTxid(*stack.back(), txid) : stack.back()->AccessCoin(COutPoint(txid, 0)); BOOST_CHECK(coin == entry); - BOOST_CHECK(!test_havecoin_before || result_havecoin == !entry.IsSpent()); + + if (test_havecoin_before) { + BOOST_CHECK(result_havecoin == !entry.IsSpent()); + } if (test_havecoin_after) { bool ret = stack.back()->HaveCoin(COutPoint(txid, 0)); @@ -167,24 +174,29 @@ void SimulationTest(CCoinsView* base, bool fake_best_block) Coin newcoin; newcoin.out.nValue = InsecureRand32(); newcoin.nHeight = 1; + + // Infrequently test adding unspendable coins. if (InsecureRandRange(16) == 0 && coin.IsSpent()) { newcoin.out.scriptPubKey.assign(1 + InsecureRandBits(6), OP_RETURN); BOOST_CHECK(newcoin.out.scriptPubKey.IsUnspendable()); added_an_unspendable_entry = true; } else { - newcoin.out.scriptPubKey.assign(InsecureRandBits(6), 0); // Random sizes so we can test memory usage accounting + // Random sizes so we can test memory usage accounting + newcoin.out.scriptPubKey.assign(InsecureRandBits(6), 0); (coin.IsSpent() ? added_an_entry : updated_an_entry) = true; coin = newcoin; } - stack.back()->AddCoin(COutPoint(txid, 0), std::move(newcoin), !coin.IsSpent() || InsecureRand32() & 1); + bool is_overwrite = !coin.IsSpent() || InsecureRand32() & 1; + stack.back()->AddCoin(COutPoint(txid, 0), std::move(newcoin), is_overwrite); } else { + // Spend the coin. removed_an_entry = true; coin.Clear(); BOOST_CHECK(stack.back()->SpendCoin(COutPoint(txid, 0))); } } - // One every 10 iterations, remove a random entry from the cache + // Once every 10 iterations, remove a random entry from the cache if (InsecureRandRange(10) == 0) { COutPoint out(txids[InsecureRand32() % txids.size()], 0); int cacheid = InsecureRand32() % stack.size(); @@ -206,7 +218,7 @@ void SimulationTest(CCoinsView* base, bool fake_best_block) found_an_entry = true; } } - for (const CCoinsViewCacheTest *test : stack) { + for (const auto& test : stack) { test->SelfTest(); } } @@ -216,7 +228,9 @@ void SimulationTest(CCoinsView* base, bool fake_best_block) if (stack.size() > 1 && InsecureRandBool() == 0) { unsigned int flushIndex = InsecureRandRange(stack.size() - 1); if (fake_best_block) stack[flushIndex]->SetBestBlock(InsecureRand256()); - BOOST_CHECK(stack[flushIndex]->Flush()); + bool should_erase = InsecureRandRange(4) < 3; + BOOST_CHECK(should_erase ? stack[flushIndex]->Flush() : stack[flushIndex]->Sync()); + flushed_without_erase |= !should_erase; } } if (InsecureRandRange(100) == 0) { @@ -224,19 +238,20 @@ void SimulationTest(CCoinsView* base, bool fake_best_block) if (stack.size() > 0 && InsecureRandBool() == 0) { //Remove the top cache if (fake_best_block) stack.back()->SetBestBlock(InsecureRand256()); - BOOST_CHECK(stack.back()->Flush()); - delete stack.back(); + bool should_erase = InsecureRandRange(4) < 3; + BOOST_CHECK(should_erase ? stack.back()->Flush() : stack.back()->Sync()); + flushed_without_erase |= !should_erase; stack.pop_back(); } if (stack.size() == 0 || (stack.size() < 4 && InsecureRandBool())) { //Add a new cache CCoinsView* tip = base; if (stack.size() > 0) { - tip = stack.back(); + tip = stack.back().get(); } else { removed_all_caches = true; } - stack.push_back(new CCoinsViewCacheTest(tip)); + stack.push_back(std::make_unique<CCoinsViewCacheTest>(tip)); if (stack.size() == 4) { reached_4_caches = true; } @@ -244,12 +259,6 @@ void SimulationTest(CCoinsView* base, bool fake_best_block) } } - // Clean up the stack. - while (stack.size() > 0) { - delete stack.back(); - stack.pop_back(); - } - // Verify coverage. BOOST_CHECK(removed_all_caches); BOOST_CHECK(reached_4_caches); @@ -260,6 +269,7 @@ void SimulationTest(CCoinsView* base, bool fake_best_block) BOOST_CHECK(found_an_entry); BOOST_CHECK(missed_an_entry); BOOST_CHECK(uncached_an_entry); + BOOST_CHECK(flushed_without_erase); } // Run the above simulation for multiple base types. @@ -304,8 +314,8 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. - std::vector<CCoinsViewCacheTest*> stack; // A stack of CCoinsViewCaches on top. - stack.push_back(new CCoinsViewCacheTest(&base)); // Start with one cache. + std::vector<std::unique_ptr<CCoinsViewCacheTest>> stack; // A stack of CCoinsViewCaches on top. + stack.push_back(std::make_unique<CCoinsViewCacheTest>(&base)); // Start with one cache. // Track the txids we've used in various sets std::set<COutPoint> coinbase_coins; @@ -470,25 +480,18 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // Every 100 iterations, change the cache stack. if (stack.size() > 0 && InsecureRandBool() == 0) { BOOST_CHECK(stack.back()->Flush()); - delete stack.back(); stack.pop_back(); } if (stack.size() == 0 || (stack.size() < 4 && InsecureRandBool())) { CCoinsView* tip = &base; if (stack.size() > 0) { - tip = stack.back(); + tip = stack.back().get(); } - stack.push_back(new CCoinsViewCacheTest(tip)); + stack.push_back(std::make_unique<CCoinsViewCacheTest>(tip)); } } } - // Clean up the stack. - while (stack.size() > 0) { - delete stack.back(); - stack.pop_back(); - } - // Verify coverage. BOOST_CHECK(spent_a_duplicate_coinbase); @@ -498,7 +501,7 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) BOOST_AUTO_TEST_CASE(ccoins_serialization) { // Good example - CDataStream ss1(ParseHex("97f23c835800816115944e077fe7c803cfa57f29b36bf87c1d35"), SER_DISK, CLIENT_VERSION); + DataStream ss1{ParseHex("97f23c835800816115944e077fe7c803cfa57f29b36bf87c1d35")}; Coin cc1; ss1 >> cc1; BOOST_CHECK_EQUAL(cc1.fCoinBase, false); @@ -507,7 +510,7 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) BOOST_CHECK_EQUAL(HexStr(cc1.out.scriptPubKey), HexStr(GetScriptForDestination(PKHash(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); // Good example - CDataStream ss2(ParseHex("8ddf77bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4"), SER_DISK, CLIENT_VERSION); + DataStream ss2{ParseHex("8ddf77bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4")}; Coin cc2; ss2 >> cc2; BOOST_CHECK_EQUAL(cc2.fCoinBase, true); @@ -516,7 +519,7 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) BOOST_CHECK_EQUAL(HexStr(cc2.out.scriptPubKey), HexStr(GetScriptForDestination(PKHash(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); // Smallest possible example - CDataStream ss3(ParseHex("000006"), SER_DISK, CLIENT_VERSION); + DataStream ss3{ParseHex("000006")}; Coin cc3; ss3 >> cc3; BOOST_CHECK_EQUAL(cc3.fCoinBase, false); @@ -525,7 +528,7 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) BOOST_CHECK_EQUAL(cc3.out.scriptPubKey.size(), 0U); // scriptPubKey that ends beyond the end of the stream - CDataStream ss4(ParseHex("000007"), SER_DISK, CLIENT_VERSION); + DataStream ss4{ParseHex("000007")}; try { Coin cc4; ss4 >> cc4; @@ -534,11 +537,11 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) } // Very large scriptPubKey (3*10^9 bytes) past the end of the stream - CDataStream tmp(SER_DISK, CLIENT_VERSION); + DataStream tmp{}; uint64_t x = 3000000000ULL; tmp << VARINT(x); BOOST_CHECK_EQUAL(HexStr(tmp), "8a95c0bb00"); - CDataStream ss5(ParseHex("00008a95c0bb00"), SER_DISK, CLIENT_VERSION); + DataStream ss5{ParseHex("00008a95c0bb00")}; try { Coin cc5; ss5 >> cc5; @@ -589,9 +592,9 @@ static size_t InsertCoinsMapEntry(CCoinsMap& map, CAmount value, char flags) return inserted.first->second.coin.DynamicMemoryUsage(); } -void GetCoinsMapEntry(const CCoinsMap& map, CAmount& value, char& flags) +void GetCoinsMapEntry(const CCoinsMap& map, CAmount& value, char& flags, const COutPoint& outp = OUTPOINT) { - auto it = map.find(OUTPOINT); + auto it = map.find(outp); if (it == map.end()) { value = ABSENT; flags = NO_ENTRY; @@ -877,4 +880,199 @@ BOOST_AUTO_TEST_CASE(ccoins_write) CheckWriteCoins(parent_value, child_value, parent_value, parent_flags, child_flags, parent_flags); } + +Coin MakeCoin() +{ + Coin coin; + coin.out.nValue = InsecureRand32(); + coin.nHeight = InsecureRandRange(4096); + coin.fCoinBase = 0; + return coin; +} + + +//! For CCoinsViewCache instances backed by either another cache instance or +//! leveldb, test cache behavior and flag state (DIRTY/FRESH) by +//! +//! 1. Adding a random coin to the child-most cache, +//! 2. Flushing all caches (without erasing), +//! 3. Ensure the entry still exists in the cache and has been written to parent, +//! 4. (if `do_erasing_flush`) Flushing the caches again (with erasing), +//! 5. (if `do_erasing_flush`) Ensure the entry has been written to the parent and is no longer in the cache, +//! 6. Spend the coin, ensure it no longer exists in the parent. +//! +void TestFlushBehavior( + CCoinsViewCacheTest* view, + CCoinsViewDB& base, + std::vector<std::unique_ptr<CCoinsViewCacheTest>>& all_caches, + bool do_erasing_flush) +{ + CAmount value; + char flags; + size_t cache_usage; + + auto flush_all = [&all_caches](bool erase) { + // Flush in reverse order to ensure that flushes happen from children up. + for (auto i = all_caches.rbegin(); i != all_caches.rend(); ++i) { + auto& cache = *i; + // hashBlock must be filled before flushing to disk; value is + // unimportant here. This is normally done during connect/disconnect block. + cache->SetBestBlock(InsecureRand256()); + erase ? cache->Flush() : cache->Sync(); + } + }; + + uint256 txid = InsecureRand256(); + COutPoint outp = COutPoint(txid, 0); + Coin coin = MakeCoin(); + // Ensure the coins views haven't seen this coin before. + BOOST_CHECK(!base.HaveCoin(outp)); + BOOST_CHECK(!view->HaveCoin(outp)); + + // --- 1. Adding a random coin to the child cache + // + view->AddCoin(outp, Coin(coin), false); + + cache_usage = view->DynamicMemoryUsage(); + // `base` shouldn't have coin (no flush yet) but `view` should have cached it. + BOOST_CHECK(!base.HaveCoin(outp)); + BOOST_CHECK(view->HaveCoin(outp)); + + GetCoinsMapEntry(view->map(), value, flags, outp); + BOOST_CHECK_EQUAL(value, coin.out.nValue); + BOOST_CHECK_EQUAL(flags, DIRTY|FRESH); + + // --- 2. Flushing all caches (without erasing) + // + flush_all(/*erase=*/ false); + + // CoinsMap usage should be unchanged since we didn't erase anything. + BOOST_CHECK_EQUAL(cache_usage, view->DynamicMemoryUsage()); + + // --- 3. Ensuring the entry still exists in the cache and has been written to parent + // + GetCoinsMapEntry(view->map(), value, flags, outp); + BOOST_CHECK_EQUAL(value, coin.out.nValue); + BOOST_CHECK_EQUAL(flags, 0); // Flags should have been wiped. + + // Both views should now have the coin. + BOOST_CHECK(base.HaveCoin(outp)); + BOOST_CHECK(view->HaveCoin(outp)); + + if (do_erasing_flush) { + // --- 4. Flushing the caches again (with erasing) + // + flush_all(/*erase=*/ true); + + // Memory usage should have gone down. + BOOST_CHECK(view->DynamicMemoryUsage() < cache_usage); + + // --- 5. Ensuring the entry is no longer in the cache + // + GetCoinsMapEntry(view->map(), value, flags, outp); + BOOST_CHECK_EQUAL(value, ABSENT); + BOOST_CHECK_EQUAL(flags, NO_ENTRY); + + view->AccessCoin(outp); + GetCoinsMapEntry(view->map(), value, flags, outp); + BOOST_CHECK_EQUAL(value, coin.out.nValue); + BOOST_CHECK_EQUAL(flags, 0); + } + + // Can't overwrite an entry without specifying that an overwrite is + // expected. + BOOST_CHECK_THROW( + view->AddCoin(outp, Coin(coin), /*possible_overwrite=*/ false), + std::logic_error); + + // --- 6. Spend the coin. + // + BOOST_CHECK(view->SpendCoin(outp)); + + // The coin should be in the cache, but spent and marked dirty. + GetCoinsMapEntry(view->map(), value, flags, outp); + BOOST_CHECK_EQUAL(value, SPENT); + BOOST_CHECK_EQUAL(flags, DIRTY); + BOOST_CHECK(!view->HaveCoin(outp)); // Coin should be considered spent in `view`. + BOOST_CHECK(base.HaveCoin(outp)); // But coin should still be unspent in `base`. + + flush_all(/*erase=*/ false); + + // Coin should be considered spent in both views. + BOOST_CHECK(!view->HaveCoin(outp)); + BOOST_CHECK(!base.HaveCoin(outp)); + + // Spent coin should not be spendable. + BOOST_CHECK(!view->SpendCoin(outp)); + + // --- Bonus check: ensure that a coin added to the base view via one cache + // can be spent by another cache which has never seen it. + // + txid = InsecureRand256(); + outp = COutPoint(txid, 0); + coin = MakeCoin(); + BOOST_CHECK(!base.HaveCoin(outp)); + BOOST_CHECK(!all_caches[0]->HaveCoin(outp)); + BOOST_CHECK(!all_caches[1]->HaveCoin(outp)); + + all_caches[0]->AddCoin(outp, std::move(coin), false); + all_caches[0]->Sync(); + BOOST_CHECK(base.HaveCoin(outp)); + BOOST_CHECK(all_caches[0]->HaveCoin(outp)); + BOOST_CHECK(!all_caches[1]->HaveCoinInCache(outp)); + + BOOST_CHECK(all_caches[1]->SpendCoin(outp)); + flush_all(/*erase=*/ false); + BOOST_CHECK(!base.HaveCoin(outp)); + BOOST_CHECK(!all_caches[0]->HaveCoin(outp)); + BOOST_CHECK(!all_caches[1]->HaveCoin(outp)); + + flush_all(/*erase=*/ true); // Erase all cache content. + + // --- Bonus check 2: ensure that a FRESH, spent coin is deleted by Sync() + // + txid = InsecureRand256(); + outp = COutPoint(txid, 0); + coin = MakeCoin(); + CAmount coin_val = coin.out.nValue; + BOOST_CHECK(!base.HaveCoin(outp)); + BOOST_CHECK(!all_caches[0]->HaveCoin(outp)); + BOOST_CHECK(!all_caches[1]->HaveCoin(outp)); + + // Add and spend from same cache without flushing. + all_caches[0]->AddCoin(outp, std::move(coin), false); + + // Coin should be FRESH in the cache. + GetCoinsMapEntry(all_caches[0]->map(), value, flags, outp); + BOOST_CHECK_EQUAL(value, coin_val); + BOOST_CHECK_EQUAL(flags, DIRTY|FRESH); + + // Base shouldn't have seen coin. + BOOST_CHECK(!base.HaveCoin(outp)); + + BOOST_CHECK(all_caches[0]->SpendCoin(outp)); + all_caches[0]->Sync(); + + // Ensure there is no sign of the coin after spend/flush. + GetCoinsMapEntry(all_caches[0]->map(), value, flags, outp); + BOOST_CHECK_EQUAL(value, ABSENT); + BOOST_CHECK_EQUAL(flags, NO_ENTRY); + BOOST_CHECK(!all_caches[0]->HaveCoinInCache(outp)); + BOOST_CHECK(!base.HaveCoin(outp)); +} + +BOOST_AUTO_TEST_CASE(ccoins_flush_behavior) +{ + // Create two in-memory caches atop a leveldb view. + CCoinsViewDB base{"test", /*nCacheSize=*/ 1 << 23, /*fMemory=*/ true, /*fWipe=*/ false}; + std::vector<std::unique_ptr<CCoinsViewCacheTest>> caches; + caches.push_back(std::make_unique<CCoinsViewCacheTest>(&base)); + caches.push_back(std::make_unique<CCoinsViewCacheTest>(caches.back().get())); + + for (const auto& view : caches) { + TestFlushBehavior(view.get(), base, caches, /*do_erasing_flush=*/false); + TestFlushBehavior(view.get(), base, caches, /*do_erasing_flush=*/true); + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 9b369a5c50..d3eef7beb7 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -925,7 +925,7 @@ BOOST_AUTO_TEST_CASE(muhash_tests) // Test MuHash3072 serialization MuHash3072 serchk = FromInt(1); serchk *= FromInt(2); std::string ser_exp = "1fa093295ea30a6a3acdc7b3f770fa538eff537528e990e2910e40bbcfd7f6696b1256901929094694b56316de342f593303dd12ac43e06dce1be1ff8301c845beb15468fff0ef002dbf80c29f26e6452bccc91b5cb9437ad410d2a67ea847887fa3c6a6553309946880fe20db2c73fe0641adbd4e86edfee0d9f8cd0ee1230898873dc13ed8ddcaf045c80faa082774279007a2253f8922ee3ef361d378a6af3ddaf180b190ac97e556888c36b3d1fb1c85aab9ccd46e3deaeb7b7cf5db067a7e9ff86b658cf3acd6662bbcce37232daa753c48b794356c020090c831a8304416e2aa7ad633c0ddb2f11be1be316a81be7f7e472071c042cb68faef549c221ebff209273638b741aba5a81675c45a5fa92fea4ca821d7a324cb1e1a2ccd3b76c4228ec8066dad2a5df6e1bd0de45c7dd5de8070bdb46db6c554cf9aefc9b7b2bbf9f75b1864d9f95005314593905c0109b71f703d49944ae94477b51dac10a816bb6d1c700bafabc8bd86fac8df24be519a2f2836b16392e18036cb13e48c5c010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"; - CDataStream ss_chk(SER_DISK, PROTOCOL_VERSION); + DataStream ss_chk{}; ss_chk << serchk; BOOST_CHECK_EQUAL(ser_exp, HexStr(ss_chk.str())); @@ -938,7 +938,7 @@ BOOST_AUTO_TEST_CASE(muhash_tests) BOOST_CHECK_EQUAL(HexStr(out), HexStr(out3)); // Test MuHash3072 overflow, meaning the internal data is larger than the modulus. - CDataStream ss_max(ParseHex("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), SER_DISK, PROTOCOL_VERSION); + DataStream ss_max{ParseHex("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")}; MuHash3072 overflowchk; ss_max >> overflowchk; diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index 2953cf149d..a59e41dbb5 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -117,7 +117,7 @@ void FillAddrman(AddrMan& addrman, FuzzedDataProvider& fuzzed_data_provider) const std::chrono::seconds time_penalty{fast_random_context.randrange(100000001)}; addrman.Add({addr}, source, time_penalty); - if (n > 0 && addrman.size() % n == 0) { + if (n > 0 && addrman.Size() % n == 0) { addrman.Good(addr, Now<NodeSeconds>()); } @@ -304,7 +304,7 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) /*max_pct=*/fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), /*network=*/std::nullopt); (void)const_addr_man.Select(fuzzed_data_provider.ConsumeBool()); - (void)const_addr_man.size(); + (void)const_addr_man.Size(); CDataStream data_stream(SER_NETWORK, PROTOCOL_VERSION); data_stream << const_addr_man; } diff --git a/src/test/fuzz/base_encode_decode.cpp b/src/test/fuzz/base_encode_decode.cpp index 630f32a1cb..d322416d34 100644 --- a/src/test/fuzz/base_encode_decode.cpp +++ b/src/test/fuzz/base_encode_decode.cpp @@ -14,12 +14,7 @@ #include <string> #include <vector> -void initialize_base_encode_decode() -{ - static const ECCVerifyHandle verify_handle; -} - -FUZZ_TARGET_INIT(base_encode_decode, initialize_base_encode_decode) +FUZZ_TARGET(base_encode_decode) { const std::string random_encoded_string(buffer.begin(), buffer.end()); diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index b7ed2c6abd..c3e17724eb 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -19,7 +19,6 @@ void initialize_block() { - static const ECCVerifyHandle verify_handle; SelectParams(CBaseChainParams::REGTEST); } diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 46026d8df3..e75dc3ce91 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -75,6 +75,9 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) (void)coins_view_cache.Flush(); }, [&] { + (void)coins_view_cache.Sync(); + }, + [&] { coins_view_cache.SetBestBlock(ConsumeUInt256(fuzzed_data_provider)); }, [&] { diff --git a/src/test/fuzz/descriptor_parse.cpp b/src/test/fuzz/descriptor_parse.cpp index f5f86a574a..1f5601ca9f 100644 --- a/src/test/fuzz/descriptor_parse.cpp +++ b/src/test/fuzz/descriptor_parse.cpp @@ -9,7 +9,6 @@ void initialize_descriptor_parse() { - static const ECCVerifyHandle verify_handle; ECC_Start(); SelectParams(CBaseChainParams::MAIN); } diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 0a7d0c55bd..7cd78e0461 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -46,9 +46,6 @@ void initialize_deserialize() { static const auto testing_setup = MakeNoLogFileContext<>(); g_setup = testing_setup.get(); - - // Fuzzers using pubkey must hold an ECCVerifyHandle. - static const ECCVerifyHandle verify_handle; } #define FUZZ_TARGET_DESERIALIZE(name, code) \ diff --git a/src/test/fuzz/eval_script.cpp b/src/test/fuzz/eval_script.cpp index e7c49c2dbc..d762676c3c 100644 --- a/src/test/fuzz/eval_script.cpp +++ b/src/test/fuzz/eval_script.cpp @@ -9,12 +9,7 @@ #include <limits> -void initialize_eval_script() -{ - static const ECCVerifyHandle verify_handle; -} - -FUZZ_TARGET_INIT(eval_script, initialize_eval_script) +FUZZ_TARGET(eval_script) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const unsigned int flags = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); diff --git a/src/test/fuzz/hex.cpp b/src/test/fuzz/hex.cpp index aed8ac455c..f67b820d11 100644 --- a/src/test/fuzz/hex.cpp +++ b/src/test/fuzz/hex.cpp @@ -16,12 +16,7 @@ #include <string> #include <vector> -void initialize_hex() -{ - static const ECCVerifyHandle verify_handle; -} - -FUZZ_TARGET_INIT(hex, initialize_hex) +FUZZ_TARGET(hex) { const std::string random_hex_string(buffer.begin(), buffer.end()); const std::vector<unsigned char> data = ParseHex(random_hex_string); diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index 7965f90dc7..c0aefe6067 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -152,7 +152,7 @@ FUZZ_TARGET_INIT(integer, initialize_integer) const CScriptID script_id{u160}; { - CDataStream stream(SER_NETWORK, INIT_PROTO_VERSION); + DataStream stream{}; uint256 deserialized_u256; stream << u256; @@ -217,7 +217,7 @@ FUZZ_TARGET_INIT(integer, initialize_integer) } { - CDataStream stream(SER_NETWORK, INIT_PROTO_VERSION); + DataStream stream{}; ser_writedata64(stream, u64); const uint64_t deserialized_u64 = ser_readdata64(stream); @@ -245,7 +245,7 @@ FUZZ_TARGET_INIT(integer, initialize_integer) } { - CDataStream stream(SER_NETWORK, INIT_PROTO_VERSION); + DataStream stream{}; WriteCompactSize(stream, u64); try { diff --git a/src/test/fuzz/key.cpp b/src/test/fuzz/key.cpp index 042f66ee1d..ea6883c08d 100644 --- a/src/test/fuzz/key.cpp +++ b/src/test/fuzz/key.cpp @@ -27,7 +27,6 @@ void initialize_key() { - static const ECCVerifyHandle ecc_verify_handle; ECC_Start(); SelectParams(CBaseChainParams::REGTEST); } @@ -112,7 +111,7 @@ FUZZ_TARGET_INIT(key, initialize_key) } { - CDataStream data_stream{SER_NETWORK, INIT_PROTO_VERSION}; + DataStream data_stream{}; pubkey.Serialize(data_stream); CPubKey pubkey_deserialized; diff --git a/src/test/fuzz/key_io.cpp b/src/test/fuzz/key_io.cpp index 32a81c2e17..29c6996365 100644 --- a/src/test/fuzz/key_io.cpp +++ b/src/test/fuzz/key_io.cpp @@ -13,7 +13,6 @@ void initialize_key_io() { - static const ECCVerifyHandle verify_handle; ECC_Start(); SelectParams(CBaseChainParams::MAIN); } diff --git a/src/test/fuzz/message.cpp b/src/test/fuzz/message.cpp index 06cd0afe2a..63e24aacdd 100644 --- a/src/test/fuzz/message.cpp +++ b/src/test/fuzz/message.cpp @@ -18,7 +18,6 @@ void initialize_message() { - static const ECCVerifyHandle ecc_verify_handle; ECC_Start(); SelectParams(CBaseChainParams::REGTEST); } diff --git a/src/test/fuzz/miniscript.cpp b/src/test/fuzz/miniscript.cpp index 1d6a8d89e4..d5667e0cf3 100644 --- a/src/test/fuzz/miniscript.cpp +++ b/src/test/fuzz/miniscript.cpp @@ -104,7 +104,7 @@ struct ScriptParserContext { return key.data; } - const std::vector<unsigned char> ToPKHBytes(const Key& key) const + std::vector<unsigned char> ToPKHBytes(const Key& key) const { if (key.is_hash) return key.data; const auto h = Hash160(key.data); diff --git a/src/test/fuzz/parse_univalue.cpp b/src/test/fuzz/parse_univalue.cpp index 417d457395..16486f6b96 100644 --- a/src/test/fuzz/parse_univalue.cpp +++ b/src/test/fuzz/parse_univalue.cpp @@ -13,7 +13,6 @@ void initialize_parse_univalue() { - static const ECCVerifyHandle verify_handle; SelectParams(CBaseChainParams::REGTEST); } diff --git a/src/test/fuzz/partially_downloaded_block.cpp b/src/test/fuzz/partially_downloaded_block.cpp new file mode 100644 index 0000000000..f8ba4f08d9 --- /dev/null +++ b/src/test/fuzz/partially_downloaded_block.cpp @@ -0,0 +1,142 @@ +#include <blockencodings.h> +#include <consensus/merkle.h> +#include <consensus/validation.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/fuzz/util/mempool.h> +#include <test/util/setup_common.h> +#include <test/util/txmempool.h> +#include <txmempool.h> + +#include <cstddef> +#include <cstdint> +#include <limits> +#include <memory> +#include <optional> +#include <set> +#include <vector> + +namespace { +const TestingSetup* g_setup; +} // namespace + +void initialize_pdb() +{ + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); +} + +PartiallyDownloadedBlock::CheckBlockFn FuzzedCheckBlock(std::optional<BlockValidationResult> result) +{ + return [result](const CBlock&, BlockValidationState& state, const Consensus::Params&, bool, bool) { + if (result) { + return state.Invalid(*result); + } + + return true; + }; +} + +FUZZ_TARGET_INIT(partially_downloaded_block, initialize_pdb) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + auto block{ConsumeDeserializable<CBlock>(fuzzed_data_provider)}; + if (!block || block->vtx.size() == 0 || + block->vtx.size() >= std::numeric_limits<uint16_t>::max()) { + return; + } + + CBlockHeaderAndShortTxIDs cmpctblock{*block}; + + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + PartiallyDownloadedBlock pdb{&pool}; + + // Set of available transactions (mempool or extra_txn) + std::set<uint16_t> available; + // The coinbase is always available + available.insert(0); + + std::vector<std::pair<uint256, CTransactionRef>> extra_txn; + for (size_t i = 1; i < block->vtx.size(); ++i) { + auto tx{block->vtx[i]}; + + bool add_to_extra_txn{fuzzed_data_provider.ConsumeBool()}; + bool add_to_mempool{fuzzed_data_provider.ConsumeBool()}; + + if (add_to_extra_txn) { + extra_txn.emplace_back(tx->GetWitnessHash(), tx); + available.insert(i); + } + + if (add_to_mempool) { + LOCK2(cs_main, pool.cs); + pool.addUnchecked(ConsumeTxMemPoolEntry(fuzzed_data_provider, *tx)); + available.insert(i); + } + } + + auto init_status{pdb.InitData(cmpctblock, extra_txn)}; + + std::vector<CTransactionRef> missing; + // Whether we skipped a transaction that should be included in `missing`. + // FillBlock should never return READ_STATUS_OK if that is the case. + bool skipped_missing{false}; + for (size_t i = 0; i < cmpctblock.BlockTxCount(); i++) { + // If init_status == READ_STATUS_OK then a available transaction in the + // compact block (i.e. IsTxAvailable(i) == true) implies that we marked + // that transaction as available above (i.e. available.count(i) > 0). + // The reverse is not true, due to possible compact block short id + // collisions (i.e. available.count(i) > 0 does not imply + // IsTxAvailable(i) == true). + if (init_status == READ_STATUS_OK) { + assert(!pdb.IsTxAvailable(i) || available.count(i) > 0); + } + + bool skip{fuzzed_data_provider.ConsumeBool()}; + if (!pdb.IsTxAvailable(i) && !skip) { + missing.push_back(block->vtx[i]); + } + + skipped_missing |= (!pdb.IsTxAvailable(i) && skip); + } + + // Mock CheckBlock + bool fail_check_block{fuzzed_data_provider.ConsumeBool()}; + auto validation_result = + fuzzed_data_provider.PickValueInArray( + {BlockValidationResult::BLOCK_RESULT_UNSET, + BlockValidationResult::BLOCK_CONSENSUS, + BlockValidationResult::BLOCK_RECENT_CONSENSUS_CHANGE, + BlockValidationResult::BLOCK_CACHED_INVALID, + BlockValidationResult::BLOCK_INVALID_HEADER, + BlockValidationResult::BLOCK_MUTATED, + BlockValidationResult::BLOCK_MISSING_PREV, + BlockValidationResult::BLOCK_INVALID_PREV, + BlockValidationResult::BLOCK_TIME_FUTURE, + BlockValidationResult::BLOCK_CHECKPOINT, + BlockValidationResult::BLOCK_HEADER_LOW_WORK}); + pdb.m_check_block_mock = FuzzedCheckBlock( + fail_check_block ? + std::optional<BlockValidationResult>{validation_result} : + std::nullopt); + + CBlock reconstructed_block; + auto fill_status{pdb.FillBlock(reconstructed_block, missing)}; + switch (fill_status) { + case READ_STATUS_OK: + assert(!skipped_missing); + assert(!fail_check_block); + assert(block->GetHash() == reconstructed_block.GetHash()); + break; + case READ_STATUS_CHECKBLOCK_FAILED: [[fallthrough]]; + case READ_STATUS_FAILED: + assert(fail_check_block); + break; + case READ_STATUS_INVALID: + break; + } +} diff --git a/src/test/fuzz/prevector.cpp b/src/test/fuzz/prevector.cpp index c8fd9aca30..9cea32e304 100644 --- a/src/test/fuzz/prevector.cpp +++ b/src/test/fuzz/prevector.cpp @@ -59,8 +59,8 @@ public: --pos; assert(v == real_vector[pos]); } - CDataStream ss1(SER_DISK, 0); - CDataStream ss2(SER_DISK, 0); + DataStream ss1{}; + DataStream ss2{}; ss1 << real_vector; ss2 << pre_vector; assert(ss1.size() == ss2.size()); diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 731e0d22e0..0a7924f226 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -56,7 +56,9 @@ void initialize_process_message() { Assert(GetNumMsgTypes() == getAllNetMessageTypes().size()); // If this fails, add or remove the message type below - static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>( + /*chain_name=*/CBaseChainParams::REGTEST, + /*extra_args=*/{"-txreconciliation"}); g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { MineBlock(g_setup->m_node, CScript() << OP_TRUE); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 465184d57d..96339743ba 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -23,7 +23,9 @@ const TestingSetup* g_setup; void initialize_process_messages() { - static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>( + /*chain_name=*/CBaseChainParams::REGTEST, + /*extra_args=*/{"-txreconciliation"}); g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { MineBlock(g_setup->m_node, CScript() << OP_TRUE); diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp index 804d55528d..825ed67ec1 100644 --- a/src/test/fuzz/psbt.cpp +++ b/src/test/fuzz/psbt.cpp @@ -22,12 +22,7 @@ using node::AnalyzePSBT; using node::PSBTAnalysis; using node::PSBTInputAnalysis; -void initialize_psbt() -{ - static const ECCVerifyHandle verify_handle; -} - -FUZZ_TARGET_INIT(psbt, initialize_psbt) +FUZZ_TARGET(psbt) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; PartiallySignedTransaction psbt_mut; diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 361cfa6cb6..2578137471 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -253,7 +253,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) if (!opt_block_header) { return; } - CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; + DataStream data_stream{}; data_stream << *opt_block_header; r = HexStr(data_stream); }, diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index 0eac34b8a5..1037dd934a 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -32,9 +32,6 @@ void initialize_script() { - // Fuzzers using pubkey must hold an ECCVerifyHandle. - static const ECCVerifyHandle verify_handle; - SelectParams(CBaseChainParams::REGTEST); } diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index 3635b3ef47..206d219afe 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -184,10 +184,7 @@ void Test(const std::string& str) } } -void test_init() -{ - static ECCVerifyHandle handle; -} +void test_init() {} FUZZ_TARGET_INIT_HIDDEN(script_assets_test_minimizer, test_init, /*hidden=*/true) { diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index 8dc99ee069..f8594fc233 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -11,12 +11,7 @@ #include <test/fuzz/fuzz.h> -void initialize_script_flags() -{ - static const ECCVerifyHandle verify_handle; -} - -FUZZ_TARGET_INIT(script_flags, initialize_script_flags) +FUZZ_TARGET(script_flags) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); try { diff --git a/src/test/fuzz/script_sign.cpp b/src/test/fuzz/script_sign.cpp index 3ddb30d870..3cef81c251 100644 --- a/src/test/fuzz/script_sign.cpp +++ b/src/test/fuzz/script_sign.cpp @@ -26,7 +26,6 @@ void initialize_script_sign() { - static const ECCVerifyHandle ecc_verify_handle; ECC_Start(); SelectParams(CBaseChainParams::REGTEST); } diff --git a/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp b/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp index f437d53b57..74ef6bfd4e 100644 --- a/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp +++ b/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp @@ -12,7 +12,7 @@ #include <vector> bool SigHasLowR(const secp256k1_ecdsa_signature* sig); -int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_signature* sig, const unsigned char* input, size_t inputlen); +int ecdsa_signature_parse_der_lax(secp256k1_ecdsa_signature* sig, const unsigned char* input, size_t inputlen); FUZZ_TARGET(secp256k1_ecdsa_signature_parse_der_lax) { @@ -21,13 +21,11 @@ FUZZ_TARGET(secp256k1_ecdsa_signature_parse_der_lax) if (signature_bytes.data() == nullptr) { return; } - secp256k1_context* secp256k1_context_verify = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); secp256k1_ecdsa_signature sig_der_lax; - const bool parsed_der_lax = ecdsa_signature_parse_der_lax(secp256k1_context_verify, &sig_der_lax, signature_bytes.data(), signature_bytes.size()) == 1; + const bool parsed_der_lax = ecdsa_signature_parse_der_lax(&sig_der_lax, signature_bytes.data(), signature_bytes.size()) == 1; if (parsed_der_lax) { ECC_Start(); (void)SigHasLowR(&sig_der_lax); ECC_Stop(); } - secp256k1_context_destroy(secp256k1_context_verify); } diff --git a/src/test/fuzz/signature_checker.cpp b/src/test/fuzz/signature_checker.cpp index 7c34b6440a..59f4792961 100644 --- a/src/test/fuzz/signature_checker.cpp +++ b/src/test/fuzz/signature_checker.cpp @@ -14,11 +14,6 @@ #include <string> #include <vector> -void initialize_signature_checker() -{ - static const auto verify_handle = std::make_unique<ECCVerifyHandle>(); -} - namespace { class FuzzedSignatureChecker : public BaseSignatureChecker { @@ -53,7 +48,7 @@ public: }; } // namespace -FUZZ_TARGET_INIT(signature_checker, initialize_signature_checker) +FUZZ_TARGET(signature_checker) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const unsigned int flags = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 3c427b9bef..9890e4c0e5 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -196,7 +196,7 @@ FUZZ_TARGET(string) } { - CDataStream data_stream{SER_NETWORK, INIT_PROTO_VERSION}; + DataStream data_stream{}; std::string s; auto limited_string = LIMITED_STRING(s, 10); data_stream << random_string_1; @@ -212,7 +212,7 @@ FUZZ_TARGET(string) } } { - CDataStream data_stream{SER_NETWORK, INIT_PROTO_VERSION}; + DataStream data_stream{}; const auto limited_string = LIMITED_STRING(random_string_1, 10); data_stream << limited_string; std::string deserialized_string; diff --git a/src/test/fuzz/tx_in.cpp b/src/test/fuzz/tx_in.cpp index f8247c1fa4..fc16f80cde 100644 --- a/src/test/fuzz/tx_in.cpp +++ b/src/test/fuzz/tx_in.cpp @@ -14,12 +14,9 @@ FUZZ_TARGET(tx_in) { - CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); + DataStream ds{buffer}; CTxIn tx_in; try { - int version; - ds >> version; - ds.SetVersion(version); ds >> tx_in; } catch (const std::ios_base::failure&) { return; diff --git a/src/test/fuzz/tx_out.cpp b/src/test/fuzz/tx_out.cpp index 337b8e2771..806216fbf5 100644 --- a/src/test/fuzz/tx_out.cpp +++ b/src/test/fuzz/tx_out.cpp @@ -13,12 +13,9 @@ FUZZ_TARGET(tx_out) { - CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); + DataStream ds{buffer}; CTxOut tx_out; try { - int version; - ds >> version; - ds.SetVersion(version); ds >> tx_out; } catch (const std::ios_base::failure&) { return; diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index e933167341..0cabaf323b 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -311,7 +311,7 @@ FUZZ_TARGET_INIT(tx_pool, initialize_tx_pool) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const auto& node = g_setup->m_node; - auto& chainstate = node.chainman->ActiveChainstate(); + auto& chainstate{static_cast<DummyChainState&>(node.chainman->ActiveChainstate())}; MockTime(fuzzed_data_provider, chainstate); @@ -329,6 +329,8 @@ FUZZ_TARGET_INIT(tx_pool, initialize_tx_pool) CTxMemPool tx_pool_{MakeMempool(fuzzed_data_provider, node)}; MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); + chainstate.SetMempool(&tx_pool); + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 300) { const auto mut_tx = ConsumeTransaction(fuzzed_data_provider, txids); diff --git a/src/test/fuzz/txorphan.cpp b/src/test/fuzz/txorphan.cpp index dafe8249c0..ed55e3fad5 100644 --- a/src/test/fuzz/txorphan.cpp +++ b/src/test/fuzz/txorphan.cpp @@ -85,16 +85,12 @@ FUZZ_TARGET_INIT(txorphan, initialize_orphanage) CallOneOf( fuzzed_data_provider, [&] { - orphanage.AddChildrenToWorkSet(*tx, peer_id); + orphanage.AddChildrenToWorkSet(*tx); }, [&] { { - NodeId originator; - bool more = true; - CTransactionRef ref = orphanage.GetTxToReconsider(peer_id, originator, more); - if (!ref) { - Assert(!more); - } else { + CTransactionRef ref = orphanage.GetTxToReconsider(peer_id); + if (ref) { bool have_tx = orphanage.HaveTx(GenTxid::Txid(ref->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(ref->GetHash())); Assert(have_tx); } diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index af1d65cd38..c14f633029 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -47,7 +47,7 @@ size_t CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callable template <typename Collection> auto& PickValue(FuzzedDataProvider& fuzzed_data_provider, Collection& col) { - const auto sz = col.size(); + auto sz{col.size()}; assert(sz >= 1); auto it = col.begin(); std::advance(it, fuzzed_data_provider.ConsumeIntegralInRange<decltype(sz)>(0, sz - 1)); diff --git a/src/test/key_io_tests.cpp b/src/test/key_io_tests.cpp index fb0a07934d..a400afee71 100644 --- a/src/test/key_io_tests.cpp +++ b/src/test/key_io_tests.cpp @@ -8,6 +8,7 @@ #include <key.h> #include <key_io.h> #include <script/script.h> +#include <test/util/json.h> #include <test/util/setup_common.h> #include <util/strencodings.h> @@ -15,8 +16,6 @@ #include <univalue.h> -UniValue read_json(const std::string& jsondata); - BOOST_FIXTURE_TEST_SUITE(key_io_tests, BasicTestingSetup) // Goal: check that parsed keys match test payload diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 21ed2f1080..edf28cfbfc 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -205,8 +205,7 @@ BOOST_AUTO_TEST_CASE(key_key_negation) unsigned char rnd[8]; std::string str = "Bitcoin key verification\n"; GetRandBytes(rnd); - uint256 hash; - CHash256().Write(MakeUCharSpan(str)).Write(rnd).Finalize(hash); + uint256 hash{Hash(str, rnd)}; // import the static test key CKey key = DecodeSecret(strSecret1C); @@ -233,7 +232,7 @@ BOOST_AUTO_TEST_CASE(key_key_negation) static CPubKey UnserializePubkey(const std::vector<uint8_t>& data) { - CDataStream stream{SER_NETWORK, INIT_PROTO_VERSION}; + DataStream stream{}; stream << data; CPubKey pubkey; stream >> pubkey; @@ -251,7 +250,7 @@ static unsigned int GetLen(unsigned char chHeader) static void CmpSerializationPubkey(const CPubKey& pubkey) { - CDataStream stream{SER_NETWORK, INIT_PROTO_VERSION}; + DataStream stream{}; stream << pubkey; CPubKey pubkey2; stream >> pubkey2; diff --git a/src/test/logging_tests.cpp b/src/test/logging_tests.cpp index 022e33f99d..beb9398c74 100644 --- a/src/test/logging_tests.cpp +++ b/src/test/logging_tests.cpp @@ -75,20 +75,9 @@ struct LogSetup : public BasicTestingSetup { BOOST_AUTO_TEST_CASE(logging_timer) { - SetMockTime(1); auto micro_timer = BCLog::Timer<std::chrono::microseconds>("tests", "end_msg"); - SetMockTime(2); - BOOST_CHECK_EQUAL(micro_timer.LogMsg("test micros"), "tests: test micros (1000000μs)"); - - SetMockTime(1); - auto ms_timer = BCLog::Timer<std::chrono::milliseconds>("tests", "end_msg"); - SetMockTime(2); - BOOST_CHECK_EQUAL(ms_timer.LogMsg("test ms"), "tests: test ms (1000.00ms)"); - - SetMockTime(1); - auto sec_timer = BCLog::Timer<std::chrono::seconds>("tests", "end_msg"); - SetMockTime(2); - BOOST_CHECK_EQUAL(sec_timer.LogMsg("test secs"), "tests: test secs (1.00s)"); + const std::string_view result_prefix{"tests: msg ("}; + BOOST_CHECK_EQUAL(micro_timer.LogMsg("msg").substr(0, result_prefix.size()), result_prefix); } BOOST_FIXTURE_TEST_CASE(logging_LogPrintf_, LogSetup) diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 7e9079743e..94e553a304 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -64,7 +64,7 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) CTxMemPool& testPool = *Assert(m_node.mempool); - LOCK2(cs_main, testPool.cs); + LOCK2(::cs_main, testPool.cs); // Nothing in pool, remove should do nothing: unsigned int poolSize = testPool.size(); diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp index bba103d1b0..74e01fc2a5 100644 --- a/src/test/merkle_tests.cpp +++ b/src/test/merkle_tests.cpp @@ -60,7 +60,7 @@ static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot } } mutated |= (inner[level] == h); - CHash256().Write(inner[level]).Write(h).Finalize(h); + h = Hash(inner[level], h); } // Store the resulting hash at inner position level. inner[level] = h; @@ -86,7 +86,7 @@ static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot if (pbranch && matchh) { pbranch->push_back(h); } - CHash256().Write(h).Write(h).Finalize(h); + h = Hash(h, h); // Increment count to the value it would have if two entries at this // level had existed. count += ((uint32_t{1}) << level); @@ -101,7 +101,7 @@ static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot matchh = true; } } - CHash256().Write(inner[level]).Write(h).Finalize(h); + h = Hash(inner[level], h); level++; } } diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index d6aee472a8..21e0dd2fc5 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -69,7 +69,7 @@ BOOST_AUTO_TEST_CASE(pmt_test1) CPartialMerkleTree pmt1(vTxid, vMatch); // serialize - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + DataStream ss{}; ss << pmt1; // verify CPartialMerkleTree's size guarantees diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index 4068775cfa..5f4d307048 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -66,8 +66,8 @@ class prevector_tester { for (const T& v : reverse_iterate(const_pre_vector)) { local_check(v == real_vector[--pos]); } - CDataStream ss1(SER_DISK, 0); - CDataStream ss2(SER_DISK, 0); + DataStream ss1{}; + DataStream ss2{}; ss1 << real_vector; ss2 << pre_vector; local_check_equal(ss1.size(), ss2.size()); diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index db659af0b8..e5cf767614 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -101,7 +101,7 @@ BOOST_AUTO_TEST_CASE(fastrandom_randbits) } } -/** Does-it-compile test for compatibility with standard C++11 RNG interface. */ +/** Does-it-compile test for compatibility with standard library RNG interface. */ BOOST_AUTO_TEST_CASE(stdrandom_test) { FastRandomContext ctx; diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index 88df34ffe6..7bebadf224 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -400,12 +400,11 @@ BOOST_AUTO_TEST_CASE(bip341_spk_test_vectors) for (const auto& vec : vectors.getValues()) { TaprootBuilder spktest; - std::map<std::pair<CScript, int>, int> scriptposes; + std::map<std::pair<std::vector<unsigned char>, int>, int> scriptposes; std::function<void (const UniValue&, int)> parse_tree = [&](const UniValue& node, int depth) { if (node.isNull()) return; if (node.isObject()) { - auto script_bytes = ParseHex(node["script"].get_str()); - CScript script(script_bytes.begin(), script_bytes.end()); + auto script = ParseHex(node["script"].get_str()); int idx = node["id"].getInt<int>(); int leaf_version = node["leafVersion"].getInt<int>(); scriptposes[{script, leaf_version}] = idx; diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 472cba2aac..22f6cfd164 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -15,6 +15,7 @@ #include <script/sign.h> #include <script/signingprovider.h> #include <streams.h> +#include <test/util/json.h> #include <test/util/setup_common.h> #include <test/util/transaction_utils.h> #include <util/strencodings.h> @@ -41,18 +42,6 @@ static const unsigned int gFlags = SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC; unsigned int ParseScriptFlags(std::string strFlags); std::string FormatScriptFlags(unsigned int flags); -UniValue read_json(const std::string& jsondata) -{ - UniValue v; - - if (!v.read(jsondata) || !v.isArray()) - { - BOOST_ERROR("Parse error."); - return UniValue(UniValue::VARR); - } - return v.get_array(); -} - struct ScriptErrorDesc { ScriptError_t err; @@ -1817,7 +1806,24 @@ BOOST_AUTO_TEST_CASE(bip341_keypath_test_vectors) } } +} + +BOOST_AUTO_TEST_CASE(compute_tapbranch) +{ + uint256 hash1 = uint256S("8ad69ec7cf41c2a4001fd1f738bf1e505ce2277acdcaa63fe4765192497f47a7"); + uint256 hash2 = uint256S("f224a923cd0021ab202ab139cc56802ddb92dcfc172b9212261a539df79a112a"); + uint256 result = uint256S("a64c5b7b943315f9b805d7a7296bedfcfd08919270a1f7a1466e98f8693d8cd9"); + BOOST_CHECK_EQUAL(ComputeTapbranchHash(hash1, hash2), result); +} + +BOOST_AUTO_TEST_CASE(compute_tapleaf) +{ + const uint8_t script[6] = {'f','o','o','b','a','r'}; + uint256 tlc0 = uint256S("edbc10c272a1215dcdcc11d605b9027b5ad6ed97cd45521203f136767b5b9c06"); + uint256 tlc2 = uint256S("8b5c4f90ae6bf76e259dbef5d8a59df06359c391b59263741b25eca76451b27a"); + BOOST_CHECK_EQUAL(ComputeTapleafHash(0xc0, Span(script)), tlc0); + BOOST_CHECK_EQUAL(ComputeTapleafHash(0xc2, Span(script)), tlc2); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/serfloat_tests.cpp b/src/test/serfloat_tests.cpp index ed1f081913..f6af32cf6c 100644 --- a/src/test/serfloat_tests.cpp +++ b/src/test/serfloat_tests.cpp @@ -111,7 +111,7 @@ Python code to generate the below hashes: */ BOOST_AUTO_TEST_CASE(doubles) { - CDataStream ss(SER_DISK, 0); + DataStream ss{}; // encode for (int i = 0; i < 1000; i++) { ss << EncodeDouble(i); diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index c90ae38ae8..09f77d2b61 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -90,8 +90,8 @@ BOOST_AUTO_TEST_CASE(varints) { // encode - CDataStream ss(SER_DISK, 0); - CDataStream::size_type size = 0; + DataStream ss{}; + DataStream::size_type size = 0; for (int i = 0; i < 100000; i++) { ss << VARINT_MODE(i, VarIntMode::NONNEGATIVE_SIGNED); size += ::GetSerializeSize(VARINT_MODE(i, VarIntMode::NONNEGATIVE_SIGNED), 0); @@ -120,7 +120,7 @@ BOOST_AUTO_TEST_CASE(varints) BOOST_AUTO_TEST_CASE(varints_bitpatterns) { - CDataStream ss(SER_DISK, 0); + DataStream ss{}; ss << VARINT_MODE(0, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "00"); ss.clear(); ss << VARINT_MODE(0x7f, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "7f"); ss.clear(); ss << VARINT_MODE(int8_t{0x7f}, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "7f"); ss.clear(); @@ -141,7 +141,7 @@ BOOST_AUTO_TEST_CASE(varints_bitpatterns) BOOST_AUTO_TEST_CASE(compactsize) { - CDataStream ss(SER_DISK, 0); + DataStream ss{}; std::vector<char>::size_type i, j; for (i = 1; i <= MAX_SIZE; i *= 2) @@ -182,7 +182,7 @@ BOOST_AUTO_TEST_CASE(noncanonical) { // Write some non-canonical CompactSize encodings, and // make sure an exception is thrown when read back. - CDataStream ss(SER_DISK, 0); + DataStream ss{}; std::vector<char>::size_type n; // zero encoded with three bytes: @@ -237,7 +237,8 @@ BOOST_AUTO_TEST_CASE(class_methods) BOOST_CHECK(methodtest2 == methodtest3); BOOST_CHECK(methodtest3 == methodtest4); - CDataStream ss2(SER_DISK, PROTOCOL_VERSION, intval, boolval, stringval, charstrval, txval); + CDataStream ss2{SER_DISK, PROTOCOL_VERSION}; + ss2 << intval << boolval << stringval << charstrval << txval; ss2 >> methodtest3; BOOST_CHECK(methodtest3 == methodtest4); } diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 1ce694b8c6..368f9e6047 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -10,6 +10,7 @@ #include <serialize.h> #include <streams.h> #include <test/data/sighash.json.h> +#include <test/util/json.h> #include <test/util/setup_common.h> #include <util/strencodings.h> #include <util/system.h> @@ -21,8 +22,6 @@ #include <univalue.h> -UniValue read_json(const std::string& jsondata); - // Old script.cpp SignatureHash function uint256 static SignatureHashOld(CScript scriptCode, const CTransaction& txTo, unsigned int nIn, int nHashType) { diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index dce230ac10..b7c1ce5066 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -128,9 +128,9 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader_rvalue) BOOST_AUTO_TEST_CASE(bitstream_reader_writer) { - CDataStream data(SER_NETWORK, INIT_PROTO_VERSION); + DataStream data{}; - BitStreamWriter<CDataStream> bit_writer(data); + BitStreamWriter bit_writer{data}; bit_writer.Write(0, 1); bit_writer.Write(2, 2); bit_writer.Write(6, 3); @@ -141,7 +141,7 @@ BOOST_AUTO_TEST_CASE(bitstream_reader_writer) bit_writer.Write(30497, 16); bit_writer.Flush(); - CDataStream data_copy(data); + DataStream data_copy{data}; uint32_t serialized_int1; data >> serialized_int1; BOOST_CHECK_EQUAL(serialized_int1, uint32_t{0x7700C35A}); // NOTE: Serialized as LE @@ -149,7 +149,7 @@ BOOST_AUTO_TEST_CASE(bitstream_reader_writer) data >> serialized_int2; BOOST_CHECK_EQUAL(serialized_int2, uint16_t{0x1072}); // NOTE: Serialized as LE - BitStreamReader<CDataStream> bit_reader(data_copy); + BitStreamReader bit_reader{data_copy}; BOOST_CHECK_EQUAL(bit_reader.Read(1), 0U); BOOST_CHECK_EQUAL(bit_reader.Read(2), 2U); BOOST_CHECK_EQUAL(bit_reader.Read(3), 6U); @@ -167,7 +167,7 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor) // Degenerate case { - CDataStream ds{in, 0, 0}; + DataStream ds{in}; ds.Xor({0x00, 0x00}); BOOST_CHECK_EQUAL(""s, ds.str()); } @@ -177,7 +177,7 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor) // Single character key { - CDataStream ds{in, 0, 0}; + DataStream ds{in}; ds.Xor({0xff}); BOOST_CHECK_EQUAL("\xf0\x0f"s, ds.str()); } @@ -189,7 +189,7 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor) in.push_back(std::byte{0x0f}); { - CDataStream ds{in, 0, 0}; + DataStream ds{in}; ds.Xor({0xff, 0x0f}); BOOST_CHECK_EQUAL("\x0f\x00"s, ds.str()); } @@ -500,4 +500,18 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) fs::remove(streams_test_filename); } +BOOST_AUTO_TEST_CASE(streams_hashed) +{ + CDataStream stream(SER_NETWORK, INIT_PROTO_VERSION); + HashedSourceWriter hash_writer{stream}; + const std::string data{"bitcoin"}; + hash_writer << data; + + CHashVerifier hash_verifier{&stream}; + std::string result; + hash_verifier >> result; + BOOST_CHECK_EQUAL(data, result); + BOOST_CHECK_EQUAL(hash_writer.GetHash(), hash_verifier.GetHash()); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 4a2417818b..0180fa47e8 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -21,6 +21,7 @@ #include <script/signingprovider.h> #include <script/standard.h> #include <streams.h> +#include <test/util/json.h> #include <test/util/script.h> #include <test/util/transaction_utils.h> #include <util/strencodings.h> @@ -37,9 +38,6 @@ typedef std::vector<unsigned char> valtype; -// In script_tests.cpp -UniValue read_json(const std::string& jsondata); - static CFeeRate g_dust{DUST_RELAY_TX_FEE}; static bool g_bare_multi{DEFAULT_PERMIT_BAREMULTISIG}; diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 52c0b1d938..8cdea3890e 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -117,7 +117,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, Dersig100Setup) // should fail. // Capture this interaction with the upgraded_nop argument: set it when evaluating // any script flag that is implemented as an upgraded NOP code. -static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache, CCoinsViewCache& active_coins_tip) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache, CCoinsViewCache& active_coins_tip) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { PrecomputedTransactionData txdata; diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index bc206fc945..9caefe43e2 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -187,7 +187,7 @@ BOOST_AUTO_TEST_CASE( methods ) // GetHex SetHex begin() end() size() GetLow64 G BOOST_CHECK(GetSerializeSize(R1L, PROTOCOL_VERSION) == 32); BOOST_CHECK(GetSerializeSize(ZeroL, PROTOCOL_VERSION) == 32); - CDataStream ss(0, PROTOCOL_VERSION); + DataStream ss{}; ss << R1L; BOOST_CHECK(ss.str() == std::string(R1Array,R1Array+32)); ss >> TmpL; diff --git a/src/test/util/json.cpp b/src/test/util/json.cpp new file mode 100644 index 0000000000..ad3c346c84 --- /dev/null +++ b/src/test/util/json.cpp @@ -0,0 +1,17 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/json.h> + +#include <string> +#include <util/check.h> + +#include <univalue.h> + +UniValue read_json(const std::string& jsondata) +{ + UniValue v; + Assert(v.read(jsondata) && v.isArray()); + return v.get_array(); +} diff --git a/src/test/util/json.h b/src/test/util/json.h new file mode 100644 index 0000000000..5b1026762e --- /dev/null +++ b/src/test/util/json.h @@ -0,0 +1,14 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_JSON_H +#define BITCOIN_TEST_UTIL_JSON_H + +#include <string> + +#include <univalue.h> + +UniValue read_json(const std::string& jsondata); + +#endif // BITCOIN_TEST_UTIL_JSON_H diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index 55e9d4d63b..0df1db84c4 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -8,7 +8,6 @@ #include <consensus/merkle.h> #include <key_io.h> #include <node/context.h> -#include <node/miner.h> #include <pow.h> #include <script/standard.h> #include <test/util/script.h> @@ -74,10 +73,11 @@ CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) return CTxIn{block->vtx[0]->GetHash(), 0}; } -std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) +std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey, + const BlockAssembler::Options& assembler_options) { auto block = std::make_shared<CBlock>( - BlockAssembler{Assert(node.chainman)->ActiveChainstate(), Assert(node.mempool.get())} + BlockAssembler{Assert(node.chainman)->ActiveChainstate(), Assert(node.mempool.get()), assembler_options} .CreateNewBlock(coinbase_scriptPubKey) ->block); @@ -87,3 +87,9 @@ std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coi return block; } +std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) +{ + BlockAssembler::Options assembler_options; + ApplyArgsManOptions(*node.args, assembler_options); + return PrepareBlock(node, coinbase_scriptPubKey, assembler_options); +} diff --git a/src/test/util/mining.h b/src/test/util/mining.h index 09e712cd35..70b1f7b3fb 100644 --- a/src/test/util/mining.h +++ b/src/test/util/mining.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_TEST_UTIL_MINING_H #define BITCOIN_TEST_UTIL_MINING_H +#include <node/miner.h> + #include <memory> #include <string> #include <vector> @@ -25,6 +27,8 @@ CTxIn MineBlock(const node::NodeContext&, const CScript& coinbase_scriptPubKey); /** Prepare a block to be mined */ std::shared_ptr<CBlock> PrepareBlock(const node::NodeContext&, const CScript& coinbase_scriptPubKey); +std::shared_ptr<CBlock> PrepareBlock(const node::NodeContext& node, const CScript& coinbase_scriptPubKey, + const node::BlockAssembler::Options& assembler_options); /** RPC-like helper function, returns the generated coin */ CTxIn generatetoaddress(const node::NodeContext&, const std::string& address); diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 975aff13c0..ac5dfe9e73 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -67,15 +67,14 @@ void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_by assert(node.ReceiveMsgBytes(msg_bytes, complete)); if (complete) { size_t nSizeAdded = 0; - auto it(node.vRecvMsg.begin()); - for (; it != node.vRecvMsg.end(); ++it) { + for (const auto& msg : node.vRecvMsg) { // vRecvMsg contains only completed CNetMessage // the single possible partially deserialized message are held by TransportDeserializer - nSizeAdded += it->m_raw_message_size; + nSizeAdded += msg.m_raw_message_size; } { LOCK(node.cs_vProcessMsg); - node.vProcessMsg.splice(node.vProcessMsg.end(), node.vRecvMsg, node.vRecvMsg.begin(), it); + node.vProcessMsg.splice(node.vProcessMsg.end(), node.vRecvMsg); node.nProcessQueueSize += nSizeAdded; node.fPauseRecv = node.nProcessQueueSize > nReceiveFloodSize; } diff --git a/src/test/util/net.h b/src/test/util/net.h index 90c606306f..e6506b0d08 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -103,7 +103,7 @@ constexpr auto ALL_NETWORKS = std::array{ class StaticContentsSock : public Sock { public: - explicit StaticContentsSock(const std::string& contents) : m_contents{contents}, m_consumed{0} + explicit StaticContentsSock(const std::string& contents) : m_contents{contents} { // Just a dummy number that is not INVALID_SOCKET. m_socket = INVALID_SOCKET - 1; @@ -191,7 +191,7 @@ public: private: const std::string m_contents; - mutable size_t m_consumed; + mutable size_t m_consumed{0}; }; std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candidates, FastRandomContext& random_context); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index efc8529bf6..6e72f69968 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -208,23 +208,24 @@ ChainTestingSetup::~ChainTestingSetup() void TestingSetup::LoadVerifyActivateChainstate() { + auto& chainman{*Assert(m_node.chainman)}; node::ChainstateLoadOptions options; options.mempool = Assert(m_node.mempool.get()); options.block_tree_db_in_memory = m_block_tree_db_in_memory; options.coins_db_in_memory = m_coins_db_in_memory; options.reindex = node::fReindex; options.reindex_chainstate = m_args.GetBoolArg("-reindex-chainstate", false); - options.prune = node::fPruneMode; + options.prune = chainman.m_blockman.IsPruneMode(); options.check_blocks = m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); options.check_level = m_args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); - auto [status, error] = LoadChainstate(*Assert(m_node.chainman), m_cache_sizes, options); + auto [status, error] = LoadChainstate(chainman, m_cache_sizes, options); assert(status == node::ChainstateLoadStatus::SUCCESS); - std::tie(status, error) = VerifyLoadedChainstate(*Assert(m_node.chainman), options); + std::tie(status, error) = VerifyLoadedChainstate(chainman, options); assert(status == node::ChainstateLoadStatus::SUCCESS); BlockValidationState state; - if (!m_node.chainman->ActiveChainstate().ActivateBestChain(state)) { + if (!chainman.ActiveChainstate().ActivateBestChain(state)) { throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } } @@ -397,15 +398,15 @@ std::vector<CTransactionRef> TestChain100Setup::PopulateMempool(FastRandomContex unspent_prevouts.pop_front(); } const size_t num_outputs = det_rand.randrange(24) + 1; - // Approximately 1000sat "fee," equal output amounts. - const CAmount amount_per_output = (total_in - 1000) / num_outputs; + const CAmount fee = 100 * det_rand.randrange(30); + const CAmount amount_per_output = (total_in - fee) / num_outputs; for (size_t n{0}; n < num_outputs; ++n) { CScript spk = CScript() << CScriptNum(num_transactions + n); mtx.vout.push_back(CTxOut(amount_per_output, spk)); } CTransactionRef ptx = MakeTransactionRef(mtx); mempool_transactions.push_back(ptx); - if (amount_per_output > 2000) { + if (amount_per_output > 3000) { // If the value is high enough to fund another transaction + fees, keep track of it so // it can be used to build a more complex transaction graph. Insert randomly into // unspent_prevouts for extra randomness in the resulting structures. @@ -415,9 +416,11 @@ std::vector<CTransactionRef> TestChain100Setup::PopulateMempool(FastRandomContex } } if (submit) { - LOCK2(m_node.mempool->cs, cs_main); + LOCK2(cs_main, m_node.mempool->cs); LockPoints lp; - m_node.mempool->addUnchecked(CTxMemPoolEntry(ptx, 1000, 0, 1, false, 4, lp)); + m_node.mempool->addUnchecked(CTxMemPoolEntry(ptx, /*fee=*/(total_in - num_outputs * amount_per_output), + /*time=*/0, /*entry_height=*/1, + /*spends_coinbase=*/false, /*sigops_cost=*/4, lp)); } --num_transactions; } diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 620c41ff0c..56867a584b 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -116,7 +116,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a legacy (IBD) chainstate. // - Chainstate& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool)); + Chainstate& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(&mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 4a42cec4af..91383ee4a5 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -13,7 +13,7 @@ /* Define a virtual block time, one block per 10 minutes after Nov 14 2014, 0:55:36am */ static int32_t TestTime(int nHeight) { return 1415926536 + 600 * nHeight; } -static const std::string StateName(ThresholdState state) +static std::string StateName(ThresholdState state) { switch (state) { case ThresholdState::DEFINED: return "DEFINED"; diff --git a/src/tinyformat.h b/src/tinyformat.h index 8eded00add..3ec385bc95 100644 --- a/src/tinyformat.h +++ b/src/tinyformat.h @@ -508,9 +508,6 @@ class FormatArg { public: FormatArg() - : m_value(nullptr), - m_formatImpl(nullptr), - m_toIntImpl(nullptr) { } template<typename T> @@ -549,10 +546,10 @@ class FormatArg return convertToInt<T>::invoke(*static_cast<const T*>(value)); } - const void* m_value; + const void* m_value{nullptr}; void (*m_formatImpl)(std::ostream& out, const char* fmtBegin, - const char* fmtEnd, int ntrunc, const void* value); - int (*m_toIntImpl)(const void* value); + const char* fmtEnd, int ntrunc, const void* value){nullptr}; + int (*m_toIntImpl)(const void* value){nullptr}; }; diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index b5f1fa7138..ece77f9023 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -53,8 +53,8 @@ static const uint16_t DEFAULT_TOR_SOCKS_PORT = 9050; /****** Low-level TorControlConnection ********/ -TorControlConnection::TorControlConnection(struct event_base *_base): - base(_base), b_conn(nullptr) +TorControlConnection::TorControlConnection(struct event_base* _base) + : base(_base) { } diff --git a/src/torcontrol.h b/src/torcontrol.h index 81475aee74..6563a2ef42 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -93,7 +93,7 @@ private: /** Libevent event base */ struct event_base *base; /** Connection to control socket */ - struct bufferevent *b_conn; + struct bufferevent* b_conn{nullptr}; /** Message being received */ TorControlReply message; /** Response handlers */ diff --git a/src/txdb.cpp b/src/txdb.cpp index f04a4e9800..c12b540b9b 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -111,7 +111,7 @@ std::vector<uint256> CCoinsViewDB::GetHeadBlocks() const { return vhashHeadBlocks; } -bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { +bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, bool erase) { CDBBatch batch(*m_db); size_t count = 0; size_t changed = 0; @@ -146,8 +146,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { changed++; } count++; - CCoinsMap::iterator itOld = it++; - mapCoins.erase(itOld); + it = erase ? mapCoins.erase(it) : std::next(it); if (batch.SizeEstimate() > batch_size) { LogPrint(BCLog::COINDB, "Writing partial batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0)); m_db->WriteBatch(batch); diff --git a/src/txdb.h b/src/txdb.h index 0570355016..e3422846c0 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -8,6 +8,7 @@ #include <coins.h> #include <dbwrapper.h> +#include <kernel/cs_main.h> #include <sync.h> #include <fs.h> @@ -44,9 +45,6 @@ static const int64_t max_filter_index_cache = 1024; //! Max memory allocated to coin DB specific cache (MiB) static const int64_t nMaxCoinsDBCache = 8; -// Actually declared in validation.cpp; can't include because of circular dependency. -extern RecursiveMutex cs_main; - /** CCoinsView backed by the coin database (chainstate/) */ class CCoinsViewDB final : public CCoinsView { @@ -64,7 +62,7 @@ public: bool HaveCoin(const COutPoint &outpoint) const override; uint256 GetBestBlock() const override; std::vector<uint256> GetHeadBlocks() const override; - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, bool erase = true) override; std::unique_ptr<CCoinsViewCursor> Cursor() const override; //! Whether an unsupported database format is used. diff --git a/src/txmempool.cpp b/src/txmempool.cpp index aa04f8a4d0..378123ce0f 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1128,7 +1128,7 @@ void CTxMemPool::SetLoadTried(bool load_tried) } -const std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept +std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept { switch (r) { case MemPoolRemovalReason::EXPIRY: return "expiry"; diff --git a/src/txmempool.h b/src/txmempool.h index c3bc86cc04..2c3cb7e9db 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -21,6 +21,7 @@ #include <coins.h> #include <consensus/amount.h> #include <indirectmap.h> +#include <kernel/cs_main.h> #include <kernel/mempool_entry.h> #include <policy/feerate.h> #include <policy/packages.h> @@ -39,7 +40,6 @@ class CBlockIndex; class CChain; class Chainstate; -extern RecursiveMutex cs_main; /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; @@ -237,7 +237,7 @@ enum class MemPoolRemovalReason { REPLACED, //!< Removed for replacement }; -const std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept; +std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept; /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions diff --git a/src/txorphanage.cpp b/src/txorphanage.cpp index 94f64abca7..19f9fae998 100644 --- a/src/txorphanage.cpp +++ b/src/txorphanage.cpp @@ -145,17 +145,19 @@ void TxOrphanage::LimitOrphans(unsigned int max_orphans) if (nEvicted > 0) LogPrint(BCLog::MEMPOOL, "orphanage overflow, removed %u tx\n", nEvicted); } -void TxOrphanage::AddChildrenToWorkSet(const CTransaction& tx, NodeId peer) +void TxOrphanage::AddChildrenToWorkSet(const CTransaction& tx) { LOCK(m_mutex); - // Get this peer's work set, emplacing an empty set it didn't exist - std::set<uint256>& orphan_work_set = m_peer_work_set.try_emplace(peer).first->second; for (unsigned int i = 0; i < tx.vout.size(); i++) { const auto it_by_prev = m_outpoint_to_orphan_it.find(COutPoint(tx.GetHash(), i)); if (it_by_prev != m_outpoint_to_orphan_it.end()) { for (const auto& elem : it_by_prev->second) { + // Get this source peer's work set, emplacing an empty set if it didn't exist + // (note: if this peer wasn't still connected, we would have removed the orphan tx already) + std::set<uint256>& orphan_work_set = m_peer_work_set.try_emplace(elem->second.fromPeer).first->second; + // Add this tx to the work set orphan_work_set.insert(elem->first); } } @@ -172,7 +174,7 @@ bool TxOrphanage::HaveTx(const GenTxid& gtxid) const } } -CTransactionRef TxOrphanage::GetTxToReconsider(NodeId peer, NodeId& originator, bool& more) +CTransactionRef TxOrphanage::GetTxToReconsider(NodeId peer) { LOCK(m_mutex); @@ -185,16 +187,25 @@ CTransactionRef TxOrphanage::GetTxToReconsider(NodeId peer, NodeId& originator, const auto orphan_it = m_orphans.find(txid); if (orphan_it != m_orphans.end()) { - more = !work_set.empty(); - originator = orphan_it->second.fromPeer; return orphan_it->second.tx; } } } - more = false; return nullptr; } +bool TxOrphanage::HaveTxToReconsider(NodeId peer) +{ + LOCK(m_mutex); + + auto work_set_it = m_peer_work_set.find(peer); + if (work_set_it != m_peer_work_set.end()) { + auto& work_set = work_set_it->second; + return !work_set.empty(); + } + return false; +} + void TxOrphanage::EraseForBlock(const CBlock& block) { LOCK(m_mutex); diff --git a/src/txorphanage.h b/src/txorphanage.h index cd7587fab5..45276c6c98 100644 --- a/src/txorphanage.h +++ b/src/txorphanage.h @@ -27,13 +27,11 @@ public: bool HaveTx(const GenTxid& gtxid) const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); /** Extract a transaction from a peer's work set - * Returns nullptr and sets more to false if there are no transactions - * to work on. Otherwise returns the transaction reference, removes - * the transaction from the work set, and populates its arguments with - * the originating peer, and whether there are more orphans for this peer - * to work on after this tx. + * Returns nullptr if there are no transactions to work on. + * Otherwise returns the transaction reference, and removes + * it from the work set. */ - CTransactionRef GetTxToReconsider(NodeId peer, NodeId& originator, bool& more) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + CTransactionRef GetTxToReconsider(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); /** Erase an orphan by txid */ int EraseTx(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); @@ -47,8 +45,11 @@ public: /** Limit the orphanage to the given maximum */ void LimitOrphans(unsigned int max_orphans) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); - /** Add any orphans that list a particular tx as a parent into a peer's work set */ - void AddChildrenToWorkSet(const CTransaction& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + /** Add any orphans that list a particular tx as a parent into the from peer's work set */ + void AddChildrenToWorkSet(const CTransaction& tx) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex);; + + /** Does this peer have any work to do? */ + bool HaveTxToReconsider(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex);; /** Return how many entries exist in the orphange */ size_t Size() EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) @@ -72,7 +73,7 @@ protected: * -maxorphantx/DEFAULT_MAX_ORPHAN_TRANSACTIONS */ std::map<uint256, OrphanTx> m_orphans GUARDED_BY(m_mutex); - /** Which peer provided a parent tx of orphans that need to be reconsidered */ + /** Which peer provided the orphans that need to be reconsidered */ std::map<NodeId, std::set<uint256>> m_peer_work_set GUARDED_BY(m_mutex); using OrphanMap = decltype(m_orphans); diff --git a/src/univalue/include/univalue_utffilter.h b/src/univalue/include/univalue_utffilter.h index f688eaaa30..41d8e6bb05 100644 --- a/src/univalue/include/univalue_utffilter.h +++ b/src/univalue/include/univalue_utffilter.h @@ -13,8 +13,8 @@ class JSONUTF8StringFilter { public: - explicit JSONUTF8StringFilter(std::string &s): - str(s), is_valid(true), codepoint(0), state(0), surpair(0) + explicit JSONUTF8StringFilter(std::string& s) + : str(s) { } // Write single 8-bit char (may be part of UTF-8 sequence) @@ -79,10 +79,10 @@ public: } private: std::string &str; - bool is_valid; + bool is_valid{true}; // Current UTF-8 decoding state - unsigned int codepoint; - int state; // Top bit to be filled in for next UTF-8 byte, or 0 + unsigned int codepoint{0}; + int state{0}; // Top bit to be filled in for next UTF-8 byte, or 0 // Keep track of the following state to handle the following section of // RFC4627: @@ -94,7 +94,7 @@ private: // "\uD834\uDD1E". // // Two subsequent \u.... may have to be replaced with one actual codepoint. - unsigned int surpair; // First half of open UTF-16 surrogate pair, or 0 + unsigned int surpair{0}; // First half of open UTF-16 surrogate pair, or 0 void append_codepoint(unsigned int codepoint_) { diff --git a/src/util/check.cpp b/src/util/check.cpp index 34b9d376a7..795dce7124 100644 --- a/src/util/check.cpp +++ b/src/util/check.cpp @@ -14,8 +14,9 @@ #include <cstdio> #include <cstdlib> #include <string> +#include <string_view> -std::string StrFormatInternalBug(const char* msg, const char* file, int line, const char* func) +std::string StrFormatInternalBug(std::string_view msg, std::string_view file, int line, std::string_view func) { return strprintf("Internal bug detected: \"%s\"\n%s:%d (%s)\n" "%s %s\n" @@ -23,12 +24,12 @@ std::string StrFormatInternalBug(const char* msg, const char* file, int line, co msg, file, line, func, PACKAGE_NAME, FormatFullVersion(), PACKAGE_BUGREPORT); } -NonFatalCheckError::NonFatalCheckError(const char* msg, const char* file, int line, const char* func) +NonFatalCheckError::NonFatalCheckError(std::string_view msg, std::string_view file, int line, std::string_view func) : std::runtime_error{StrFormatInternalBug(msg, file, line, func)} { } -void assertion_fail(const char* file, int line, const char* func, const char* assertion) +void assertion_fail(std::string_view file, int line, std::string_view func, std::string_view assertion) { auto str = strprintf("%s:%s %s: Assertion `%s' failed.\n", file, line, func, assertion); fwrite(str.data(), 1, str.size(), stderr); diff --git a/src/util/check.h b/src/util/check.h index 96cd905d47..7ddcebf506 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -8,14 +8,16 @@ #include <attributes.h> #include <stdexcept> +#include <string> +#include <string_view> #include <utility> -std::string StrFormatInternalBug(const char* msg, const char* file, int line, const char* func); +std::string StrFormatInternalBug(std::string_view msg, std::string_view file, int line, std::string_view func); class NonFatalCheckError : public std::runtime_error { public: - NonFatalCheckError(const char* msg, const char* file, int line, const char* func); + NonFatalCheckError(std::string_view msg, std::string_view file, int line, std::string_view func); }; #define STR_INTERNAL_BUG(msg) StrFormatInternalBug((msg), __FILE__, __LINE__, __func__) @@ -49,7 +51,7 @@ T&& inline_check_non_fatal(LIFETIMEBOUND T&& val, const char* file, int line, co #endif /** Helper for Assert() */ -void assertion_fail(const char* file, int line, const char* func, const char* assertion); +void assertion_fail(std::string_view file, int line, std::string_view func, std::string_view assertion); /** Helper for Assert()/Assume() */ template <bool IS_ASSERT, typename T> diff --git a/src/util/fees.cpp b/src/util/fees.cpp index cbefe18dbb..8ada02ce54 100644 --- a/src/util/fees.cpp +++ b/src/util/fees.cpp @@ -49,7 +49,7 @@ std::string FeeModes(const std::string& delimiter) return Join(FeeModeMap(), delimiter, [&](const std::pair<std::string, FeeEstimateMode>& i) { return i.first; }); } -const std::string InvalidEstimateModeErrorMessage() +std::string InvalidEstimateModeErrorMessage() { return "Invalid estimate_mode parameter, must be one of: \"" + FeeModes("\", \"") + "\""; } diff --git a/src/util/fees.h b/src/util/fees.h index 9ef2389d3e..10ba1e4f85 100644 --- a/src/util/fees.h +++ b/src/util/fees.h @@ -13,6 +13,6 @@ enum class FeeReason; bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode); std::string StringForFeeReason(FeeReason reason); std::string FeeModes(const std::string& delimiter); -const std::string InvalidEstimateModeErrorMessage(); +std::string InvalidEstimateModeErrorMessage(); #endif // BITCOIN_UTIL_FEES_H diff --git a/src/util/sock.h b/src/util/sock.h index adcca377e3..6bac2dfd34 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -181,9 +181,9 @@ public: * Auxiliary requested/occurred events to wait for in `WaitMany()`. */ struct Events { - explicit Events(Event req) : requested{req}, occurred{0} {} + explicit Events(Event req) : requested{req} {} Event requested; - Event occurred; + Event occurred{0}; }; struct HashSharedPtrSock { diff --git a/src/util/system.cpp b/src/util/system.cpp index 309595ff5b..0cea283ece 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -242,7 +242,7 @@ static std::optional<util::SettingsValue> InterpretValue(const KeyInfo& key, con ArgsManager::ArgsManager() = default; ArgsManager::~ArgsManager() = default; -const std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const +std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const { std::set<std::string> unsuitables; @@ -262,7 +262,7 @@ const std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const return unsuitables; } -const std::list<SectionInfo> ArgsManager::GetUnrecognizedSections() const +std::list<SectionInfo> ArgsManager::GetUnrecognizedSections() const { // Section names to be recognized in the config file. static const std::set<std::string> available_sections{ diff --git a/src/util/system.h b/src/util/system.h index 671491f2ff..c053adf8c3 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -250,12 +250,12 @@ protected: * on the command line or in a network-specific section in the * config file. */ - const std::set<std::string> GetUnsuitableSectionOnlyArgs() const; + std::set<std::string> GetUnsuitableSectionOnlyArgs() const; /** * Log warnings for unrecognized section names in the config file. */ - const std::list<SectionInfo> GetUnrecognizedSections() const; + std::list<SectionInfo> GetUnrecognizedSections() const; struct Command { /** The command (if one has been registered with AddCommand), or empty */ diff --git a/src/validation.cpp b/src/validation.cpp index aff86c4736..62ce1f1162 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -76,8 +76,6 @@ using node::BlockManager; using node::BlockMap; using node::CBlockIndexHeightOnlyComparator; using node::CBlockIndexWorkComparator; -using node::fImporting; -using node::fPruneMode; using node::fReindex; using node::ReadBlockFromDisk; using node::SnapshotMetadata; @@ -107,18 +105,6 @@ const std::vector<std::string> CHECKLEVEL_DOC { * */ static constexpr int PRUNE_LOCK_BUFFER{10}; -/** - * Mutex to guard access to validation specific variables, such as reading - * or changing the chainstate. - * - * This may also need to be locked when updating the transaction pool, e.g. on - * AcceptToMemoryPool. See CTxMemPool::cs comment for details. - * - * The transaction pool has a separate lock to allow reading from it and the - * chainstate at the same time. - */ -RecursiveMutex cs_main; - GlobalMutex g_best_block_mutex; std::condition_variable g_best_block_cv; uint256 g_best_block; @@ -1585,8 +1571,9 @@ bool Chainstate::IsInitialBlockDownload() const LOCK(cs_main); if (m_cached_finished_ibd.load(std::memory_order_relaxed)) return false; - if (fImporting || fReindex) + if (m_chainman.m_blockman.LoadingBlocks()) { return true; + } if (m_chain.Tip() == nullptr) return true; if (m_chain.Tip()->nChainWork < m_chainman.MinimumChainWork()) { @@ -2423,7 +2410,7 @@ bool Chainstate::FlushStateToDisk( CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(); LOCK(m_blockman.cs_LastBlockFile); - if (fPruneMode && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !fReindex) { + if (m_blockman.IsPruneMode() && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !fReindex) { // make sure we don't prune above any of the prune locks bestblocks // pruning is height-based int last_prune{m_chain.Height()}; // last height we can prune @@ -4109,7 +4096,7 @@ bool CVerifyDB::VerifyDB( if (pindex->nHeight <= chainstate.m_chain.Height() - nCheckDepth) { break; } - if ((fPruneMode || is_snapshot_cs) && !(pindex->nStatus & BLOCK_HAVE_DATA)) { + if ((chainstate.m_blockman.IsPruneMode() || is_snapshot_cs) && !(pindex->nStatus & BLOCK_HAVE_DATA)) { // If pruning or running under an assumeutxo snapshot, only go // back as far as we have data. LogPrintf("VerifyDB(): block verification stopping at height %d (pruning, no data)\n", pindex->nHeight); diff --git a/src/validation.h b/src/validation.h index 900f380563..7170467b00 100644 --- a/src/validation.h +++ b/src/validation.h @@ -18,6 +18,7 @@ #include <consensus/amount.h> #include <deploymentstatus.h> #include <fs.h> +#include <kernel/cs_main.h> // IWYU pragma: export #include <node/blockstorage.h> #include <policy/feerate.h> #include <policy/packages.h> @@ -86,7 +87,6 @@ enum class SynchronizationState { POST_INIT }; -extern RecursiveMutex cs_main; extern GlobalMutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; /** Used to notify getblocktemplate RPC of new tips. */ diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 900cb0474a..d344c8bfbd 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -17,7 +17,7 @@ #include <unordered_map> #include <utility> -const std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept; +std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept; /** * MainSignalsImpl manages a list of shared_ptr<CValidationInterface> callbacks. diff --git a/src/validationinterface.h b/src/validationinterface.h index acd775ada4..8c20cc8ffb 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -6,13 +6,13 @@ #ifndef BITCOIN_VALIDATIONINTERFACE_H #define BITCOIN_VALIDATIONINTERFACE_H +#include <kernel/cs_main.h> #include <primitives/transaction.h> // CTransaction(Ref) #include <sync.h> #include <functional> #include <memory> -extern RecursiveMutex cs_main; class BlockValidationState; class CBlock; class CBlockIndex; diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 4ec3ac2189..653115aa81 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -8,6 +8,7 @@ #include <wallet/bdb.h> #include <wallet/db.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/translation.h> @@ -220,17 +221,17 @@ BerkeleyEnvironment::BerkeleyEnvironment() : m_use_shared_memory(false) fMockDb = true; } -BerkeleyBatch::SafeDbt::SafeDbt() +SafeDbt::SafeDbt() { m_dbt.set_flags(DB_DBT_MALLOC); } -BerkeleyBatch::SafeDbt::SafeDbt(void* data, size_t size) +SafeDbt::SafeDbt(void* data, size_t size) : m_dbt(data, size) { } -BerkeleyBatch::SafeDbt::~SafeDbt() +SafeDbt::~SafeDbt() { if (m_dbt.get_data() != nullptr) { // Clear memory, e.g. in case it was a private key @@ -244,17 +245,17 @@ BerkeleyBatch::SafeDbt::~SafeDbt() } } -const void* BerkeleyBatch::SafeDbt::get_data() const +const void* SafeDbt::get_data() const { return m_dbt.get_data(); } -uint32_t BerkeleyBatch::SafeDbt::get_size() const +uint32_t SafeDbt::get_size() const { return m_dbt.get_size(); } -BerkeleyBatch::SafeDbt::operator Dbt*() +SafeDbt::operator Dbt*() { return &m_dbt; } @@ -307,7 +308,7 @@ BerkeleyDatabase::~BerkeleyDatabase() } } -BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const bool read_only, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr), m_cursor(nullptr), m_database(database) +BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const bool read_only, bool fFlushOnCloseIn) : m_database(database) { database.AddRef(); database.Open(); @@ -398,7 +399,6 @@ void BerkeleyBatch::Close() activeTxn->abort(); activeTxn = nullptr; pdb = nullptr; - CloseCursor(); if (fFlushOnClose) Flush(); @@ -476,15 +476,15 @@ bool BerkeleyDatabase::Rewrite(const char* pszSkip) fSuccess = false; } - if (db.StartCursor()) { + std::unique_ptr<DatabaseCursor> cursor = db.GetNewCursor(); + if (cursor) { while (fSuccess) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - bool complete; - bool ret1 = db.ReadAtCursor(ssKey, ssValue, complete); - if (complete) { + DataStream ssKey{}; + DataStream ssValue{}; + DatabaseCursor::Status ret1 = cursor->Next(ssKey, ssValue); + if (ret1 == DatabaseCursor::Status::DONE) { break; - } else if (!ret1) { + } else if (ret1 == DatabaseCursor::Status::FAIL) { fSuccess = false; break; } @@ -502,7 +502,7 @@ bool BerkeleyDatabase::Rewrite(const char* pszSkip) if (ret2 > 0) fSuccess = false; } - db.CloseCursor(); + cursor.reset(); } if (fSuccess) { db.Close(); @@ -656,48 +656,52 @@ void BerkeleyDatabase::ReloadDbEnv() env->ReloadDbEnv(); } -bool BerkeleyBatch::StartCursor() +BerkeleyCursor::BerkeleyCursor(BerkeleyDatabase& database) { - assert(!m_cursor); - if (!pdb) - return false; - int ret = pdb->cursor(nullptr, &m_cursor, 0); - return ret == 0; + if (!database.m_db.get()) { + throw std::runtime_error(STR_INTERNAL_BUG("BerkeleyDatabase does not exist")); + } + int ret = database.m_db->cursor(nullptr, &m_cursor, 0); + if (ret != 0) { + throw std::runtime_error(STR_INTERNAL_BUG(strprintf("BDB Cursor could not be created. Returned %d", ret))); + } } -bool BerkeleyBatch::ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) +DatabaseCursor::Status BerkeleyCursor::Next(DataStream& ssKey, DataStream& ssValue) { - complete = false; - if (m_cursor == nullptr) return false; + if (m_cursor == nullptr) return Status::FAIL; // Read at cursor SafeDbt datKey; SafeDbt datValue; int ret = m_cursor->get(datKey, datValue, DB_NEXT); if (ret == DB_NOTFOUND) { - complete = true; + return Status::DONE; + } + if (ret != 0 || datKey.get_data() == nullptr || datValue.get_data() == nullptr) { + return Status::FAIL; } - if (ret != 0) - return false; - else if (datKey.get_data() == nullptr || datValue.get_data() == nullptr) - return false; // Convert to streams - ssKey.SetType(SER_DISK); ssKey.clear(); ssKey.write({AsBytePtr(datKey.get_data()), datKey.get_size()}); - ssValue.SetType(SER_DISK); ssValue.clear(); ssValue.write({AsBytePtr(datValue.get_data()), datValue.get_size()}); - return true; + return Status::MORE; } -void BerkeleyBatch::CloseCursor() +BerkeleyCursor::~BerkeleyCursor() { if (!m_cursor) return; m_cursor->close(); m_cursor = nullptr; } +std::unique_ptr<DatabaseCursor> BerkeleyBatch::GetNewCursor() +{ + if (!pdb) return nullptr; + return std::make_unique<BerkeleyCursor>(m_database); +} + bool BerkeleyBatch::TxnBegin() { if (!pdb || activeTxn) @@ -749,7 +753,7 @@ std::string BerkeleyDatabaseVersion() return DbEnv::version(nullptr, nullptr, nullptr); } -bool BerkeleyBatch::ReadKey(CDataStream&& key, CDataStream& value) +bool BerkeleyBatch::ReadKey(DataStream&& key, DataStream& value) { if (!pdb) return false; @@ -765,7 +769,7 @@ bool BerkeleyBatch::ReadKey(CDataStream&& key, CDataStream& value) return false; } -bool BerkeleyBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite) +bool BerkeleyBatch::WriteKey(DataStream&& key, DataStream&& value, bool overwrite) { if (!pdb) return false; @@ -780,7 +784,7 @@ bool BerkeleyBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwr return (ret == 0); } -bool BerkeleyBatch::EraseKey(CDataStream&& key) +bool BerkeleyBatch::EraseKey(DataStream&& key) { if (!pdb) return false; @@ -793,7 +797,7 @@ bool BerkeleyBatch::EraseKey(CDataStream&& key) return (ret == 0 || ret == DB_NOTFOUND); } -bool BerkeleyBatch::HasKey(CDataStream&& key) +bool BerkeleyBatch::HasKey(DataStream&& key) { if (!pdb) return false; diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index 40a1031c8e..06c98972b0 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -165,40 +165,51 @@ public: std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override; }; -/** RAII class that provides access to a Berkeley database */ -class BerkeleyBatch : public DatabaseBatch +/** RAII class that automatically cleanses its data on destruction */ +class SafeDbt final { - /** RAII class that automatically cleanses its data on destruction */ - class SafeDbt final - { - Dbt m_dbt; + Dbt m_dbt; - public: - // construct Dbt with internally-managed data - SafeDbt(); - // construct Dbt with provided data - SafeDbt(void* data, size_t size); - ~SafeDbt(); +public: + // construct Dbt with internally-managed data + SafeDbt(); + // construct Dbt with provided data + SafeDbt(void* data, size_t size); + ~SafeDbt(); + + // delegate to Dbt + const void* get_data() const; + uint32_t get_size() const; + + // conversion operator to access the underlying Dbt + operator Dbt*(); +}; - // delegate to Dbt - const void* get_data() const; - uint32_t get_size() const; +class BerkeleyCursor : public DatabaseCursor +{ +private: + Dbc* m_cursor; - // conversion operator to access the underlying Dbt - operator Dbt*(); - }; +public: + explicit BerkeleyCursor(BerkeleyDatabase& database); + ~BerkeleyCursor() override; + + Status Next(DataStream& key, DataStream& value) override; +}; +/** RAII class that provides access to a Berkeley database */ +class BerkeleyBatch : public DatabaseBatch +{ private: - bool ReadKey(CDataStream&& key, CDataStream& value) override; - bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite = true) override; - bool EraseKey(CDataStream&& key) override; - bool HasKey(CDataStream&& key) override; + bool ReadKey(DataStream&& key, DataStream& value) override; + bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) override; + bool EraseKey(DataStream&& key) override; + bool HasKey(DataStream&& key) override; protected: - Db* pdb; + Db* pdb{nullptr}; std::string strFile; - DbTxn* activeTxn; - Dbc* m_cursor; + DbTxn* activeTxn{nullptr}; bool fReadOnly; bool fFlushOnClose; BerkeleyEnvironment *env; @@ -214,9 +225,7 @@ public: void Flush() override; void Close() override; - bool StartCursor() override; - bool ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) override; - void CloseCursor() override; + std::unique_ptr<DatabaseCursor> GetNewCursor() override; bool TxnBegin() override; bool TxnCommit() override; bool TxnAbort() override; diff --git a/src/wallet/db.h b/src/wallet/db.h index f09844c37e..d4c590fac7 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -22,14 +22,33 @@ struct bilingual_str; namespace wallet { void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename); +class DatabaseCursor +{ +public: + explicit DatabaseCursor() {} + virtual ~DatabaseCursor() {} + + DatabaseCursor(const DatabaseCursor&) = delete; + DatabaseCursor& operator=(const DatabaseCursor&) = delete; + + enum class Status + { + FAIL, + MORE, + DONE, + }; + + virtual Status Next(DataStream& key, DataStream& value) { return Status::FAIL; } +}; + /** RAII class that provides access to a WalletDatabase */ class DatabaseBatch { private: - virtual bool ReadKey(CDataStream&& key, CDataStream& value) = 0; - virtual bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite=true) = 0; - virtual bool EraseKey(CDataStream&& key) = 0; - virtual bool HasKey(CDataStream&& key) = 0; + virtual bool ReadKey(DataStream&& key, DataStream& value) = 0; + virtual bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) = 0; + virtual bool EraseKey(DataStream&& key) = 0; + virtual bool HasKey(DataStream&& key) = 0; public: explicit DatabaseBatch() {} @@ -44,7 +63,7 @@ public: template <typename K, typename T> bool Read(const K& key, T& value) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); + DataStream ssKey{}; ssKey.reserve(1000); ssKey << key; @@ -61,7 +80,7 @@ public: template <typename K, typename T> bool Write(const K& key, const T& value, bool fOverwrite = true) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); + DataStream ssKey{}; ssKey.reserve(1000); ssKey << key; @@ -75,7 +94,7 @@ public: template <typename K> bool Erase(const K& key) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); + DataStream ssKey{}; ssKey.reserve(1000); ssKey << key; @@ -85,16 +104,14 @@ public: template <typename K> bool Exists(const K& key) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); + DataStream ssKey{}; ssKey.reserve(1000); ssKey << key; return HasKey(std::move(ssKey)); } - virtual bool StartCursor() = 0; - virtual bool ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) = 0; - virtual void CloseCursor() = 0; + virtual std::unique_ptr<DatabaseCursor> GetNewCursor() = 0; virtual bool TxnBegin() = 0; virtual bool TxnCommit() = 0; virtual bool TxnAbort() = 0; @@ -106,7 +123,7 @@ class WalletDatabase { public: /** Create dummy DB handle */ - WalletDatabase() : nUpdateCounter(0), nLastSeen(0), nLastFlushed(0), nLastWalletUpdate(0) {} + WalletDatabase() : nUpdateCounter(0) {} virtual ~WalletDatabase() {}; /** Open the database if it is not already opened. */ @@ -148,30 +165,33 @@ public: virtual std::string Format() = 0; std::atomic<unsigned int> nUpdateCounter; - unsigned int nLastSeen; - unsigned int nLastFlushed; - int64_t nLastWalletUpdate; + unsigned int nLastSeen{0}; + unsigned int nLastFlushed{0}; + int64_t nLastWalletUpdate{0}; /** Make a DatabaseBatch connected to this database */ virtual std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) = 0; }; +class DummyCursor : public DatabaseCursor +{ + Status Next(DataStream& key, DataStream& value) override { return Status::FAIL; } +}; + /** RAII class that provides access to a DummyDatabase. Never fails. */ class DummyBatch : public DatabaseBatch { private: - bool ReadKey(CDataStream&& key, CDataStream& value) override { return true; } - bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite=true) override { return true; } - bool EraseKey(CDataStream&& key) override { return true; } - bool HasKey(CDataStream&& key) override { return true; } + bool ReadKey(DataStream&& key, DataStream& value) override { return true; } + bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) override { return true; } + bool EraseKey(DataStream&& key) override { return true; } + bool HasKey(DataStream&& key) override { return true; } public: void Flush() override {} void Close() override {} - bool StartCursor() override { return true; } - bool ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) override { return true; } - void CloseCursor() override {} + std::unique_ptr<DatabaseCursor> GetNewCursor() override { return std::make_unique<DummyCursor>(); } bool TxnBegin() override { return true; } bool TxnCommit() override { return true; } bool TxnAbort() override { return true; } diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index efa548ad91..403ec711ff 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -47,7 +47,8 @@ bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error) std::unique_ptr<DatabaseBatch> batch = db.MakeBatch(); bool ret = true; - if (!batch->StartCursor()) { + std::unique_ptr<DatabaseCursor> cursor = batch->GetNewCursor(); + if (!cursor) { error = _("Error: Couldn't create cursor into database"); ret = false; } @@ -66,15 +67,15 @@ bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error) // Read the records while (true) { - CDataStream ss_key(SER_DISK, CLIENT_VERSION); - CDataStream ss_value(SER_DISK, CLIENT_VERSION); - bool complete; - ret = batch->ReadAtCursor(ss_key, ss_value, complete); - if (complete) { + DataStream ss_key{}; + DataStream ss_value{}; + DatabaseCursor::Status status = cursor->Next(ss_key, ss_value); + if (status == DatabaseCursor::Status::DONE) { ret = true; break; - } else if (!ret) { + } else if (status == DatabaseCursor::Status::FAIL) { error = _("Error reading next record from wallet database"); + ret = false; break; } std::string key_str = HexStr(ss_key); @@ -85,7 +86,7 @@ bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error) } } - batch->CloseCursor(); + cursor.reset(); batch.reset(); // Close the wallet after we're done with it. The caller won't be doing this @@ -254,8 +255,8 @@ bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs:: std::vector<unsigned char> k = ParseHex(key); std::vector<unsigned char> v = ParseHex(value); - CDataStream ss_key(k, SER_DISK, CLIENT_VERSION); - CDataStream ss_value(v, SER_DISK, CLIENT_VERSION); + DataStream ss_key{k}; + DataStream ss_value{v}; if (!batch->Write(ss_key, ss_value)) { error = strprintf(_("Error: Unable to write record to new wallet")); diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index 95e1ba4dd9..da63d45d11 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -226,7 +226,7 @@ RPCHelpMan addmultisigaddress() {"key", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "bitcoin address or hex-encoded public key"}, }, }, - {"label", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "A label to assign the addresses to."}, + {"label", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "A label to assign the addresses to."}, {"address_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -addresstype"}, "The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, }, RPCResult{ @@ -696,7 +696,7 @@ RPCHelpMan listlabels() return RPCHelpMan{"listlabels", "\nReturns the list of all labels, or labels that are assigned to addresses with a specific purpose.\n", { - {"purpose", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Address purpose to list labels for ('send','receive'). An empty string is the same as not providing this argument."}, + {"purpose", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Address purpose to list labels for ('send','receive'). An empty string is the same as not providing this argument."}, }, RPCResult{ RPCResult::Type::ARR, "", "", diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 64253789ec..93a6bbde20 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -6,6 +6,7 @@ #include <clientversion.h> #include <core_io.h> #include <fs.h> +#include <hash.h> #include <interfaces/chain.h> #include <key_io.h> #include <merkleblock.h> @@ -14,6 +15,7 @@ #include <script/script.h> #include <script/standard.h> #include <sync.h> +#include <uint256.h> #include <util/bip32.h> #include <util/system.h> #include <util/time.h> @@ -334,7 +336,7 @@ RPCHelpMan importprunedfunds() } uint256 hashTx = tx.GetHash(); - CDataStream ssMB(ParseHexV(request.params[1], "proof"), SER_NETWORK, PROTOCOL_VERSION); + DataStream ssMB{ParseHexV(request.params[1], "proof")}; CMerkleBlock merkleBlock; ssMB >> merkleBlock; @@ -886,9 +888,7 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d } case TxoutType::WITNESS_V0_SCRIPTHASH: { if (script_ctx == ScriptContext::WITNESS_V0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2WSH inside another P2WSH"); - uint256 fullid(solverdata[0]); - CScriptID id; - CRIPEMD160().Write(fullid.begin(), fullid.size()).Finalize(id.begin()); + CScriptID id{RIPEMD160(solverdata[0])}; auto subscript = std::move(import_data.witnessscript); // Remove redeemscript from import_data to check for superfluous script later. if (!subscript) return "missing witnessscript"; if (CScriptID(*subscript) != id) return "witnessScript does not match the scriptPubKey or redeemScript"; @@ -1292,7 +1292,7 @@ RPCHelpMan importmulti() }, }, RPCArgOptions{.oneline_description="\"requests\""}}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions after all imports."}, }, @@ -1330,8 +1330,6 @@ RPCHelpMan importmulti() // the user could have gotten from another RPC command prior to now wallet.BlockUntilSyncedToCurrentChain(); - RPCTypeCheck(mainRequest.params, {UniValue::VARR, UniValue::VOBJ}); - EnsureLegacyScriptPubKeyMan(*pwallet, true); const UniValue& requests = mainRequest.params[0]; @@ -1652,8 +1650,6 @@ RPCHelpMan importdescriptors() throw JSONRPCError(RPC_WALLET_ERROR, "importdescriptors is not available for non-descriptor wallets"); } - RPCTypeCheck(main_request.params, {UniValue::VARR, UniValue::VOBJ}); - WalletRescanReserver reserver(*pwallet); if (!reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); @@ -1895,7 +1891,7 @@ RPCHelpMan restorewallet() { {"wallet_name", RPCArg::Type::STR, RPCArg::Optional::NO, "The name that will be applied to the restored wallet"}, {"backup_file", RPCArg::Type::STR, RPCArg::Optional::NO, "The backup file that will be used to restore the wallet."}, - {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, + {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, }, RPCResult{ RPCResult::Type::OBJ, "", "", diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index 82642194c2..4c386789f1 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <core_io.h> +#include <hash.h> #include <key_io.h> #include <rpc/util.h> #include <util/moneystr.h> @@ -165,7 +166,7 @@ RPCHelpMan getbalance() "The available balance is what the wallet considers currently spendable, and is\n" "thus affected by options which limit spendability such as -spendzeroconfchange.\n", { - {"dummy", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Remains for backward compatibility. Must be excluded or set to \"*\"."}, + {"dummy", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Remains for backward compatibility. Must be excluded or set to \"*\"."}, {"minconf", RPCArg::Type::NUM, RPCArg::Default{0}, "Only include transactions confirmed at least this many times."}, {"include_watchonly", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also include balance in watch-only addresses (see 'importaddress')"}, {"avoid_reuse", RPCArg::Type::BOOL, RPCArg::Default{true}, "(only available if avoid_reuse wallet flag is set) Do not include balance in dirty outputs; addresses are considered dirty if they have previously been used in a transaction."}, @@ -509,7 +510,7 @@ RPCHelpMan listunspent() }, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{true}, "Include outputs that are not safe to spend\n" "See description of \"safe\" attribute below."}, - {"query_options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "JSON with query options", + {"query_options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "JSON with query options", { {"minimumAmount", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(0)}, "Minimum value of each UTXO in " + CURRENCY_UNIT + ""}, {"maximumAmount", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"unlimited"}, "Maximum value of each UTXO in " + CURRENCY_UNIT + ""}, @@ -679,8 +680,7 @@ RPCHelpMan listunspent() CHECK_NONFATAL(extracted); // Also return the witness script const WitnessV0ScriptHash& whash = std::get<WitnessV0ScriptHash>(witness_destination); - CScriptID id; - CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); + CScriptID id{RIPEMD160(whash)}; CScript witnessScript; if (provider->GetCScript(id, witnessScript)) { entry.pushKV("witnessScript", HexStr(witnessScript)); @@ -689,8 +689,7 @@ RPCHelpMan listunspent() } } else if (scriptPubKey.IsPayToWitnessScriptHash()) { const WitnessV0ScriptHash& whash = std::get<WitnessV0ScriptHash>(address); - CScriptID id; - CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); + CScriptID id{RIPEMD160(whash)}; CScript witnessScript; if (provider->GetCScript(id, witnessScript)) { entry.pushKV("witnessScript", HexStr(witnessScript)); diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index be1ad37592..cab797bbce 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -217,9 +217,9 @@ RPCHelpMan sendtoaddress() { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to send to."}, {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The amount in " + CURRENCY_UNIT + " to send. eg 0.1"}, - {"comment", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "A comment used to store what the transaction is for.\n" + {"comment", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "A comment used to store what the transaction is for.\n" "This is not part of the transaction, just kept in your wallet."}, - {"comment_to", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "A comment to store the name of the person or organization\n" + {"comment_to", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "A comment to store the name of the person or organization\n" "to which you're sending the transaction. This is not part of the \n" "transaction, just kept in your wallet."}, {"subtractfeefromamount", RPCArg::Type::BOOL, RPCArg::Default{false}, "The fee will be deducted from the amount being sent.\n" @@ -314,18 +314,21 @@ RPCHelpMan sendtoaddress() RPCHelpMan sendmany() { return RPCHelpMan{"sendmany", - "\nSend multiple times. Amounts are double-precision floating point numbers." + + "Send multiple times. Amounts are double-precision floating point numbers." + HELP_REQUIRING_PASSPHRASE, { - {"dummy", RPCArg::Type::STR, RPCArg::Optional::NO, "Must be set to \"\" for backwards compatibility.", RPCArgOptions{.oneline_description="\"\""}}, + {"dummy", RPCArg::Type::STR, RPCArg::Default{"\"\""}, "Must be set to \"\" for backwards compatibility.", + RPCArgOptions{ + .oneline_description = "\"\"", + }}, {"amounts", RPCArg::Type::OBJ_USER_KEYS, RPCArg::Optional::NO, "The addresses and amounts", { {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The bitcoin address is the key, the numeric amount (can be string) in " + CURRENCY_UNIT + " is the value"}, }, }, - {"minconf", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "Ignored dummy value"}, - {"comment", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "A comment"}, - {"subtractfeefrom", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "The addresses.\n" + {"minconf", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "Ignored dummy value"}, + {"comment", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "A comment"}, + {"subtractfeefrom", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "The addresses.\n" "The fee will be equally deducted from the amount of each selected address.\n" "Those recipients will receive less bitcoins than you enter in their corresponding amount field.\n" "If no addresses are specified here, the sender pays the fee.", @@ -459,7 +462,7 @@ static std::vector<RPCArg> FundTxDoc(bool solving_data = true) }, }; if (solving_data) { - args.push_back({"solving_data", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "Keys and scripts needed for producing a final transaction with a dummy signature.\n" + args.push_back({"solving_data", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "Keys and scripts needed for producing a final transaction with a dummy signature.\n" "Used for fee estimation during coin selection.", { { @@ -528,6 +531,8 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, {"replaceable", UniValueType(UniValue::VBOOL)}, {"conf_target", UniValueType(UniValue::VNUM)}, {"estimate_mode", UniValueType(UniValue::VSTR)}, + {"minconf", UniValueType(UniValue::VNUM)}, + {"maxconf", UniValueType(UniValue::VNUM)}, {"input_weights", UniValueType(UniValue::VARR)}, }, true, true); @@ -593,6 +598,22 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, if (options.exists("replaceable")) { coinControl.m_signal_bip125_rbf = options["replaceable"].get_bool(); } + + if (options.exists("minconf")) { + coinControl.m_min_depth = options["minconf"].getInt<int>(); + + if (coinControl.m_min_depth < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative minconf"); + } + } + + if (options.exists("maxconf")) { + coinControl.m_max_depth = options["maxconf"].getInt<int>(); + + if (coinControl.m_max_depth < coinControl.m_min_depth) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("maxconf can't be lower than minconf: %d < %d", coinControl.m_max_depth, coinControl.m_min_depth)); + } + } SetFeeEstimateMode(wallet, coinControl, options["conf_target"], options["estimate_mode"], options["fee_rate"], override_min_fee); } } else { @@ -737,13 +758,15 @@ RPCHelpMan fundrawtransaction() "Only pay-to-pubkey, multisig, and P2SH versions thereof are currently supported for watch-only\n", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", + {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::Default{true}, "For a transaction with existing inputs, automatically include more if they are not enough."}, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" "Warning: the resulting transaction may become invalid if one of the unsafe inputs disappears.\n" "If that happens, you will need to fund the transaction with different inputs and republish it."}, + {"minconf", RPCArg::Type::NUM, RPCArg::Default{0}, "If add_inputs is specified, require inputs with at least this many confirmations."}, + {"maxconf", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "If add_inputs is specified, require inputs with at most this many confirmations."}, {"changeAddress", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"changePosition", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", \"bech32\", and \"bech32m\"."}, @@ -761,7 +784,7 @@ RPCHelpMan fundrawtransaction() {"vout_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The zero-based output index, before a change output is added."}, }, }, - {"input_weights", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "Inputs and their corresponding weights", + {"input_weights", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "Inputs and their corresponding weights", { {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { @@ -778,7 +801,10 @@ RPCHelpMan fundrawtransaction() }, }, FundTxDoc()), - RPCArgOptions{.oneline_description="options"}}, + RPCArgOptions{ + .skip_type_check = true, + .oneline_description = "options", + }}, {"iswitness", RPCArg::Type::BOOL, RPCArg::DefaultHint{"depends on heuristic tests"}, "Whether the transaction hex is a serialized witness transaction.\n" "If iswitness is not present, heuristic tests will be used in decoding.\n" "If true, only witness deserialization will be tried.\n" @@ -810,8 +836,6 @@ RPCHelpMan fundrawtransaction() std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return UniValue::VNULL; - RPCTypeCheck(request.params, {UniValue::VSTR, UniValueType(), UniValue::VBOOL}); - // parse hex string from parameter CMutableTransaction tx; bool try_witness = request.params[2].isNull() ? true : request.params[2].get_bool(); @@ -846,7 +870,7 @@ RPCHelpMan signrawtransactionwithwallet() HELP_REQUIRING_PASSPHRASE, { {"hexstring", RPCArg::Type::STR, RPCArg::Optional::NO, "The transaction hex string"}, - {"prevtxs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "The previous dependent transaction outputs", + {"prevtxs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "The previous dependent transaction outputs", { {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { @@ -900,8 +924,6 @@ RPCHelpMan signrawtransactionwithwallet() const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return UniValue::VNULL; - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VARR, UniValue::VSTR}, true); - CMutableTransaction mtx; if (!DecodeHexTx(mtx, request.params[0].get_str())) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input."); @@ -955,7 +977,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) "* WARNING: before version 0.21, fee_rate was in " + CURRENCY_UNIT + "/kvB. As of 0.21, fee_rate is in " + CURRENCY_ATOM + "/vB. *\n", { {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The txid to be bumped"}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks\n"}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, @@ -1001,7 +1023,6 @@ static RPCHelpMan bumpfee_helper(std::string method_name) throw JSONRPCError(RPC_WALLET_ERROR, "bumpfee is not available with wallets that have private keys disabled. Use psbtbumpfee instead."); } - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VOBJ}); uint256 hash(ParseHashV(request.params[0], "txid")); CCoinControl coin_control; @@ -1135,18 +1156,20 @@ RPCHelpMan send() }, }, }, - }, + RPCArgOptions{.skip_type_check = true}}, {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, "The fee estimate mode, must be one of (case insensitive):\n" "\"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::DefaultHint{"false when \"inputs\" are specified, true otherwise"},"Automatically include coins from the wallet to cover the target amount.\n"}, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" "Warning: the resulting transaction may become invalid if one of the unsafe inputs disappears.\n" "If that happens, you will need to fund the transaction with different inputs and republish it."}, + {"minconf", RPCArg::Type::NUM, RPCArg::Default{0}, "If add_inputs is specified, require inputs with at least this many confirmations."}, + {"maxconf", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "If add_inputs is specified, require inputs with at most this many confirmations."}, {"add_to_wallet", RPCArg::Type::BOOL, RPCArg::Default{true}, "When false, returns a serialized transaction which will not be added to the wallet or broadcast"}, {"change_address", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"change_position", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, @@ -1205,15 +1228,6 @@ RPCHelpMan send() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, { - UniValueType(), // outputs (ARR or OBJ, checked later) - UniValue::VNUM, // conf_target - UniValue::VSTR, // estimate_mode - UniValueType(), // fee_rate, will be checked by AmountFromValue() in SetFeeEstimateMode() - UniValue::VOBJ, // options - }, true - ); - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return UniValue::VNULL; @@ -1262,7 +1276,7 @@ RPCHelpMan sendall() "\"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, { - "options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + "options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", Cat<std::vector<RPCArg>>( { {"add_to_wallet", RPCArg::Type::BOOL, RPCArg::Default{true}, "When false, returns the serialized transaction without broadcasting or adding it to the wallet"}, @@ -1270,7 +1284,7 @@ RPCHelpMan sendall() {"include_watching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch-only.\n" "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" "e.g. with 'importpubkey' or 'importmulti' with the 'pubkeys' or 'desc' field."}, - {"inputs", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Use exactly the specified inputs to build the transaction. Specifying inputs is incompatible with send_max.", + {"inputs", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Use exactly the specified inputs to build the transaction. Specifying inputs is incompatible with the send_max, minconf, and maxconf options.", { {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { @@ -1285,6 +1299,8 @@ RPCHelpMan sendall() {"lock_unspents", RPCArg::Type::BOOL, RPCArg::Default{false}, "Lock selected unspent outputs"}, {"psbt", RPCArg::Type::BOOL, RPCArg::DefaultHint{"automatic"}, "Always return a PSBT, implies add_to_wallet=false."}, {"send_max", RPCArg::Type::BOOL, RPCArg::Default{false}, "When true, only use UTXOs that can pay for their own fees to maximize the output amount. When 'false' (default), no UTXO is left behind. send_max is incompatible with providing specific inputs."}, + {"minconf", RPCArg::Type::NUM, RPCArg::Default{0}, "Require inputs with at least this many confirmations."}, + {"maxconf", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "Require inputs with at most this many confirmations."}, }, FundTxDoc() ), @@ -1314,15 +1330,6 @@ RPCHelpMan sendall() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - RPCTypeCheck(request.params, { - UniValue::VARR, // recipients - UniValue::VNUM, // conf_target - UniValue::VSTR, // estimate_mode - UniValueType(), // fee_rate, will be checked by AmountFromValue() in SetFeeEstimateMode() - UniValue::VOBJ, // options - }, true - ); - std::shared_ptr<CWallet> const pwallet{GetWalletForJSONRPCRequest(request)}; if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block @@ -1359,6 +1366,23 @@ RPCHelpMan sendall() coin_control.fAllowWatchOnly = ParseIncludeWatchonly(options["include_watching"], *pwallet); + if (options.exists("minconf")) { + if (options["minconf"].getInt<int>() < 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid minconf (minconf cannot be negative): %s", options["minconf"].getInt<int>())); + } + + coin_control.m_min_depth = options["minconf"].getInt<int>(); + } + + if (options.exists("maxconf")) { + coin_control.m_max_depth = options["maxconf"].getInt<int>(); + + if (coin_control.m_max_depth < coin_control.m_min_depth) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("maxconf can't be lower than minconf: %d < %d", coin_control.m_max_depth, coin_control.m_min_depth)); + } + } + const bool rbf{options.exists("replaceable") ? options["replaceable"].get_bool() : pwallet->m_signal_rbf}; FeeCalculation fee_calc_out; @@ -1380,6 +1404,8 @@ RPCHelpMan sendall() bool send_max{options.exists("send_max") ? options["send_max"].get_bool() : false}; if (options.exists("inputs") && options.exists("send_max")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot combine send_max with specific inputs."); + } else if (options.exists("inputs") && (options.exists("minconf") || options.exists("maxconf"))) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot combine minconf or maxconf with specific inputs."); } else if (options.exists("inputs")) { for (const CTxIn& input : rawTx.vin) { if (pwallet->IsSpent(input.prevout)) { @@ -1518,8 +1544,6 @@ RPCHelpMan walletprocesspsbt() // the user could have gotten from another RPC command prior to now wallet.BlockUntilSyncedToCurrentChain(); - RPCTypeCheck(request.params, {UniValue::VSTR}); - // Unserialize the transaction PartiallySignedTransaction psbtx; std::string error; @@ -1562,7 +1586,7 @@ RPCHelpMan walletcreatefundedpsbt() "All existing inputs must either have their previous output transaction be in the wallet\n" "or be in the UTXO set. Solving data must be provided for non-wallet inputs.\n", { - {"inputs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "Leave empty to add inputs automatically. See add_inputs option.", + {"inputs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "Leave empty to add inputs automatically. See add_inputs option.", { {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { @@ -1594,15 +1618,17 @@ RPCHelpMan walletcreatefundedpsbt() }, }, }, - }, + RPCArgOptions{.skip_type_check = true}}, {"locktime", RPCArg::Type::NUM, RPCArg::Default{0}, "Raw locktime. Non-0 value also locktime-activates inputs"}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::DefaultHint{"false when \"inputs\" are specified, true otherwise"}, "Automatically include coins from the wallet to cover the target amount.\n"}, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" "Warning: the resulting transaction may become invalid if one of the unsafe inputs disappears.\n" "If that happens, you will need to fund the transaction with different inputs and republish it."}, + {"minconf", RPCArg::Type::NUM, RPCArg::Default{0}, "If add_inputs is specified, require inputs with at least this many confirmations."}, + {"maxconf", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "If add_inputs is specified, require inputs with at most this many confirmations."}, {"changeAddress", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"changePosition", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", \"bech32\", and \"bech32m\"."}, @@ -1645,15 +1671,6 @@ RPCHelpMan walletcreatefundedpsbt() // the user could have gotten from another RPC command prior to now wallet.BlockUntilSyncedToCurrentChain(); - RPCTypeCheck(request.params, { - UniValue::VARR, - UniValueType(), // ARR or OBJ, checked later - UniValue::VNUM, - UniValue::VOBJ, - UniValue::VBOOL - }, true - ); - UniValue options{request.params[3].isNull() ? UniValue::VOBJ : request.params[3]}; CAmount fee; diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index f571f8bcb2..e590aa1f08 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -206,7 +206,7 @@ RPCHelpMan listreceivedbyaddress() {"minconf", RPCArg::Type::NUM, RPCArg::Default{1}, "The minimum number of confirmations before payments are included."}, {"include_empty", RPCArg::Type::BOOL, RPCArg::Default{false}, "Whether to include addresses that haven't received any payments."}, {"include_watchonly", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Whether to include watch-only addresses (see 'importaddress')"}, - {"address_filter", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "If present and non-empty, only return information on this address."}, + {"address_filter", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "If present and non-empty, only return information on this address."}, {"include_immature_coinbase", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include immature coinbase transactions."}, }, RPCResult{ @@ -397,7 +397,7 @@ static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nM } -static const std::vector<RPCResult> TransactionDescriptionString() +static std::vector<RPCResult> TransactionDescriptionString() { return{{RPCResult::Type::NUM, "confirmations", "The number of confirmations for the transaction. Negative confirmations means the\n" "transaction conflicted that many blocks ago."}, @@ -434,7 +434,7 @@ RPCHelpMan listtransactions() "\nIf a label name is provided, this will return only incoming transactions paying to addresses with the specified label.\n" "\nReturns up to 'count' most recent transactions skipping the first 'from' transactions.\n", { - {"label|dummy", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "If set, should be a valid label name to return only incoming transactions\n" + {"label|dummy", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "If set, should be a valid label name to return only incoming transactions\n" "with the specified label, or \"*\" to disable filtering and return all transactions."}, {"count", RPCArg::Type::NUM, RPCArg::Default{10}, "The number of transactions to return"}, {"skip", RPCArg::Type::NUM, RPCArg::Default{0}, "The number of transactions to skip"}, @@ -545,13 +545,13 @@ RPCHelpMan listsinceblock() "If \"blockhash\" is no longer a part of the main chain, transactions from the fork point onward are included.\n" "Additionally, if include_removed is set, transactions affecting the wallet which were removed are returned in the \"removed\" array.\n", { - {"blockhash", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "If set, the block hash to list transactions since, otherwise list all transactions."}, + {"blockhash", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "If set, the block hash to list transactions since, otherwise list all transactions."}, {"target_confirmations", RPCArg::Type::NUM, RPCArg::Default{1}, "Return the nth block hash from the main chain. e.g. 1 would mean the best block hash. Note: this is not used as a filter, but only affects [lastblock] in the return value"}, {"include_watchonly", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Include transactions to watch-only addresses (see 'importaddress')"}, {"include_removed", RPCArg::Type::BOOL, RPCArg::Default{true}, "Show transactions that were removed due to a reorg in the \"removed\" array\n" "(not guaranteed to work on pruned nodes)"}, {"include_change", RPCArg::Type::BOOL, RPCArg::Default{false}, "Also add entries for change outputs.\n"}, - {"label", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Return only incoming transactions paying to addresses with the specified label.\n"}, + {"label", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Return only incoming transactions paying to addresses with the specified label.\n"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -848,7 +848,7 @@ RPCHelpMan rescanblockchain() "and block filters are available (using startup option \"-blockfilterindex=1\").\n", { {"start_height", RPCArg::Type::NUM, RPCArg::Default{0}, "block height where the rescan should start"}, - {"stop_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "the last block height that should be scanned. If none is provided it will rescan up to the tip at return time of this call."}, + {"stop_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "the last block height that should be scanned. If none is provided it will rescan up to the tip at return time of this call."}, }, RPCResult{ RPCResult::Type::OBJ, "", "", diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index aeab9b7634..23a88cd51b 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -201,7 +201,7 @@ static RPCHelpMan loadwallet() "\napplied to the new wallet.\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet directory or .dat file."}, - {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, + {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -323,12 +323,12 @@ static RPCHelpMan createwallet() {"wallet_name", RPCArg::Type::STR, RPCArg::Optional::NO, "The name for the new wallet. If this is a path, the wallet will be created at the path location."}, {"disable_private_keys", RPCArg::Type::BOOL, RPCArg::Default{false}, "Disable the possibility of private keys (only watchonlys are possible in this mode)."}, {"blank", RPCArg::Type::BOOL, RPCArg::Default{false}, "Create a blank wallet. A blank wallet has no keys or HD seed. One can be set using sethdseed."}, - {"passphrase", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Encrypt the wallet with this passphrase."}, + {"passphrase", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Encrypt the wallet with this passphrase."}, {"avoid_reuse", RPCArg::Type::BOOL, RPCArg::Default{false}, "Keep track of coin reuse, and treat dirty and clean coins differently with privacy considerations in mind."}, {"descriptors", RPCArg::Type::BOOL, RPCArg::Default{true}, "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation." " Setting to \"false\" will create a legacy wallet; however, the legacy wallet type is being deprecated and" " support for creating and opening legacy wallets will be removed in the future."}, - {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, + {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, {"external_signer", RPCArg::Type::BOOL, RPCArg::Default{false}, "Use an external signer such as a hardware wallet. Requires -signer to be configured. Wallet creation will fail if keys cannot be fetched. Requires disable_private_keys and descriptors set to true."}, }, RPCResult{ @@ -419,7 +419,7 @@ static RPCHelpMan unloadwallet() "Specifying the wallet name on a wallet endpoint is invalid.", { {"wallet_name", RPCArg::Type::STR, RPCArg::DefaultHint{"the wallet name from the RPC endpoint"}, "The name of the wallet to unload. If provided both here and in the RPC endpoint, the two must be identical."}, - {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, + {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, }, RPCResult{RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR, "warning", "Warning message if wallet was not unloaded cleanly."}, @@ -568,8 +568,6 @@ static RPCHelpMan upgradewallet() std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return UniValue::VNULL; - RPCTypeCheck(request.params, {UniValue::VNUM}, true); - EnsureWalletIsUnlocked(*pwallet); int version = 0; @@ -610,12 +608,12 @@ RPCHelpMan simulaterawtransaction() return RPCHelpMan{"simulaterawtransaction", "\nCalculate the balance change resulting in the signing and broadcasting of the given transaction(s).\n", { - {"rawtxs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "An array of hex strings of raw transactions.\n", + {"rawtxs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "An array of hex strings of raw transactions.\n", { {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, }, }, - {"options", RPCArg::Type::OBJ_USER_KEYS, RPCArg::Optional::OMITTED_NAMED_ARG, "Options", + {"options", RPCArg::Type::OBJ_USER_KEYS, RPCArg::Optional::OMITTED, "Options", { {"include_watchonly", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Whether to include watch-only addresses (see RPC importaddress)"}, }, @@ -637,8 +635,6 @@ RPCHelpMan simulaterawtransaction() if (!rpc_wallet) return UniValue::VNULL; const CWallet& wallet = *rpc_wallet; - RPCTypeCheck(request.params, {UniValue::VARR, UniValue::VOBJ}, true); - LOCK(wallet.cs_wallet); UniValue include_watchonly(UniValue::VNULL); diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index 9ba3c7fd2c..84f33e50b3 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -139,7 +139,7 @@ bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bil for (KeyValPair& row : salvagedData) { /* Filter for only private key type KV pairs to be added to the salvaged wallet */ - CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); + DataStream ssKey{row.first}; CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); std::string strType, strErr; bool fReadOK; diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index d8f34dd2b0..59cf87355b 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <hash.h> #include <key_io.h> #include <logging.h> #include <outputtype.h> @@ -166,9 +167,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s if (sigversion == IsMineSigVersion::TOP && !keystore.HaveCScript(CScriptID(CScript() << OP_0 << vSolutions[0]))) { break; } - uint160 hash; - CRIPEMD160().Write(vSolutions[0].data(), vSolutions[0].size()).Finalize(hash.begin()); - CScriptID scriptID = CScriptID(hash); + CScriptID scriptID{RIPEMD160(vSolutions[0])}; CScript subscript; if (keystore.GetCScript(scriptID, subscript)) { ret = std::max(ret, recurse_scripthash ? IsMineInner(keystore, subscript, IsMineSigVersion::WITNESS_V0) : IsMineResult::SPENDABLE); @@ -1665,7 +1664,7 @@ std::set<CKeyID> LegacyScriptPubKeyMan::GetKeys() const return set_address; } -const std::unordered_set<CScript, SaltedSipHasher> LegacyScriptPubKeyMan::GetScriptPubKeys() const +std::unordered_set<CScript, SaltedSipHasher> LegacyScriptPubKeyMan::GetScriptPubKeys() const { LOCK(cs_KeyStore); std::unordered_set<CScript, SaltedSipHasher> spks; @@ -2651,17 +2650,17 @@ void DescriptorScriptPubKeyMan::WriteDescriptor() } } -const WalletDescriptor DescriptorScriptPubKeyMan::GetWalletDescriptor() const +WalletDescriptor DescriptorScriptPubKeyMan::GetWalletDescriptor() const { return m_wallet_descriptor; } -const std::unordered_set<CScript, SaltedSipHasher> DescriptorScriptPubKeyMan::GetScriptPubKeys() const +std::unordered_set<CScript, SaltedSipHasher> DescriptorScriptPubKeyMan::GetScriptPubKeys() const { return GetScriptPubKeys(0); } -const std::unordered_set<CScript, SaltedSipHasher> DescriptorScriptPubKeyMan::GetScriptPubKeys(int32_t minimum_index) const +std::unordered_set<CScript, SaltedSipHasher> DescriptorScriptPubKeyMan::GetScriptPubKeys(int32_t minimum_index) const { LOCK(cs_desc_man); std::unordered_set<CScript, SaltedSipHasher> script_pub_keys; diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index d74388b3e8..4399ac2087 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -36,7 +36,7 @@ class WalletStorage { public: virtual ~WalletStorage() = default; - virtual const std::string GetDisplayName() const = 0; + virtual std::string GetDisplayName() const = 0; virtual WalletDatabase& GetDatabase() const = 0; virtual bool IsWalletFlagSet(uint64_t) const = 0; virtual void UnsetBlankWalletFlag(WalletBatch&) = 0; @@ -243,7 +243,7 @@ public: virtual uint256 GetID() const { return uint256(); } /** Returns a set of all the scriptPubKeys that this ScriptPubKeyMan watches */ - virtual const std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys() const { return {}; }; + virtual std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys() const { return {}; }; /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ template<typename... Params> @@ -512,7 +512,7 @@ public: const std::map<CKeyID, int64_t>& GetAllReserveKeys() const { return m_pool_key_to_index; } std::set<CKeyID> GetKeys() const override; - const std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys() const override; + std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys() const override; /** Get the DescriptorScriptPubKeyMans (with private keys) that have the same scriptPubKeys as this LegacyScriptPubKeyMan. * Does not modify this ScriptPubKeyMan. */ @@ -641,9 +641,9 @@ public: void AddDescriptorKey(const CKey& key, const CPubKey &pubkey); void WriteDescriptor(); - const WalletDescriptor GetWalletDescriptor() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); - const std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys() const override; - const std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys(int32_t minimum_index) const; + WalletDescriptor GetWalletDescriptor() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); + std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys() const override; + std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys(int32_t minimum_index) const; int32_t GetEndRange() const; bool GetDescriptorString(std::string& out, const bool priv) const; diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 8bb970936b..e66ff8c97c 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -362,67 +362,45 @@ CAmount GetAvailableBalance(const CWallet& wallet, const CCoinControl* coinContr return AvailableCoins(wallet, coinControl).GetTotalAmount(); } -const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const CTransaction& tx, int output) +const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const COutPoint& outpoint) { AssertLockHeld(wallet.cs_wallet); - const CTransaction* ptx = &tx; - int n = output; + const CWalletTx* wtx{Assert(wallet.GetWalletTx(outpoint.hash))}; + + const CTransaction* ptx = wtx->tx.get(); + int n = outpoint.n; while (OutputIsChange(wallet, ptx->vout[n]) && ptx->vin.size() > 0) { const COutPoint& prevout = ptx->vin[0].prevout; - auto it = wallet.mapWallet.find(prevout.hash); - if (it == wallet.mapWallet.end() || it->second.tx->vout.size() <= prevout.n || - !wallet.IsMine(it->second.tx->vout[prevout.n])) { + const CWalletTx* it = wallet.GetWalletTx(prevout.hash); + if (!it || it->tx->vout.size() <= prevout.n || + !wallet.IsMine(it->tx->vout[prevout.n])) { break; } - ptx = it->second.tx.get(); + ptx = it->tx.get(); n = prevout.n; } return ptx->vout[n]; } -const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const COutPoint& outpoint) -{ - AssertLockHeld(wallet.cs_wallet); - return FindNonChangeParentOutput(wallet, *wallet.GetWalletTx(outpoint.hash)->tx, outpoint.n); -} - std::map<CTxDestination, std::vector<COutput>> ListCoins(const CWallet& wallet) { AssertLockHeld(wallet.cs_wallet); std::map<CTxDestination, std::vector<COutput>> result; - for (COutput& coin : AvailableCoinsListUnspent(wallet).All()) { + CCoinControl coin_control; + // Include watch-only for LegacyScriptPubKeyMan wallets without private keys + coin_control.fAllowWatchOnly = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + CoinFilterParams coins_params; + coins_params.only_spendable = false; + coins_params.skip_locked = false; + for (const COutput& coin : AvailableCoins(wallet, &coin_control, /*feerate=*/std::nullopt, coins_params).All()) { CTxDestination address; if ((coin.spendable || (wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && coin.solvable)) && ExtractDestination(FindNonChangeParentOutput(wallet, coin.outpoint).scriptPubKey, address)) { - result[address].emplace_back(std::move(coin)); + result[address].emplace_back(coin); } } - - std::vector<COutPoint> lockedCoins; - wallet.ListLockedCoins(lockedCoins); - // Include watch-only for LegacyScriptPubKeyMan wallets without private keys - const bool include_watch_only = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); - const isminetype is_mine_filter = include_watch_only ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; - for (const COutPoint& output : lockedCoins) { - auto it = wallet.mapWallet.find(output.hash); - if (it != wallet.mapWallet.end()) { - const auto& wtx = it->second; - int depth = wallet.GetTxDepthInMainChain(wtx); - if (depth >= 0 && output.n < wtx.tx->vout.size() && - wallet.IsMine(wtx.tx->vout[output.n]) == is_mine_filter - ) { - CTxDestination address; - if (ExtractDestination(FindNonChangeParentOutput(wallet, *wtx.tx, output.n).scriptPubKey, address)) { - const auto out = wtx.tx->vout.at(output.n); - result[address].emplace_back( - COutPoint(wtx.GetHash(), output.n), out, depth, CalculateMaximumSignedInputSize(out, &wallet, /*coin_control=*/nullptr), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ false, wtx.GetTxTime(), CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)); - } - } - } - } - return result; } diff --git a/src/wallet/spend.h b/src/wallet/spend.h index 5ffdd11813..d8da556d29 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -99,7 +99,6 @@ CAmount GetAvailableBalance(const CWallet& wallet, const CCoinControl* coinContr /** * Find non-change parent output. */ -const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const CTransaction& tx, int output) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const COutPoint& outpoint) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); /** diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index f2b9909851..4af49db609 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -125,7 +125,6 @@ void SQLiteBatch::SetupSQLStatements() {&m_insert_stmt, "INSERT INTO main VALUES(?, ?)"}, {&m_overwrite_stmt, "INSERT or REPLACE into main values(?, ?)"}, {&m_delete_stmt, "DELETE FROM main WHERE key = ?"}, - {&m_cursor_stmt, "SELECT key, value FROM main"}, }; for (const auto& [stmt_prepared, stmt_text] : statements) { @@ -374,7 +373,6 @@ void SQLiteBatch::Close() {&m_insert_stmt, "insert"}, {&m_overwrite_stmt, "overwrite"}, {&m_delete_stmt, "delete"}, - {&m_cursor_stmt, "cursor"}, }; for (const auto& [stmt_prepared, stmt_description] : statements) { @@ -387,7 +385,7 @@ void SQLiteBatch::Close() } } -bool SQLiteBatch::ReadKey(CDataStream&& key, CDataStream& value) +bool SQLiteBatch::ReadKey(DataStream&& key, DataStream& value) { if (!m_database.m_db) return false; assert(m_read_stmt); @@ -414,7 +412,7 @@ bool SQLiteBatch::ReadKey(CDataStream&& key, CDataStream& value) return true; } -bool SQLiteBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite) +bool SQLiteBatch::WriteKey(DataStream&& key, DataStream&& value, bool overwrite) { if (!m_database.m_db) return false; assert(m_insert_stmt && m_overwrite_stmt); @@ -441,7 +439,7 @@ bool SQLiteBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwrit return res == SQLITE_DONE; } -bool SQLiteBatch::EraseKey(CDataStream&& key) +bool SQLiteBatch::EraseKey(DataStream&& key) { if (!m_database.m_db) return false; assert(m_delete_stmt); @@ -459,7 +457,7 @@ bool SQLiteBatch::EraseKey(CDataStream&& key) return res == SQLITE_DONE; } -bool SQLiteBatch::HasKey(CDataStream&& key) +bool SQLiteBatch::HasKey(DataStream&& key) { if (!m_database.m_db) return false; assert(m_read_stmt); @@ -472,28 +470,15 @@ bool SQLiteBatch::HasKey(CDataStream&& key) return res == SQLITE_ROW; } -bool SQLiteBatch::StartCursor() +DatabaseCursor::Status SQLiteCursor::Next(DataStream& key, DataStream& value) { - assert(!m_cursor_init); - if (!m_database.m_db) return false; - m_cursor_init = true; - return true; -} - -bool SQLiteBatch::ReadAtCursor(CDataStream& key, CDataStream& value, bool& complete) -{ - complete = false; - - if (!m_cursor_init) return false; - int res = sqlite3_step(m_cursor_stmt); if (res == SQLITE_DONE) { - complete = true; - return true; + return Status::DONE; } if (res != SQLITE_ROW) { - LogPrintf("SQLiteBatch::ReadAtCursor: Unable to execute cursor step: %s\n", sqlite3_errstr(res)); - return false; + LogPrintf("%s: Unable to execute cursor step: %s\n", __func__, sqlite3_errstr(res)); + return Status::FAIL; } // Leftmost column in result is index 0 @@ -503,13 +488,32 @@ bool SQLiteBatch::ReadAtCursor(CDataStream& key, CDataStream& value, bool& compl const std::byte* value_data{AsBytePtr(sqlite3_column_blob(m_cursor_stmt, 1))}; size_t value_data_size(sqlite3_column_bytes(m_cursor_stmt, 1)); value.write({value_data, value_data_size}); - return true; + return Status::MORE; } -void SQLiteBatch::CloseCursor() +SQLiteCursor::~SQLiteCursor() { sqlite3_reset(m_cursor_stmt); - m_cursor_init = false; + int res = sqlite3_finalize(m_cursor_stmt); + if (res != SQLITE_OK) { + LogPrintf("%s: cursor closed but could not finalize cursor statement: %s\n", + __func__, sqlite3_errstr(res)); + } +} + +std::unique_ptr<DatabaseCursor> SQLiteBatch::GetNewCursor() +{ + if (!m_database.m_db) return nullptr; + auto cursor = std::make_unique<SQLiteCursor>(); + + const char* stmt_text = "SELECT key, value FROM main"; + int res = sqlite3_prepare_v2(m_database.m_db, stmt_text, -1, &cursor->m_cursor_stmt, nullptr); + if (res != SQLITE_OK) { + throw std::runtime_error(strprintf( + "%s: Failed to setup cursor SQL statement: %s\n", __func__, sqlite3_errstr(res))); + } + + return cursor; } bool SQLiteBatch::TxnBegin() diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index 7680bdd07b..c6745d7a7e 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -14,26 +14,34 @@ struct bilingual_str; namespace wallet { class SQLiteDatabase; +class SQLiteCursor : public DatabaseCursor +{ +public: + sqlite3_stmt* m_cursor_stmt{nullptr}; + + explicit SQLiteCursor() {} + ~SQLiteCursor() override; + + Status Next(DataStream& key, DataStream& value) override; +}; + /** RAII class that provides access to a WalletDatabase */ class SQLiteBatch : public DatabaseBatch { private: SQLiteDatabase& m_database; - bool m_cursor_init = false; - sqlite3_stmt* m_read_stmt{nullptr}; sqlite3_stmt* m_insert_stmt{nullptr}; sqlite3_stmt* m_overwrite_stmt{nullptr}; sqlite3_stmt* m_delete_stmt{nullptr}; - sqlite3_stmt* m_cursor_stmt{nullptr}; void SetupSQLStatements(); - bool ReadKey(CDataStream&& key, CDataStream& value) override; - bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite = true) override; - bool EraseKey(CDataStream&& key) override; - bool HasKey(CDataStream&& key) override; + bool ReadKey(DataStream&& key, DataStream& value) override; + bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) override; + bool EraseKey(DataStream&& key) override; + bool HasKey(DataStream&& key) override; public: explicit SQLiteBatch(SQLiteDatabase& database); @@ -44,9 +52,7 @@ public: void Close() override; - bool StartCursor() override; - bool ReadAtCursor(CDataStream& key, CDataStream& value, bool& complete) override; - void CloseCursor() override; + std::unique_ptr<DatabaseCursor> GetNewCursor() override; bool TxnBegin() override; bool TxnCommit() override; bool TxnAbort() override; diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index 88597bd320..225871fd91 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -50,18 +50,18 @@ std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database, // Get a cursor to the original database auto batch = database.MakeBatch(); - batch->StartCursor(); + std::unique_ptr<wallet::DatabaseCursor> cursor = batch->GetNewCursor(); // Get a batch for the new database auto new_batch = new_database->MakeBatch(); // Read all records from the original database and write them to the new one while (true) { - CDataStream key(SER_DISK, CLIENT_VERSION); - CDataStream value(SER_DISK, CLIENT_VERSION); - bool complete; - batch->ReadAtCursor(key, value, complete); - if (complete) break; + DataStream key{}; + DataStream value{}; + DatabaseCursor::Status status = cursor->Next(key, value); + assert(status != DatabaseCursor::Status::FAIL); + if (status == DatabaseCursor::Status::DONE) break; new_batch->Write(key, value); } diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index b6e50e961a..f056c54734 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -43,7 +43,7 @@ static_assert(WALLET_INCREMENTAL_RELAY_FEE >= DEFAULT_INCREMENTAL_RELAY_FEE, "wa BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) -static const std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) +static std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) { DatabaseOptions options; options.create_flags = WALLET_FLAG_DESCRIPTORS; @@ -907,24 +907,28 @@ BOOST_FIXTURE_TEST_CASE(ZapSelectTx, TestChain100Setup) TestUnloadWallet(std::move(wallet)); } +class FailCursor : public DatabaseCursor +{ +public: + Status Next(DataStream& key, DataStream& value) override { return Status::FAIL; } +}; + /** RAII class that provides access to a FailDatabase. Which fails if needed. */ class FailBatch : public DatabaseBatch { private: bool m_pass{true}; - bool ReadKey(CDataStream&& key, CDataStream& value) override { return m_pass; } - bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite=true) override { return m_pass; } - bool EraseKey(CDataStream&& key) override { return m_pass; } - bool HasKey(CDataStream&& key) override { return m_pass; } + bool ReadKey(DataStream&& key, DataStream& value) override { return m_pass; } + bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) override { return m_pass; } + bool EraseKey(DataStream&& key) override { return m_pass; } + bool HasKey(DataStream&& key) override { return m_pass; } public: explicit FailBatch(bool pass) : m_pass(pass) {} void Flush() override {} void Close() override {} - bool StartCursor() override { return true; } - bool ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) override { return false; } - void CloseCursor() override {} + std::unique_ptr<DatabaseCursor> GetNewCursor() override { return std::make_unique<FailCursor>(); } bool TxnBegin() override { return false; } bool TxnCommit() override { return false; } bool TxnAbort() override { return false; } diff --git a/src/wallet/test/walletload_tests.cpp b/src/wallet/test/walletload_tests.cpp index 24d21c2f22..f1feb28e7d 100644 --- a/src/wallet/test/walletload_tests.cpp +++ b/src/wallet/test/walletload_tests.cpp @@ -54,13 +54,15 @@ BOOST_FIXTURE_TEST_CASE(wallet_load_unknown_descriptor, TestingSetup) bool HasAnyRecordOfType(WalletDatabase& db, const std::string& key) { std::unique_ptr<DatabaseBatch> batch = db.MakeBatch(false); - BOOST_CHECK(batch->StartCursor()); + BOOST_CHECK(batch); + std::unique_ptr<DatabaseCursor> cursor = batch->GetNewCursor(); + BOOST_CHECK(cursor); while (true) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - bool complete; - BOOST_CHECK(batch->ReadAtCursor(ssKey, ssValue, complete)); - if (complete) break; + DataStream ssKey{}; + DataStream ssValue{}; + DatabaseCursor::Status status = cursor->Next(ssKey, ssValue); + assert(status != DatabaseCursor::Status::FAIL); + if (status == DatabaseCursor::Status::DONE) break; std::string type; ssKey >> type; if (type == key) return true; diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index 6ad222864a..290ef4eaa9 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -293,6 +293,7 @@ public: bool isAbandoned() const { return state<TxStateInactive>() && state<TxStateInactive>()->abandoned; } bool isConflicted() const { return state<TxStateConflicted>(); } + bool isInactive() const { return state<TxStateInactive>(); } bool isUnconfirmed() const { return !isAbandoned() && !isConflicted() && !isConfirmed(); } bool isConfirmed() const { return state<TxStateConfirmed>(); } const uint256& GetHash() const { return tx->GetHash(); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 0f761fb01a..5a92dbe428 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -647,8 +647,7 @@ bool CWallet::HasWalletSpend(const CTransactionRef& tx) const AssertLockHeld(cs_wallet); const uint256& txid = tx->GetHash(); for (unsigned int i = 0; i < tx->vout.size(); ++i) { - auto iter = mapTxSpends.find(COutPoint(txid, i)); - if (iter != mapTxSpends.end()) { + if (IsSpent(COutPoint(txid, i))) { return true; } } @@ -1066,6 +1065,33 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const TxState& state, const } } + // Mark inactive coinbase transactions and their descendants as abandoned + if (wtx.IsCoinBase() && wtx.isInactive()) { + std::vector<CWalletTx*> txs{&wtx}; + + TxStateInactive inactive_state = TxStateInactive{/*abandoned=*/true}; + + while (!txs.empty()) { + CWalletTx* desc_tx = txs.back(); + txs.pop_back(); + desc_tx->m_state = inactive_state; + // Break caches since we have changed the state + desc_tx->MarkDirty(); + batch.WriteTx(*desc_tx); + MarkInputsDirty(desc_tx->tx); + for (unsigned int i = 0; i < desc_tx->tx->vout.size(); ++i) { + COutPoint outpoint(desc_tx->GetHash(), i); + std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(outpoint); + for (TxSpends::const_iterator it = range.first; it != range.second; ++it) { + const auto wit = mapWallet.find(it->second); + if (wit != mapWallet.end()) { + txs.push_back(&wit->second); + } + } + } + } + } + //// debug print WalletLogPrintf("AddToWallet %s %s%s\n", hash.ToString(), (fInsertedNew ? "new" : ""), (fUpdated ? "update" : "")); @@ -1275,7 +1301,11 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) wtx.MarkDirty(); batch.WriteTx(wtx); NotifyTransactionChanged(wtx.GetHash(), CT_UPDATED); - // Iterate over all its outputs, and mark transactions in the wallet that spend them abandoned too + // Iterate over all its outputs, and mark transactions in the wallet that spend them abandoned too. + // States are not permanent, so these transactions can become unabandoned if they are re-added to the + // mempool, or confirmed in a block, or conflicted. + // Note: If the reorged coinbase is re-added to the main chain, the descendants that have not had their + // states change will remain abandoned and will require manual broadcast if the user wants them. for (unsigned int i = 0; i < wtx.tx->vout.size(); ++i) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(COutPoint(now, i)); for (TxSpends::const_iterator iter = range.first; iter != range.second; ++iter) { @@ -2996,7 +3026,7 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri if (args.IsArgSet("-mintxfee")) { std::optional<CAmount> min_tx_fee = ParseMoney(args.GetArg("-mintxfee", "")); - if (!min_tx_fee || min_tx_fee.value() == 0) { + if (!min_tx_fee) { error = AmountErrMsg("mintxfee", args.GetArg("-mintxfee", "")); return nullptr; } else if (min_tx_fee.value() > HIGH_TX_FEE_PER_KB) { @@ -3776,26 +3806,27 @@ bool CWallet::MigrateToSQLite(bilingual_str& error) // Get all of the records for DB type migration std::unique_ptr<DatabaseBatch> batch = m_database->MakeBatch(); + std::unique_ptr<DatabaseCursor> cursor = batch->GetNewCursor(); std::vector<std::pair<SerializeData, SerializeData>> records; - if (!batch->StartCursor()) { + if (!cursor) { error = _("Error: Unable to begin reading all records in the database"); return false; } - bool complete = false; + DatabaseCursor::Status status = DatabaseCursor::Status::FAIL; while (true) { - CDataStream ss_key(SER_DISK, CLIENT_VERSION); - CDataStream ss_value(SER_DISK, CLIENT_VERSION); - bool ret = batch->ReadAtCursor(ss_key, ss_value, complete); - if (!ret) { + DataStream ss_key{}; + DataStream ss_value{}; + status = cursor->Next(ss_key, ss_value); + if (status != DatabaseCursor::Status::MORE) { break; } SerializeData key(ss_key.begin(), ss_key.end()); SerializeData value(ss_value.begin(), ss_value.end()); records.emplace_back(key, value); } - batch->CloseCursor(); + cursor.reset(); batch.reset(); - if (!complete) { + if (status != DatabaseCursor::Status::DONE) { error = _("Error: Unable to read all records in the database"); return false; } @@ -3821,8 +3852,8 @@ bool CWallet::MigrateToSQLite(bilingual_str& error) bool began = batch->TxnBegin(); assert(began); // This is a critical error, the new db could not be written to. The original db exists as a backup, but we should not continue execution. for (const auto& [key, value] : records) { - CDataStream ss_key(key, SER_DISK, CLIENT_VERSION); - CDataStream ss_value(value, SER_DISK, CLIENT_VERSION); + DataStream ss_key{key}; + DataStream ss_value{value}; if (!batch->Write(ss_key, ss_value)) { batch->TxnAbort(); m_database->Close(); @@ -3840,10 +3871,7 @@ std::optional<MigrationData> CWallet::GetDescriptorsForLegacy(bilingual_str& err AssertLockHeld(cs_wallet); LegacyScriptPubKeyMan* legacy_spkm = GetLegacyScriptPubKeyMan(); - if (!legacy_spkm) { - error = _("Error: This wallet is already a descriptor wallet"); - return std::nullopt; - } + assert(legacy_spkm); std::optional<MigrationData> res = legacy_spkm->MigrateToDescriptor(); if (res == std::nullopt) { @@ -4139,6 +4167,11 @@ bool DoMigration(CWallet& wallet, WalletContext& context, bilingual_str& error, util::Result<MigrationResult> MigrateLegacyToDescriptor(std::shared_ptr<CWallet>&& wallet, WalletContext& context) { + // Before anything else, check if there is something to migrate. + if (!wallet->GetLegacyScriptPubKeyMan()) { + return util::Error{_("Error: This wallet is already a descriptor wallet")}; + } + MigrationResult res; bilingual_str error; std::vector<bilingual_str> warnings; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 8b7e6dd526..f104a15f98 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -821,7 +821,8 @@ public: bool IsLegacy() const; /** Returns a bracketed wallet name for displaying in logs, will return [default wallet] if the wallet has no name */ - const std::string GetDisplayName() const override { + std::string GetDisplayName() const override + { std::string wallet_name = GetName().length() == 0 ? "default wallet" : GetName(); return strprintf("[%s]", wallet_name); }; @@ -954,10 +955,10 @@ private: using Clock = std::chrono::steady_clock; using NowFn = std::function<Clock::time_point()>; CWallet& m_wallet; - bool m_could_reserve; + bool m_could_reserve{false}; NowFn m_now; public: - explicit WalletRescanReserver(CWallet& w) : m_wallet(w), m_could_reserve(false) {} + explicit WalletRescanReserver(CWallet& w) : m_wallet(w) {} bool reserve() { diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index b393c35112..2cd35ae40e 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -321,7 +321,7 @@ public: }; static bool -ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, +ReadKeyValue(CWallet* pwallet, DataStream& ssKey, CDataStream& ssValue, CWalletScanState &wss, std::string& strType, std::string& strErr, const KeyFilterFn& filter_fn = nullptr) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { try { @@ -759,7 +759,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, return true; } -bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr, const KeyFilterFn& filter_fn) +bool ReadKeyValue(CWallet* pwallet, DataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr, const KeyFilterFn& filter_fn) { CWalletScanState dummy_wss; LOCK(pwallet->cs_wallet); @@ -812,7 +812,8 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) #endif // Get cursor - if (!m_batch->StartCursor()) + std::unique_ptr<DatabaseCursor> cursor = m_batch->GetNewCursor(); + if (!cursor) { pwallet->WalletLogPrintf("Error getting wallet database cursor\n"); return DBErrors::CORRUPT; @@ -821,16 +822,13 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) while (true) { // Read next record - CDataStream ssKey(SER_DISK, CLIENT_VERSION); + DataStream ssKey{}; CDataStream ssValue(SER_DISK, CLIENT_VERSION); - bool complete; - bool ret = m_batch->ReadAtCursor(ssKey, ssValue, complete); - if (complete) { + DatabaseCursor::Status status = cursor->Next(ssKey, ssValue); + if (status == DatabaseCursor::Status::DONE) { break; - } - else if (!ret) - { - m_batch->CloseCursor(); + } else if (status == DatabaseCursor::Status::FAIL) { + cursor.reset(); pwallet->WalletLogPrintf("Error reading next record from wallet database\n"); return DBErrors::CORRUPT; } @@ -878,7 +876,6 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) } catch (...) { result = DBErrors::CORRUPT; } - m_batch->CloseCursor(); // Set the active ScriptPubKeyMans for (auto spk_man_pair : wss.m_active_external_spks) { @@ -986,7 +983,8 @@ DBErrors WalletBatch::FindWalletTxHashes(std::vector<uint256>& tx_hashes) } // Get cursor - if (!m_batch->StartCursor()) + std::unique_ptr<DatabaseCursor> cursor = m_batch->GetNewCursor(); + if (!cursor) { LogPrintf("Error getting wallet database cursor\n"); return DBErrors::CORRUPT; @@ -995,14 +993,12 @@ DBErrors WalletBatch::FindWalletTxHashes(std::vector<uint256>& tx_hashes) while (true) { // Read next record - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - bool complete; - bool ret = m_batch->ReadAtCursor(ssKey, ssValue, complete); - if (complete) { + DataStream ssKey{}; + DataStream ssValue{}; + DatabaseCursor::Status status = cursor->Next(ssKey, ssValue); + if (status == DatabaseCursor::Status::DONE) { break; - } else if (!ret) { - m_batch->CloseCursor(); + } else if (status == DatabaseCursor::Status::FAIL) { LogPrintf("Error reading next record from wallet database\n"); return DBErrors::CORRUPT; } @@ -1018,7 +1014,6 @@ DBErrors WalletBatch::FindWalletTxHashes(std::vector<uint256>& tx_hashes) } catch (...) { result = DBErrors::CORRUPT; } - m_batch->CloseCursor(); return result; } @@ -1111,7 +1106,8 @@ bool WalletBatch::WriteWalletFlags(const uint64_t flags) bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) { // Get cursor - if (!m_batch->StartCursor()) + std::unique_ptr<DatabaseCursor> cursor = m_batch->GetNewCursor(); + if (!cursor) { return false; } @@ -1120,16 +1116,12 @@ bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) while (true) { // Read next record - CDataStream key(SER_DISK, CLIENT_VERSION); - CDataStream value(SER_DISK, CLIENT_VERSION); - bool complete; - bool ret = m_batch->ReadAtCursor(key, value, complete); - if (complete) { + DataStream key{}; + DataStream value{}; + DatabaseCursor::Status status = cursor->Next(key, value); + if (status == DatabaseCursor::Status::DONE) { break; - } - else if (!ret) - { - m_batch->CloseCursor(); + } else if (status == DatabaseCursor::Status::FAIL) { return false; } @@ -1143,7 +1135,6 @@ bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) m_batch->Erase(key_data); } } - m_batch->CloseCursor(); return true; } diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 97e8fad278..c97356a71f 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -303,7 +303,7 @@ void MaybeCompactWalletDB(WalletContext& context); using KeyFilterFn = std::function<bool(const std::string&)>; //! Unserialize a given Key-Value pair and load it into the wallet -bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr, const KeyFilterFn& filter_fn = nullptr); +bool ReadKeyValue(CWallet* pwallet, DataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr, const KeyFilterFn& filter_fn = nullptr); /** Return object for accessing dummy database with no read/write capabilities. */ std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase(); diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index f93b666bd5..96537b9873 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -47,7 +47,7 @@ static void WalletCreate(CWallet* wallet_instance, uint64_t wallet_creation_flag wallet_instance->TopUpKeyPool(); } -static const std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::path& path, const ArgsManager& args, DatabaseOptions options) +static std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::path& path, const ArgsManager& args, DatabaseOptions options) { DatabaseStatus status; bilingual_str error; diff --git a/src/zmq/zmqabstractnotifier.h b/src/zmq/zmqabstractnotifier.h index 6899ee8fa6..cf0ee48f47 100644 --- a/src/zmq/zmqabstractnotifier.h +++ b/src/zmq/zmqabstractnotifier.h @@ -20,7 +20,7 @@ class CZMQAbstractNotifier public: static const int DEFAULT_ZMQ_SNDHWM {1000}; - CZMQAbstractNotifier() : psocket(nullptr), outbound_message_high_water_mark(DEFAULT_ZMQ_SNDHWM) { } + CZMQAbstractNotifier() : outbound_message_high_water_mark(DEFAULT_ZMQ_SNDHWM) {} virtual ~CZMQAbstractNotifier(); template <typename T> @@ -57,7 +57,7 @@ public: virtual bool NotifyTransaction(const CTransaction &transaction); protected: - void *psocket; + void* psocket{nullptr}; std::string type; std::string address; int outbound_message_high_water_mark; // aka SNDHWM diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index 6dc4737d0a..df129c0830 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -21,7 +21,7 @@ #include <utility> #include <vector> -CZMQNotificationInterface::CZMQNotificationInterface() : pcontext(nullptr) +CZMQNotificationInterface::CZMQNotificationInterface() { } diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index b24d4664da..a43f9bfef3 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -39,7 +39,7 @@ protected: private: CZMQNotificationInterface(); - void *pcontext; + void* pcontext{nullptr}; std::list<std::unique_ptr<CZMQAbstractNotifier>> notifiers; }; diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 5ba13ecc9c..6418455d19 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -7,6 +7,7 @@ #include <chain.h> #include <chainparams.h> #include <crypto/common.h> +#include <kernel/cs_main.h> #include <logging.h> #include <netaddress.h> #include <netbase.h> diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index d63821fbbc..894afffc79 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -32,6 +32,7 @@ from test_framework.util import ( assert_raises_rpc_error, softfork_active, ) +from test_framework.wallet import MiniWallet SCRIPT_W0_SH_OP_TRUE = script_to_p2wsh_script(CScript([OP_TRUE])) @@ -58,14 +59,9 @@ class BIP68Test(BitcoinTestFramework): ], ] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] - - # Generate some coins - self.generate(self.nodes[0], 110) + self.wallet = MiniWallet(self.nodes[0]) self.log.info("Running test disable flag") self.test_disable_flag() @@ -92,16 +88,10 @@ class BIP68Test(BitcoinTestFramework): # the first sequence bit is set. def test_disable_flag(self): # Create some unconfirmed inputs - new_addr = self.nodes[0].getnewaddress() - self.nodes[0].sendtoaddress(new_addr, 2) # send 2 BTC - - utxos = self.nodes[0].listunspent(0, 0) - assert len(utxos) > 0 - - utxo = utxos[0] + utxo = self.wallet.send_self_transfer(from_node=self.nodes[0])["new_utxo"] tx1 = CTransaction() - value = int((utxo["amount"] - self.relayfee) * COIN) + value = int((utxo["value"] - self.relayfee) * COIN) # Check that the disable flag disables relative locktime. # If sequence locks were used, this would require 1 block for the @@ -110,8 +100,8 @@ class BIP68Test(BitcoinTestFramework): tx1.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), nSequence=sequence_value)] tx1.vout = [CTxOut(value, SCRIPT_W0_SH_OP_TRUE)] - tx1_signed = self.nodes[0].signrawtransactionwithwallet(tx1.serialize().hex())["hex"] - tx1_id = self.nodes[0].sendrawtransaction(tx1_signed) + self.wallet.sign_tx(tx=tx1) + tx1_id = self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx1.serialize().hex()) tx1_id = int(tx1_id, 16) # This transaction will enable sequence-locks, so this transaction should @@ -125,13 +115,13 @@ class BIP68Test(BitcoinTestFramework): tx2.vout = [CTxOut(int(value - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] tx2.rehash() - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, tx2.serialize().hex()) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=self.nodes[0], tx_hex=tx2.serialize().hex()) # Setting the version back down to 1 should disable the sequence lock, # so this should be accepted. tx2.nVersion = 1 - self.nodes[0].sendrawtransaction(tx2.serialize().hex()) + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx2.serialize().hex()) # Calculate the median time past of a prior block ("confirmations" before # the current tip). @@ -144,20 +134,13 @@ class BIP68Test(BitcoinTestFramework): # Create lots of confirmed utxos, and use them to generate lots of random # transactions. max_outputs = 50 - addresses = [] - while len(addresses) < max_outputs: - addresses.append(self.nodes[0].getnewaddress()) - while len(self.nodes[0].listunspent()) < 200: + while len(self.wallet.get_utxos(include_immature_coinbase=False, mark_as_spent=False)) < 200: import random - random.shuffle(addresses) num_outputs = random.randint(1, max_outputs) - outputs = {} - for i in range(num_outputs): - outputs[addresses[i]] = random.randint(1, 20)*0.01 - self.nodes[0].sendmany("", outputs) - self.generate(self.nodes[0], 1) + self.wallet.send_self_transfer_multi(from_node=self.nodes[0], num_outputs=num_outputs) + self.generate(self.wallet, 1) - utxos = self.nodes[0].listunspent() + utxos = self.wallet.get_utxos(include_immature_coinbase=False) # Try creating a lot of random transactions. # Each time, choose a random number of inputs, and randomly set @@ -214,19 +197,20 @@ class BIP68Test(BitcoinTestFramework): sequence_value = ((cur_time - orig_time) >> SEQUENCE_LOCKTIME_GRANULARITY)+1 sequence_value |= SEQUENCE_LOCKTIME_TYPE_FLAG tx.vin.append(CTxIn(COutPoint(int(utxos[j]["txid"], 16), utxos[j]["vout"]), nSequence=sequence_value)) - value += utxos[j]["amount"]*COIN + value += utxos[j]["value"]*COIN # Overestimate the size of the tx - signatures should be less than 120 bytes, and leave 50 for the output tx_size = len(tx.serialize().hex())//2 + 120*num_inputs + 50 tx.vout.append(CTxOut(int(value - self.relayfee * tx_size * COIN / 1000), SCRIPT_W0_SH_OP_TRUE)) - rawtx = self.nodes[0].signrawtransactionwithwallet(tx.serialize().hex())["hex"] + self.wallet.sign_tx(tx=tx) if (using_sequence_locks and not should_pass): # This transaction should be rejected - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, rawtx) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=self.nodes[0], tx_hex=tx.serialize().hex()) else: # This raw transaction should be accepted - self.nodes[0].sendrawtransaction(rawtx) - utxos = self.nodes[0].listunspent() + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx.serialize().hex()) + self.wallet.rescan_utxos() + utxos = self.wallet.get_utxos(include_immature_coinbase=False) # Test that sequence locks on unconfirmed inputs must have nSequence # height or time of 0 to be accepted. @@ -237,8 +221,8 @@ class BIP68Test(BitcoinTestFramework): cur_height = self.nodes[0].getblockcount() # Create a mempool tx. - txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 2) - tx1 = tx_from_hex(self.nodes[0].getrawtransaction(txid)) + self.wallet.rescan_utxos() + tx1 = self.wallet.send_self_transfer(from_node=self.nodes[0])["tx"] tx1.rehash() # Anyone-can-spend mempool tx. @@ -247,11 +231,11 @@ class BIP68Test(BitcoinTestFramework): tx2.nVersion = 2 tx2.vin = [CTxIn(COutPoint(tx1.sha256, 0), nSequence=0)] tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] - tx2_raw = self.nodes[0].signrawtransactionwithwallet(tx2.serialize().hex())["hex"] - tx2 = tx_from_hex(tx2_raw) + self.wallet.sign_tx(tx=tx2) + tx2_raw = tx2.serialize().hex() tx2.rehash() - self.nodes[0].sendrawtransaction(tx2_raw) + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx2_raw) # Create a spend of the 0th output of orig_tx with a sequence lock # of 1, and test what happens when submitting. @@ -271,10 +255,10 @@ class BIP68Test(BitcoinTestFramework): if (orig_tx.hash in node.getrawmempool()): # sendrawtransaction should fail if the tx is in the mempool - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, node.sendrawtransaction, tx.serialize().hex()) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=node, tx_hex=tx.serialize().hex()) else: # sendrawtransaction should succeed if the tx is not in the mempool - node.sendrawtransaction(tx.serialize().hex()) + self.wallet.sendrawtransaction(from_node=node, tx_hex=tx.serialize().hex()) return tx @@ -287,7 +271,7 @@ class BIP68Test(BitcoinTestFramework): cur_time = int(time.time()) for _ in range(10): self.nodes[0].setmocktime(cur_time + 600) - self.generate(self.nodes[0], 1, sync_fun=self.no_op) + self.generate(self.wallet, 1, sync_fun=self.no_op) cur_time += 600 assert tx2.hash in self.nodes[0].getrawmempool() @@ -321,12 +305,12 @@ class BIP68Test(BitcoinTestFramework): tx5 = test_nonzero_locks(tx4, self.nodes[0], self.relayfee, use_height_lock=True) assert tx5.hash not in self.nodes[0].getrawmempool() - utxos = self.nodes[0].listunspent() - tx5.vin.append(CTxIn(COutPoint(int(utxos[0]["txid"], 16), utxos[0]["vout"]), nSequence=1)) - tx5.vout[0].nValue += int(utxos[0]["amount"]*COIN) - raw_tx5 = self.nodes[0].signrawtransactionwithwallet(tx5.serialize().hex())["hex"] + utxo = self.wallet.get_utxo() + tx5.vin.append(CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), nSequence=1)) + tx5.vout[0].nValue += int(utxo["value"]*COIN) + self.wallet.sign_tx(tx=tx5) - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, raw_tx5) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=self.nodes[0], tx_hex=tx5.serialize().hex()) # Test mempool-BIP68 consistency after reorg # @@ -362,7 +346,7 @@ class BIP68Test(BitcoinTestFramework): # Reset the chain and get rid of the mocktimed-blocks self.nodes[0].setmocktime(0) self.nodes[0].invalidateblock(self.nodes[0].getblockhash(cur_height+1)) - self.generate(self.nodes[0], 10, sync_fun=self.no_op) + self.generate(self.wallet, 10, sync_fun=self.no_op) # Make sure that BIP68 isn't being used to validate blocks prior to # activation height. If more blocks are mined prior to this test @@ -370,9 +354,8 @@ class BIP68Test(BitcoinTestFramework): # this test should be moved to run earlier, or deleted. def test_bip68_not_consensus(self): assert not softfork_active(self.nodes[0], 'csv') - txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 2) - tx1 = tx_from_hex(self.nodes[0].getrawtransaction(txid)) + tx1 = self.wallet.send_self_transfer(from_node=self.nodes[0])["tx"] tx1.rehash() # Make an anyone-can-spend transaction @@ -382,11 +365,12 @@ class BIP68Test(BitcoinTestFramework): tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] # sign tx2 - tx2_raw = self.nodes[0].signrawtransactionwithwallet(tx2.serialize().hex())["hex"] + self.wallet.sign_tx(tx=tx2) + tx2_raw = tx2.serialize().hex() tx2 = tx_from_hex(tx2_raw) tx2.rehash() - self.nodes[0].sendrawtransaction(tx2.serialize().hex()) + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx2_raw) # Now make an invalid spend of tx2 according to BIP68 sequence_value = 100 # 100 block relative locktime @@ -399,7 +383,7 @@ class BIP68Test(BitcoinTestFramework): tx3.vout = [CTxOut(int(tx2.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] tx3.rehash() - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, tx3.serialize().hex()) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=self.nodes[0], tx_hex=tx3.serialize().hex()) # make a block that violates bip68; ensure that the tip updates block = create_block(tmpl=self.nodes[0].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS), txlist=[tx1, tx2, tx3]) @@ -415,22 +399,19 @@ class BIP68Test(BitcoinTestFramework): min_activation_height = 432 height = self.nodes[0].getblockcount() assert_greater_than(min_activation_height - height, 2) - self.generate(self.nodes[0], min_activation_height - height - 2, sync_fun=self.no_op) + self.generate(self.wallet, min_activation_height - height - 2, sync_fun=self.no_op) assert not softfork_active(self.nodes[0], 'csv') - self.generate(self.nodes[0], 1, sync_fun=self.no_op) + self.generate(self.wallet, 1, sync_fun=self.no_op) assert softfork_active(self.nodes[0], 'csv') self.sync_blocks() # Use self.nodes[1] to test that version 2 transactions are standard. def test_version2_relay(self): - inputs = [ ] - outputs = { self.nodes[1].getnewaddress() : 1.0 } - rawtx = self.nodes[1].createrawtransaction(inputs, outputs) - rawtxfund = self.nodes[1].fundrawtransaction(rawtx)['hex'] - tx = tx_from_hex(rawtxfund) + mini_wallet = MiniWallet(self.nodes[1]) + mini_wallet.rescan_utxos() + tx = mini_wallet.create_self_transfer()["tx"] tx.nVersion = 2 - tx_signed = self.nodes[1].signrawtransactionwithwallet(tx.serialize().hex())["hex"] - self.nodes[1].sendrawtransaction(tx_signed) + mini_wallet.sendrawtransaction(from_node=self.nodes[1], tx_hex=tx.serialize().hex()) if __name__ == '__main__': BIP68Test().main() diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index d5e5ed47d6..a37d614535 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -126,7 +126,6 @@ class ConfArgsTest(BitcoinTestFramework): expected_msgs=[ 'Command-line arg: addnode="some.node"', 'Command-line arg: rpcauth=****', - 'Command-line arg: rpcbind=****', 'Command-line arg: rpcpassword=****', 'Command-line arg: rpcuser=****', 'Command-line arg: torpassword=****', @@ -135,14 +134,17 @@ class ConfArgsTest(BitcoinTestFramework): ], unexpected_msgs=[ 'alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc0', - '127.1.1.1', 'secret-rpcuser', 'secret-torpassword', + 'Command-line arg: rpcbind=****', + 'Command-line arg: rpcallowip=****', ]): self.start_node(0, extra_args=[ '-addnode=some.node', '-rpcauth=alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc0', '-rpcbind=127.1.1.1', + '-rpcbind=127.0.0.1', + "-rpcallowip=127.0.0.1", '-rpcpassword=', '-rpcuser=secret-rpcuser', '-torpassword=secret-torpassword', diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index ff770e7707..e2bc566f53 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -202,7 +202,6 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[3]) - self.wallet.rescan_utxos() initial_height = self.nodes[3].getblockcount() self.generate(self.nodes[3], COINBASE_MATURITY, sync_fun=self.no_op) diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 6b2d5b9455..05ee556ece 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -297,7 +297,6 @@ class EstimateFeeTest(BitcoinTestFramework): # Split two coinbases into many small utxos self.start_node(0) self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() self.initial_split(self.nodes[0]) self.log.info("Finished splitting") diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index 32fea18f37..8cb633d454 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -31,7 +31,7 @@ class NotificationsTest(BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = True # The experimental syscall sandbox feature (-sandbox) is not compatible with -alertnotify, - # -blocknotify or -walletnotify (which all invoke execve). + # -blocknotify, -walletnotify or -shutdownnotify (which all invoke execve). self.disable_syscall_sandbox = True def setup_network(self): @@ -39,14 +39,18 @@ class NotificationsTest(BitcoinTestFramework): self.alertnotify_dir = os.path.join(self.options.tmpdir, "alertnotify") self.blocknotify_dir = os.path.join(self.options.tmpdir, "blocknotify") self.walletnotify_dir = os.path.join(self.options.tmpdir, "walletnotify") + self.shutdownnotify_dir = os.path.join(self.options.tmpdir, "shutdownnotify") + self.shutdownnotify_file = os.path.join(self.shutdownnotify_dir, "shutdownnotify.txt") os.mkdir(self.alertnotify_dir) os.mkdir(self.blocknotify_dir) os.mkdir(self.walletnotify_dir) + os.mkdir(self.shutdownnotify_dir) # -alertnotify and -blocknotify on node0, walletnotify on node1 self.extra_args = [[ f"-alertnotify=echo > {os.path.join(self.alertnotify_dir, '%s')}", f"-blocknotify=echo > {os.path.join(self.blocknotify_dir, '%s')}", + f"-shutdownnotify=echo > {self.shutdownnotify_file}", ], [ f"-walletnotify=echo %h_%b > {os.path.join(self.walletnotify_dir, notify_outputname('%w', '%s'))}", ]] @@ -162,6 +166,10 @@ class NotificationsTest(BitcoinTestFramework): # TODO: add test for `-alertnotify` large fork notifications + self.log.info("test -shutdownnotify") + self.stop_nodes() + self.wait_until(lambda: os.path.isfile(self.shutdownnotify_file), timeout=10) + def expect_wallet_notify(self, tx_details): self.wait_until(lambda: len(os.listdir(self.walletnotify_dir)) >= len(tx_details), timeout=10) # Should have no more and no less files than expected diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index f68416dc03..664ed779db 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -84,7 +84,7 @@ class PruneTest(BitcoinTestFramework): ["-maxreceivebuffer=20000", "-prune=550"], ["-maxreceivebuffer=20000"], ["-maxreceivebuffer=20000"], - ["-prune=550"], + ["-prune=550", "-blockfilterindex=1"], ] self.rpc_timeout = 120 @@ -356,7 +356,7 @@ class PruneTest(BitcoinTestFramework): self.connect_nodes(0, 5) nds = [self.nodes[0], self.nodes[5]] self.sync_blocks(nds, wait=5, timeout=300) - self.restart_node(5, extra_args=["-prune=550"]) # restart to trigger rescan + self.restart_node(5, extra_args=["-prune=550", "-blockfilterindex=1"]) # restart to trigger rescan self.log.info("Success") def run_test(self): @@ -472,7 +472,20 @@ class PruneTest(BitcoinTestFramework): self.log.info("Test invalid pruning command line options") self.test_invalid_command_line_options() + self.test_scanblocks_pruned() + self.log.info("Done") + def test_scanblocks_pruned(self): + node = self.nodes[5] + genesis_blockhash = node.getblockhash(0) + false_positive_spk = bytes.fromhex("001400000000000000000000000000000000000cadcb") + + assert genesis_blockhash in node.scanblocks( + "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0)['relevant_blocks'] + + assert_raises_rpc_error(-1, "Block not available (pruned data)", node.scanblocks, + "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0, "basic", {"filter_false_positives": True}) + if __name__ == '__main__': PruneTest().main() diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index 085ff3a2e3..947d2e8273 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -42,10 +42,6 @@ class ReplaceByFeeTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - # the pre-mined test framework chain contains coinbase outputs to the - # MiniWallet's default address in blocks 76-100 (see method - # BitcoinTestFramework._initialize_chain()) - self.wallet.rescan_utxos() self.log.info("Running test simple doublespend...") self.test_simple_doublespend() @@ -396,12 +392,11 @@ class ReplaceByFeeTest(BitcoinTestFramework): enough transactions off of each root UTXO to exceed the MAX_REPLACEMENT_LIMIT. Then create a conflicting RBF replacement transaction. """ - normal_node = self.nodes[1] - wallet = MiniWallet(normal_node) - wallet.rescan_utxos() # Clear mempools to avoid cross-node sync failure. for node in self.nodes: self.generate(node, 1) + normal_node = self.nodes[1] + wallet = MiniWallet(normal_node) # This has to be chosen so that the total number of transactions can exceed # MAX_REPLACEMENT_LIMIT without having any one tx graph run into the descendant diff --git a/test/functional/feature_txindex_compatibility.py b/test/functional/feature_txindex_compatibility.py index dd18c5fd99..48fefaa0ba 100755 --- a/test/functional/feature_txindex_compatibility.py +++ b/test/functional/feature_txindex_compatibility.py @@ -42,7 +42,6 @@ class TxindexCompatibilityTest(BitcoinTestFramework): def run_test(self): mini_wallet = MiniWallet(self.nodes[1]) - mini_wallet.rescan_utxos() spend_utxo = mini_wallet.get_utxo() mini_wallet.send_self_transfer(from_node=self.nodes[1], utxo_to_spend=spend_utxo) self.generate(self.nodes[1], 1) diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index db2ff19b44..0dddc8946a 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -96,7 +96,6 @@ class RESTTest (BitcoinTestFramework): def run_test(self): self.url = urllib.parse.urlparse(self.nodes[0].url) self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() self.log.info("Broadcast test transaction and sync nodes") txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=int(0.1 * COIN)) diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py index e3b0b32f0d..23785b1e8a 100755 --- a/test/functional/interface_usdt_utxocache.py +++ b/test/functional/interface_usdt_utxocache.py @@ -144,7 +144,6 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.generate(self.wallet, 101) self.test_uncache() self.test_add_spent() diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 5357bdf2b8..2f41f9aa53 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -215,7 +215,6 @@ class ZMQTest (BitcoinTestFramework): assert_equal([txid.hex()], self.nodes[1].getblock(hash)["tx"]) - self.wallet.rescan_utxos() self.log.info("Wait for tx from second node") payment_tx = self.wallet.send_self_transfer(from_node=self.nodes[1]) payment_txid = payment_tx['txid'] diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 19cb65be36..362b407062 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -69,7 +69,6 @@ class MempoolAcceptanceTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] self.wallet = MiniWallet(node) - self.wallet.rescan_utxos() self.log.info('Start with empty mempool, and 200 blocks') self.mempool_size = 0 diff --git a/test/functional/mempool_datacarrier.py b/test/functional/mempool_datacarrier.py index 9c82964a24..c370d8fa91 100755 --- a/test/functional/mempool_datacarrier.py +++ b/test/functional/mempool_datacarrier.py @@ -44,7 +44,6 @@ class DataCarrierTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() # By default, only 80 bytes are used for data (+1 for OP_RETURN, +2 for the pushdata opcodes). default_size_data = random_bytes(MAX_OP_RETURN_RELAY - 3) diff --git a/test/functional/mempool_dust.py b/test/functional/mempool_dust.py new file mode 100755 index 0000000000..41a26e82da --- /dev/null +++ b/test/functional/mempool_dust.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test dust limit mempool policy (`-dustrelayfee` parameter)""" +from decimal import Decimal + +from test_framework.key import ECKey +from test_framework.messages import ( + COIN, + CTxOut, +) +from test_framework.script import ( + CScript, + OP_RETURN, + OP_TRUE, +) +from test_framework.script_util import ( + key_to_p2pk_script, + key_to_p2pkh_script, + key_to_p2wpkh_script, + keys_to_multisig_script, + output_key_to_p2tr_script, + program_to_witness_script, + script_to_p2sh_script, + script_to_p2wsh_script, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_node import TestNode +from test_framework.util import ( + assert_equal, + get_fee, +) +from test_framework.wallet import MiniWallet + + +DUST_RELAY_TX_FEE = 3000 # default setting [sat/kvB] + + +class DustRelayFeeTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def test_dust_output(self, node: TestNode, dust_relay_fee: Decimal, + output_script: CScript, type_desc: str) -> None: + # determine dust threshold (see `GetDustThreshold`) + if output_script[0] == OP_RETURN: + dust_threshold = 0 + else: + tx_size = len(CTxOut(nValue=0, scriptPubKey=output_script).serialize()) + tx_size += 67 if output_script.IsWitnessProgram() else 148 + dust_threshold = int(get_fee(tx_size, dust_relay_fee) * COIN) + self.log.info(f"-> Test {type_desc} output (size {len(output_script)}, limit {dust_threshold})") + + # amount right on the dust threshold should pass + tx = self.wallet.create_self_transfer()["tx"] + tx.vout.append(CTxOut(nValue=dust_threshold, scriptPubKey=output_script)) + tx.vout[0].nValue -= dust_threshold # keep total output value constant + tx_good_hex = tx.serialize().hex() + res = node.testmempoolaccept([tx_good_hex])[0] + assert_equal(res['allowed'], True) + + # amount just below the dust threshold should fail + if dust_threshold > 0: + tx.vout[1].nValue -= 1 + res = node.testmempoolaccept([tx.serialize().hex()])[0] + assert_equal(res['allowed'], False) + assert_equal(res['reject-reason'], 'dust') + + # finally send the transaction to avoid running out of MiniWallet UTXOs + self.wallet.sendrawtransaction(from_node=node, tx_hex=tx_good_hex) + + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + + # prepare output scripts of each standard type + key = ECKey() + key.generate(compressed=False) + uncompressed_pubkey = key.get_pubkey().get_bytes() + key.generate(compressed=True) + pubkey = key.get_pubkey().get_bytes() + + output_scripts = ( + (key_to_p2pk_script(uncompressed_pubkey), "P2PK (uncompressed)"), + (key_to_p2pk_script(pubkey), "P2PK (compressed)"), + (key_to_p2pkh_script(pubkey), "P2PKH"), + (script_to_p2sh_script(CScript([OP_TRUE])), "P2SH"), + (key_to_p2wpkh_script(pubkey), "P2WPKH"), + (script_to_p2wsh_script(CScript([OP_TRUE])), "P2WSH"), + (output_key_to_p2tr_script(pubkey[1:]), "P2TR"), + # witness programs for segwitv2+ can be between 2 and 40 bytes + (program_to_witness_script(2, b'\x66' * 2), "P2?? (future witness version 2)"), + (program_to_witness_script(16, b'\x77' * 40), "P2?? (future witness version 16)"), + # largest possible output script considered standard + (keys_to_multisig_script([uncompressed_pubkey]*3), "bare multisig (m-of-3)"), + (CScript([OP_RETURN, b'superimportanthash']), "null data (OP_RETURN)"), + ) + + # test default (no parameter), disabled (=0) and a bunch of arbitrary dust fee rates [sat/kvB] + for dustfee_sat_kvb in (DUST_RELAY_TX_FEE, 0, 1, 66, 500, 1337, 12345, 21212, 333333): + dustfee_btc_kvb = dustfee_sat_kvb / Decimal(COIN) + if dustfee_sat_kvb == DUST_RELAY_TX_FEE: + self.log.info(f"Test default dust limit setting ({dustfee_sat_kvb} sat/kvB)...") + else: + dust_parameter = f"-dustrelayfee={dustfee_btc_kvb:.8f}" + self.log.info(f"Test dust limit setting {dust_parameter} ({dustfee_sat_kvb} sat/kvB)...") + self.restart_node(0, extra_args=[dust_parameter]) + + for output_script, description in output_scripts: + self.test_dust_output(self.nodes[0], dustfee_btc_kvb, output_script, description) + self.generate(self.nodes[0], 1) + + +if __name__ == '__main__': + DustRelayFeeTest().main() diff --git a/test/functional/mempool_expiry.py b/test/functional/mempool_expiry.py index 0c91da901f..15a5f765df 100755 --- a/test/functional/mempool_expiry.py +++ b/test/functional/mempool_expiry.py @@ -12,7 +12,6 @@ definable expiry timeout via the '-mempoolexpiry=<n>' command line argument from datetime import timedelta -from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import DEFAULT_MEMPOOL_EXPIRY_HOURS from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -27,17 +26,11 @@ CUSTOM_MEMPOOL_EXPIRY = 10 # hours class MempoolExpiryTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.setup_clean_chain = True def test_transaction_expiry(self, timeout): """Tests that a transaction expires after the expiry timeout and its children are removed as well.""" node = self.nodes[0] - self.wallet = MiniWallet(node) - - # Add enough mature utxos to the wallet so that all txs spend confirmed coins. - self.generate(self.wallet, 4) - self.generate(node, COINBASE_MATURITY) # Send a parent transaction that will expire. parent_txid = self.wallet.send_self_transfer(from_node=node)['txid'] @@ -97,6 +90,8 @@ class MempoolExpiryTest(BitcoinTestFramework): assert_equal(half_expiry_time, node.getmempoolentry(independent_txid)['time']) def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.log.info('Test default mempool expiry timeout of %d hours.' % DEFAULT_MEMPOOL_EXPIRY_HOURS) self.test_transaction_expiry(DEFAULT_MEMPOOL_EXPIRY_HOURS) diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index 07636439d0..d38a37f952 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -25,7 +25,6 @@ class MempoolLimitTest(BitcoinTestFramework): self.extra_args = [[ "-datacarriersize=100000", "-maxmempool=5", - "-spendzeroconfchange=0", ]] self.supports_cli = False diff --git a/test/functional/mempool_package_limits.py b/test/functional/mempool_package_limits.py index 47b7be7d88..63f5a3e3d3 100755 --- a/test/functional/mempool_package_limits.py +++ b/test/functional/mempool_package_limits.py @@ -45,7 +45,7 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): assert_equal(0, node.getmempoolinfo()["size"]) chain_hex = [] - chaintip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=mempool_count) + chaintip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=mempool_count)[-1]["new_utxo"] # in-package transactions for _ in range(package_count): tx = self.wallet.create_self_transfer(utxo_to_spend=chaintip_utxo) @@ -100,13 +100,13 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): package_hex = [] # Chain A (M2a... M12a) - chain_a_tip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=11, utxo_to_spend=m1_utxos[0]) + chain_a_tip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=11, utxo_to_spend=m1_utxos[0])[-1]["new_utxo"] # Pa pa_hex = self.wallet.create_self_transfer(utxo_to_spend=chain_a_tip_utxo)["hex"] package_hex.append(pa_hex) # Chain B (M2b... M13b) - chain_b_tip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=12, utxo_to_spend=m1_utxos[1]) + chain_b_tip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=12, utxo_to_spend=m1_utxos[1])[-1]["new_utxo"] # Pb pb_hex = self.wallet.create_self_transfer(utxo_to_spend=chain_b_tip_utxo)["hex"] package_hex.append(pb_hex) @@ -145,7 +145,7 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): m1_utxos = self.wallet.send_self_transfer_multi(from_node=node, num_outputs=2)['new_utxos'] # Chain M2...M24 - self.wallet.send_self_transfer_chain(from_node=node, chain_length=23, utxo_to_spend=m1_utxos[0]) + self.wallet.send_self_transfer_chain(from_node=node, chain_length=23, utxo_to_spend=m1_utxos[0])[-1]["new_utxo"] # P1 p1_tx = self.wallet.create_self_transfer(utxo_to_spend=m1_utxos[1]) @@ -191,7 +191,7 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): # Two chains of 13 transactions each for _ in range(2): - chain_tip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=12) + chain_tip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=12)[-1]["new_utxo"] # Save the 13th transaction for the package tx = self.wallet.create_self_transfer(utxo_to_spend=chain_tip_utxo) package_hex.append(tx["hex"]) @@ -234,7 +234,7 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): self.log.info("Check that in-mempool and in-package ancestors are calculated properly in packages") # Two chains of 12 transactions each for _ in range(2): - chaintip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=12) + chaintip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=12)[-1]["new_utxo"] # last 2 transactions will be the parents of Pc pc_parent_utxos.append(chaintip_utxo) diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index 23ee587098..921c190668 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -31,7 +31,6 @@ class MempoolPackagesTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() # DEFAULT_ANCESTOR_LIMIT transactions off a confirmed tx should be fine chain = [] diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index c89528101e..0387282862 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -6,7 +6,6 @@ from decimal import Decimal -from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import ( COIN, DEFAULT_ANCESTOR_LIMIT, @@ -17,9 +16,8 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - chain_transaction, ) - +from test_framework.wallet import MiniWallet # custom limits for node1 CUSTOM_ANCESTOR_LIMIT = 5 @@ -45,43 +43,33 @@ class MempoolPackagesTest(BitcoinTestFramework): ], ] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): - # Mine some blocks and have them mature. + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() + + if self.is_specified_wallet_compiled(): + self.nodes[0].createwallet("watch_wallet", disable_private_keys=True) + self.nodes[0].importaddress(self.wallet.get_address()) + peer_inv_store = self.nodes[0].add_p2p_connection(P2PTxInvStore()) # keep track of invs - self.generate(self.nodes[0], COINBASE_MATURITY + 1) - utxo = self.nodes[0].listunspent(10) - txid = utxo[0]['txid'] - vout = utxo[0]['vout'] - value = utxo[0]['amount'] - assert 'ancestorcount' not in utxo[0] - assert 'ancestorsize' not in utxo[0] - assert 'ancestorfees' not in utxo[0] - - fee = Decimal("0.0001") + # DEFAULT_ANCESTOR_LIMIT transactions off a confirmed tx should be fine - chain = [] - witness_chain = [] + chain = self.wallet.create_self_transfer_chain(chain_length=DEFAULT_ANCESTOR_LIMIT) + witness_chain = [t["wtxid"] for t in chain] ancestor_vsize = 0 ancestor_fees = Decimal(0) - for i in range(DEFAULT_ANCESTOR_LIMIT): - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [0], value, fee, 1) - value = sent_value - chain.append(txid) - # We need the wtxids to check P2P announcements - witnesstx = self.nodes[0].gettransaction(txid=txid, verbose=True)['decoded'] - witness_chain.append(witnesstx['hash']) + for i, t in enumerate(chain): + ancestor_vsize += t["tx"].get_vsize() + ancestor_fees += t["fee"] + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=t["hex"]) # Check that listunspent ancestor{count, size, fees} yield the correct results - wallet_unspent = self.nodes[0].listunspent(minconf=0) - this_unspent = next(utxo_info for utxo_info in wallet_unspent if utxo_info['txid'] == txid) - assert_equal(this_unspent['ancestorcount'], i + 1) - ancestor_vsize += self.nodes[0].getrawtransaction(txid=txid, verbose=True)['vsize'] - assert_equal(this_unspent['ancestorsize'], ancestor_vsize) - ancestor_fees -= self.nodes[0].gettransaction(txid=txid)['fee'] - assert_equal(this_unspent['ancestorfees'], ancestor_fees * COIN) + if self.is_specified_wallet_compiled(): + wallet_unspent = self.nodes[0].listunspent(minconf=0) + this_unspent = next(utxo_info for utxo_info in wallet_unspent if utxo_info["txid"] == t["txid"]) + assert_equal(this_unspent['ancestorcount'], i + 1) + assert_equal(this_unspent['ancestorsize'], ancestor_vsize) + assert_equal(this_unspent['ancestorfees'], ancestor_fees * COIN) # Wait until mempool transactions have passed initial broadcast (sent inv and received getdata) # Otherwise, getrawmempool may be inconsistent with getmempoolentry if unbroadcast changes in between @@ -99,15 +87,20 @@ class MempoolPackagesTest(BitcoinTestFramework): ancestor_count = DEFAULT_ANCESTOR_LIMIT assert_equal(ancestor_fees, sum([mempool[tx]['fees']['base'] for tx in mempool])) + # Adding one more transaction on to the chain should fail. + next_hop = self.wallet.create_self_transfer(utxo_to_spend=chain[-1]["new_utxo"])["hex"] + assert_raises_rpc_error(-26, "too-long-mempool-chain", lambda: self.nodes[0].sendrawtransaction(next_hop)) + descendants = [] - ancestors = list(chain) + ancestors = [t["txid"] for t in chain] + chain = [t["txid"] for t in chain] for x in reversed(chain): # Check that getmempoolentry is consistent with getrawmempool entry = self.nodes[0].getmempoolentry(x) assert_equal(entry, mempool[x]) # Check that gettxspendingprevout is consistent with getrawmempool - witnesstx = self.nodes[0].gettransaction(txid=x, verbose=True)['decoded'] + witnesstx = self.nodes[0].getrawtransaction(txid=x, verbose=True) for tx_in in witnesstx["vin"]: spending_result = self.nodes[0].gettxspendingprevout([ {'txid' : tx_in["txid"], 'vout' : tx_in["vout"]} ]) assert_equal(spending_result, [ {'txid' : tx_in["txid"], 'vout' : tx_in["vout"], 'spendingtxid' : x} ]) @@ -193,9 +186,6 @@ class MempoolPackagesTest(BitcoinTestFramework): descendant_fees += entry['fees']['base'] assert_equal(entry['fees']['descendant'], descendant_fees + Decimal('0.00001')) - # Adding one more transaction on to the chain should fail. - assert_raises_rpc_error(-26, "too-long-mempool-chain", chain_transaction, self.nodes[0], [txid], [vout], value, fee, 1) - # Check that prioritising a tx before it's added to the mempool works # First clear the mempool by mining a block. self.generate(self.nodes[0], 1) @@ -232,28 +222,23 @@ class MempoolPackagesTest(BitcoinTestFramework): # TODO: test ancestor size limits # Now test descendant chain limits - txid = utxo[1]['txid'] - value = utxo[1]['amount'] - vout = utxo[1]['vout'] - transaction_package = [] tx_children = [] # First create one parent tx with 10 children - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [vout], value, fee, 10) - parent_transaction = txid - for i in range(10): - transaction_package.append({'txid': txid, 'vout': i, 'amount': sent_value}) + tx_with_children = self.wallet.send_self_transfer_multi(from_node=self.nodes[0], num_outputs=10) + parent_transaction = tx_with_children["txid"] + transaction_package = tx_with_children["new_utxos"] # Sign and send up to MAX_DESCENDANT transactions chained off the parent tx chain = [] # save sent txs for the purpose of checking node1's mempool later (see below) for _ in range(DEFAULT_DESCENDANT_LIMIT - 1): utxo = transaction_package.pop(0) - (txid, sent_value) = chain_transaction(self.nodes[0], [utxo['txid']], [utxo['vout']], utxo['amount'], fee, 10) + new_tx = self.wallet.send_self_transfer_multi(from_node=self.nodes[0], num_outputs=10, utxos_to_spend=[utxo]) + txid = new_tx["txid"] chain.append(txid) if utxo['txid'] is parent_transaction: tx_children.append(txid) - for j in range(10): - transaction_package.append({'txid': txid, 'vout': j, 'amount': sent_value}) + transaction_package.extend(new_tx["new_utxos"]) mempool = self.nodes[0].getrawmempool(True) assert_equal(mempool[parent_transaction]['descendantcount'], DEFAULT_DESCENDANT_LIMIT) @@ -263,8 +248,8 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(mempool[child]['depends'], [parent_transaction]) # Sending one more chained transaction will fail - utxo = transaction_package.pop(0) - assert_raises_rpc_error(-26, "too-long-mempool-chain", chain_transaction, self.nodes[0], [utxo['txid']], [utxo['vout']], utxo['amount'], fee, 10) + next_hop = self.wallet.create_self_transfer(utxo_to_spend=transaction_package.pop(0))["hex"] + assert_raises_rpc_error(-26, "too-long-mempool-chain", lambda: self.nodes[0].sendrawtransaction(next_hop)) # Check that node1's mempool is as expected, containing: # - txs from previous ancestor test (-> custom ancestor limit) @@ -304,42 +289,19 @@ class MempoolPackagesTest(BitcoinTestFramework): # last block. # Create tx0 with 2 outputs - utxo = self.nodes[0].listunspent() - txid = utxo[0]['txid'] - value = utxo[0]['amount'] - vout = utxo[0]['vout'] - - send_value = (value - fee) / 2 - inputs = [ {'txid' : txid, 'vout' : vout} ] - outputs = {} - for _ in range(2): - outputs[self.nodes[0].getnewaddress()] = send_value - rawtx = self.nodes[0].createrawtransaction(inputs, outputs) - signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx) - txid = self.nodes[0].sendrawtransaction(signedtx['hex']) - tx0_id = txid - value = send_value + tx0 = self.wallet.send_self_transfer_multi(from_node=self.nodes[0], num_outputs=2) # Create tx1 - tx1_id, _ = chain_transaction(self.nodes[0], [tx0_id], [0], value, fee, 1) + tx1 = self.wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=tx0["new_utxos"][0]) # Create tx2-7 - vout = 1 - txid = tx0_id - for _ in range(6): - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [vout], value, fee, 1) - vout = 0 - value = sent_value + tx7 = self.wallet.send_self_transfer_chain(from_node=self.nodes[0], utxo_to_spend=tx0["new_utxos"][1], chain_length=6)[-1] # Mine these in a block self.generate(self.nodes[0], 1) # Now generate tx8, with a big fee - inputs = [ {'txid' : tx1_id, 'vout': 0}, {'txid' : txid, 'vout': 0} ] - outputs = { self.nodes[0].getnewaddress() : send_value + value - 4*fee } - rawtx = self.nodes[0].createrawtransaction(inputs, outputs) - signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx) - txid = self.nodes[0].sendrawtransaction(signedtx['hex']) + self.wallet.send_self_transfer_multi(from_node=self.nodes[0], utxos_to_spend=[tx1["new_utxo"], tx7["new_utxo"]], fee_per_output=40000) self.sync_mempools() # Now try to disconnect the tip on each node... diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index dca4a71bd0..f818801136 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -59,7 +59,6 @@ class MempoolPersistTest(BitcoinTestFramework): def run_test(self): self.mini_wallet = MiniWallet(self.nodes[2]) - self.mini_wallet.rescan_utxos() if self.is_sqlite_compiled(): self.nodes[2].createwallet( wallet_name="watch", diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index 83ab65f1ba..3a5bc1ebcd 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -31,7 +31,6 @@ class MempoolCoinbaseTest(BitcoinTestFramework): self.log.info("Add 4 coinbase utxos to the miniwallet") # Block 76 contains the first spendable coinbase txs. first_block = 76 - wallet.rescan_utxos() # Three scenarios for re-orging coinbase spends in the memory pool: # 1. Direct coinbase spend : spend_1 diff --git a/test/functional/mempool_resurrect.py b/test/functional/mempool_resurrect.py index 3e610d02ac..c10052372d 100755 --- a/test/functional/mempool_resurrect.py +++ b/test/functional/mempool_resurrect.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test resurrection of mined transactions when the blockchain is re-organized.""" -from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal from test_framework.wallet import MiniWallet @@ -13,16 +12,11 @@ from test_framework.wallet import MiniWallet class MempoolCoinbaseTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.setup_clean_chain = True def run_test(self): node = self.nodes[0] wallet = MiniWallet(node) - # Add enough mature utxos to the wallet so that all txs spend confirmed coins - self.generate(wallet, 3) - self.generate(node, COINBASE_MATURITY) - # Spend block 1/2/3's coinbase transactions # Mine a block # Create three more transactions, spending the spends diff --git a/test/functional/mempool_spend_coinbase.py b/test/functional/mempool_spend_coinbase.py index bca512445c..a7cb2ba602 100755 --- a/test/functional/mempool_spend_coinbase.py +++ b/test/functional/mempool_spend_coinbase.py @@ -28,7 +28,6 @@ class MempoolSpendCoinbaseTest(BitcoinTestFramework): chain_height = 198 self.nodes[0].invalidateblock(self.nodes[0].getblockhash(chain_height + 1)) assert_equal(chain_height, self.nodes[0].getblockcount()) - wallet.rescan_utxos() # Coinbase at height chain_height-100+1 ok in mempool, should # get mined. Coinbase at height chain_height-100+2 is diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index 1b0097d578..12de750731 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -23,7 +23,6 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() self.test_broadcast() self.test_txn_removal() diff --git a/test/functional/mining_getblocktemplate_longpoll.py b/test/functional/mining_getblocktemplate_longpoll.py index e928ee4936..ec492f9e72 100755 --- a/test/functional/mining_getblocktemplate_longpoll.py +++ b/test/functional/mining_getblocktemplate_longpoll.py @@ -8,7 +8,6 @@ from decimal import Decimal import random import threading -from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import get_rpc_proxy from test_framework.wallet import MiniWallet @@ -62,9 +61,6 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): thr.join(5) # wait 5 seconds or until thread exits assert not thr.is_alive() - # Add enough mature utxos to the wallets, so that all txs spend confirmed coins - self.generate(self.nodes[0], COINBASE_MATURITY) - self.log.info("Test that introducing a new transaction into the mempool will terminate the longpoll") thr = LongpollThread(self.nodes[0]) thr.start() diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index 4a54f82b58..a4481c15a0 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -106,7 +106,6 @@ class PrioritiseTransactionTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() # Test `prioritisetransaction` required parameters assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction) diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index fa9ddf7ebe..110a1bd03f 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -20,8 +20,6 @@ class P2PBlocksOnly(BitcoinTestFramework): def run_test(self): self.miniwallet = MiniWallet(self.nodes[0]) - # Add enough mature utxos to the wallet, so that all txs spend confirmed coins - self.miniwallet.rescan_utxos() self.blocksonly_mode_tests() self.blocks_relay_conn_tests() diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 7169cc8937..b2f0659eda 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -107,7 +107,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("disconnectnode: fail to disconnect when calling with address and nodeid") address1 = self.nodes[0].getpeerinfo()[0]['addr'] - node1 = self.nodes[0].getpeerinfo()[0]['addr'] + node1 = self.nodes[0].getpeerinfo()[0]["id"] assert_raises_rpc_error(-32602, "Only one of address and nodeid should be provided.", self.nodes[0].disconnectnode, address=address1, nodeid=node1) self.log.info("disconnectnode: fail to disconnect when calling with junk address") diff --git a/test/functional/p2p_eviction.py b/test/functional/p2p_eviction.py index 1f4797a89d..8b31dfa549 100755 --- a/test/functional/p2p_eviction.py +++ b/test/functional/p2p_eviction.py @@ -12,22 +12,23 @@ address/netgroup since in the current framework, all peers are connecting from the same local address. See Issue #14210 for more info. Therefore, this test is limited to the remaining protection criteria. """ - import time from test_framework.blocktools import ( - COINBASE_MATURITY, create_block, create_coinbase, ) from test_framework.messages import ( msg_pong, msg_tx, - tx_from_hex, ) -from test_framework.p2p import P2PDataStore, P2PInterface +from test_framework.p2p import ( + P2PDataStore, + P2PInterface, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal +from test_framework.wallet import MiniWallet class SlowP2PDataStore(P2PDataStore): @@ -35,14 +36,15 @@ class SlowP2PDataStore(P2PDataStore): time.sleep(0.1) self.send_message(msg_pong(message.nonce)) + class SlowP2PInterface(P2PInterface): def on_ping(self, message): time.sleep(0.1) self.send_message(msg_pong(message.nonce)) + class P2PEvict(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = True self.num_nodes = 1 # The choice of maxconnections=32 results in a maximum of 21 inbound connections # (32 - 10 outbound - 1 feeler). 20 inbound peers are protected from eviction: @@ -53,7 +55,7 @@ class P2PEvict(BitcoinTestFramework): protected_peers = set() # peers that we expect to be protected from eviction current_peer = -1 node = self.nodes[0] - self.generatetoaddress(node, COINBASE_MATURITY + 1, node.get_deterministic_priv_key().address) + self.wallet = MiniWallet(node) self.log.info("Create 4 peers and protect them from eviction by sending us a block") for _ in range(4): @@ -79,21 +81,8 @@ class P2PEvict(BitcoinTestFramework): current_peer += 1 txpeer.sync_with_ping() - prevtx = node.getblock(node.getblockhash(i + 1), 2)['tx'][0] - rawtx = node.createrawtransaction( - inputs=[{'txid': prevtx['txid'], 'vout': 0}], - outputs=[{node.get_deterministic_priv_key().address: 50 - 0.00125}], - ) - sigtx = node.signrawtransactionwithkey( - hexstring=rawtx, - privkeys=[node.get_deterministic_priv_key().key], - prevtxs=[{ - 'txid': prevtx['txid'], - 'vout': 0, - 'scriptPubKey': prevtx['vout'][0]['scriptPubKey']['hex'], - }], - )['hex'] - txpeer.send_message(msg_tx(tx_from_hex(sigtx))) + tx = self.wallet.create_self_transfer()['tx'] + txpeer.send_message(msg_tx(tx)) protected_peers.add(current_peer) self.log.info("Create 8 peers and protect them from eviction by having faster pings") @@ -133,5 +122,6 @@ class P2PEvict(BitcoinTestFramework): self.log.debug("{} protected peers: {}".format(len(protected_peers), protected_peers)) assert evicted_peers[0] not in protected_peers + if __name__ == '__main__': P2PEvict().main() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index b65e927d5b..6b03cdf877 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -6,7 +6,6 @@ from decimal import Decimal -from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import MSG_TX, MSG_WTX, msg_feefilter from test_framework.p2p import P2PInterface, p2p_lock from test_framework.test_framework import BitcoinTestFramework @@ -80,9 +79,6 @@ class FeeFilterTest(BitcoinTestFramework): node1 = self.nodes[1] node0 = self.nodes[0] miniwallet = MiniWallet(node1) - # Add enough mature utxos to the wallet, so that all txs spend confirmed coins - self.generate(miniwallet, 5) - self.generate(node1, COINBASE_MATURITY) conn = self.nodes[0].add_p2p_connection(TestP2PConn()) diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 3cf92b0316..b3e68ca536 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -214,7 +214,6 @@ class FilterTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() filter_peer = self.nodes[0].add_p2p_connection(P2PBloomFilter()) self.log.info('Test filter size limits') diff --git a/test/functional/p2p_headers_sync_with_minchainwork.py b/test/functional/p2p_headers_sync_with_minchainwork.py index b07077c668..832fd7e0e9 100755 --- a/test/functional/p2p_headers_sync_with_minchainwork.py +++ b/test/functional/p2p_headers_sync_with_minchainwork.py @@ -27,6 +27,7 @@ NODE2_BLOCKS_REQUIRED = 2047 class RejectLowDifficultyHeadersTest(BitcoinTestFramework): def set_test_params(self): + self.rpc_timeout *= 4 # To avoid timeout when generating BLOCKS_TO_MINE self.setup_clean_chain = True self.num_nodes = 4 # Node0 has no required chainwork; node1 requires 15 blocks on top of the genesis block; node2 requires 2047 diff --git a/test/functional/p2p_ibd_stalling.py b/test/functional/p2p_ibd_stalling.py new file mode 100755 index 0000000000..aca98ceb3f --- /dev/null +++ b/test/functional/p2p_ibd_stalling.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022- The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test stalling logic during IBD +""" + +import time + +from test_framework.blocktools import ( + create_block, + create_coinbase +) +from test_framework.messages import ( + MSG_BLOCK, + MSG_TYPE_MASK, +) +from test_framework.p2p import ( + CBlockHeader, + msg_block, + msg_headers, + P2PDataStore, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) + + +class P2PStaller(P2PDataStore): + def __init__(self, stall_block): + self.stall_block = stall_block + super().__init__() + + def on_getdata(self, message): + for inv in message.inv: + self.getdata_requests.append(inv.hash) + if (inv.type & MSG_TYPE_MASK) == MSG_BLOCK: + if (inv.hash != self.stall_block): + self.send_message(msg_block(self.block_store[inv.hash])) + + def on_getheaders(self, message): + pass + + +class P2PIBDStallingTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + NUM_BLOCKS = 1025 + NUM_PEERS = 4 + node = self.nodes[0] + tip = int(node.getbestblockhash(), 16) + blocks = [] + height = 1 + block_time = node.getblock(node.getbestblockhash())['time'] + 1 + self.log.info("Prepare blocks without sending them to the node") + block_dict = {} + for _ in range(NUM_BLOCKS): + blocks.append(create_block(tip, create_coinbase(height), block_time)) + blocks[-1].solve() + tip = blocks[-1].sha256 + block_time += 1 + height += 1 + block_dict[blocks[-1].sha256] = blocks[-1] + stall_block = blocks[0].sha256 + + headers_message = msg_headers() + headers_message.headers = [CBlockHeader(b) for b in blocks[:NUM_BLOCKS-1]] + peers = [] + + self.log.info("Check that a staller does not get disconnected if the 1024 block lookahead buffer is filled") + for id in range(NUM_PEERS): + peers.append(node.add_outbound_p2p_connection(P2PStaller(stall_block), p2p_idx=id, connection_type="outbound-full-relay")) + peers[-1].block_store = block_dict + peers[-1].send_message(headers_message) + + # Need to wait until 1023 blocks are received - the magic total bytes number is a workaround in lack of an rpc + # returning the number of downloaded (but not connected) blocks. + self.wait_until(lambda: self.total_bytes_recv_for_blocks() == 172761) + + self.all_sync_send_with_ping(peers) + # If there was a peer marked for stalling, it would get disconnected + self.mocktime = int(time.time()) + 3 + node.setmocktime(self.mocktime) + self.all_sync_send_with_ping(peers) + assert_equal(node.num_test_p2p_connections(), NUM_PEERS) + + self.log.info("Check that increasing the window beyond 1024 blocks triggers stalling logic") + headers_message.headers = [CBlockHeader(b) for b in blocks] + with node.assert_debug_log(expected_msgs=['Stall started']): + for p in peers: + p.send_message(headers_message) + self.all_sync_send_with_ping(peers) + + self.log.info("Check that the stalling peer is disconnected after 2 seconds") + self.mocktime += 3 + node.setmocktime(self.mocktime) + peers[0].wait_for_disconnect() + assert_equal(node.num_test_p2p_connections(), NUM_PEERS - 1) + self.wait_until(lambda: self.is_block_requested(peers, stall_block)) + # Make sure that SendMessages() is invoked, which assigns the missing block + # to another peer and starts the stalling logic for them + self.all_sync_send_with_ping(peers) + + self.log.info("Check that the stalling timeout gets doubled to 4 seconds for the next staller") + # No disconnect after just 3 seconds + self.mocktime += 3 + node.setmocktime(self.mocktime) + self.all_sync_send_with_ping(peers) + assert_equal(node.num_test_p2p_connections(), NUM_PEERS - 1) + + self.mocktime += 2 + node.setmocktime(self.mocktime) + self.wait_until(lambda: sum(x.is_connected for x in node.p2ps) == NUM_PEERS - 2) + self.wait_until(lambda: self.is_block_requested(peers, stall_block)) + self.all_sync_send_with_ping(peers) + + self.log.info("Check that the stalling timeout gets doubled to 8 seconds for the next staller") + # No disconnect after just 7 seconds + self.mocktime += 7 + node.setmocktime(self.mocktime) + self.all_sync_send_with_ping(peers) + assert_equal(node.num_test_p2p_connections(), NUM_PEERS - 2) + + self.mocktime += 2 + node.setmocktime(self.mocktime) + self.wait_until(lambda: sum(x.is_connected for x in node.p2ps) == NUM_PEERS - 3) + self.wait_until(lambda: self.is_block_requested(peers, stall_block)) + self.all_sync_send_with_ping(peers) + + self.log.info("Provide the withheld block and check that stalling timeout gets reduced back to 2 seconds") + with node.assert_debug_log(expected_msgs=['Decreased stalling timeout to 2 seconds']): + for p in peers: + if p.is_connected and (stall_block in p.getdata_requests): + p.send_message(msg_block(block_dict[stall_block])) + + self.log.info("Check that all outstanding blocks get connected") + self.wait_until(lambda: node.getblockcount() == NUM_BLOCKS) + + def total_bytes_recv_for_blocks(self): + total = 0 + for info in self.nodes[0].getpeerinfo(): + if ("block" in info["bytesrecv_per_msg"].keys()): + total += info["bytesrecv_per_msg"]["block"] + return total + + def all_sync_send_with_ping(self, peers): + for p in peers: + if p.is_connected: + p.sync_send_with_ping() + + def is_block_requested(self, peers, hash): + for p in peers: + if p.is_connected and (hash in p.getdata_requests): + return True + return False + + +if __name__ == '__main__': + P2PIBDStallingTest().main() diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py index 4c064b359e..6283dd89ac 100755 --- a/test/functional/p2p_leak_tx.py +++ b/test/functional/p2p_leak_tx.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that we don't leak txs to inbound peers that we haven't yet announced to""" -from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import msg_getdata, CInv, MSG_TX from test_framework.p2p import p2p_lock, P2PDataStore from test_framework.test_framework import BitcoinTestFramework @@ -26,9 +25,6 @@ class P2PLeakTxTest(BitcoinTestFramework): def run_test(self): gen_node = self.nodes[0] # The block and tx generating node miniwallet = MiniWallet(gen_node) - # Add enough mature utxos to the wallet, so that all txs spend confirmed coins - self.generate(miniwallet, 1) - self.generate(gen_node, COINBASE_MATURITY) inbound_peer = self.nodes[0].add_p2p_connection(P2PNode()) # An "attacking" inbound peer diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index f8d3fd919d..f84bbf67e6 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -7,30 +7,26 @@ Test that permissions are correctly calculated and applied """ -from test_framework.address import ADDRESS_BCRT1_P2WSH_OP_TRUE from test_framework.messages import ( - CTxInWitness, - tx_from_hex, + SEQUENCE_FINAL, ) from test_framework.p2p import P2PDataStore -from test_framework.script import ( - CScript, - OP_TRUE, -) from test_framework.test_node import ErrorMatch from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, p2p_port, ) +from test_framework.wallet import MiniWallet class P2PPermissionsTests(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.setup_clean_chain = True def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.check_tx_relay() self.checkpermission( @@ -60,12 +56,12 @@ class P2PPermissionsTests(BitcoinTestFramework): # For this, we need to use whitebind instead of bind # by modifying the configuration file. ip_port = "127.0.0.1:{}".format(p2p_port(1)) - self.replaceinconfig(1, "bind=127.0.0.1", "whitebind=bloomfilter,forcerelay@" + ip_port) + self.nodes[1].replace_in_config([("bind=127.0.0.1", "whitebind=bloomfilter,forcerelay@" + ip_port)]) self.checkpermission( ["-whitelist=noban@127.0.0.1"], # Check parameter interaction forcerelay should activate relay ["noban", "bloomfilter", "forcerelay", "relay", "download"]) - self.replaceinconfig(1, "whitebind=bloomfilter,forcerelay@" + ip_port, "bind=127.0.0.1") + self.nodes[1].replace_in_config([("whitebind=bloomfilter,forcerelay@" + ip_port, "bind=127.0.0.1")]) self.checkpermission( # legacy whitelistrelay should be ignored @@ -94,8 +90,6 @@ class P2PPermissionsTests(BitcoinTestFramework): self.nodes[1].assert_start_raises_init_error(["-whitebind=noban@127.0.0.1", "-bind=127.0.0.1", "-listen=0"], "Cannot set -bind or -whitebind together with -listen=0", match=ErrorMatch.PARTIAL_REGEX) def check_tx_relay(self): - block_op_true = self.nodes[0].getblock(self.generatetoaddress(self.nodes[0], 100, ADDRESS_BCRT1_P2WSH_OP_TRUE)[0]) - self.log.debug("Create a connection from a forcerelay peer that rebroadcasts raw txs") # A test framework p2p connection is needed to send the raw transaction directly. If a full node was used, it could only # rebroadcast via the inv-getdata mechanism. However, even for forcerelay connections, a full node would @@ -104,18 +98,7 @@ class P2PPermissionsTests(BitcoinTestFramework): p2p_rebroadcast_wallet = self.nodes[1].add_p2p_connection(P2PDataStore()) self.log.debug("Send a tx from the wallet initially") - tx = tx_from_hex( - self.nodes[0].createrawtransaction( - inputs=[{ - 'txid': block_op_true['tx'][0], - 'vout': 0, - }], outputs=[{ - ADDRESS_BCRT1_P2WSH_OP_TRUE: 5, - }], - replaceable=False), - ) - tx.wit.vtxinwit = [CTxInWitness()] - tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] + tx = self.wallet.create_self_transfer(sequence=SEQUENCE_FINAL)['tx'] txid = tx.rehash() self.log.debug("Wait until tx is in node[1]'s mempool") @@ -155,12 +138,6 @@ class P2PPermissionsTests(BitcoinTestFramework): if p not in peerinfo['permissions']: raise AssertionError("Expected permissions %r is not granted." % p) - def replaceinconfig(self, nodeid, old, new): - with open(self.nodes[nodeid].bitcoinconf, encoding="utf8") as f: - newText = f.read().replace(old, new) - with open(self.nodes[nodeid].bitcoinconf, 'w', encoding="utf8") as f: - f.write(newText) - if __name__ == '__main__': P2PPermissionsTests().main() diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 7356b8bbb3..0e463c5072 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -5,6 +5,7 @@ """ Test transaction download behavior """ +import time from test_framework.messages import ( CInv, @@ -13,7 +14,6 @@ from test_framework.messages import ( MSG_WTX, msg_inv, msg_notfound, - tx_from_hex, ) from test_framework.p2p import ( P2PInterface, @@ -23,9 +23,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, ) -from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE - -import time +from test_framework.wallet import MiniWallet class TestP2PConn(P2PInterface): @@ -88,19 +86,8 @@ class TxDownloadTest(BitcoinTestFramework): def test_inv_block(self): self.log.info("Generate a transaction on node 0") - tx = self.nodes[0].createrawtransaction( - inputs=[{ # coinbase - "txid": self.nodes[0].getblock(self.nodes[0].getblockhash(1))['tx'][0], - "vout": 0 - }], - outputs={ADDRESS_BCRT1_UNSPENDABLE: 50 - 0.00025}, - ) - tx = self.nodes[0].signrawtransactionwithkey( - hexstring=tx, - privkeys=[self.nodes[0].get_deterministic_priv_key().key], - )['hex'] - ctx = tx_from_hex(tx) - txid = int(ctx.rehash(), 16) + tx = self.wallet.create_self_transfer() + txid = int(tx['txid'], 16) self.log.info( "Announce the transaction to all nodes from all {} incoming peers, but never send it".format(NUM_INBOUND)) @@ -109,7 +96,7 @@ class TxDownloadTest(BitcoinTestFramework): p.send_and_ping(msg) self.log.info("Put the tx in node 0's mempool") - self.nodes[0].sendrawtransaction(tx) + self.nodes[0].sendrawtransaction(tx['hex']) # Since node 1 is connected outbound to an honest peer (node 0), it # should get the tx within a timeout. (Assuming that node 0 @@ -255,6 +242,8 @@ class TxDownloadTest(BitcoinTestFramework): self.nodes[0].p2ps[0].send_message(msg_notfound(vec=[CInv(MSG_TX, 1)])) def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + # Run tests without mocktime that only need one peer-connection first, to avoid restarting the nodes self.test_expiry_fallback() self.test_disconnect_fallback() diff --git a/test/functional/p2p_tx_privacy.py b/test/functional/p2p_tx_privacy.py index b885ccdf5d..e674f6c3eb 100755 --- a/test/functional/p2p_tx_privacy.py +++ b/test/functional/p2p_tx_privacy.py @@ -53,7 +53,6 @@ class TxPrivacyTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() tx_originator = self.nodes[0].add_p2p_connection(P2PInterface()) spy = self.nodes[0].add_p2p_connection(P2PTxSpy(), wait_for_verack=False) diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 19c73eebf0..7a0cedb1f5 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -25,6 +25,7 @@ from decimal import Decimal import http.client import os import subprocess +import textwrap from test_framework.blocktools import ( MAX_FUTURE_BLOCK_TIME, @@ -429,6 +430,17 @@ class BlockchainTest(BitcoinTestFramework): def _test_getnetworkhashps(self): self.log.info("Test getnetworkhashps") hashes_per_second = self.nodes[0].getnetworkhashps() + assert_raises_rpc_error( + -3, + textwrap.dedent(""" + Wrong type passed: + { + "Position 1 (nblocks)": "JSON value of type string is not of expected type number", + "Position 2 (height)": "JSON value of type array is not of expected type number" + } + """).strip(), + lambda: self.nodes[0].getnetworkhashps("a", []), + ) # This should be 2 hashes every 10 minutes or 1/300 assert abs(hashes_per_second * 300 - 1) < 0.0001 diff --git a/test/functional/rpc_generate.py b/test/functional/rpc_generate.py index 89b410e37e..8948ccb48d 100755 --- a/test/functional/rpc_generate.py +++ b/test/functional/rpc_generate.py @@ -28,7 +28,6 @@ class RPCGenerateTest(BitcoinTestFramework): def test_generateblock(self): node = self.nodes[0] miniwallet = MiniWallet(node) - miniwallet.rescan_utxos() self.log.info('Generate an empty block to address') address = miniwallet.get_address() diff --git a/test/functional/rpc_invalid_address_message.py b/test/functional/rpc_invalid_address_message.py index 452f857a44..0c29efb85a 100755 --- a/test/functional/rpc_invalid_address_message.py +++ b/test/functional/rpc_invalid_address_message.py @@ -95,15 +95,19 @@ class InvalidAddressErrorMessageTest(BitcoinTestFramework): self.check_invalid(INVALID_ADDRESS, 'Not a valid Bech32 or Base58 encoding') self.check_invalid(INVALID_ADDRESS_2, 'Not a valid Bech32 or Base58 encoding') + node = self.nodes[0] + + # Missing arg returns the help text + assert_raises_rpc_error(-1, "Return information about the given bitcoin address.", node.validateaddress) + # Explicit None is not allowed for required parameters + assert_raises_rpc_error(-3, "JSON value of type null is not of expected type string", node.validateaddress, None) + def test_getaddressinfo(self): node = self.nodes[0] assert_raises_rpc_error(-5, "Invalid Bech32 address data size", node.getaddressinfo, BECH32_INVALID_SIZE) - assert_raises_rpc_error(-5, "Not a valid Bech32 or Base58 encoding", node.getaddressinfo, BECH32_INVALID_PREFIX) - assert_raises_rpc_error(-5, "Invalid prefix for Base58-encoded address", node.getaddressinfo, BASE58_INVALID_PREFIX) - assert_raises_rpc_error(-5, "Not a valid Bech32 or Base58 encoding", node.getaddressinfo, INVALID_ADDRESS) def run_test(self): diff --git a/test/functional/rpc_mempool_info.py b/test/functional/rpc_mempool_info.py index ae9c6572cf..246af22e50 100755 --- a/test/functional/rpc_mempool_info.py +++ b/test/functional/rpc_mempool_info.py @@ -18,7 +18,6 @@ class RPCMempoolInfoTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() confirmed_utxo = self.wallet.get_utxo() # Create a tree of unconfirmed transactions in the mempool: diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index af8b2ad72b..5fdd5daddf 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -11,7 +11,6 @@ from decimal import Decimal from itertools import product import time -from test_framework.blocktools import COINBASE_MATURITY import test_framework.messages from test_framework.p2p import ( P2PInterface, @@ -43,7 +42,6 @@ def assert_net_servicesnames(servicesflag, servicenames): class NetTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = True self.num_nodes = 2 self.extra_args = [["-minrelaytxfee=0.00001000"], ["-minrelaytxfee=0.00000500"]] self.supports_cli = False @@ -51,9 +49,6 @@ class NetTest(BitcoinTestFramework): def run_test(self): # We need miniwallet to make a transaction self.wallet = MiniWallet(self.nodes[0]) - self.generate(self.wallet, 1) - # Get out of IBD for the minfeefilter and getpeerinfo tests. - self.generate(self.nodes[0], COINBASE_MATURITY + 1) # By default, the test framework sets up an addnode connection from # node 1 --> node0. By connecting node0 --> node 1, we're left with diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index 10388399ad..6cb9760b3d 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -128,8 +128,8 @@ class RPCPackagesTest(BitcoinTestFramework): node = self.nodes[0] chain = self.wallet.create_self_transfer_chain(chain_length=25) - chain_hex = chain["chain_hex"] - chain_txns = chain["chain_txns"] + chain_hex = [t["hex"] for t in chain] + chain_txns = [t["tx"] for t in chain] self.log.info("Check that testmempoolaccept requires packages to be sorted by dependency") assert_equal(node.testmempoolaccept(rawtxs=chain_hex[::-1]), @@ -374,7 +374,7 @@ class RPCPackagesTest(BitcoinTestFramework): self.log.info("Submitpackage only allows packages of 1 child with its parents") # Chain of 3 transactions has too many generations - chain_hex = self.wallet.create_self_transfer_chain(chain_length=25)["chain_hex"] + chain_hex = [t["hex"] for t in self.wallet.create_self_transfer_chain(chain_length=25)] assert_raises_rpc_error(-25, "not-child-with-parents", node.submitpackage, chain_hex) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index a50e0fb244..58a80e37a2 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -36,6 +36,7 @@ from test_framework.util import ( assert_approx, assert_equal, assert_greater_than, + assert_greater_than_or_equal, assert_raises_rpc_error, find_output, find_vout_for_address, @@ -106,6 +107,65 @@ class PSBTTest(BitcoinTestFramework): self.connect_nodes(0, 1) self.connect_nodes(0, 2) + def test_input_confs_control(self): + self.nodes[0].createwallet("minconf") + wallet = self.nodes[0].get_wallet_rpc("minconf") + + # Fund the wallet with different chain heights + for _ in range(2): + self.nodes[1].sendmany("", {wallet.getnewaddress():1, wallet.getnewaddress():1}) + self.generate(self.nodes[1], 1) + + unconfirmed_txid = wallet.sendtoaddress(wallet.getnewaddress(), 0.5) + + self.log.info("Crafting PSBT using an unconfirmed input") + target_address = self.nodes[1].getnewaddress() + psbtx1 = wallet.walletcreatefundedpsbt([], {target_address: 0.1}, 0, {'fee_rate': 1, 'maxconf': 0})['psbt'] + + # Make sure we only had the one input + tx1_inputs = self.nodes[0].decodepsbt(psbtx1)['tx']['vin'] + assert_equal(len(tx1_inputs), 1) + + utxo1 = tx1_inputs[0] + assert_equal(unconfirmed_txid, utxo1['txid']) + + signed_tx1 = wallet.walletprocesspsbt(psbtx1)['psbt'] + final_tx1 = wallet.finalizepsbt(signed_tx1)['hex'] + txid1 = self.nodes[0].sendrawtransaction(final_tx1) + + mempool = self.nodes[0].getrawmempool() + assert txid1 in mempool + + self.log.info("Fail to craft a new PSBT that sends more funds with add_inputs = False") + assert_raises_rpc_error(-4, "The preselected coins total amount does not cover the transaction target. Please allow other inputs to be automatically selected or include more coins manually", wallet.walletcreatefundedpsbt, [{'txid': utxo1['txid'], 'vout': utxo1['vout']}], {target_address: 1}, 0, {'add_inputs': False}) + + self.log.info("Fail to craft a new PSBT with minconf above highest one") + assert_raises_rpc_error(-4, "Insufficient funds", wallet.walletcreatefundedpsbt, [{'txid': utxo1['txid'], 'vout': utxo1['vout']}], {target_address: 1}, 0, {'add_inputs': True, 'minconf': 3, 'fee_rate': 10}) + + self.log.info("Fail to broadcast a new PSBT with maxconf 0 due to BIP125 rules to verify it actually chose unconfirmed outputs") + psbt_invalid = wallet.walletcreatefundedpsbt([{'txid': utxo1['txid'], 'vout': utxo1['vout']}], {target_address: 1}, 0, {'add_inputs': True, 'maxconf': 0, 'fee_rate': 10})['psbt'] + signed_invalid = wallet.walletprocesspsbt(psbt_invalid)['psbt'] + final_invalid = wallet.finalizepsbt(signed_invalid)['hex'] + assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, final_invalid) + + self.log.info("Craft a replacement adding inputs with highest confs possible") + psbtx2 = wallet.walletcreatefundedpsbt([{'txid': utxo1['txid'], 'vout': utxo1['vout']}], {target_address: 1}, 0, {'add_inputs': True, 'minconf': 2, 'fee_rate': 10})['psbt'] + tx2_inputs = self.nodes[0].decodepsbt(psbtx2)['tx']['vin'] + assert_greater_than_or_equal(len(tx2_inputs), 2) + for vin in tx2_inputs: + if vin['txid'] != unconfirmed_txid: + assert_greater_than_or_equal(self.nodes[0].gettxout(vin['txid'], vin['vout'])['confirmations'], 2) + + signed_tx2 = wallet.walletprocesspsbt(psbtx2)['psbt'] + final_tx2 = wallet.finalizepsbt(signed_tx2)['hex'] + txid2 = self.nodes[0].sendrawtransaction(final_tx2) + + mempool = self.nodes[0].getrawmempool() + assert txid1 not in mempool + assert txid2 in mempool + + wallet.unloadwallet() + def assert_change_type(self, psbtx, expected_type): """Assert that the given PSBT has a change output with the given type.""" @@ -514,6 +574,8 @@ class PSBTTest(BitcoinTestFramework): # TODO: Re-enable this for segwit v1 # self.test_utxo_conversion() + self.test_input_confs_control() + # Test that psbts with p2pkh outputs are created properly p2pkh = self.nodes[0].getnewaddress(address_type='legacy') psbt = self.nodes[1].walletcreatefundedpsbt([], [{p2pkh : 1}], 0, {"includeWatching" : True}, True) diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 1c91ab6f5f..cdec4b2a85 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -16,7 +16,6 @@ from collections import OrderedDict from decimal import Decimal from itertools import product -from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import ( MAX_BIP125_RBF_SEQUENCE, CTransaction, @@ -59,7 +58,6 @@ class RawTransactionsTest(BitcoinTestFramework): self.add_wallet_options(parser, descriptors=False) def set_test_params(self): - self.setup_clean_chain = True self.num_nodes = 3 self.extra_args = [ ["-txindex"], @@ -77,9 +75,6 @@ class RawTransactionsTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.log.info("Prepare some coins for multiple *rawtransaction commands") - self.generate(self.wallet, 10) - self.generate(self.nodes[0], COINBASE_MATURITY + 1) self.getrawtransaction_tests() self.getrawtransaction_verbosity_tests() diff --git a/test/functional/rpc_scanblocks.py b/test/functional/rpc_scanblocks.py index 9a00518150..126e95362b 100755 --- a/test/functional/rpc_scanblocks.py +++ b/test/functional/rpc_scanblocks.py @@ -27,7 +27,6 @@ class ScanblocksTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] wallet = MiniWallet(node) - wallet.rescan_utxos() # send 1.0, mempool only _, spk_1, addr_1 = getnewdestination() @@ -62,6 +61,12 @@ class ScanblocksTest(BitcoinTestFramework): # make sure the blockhash is present when using the first mined block as start_height assert blockhash in node.scanblocks( "start", [f"addr({addr_1})"], height)['relevant_blocks'] + for v in [False, True]: + assert blockhash in node.scanblocks( + action="start", + scanobjects=[f"addr({addr_1})"], + start_height=height, + options={"filter_false_positives": v})['relevant_blocks'] # also test the stop height assert blockhash in node.scanblocks( @@ -94,8 +99,11 @@ class ScanblocksTest(BitcoinTestFramework): assert genesis_blockhash in node.scanblocks( "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0)['relevant_blocks'] - # TODO: after an "accurate" mode for scanblocks is implemented (e.g. PR #26325) - # check here that it filters out the false-positive + # check that the filter_false_positives option works + assert genesis_blockhash in node.scanblocks( + "start", [{"desc": f"raw({genesis_coinbase_spk.hex()})"}], 0, 0, "basic", {"filter_false_positives": True})['relevant_blocks'] + assert genesis_blockhash not in node.scanblocks( + "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0, "basic", {"filter_false_positives": True})['relevant_blocks'] # test node with disabled blockfilterindex assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index af3e7a6d19..507a4f48e5 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -31,7 +31,6 @@ class ScantxoutsetTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() self.log.info("Test if we find coinbase outputs.") assert_equal(sum(u["coinbase"] for u in self.nodes[0].scantxoutset("start", [self.wallet.get_descriptor()])["unspents"]), 49) diff --git a/test/functional/rpc_txoutproof.py b/test/functional/rpc_txoutproof.py index d04d05962f..60b7ce8d20 100755 --- a/test/functional/rpc_txoutproof.py +++ b/test/functional/rpc_txoutproof.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test gettxoutproof and verifytxoutproof RPCs.""" -from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import ( CMerkleBlock, from_hex, @@ -20,7 +19,6 @@ from test_framework.wallet import MiniWallet class MerkleBlockTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.setup_clean_chain = True self.extra_args = [ [], ["-txindex"], @@ -28,12 +26,9 @@ class MerkleBlockTest(BitcoinTestFramework): def run_test(self): miniwallet = MiniWallet(self.nodes[0]) - # Add enough mature utxos to the wallet, so that all txs spend confirmed coins - self.generate(miniwallet, 5) - self.generate(self.nodes[0], COINBASE_MATURITY) chain_height = self.nodes[1].getblockcount() - assert_equal(chain_height, 105) + assert_equal(chain_height, 200) txid1 = miniwallet.send_self_transfer(from_node=self.nodes[0])['txid'] txid2 = miniwallet.send_self_transfer(from_node=self.nodes[0])['txid'] diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index 560f226469..8cc3ec401e 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -53,13 +53,13 @@ class HTTPBasicsTest(BitcoinTestFramework): # Generate RPCAUTH with specified password self.rt2password = "8/F3uMDw4KSEbw96U3CA1C4X05dkHDN2BPFjTgZW4KI=" - p = subprocess.Popen([sys.executable, gen_rpcauth, 'rt2', self.rt2password], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.Popen([sys.executable, gen_rpcauth, 'rt2', self.rt2password], stdout=subprocess.PIPE, text=True) lines = p.stdout.read().splitlines() rpcauth2 = lines[1] # Generate RPCAUTH without specifying password self.user = ''.join(SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(10)) - p = subprocess.Popen([sys.executable, gen_rpcauth, self.user], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.Popen([sys.executable, gen_rpcauth, self.user], stdout=subprocess.PIPE, text=True) lines = p.stdout.read().splitlines() rpcauth3 = lines[1] self.password = lines[3] diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index f345bf02db..443cae86a1 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -597,6 +597,13 @@ class CScript(bytes): lastOpcode = opcode return n + def IsWitnessProgram(self): + """A witness program is any valid CScript that consists of a 1-byte + push opcode followed by a data push between 2 and 40 bytes.""" + return ((4 <= len(self) <= 42) and + (self[0] == OP_0 or (OP_1 <= self[0] <= OP_16)) and + (self[1] + 2 == len(self))) + SIGHASH_DEFAULT = 0 # Taproot-only default, semantics same as SIGHASH_ALL SIGHASH_ALL = 1 diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index f7dd4551c8..6ff4e4ee54 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -533,11 +533,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.nodes.append(test_node_i) if not test_node_i.version_is_at_least(170000): # adjust conf for pre 17 - conf_file = test_node_i.bitcoinconf - with open(conf_file, 'r', encoding='utf8') as conf: - conf_data = conf.read() - with open(conf_file, 'w', encoding='utf8') as conf: - conf.write(conf_data.replace('[regtest]', '')) + test_node_i.replace_in_config([('[regtest]', '')]) def start_node(self, i, *args, **kwargs): """Start a bitcoind""" @@ -608,6 +604,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.wait_until(lambda: sum(peer['version'] != 0 for peer in to_connection.getpeerinfo()) == to_num_peers) self.wait_until(lambda: sum(peer['bytesrecv_per_msg'].pop('verack', 0) == 24 for peer in from_connection.getpeerinfo()) == from_num_peers) self.wait_until(lambda: sum(peer['bytesrecv_per_msg'].pop('verack', 0) == 24 for peer in to_connection.getpeerinfo()) == to_num_peers) + # The message bytes are counted before processing the message, so make + # sure it was fully processed by waiting for a ping. + self.wait_until(lambda: sum(peer["bytesrecv_per_msg"].pop("pong", 0) >= 32 for peer in from_connection.getpeerinfo()) == from_num_peers) + self.wait_until(lambda: sum(peer["bytesrecv_per_msg"].pop("pong", 0) >= 32 for peer in to_connection.getpeerinfo()) == to_num_peers) def disconnect_nodes(self, a, b): def disconnect_nodes_helper(node_a, node_b): @@ -850,6 +850,13 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): except ImportError: raise SkipTest("python3-zmq module not available.") + def skip_if_no_py_sqlite3(self): + """Attempt to import the sqlite3 package and skip the test if the import fails.""" + try: + import sqlite3 # noqa + except ImportError: + raise SkipTest("sqlite3 module not available.") + def skip_if_no_python_bcc(self): """Attempt to import the bcc package and skip the tests if the import fails.""" try: diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 8585972cb3..b515538a1a 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -387,6 +387,21 @@ class TestNode(): def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT): wait_until_helper(self.is_node_stopped, timeout=timeout, timeout_factor=self.timeout_factor) + def replace_in_config(self, replacements): + """ + Perform replacements in the configuration file. + The substitutions are passed as a list of search-replace-tuples, e.g. + [("old", "new"), ("foo", "bar"), ...] + """ + with open(self.bitcoinconf, 'r', encoding='utf8') as conf: + conf_data = conf.read() + for replacement in replacements: + assert_equal(len(replacement), 2) + old, new = replacement[0], replacement[1] + conf_data = conf_data.replace(old, new) + with open(self.bitcoinconf, 'w', encoding='utf8') as conf: + conf.write(conf_data) + @property def chain_path(self) -> Path: return Path(self.datadir) / self.chain @@ -730,7 +745,7 @@ class TestNodeCLI(): p_args += [command] p_args += pos_args + named_args self.log.debug("Running bitcoin-cli {}".format(p_args[2:])) - process = subprocess.Popen(p_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + process = subprocess.Popen(p_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) cli_stdout, cli_stderr = process.communicate(input=self.input) returncode = process.poll() if returncode: diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 1d5108c31a..9048a915b2 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -488,28 +488,6 @@ def find_output(node, txid, amount, *, blockhash=None): raise RuntimeError("find_output txid %s : %s not found" % (txid, str(amount))) -def chain_transaction(node, parent_txids, vouts, value, fee, num_outputs): - """Build and send a transaction that spends the given inputs (specified - by lists of parent_txid:vout each), with the desired total value and fee, - equally divided up to the desired number of outputs. - - Returns a tuple with the txid and the amount sent per output. - """ - send_value = satoshi_round((value - fee)/num_outputs) - inputs = [] - for (txid, vout) in zip(parent_txids, vouts): - inputs.append({'txid' : txid, 'vout' : vout}) - outputs = {} - for _ in range(num_outputs): - outputs[node.getnewaddress()] = send_value - rawtx = node.createrawtransaction(inputs, outputs, 0, True) - signedtx = node.signrawtransactionwithwallet(rawtx) - txid = node.sendrawtransaction(signedtx['hex']) - fulltx = node.getrawtransaction(txid, 1) - assert len(fulltx['vout']) == num_outputs # make sure we didn't generate a change output - return (txid, send_value) - - # Create large OP_RETURN txouts that can be appended to a transaction # to make it large (helper for constructing large transactions). The # total serialized size of the txouts is about 66k vbytes. diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index a72b5e5891..f3253630c4 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -55,6 +55,7 @@ from test_framework.util import ( assert_equal, assert_greater_than_or_equal, ) +from test_framework.blocktools import COINBASE_MATURITY DEFAULT_FEE = Decimal("0.0001") @@ -100,8 +101,15 @@ class MiniWallet: self._address, self._internal_key = create_deterministic_address_bcrt1_p2tr_op_true() self._scriptPubKey = bytes.fromhex(self._test_node.validateaddress(self._address)['scriptPubKey']) - def _create_utxo(self, *, txid, vout, value, height): - return {"txid": txid, "vout": vout, "value": value, "height": height} + # When the pre-mined test framework chain is used, it contains coinbase + # outputs to the MiniWallet's default address in blocks 76-100 + # (see method BitcoinTestFramework._initialize_chain()) + # The MiniWallet needs to rescan_utxos() in order to account + # for those mature UTXOs, so that all txs spend confirmed coins + self.rescan_utxos() + + def _create_utxo(self, *, txid, vout, value, height, coinbase, confirmations): + return {"txid": txid, "vout": vout, "value": value, "height": height, "coinbase": coinbase, "confirmations": confirmations} def _bulk_tx(self, tx, target_weight): """Pad a transaction with extra outputs until it reaches a target weight (or higher). @@ -118,13 +126,25 @@ class MiniWallet: def get_balance(self): return sum(u['value'] for u in self._utxos) - def rescan_utxos(self): + def rescan_utxos(self, *, include_mempool=True): """Drop all utxos and rescan the utxo set""" self._utxos = [] res = self._test_node.scantxoutset(action="start", scanobjects=[self.get_descriptor()]) assert_equal(True, res['success']) for utxo in res['unspents']: - self._utxos.append(self._create_utxo(txid=utxo["txid"], vout=utxo["vout"], value=utxo["amount"], height=utxo["height"])) + self._utxos.append( + self._create_utxo(txid=utxo["txid"], + vout=utxo["vout"], + value=utxo["amount"], + height=utxo["height"], + coinbase=utxo["coinbase"], + confirmations=res["height"] - utxo["height"] + 1)) + if include_mempool: + mempool = self._test_node.getrawmempool(verbose=True) + # Sort tx by ancestor count. See BlockAssembler::SortForBlock in src/node/miner.cpp + sorted_mempool = sorted(mempool.items(), key=lambda item: (item[1]["ancestorcount"], int(item[0], 16))) + for txid, _ in sorted_mempool: + self.scan_tx(self._test_node.getrawtransaction(txid=txid, verbose=True)) def scan_tx(self, tx): """Scan the tx and adjust the internal list of owned utxos""" @@ -139,27 +159,35 @@ class MiniWallet: pass for out in tx['vout']: if out['scriptPubKey']['hex'] == self._scriptPubKey.hex(): - self._utxos.append(self._create_utxo(txid=tx["txid"], vout=out["n"], value=out["value"], height=0)) + self._utxos.append(self._create_utxo(txid=tx["txid"], vout=out["n"], value=out["value"], height=0, coinbase=False, confirmations=0)) def scan_txs(self, txs): for tx in txs: self.scan_tx(tx) def sign_tx(self, tx, fixed_length=True): - """Sign tx that has been created by MiniWallet in P2PK mode""" - assert_equal(self._mode, MiniWalletMode.RAW_P2PK) - (sighash, err) = LegacySignatureHash(CScript(self._scriptPubKey), tx, 0, SIGHASH_ALL) - assert err is None - # for exact fee calculation, create only signatures with fixed size by default (>49.89% probability): - # 65 bytes: high-R val (33 bytes) + low-S val (32 bytes) - # with the DER header/skeleton data of 6 bytes added, this leads to a target size of 71 bytes - der_sig = b'' - while not len(der_sig) == 71: - der_sig = self._priv_key.sign_ecdsa(sighash) - if not fixed_length: - break - tx.vin[0].scriptSig = CScript([der_sig + bytes(bytearray([SIGHASH_ALL]))]) - tx.rehash() + if self._mode == MiniWalletMode.RAW_P2PK: + (sighash, err) = LegacySignatureHash(CScript(self._scriptPubKey), tx, 0, SIGHASH_ALL) + assert err is None + # for exact fee calculation, create only signatures with fixed size by default (>49.89% probability): + # 65 bytes: high-R val (33 bytes) + low-S val (32 bytes) + # with the DER header/skeleton data of 6 bytes added, this leads to a target size of 71 bytes + der_sig = b'' + while not len(der_sig) == 71: + der_sig = self._priv_key.sign_ecdsa(sighash) + if not fixed_length: + break + tx.vin[0].scriptSig = CScript([der_sig + bytes(bytearray([SIGHASH_ALL]))]) + tx.rehash() + elif self._mode == MiniWalletMode.RAW_OP_TRUE: + for i in tx.vin: + i.scriptSig = CScript([OP_NOP] * 43) # pad to identical size + elif self._mode == MiniWalletMode.ADDRESS_OP_TRUE: + tx.wit.vtxinwit = [CTxInWitness()] * len(tx.vin) + for i in tx.wit.vtxinwit: + i.scriptWitness.stack = [CScript([OP_TRUE]), bytes([LEAF_VERSION_TAPSCRIPT]) + self._internal_key] + else: + assert False def generate(self, num_blocks, **kwargs): """Generate blocks with coinbase outputs to the internal address, and call rescan_utxos""" @@ -204,9 +232,13 @@ class MiniWallet: else: return self._utxos[index] - def get_utxos(self, *, mark_as_spent=True): + def get_utxos(self, *, include_immature_coinbase=False, mark_as_spent=True): """Returns the list of all utxos and optionally mark them as spent""" - utxos = deepcopy(self._utxos) + if not include_immature_coinbase: + utxo_filter = filter(lambda utxo: not utxo['coinbase'] or COINBASE_MATURITY <= utxo['confirmations'], self._utxos) + else: + utxo_filter = self._utxos + utxos = deepcopy(list(utxo_filter)) if mark_as_spent: self._utxos = [] return utxos @@ -266,24 +298,17 @@ class MiniWallet: inputs_value_total = sum([int(COIN * utxo['value']) for utxo in utxos_to_spend]) outputs_value_total = inputs_value_total - fee_per_output * num_outputs amount_per_output = amount_per_output or (outputs_value_total // num_outputs) + assert amount_per_output > 0 + outputs_value_total = amount_per_output * num_outputs + fee = Decimal(inputs_value_total - outputs_value_total) / COIN # create tx tx = CTransaction() - tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']), nSequence=seq) for utxo_to_spend,seq in zip(utxos_to_spend, sequence)] + tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']), nSequence=seq) for utxo_to_spend, seq in zip(utxos_to_spend, sequence)] tx.vout = [CTxOut(amount_per_output, bytearray(self._scriptPubKey)) for _ in range(num_outputs)] tx.nLockTime = locktime - if self._mode == MiniWalletMode.RAW_P2PK: - self.sign_tx(tx) - elif self._mode == MiniWalletMode.RAW_OP_TRUE: - for i in range(len(utxos_to_spend)): - tx.vin[i].scriptSig = CScript([OP_NOP] * 43) # pad to identical size - elif self._mode == MiniWalletMode.ADDRESS_OP_TRUE: - tx.wit.vtxinwit = [CTxInWitness()] * len(utxos_to_spend) - for i in range(len(utxos_to_spend)): - tx.wit.vtxinwit[i].scriptWitness.stack = [CScript([OP_TRUE]), bytes([LEAF_VERSION_TAPSCRIPT]) + self._internal_key] - else: - assert False + self.sign_tx(tx) if target_weight: self._bulk_tx(tx, target_weight) @@ -295,8 +320,12 @@ class MiniWallet: vout=i, value=Decimal(tx.vout[i].nValue) / COIN, height=0, + coinbase=False, + confirmations=0, ) for i in range(len(tx.vout))], + "fee": fee, "txid": txid, + "wtxid": tx.getwtxid(), "hex": tx.serialize().hex(), "tx": tx, } @@ -314,52 +343,45 @@ class MiniWallet: else: assert False send_value = utxo_to_spend["value"] - (fee or (fee_rate * vsize / 1000)) - assert send_value > 0 # create tx tx = self.create_self_transfer_multi(utxos_to_spend=[utxo_to_spend], locktime=locktime, sequence=sequence, amount_per_output=int(COIN * send_value), target_weight=target_weight) if not target_weight: assert_equal(tx["tx"].get_vsize(), vsize) + tx["new_utxo"] = tx.pop("new_utxos")[0] - return {"txid": tx["txid"], "wtxid": tx["tx"].getwtxid(), "hex": tx["hex"], "tx": tx["tx"], "new_utxo": tx["new_utxos"][0]} + return tx def sendrawtransaction(self, *, from_node, tx_hex, maxfeerate=0, **kwargs): txid = from_node.sendrawtransaction(hexstring=tx_hex, maxfeerate=maxfeerate, **kwargs) self.scan_tx(from_node.decoderawtransaction(tx_hex)) return txid - def create_self_transfer_chain(self, *, chain_length): + def create_self_transfer_chain(self, *, chain_length, utxo_to_spend=None): """ Create a "chain" of chain_length transactions. The nth transaction in the chain is a child of the n-1th transaction and parent of the n+1th transaction. - - Returns a dic {"chain_hex": chain_hex, "chain_txns" : chain_txns} - - "chain_hex" is a list representing the chain's transactions in hexadecimal. - "chain_txns" is a list representing the chain's transactions in the CTransaction object. """ - chaintip_utxo = self.get_utxo() - chain_hex = [] - chain_txns = [] + chaintip_utxo = utxo_to_spend or self.get_utxo() + chain = [] for _ in range(chain_length): tx = self.create_self_transfer(utxo_to_spend=chaintip_utxo) chaintip_utxo = tx["new_utxo"] - chain_hex.append(tx["hex"]) - chain_txns.append(tx["tx"]) + chain.append(tx) - return {"chain_hex": chain_hex, "chain_txns" : chain_txns} + return chain - def send_self_transfer_chain(self, *, from_node, chain_length, utxo_to_spend=None): + def send_self_transfer_chain(self, *, from_node, **kwargs): """Create and send a "chain" of chain_length transactions. The nth transaction in the chain is a child of the n-1th transaction and parent of the n+1th transaction. - Returns the chaintip (nth) utxo + Returns a list of objects for each tx (see create_self_transfer_multi). """ - chaintip_utxo = utxo_to_spend or self.get_utxo() - for _ in range(chain_length): - chaintip_utxo = self.send_self_transfer(utxo_to_spend=chaintip_utxo, from_node=from_node)["new_utxo"] - return chaintip_utxo + chain = self.create_self_transfer_chain(**kwargs) + for t in chain: + self.sendrawtransaction(from_node=from_node, tx_hex=t["hex"]) + return chain def getnewdestination(address_type='bech32m'): diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 86a9e4cd9a..a301cf9184 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -254,6 +254,7 @@ BASE_SCRIPTS = [ 'wallet_importprunedfunds.py --descriptors', 'p2p_leak_tx.py', 'p2p_eviction.py', + 'p2p_ibd_stalling.py', 'wallet_signmessagewithaddress.py', 'rpc_signmessagewithprivkey.py', 'rpc_generate.py', @@ -318,6 +319,7 @@ BASE_SCRIPTS = [ 'mempool_unbroadcast.py', 'mempool_compatibility.py', 'mempool_accept_wtxid.py', + 'mempool_dust.py', 'rpc_deriveaddresses.py', 'rpc_deriveaddresses.py --usecli', 'p2p_ping.py', @@ -584,7 +586,7 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= combined_logs_args = [sys.executable, os.path.join(tests_dir, 'combine_logs.py'), testdir] if BOLD[0]: combined_logs_args += ['--color'] - combined_logs, _ = subprocess.Popen(combined_logs_args, universal_newlines=True, stdout=subprocess.PIPE).communicate() + combined_logs, _ = subprocess.Popen(combined_logs_args, text=True, stdout=subprocess.PIPE).communicate() print("\n".join(deque(combined_logs.splitlines(), combined_logs_len))) if failfast: @@ -669,7 +671,7 @@ class TestHandler: self.jobs.append((test, time.time(), subprocess.Popen([sys.executable, self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir_arg, - universal_newlines=True, + text=True, stdout=log_stdout, stderr=log_stderr), testdir, diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 4a321c2fc4..a888f93b03 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -37,7 +37,7 @@ class ToolWalletTest(BitcoinTestFramework): if not self.options.descriptors and 'create' in args: default_args.append('-legacy') - return subprocess.Popen([binary] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + return subprocess.Popen([binary] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) def assert_raises_tool_error(self, error, *args): p = self.bitcoin_wallet_process(*args) diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 86cfd4a230..52022a2eee 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -731,6 +731,45 @@ class WalletTest(BitcoinTestFramework): assert_equal(coin_b["parent_descs"][0], multi_b) self.nodes[0].unloadwallet("wo") + self.log.info("Test -spendzeroconfchange") + self.restart_node(0, ["-spendzeroconfchange=0"]) + + # create new wallet and fund it with a confirmed UTXO + self.nodes[0].createwallet(wallet_name="zeroconf", load_on_startup=True) + zeroconf_wallet = self.nodes[0].get_wallet_rpc("zeroconf") + default_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + default_wallet.sendtoaddress(zeroconf_wallet.getnewaddress(), Decimal('1.0')) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) + utxos = zeroconf_wallet.listunspent(minconf=0) + assert_equal(len(utxos), 1) + assert_equal(utxos[0]['confirmations'], 1) + + # spend confirmed UTXO to ourselves + zeroconf_wallet.sendall(recipients=[zeroconf_wallet.getnewaddress()]) + utxos = zeroconf_wallet.listunspent(minconf=0) + assert_equal(len(utxos), 1) + assert_equal(utxos[0]['confirmations'], 0) + # accounts for untrusted pending balance + bal = zeroconf_wallet.getbalances() + assert_equal(bal['mine']['trusted'], 0) + assert_equal(bal['mine']['untrusted_pending'], utxos[0]['amount']) + + # spending an unconfirmed UTXO sent to ourselves should fail + assert_raises_rpc_error(-6, "Insufficient funds", zeroconf_wallet.sendtoaddress, zeroconf_wallet.getnewaddress(), Decimal('0.5')) + + # check that it works again with -spendzeroconfchange set (=default) + self.restart_node(0, ["-spendzeroconfchange=1"]) + zeroconf_wallet = self.nodes[0].get_wallet_rpc("zeroconf") + utxos = zeroconf_wallet.listunspent(minconf=0) + assert_equal(len(utxos), 1) + assert_equal(utxos[0]['confirmations'], 0) + # accounts for trusted balance + bal = zeroconf_wallet.getbalances() + assert_equal(bal['mine']['trusted'], utxos[0]['amount']) + assert_equal(bal['mine']['untrusted_pending'], 0) + + zeroconf_wallet.sendtoaddress(zeroconf_wallet.getnewaddress(), Decimal('0.5')) + if __name__ == '__main__': WalletTest().main() diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 1cb3f434e8..a2ae997ecb 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -88,6 +88,7 @@ class BumpFeeTest(BitcoinTestFramework): test_nonrbf_bumpfee_fails(self, peer_node, dest_address) test_notmine_bumpfee(self, rbf_node, peer_node, dest_address) test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_address) + test_bumpfee_with_abandoned_descendant_succeeds(self, rbf_node, rbf_node_address, dest_address) test_dust_to_fee(self, rbf_node, dest_address) test_watchonly_psbt(self, peer_node, rbf_node, dest_address) test_rebumping(self, rbf_node, dest_address) @@ -286,6 +287,35 @@ def test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_ad self.clear_mempool() +def test_bumpfee_with_abandoned_descendant_succeeds(self, rbf_node, rbf_node_address, dest_address): + self.log.info('Test that fee can be bumped when it has abandoned descendant') + # parent is send-to-self, so we don't have to check which output is change when creating the child tx + parent_id = spend_one_input(rbf_node, rbf_node_address) + # Submit child transaction with low fee + child_id = rbf_node.send(outputs={dest_address: 0.00020000}, + options={"inputs": [{"txid": parent_id, "vout": 0}], "fee_rate": 2})["txid"] + assert child_id in rbf_node.getrawmempool() + + # Restart the node with higher min relay fee so the descendant tx is no longer in mempool so that we can abandon it + self.restart_node(1, ['-minrelaytxfee=0.00005'] + self.extra_args[1]) + rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + self.connect_nodes(1, 0) + assert parent_id in rbf_node.getrawmempool() + assert child_id not in rbf_node.getrawmempool() + # Should still raise an error even if not in mempool + assert_raises_rpc_error(-8, "Transaction has descendants in the wallet", rbf_node.bumpfee, parent_id) + # Now abandon the child transaction and bump the original + rbf_node.abandontransaction(child_id) + bumped_result = rbf_node.bumpfee(parent_id, {"fee_rate": HIGH}) + assert bumped_result['txid'] in rbf_node.getrawmempool() + assert parent_id not in rbf_node.getrawmempool() + # Cleanup + self.restart_node(1, self.extra_args[1]) + rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + self.connect_nodes(1, 0) + self.clear_mempool() + + def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): self.log.info('Testing small output with feerate bump succeeds') diff --git a/test/functional/wallet_crosschain.py b/test/functional/wallet_crosschain.py index 6f93ad4e3b..7a1297e65f 100755 --- a/test/functional/wallet_crosschain.py +++ b/test/functional/wallet_crosschain.py @@ -25,11 +25,7 @@ class WalletCrossChain(BitcoinTestFramework): # Switch node 1 to testnet before starting it. self.nodes[1].chain = 'testnet3' self.nodes[1].extra_args = ['-maxconnections=0', '-prune=550'] # disable testnet sync - with open(self.nodes[1].bitcoinconf, 'r', encoding='utf8') as conf: - conf_data = conf.read() - with open (self.nodes[1].bitcoinconf, 'w', encoding='utf8') as conf: - conf.write(conf_data.replace('regtest=', 'testnet=').replace('[regtest]', '[test]')) - + self.nodes[1].replace_in_config([('regtest=', 'testnet='), ('[regtest]', '[test]')]) self.start_nodes() def run_test(self): diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index 2b70e5ecc9..b0e93df36a 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -4,7 +4,11 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test descriptor wallet function.""" import os -import sqlite3 + +try: + import sqlite3 +except ImportError: + pass from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework @@ -27,6 +31,7 @@ class WalletDescriptorTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() self.skip_if_no_sqlite() + self.skip_if_no_py_sqlite3() def run_test(self): if self.is_bdb_compiled(): @@ -109,7 +114,7 @@ class WalletDescriptorTest(BitcoinTestFramework): # Make sure things are disabled self.log.info("Test disabled RPCs") assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importprivkey, "cVpF924EspNh8KjYsfhgY96mmxvT6DgdWiTYMtMjuM74hJaU5psW") - assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importpubkey, send_wrpc.getaddressinfo(send_wrpc.getnewaddress())) + assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importpubkey, send_wrpc.getaddressinfo(send_wrpc.getnewaddress())["pubkey"]) assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importaddress, recv_wrpc.getnewaddress()) assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importmulti, []) assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.addmultisigaddress, 1, [recv_wrpc.getnewaddress()]) diff --git a/test/functional/wallet_fast_rescan.py b/test/functional/wallet_fast_rescan.py index 52e33acb24..1ab24f1a96 100755 --- a/test/functional/wallet_fast_rescan.py +++ b/test/functional/wallet_fast_rescan.py @@ -40,7 +40,6 @@ class WalletFastRescanTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] wallet = MiniWallet(node) - wallet.rescan_utxos() self.log.info("Create descriptor wallet with backup") WALLET_BACKUP_FILENAME = os.path.join(node.datadir, 'wallet.bak') diff --git a/test/functional/wallet_fundrawtransaction.py b/test/functional/wallet_fundrawtransaction.py index 98b0f70b01..29ddb77b41 100755 --- a/test/functional/wallet_fundrawtransaction.py +++ b/test/functional/wallet_fundrawtransaction.py @@ -148,6 +148,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.test_external_inputs() self.test_22670() self.test_feerate_rounding() + self.test_input_confs_control() def test_change_position(self): """Ensure setting changePosition in fundraw with an exact match is handled properly.""" @@ -1403,6 +1404,66 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = w.createrawtransaction(inputs=[], outputs=[{self.nodes[0].getnewaddress(address_type="bech32"): 1 - 0.00000202}]) assert_raises_rpc_error(-4, "Insufficient funds", w.fundrawtransaction, rawtx, {"fee_rate": 1.85}) + def test_input_confs_control(self): + self.nodes[0].createwallet("minconf") + wallet = self.nodes[0].get_wallet_rpc("minconf") + + # Fund the wallet with different chain heights + for _ in range(2): + self.nodes[2].sendmany("", {wallet.getnewaddress():1, wallet.getnewaddress():1}) + self.generate(self.nodes[2], 1) + + unconfirmed_txid = wallet.sendtoaddress(wallet.getnewaddress(), 0.5) + + self.log.info("Crafting TX using an unconfirmed input") + target_address = self.nodes[2].getnewaddress() + raw_tx1 = wallet.createrawtransaction([], {target_address: 0.1}, 0, True) + funded_tx1 = wallet.fundrawtransaction(raw_tx1, {'fee_rate': 1, 'maxconf': 0})['hex'] + + # Make sure we only had the one input + tx1_inputs = self.nodes[0].decoderawtransaction(funded_tx1)['vin'] + assert_equal(len(tx1_inputs), 1) + + utxo1 = tx1_inputs[0] + assert unconfirmed_txid == utxo1['txid'] + + final_tx1 = wallet.signrawtransactionwithwallet(funded_tx1)['hex'] + txid1 = self.nodes[0].sendrawtransaction(final_tx1) + + mempool = self.nodes[0].getrawmempool() + assert txid1 in mempool + + self.log.info("Fail to craft a new TX with minconf above highest one") + # Create a replacement tx to 'final_tx1' that has 1 BTC target instead of 0.1. + raw_tx2 = wallet.createrawtransaction([{'txid': utxo1['txid'], 'vout': utxo1['vout']}], {target_address: 1}) + assert_raises_rpc_error(-4, "Insufficient funds", wallet.fundrawtransaction, raw_tx2, {'add_inputs': True, 'minconf': 3, 'fee_rate': 10}) + + self.log.info("Fail to broadcast a new TX with maxconf 0 due to BIP125 rules to verify it actually chose unconfirmed outputs") + # Now fund 'raw_tx2' to fulfill the total target (1 BTC) by using all the wallet unconfirmed outputs. + # As it was created with the first unconfirmed output, 'raw_tx2' only has 0.1 BTC covered (need to fund 0.9 BTC more). + # So, the selection process, to cover the amount, will pick up the 'final_tx1' output as well, which is an output of the tx that this + # new tx is replacing!. So, once we send it to the mempool, it will return a "bad-txns-spends-conflicting-tx" + # because the input will no longer exist once the first tx gets replaced by this new one). + funded_invalid = wallet.fundrawtransaction(raw_tx2, {'add_inputs': True, 'maxconf': 0, 'fee_rate': 10})['hex'] + final_invalid = wallet.signrawtransactionwithwallet(funded_invalid)['hex'] + assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, final_invalid) + + self.log.info("Craft a replacement adding inputs with highest depth possible") + funded_tx2 = wallet.fundrawtransaction(raw_tx2, {'add_inputs': True, 'minconf': 2, 'fee_rate': 10})['hex'] + tx2_inputs = self.nodes[0].decoderawtransaction(funded_tx2)['vin'] + assert_greater_than_or_equal(len(tx2_inputs), 2) + for vin in tx2_inputs: + if vin['txid'] != unconfirmed_txid: + assert_greater_than_or_equal(self.nodes[0].gettxout(vin['txid'], vin['vout'])['confirmations'], 2) + + final_tx2 = wallet.signrawtransactionwithwallet(funded_tx2)['hex'] + txid2 = self.nodes[0].sendrawtransaction(final_tx2) + + mempool = self.nodes[0].getrawmempool() + assert txid1 not in mempool + assert txid2 in mempool + + wallet.unloadwallet() if __name__ == '__main__': RawTransactionsTest().main() diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index 9e813166c5..ca0209b61d 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -15,7 +15,6 @@ variants. - `test_address()` is called to call getaddressinfo for an address on node1 and test the values returned.""" -from test_framework.address import key_to_p2pkh from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.descriptors import descsum_create @@ -120,12 +119,11 @@ class ImportDescriptorsTest(BitcoinTestFramework): self.log.info("Internal addresses should be detected as such") key = get_generate_key() - addr = key_to_p2pkh(key.pubkey) self.test_importdesc({"desc": descsum_create("pkh(" + key.pubkey + ")"), "timestamp": "now", "internal": True}, success=True) - info = w1.getaddressinfo(addr) + info = w1.getaddressinfo(key.p2pkh_addr) assert_equal(info["ismine"], True) assert_equal(info["ischange"], True) diff --git a/test/functional/wallet_migration.py b/test/functional/wallet_migration.py index 688ac98617..72c5fe7b84 100755 --- a/test/functional/wallet_migration.py +++ b/test/functional/wallet_migration.py @@ -163,6 +163,10 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(basic2.getbalance(), basic2_balance) self.assert_list_txs_equal(basic2.listtransactions(), basic2_txs) + # Now test migration on a descriptor wallet + self.log.info("Test \"nothing to migrate\" when the user tries to migrate a wallet with no legacy data") + assert_raises_rpc_error(-4, "Error: This wallet is already a descriptor wallet", basic2.migratewallet) + def test_multisig(self): default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) diff --git a/test/functional/wallet_orphanedreward.py b/test/functional/wallet_orphanedreward.py index d9f7c14ded..d8931fa620 100755 --- a/test/functional/wallet_orphanedreward.py +++ b/test/functional/wallet_orphanedreward.py @@ -34,29 +34,40 @@ class OrphanedBlockRewardTest(BitcoinTestFramework): # the existing balance and the block reward. self.generate(self.nodes[0], 150) assert_equal(self.nodes[1].getbalance(), 10 + 25) + pre_reorg_conf_bals = self.nodes[1].getbalances() txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 30) + orig_chain_tip = self.nodes[0].getbestblockhash() + self.sync_mempools() # Orphan the block reward and make sure that the original coins # from the wallet can still be spent. self.nodes[0].invalidateblock(blk) - self.generate(self.nodes[0], 152) - # Without the following abandontransaction call, the coins are - # not considered available yet. - assert_equal(self.nodes[1].getbalances()["mine"], { - "trusted": 0, - "untrusted_pending": 0, - "immature": 0, - }) - # The following abandontransaction is necessary to make the later - # lines succeed, and probably should not be needed; see - # https://github.com/bitcoin/bitcoin/issues/14148. - self.nodes[1].abandontransaction(txid) + blocks = self.generate(self.nodes[0], 152) + conflict_block = blocks[0] + # We expect the descendants of orphaned rewards to no longer be considered assert_equal(self.nodes[1].getbalances()["mine"], { "trusted": 10, "untrusted_pending": 0, "immature": 0, }) - self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 9) + # And the unconfirmed tx to be abandoned + assert_equal(self.nodes[1].gettransaction(txid)["details"][0]["abandoned"], True) + + # The abandoning should persist through reloading + self.nodes[1].unloadwallet(self.default_wallet_name) + self.nodes[1].loadwallet(self.default_wallet_name) + assert_equal(self.nodes[1].gettransaction(txid)["details"][0]["abandoned"], True) + + # If the orphaned reward is reorged back into the main chain, any unconfirmed + # descendant txs at the time of the original reorg remain abandoned. + self.nodes[0].invalidateblock(conflict_block) + self.nodes[0].reconsiderblock(blk) + assert_equal(self.nodes[0].getbestblockhash(), orig_chain_tip) + self.generate(self.nodes[0], 3) + + assert_equal(self.nodes[1].getbalances(), pre_reorg_conf_bals) + assert_equal(self.nodes[1].gettransaction(txid)["details"][0]["abandoned"], True) + if __name__ == '__main__': OrphanedBlockRewardTest().main() diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 424834323f..ac3ec06eec 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -45,7 +45,7 @@ class WalletSendTest(BitcoinTestFramework): conf_target=None, estimate_mode=None, fee_rate=None, add_to_wallet=None, psbt=None, inputs=None, add_inputs=None, include_unsafe=None, change_address=None, change_position=None, change_type=None, include_watching=None, locktime=None, lock_unspents=None, replaceable=None, subtract_fee_from_outputs=None, - expect_error=None, solving_data=None): + expect_error=None, solving_data=None, minconf=None): assert (amount is None) != (data is None) from_balance_before = from_wallet.getbalances()["mine"]["trusted"] @@ -106,6 +106,8 @@ class WalletSendTest(BitcoinTestFramework): options["subtract_fee_from_outputs"] = subtract_fee_from_outputs if solving_data is not None: options["solving_data"] = solving_data + if minconf is not None: + options["minconf"] = minconf if len(options.keys()) == 0: options = None @@ -487,6 +489,16 @@ class WalletSendTest(BitcoinTestFramework): res = self.test_send(from_wallet=w5, to_wallet=w0, amount=1, include_unsafe=True) assert res["complete"] + self.log.info("Minconf") + self.nodes[1].createwallet(wallet_name="minconfw") + minconfw= self.nodes[1].get_wallet_rpc("minconfw") + self.test_send(from_wallet=w0, to_wallet=minconfw, amount=2) + self.generate(self.nodes[0], 3) + self.test_send(from_wallet=minconfw, to_wallet=w0, amount=1, minconf=4, expect_error=(-4, "Insufficient funds")) + self.test_send(from_wallet=minconfw, to_wallet=w0, amount=1, minconf=-4, expect_error=(-8, "Negative minconf")) + res = self.test_send(from_wallet=minconfw, to_wallet=w0, amount=1, minconf=3) + assert res["complete"] + self.log.info("External outputs") eckey = ECKey() eckey.generate() diff --git a/test/functional/wallet_sendall.py b/test/functional/wallet_sendall.py index 778c8a5b9e..f6440f07d7 100755 --- a/test/functional/wallet_sendall.py +++ b/test/functional/wallet_sendall.py @@ -317,6 +317,68 @@ class SendallTest(BitcoinTestFramework): assert_equal(decoded["tx"]["vin"][0]["vout"], utxo["vout"]) assert_equal(decoded["tx"]["vout"][0]["scriptPubKey"]["address"], self.remainder_target) + @cleanup + def sendall_with_minconf(self): + # utxo of 17 bicoin has 6 confirmations, utxo of 4 has 3 + self.add_utxos([17]) + self.generate(self.nodes[0], 2) + self.add_utxos([4]) + self.generate(self.nodes[0], 2) + + self.log.info("Test sendall fails because minconf is negative") + + assert_raises_rpc_error(-8, + "Invalid minconf (minconf cannot be negative): -2", + self.wallet.sendall, + recipients=[self.remainder_target], + options={"minconf": -2}) + self.log.info("Test sendall fails because minconf is used while specific inputs are provided") + + utxo = self.wallet.listunspent()[0] + assert_raises_rpc_error(-8, + "Cannot combine minconf or maxconf with specific inputs.", + self.wallet.sendall, + recipients=[self.remainder_target], + options={"inputs": [utxo], "minconf": 2}) + + self.log.info("Test sendall fails because there are no utxos with enough confirmations specified by minconf") + + assert_raises_rpc_error(-6, + "Total value of UTXO pool too low to pay for transaction. Try using lower feerate or excluding uneconomic UTXOs with 'send_max' option.", + self.wallet.sendall, + recipients=[self.remainder_target], + options={"minconf": 7}) + + self.log.info("Test sendall only spends utxos with a specified number of confirmations when minconf is used") + self.wallet.sendall(recipients=[self.remainder_target], fee_rate=300, options={"minconf": 6}) + + assert_equal(len(self.wallet.listunspent()), 1) + assert_equal(self.wallet.listunspent()[0]['confirmations'], 3) + + # decrease minconf and show the remaining utxo is picked up + self.wallet.sendall(recipients=[self.remainder_target], fee_rate=300, options={"minconf": 3}) + assert_equal(self.wallet.getbalance(), 0) + + @cleanup + def sendall_with_maxconf(self): + # utxo of 17 bicoin has 6 confirmations, utxo of 4 has 3 + self.add_utxos([17]) + self.generate(self.nodes[0], 2) + self.add_utxos([4]) + self.generate(self.nodes[0], 2) + + self.log.info("Test sendall fails because there are no utxos with enough confirmations specified by maxconf") + assert_raises_rpc_error(-6, + "Total value of UTXO pool too low to pay for transaction. Try using lower feerate or excluding uneconomic UTXOs with 'send_max' option.", + self.wallet.sendall, + recipients=[self.remainder_target], + options={"maxconf": 1}) + + self.log.info("Test sendall only spends utxos with a specified number of confirmations when maxconf is used") + self.wallet.sendall(recipients=[self.remainder_target], fee_rate=300, options={"maxconf":4}) + assert_equal(len(self.wallet.listunspent()), 1) + assert_equal(self.wallet.listunspent()[0]['confirmations'], 6) + # This tests needs to be the last one otherwise @cleanup will fail with "Transaction too large" error def sendall_fails_with_transaction_too_large(self): self.log.info("Test that sendall fails if resulting transaction is too large") @@ -392,6 +454,12 @@ class SendallTest(BitcoinTestFramework): # Sendall succeeds with watchonly wallets spending specific UTXOs self.sendall_watchonly_specific_inputs() + # Sendall only uses outputs with at least a give number of confirmations when using minconf + self.sendall_with_minconf() + + # Sendall only uses outputs with less than a given number of confirmation when using minconf + self.sendall_with_maxconf() + # Sendall fails when many inputs result to too large transaction self.sendall_fails_with_transaction_too_large() diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index e2eab2a0fe..af21e7b956 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -143,7 +143,7 @@ def main(): timeout=20, check=True, stderr=subprocess.PIPE, - universal_newlines=True, + text=True, ).stderr if "libFuzzer" not in help_output: logging.error("Must be built with libFuzzer") @@ -200,7 +200,7 @@ def generate_corpus(*, fuzz_pool, src_dir, build_dir, corpus_dir, targets): env=get_fuzz_env(target=t, source_dir=src_dir), check=True, stderr=subprocess.PIPE, - universal_newlines=True, + text=True, ).stderr)) futures = [] @@ -241,7 +241,7 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, build_dir, merge_dir) env=get_fuzz_env(target=t, source_dir=src_dir), check=True, stderr=subprocess.PIPE, - universal_newlines=True, + text=True, ).stderr logging.debug(output) @@ -270,7 +270,7 @@ def run_once(*, fuzz_pool, corpus, test_list, src_dir, build_dir, use_valgrind): args, env=get_fuzz_env(target=t, source_dir=src_dir), stderr=subprocess.PIPE, - universal_newlines=True, + text=True, ) output += result.stderr return output, result @@ -299,7 +299,7 @@ def parse_test_list(*, fuzz_bin): }, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, - universal_newlines=True, + text=True, ).stdout.splitlines() return test_list_all diff --git a/test/lint/README.md b/test/lint/README.md index 8d592c3282..704922d7ab 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -1,5 +1,23 @@ This folder contains lint scripts. +Running locally +=============== + +To run linters locally with the same versions as the CI environment, use the included +Dockerfile: + +```sh +cd ./ci/lint +docker build -t bitcoin-linter . + +cd /root/of/bitcoin/repo +docker run --rm -v $(pwd):/bitcoin -it bitcoin-linter +``` + +After building the container once, you can simply run the last command any time you +want to lint. + + check-doc.py ============ Check for missing documentation of command line options. diff --git a/test/lint/lint-assertions.py b/test/lint/lint-assertions.py index 195ff33d11..e7eecebce5 100755 --- a/test/lint/lint-assertions.py +++ b/test/lint/lint-assertions.py @@ -12,7 +12,7 @@ import subprocess def git_grep(params: [], error_msg: ""): try: - output = subprocess.check_output(["git", "grep", *params], universal_newlines=True, encoding="utf8") + output = subprocess.check_output(["git", "grep", *params], text=True, encoding="utf8") print(error_msg) print(output) return 1 diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py index b69bbe7cd0..cf6a5f81f1 100755 --- a/test/lint/lint-circular-dependencies.py +++ b/test/lint/lint-circular-dependencies.py @@ -38,14 +38,14 @@ def main(): os.chdir(CODE_DIR) files = subprocess.check_output( ['git', 'ls-files', '--', '*.h', '*.cpp'], - universal_newlines=True, + text=True, ).splitlines() command = [sys.executable, "../contrib/devtools/circular-dependencies.py", *files] dependencies_output = subprocess.run( command, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, ) for dependency_str in dependencies_output.stdout.rstrip().split("\n"): diff --git a/test/lint/lint-git-commit-check.py b/test/lint/lint-git-commit-check.py index 049104398a..5897a17e70 100755 --- a/test/lint/lint-git-commit-check.py +++ b/test/lint/lint-git-commit-check.py @@ -42,17 +42,17 @@ def main(): commit_range = "HEAD~" + args.prev_commits + "...HEAD" else: # This assumes that the target branch of the pull request will be master. - merge_base = check_output(["git", "merge-base", "HEAD", "master"], universal_newlines=True, encoding="utf8").rstrip("\n") + merge_base = check_output(["git", "merge-base", "HEAD", "master"], text=True, encoding="utf8").rstrip("\n") commit_range = merge_base + "..HEAD" else: commit_range = os.getenv("COMMIT_RANGE") if commit_range == "SKIP_EMPTY_NOT_A_PR": sys.exit(0) - commit_hashes = check_output(["git", "log", commit_range, "--format=%H"], universal_newlines=True, encoding="utf8").splitlines() + commit_hashes = check_output(["git", "log", commit_range, "--format=%H"], text=True, encoding="utf8").splitlines() for hash in commit_hashes: - commit_info = check_output(["git", "log", "--format=%B", "-n", "1", hash], universal_newlines=True, encoding="utf8").splitlines() + commit_info = check_output(["git", "log", "--format=%B", "-n", "1", hash], text=True, encoding="utf8").splitlines() if len(commit_info) >= 2: if commit_info[1]: print(f"The subject line of commit hash {hash} is followed by a non-empty line. Subject lines should always be followed by a blank line.") diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py index b3fa4b9303..459030bb0b 100755 --- a/test/lint/lint-includes.py +++ b/test/lint/lint-includes.py @@ -35,13 +35,13 @@ EXPECTED_BOOST_INCLUDES = ["boost/date_time/posix_time/posix_time.hpp", def get_toplevel(): - return check_output(["git", "rev-parse", "--show-toplevel"], universal_newlines=True, encoding="utf8").rstrip("\n") + return check_output(["git", "rev-parse", "--show-toplevel"], text=True, encoding="utf8").rstrip("\n") def list_files_by_suffix(suffixes): exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] - files_list = check_output(["git", "ls-files", "src"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + files_list = check_output(["git", "ls-files", "src"] + exclude_args, text=True, encoding="utf8").splitlines() return [file for file in files_list if file.endswith(suffixes)] @@ -63,7 +63,7 @@ def find_included_cpps(): included_cpps = list() try: - included_cpps = check_output(["git", "grep", "-E", r"^#include [<\"][^>\"]+\.cpp[>\"]", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + included_cpps = check_output(["git", "grep", "-E", r"^#include [<\"][^>\"]+\.cpp[>\"]", "--", "*.cpp", "*.h"], text=True, encoding="utf8").splitlines() except CalledProcessError as e: if e.returncode > 1: raise e @@ -77,7 +77,7 @@ def find_extra_boosts(): exclusion_set = set() try: - included_boosts = check_output(["git", "grep", "-E", r"^#include <boost/", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + included_boosts = check_output(["git", "grep", "-E", r"^#include <boost/", "--", "*.cpp", "*.h"], text=True, encoding="utf8").splitlines() except CalledProcessError as e: if e.returncode > 1: raise e @@ -100,7 +100,7 @@ def find_quote_syntax_inclusions(): quote_syntax_inclusions = list() try: - quote_syntax_inclusions = check_output(["git", "grep", r"^#include \"", "--", "*.cpp", "*.h"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + quote_syntax_inclusions = check_output(["git", "grep", r"^#include \"", "--", "*.cpp", "*.h"] + exclude_args, text=True, encoding="utf8").splitlines() except CalledProcessError as e: if e.returncode > 1: raise e @@ -143,13 +143,13 @@ def main(): if extra_boosts: for boost in extra_boosts: print(f"A new Boost dependency in the form of \"{boost}\" appears to have been introduced:") - print(check_output(["git", "grep", boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8")) + print(check_output(["git", "grep", boost, "--", "*.cpp", "*.h"], text=True, encoding="utf8")) exit_code = 1 # Check if Boost dependencies are no longer used for expected_boost in EXPECTED_BOOST_INCLUDES: try: - check_output(["git", "grep", "-q", r"^#include <%s>" % expected_boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8") + check_output(["git", "grep", "-q", r"^#include <%s>" % expected_boost, "--", "*.cpp", "*.h"], text=True, encoding="utf8") except CalledProcessError as e: if e.returncode > 1: raise e diff --git a/test/lint/lint-locale-dependence.py b/test/lint/lint-locale-dependence.py index ce7444cd1a..c5cb34b20a 100755 --- a/test/lint/lint-locale-dependence.py +++ b/test/lint/lint-locale-dependence.py @@ -223,7 +223,7 @@ def find_locale_dependent_function_uses(): git_grep_output = list() try: - git_grep_output = check_output(git_grep_command, universal_newlines=True, encoding="utf8").splitlines() + git_grep_output = check_output(git_grep_command, text=True, encoding="utf8").splitlines() except CalledProcessError as e: if e.returncode > 1: raise e diff --git a/test/lint/lint-logs.py b/test/lint/lint-logs.py index aaf697467d..de04a1aeca 100755 --- a/test/lint/lint-logs.py +++ b/test/lint/lint-logs.py @@ -16,7 +16,7 @@ from subprocess import check_output def main(): - logs_list = check_output(["git", "grep", "--extended-regexp", r"(LogPrintLevel|LogPrintfCategory|LogPrintf?)\(", "--", "*.cpp"], universal_newlines=True, encoding="utf8").splitlines() + logs_list = check_output(["git", "grep", "--extended-regexp", r"(LogPrintLevel|LogPrintfCategory|LogPrintf?)\(", "--", "*.cpp"], text=True, encoding="utf8").splitlines() unterminated_logs = [line for line in logs_list if not re.search(r'(\\n"|/\* Continued \*/)', line)] diff --git a/test/lint/lint-python-mutable-default-parameters.py b/test/lint/lint-python-mutable-default-parameters.py index 3dfc5940f7..820595ea34 100755 --- a/test/lint/lint-python-mutable-default-parameters.py +++ b/test/lint/lint-python-mutable-default-parameters.py @@ -21,7 +21,7 @@ def main(): "--", "*.py", ] - output = subprocess.run(command, stdout=subprocess.PIPE, universal_newlines=True) + output = subprocess.run(command, stdout=subprocess.PIPE, text=True) if len(output.stdout) > 0: error_msg = ( "A mutable list or dict seems to be used as default parameter value:\n\n" diff --git a/test/lint/lint-python-utf8-encoding.py b/test/lint/lint-python-utf8-encoding.py index 62fdc34d50..364da63468 100755 --- a/test/lint/lint-python-utf8-encoding.py +++ b/test/lint/lint-python-utf8-encoding.py @@ -23,7 +23,7 @@ def check_fileopens(): fileopens = list() try: - fileopens = check_output(["git", "grep", r" open(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + fileopens = check_output(["git", "grep", r" open(", "--", "*.py"] + get_exclude_args(), text=True, encoding="utf8").splitlines() except CalledProcessError as e: if e.returncode > 1: raise e @@ -37,12 +37,12 @@ def check_checked_outputs(): checked_outputs = list() try: - checked_outputs = check_output(["git", "grep", "check_output(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + checked_outputs = check_output(["git", "grep", "check_output(", "--", "*.py"] + get_exclude_args(), text=True, encoding="utf8").splitlines() except CalledProcessError as e: if e.returncode > 1: raise e - filtered_checked_outputs = [checked_output for checked_output in checked_outputs if re.search(r"universal_newlines=True", checked_output) and not re.search(r"encoding=.(ascii|utf8|utf-8).", checked_output)] + filtered_checked_outputs = [checked_output for checked_output in checked_outputs if re.search(r"text=True", checked_output) and not re.search(r"encoding=.(ascii|utf8|utf-8).", checked_output)] return filtered_checked_outputs diff --git a/test/lint/lint-shell.py b/test/lint/lint-shell.py index ed95024ef5..1646bf0d3e 100755 --- a/test/lint/lint-shell.py +++ b/test/lint/lint-shell.py @@ -25,7 +25,7 @@ def check_shellcheck_install(): sys.exit(0) def get_files(command): - output = subprocess.run(command, stdout=subprocess.PIPE, universal_newlines=True) + output = subprocess.run(command, stdout=subprocess.PIPE, text=True) files = output.stdout.split('\n') # remove whitespace element diff --git a/test/lint/lint-submodule.py b/test/lint/lint-submodule.py index 89d4c80f55..4d2fbf088f 100755 --- a/test/lint/lint-submodule.py +++ b/test/lint/lint-submodule.py @@ -13,7 +13,7 @@ import sys def main(): submodules_list = subprocess.check_output(['git', 'submodule', 'status', '--recursive'], - universal_newlines = True, encoding = 'utf8').rstrip('\n') + text = True, encoding = 'utf8').rstrip('\n') if submodules_list: print("These submodules were found, delete them:\n", submodules_list) sys.exit(1) diff --git a/test/lint/lint-tests.py b/test/lint/lint-tests.py index 849ddcb961..1eeb7bb014 100755 --- a/test/lint/lint-tests.py +++ b/test/lint/lint-tests.py @@ -23,7 +23,7 @@ def grep_boost_fixture_test_suite(): "src/test/**.cpp", "src/wallet/test/**.cpp", ] - return subprocess.check_output(command, universal_newlines=True, encoding="utf8") + return subprocess.check_output(command, text=True, encoding="utf8") def check_matching_test_names(test_suite_list): diff --git a/test/lint/lint-whitespace.py b/test/lint/lint-whitespace.py index 72b7ebc394..f5e4a776d0 100755 --- a/test/lint/lint-whitespace.py +++ b/test/lint/lint-whitespace.py @@ -80,7 +80,7 @@ def get_diff(commit_range, check_only_code): else: what_files = ["."] - diff = check_output(["git", "diff", "-U0", commit_range, "--"] + what_files + exclude_args, universal_newlines=True, encoding="utf8") + diff = check_output(["git", "diff", "-U0", commit_range, "--"] + what_files + exclude_args, text=True, encoding="utf8") return diff @@ -93,7 +93,7 @@ def main(): commit_range = "HEAD~" + args.prev_commits + "...HEAD" else: # This assumes that the target branch of the pull request will be master. - merge_base = check_output(["git", "merge-base", "HEAD", "master"], universal_newlines=True, encoding="utf8").rstrip("\n") + merge_base = check_output(["git", "merge-base", "HEAD", "master"], text=True, encoding="utf8").rstrip("\n") commit_range = merge_base + "..HEAD" else: commit_range = os.getenv("COMMIT_RANGE") diff --git a/test/util/test_runner.py b/test/util/test_runner.py index 03db05c563..ea3626fa65 100755 --- a/test/util/test_runner.py +++ b/test/util/test_runner.py @@ -107,7 +107,7 @@ def bctest(testDir, testObj, buildenv): raise Exception # Run the test - proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) try: outs = proc.communicate(input=inputData) except OSError: |