diff options
37 files changed, 663 insertions, 296 deletions
diff --git a/.travis.yml b/.travis.yml index bfb4cb1d58..4f9dbeded4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,7 +43,6 @@ before_cache: stages: - lint - test - - extended-lint env: global: - CI_RETRY_EXE="travis_retry" @@ -82,28 +81,15 @@ jobs: script: - set -o errexit; source ./ci/lint/06_script.sh - - stage: extended-lint - name: 'extended lint [runtime >= 60 seconds]' - env: - cache: false - language: python - python: '3.5' - install: - - set -o errexit; source ./ci/extended_lint/04_install.sh - before_script: - - set -o errexit; source ./ci/lint/05_before_script.sh - script: - - set -o errexit; source ./ci/extended_lint/06_script.sh - - stage: test - name: 'ARM [GOAL: install] [unit tests, functional tests]' + name: 'ARM [GOAL: install] [bionic] [unit tests, functional tests]' arch: arm64 env: >- FILE_ENV="./ci/test/00_setup_env_arm.sh" QEMU_USER_CMD="" # Can run the tests natively without qemu - stage: test - name: 'S390x [GOAL: install] [unit tests, functional tests]' + name: 'S390x [GOAL: install] [bionic] [unit tests, functional tests]' arch: s390x env: >- FILE_ENV="./ci/test/00_setup_env_s390x.sh" diff --git a/ci/extended_lint/04_install.sh b/ci/extended_lint/04_install.sh deleted file mode 100755 index 123d874a84..0000000000 --- a/ci/extended_lint/04_install.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -CPPCHECK_VERSION=1.86 -curl -s https://codeload.github.com/danmar/cppcheck/tar.gz/${CPPCHECK_VERSION} | tar -zxf - --directory /tmp/ -(cd /tmp/cppcheck-${CPPCHECK_VERSION}/ && make CFGDIR=/tmp/cppcheck-${CPPCHECK_VERSION}/cfg/ > /dev/null) -export PATH="$PATH:/tmp/cppcheck-${CPPCHECK_VERSION}/" diff --git a/ci/extended_lint/06_script.sh b/ci/extended_lint/06_script.sh deleted file mode 100755 index e8228c9c4d..0000000000 --- a/ci/extended_lint/06_script.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -test/lint/extended-lint-all.sh diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 2919072fe4..1f485fbec4 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -6,6 +6,12 @@ export LC_ALL=C.UTF-8 +# The root dir. +# The ci system copies this folder. +# This is where the build is done (depends and dist). +BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) +export BASE_ROOT_DIR + echo "Setting specific values in env" if [ -n "${FILE_ENV}" ]; then set -o errexit; @@ -13,13 +19,11 @@ if [ -n "${FILE_ENV}" ]; then source "${FILE_ENV}" fi -BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) -export BASE_ROOT_DIR - echo "Fallback to default values in env (if not yet set)" # The number of parallel jobs to pass down to make and test_runner.py export MAKEJOBS=${MAKEJOBS:--j4} # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) +# This folder only exists on the ci host. export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/} # What host to compile for. See also ./depends/README.md # Tests that need cross-compilation export the appropriate HOST. @@ -37,15 +41,18 @@ export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} export CCACHE_SIZE=${CCACHE_SIZE:-100M} export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} +# The cache dir. +# This folder exists on the ci host and ci guest. Changes are propagated back and forth. export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache} -# Folder where the build is done (depends and dist). Can not be changed and is equal to the root of the git repo -export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR} -# Folder where the build is done (bin and lib). Can not be changed. -export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST} +# The depends dir. +# This folder exists on the ci host and ci guest. Changes are propagated back and forth. +export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} +# Folder where the build is done (bin and lib). +export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export WINEDEBUG=${WINEDEBUG:-fixme-all} export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git} export GOAL=${GOAL:-install} -export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets} +export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH export CI_RETRY_EXE=${CI_RETRY_EXE:retry} diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index b354940d35..2ffd3c5107 100644 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -6,7 +6,8 @@ export LC_ALL=C.UTF-8 -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export PACKAGES="clang-8 llvm-8 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +# Use clang-8 instead of default clang (which is clang-6 on Bionic) to avoid spurious segfaults when running on ppc64le export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang-8 CXX=clang++-8" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index f7a16206ab..e9b7a7bba1 100644 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -11,3 +11,7 @@ export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent- export NO_DEPENDS=1 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++" + +# xenial comes with old clang versions that can not parse the sanitizer suppressions files +# Remove unparseable lines as a hacky workaround +sed -i '/^implicit-/d' "${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan" diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index b41d44c61a..89660c7fa4 100644 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -7,12 +7,9 @@ export LC_ALL=C.UTF-8 export HOST=s390x-unknown-linux-gnu -export DOCKER_NAME_TAG=s390x/ubuntu:18.04 -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=true -export RUN_FUNCTIONAL_TESTS=false +export RUN_FUNCTIONAL_TESTS=true export GOAL="install" export BITCOIN_CONFIG="--enable-reduce-exports --with-incompatible-bdb" - -lscpu diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 318ee94fde..c721345250 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -37,10 +37,10 @@ fi mkdir -p "${BASE_SCRATCH_DIR}" mkdir -p "${CCACHE_DIR}" -export ASAN_OPTIONS="detect_stack_use_after_return=1" -export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan" -export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan" -export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1" +export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" +export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" +export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" +export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" env | grep -E '^(BITCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env if [[ $HOST = *-mingw32 ]]; then DOCKER_ADMIN="--cap-add SYS_ADMIN" @@ -48,25 +48,27 @@ elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (A DOCKER_ADMIN="--cap-add SYS_PTRACE" fi +export P_CI_DIR="$PWD" + if [ -z "$RUN_CI_ON_HOST" ]; then echo "Creating $DOCKER_NAME_TAG container to run in" ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \ - --mount type=bind,src=$BASE_BUILD_DIR,dst=/ro_base,readonly \ + --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ - --mount type=bind,src=$BASE_BUILD_DIR/depends,dst=$BASE_BUILD_DIR/depends \ - -w $BASE_BUILD_DIR \ + --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \ + -w $BASE_ROOT_DIR \ --env-file /tmp/env \ $DOCKER_NAME_TAG) DOCKER_EXEC () { - docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*" + docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" } else echo "Running on host system without docker wrapper" DOCKER_EXEC () { - bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*" + bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" } fi @@ -76,6 +78,8 @@ if [ "$TRAVIS_OS_NAME" == "osx" ]; then else DOCKER_EXEC free -m -h DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) + DOCKER_EXEC echo "Free disk space:" + DOCKER_EXEC df -h fi if [ -n "$DPKG_ADD_ARCH" ]; then @@ -95,11 +99,11 @@ if [ ! -d ${DIR_QA_ASSETS} ]; then fi export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ -DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" +DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" if [ -z "$RUN_CI_ON_HOST" ]; then - echo "Create $BASE_BUILD_DIR" - DOCKER_EXEC rsync -a /ro_base/ $BASE_BUILD_DIR + echo "Create $BASE_ROOT_DIR" + DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR fi if [ "$USE_BUSY_BOX" = "true" ]; then diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 3f0bba19ac..51a5830682 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -13,13 +13,13 @@ else DOCKER_EXEC echo \> \$HOME/.bitcoin fi -DOCKER_EXEC mkdir -p depends/SDKs depends/sdk-sources +DOCKER_EXEC mkdir -p ${DEPENDS_DIR}/SDKs ${DEPENDS_DIR}/sdk-sources -if [ -n "$OSX_SDK" ] && [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz +if [ -n "$OSX_SDK" ] && [ ! -f ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then + curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz fi -if [ -n "$OSX_SDK" ] && [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - DOCKER_EXEC tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz +if [ -n "$OSX_SDK" ] && [ -f ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then + DOCKER_EXEC tar -C ${DEPENDS_DIR}/SDKs -xf ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz fi if [[ $HOST = *-mingw32 ]]; then DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\) diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 34b8477197..98b75d7497 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$BASE_BUILD_DIR/depends/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" +BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" if [ -z "$NO_DEPENDS" ]; then DOCKER_EXEC ccache --max-size=$CCACHE_SIZE fi @@ -19,14 +19,8 @@ else fi END_FOLD -# Create folder on host and docker, so that `cd` works -mkdir -p build DOCKER_EXEC mkdir -p build - -# Temporarily disable errexit, because Travis macOS fails without error message -set +o errexit -cd build || (echo "could not enter build directory"; exit 1) -set -o errexit +export P_CI_DIR="$P_CI_DIR/build" BEGIN_FOLD configure DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) @@ -38,21 +32,15 @@ mkdir -p "bitcoin-$HOST" DOCKER_EXEC make distdir VERSION=$HOST END_FOLD -set +o errexit -cd "bitcoin-$HOST" || (echo "could not enter distdir bitcoin-$HOST"; exit 1) -set -o errexit +export P_CI_DIR="$P_CI_DIR/bitcoin-$HOST" BEGIN_FOLD configure DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) END_FOLD set -o errtrace -trap 'DOCKER_EXEC "cat ${BASE_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR +trap 'DOCKER_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR BEGIN_FOLD build DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) END_FOLD - -set +o errexit -cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1) -set -o errexit diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 49962bb49b..a8e0a50f36 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -6,11 +6,6 @@ export LC_ALL=C.UTF-8 -# Temporarily disable errexit, because Travis macOS fails without error message -set +o errexit -cd "build/bitcoin-$HOST" || (echo "could not enter distdir build/bitcoin-$HOST"; exit 1) -set -o errexit - if [ -n "$QEMU_USER_CMD" ]; then BEGIN_FOLD wrap-qemu echo "Prepare to run functional tests for HOST=$HOST" @@ -33,7 +28,7 @@ fi if [ "$RUN_UNIT_TESTS" = "true" ]; then BEGIN_FOLD unit-tests bash -c "${CI_WAIT}" & # Print dots in case the unit tests take a long time to run - DOCKER_EXEC LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1 + DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib make $MAKEJOBS check VERBOSE=1 END_FOLD fi diff --git a/configure.ac b/configure.ac index 2025037bd4..e0bec0882e 100644 --- a/configure.ac +++ b/configure.ac @@ -783,6 +783,7 @@ fi dnl this flag screws up non-darwin gcc even when the check fails. special-case it. if test x$TARGET_OS = xdarwin; then AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"]) + AX_CHECK_LINK_FLAG([[-Wl,-dead_strip_dylibs]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"]) fi AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) @@ -1471,18 +1472,16 @@ if test x$bitcoin_enable_qt != xno; then AC_MSG_CHECKING([whether to build GUI with support for QR codes]) if test x$have_qrencode = xno; then if test x$use_qr = xyes; then - AC_MSG_ERROR("QR support requested but cannot be built. use --without-qrencode") + AC_MSG_ERROR([QR support requested but cannot be built. Use --without-qrencode]) fi - AC_MSG_RESULT(no) + use_qr=no else if test x$use_qr != xno; then - AC_MSG_RESULT(yes) AC_DEFINE([USE_QRCODE],[1],[Define if QR support should be compiled in]) use_qr=yes - else - AC_MSG_RESULT(no) fi fi + AC_MSG_RESULT([$use_qr]) if test x$XGETTEXT = x; then AC_MSG_WARN("xgettext is required to update qt translations") diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md index 1215962a16..e95a57586f 100644 --- a/contrib/verify-commits/README.md +++ b/contrib/verify-commits/README.md @@ -40,7 +40,7 @@ Import trusted keys In order to check the commit signatures, you must add the trusted PGP keys to your machine. [GnuPG](https://gnupg.org/) may be used to import the trusted keys by running the following command: ```sh -gpg --recv-keys $(<contrib/verify-commits/trusted-keys) +gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys $(<contrib/verify-commits/trusted-keys) ``` Key expiry/revocation diff --git a/src/Makefile.test.include b/src/Makefile.test.include index b4ca01b1fe..6ae15cc553 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -4,11 +4,15 @@ FUZZ_TARGETS = \ + test/fuzz/addr_info_deserialize \ test/fuzz/address_deserialize \ test/fuzz/addrman_deserialize \ test/fuzz/banentry_deserialize \ test/fuzz/bech32 \ test/fuzz/block_deserialize \ + test/fuzz/block_file_info_deserialize \ + test/fuzz/block_filter_deserialize \ + test/fuzz/block_header_and_short_txids_deserialize \ test/fuzz/blockheader_deserialize \ test/fuzz/blocklocator_deserialize \ test/fuzz/blockmerkleroot \ @@ -20,16 +24,30 @@ FUZZ_TARGETS = \ test/fuzz/descriptor_parse \ test/fuzz/diskblockindex_deserialize \ test/fuzz/eval_script \ + test/fuzz/fee_rate_deserialize \ + test/fuzz/flat_file_pos_deserialize \ test/fuzz/inv_deserialize \ + test/fuzz/key_origin_info_deserialize \ + test/fuzz/merkle_block_deserialize \ test/fuzz/messageheader_deserialize \ test/fuzz/netaddr_deserialize \ + test/fuzz/out_point_deserialize \ test/fuzz/parse_iso8601 \ + test/fuzz/partial_merkle_tree_deserialize \ + test/fuzz/partially_signed_transaction_deserialize \ + test/fuzz/prefilled_transaction_deserialize \ test/fuzz/psbt \ + test/fuzz/psbt_input_deserialize \ + test/fuzz/psbt_output_deserialize \ + test/fuzz/pub_key_deserialize \ test/fuzz/script \ + test/fuzz/script_deserialize \ test/fuzz/script_flags \ test/fuzz/service_deserialize \ test/fuzz/spanparsing \ + test/fuzz/sub_net_deserialize \ test/fuzz/transaction \ + test/fuzz/tx_in_deserialize \ test/fuzz/txoutcompressor_deserialize \ test/fuzz/txundo_deserialize @@ -371,6 +389,114 @@ test_fuzz_transaction_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_transaction_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_transaction_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_addr_info_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_addr_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDR_INFO_DESERIALIZE=1 +test_fuzz_addr_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_addr_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_addr_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_block_file_info_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_block_file_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_FILE_INFO_DESERIALIZE=1 +test_fuzz_block_file_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_block_file_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_file_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_block_filter_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_block_filter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_FILTER_DESERIALIZE=1 +test_fuzz_block_filter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_block_filter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_filter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_block_header_and_short_txids_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_block_header_and_short_txids_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_HEADER_AND_SHORT_TXIDS_DESERIALIZE=1 +test_fuzz_block_header_and_short_txids_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_block_header_and_short_txids_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_header_and_short_txids_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_fee_rate_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_fee_rate_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DFEE_RATE_DESERIALIZE=1 +test_fuzz_fee_rate_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_fee_rate_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_fee_rate_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_flat_file_pos_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_flat_file_pos_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DFLAT_FILE_POS_DESERIALIZE=1 +test_fuzz_flat_file_pos_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_flat_file_pos_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_flat_file_pos_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_key_origin_info_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_key_origin_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DKEY_ORIGIN_INFO_DESERIALIZE=1 +test_fuzz_key_origin_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_key_origin_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_key_origin_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_merkle_block_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_merkle_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMERKLE_BLOCK_DESERIALIZE=1 +test_fuzz_merkle_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_merkle_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_merkle_block_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_out_point_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_out_point_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DOUT_POINT_DESERIALIZE=1 +test_fuzz_out_point_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_out_point_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_out_point_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_partially_signed_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_partially_signed_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIALLY_SIGNED_TRANSACTION_DESERIALIZE=1 +test_fuzz_partially_signed_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_partially_signed_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_partially_signed_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_partial_merkle_tree_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_partial_merkle_tree_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIAL_MERKLE_TREE_DESERIALIZE=1 +test_fuzz_partial_merkle_tree_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_partial_merkle_tree_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_partial_merkle_tree_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_prefilled_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_prefilled_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPREFILLED_TRANSACTION_DESERIALIZE=1 +test_fuzz_prefilled_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_prefilled_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_prefilled_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_psbt_input_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_psbt_input_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPSBT_INPUT_DESERIALIZE=1 +test_fuzz_psbt_input_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_psbt_input_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_psbt_input_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_psbt_output_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_psbt_output_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPSBT_OUTPUT_DESERIALIZE=1 +test_fuzz_psbt_output_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_psbt_output_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_psbt_output_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_pub_key_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_pub_key_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPUB_KEY_DESERIALIZE=1 +test_fuzz_pub_key_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_pub_key_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_pub_key_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_script_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_script_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSCRIPT_DESERIALIZE=1 +test_fuzz_script_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_script_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_sub_net_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_sub_net_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSUB_NET_DESERIALIZE=1 +test_fuzz_sub_net_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_sub_net_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_sub_net_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_tx_in_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp +test_fuzz_tx_in_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTX_IN_DESERIALIZE=1 +test_fuzz_tx_in_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_tx_in_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_tx_in_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) + endif # ENABLE_FUZZ nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 592fcbe8dd..1035e730b8 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -256,7 +256,6 @@ public: return batch[ID_BLOCKCHAININFO]; } result.pushKV("version", batch[ID_NETWORKINFO]["result"]["version"]); - result.pushKV("protocolversion", batch[ID_NETWORKINFO]["result"]["protocolversion"]); result.pushKV("blocks", batch[ID_BLOCKCHAININFO]["result"]["blocks"]); result.pushKV("headers", batch[ID_BLOCKCHAININFO]["result"]["headers"]); result.pushKV("verificationprogress", batch[ID_BLOCKCHAININFO]["result"]["verificationprogress"]); @@ -266,9 +265,7 @@ public: result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); if (!batch[ID_WALLETINFO]["result"].isNull()) { - result.pushKV("walletversion", batch[ID_WALLETINFO]["result"]["walletversion"]); result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); - result.pushKV("keypoololdest", batch[ID_WALLETINFO]["result"]["keypoololdest"]); result.pushKV("keypoolsize", batch[ID_WALLETINFO]["result"]["keypoolsize"]); if (!batch[ID_WALLETINFO]["result"]["unlocked_until"].isNull()) { result.pushKV("unlocked_until", batch[ID_WALLETINFO]["result"]["unlocked_until"]); diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 917ecd71c5..a2013edc05 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -41,8 +41,8 @@ static bool WalletAppInit(int argc, char* argv[]) } if (argc < 2 || HelpRequested(gArgs)) { std::string usage = strprintf("%s bitcoin-wallet version", PACKAGE_NAME) + " " + FormatFullVersion() + "\n\n" + - "wallet-tool is an offline tool for creating and interacting with Bitcoin Core wallet files.\n" + - "By default wallet-tool will act on wallets in the default mainnet wallet directory in the datadir.\n" + + "bitcoin-wallet is an offline tool for creating and interacting with Bitcoin Core wallet files.\n" + + "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" + "To change the target wallet, use the -datadir, -wallet and -testnet/-regtest arguments.\n\n" + "Usage:\n" + " bitcoin-wallet [options] <command>\n\n" + diff --git a/src/net.cpp b/src/net.cpp index 84692d2a79..99dae88bab 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -16,6 +16,7 @@ #include <crypto/sha256.h> #include <netbase.h> #include <net_permissions.h> +#include <random.h> #include <scheduler.h> #include <ui_interface.h> #include <util/strencodings.h> @@ -445,6 +446,9 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", false, block_relay_only); pnode->AddRef(); + // We're making a new connection, harvest entropy from the time (and our peer count) + RandAddEvent((uint32_t)id); + return pnode; } @@ -693,6 +697,9 @@ CNetMessage V1TransportDeserializer::GetMessage(const CMessageHeader::MessageSta msg.m_message_size = hdr.nMessageSize; msg.m_raw_message_size = hdr.nMessageSize + CMessageHeader::HEADER_SIZE; + // We just received a message off the wire, harvest entropy from the time (and the message checksum) + RandAddEvent(ReadLE32(hash.begin())); + msg.m_valid_checksum = (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) == 0); if (!msg.m_valid_checksum) { LogPrint(BCLog::NET, "CHECKSUM ERROR (%s, %u bytes), expected %s was %s\n", @@ -1017,6 +1024,9 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { LOCK(cs_vNodes); vNodes.push_back(pnode); } + + // We received a new connection, harvest entropy from the time (and our peer count) + RandAddEvent((uint32_t)id); } void CConnman::DisconnectNodes() diff --git a/src/pubkey.cpp b/src/pubkey.cpp index 10953adc35..21e51a380d 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -171,6 +171,7 @@ bool CPubKey::Verify(const uint256 &hash, const std::vector<unsigned char>& vchS return false; secp256k1_pubkey pubkey; secp256k1_ecdsa_signature sig; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { return false; } @@ -190,6 +191,7 @@ bool CPubKey::RecoverCompact(const uint256 &hash, const std::vector<unsigned cha bool fComp = ((vchSig[0] - 27) & 4) != 0; secp256k1_pubkey pubkey; secp256k1_ecdsa_recoverable_signature sig; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); if (!secp256k1_ecdsa_recoverable_signature_parse_compact(secp256k1_context_verify, &sig, &vchSig[1], recid)) { return false; } @@ -207,6 +209,7 @@ bool CPubKey::IsFullyValid() const { if (!IsValid()) return false; secp256k1_pubkey pubkey; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); return secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size()); } @@ -214,6 +217,7 @@ bool CPubKey::Decompress() { if (!IsValid()) return false; secp256k1_pubkey pubkey; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { return false; } @@ -232,6 +236,7 @@ bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChi BIP32Hash(cc, nChild, *begin(), begin()+1, out); memcpy(ccChild.begin(), out+32, 32); secp256k1_pubkey pubkey; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { return false; } @@ -273,6 +278,7 @@ bool CExtPubKey::Derive(CExtPubKey &out, unsigned int _nChild) const { /* static */ bool CPubKey::CheckLowS(const std::vector<unsigned char>& vchSig) { secp256k1_ecdsa_signature sig; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); if (!ecdsa_signature_parse_der_lax(secp256k1_context_verify, &sig, vchSig.data(), vchSig.size())) { return false; } diff --git a/src/random.cpp b/src/random.cpp index 47d76d8700..50b8477733 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -6,6 +6,7 @@ #include <random.h> #include <compat/cpuid.h> +#include <crypto/sha256.h> #include <crypto/sha512.h> #include <support/cleanse.h> #ifdef WIN32 @@ -178,7 +179,7 @@ static uint64_t GetRdSeed() noexcept /* Access to other hardware random number generators could be added here later, * assuming it is sufficiently fast (in the order of a few hundred CPU cycles). * Slower sources should probably be invoked separately, and/or only from - * RandAddSeedSleep (which is called during idle background operation). + * RandAddPeriodic (which is called once a minute). */ static void InitHardwareRand() {} static void ReportHardwareRand() {} @@ -359,6 +360,9 @@ class RNGState { uint64_t m_counter GUARDED_BY(m_mutex) = 0; bool m_strongly_seeded GUARDED_BY(m_mutex) = false; + Mutex m_events_mutex; + CSHA256 m_events_hasher GUARDED_BY(m_events_mutex); + public: RNGState() noexcept { @@ -369,6 +373,35 @@ public: { } + void AddEvent(uint32_t event_info) noexcept + { + LOCK(m_events_mutex); + + m_events_hasher.Write((const unsigned char *)&event_info, sizeof(event_info)); + // Get the low four bytes of the performance counter. This translates to roughly the + // subsecond part. + uint32_t perfcounter = (GetPerformanceCounter() & 0xffffffff); + m_events_hasher.Write((const unsigned char*)&perfcounter, sizeof(perfcounter)); + } + + /** + * Feed (the hash of) all events added through AddEvent() to hasher. + */ + void SeedEvents(CSHA512& hasher) noexcept + { + // We use only SHA256 for the events hashing to get the ASM speedups we have for SHA256, + // since we want it to be fast as network peers may be able to trigger it repeatedly. + LOCK(m_events_mutex); + + unsigned char events_hash[32]; + m_events_hasher.Finalize(events_hash); + hasher.Write(events_hash, 32); + + // Re-initialize the hasher with the finalized state to use later. + m_events_hasher.Reset(); + m_events_hasher.Write(events_hash, 32); + } + /** Extract up to 32 bytes of entropy from the RNG state, mixing in new entropy from hasher. * * If this function has never been called with strong_seed = true, false is returned. @@ -415,17 +448,7 @@ RNGState& GetRNGState() noexcept /* A note on the use of noexcept in the seeding functions below: * - * None of the RNG code should ever throw any exception, with the sole exception - * of MilliSleep in SeedSleep, which can (and does) support interruptions which - * cause a boost::thread_interrupted to be thrown. - * - * This means that SeedSleep, and all functions that invoke it are throwing. - * However, we know that GetRandBytes() and GetStrongRandBytes() never trigger - * this sleeping logic, so they are noexcept. The same is true for all the - * GetRand*() functions that use GetRandBytes() indirectly. - * - * TODO: After moving away from interruptible boost-based thread management, - * everything can become noexcept here. + * None of the RNG code should ever throw any exception. */ static void SeedTimestamp(CSHA512& hasher) noexcept @@ -449,7 +472,7 @@ static void SeedFast(CSHA512& hasher) noexcept SeedTimestamp(hasher); } -static void SeedSlow(CSHA512& hasher) noexcept +static void SeedSlow(CSHA512& hasher, RNGState& rng) noexcept { unsigned char buffer[32]; @@ -460,6 +483,9 @@ static void SeedSlow(CSHA512& hasher) noexcept GetOSRand(buffer); hasher.Write(buffer, sizeof(buffer)); + // Add the events hasher into the mix + rng.SeedEvents(hasher); + // High-precision timestamp. // // Note that we also commit to a timestamp in the Fast seeder, so we indirectly commit to a @@ -477,7 +503,7 @@ static void SeedStrengthen(CSHA512& hasher, RNGState& rng, int microseconds) noe Strengthen(strengthen_seed, microseconds, hasher); } -static void SeedPeriodic(CSHA512& hasher, RNGState& rng) +static void SeedPeriodic(CSHA512& hasher, RNGState& rng) noexcept { // Everything that the 'fast' seeder includes SeedFast(hasher); @@ -485,6 +511,9 @@ static void SeedPeriodic(CSHA512& hasher, RNGState& rng) // High-precision timestamp SeedTimestamp(hasher); + // Add the events hasher into the mix + rng.SeedEvents(hasher); + // Dynamic environment data (performance monitoring, ...) auto old_size = hasher.Size(); RandAddDynamicEnv(hasher); @@ -500,7 +529,7 @@ static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept SeedHardwareSlow(hasher); // Everything that the 'slow' seeder includes. - SeedSlow(hasher); + SeedSlow(hasher, rng); // Dynamic environment data (performance monitoring, ...) auto old_size = hasher.Size(); @@ -520,7 +549,7 @@ enum class RNGLevel { PERIODIC, //!< Called by RandAddPeriodic() }; -static void ProcRand(unsigned char* out, int num, RNGLevel level) +static void ProcRand(unsigned char* out, int num, RNGLevel level) noexcept { // Make sure the RNG is initialized first (as all Seed* function possibly need hwrand to be available). RNGState& rng = GetRNGState(); @@ -533,7 +562,7 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) SeedFast(hasher); break; case RNGLevel::SLOW: - SeedSlow(hasher); + SeedSlow(hasher, rng); break; case RNGLevel::PERIODIC: SeedPeriodic(hasher, rng); @@ -551,7 +580,8 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::FAST); } void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } -void RandAddPeriodic() { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } +void RandAddPeriodic() noexcept { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } +void RandAddEvent(const uint32_t event_info) noexcept { GetRNGState().AddEvent(event_info); } bool g_mock_deterministic_tests{false}; diff --git a/src/random.h b/src/random.h index 2d8ab085e3..e1b105168d 100644 --- a/src/random.h +++ b/src/random.h @@ -40,17 +40,17 @@ * These entropy sources are slower, but designed to make sure the RNG state contains * fresh data that is unpredictable to attackers. * - * - RandAddSeedSleep() seeds everything that fast seeding includes, but additionally: - * - A high-precision timestamp before and after sleeping 1ms. - * - (On Windows) Once every 10 minutes, performance monitoring data from the OS. - - - Once every minute, strengthen the entropy for 10 ms using repeated SHA512. - * These just exploit the fact the system is idle to improve the quality of the RNG - * slightly. + * - RandAddPeriodic() seeds everything that fast seeding includes, but additionally: + * - A high-precision timestamp + * - Dynamic environment data (performance monitoring, ...) + * - Strengthen the entropy for 10 ms using repeated SHA512. + * This is run once every minute. * * On first use of the RNG (regardless of what function is called first), all entropy * sources used in the 'slow' seeder are included, but also: * - 256 bits from the hardware RNG (rdseed or rdrand) when available. - * - (On Windows) Performance monitoring data from the OS. + * - Dynamic environment data (performance monitoring, ...) + * - Static environment data * - Strengthen the entropy for 100 ms using repeated SHA512. * * When mixing in new entropy, H = SHA512(entropy || old_rng_state) is computed, and @@ -87,7 +87,15 @@ void GetStrongRandBytes(unsigned char* buf, int num) noexcept; * * Thread-safe. */ -void RandAddPeriodic(); +void RandAddPeriodic() noexcept; + +/** + * Gathers entropy from the low bits of the time at which events occur. Should + * be called with a uint32_t describing the event at the time an event occurs. + * + * Thread-safe. + */ +void RandAddEvent(const uint32_t event_info) noexcept; /** * Fast randomness source. This is seeded once with secure random data, but diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 591a317d17..2deb0c5bfc 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -186,14 +186,15 @@ static void TestHKDF_SHA256_32(const std::string &ikm_hex, const std::string &sa BOOST_CHECK(HexStr(out, out + 32) == okm_check_hex); } -static std::string LongTestString() { +static std::string LongTestString() +{ std::string ret; - for (int i=0; i<200000; i++) { - ret += (unsigned char)(i); - ret += (unsigned char)(i >> 4); - ret += (unsigned char)(i >> 8); - ret += (unsigned char)(i >> 12); - ret += (unsigned char)(i >> 16); + for (int i = 0; i < 200000; i++) { + ret += (char)(i); + ret += (char)(i >> 4); + ret += (char)(i >> 8); + ret += (char)(i >> 12); + ret += (char)(i >> 16); } return ret; } diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 57d5b2bb5c..b647c0f70b 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -397,6 +397,18 @@ BOOST_AUTO_TEST_CASE(iterator_string_ordering) } } +BOOST_AUTO_TEST_CASE(unicodepath) +{ + // Attempt to create a database with a utf8 character in the path. + // On Windows this test will fail if the directory is created using + // the ANSI CreateDirectoryA call and the code page isn't UTF8. + // It will succeed if the created with CreateDirectoryW. + fs::path ph = GetDataDir() / "test_runner_₿_🏃_20191128_104644"; + CDBWrapper dbw(ph, (1 << 20)); + + fs::path lockPath = ph / "LOCK"; + BOOST_CHECK(boost::filesystem::exists(lockPath)); +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index bcd8691359..46bc38fdab 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -5,18 +5,24 @@ #include <addrdb.h> #include <addrman.h> #include <blockencodings.h> +#include <blockfilter.h> #include <chain.h> #include <coins.h> #include <compressor.h> #include <consensus/merkle.h> +#include <key.h> +#include <merkleblock.h> #include <net.h> #include <primitives/block.h> #include <protocol.h> +#include <psbt.h> #include <pubkey.h> +#include <script/keyorigin.h> #include <streams.h> #include <undo.h> #include <version.h> +#include <stdexcept> #include <stdint.h> #include <unistd.h> @@ -37,129 +43,237 @@ void test_one_input(const std::vector<uint8_t>& buffer) int nVersion; ds >> nVersion; ds.SetVersion(nVersion); - } catch (const std::ios_base::failure& e) { + } catch (const std::ios_base::failure&) { return; } -#if BLOCK_DESERIALIZE - try - { - CBlock block; - ds >> block; - } catch (const std::ios_base::failure& e) {return;} +#if BLOCK_FILTER_DESERIALIZE + try { + BlockFilter block_filter; + ds >> block_filter; + } catch (const std::ios_base::failure&) { + } +#elif ADDR_INFO_DESERIALIZE + try { + CAddrInfo addr_info; + ds >> addr_info; + } catch (const std::ios_base::failure&) { + } +#elif BLOCK_FILE_INFO_DESERIALIZE + try { + CBlockFileInfo block_file_info; + ds >> block_file_info; + } catch (const std::ios_base::failure&) { + } +#elif BLOCK_HEADER_AND_SHORT_TXIDS_DESERIALIZE + try { + CBlockHeaderAndShortTxIDs block_header_and_short_txids; + ds >> block_header_and_short_txids; + } catch (const std::ios_base::failure&) { + } +#elif FEE_RATE_DESERIALIZE + try { + CFeeRate fee_rate; + ds >> fee_rate; + } catch (const std::ios_base::failure&) { + } +#elif MERKLE_BLOCK_DESERIALIZE + try { + CMerkleBlock merkle_block; + ds >> merkle_block; + } catch (const std::ios_base::failure&) { + } +#elif OUT_POINT_DESERIALIZE + try { + COutPoint out_point; + ds >> out_point; + } catch (const std::ios_base::failure&) { + } +#elif PARTIAL_MERKLE_TREE_DESERIALIZE + try { + CPartialMerkleTree partial_merkle_tree; + ds >> partial_merkle_tree; + } catch (const std::ios_base::failure&) { + } +#elif PUB_KEY_DESERIALIZE + try { + CPubKey pub_key; + ds >> pub_key; + } catch (const std::ios_base::failure&) { + } +#elif SCRIPT_DESERIALIZE + try { + CScript script; + ds >> script; + } catch (const std::ios_base::failure&) { + } +#elif SUB_NET_DESERIALIZE + try { + CSubNet sub_net; + ds >> sub_net; + } catch (const std::ios_base::failure&) { + } +#elif TX_IN_DESERIALIZE + try { + CTxIn tx_in; + ds >> tx_in; + } catch (const std::ios_base::failure&) { + } +#elif FLAT_FILE_POS_DESERIALIZE + try { + FlatFilePos flat_file_pos; + ds >> flat_file_pos; + } catch (const std::ios_base::failure&) { + } +#elif KEY_ORIGIN_INFO_DESERIALIZE + try { + KeyOriginInfo key_origin_info; + ds >> key_origin_info; + } catch (const std::ios_base::failure&) { + } +#elif PARTIALLY_SIGNED_TRANSACTION_DESERIALIZE + try { + PartiallySignedTransaction partially_signed_transaction; + ds >> partially_signed_transaction; + } catch (const std::ios_base::failure&) { + } +#elif PREFILLED_TRANSACTION_DESERIALIZE + try { + PrefilledTransaction prefilled_transaction; + ds >> prefilled_transaction; + } catch (const std::ios_base::failure&) { + } +#elif PSBT_INPUT_DESERIALIZE + try { + PSBTInput psbt_input; + ds >> psbt_input; + } catch (const std::ios_base::failure&) { + } +#elif PSBT_OUTPUT_DESERIALIZE + try { + PSBTOutput psbt_output; + ds >> psbt_output; + } catch (const std::ios_base::failure&) { + } +#elif BLOCK_DESERIALIZE + try { + CBlock block; + ds >> block; + } catch (const std::ios_base::failure&) { + } #elif BLOCKLOCATOR_DESERIALIZE - try - { - CBlockLocator bl; - ds >> bl; - } catch (const std::ios_base::failure& e) {return;} + try { + CBlockLocator bl; + ds >> bl; + } catch (const std::ios_base::failure&) { + } #elif BLOCKMERKLEROOT - try - { - CBlock block; - ds >> block; - bool mutated; - BlockMerkleRoot(block, &mutated); - } catch (const std::ios_base::failure& e) {return;} + try { + CBlock block; + ds >> block; + bool mutated; + BlockMerkleRoot(block, &mutated); + } catch (const std::ios_base::failure&) { + } #elif ADDRMAN_DESERIALIZE - try - { - CAddrMan am; - ds >> am; - } catch (const std::ios_base::failure& e) {return;} + try { + CAddrMan am; + ds >> am; + } catch (const std::ios_base::failure&) { + } #elif BLOCKHEADER_DESERIALIZE - try - { - CBlockHeader bh; - ds >> bh; - } catch (const std::ios_base::failure& e) {return;} + try { + CBlockHeader bh; + ds >> bh; + } catch (const std::ios_base::failure&) { + } #elif BANENTRY_DESERIALIZE - try - { - CBanEntry be; - ds >> be; - } catch (const std::ios_base::failure& e) {return;} + try { + CBanEntry be; + ds >> be; + } catch (const std::ios_base::failure&) { + } #elif TXUNDO_DESERIALIZE - try - { - CTxUndo tu; - ds >> tu; - } catch (const std::ios_base::failure& e) {return;} + try { + CTxUndo tu; + ds >> tu; + } catch (const std::ios_base::failure&) { + } #elif BLOCKUNDO_DESERIALIZE - try - { - CBlockUndo bu; - ds >> bu; - } catch (const std::ios_base::failure& e) {return;} + try { + CBlockUndo bu; + ds >> bu; + } catch (const std::ios_base::failure&) { + } #elif COINS_DESERIALIZE - try - { - Coin coin; - ds >> coin; - } catch (const std::ios_base::failure& e) {return;} + try { + Coin coin; + ds >> coin; + } catch (const std::ios_base::failure&) { + } #elif NETADDR_DESERIALIZE - try - { - CNetAddr na; - ds >> na; - } catch (const std::ios_base::failure& e) {return;} + try { + CNetAddr na; + ds >> na; + } catch (const std::ios_base::failure&) { + } #elif SERVICE_DESERIALIZE - try - { - CService s; - ds >> s; - } catch (const std::ios_base::failure& e) {return;} + try { + CService s; + ds >> s; + } catch (const std::ios_base::failure&) { + } #elif MESSAGEHEADER_DESERIALIZE - CMessageHeader::MessageStartChars pchMessageStart = {0x00, 0x00, 0x00, 0x00}; - try - { - CMessageHeader mh(pchMessageStart); - ds >> mh; - if (!mh.IsValid(pchMessageStart)) {return;} - } catch (const std::ios_base::failure& e) {return;} + CMessageHeader::MessageStartChars pchMessageStart = {0x00, 0x00, 0x00, 0x00}; + try { + CMessageHeader mh(pchMessageStart); + ds >> mh; + (void)mh.IsValid(pchMessageStart); + } catch (const std::ios_base::failure&) { + } #elif ADDRESS_DESERIALIZE - try - { - CAddress a; - ds >> a; - } catch (const std::ios_base::failure& e) {return;} + try { + CAddress a; + ds >> a; + } catch (const std::ios_base::failure&) { + } #elif INV_DESERIALIZE - try - { - CInv i; - ds >> i; - } catch (const std::ios_base::failure& e) {return;} + try { + CInv i; + ds >> i; + } catch (const std::ios_base::failure&) { + } #elif BLOOMFILTER_DESERIALIZE - try - { - CBloomFilter bf; - ds >> bf; - } catch (const std::ios_base::failure& e) {return;} + try { + CBloomFilter bf; + ds >> bf; + } catch (const std::ios_base::failure&) { + } #elif DISKBLOCKINDEX_DESERIALIZE - try - { - CDiskBlockIndex dbi; - ds >> dbi; - } catch (const std::ios_base::failure& e) {return;} + try { + CDiskBlockIndex dbi; + ds >> dbi; + } catch (const std::ios_base::failure&) { + } #elif TXOUTCOMPRESSOR_DESERIALIZE - CTxOut to; - CTxOutCompressor toc(to); - try - { - ds >> toc; - } catch (const std::ios_base::failure& e) {return;} + CTxOut to; + CTxOutCompressor toc(to); + try { + ds >> toc; + } catch (const std::ios_base::failure&) { + } #elif BLOCKTRANSACTIONS_DESERIALIZE - try - { - BlockTransactions bt; - ds >> bt; - } catch (const std::ios_base::failure& e) {return;} + try { + BlockTransactions bt; + ds >> bt; + } catch (const std::ios_base::failure&) { + } #elif BLOCKTRANSACTIONSREQUEST_DESERIALIZE - try - { - BlockTransactionsRequest btr; - ds >> btr; - } catch (const std::ios_base::failure& e) {return;} + try { + BlockTransactionsRequest btr; + ds >> btr; + } catch (const std::ios_base::failure&) { + } #else #error Need at least one fuzz target to compile #endif diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index 76b230ef3c..fefafda36b 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -26,19 +26,31 @@ void test_one_input(const std::vector<uint8_t>& buffer) int nVersion; ds >> nVersion; ds.SetVersion(nVersion); - } catch (const std::ios_base::failure& e) { + } catch (const std::ios_base::failure&) { return; } - bool valid = true; + bool valid_tx = true; const CTransaction tx = [&] { try { return CTransaction(deserialize, ds); - } catch (const std::ios_base::failure& e) { - valid = false; + } catch (const std::ios_base::failure&) { + valid_tx = false; return CTransaction(); } }(); - if (!valid) { + bool valid_mutable_tx = true; + CDataStream ds_mtx(buffer, SER_NETWORK, INIT_PROTO_VERSION); + CMutableTransaction mutable_tx; + try { + int nVersion; + ds_mtx >> nVersion; + ds_mtx.SetVersion(nVersion); + ds_mtx >> mutable_tx; + } catch (const std::ios_base::failure&) { + valid_mutable_tx = false; + } + assert(valid_tx == valid_mutable_tx); + if (!valid_tx) { return; } diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index eb0050a4a3..2f7a3132d8 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -713,6 +713,29 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) t.vout[0].nValue = nDustThreshold; BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + // Disallowed nVersion + t.nVersion = -1; + reason.clear(); + BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "version"); + + t.nVersion = 0; + reason.clear(); + BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "version"); + + t.nVersion = 3; + reason.clear(); + BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "version"); + + // Allowed nVersion + t.nVersion = 1; + BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + + t.nVersion = 2; + BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + // Check dust with odd relay fee to verify rounding: // nDustThreshold = 182 * 3702 / 1000 dustRelayFee = CFeeRate(3702); diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 8f0b495ac4..36588eb7d1 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -108,12 +108,11 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wt return feebumper::Result::OK; } -static CFeeRate EstimateFeeRate(const CWallet& wallet, const CWalletTx& wtx, CCoinControl& coin_control, CAmount& old_fee) +static CFeeRate EstimateFeeRate(const CWallet& wallet, const CWalletTx& wtx, const CAmount old_fee, CCoinControl& coin_control) { // Get the fee rate of the original transaction. This is calculated from // the tx fee/vsize, so it may have been rounded down. Add 1 satoshi to the // result. - old_fee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); CFeeRate feerate(old_fee, txSize); feerate += CFeeRate(1); @@ -309,6 +308,8 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo } } + old_fee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); + if (coin_control.m_feerate) { // The user provided a feeRate argument. // We calculate this here to avoid compiler warning on the cs_wallet lock @@ -319,7 +320,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo } } else { // The user did not provide a feeRate argument - new_coin_control.m_feerate = EstimateFeeRate(wallet, wtx, new_coin_control, old_fee); + new_coin_control.m_feerate = EstimateFeeRate(wallet, wtx, old_fee, new_coin_control); } // Fill in required inputs we are double-spending(all of them) diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 3eaaf3786c..2b9e8fbf2a 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -18,7 +18,7 @@ bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestinat // Generate a new key that is added to wallet CPubKey new_key; - if (!GetKeyFromPool(new_key)) { + if (!GetKeyFromPool(new_key, type)) { error = "Error: Keypool ran out, please call keypoolrefill first"; return false; } @@ -262,24 +262,19 @@ bool LegacyScriptPubKeyMan::EncryptKeys(CKeyingMaterial& vMasterKeyIn) return true; } -bool LegacyScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, int64_t& index, CKeyPool& keypool) +bool LegacyScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) { + if (!CanGetAddresses(internal)) { + return false; + } + if (!ReserveKeyFromKeyPool(index, keypool, internal)) { return false; } + address = GetDestinationForKey(keypool.vchPubKey, type); return true; } -void LegacyScriptPubKeyMan::KeepDestination(int64_t index) -{ - KeepKey(index); -} - -void LegacyScriptPubKeyMan::ReturnDestination(int64_t index, bool internal, const CPubKey& pubkey) -{ - ReturnKey(index, internal, pubkey); -} - void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) { AssertLockHeld(cs_wallet); @@ -460,7 +455,7 @@ size_t LegacyScriptPubKeyMan::KeypoolCountExternalKeys() unsigned int LegacyScriptPubKeyMan::GetKeyPoolSize() const { AssertLockHeld(cs_wallet); - return setInternalKeyPool.size() + setExternalKeyPool.size(); + return setInternalKeyPool.size() + setExternalKeyPool.size() + set_pre_split_keypool.size(); } int64_t LegacyScriptPubKeyMan::GetTimeFirstKey() const @@ -1092,15 +1087,20 @@ void LegacyScriptPubKeyMan::AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const m_pool_key_to_index[pubkey.GetID()] = index; } -void LegacyScriptPubKeyMan::KeepKey(int64_t nIndex) +void LegacyScriptPubKeyMan::KeepDestination(int64_t nIndex, const OutputType& type) { // Remove from key pool WalletBatch batch(m_storage.GetDatabase()); batch.ErasePool(nIndex); + CPubKey pubkey; + bool have_pk = GetPubKey(m_index_to_reserved_key.at(nIndex), pubkey); + assert(have_pk); + LearnRelatedScripts(pubkey, type); + m_index_to_reserved_key.erase(nIndex); WalletLogPrintf("keypool keep %d\n", nIndex); } -void LegacyScriptPubKeyMan::ReturnKey(int64_t nIndex, bool fInternal, const CPubKey& pubkey) +void LegacyScriptPubKeyMan::ReturnDestination(int64_t nIndex, bool fInternal, const CTxDestination&) { // Return to key pool { @@ -1112,13 +1112,15 @@ void LegacyScriptPubKeyMan::ReturnKey(int64_t nIndex, bool fInternal, const CPub } else { setExternalKeyPool.insert(nIndex); } - m_pool_key_to_index[pubkey.GetID()] = nIndex; + CKeyID& pubkey_id = m_index_to_reserved_key.at(nIndex); + m_pool_key_to_index[pubkey_id] = nIndex; + m_index_to_reserved_key.erase(nIndex); NotifyCanGetAddressesChanged(); } WalletLogPrintf("keypool return %d\n", nIndex); } -bool LegacyScriptPubKeyMan::GetKeyFromPool(CPubKey& result, bool internal) +bool LegacyScriptPubKeyMan::GetKeyFromPool(CPubKey& result, const OutputType type, bool internal) { if (!CanGetAddresses(internal)) { return false; @@ -1134,7 +1136,7 @@ bool LegacyScriptPubKeyMan::GetKeyFromPool(CPubKey& result, bool internal) result = GenerateNewKey(batch, internal); return true; } - KeepKey(nIndex); + KeepDestination(nIndex, type); result = keypool.vchPubKey; } return true; @@ -1179,6 +1181,8 @@ bool LegacyScriptPubKeyMan::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& key throw std::runtime_error(std::string(__func__) + ": keypool entry invalid"); } + assert(m_index_to_reserved_key.count(nIndex) == 0); + m_index_to_reserved_key[nIndex] = keypool.vchPubKey.GetID(); m_pool_key_to_index.erase(keypool.vchPubKey.GetID()); WalletLogPrintf("keypool reserve %d\n", nIndex); } diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 4f17156792..6ed9a4787a 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -150,9 +150,9 @@ public: virtual bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) { return false; } virtual isminetype IsMine(const CScript& script) const { return ISMINE_NO; } - virtual bool GetReservedDestination(const OutputType type, bool internal, int64_t& index, CKeyPool& keypool) { return false; } - virtual void KeepDestination(int64_t index) {} - virtual void ReturnDestination(int64_t index, bool internal, const CPubKey& pubkey) {} + virtual bool GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) { return false; } + virtual void KeepDestination(int64_t index, const OutputType& type) {} + virtual void ReturnDestination(int64_t index, bool internal, const CTxDestination& addr) {} virtual bool TopUp(unsigned int size = 0) { return false; } @@ -246,9 +246,11 @@ private: std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_wallet); int64_t m_max_keypool_index GUARDED_BY(cs_wallet) = 0; std::map<CKeyID, int64_t> m_pool_key_to_index; + // Tracks keypool indexes to CKeyIDs of keys that have been taken out of the keypool but may be returned to it + std::map<int64_t, CKeyID> m_index_to_reserved_key; //! Fetches a key from the keypool - bool GetKeyFromPool(CPubKey &key, bool internal = false); + bool GetKeyFromPool(CPubKey &key, const OutputType type, bool internal = false); /** * Reserves a key from the keypool and sets nIndex to its index @@ -266,9 +268,6 @@ private: */ bool ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal); - void KeepKey(int64_t nIndex); - void ReturnKey(int64_t nIndex, bool fInternal, const CPubKey& pubkey); - public: bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) override; isminetype IsMine(const CScript& script) const override; @@ -276,9 +275,9 @@ public: //! will encrypt previously unencrypted keys bool EncryptKeys(CKeyingMaterial& vMasterKeyIn); - bool GetReservedDestination(const OutputType type, bool internal, int64_t& index, CKeyPool& keypool) override; - void KeepDestination(int64_t index) override; - void ReturnDestination(int64_t index, bool internal, const CPubKey& pubkey) override; + bool GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) override; + void KeepDestination(int64_t index, const OutputType& type) override; + void ReturnDestination(int64_t index, bool internal, const CTxDestination&) override; bool TopUp(unsigned int size = 0) override; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 647b381b39..abee497c1d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -822,7 +822,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) void CWallet::LoadToWallet(CWalletTx& wtxIn) { - // If wallet doesn't have a chain (e.g wallet-tool), lock can't be taken. + // If wallet doesn't have a chain (e.g bitcoin-wallet), lock can't be taken. auto locked_chain = LockChain(); if (locked_chain) { Optional<int> block_height = locked_chain->getBlockHeight(wtxIn.m_confirm.hashBlock); @@ -2944,7 +2944,7 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { // Even if we don't use this lock in this function, we want to preserve // lock order in LoadToWallet if query of chain state is needed to know - // tx status. If lock can't be taken (e.g wallet-tool), tx confirmation + // tx status. If lock can't be taken (e.g bitcoin-wallet), tx confirmation // status may be not reliable. auto locked_chain = LockChain(); LOCK(cs_wallet); @@ -3295,21 +3295,15 @@ bool ReserveDestination::GetReservedDestination(CTxDestination& dest, bool inter return false; } - if (!pwallet->CanGetAddresses(internal)) { - return false; - } if (nIndex == -1) { CKeyPool keypool; - if (!m_spk_man->GetReservedDestination(type, internal, nIndex, keypool)) { + if (!m_spk_man->GetReservedDestination(type, internal, address, nIndex, keypool)) { return false; } - vchPubKey = keypool.vchPubKey; fInternal = keypool.fInternal; } - assert(vchPubKey.IsValid()); - address = GetDestinationForKey(vchPubKey, type); dest = address; return true; } @@ -3317,21 +3311,18 @@ bool ReserveDestination::GetReservedDestination(CTxDestination& dest, bool inter void ReserveDestination::KeepDestination() { if (nIndex != -1) { - m_spk_man->KeepDestination(nIndex); - m_spk_man->LearnRelatedScripts(vchPubKey, type); + m_spk_man->KeepDestination(nIndex, type); } nIndex = -1; - vchPubKey = CPubKey(); address = CNoDestination(); } void ReserveDestination::ReturnDestination() { if (nIndex != -1) { - m_spk_man->ReturnDestination(nIndex, fInternal, vchPubKey); + m_spk_man->ReturnDestination(nIndex, fInternal, address); } nIndex = -1; - vchPubKey = CPubKey(); address = CNoDestination(); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 51df4a9a8b..b02e092f0a 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -139,12 +139,11 @@ class ReserveDestination protected: //! The wallet to reserve from CWallet* const pwallet; - LegacyScriptPubKeyMan* m_spk_man{nullptr}; + //! The ScriptPubKeyMan to reserve from. Based on type when GetReservedDestination is called + ScriptPubKeyMan* m_spk_man{nullptr}; OutputType const type; //! The index of the address's key in the keypool int64_t nIndex{-1}; - //! The public key for the address - CPubKey vchPubKey; //! The destination CTxDestination address; //! Whether this is from the internal (change output) keypool diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index ec1c88ed53..f04a58cd19 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -59,7 +59,6 @@ class TestBitcoinCli(BitcoinTestFramework): blockchain_info = self.nodes[0].getblockchaininfo() assert_equal(cli_get_info['version'], network_info['version']) - assert_equal(cli_get_info['protocolversion'], network_info['protocolversion']) assert_equal(cli_get_info['blocks'], blockchain_info['blocks']) assert_equal(cli_get_info['timeoffset'], network_info['timeoffset']) assert_equal(cli_get_info['connections'], network_info['connections']) @@ -67,9 +66,7 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['difficulty'], blockchain_info['difficulty']) assert_equal(cli_get_info['chain'], blockchain_info['chain']) if self.is_wallet_compiled(): - assert_equal(cli_get_info['walletversion'], wallet_info['walletversion']) assert_equal(cli_get_info['balance'], wallet_info['balance']) - assert_equal(cli_get_info['keypoololdest'], wallet_info['keypoololdest']) assert_equal(cli_get_info['keypoolsize'], wallet_info['keypoolsize']) assert_equal(cli_get_info['paytxfee'], wallet_info['paytxfee']) assert_equal(cli_get_info['relayfee'], network_info['relayfee']) diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index a036dfc790..797fcc828a 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -151,7 +151,7 @@ class RESTTest (BitcoinTestFramework): bin_response = self.test_rest_request("/getutxos", http_method='POST', req_type=ReqType.BIN, body=bin_request, ret_type=RetType.BYTES) output = BytesIO(bin_response) - chain_height, = unpack("i", output.read(4)) + chain_height, = unpack("<i", output.read(4)) response_hash = output.read(32)[::-1].hex() assert_equal(bb_hash, response_hash) # check if getutxo's chaintip during calculation was fine diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 6b6bbfd1f9..f468f9eaec 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -12,6 +12,7 @@ import os import pdb import random import shutil +import subprocess import sys import tempfile import time @@ -121,6 +122,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): except KeyError: self.log.exception("Key error") self.success = TestStatus.FAILED + except subprocess.CalledProcessError as e: + self.log.exception("Called Process failed with '{}'".format(e.output)) + self.success = TestStatus.FAILED except Exception: self.log.exception("Unexpected exception caught during testing") self.success = TestStatus.FAILED diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index e86679bc31..1122daaf83 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -18,6 +18,7 @@ from test_framework.util import ( assert_raises_rpc_error, connect_nodes, disconnect_nodes, + wait_until, ) @@ -97,6 +98,7 @@ class AbandonConflictTest(BitcoinTestFramework): # TODO: redo with eviction self.stop_node(0) self.start_node(0, extra_args=["-minrelaytxfee=0.0001"]) + wait_until(lambda: self.nodes[0].getmempoolinfo()['loaded']) # Verify txs no longer in either node's mempool assert_equal(len(self.nodes[0].getrawmempool()), 0) @@ -124,6 +126,8 @@ class AbandonConflictTest(BitcoinTestFramework): # Verify that even with a low min relay fee, the tx is not reaccepted from wallet on startup once abandoned self.stop_node(0) self.start_node(0, extra_args=["-minrelaytxfee=0.00001"]) + wait_until(lambda: self.nodes[0].getmempoolinfo()['loaded']) + assert_equal(len(self.nodes[0].getrawmempool()), 0) assert_equal(self.nodes[0].getbalance(), balance) @@ -144,6 +148,7 @@ class AbandonConflictTest(BitcoinTestFramework): # Remove using high relay fee again self.stop_node(0) self.start_node(0, extra_args=["-minrelaytxfee=0.0001"]) + wait_until(lambda: self.nodes[0].getmempoolinfo()['loaded']) assert_equal(len(self.nodes[0].getrawmempool()), 0) newbalance = self.nodes[0].getbalance() assert_equal(newbalance, balance - Decimal("24.9996")) diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 9d6aa36c35..0c08655833 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -101,7 +101,8 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): else: bumped_tx = rbf_node.bumpfee(rbfid) assert_equal(bumped_tx["errors"], []) - assert bumped_tx["fee"] - abs(rbftx["fee"]) > 0 + assert bumped_tx["fee"] > -rbftx["fee"] + assert_equal(bumped_tx["origfee"], -rbftx["fee"]) # check that bumped_tx propagates, original tx was evicted and has a wallet conflict self.sync_mempools((rbf_node, peer_node)) assert bumped_tx["txid"] in rbf_node.getrawmempool() diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index fde99fe496..ffebb579e7 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -12,6 +12,27 @@ import sys import subprocess import logging +# Fuzzers known to lack a seed corpus in https://github.com/bitcoin-core/qa-assets/tree/master/fuzz_seed_corpus +FUZZERS_MISSING_CORPORA = [ + "addr_info_deserialize", + "block_file_info_deserialize", + "block_filter_deserialize", + "block_header_and_short_txids_deserialize", + "fee_rate_deserialize", + "flat_file_pos_deserialize", + "key_origin_info_deserialize", + "merkle_block_deserialize", + "out_point_deserialize", + "partial_merkle_tree_deserialize", + "partially_signed_transaction_deserialize", + "prefilled_transaction_deserialize", + "psbt_input_deserialize", + "psbt_output_deserialize", + "pub_key_deserialize", + "script_deserialize", + "sub_net_deserialize", + "tx_in_deserialize", +] def main(): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -100,10 +121,14 @@ def main(): def run_once(*, corpus, test_list, build_dir, export_coverage): for t in test_list: + corpus_path = os.path.join(corpus, t) + if t in FUZZERS_MISSING_CORPORA: + os.makedirs(corpus_path, exist_ok=True) args = [ os.path.join(build_dir, 'src', 'test', 'fuzz', t), '-runs=1', - os.path.join(corpus, t), + '-detect_leaks=0', + corpus_path, ] logging.debug('Run {} with args {}'.format(t, args)) result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True) diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index e7c690fabe..f5de358bcb 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -42,3 +42,45 @@ unsigned-integer-overflow:stl_bvector.h unsigned-integer-overflow:txmempool.cpp unsigned-integer-overflow:util/strencodings.cpp unsigned-integer-overflow:validation.cpp + +implicit-integer-sign-change:*/include/c++/*/bits/*.h +implicit-integer-sign-change:*/new_allocator.h +implicit-integer-sign-change:/usr/include/boost/date_time/format_date_parser.hpp +implicit-integer-sign-change:arith_uint256.cpp +implicit-integer-sign-change:bech32.cpp +implicit-integer-sign-change:bloom.cpp +implicit-integer-sign-change:chain.* +implicit-integer-sign-change:coins.h +implicit-integer-sign-change:compat/stdin.cpp +implicit-integer-sign-change:compressor.h +implicit-integer-sign-change:crypto/* +implicit-integer-sign-change:key.cpp +implicit-integer-sign-change:noui.cpp +implicit-integer-sign-change:prevector.h +implicit-integer-sign-change:protocol.cpp +implicit-integer-sign-change:script/bitcoinconsensus.cpp +implicit-integer-sign-change:script/interpreter.cpp +implicit-integer-sign-change:serialize.h +implicit-integer-sign-change:test/arith_uint256_tests.cpp +implicit-integer-sign-change:test/coins_tests.cpp +implicit-integer-sign-change:test/pow_tests.cpp +implicit-integer-sign-change:test/prevector_tests.cpp +implicit-integer-sign-change:test/sighash_tests.cpp +implicit-integer-sign-change:test/streams_tests.cpp +implicit-integer-sign-change:test/transaction_tests.cpp +implicit-integer-sign-change:txmempool.cpp +implicit-integer-sign-change:util/strencodings.* +implicit-integer-sign-change:validation.cpp +implicit-integer-sign-change:zmq/zmqpublishnotifier.cpp +implicit-signed-integer-truncation,implicit-integer-sign-change:chain.h +implicit-signed-integer-truncation,implicit-integer-sign-change:test/skiplist_tests.cpp +implicit-signed-integer-truncation:chain.h +implicit-signed-integer-truncation:crypto/* +implicit-signed-integer-truncation:cuckoocache.h +implicit-signed-integer-truncation:leveldb/* +implicit-signed-integer-truncation:streams.h +implicit-signed-integer-truncation:test/arith_uint256_tests.cpp +implicit-signed-integer-truncation:test/skiplist_tests.cpp +implicit-signed-integer-truncation:torcontrol.cpp +implicit-unsigned-integer-truncation:crypto/* +implicit-unsigned-integer-truncation:leveldb/* |