diff options
365 files changed, 7968 insertions, 3618 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index b839bfa5fb..44b49bb397 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,33 +1,3 @@ -task: - name: "FreeBsd 12.1 amd64 [GOAL: install] [no depends, only system libs]" - freebsd_instance: - image_family: freebsd-12-1 # https://cirrus-ci.org/guide/FreeBSD/ - cpu: 8 - memory: 8G - timeout_in: 60m - env: - MAKEJOBS: "-j9" - CONFIGURE_OPTS: "--disable-dependency-tracking" - GOAL: "install" - TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache - CCACHE_SIZE: "200M" - CCACHE_COMPRESS: 1 - CCACHE_DIR: "/tmp/ccache_dir" - ccache_cache: - folder: "/tmp/ccache_dir" - install_script: - - pkg install -y autoconf automake boost-libs git gmake libevent libtool pkgconf python3 ccache - - ./contrib/install_db4.sh $(pwd) - - ccache --max-size=${CCACHE_SIZE} - configure_script: - - ./autogen.sh - - ./configure ${CONFIGURE_OPTS} BDB_LIBS="-L$(pwd)/db4/lib -ldb_cxx-4.8" BDB_CFLAGS="-I$(pwd)/db4/include" || ( cat config.log && false) - make_script: - - gmake ${MAKEJOBS} ${GOAL} || ( echo "Build failure. Verbose build follows." && gmake ${GOAL} V=1 ; false ) - check_script: - - gmake check ${MAKEJOBS} VERBOSE=1 - functional_test_script: - - ./test/functional/test_runner.py --jobs 9 --ci --extended --exclude feature_dbcrash --combinedlogslen=1000 --quiet --failfast #task: # name: "Windows" # windows_container: @@ -60,9 +30,5 @@ task: folder: "/tmp/ccache_dir" depends_built_cache: folder: "/tmp/cirrus-ci-build/depends/built" - install_script: - - apt-get update - - apt-get -y install git bash ccache - - ccache --max-size=${CCACHE_SIZE} ci_script: - ./ci/test_run_all.sh diff --git a/.fuzzbuzz.yml b/.fuzzbuzz.yml new file mode 100644 index 0000000000..d44ac27eb9 --- /dev/null +++ b/.fuzzbuzz.yml @@ -0,0 +1,16 @@ +base: ubuntu:16.04 +language: c++ +engine: libFuzzer +environment: + - CXXFLAGS=-fcoverage-mapping -fno-omit-frame-pointer -fprofile-instr-generate -gline-tables-only -O1 +setup: + - sudo apt-get update + - sudo apt-get install -y autoconf bsdmainutils clang git libboost-all-dev libboost-program-options-dev libc++1 libc++abi1 libc++abi-dev libc++-dev libclang1 libclang-dev libdb5.3++ libevent-dev libllvm-ocaml-dev libomp5 libomp-dev libprotobuf-dev libqt5core5a libqt5dbus5 libqt5gui5 libssl-dev libtool llvm llvm-dev llvm-runtime pkg-config protobuf-compiler qttools5-dev qttools5-dev-tools software-properties-common + - ./autogen.sh + - CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined + - make + - git clone https://github.com/bitcoin-core/qa-assets +auto_targets: + find_targets_command: find src/test/fuzz/ -executable -type f ! -name "*.cpp" ! -name "*.h" + base_corpus_dir: qa-assets/fuzz_seed_corpus/ + memory_limit: none diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.md b/.github/ISSUE_TEMPLATE/good_first_issue.md index c441f7a307..8be78a1f6e 100644 --- a/.github/ISSUE_TEMPLATE/good_first_issue.md +++ b/.github/ISSUE_TEMPLATE/good_first_issue.md @@ -7,6 +7,9 @@ assignees: '' --- +<!-- A good first issue is an uncontroversial issue, that has a relatively unique and obvious solution --> + +<!-- Motivate the issue and explain the solution briefly --> #### Useful skills: @@ -14,8 +17,4 @@ assignees: '' #### Want to work on this issue? -The purpose of the `good first issue` label is to highlight which issues are suitable for a new contributor without a deep understanding of the codebase. - -You do not need to request permission to start working on this. You are encouraged to comment on the issue if you are planning to work on it. This will help other contributors monitor which issues are actively being addressed and is also an effective way to request assistance if and when you need it. - For guidance on contributing, please read [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md) before opening your pull request. diff --git a/.gitignore b/.gitignore index 1c487f43a7..23b6090265 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ src/bitcoin src/bitcoind src/bitcoin-cli +src/bitcoin-gui +src/bitcoin-node src/bitcoin-tx src/bitcoin-wallet src/test/fuzz @@ -125,6 +127,7 @@ linux-build win32-build test/config.ini test/cache/* +test/.mypy_cache/ !src/leveldb*/Makefile diff --git a/.travis.yml b/.travis.yml index ab066d093a..03ab9d2613 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,6 @@ stages: env: global: - CI_RETRY_EXE="travis_retry" - - CI_WAIT="while sleep 500; do echo .; done" - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. See https://docs.travis-ci.com/user/customizing-the-build#build-timeouts . Please manually re-run this job by using the travis restart button. The next run should not time out because the build cache has been saved." before_install: - set -o errexit; source ./ci/test/00_setup_env.sh @@ -113,10 +112,11 @@ jobs: FILE_ENV="./ci/test/00_setup_env_native_qt5.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [xenial] [no depends, only system libs, sanitizers: thread (TSan), no wallet]' + name: 'x86_64 Linux [GOAL: install] [focal] [depends, sanitizers: thread (TSan), no gui]' + # Not enough memory on travis machines, so feature_block is excluded for now env: >- + TEST_RUNNER_EXTRA="--exclude feature_block" FILE_ENV="./ci/test/00_setup_env_native_tsan.sh" - TEST_RUNNER_EXTRA="--exclude feature_block" # Not enough memory on travis machines - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]' @@ -129,6 +129,12 @@ jobs: FILE_ENV="./ci/test/00_setup_env_native_fuzz.sh" - stage: test + name: 'x86_64 Linux [GOAL: install] [bionic] [multiprocess]' + if: type != pull_request OR commit_message =~ /depends:|multiprocess:/ # Skip on non-depends, non-multiprocess PRs + env: >- + FILE_ENV="./ci/test/00_setup_env_native_multiprocess.sh" + + - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no wallet]' env: >- FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7216db0500..f34b1e1e15 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,15 +14,49 @@ purposes. As such there are repository "maintainers" who are responsible for merging pull requests as well as a "lead maintainer" who is responsible for the release cycle, overall merging, moderation and appointment of maintainers. +Getting Started +--------------- + +New contributors are very welcome and needed. + +Reviewing and testing is the most effective way you can contribute as a new +contributor, and it also will teach you much more about the code and process +than opening PRs. Please refer to the section [peer review](#peer-review) later +in this document. + +Before you start contributing, familiarize yourself with the Bitcoin Core build +system and tests. Refer to the documentation in the repository on how to build +Bitcoin Core and how to run the unit and functional tests. + +There are many open issues of varying difficulty waiting to be fixed. If you're looking for somewhere to start contributing, check out the [good first issue](https://github.com/bitcoin/bitcoin/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) -list or participate in a weekly [Bitcoin Core PR Review Club](https://bitcoincore.reviews/) meeting. +list or changes that are +[up for grabs](https://github.com/bitcoin/bitcoin/issues?utf8=%E2%9C%93&q=label%3A%22Up+for+grabs%22). +Some of them might no longer be applicable. So if you are interested, but +unsure, you might want to leave a comment on the issue first. + +You may also participate in the weekly +[Bitcoin Core PR Review Club](https://bitcoincore.reviews/) meeting. + +### Good First Issue Label + +The purpose of the `good first issue` label is to highlight which issues are +suitable for a new contributor without a deep understanding of the codebase. + +However, good first issues can be solved by anyone. If they remain unsolved +for a longer time, a frequent contributor might address them. + +You do not need to request permission to start working on an issue. However, +you are encouraged to leave a comment if you are planning to work on it. This +will help other contributors monitor which issues are actively being addressed +and is also an effective way to request assistance if and when you need it. Communication Channels ---------------------- Most communication about Bitcoin Core development happens on IRC, in the -#bitcoin-core-dev channel on Freenode. The easiest way to participate on IRC is +`#bitcoin-core-dev` channel on Freenode. The easiest way to participate on IRC is with the web client, [webchat.freenode.net](https://webchat.freenode.net/). Chat history logs can be found on [http://www.erisian.com.au/bitcoin-core-dev/](http://www.erisian.com.au/bitcoin-core-dev/) @@ -33,7 +67,7 @@ requests. The developer [mailing list](https://lists.linuxfoundation.org/mailman/listinfo/bitcoin-dev) -should be used to discuss complicated or controversial changes before working on +should be used to discuss complicated or controversial consensus or P2P protocol changes before working on a patch set. @@ -53,6 +87,8 @@ To contribute a patch, the workflow is as follows: The project coding conventions in the [developer notes](doc/developer-notes.md) must be followed. +### Committing Patches + In general, [commits should be atomic](https://en.wikipedia.org/wiki/Atomic_commit#Atomic_commit_convention) and diffs should be easy to read. For this reason, do not mix any formatting fixes or code moves with actual code changes. @@ -76,6 +112,8 @@ about Git. - Push changes to your fork - Create pull request +### Creating the Pull Request + The title of the pull request should be prefixed by the component or area that the pull request affects. Valid areas as: @@ -100,18 +138,24 @@ Examples: qt: Add feed bump button log: Fix typo in log message +The body of the pull request should contain enough description about what the +patch does together with any justification/reasoning. You should include +references to any discussions (for example other tickets or mailing list +discussions). + +### Translation changes + Note that translations should not be submitted as pull requests. Please see [Translation Process](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md) for more information on helping with translations. +### Work in Progress Changes and Requests for Comments + If a pull request is not to be considered for merging (yet), please prefix the title with [WIP] or use [Tasks Lists](https://help.github.com/articles/basic-writing-and-formatting-syntax/#task-lists) in the body of the pull request to indicate tasks are pending. -The body of the pull request should contain enough description about what the -patch does together with any justification/reasoning. You should include -references to any discussions (for example other tickets or mailing list -discussions). +### Address Feedback At this stage, one should expect comments and review from other contributors. You can add more commits to your pull request by committing them locally and pushing diff --git a/Makefile.am b/Makefile.am index 43790f1c23..45dab3930d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -41,7 +41,7 @@ OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus OSX_FANCY_PLIST=$(top_srcdir)/contrib/macdeploy/fancy.plist OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed -OSX_QT_TRANSLATIONS = da,de,es,hu,ru,uk,zh_CN,zh_TW +OSX_QT_TRANSLATIONS = ar,bg,ca,cs,da,de,es,fa,fi,fr,gd,gl,he,hu,it,ja,ko,lt,lv,pl,pt,ru,sk,sl,sv,uk,zh_CN,zh_TW DIST_CONTRIB = \ $(top_srcdir)/contrib/linearize/linearize-data.py \ diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index c8b2fb1658..35ba8425b3 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -41,9 +41,6 @@ /* Define to 1 to enable ZMQ functions */ #define ENABLE_ZMQ 1 -/* parameter and return value type for __fdelt_chk */ -/* #undef FDELT_TYPE */ - /* define if the Boost library is available */ #define HAVE_BOOST /**/ diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 8b2d609504..26b576c1ae 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -9,6 +9,7 @@ export LC_ALL=C travis_retry pip3 install codespell==1.15.0 travis_retry pip3 install flake8==3.7.8 travis_retry pip3 install yq +travis_retry pip3 install mypy==0.700 SHELLCHECK_VERSION=v0.6.0 curl -s "https://storage.googleapis.com/shellcheck/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index dae61c5e34..56b8fc976e 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -33,13 +33,15 @@ export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} -export TEST_PREVIOUS_RELEASES=${TEST_PREVIOUS_RELEASES:-false} +export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04} # Randomize test order. # See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/random.html export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} +# See man 7 debconf +export DEBIAN_FRONTEND=noninteractive export CCACHE_SIZE=${CCACHE_SIZE:-100M} export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh new file mode 100644 index 0000000000..0fc989085c --- /dev/null +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_native_multiprocess +export PACKAGES="cmake python3" +export DEP_OPTS="MULTIPROCESS=1" +export GOAL="install" +export BITCOIN_CONFIG="" +export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 4dedea23bc..fa07990756 100644 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -9,9 +9,9 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" -export TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash +export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export TEST_PREVIOUS_RELEASES=true +export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.1 v0.18.1 v0.19.1" export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-c++17 --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 73ab5eebb6..87b9d9da95 100644 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -7,8 +7,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export DOCKER_NAME_TAG=ubuntu:16.04 -export PACKAGES="clang-8 llvm-8 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" -export NO_DEPENDS=1 +export DOCKER_NAME_TAG=ubuntu:20.04 +export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq" +export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang-8 CXX=clang++-8" +export BITCOIN_CONFIG="--enable-zmq --with-gui=no CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++'" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 153a781b0a..ff7c5fe913 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -10,10 +10,6 @@ export CONTAINER_NAME=ci_native_valgrind export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -if [[ "${TRAVIS}" == "true" && "${TRAVIS_REPO_SLUG}" != "bitcoin/bitcoin" ]]; then - export TEST_RUNNER_EXTRA="wallet_disable" # Only run wallet_disable as a smoke test to not hit the 50 min travis time limit -else - export TEST_RUNNER_EXTRA="--exclude rpc_bind --factor=2" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 -fi +export TEST_RUNNER_EXTRA="--exclude rpc_bind --factor=2" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/ci/test/03_before_install.sh b/ci/test/03_before_install.sh index e939b9eeeb..80806aab75 100755 --- a/ci/test/03_before_install.sh +++ b/ci/test/03_before_install.sh @@ -6,10 +6,6 @@ export LC_ALL=C.UTF-8 -# Add llvm-symbolizer directory to PATH. Needed to get symbolized stack traces from the sanitizers. -PATH=$PATH:/usr/lib/llvm-6.0/bin/ -export PATH - BEGIN_FOLD () { echo "" CURRENT_FOLD_NAME=$1 diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 5dbf1b82f1..165983d906 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -24,9 +24,9 @@ mkdir -p "${PREVIOUS_RELEASES_DIR}" export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" -export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" +export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" -env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|TEST_PREVIOUS_RELEASES|PREVIOUS_RELEASES_DIR)' | tee /tmp/env +env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env if [[ $HOST = *-mingw32 ]]; then DOCKER_ADMIN="--cap-add SYS_ADMIN" elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) @@ -48,16 +48,14 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then --env-file /tmp/env \ --name $CONTAINER_NAME \ $DOCKER_NAME_TAG) - - DOCKER_EXEC () { - docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" - } + export DOCKER_CI_CMD_PREFIX="docker exec $DOCKER_ID" else echo "Running on host system without docker wrapper" - DOCKER_EXEC () { - bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" - } fi + +DOCKER_EXEC () { + $DOCKER_CI_CMD_PREFIX bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" +} export -f DOCKER_EXEC if [ -n "$DPKG_ADD_ARCH" ]; then diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index ab9d673101..efaaf154b1 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -33,10 +33,12 @@ if [ -z "$NO_DEPENDS" ]; then else SHELL_OPTS="CONFIG_SHELL=" fi - DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS + # Temporary workaround for https://github.com/bitcoin/bitcoin/issues/16368 + python3 -c 'import time; [print(".") or time.sleep(500) for _ in range(4)]' & + ( DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS ) &> /dev/null fi -if [ "$TEST_PREVIOUS_RELEASES" = "true" ]; then +if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then BEGIN_FOLD previous-versions - DOCKER_EXEC contrib/devtools/previous_release.sh -b -t "$PREVIOUS_RELEASES_DIR" v0.15.2 v0.16.3 v0.17.1 v0.18.1 v0.19.0.1 + DOCKER_EXEC contrib/devtools/previous_release.sh -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" END_FOLD fi diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index c8542862f2..51d84ee39d 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -37,12 +37,12 @@ fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then BEGIN_FOLD functional-tests - DOCKER_EXEC test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast + DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast END_FOLD fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then BEGIN_FOLD fuzz-tests - DOCKER_EXEC test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} -l DEBUG ${DIR_FUZZ_IN} + DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN} END_FOLD fi diff --git a/configure.ac b/configure.ac index cf04faf6d1..3bfbe2e7b0 100644 --- a/configure.ac +++ b/configure.ac @@ -69,7 +69,7 @@ AC_ARG_ENABLE([c++17], [use_cxx17=no]) dnl Require C++11 or C++17 compiler (no GNU extensions) -if test "x$use_cxx17" = xyes; then +if test "x$use_cxx17" = xyes -o "x$enable_fuzz" = xyes ; then AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) else AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory]) @@ -242,6 +242,24 @@ if test x$enable_bip70 != xno; then AC_MSG_ERROR([BIP70 is no longer supported!]) fi +AC_ARG_WITH([libmultiprocess], + [AS_HELP_STRING([--with-libmultiprocess=yes|no|auto], + [Build with libmultiprocess library. (default: auto, i.e. detect with pkg-config)])], + [with_libmultiprocess=$withval], + [with_libmultiprocess=auto]) + +AC_ARG_WITH([mpgen], + [AS_HELP_STRING([--with-mpgen=yes|no|auto|PREFIX], + [Build with libmultiprocess codegen tool. Useful to specify different libmultiprocess host system library and build system codegen tool prefixes when cross-compiling (default is host system libmultiprocess prefix)])], + [with_mpgen=$withval], + [with_mpgen=auto]) + +AC_ARG_ENABLE([multiprocess], + [AS_HELP_STRING([--enable-multiprocess], + [build multiprocess bitcoin-node, bitcoin-wallet, and bitcoin-gui executables in addition to monolithic bitcoind and bitcoin-qt executables. Requires libmultiprocess library. Experimental (default is no)])], + [enable_multiprocess=$enableval], + [enable_multiprocess=no]) + AC_ARG_ENABLE(man, [AS_HELP_STRING([--disable-man], [do not install man pages (default is to install)])],, @@ -360,13 +378,21 @@ if test "x$enable_werror" = "xyes"; then if test "x$CXXFLAG_WERROR" = "x"; then AC_MSG_ERROR("enable-werror set but -Werror is not usable") fi + AX_CHECK_COMPILE_FLAG([-Werror=gnu],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=gnu"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=vla],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=vla"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=shadow-field],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=shadow-field"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=switch],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=switch"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=thread-safety-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety-analysis"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=thread-safety],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=unused-variable],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unused-variable"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=date-time],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=date-time"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=return-type],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=return-type"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=conditional-uninitialized],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=conditional-uninitialized"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=sign-compare],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=sign-compare"],,[[$CXXFLAG_WERROR]]) + dnl -Wsuggest-override is broken with GCC before 9.2 + dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78010 + AX_CHECK_COMPILE_FLAG([-Werror=suggest-override],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=suggest-override"],,[[$CXXFLAG_WERROR]], + [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) + AX_CHECK_COMPILE_FLAG([-Werror=unreachable-code-loop-increment],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) fi if test "x$CXXFLAGS_overridden" = "xno"; then @@ -375,14 +401,19 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wgnu],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wgnu"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wformat],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wvla],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wshadow-field],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wswitch],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wswitch"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wformat-security],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat-security"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wthread-safety-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety-analysis"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wthread-safety],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wunused-variable],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-variable"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wdate-time],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wsign-compare],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsign-compare"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wsuggest-override],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"],,[[$CXXFLAG_WERROR]], + [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) + AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) dnl Some compilers (gcc) ignore unknown -Wno-* options, but warn about all dnl unknown options if any other warning is produced. Test the -Wfoo case, and @@ -392,8 +423,12 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wdeprecated-register],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-register"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-implicit-fallthrough"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"],,[[$CXXFLAG_WERROR]]) fi +dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. +AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers],[[CXXFLAGS="$CXXFLAGS -fno-extended-identifiers"]],,[[$CXXFLAG_WERROR]]) + enable_sse42=no enable_sse41=no enable_avx2=no @@ -576,9 +611,6 @@ case $host in fi CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB -D_WIN32_WINNT=0x0601" - if test "x$CXXFLAGS_overridden" = "xno"; then - CXXFLAGS="$CXXFLAGS -w" - fi dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against. dnl That breaks our ability to build dll's with static libgcc/libstdc++/libssp. Override @@ -588,6 +620,8 @@ case $host in archive_cmds_CXX="\$CC -shared \$libobjs \$deplibs \$compiler_flags -static -o \$output_objdir/\$soname \${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker \$lib" postdeps_CXX= + dnl We require Windows 7 (NT 6.1) or later + AX_CHECK_LINK_FLAG([[-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1]],[LDFLAGS="$LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"],,[[$LDFLAG_WERROR]]) ;; *darwin*) TARGET_OS=darwin @@ -717,22 +751,8 @@ AX_GCC_FUNC_ATTRIBUTE([dllexport]) AX_GCC_FUNC_ATTRIBUTE([dllimport]) if test x$use_glibc_compat != xno; then - - dnl __fdelt_chk's params and return type have changed from long unsigned int to long int. - dnl See which one is present here. - AC_MSG_CHECKING(__fdelt_chk type) - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#ifdef _FORTIFY_SOURCE - #undef _FORTIFY_SOURCE - #endif - #define _FORTIFY_SOURCE 2 - #include <sys/select.h> - extern "C" long unsigned int __fdelt_warn(long unsigned int);]],[[]])], - [ fdelt_type="long unsigned int"], - [ fdelt_type="long int"]) - AC_MSG_RESULT($fdelt_type) - AC_DEFINE_UNQUOTED(FDELT_TYPE, $fdelt_type,[parameter and return value type for __fdelt_chk]) - AX_CHECK_LINK_FLAG([[-Wl,--wrap=__divmoddi4]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=__divmoddi4"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,--wrap=log2f]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=log2f"],, [[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([[-Wl,--wrap=__divmoddi4]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=__divmoddi4"]) + AX_CHECK_LINK_FLAG([[-Wl,--wrap=log2f]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=log2f"]) else AC_SEARCH_LIBS([clock_gettime],[rt]) fi @@ -795,7 +815,7 @@ if test x$use_hardening != xno; then fi dnl These flags are specific to ld64, and may cause issues with other linkers. -dnl For example: GNU ld will intepret -dead_strip as -de and then try and use +dnl For example: GNU ld will interpret -dead_strip as -de and then try and use dnl "ad_strip" as the symbol for the entry point. if test x$TARGET_OS = xdarwin; then AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"],, [[$LDFLAG_WERROR]]) @@ -811,39 +831,6 @@ fi AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) -dnl FD_ZERO may be dependent on a declaration of memcpy, e.g. in SmartOS -dnl check that it fails to build without memcpy, then that it builds with -AC_MSG_CHECKING(FD_ZERO memcpy dependence) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include <cstddef> - #if HAVE_SYS_SELECT_H - #include <sys/select.h> - #endif - ]],[[ - #if HAVE_SYS_SELECT_H - fd_set fds; - FD_ZERO(&fds); - #endif - ]])], - [ AC_MSG_RESULT(no) ], - [ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include <cstring> - #if HAVE_SYS_SELECT_H - #include <sys/select.h> - #endif - ]], [[ - #if HAVE_SYS_SELECT_H - fd_set fds; - FD_ZERO(&fds); - #endif - ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_CSTRING_DEPENDENT_FD_ZERO, 1, [Define this symbol if FD_ZERO is dependent of a memcpy declaration being available]) ], - [ AC_MSG_ERROR(failed with cstring include) ] - ) - ] -) - AC_CHECK_DECLS([getifaddrs, freeifaddrs],,, [#include <sys/types.h> #include <ifaddrs.h>] @@ -1427,6 +1414,50 @@ AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes]) AC_SUBST(UNIVALUE_CFLAGS) AC_SUBST(UNIVALUE_LIBS) +dnl libmultiprocess library check + +libmultiprocess_found=no +if test "x$with_libmultiprocess" = xyes || test "x$with_libmultiprocess" = xauto; then + if test "x$use_pkgconfig" = xyes; then + m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ + libmultiprocess_found=yes; + libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`; + ], [true])]) + fi +elif test "x$with_libmultiprocess" != xno; then + AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no]) +fi +AC_SUBST(LIBMULTIPROCESS_CFLAGS) +AC_SUBST(LIBMULTIPROCESS_LIBS) + +dnl Enable multiprocess check + +if test "x$enable_multiprocess" = xyes; then + if test "x$libmultiprocess_found" != xyes; then + AC_MSG_ERROR([--enable-multiprocess=yes option specified but libmultiprocess library was not found. May need to install libmultiprocess library, or specify install path with PKG_CONFIG_PATH environment variable. Running 'pkg-config --debug libmultiprocess' may be helpful for debugging.]) + fi + build_multiprocess=yes +elif test "x$enable_multiprocess" = xauto; then + build_multiprocess=$libmultiprocess_found +else + build_multiprocess=no +fi + +AM_CONDITIONAL([BUILD_MULTIPROCESS],[test "x$build_multiprocess" = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_NODE], [test "x$build_multiprocess" = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_GUI], [test "x$build_multiprocess" = xyes]) + +dnl codegen tools check + +if test x$build_multiprocess != xno; then + if test "x$with_mpgen" = xyes || test "x$with_mpgen" = xauto; then + MPGEN_PREFIX="$libmultiprocess_prefix" + elif test "x$with_mpgen" != xno; then + MPGEN_PREFIX="$with_mpgen"; + fi + AC_SUBST(MPGEN_PREFIX) +fi + AC_MSG_CHECKING([whether to build bitcoind]) AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes]) AC_MSG_RESULT($build_bitcoind) @@ -1705,6 +1736,7 @@ esac echo echo "Options used to compile and link:" +echo " multiprocess = $build_multiprocess" echo " with wallet = $enable_wallet" echo " with gui / qt = $bitcoin_enable_qt" if test x$bitcoin_enable_qt != xno; then @@ -1724,7 +1756,7 @@ echo " gprof enabled = $enable_gprof" echo " werror = $enable_werror" echo echo " target os = $TARGET_OS" -echo " build os = $BUILD_OS" +echo " build os = $build_os" echo echo " CC = $CC" echo " CFLAGS = $CFLAGS" diff --git a/contrib/devtools/previous_release.sh b/contrib/devtools/previous_release.sh index b2ecc274fb..d375291f47 100755 --- a/contrib/devtools/previous_release.sh +++ b/contrib/devtools/previous_release.sh @@ -137,7 +137,7 @@ pushd "$TARGET" || exit 1 else BIN_PATH="bin/bitcoin-core-${tag:1}" fi - URL="https://bitcoin.org/$BIN_PATH/bitcoin-${tag:1}-$PLATFORM.tar.gz" + URL="https://bitcoincore.org/$BIN_PATH/bitcoin-${tag:1}-$PLATFORM.tar.gz" echo "Fetching: $URL" if ! curl -O -f $URL; then echo "Download failed." diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 9444271bdc..ca587ca9e5 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -12,33 +12,33 @@ import subprocess import sys import os +from typing import List, Optional + READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool') -NONFATAL = {} # checks which are non-fatal for now but only generate a warning -def check_ELF_PIE(executable): +def run_command(command) -> str: + p = subprocess.run(command, stdout=subprocess.PIPE, check=True, universal_newlines=True) + return p.stdout + +def check_ELF_PIE(executable) -> bool: ''' Check for position independent executable (PIE), allowing for address space randomization. ''' - p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([READELF_CMD, '-h', '-W', executable]) ok = False for line in stdout.splitlines(): - line = line.split() - if len(line)>=2 and line[0] == 'Type:' and line[1] == 'DYN': + tokens = line.split() + if len(line)>=2 and tokens[0] == 'Type:' and tokens[1] == 'DYN': ok = True return ok def get_ELF_program_headers(executable): '''Return type and flags for ELF program headers''' - p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([READELF_CMD, '-l', '-W', executable]) + in_headers = False count = 0 headers = [] @@ -62,7 +62,7 @@ def get_ELF_program_headers(executable): count += 1 return headers -def check_ELF_NX(executable): +def check_ELF_NX(executable) -> bool: ''' Check that no sections are writable and executable (including the stack) ''' @@ -75,7 +75,7 @@ def check_ELF_NX(executable): have_wx = True return have_gnu_stack and not have_wx -def check_ELF_RELRO(executable): +def check_ELF_RELRO(executable) -> bool: ''' Check for read-only relocations. GNU_RELRO program header must exist @@ -84,7 +84,8 @@ def check_ELF_RELRO(executable): have_gnu_relro = False for (typ, flags) in get_ELF_program_headers(executable): # Note: not checking flags == 'R': here as linkers set the permission differently - # This does not affect security: the permission flags of the GNU_RELRO program header are ignored, the PT_LOAD header determines the effective permissions. + # This does not affect security: the permission flags of the GNU_RELRO program + # header are ignored, the PT_LOAD header determines the effective permissions. # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. # See also https://marc.info/?l=binutils&m=1498883354122353 @@ -92,93 +93,69 @@ def check_ELF_RELRO(executable): have_gnu_relro = True have_bindnow = False - p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([READELF_CMD, '-d', '-W', executable]) + for line in stdout.splitlines(): tokens = line.split() if len(tokens)>1 and tokens[1] == '(BIND_NOW)' or (len(tokens)>2 and tokens[1] == '(FLAGS)' and 'BIND_NOW' in tokens[2:]): have_bindnow = True return have_gnu_relro and have_bindnow -def check_ELF_Canary(executable): +def check_ELF_Canary(executable) -> bool: ''' Check for use of stack canary ''' - p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([READELF_CMD, '--dyn-syms', '-W', executable]) + ok = False for line in stdout.splitlines(): if '__stack_chk_fail' in line: ok = True return ok -def get_PE_dll_characteristics(executable): - ''' - Get PE DllCharacteristics bits. - Returns a tuple (arch,bits) where arch is 'i386:x86-64' or 'i386' - and bits is the DllCharacteristics value. - ''' - p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - arch = '' +def get_PE_dll_characteristics(executable) -> int: + '''Get PE DllCharacteristics bits''' + stdout = run_command([OBJDUMP_CMD, '-x', executable]) + bits = 0 for line in stdout.splitlines(): tokens = line.split() - if len(tokens)>=2 and tokens[0] == 'architecture:': - arch = tokens[1].rstrip(',') if len(tokens)>=2 and tokens[0] == 'DllCharacteristics': bits = int(tokens[1],16) - return (arch,bits) + return bits IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020 IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE = 0x0040 IMAGE_DLL_CHARACTERISTICS_NX_COMPAT = 0x0100 -def check_PE_DYNAMIC_BASE(executable): +def check_PE_DYNAMIC_BASE(executable) -> bool: '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' - (arch,bits) = get_PE_dll_characteristics(executable) - reqbits = IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE - return (bits & reqbits) == reqbits + bits = get_PE_dll_characteristics(executable) + return (bits & IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE) == IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE -# On 64 bit, must support high-entropy 64-bit address space layout randomization in addition to DYNAMIC_BASE -# to have secure ASLR. -def check_PE_HIGH_ENTROPY_VA(executable): +# Must support high-entropy 64-bit address space layout randomization +# in addition to DYNAMIC_BASE to have secure ASLR. +def check_PE_HIGH_ENTROPY_VA(executable) -> bool: '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' - (arch,bits) = get_PE_dll_characteristics(executable) - if arch == 'i386:x86-64': - reqbits = IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA - else: # Unnecessary on 32-bit - assert(arch == 'i386') - reqbits = 0 - return (bits & reqbits) == reqbits + bits = get_PE_dll_characteristics(executable) + return (bits & IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA) == IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA def check_PE_RELOC_SECTION(executable) -> bool: '''Check for a reloc section. This is required for functional ASLR.''' - p = subprocess.Popen([OBJDUMP_CMD, '-h', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([OBJDUMP_CMD, '-h', executable]) + for line in stdout.splitlines(): if '.reloc' in line: return True return False -def check_PE_NX(executable): +def check_PE_NX(executable) -> bool: '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)''' - (arch,bits) = get_PE_dll_characteristics(executable) + bits = get_PE_dll_characteristics(executable) return (bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT -def get_MACHO_executable_flags(executable): - p = subprocess.Popen([OTOOL_CMD, '-vh', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') +def get_MACHO_executable_flags(executable) -> List[str]: + stdout = run_command([OTOOL_CMD, '-vh', executable]) flags = [] for line in stdout.splitlines(): @@ -222,10 +199,7 @@ def check_MACHO_LAZY_BINDINGS(executable) -> bool: Check for no lazy bindings. We don't use or check for MH_BINDATLOAD. See #18295. ''' - p = subprocess.Popen([OTOOL_CMD, '-l', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([OTOOL_CMD, '-l', executable]) for line in stdout.splitlines(): tokens = line.split() @@ -238,10 +212,8 @@ def check_MACHO_Canary(executable) -> bool: ''' Check for use of stack canary ''' - p = subprocess.Popen([OTOOL_CMD, '-Iv', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([OTOOL_CMD, '-Iv', executable]) + ok = False for line in stdout.splitlines(): if '___stack_chk_fail' in line: @@ -270,7 +242,7 @@ CHECKS = { ] } -def identify_executable(executable): +def identify_executable(executable) -> Optional[str]: with open(filename, 'rb') as f: magic = f.read(4) if magic.startswith(b'MZ'): @@ -292,18 +264,12 @@ if __name__ == '__main__': continue failed = [] - warning = [] for (name, func) in CHECKS[etype]: if not func(filename): - if name in NONFATAL: - warning.append(name) - else: - failed.append(name) + failed.append(name) if failed: print('%s: failed %s' % (filename, ' '.join(failed))) retval = 1 - if warning: - print('%s: warning %s' % (filename, ' '.join(warning))) except IOError: print('%s: cannot open' % filename) retval = 1 diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 8ce8cb97a0..dffcf99607 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -142,6 +142,11 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum If non-empty, will pass `V=1` to all `make` invocations, making `make` output verbose. + Note that any given value is ignored. The variable is only checked for + emptiness. More concretely, this means that `V=` (setting `V` to the empty + string) is interpreted the same way as not setting `V` at all, and that `V=0` + has the same effect as `V=1`. + * _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ Additional flags to be passed to `guix environment`. For a fully-bootstrapped diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 01f4518c73..5be3baeefa 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -3,6 +3,14 @@ export LC_ALL=C set -e -o pipefail export TZ=UTC +if [ -n "$V" ]; then + # Print both unexpanded (-v) and expanded (-x) forms of commands as they are + # read from this file. + set -vx + # Set VERBOSE for CMake-based builds + export VERBOSE="$V" +fi + # Check that environment variables assumed to be set by the environment are set echo "Building for platform triple ${HOST:?not set} with reference timestamp ${SOURCE_DATE_EPOCH:?not set}..." echo "At most ${MAX_JOBS:?not set} jobs will run at once..." diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index f232bb62c2..d2652119b4 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -1,7 +1,5 @@ -# Valgrind suppressions file for Bitcoin. -# -# Includes known Valgrind warnings in our dependencies that cannot be fixed -# in-tree. +# This valgrind suppressions file includes known Valgrind warnings in our +# dependencies that cannot be fixed in-tree. # # Example use: # $ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin @@ -14,6 +12,9 @@ # --error-limit=no src/test/test_bitcoin # # Note that suppressions may depend on OS and/or library versions. +# Tested on: +# * aarch64 (Ubuntu 20.04 system libs, without gui) +# * x86_64 (Ubuntu 18.04 system libs, without gui) { Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 Memcheck:Leak @@ -47,8 +48,7 @@ Suppress libdb warning Memcheck:Param pwrite64(buf) - fun:pwrite - fun:__os_io + ... obj:*/libdb_cxx-*.so } { diff --git a/depends/Makefile b/depends/Makefile index 5f5247f881..3d0784cb6b 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -4,6 +4,30 @@ print-%: @echo $* = $($*) +# When invoking a sub-make, keep only the command line variable definitions +# matching the pattern in the filter function. +# +# e.g. invoking: +# $ make A=1 C=1 print-MAKEOVERRIDES print-MAKEFLAGS +# +# with the following in the Makefile: +# MAKEOVERRIDES := $(filter A=% B=%,$(MAKEOVERRIDES)) +# +# will print: +# MAKEOVERRIDES = A=1 +# MAKEFLAGS = -- A=1 +# +# this is because as the GNU make manual says: +# The command line variable definitions really appear in the variable +# MAKEOVERRIDES, and MAKEFLAGS contains a reference to this variable. +# +# and since the GNU make manual also says: +# variables defined on the command line are passed to the sub-make through +# MAKEFLAGS +# +# this means that sub-makes will be invoked as if: +# $(MAKE) A=1 blah blah +MAKEOVERRIDES := $(filter V=%,$(MAKEOVERRIDES)) SOURCES_PATH ?= $(BASEDIR)/sources WORK_PATH = $(BASEDIR)/work BASE_CACHE ?= $(BASEDIR)/built @@ -13,6 +37,7 @@ NO_QR ?= NO_WALLET ?= NO_ZMQ ?= NO_UPNP ?= +MULTIPROCESS ?= FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources BUILD = $(shell ./config.guess) @@ -107,6 +132,7 @@ qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch wallet_packages_$(NO_WALLET) = $(wallet_packages) upnp_packages_$(NO_UPNP) = $(upnp_packages) zmq_packages_$(NO_ZMQ) = $(zmq_packages) +multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) @@ -115,6 +141,11 @@ ifneq ($(zmq_packages_),) packages += $(zmq_packages) endif +ifeq ($(multiprocess_packages_),) +packages += $(multiprocess_packages) +native_packages += $(multiprocess_native_packages) +endif + all_packages = $(packages) $(native_packages) meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk @@ -155,6 +186,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@no_zmq@|$(NO_ZMQ)|' \ -e 's|@no_wallet@|$(NO_WALLET)|' \ -e 's|@no_upnp@|$(NO_UPNP)|' \ + -e 's|@multiprocess@|$(MULTIPROCESS)|' \ -e 's|@debug@|$(DEBUG)|' \ $< > $@ $(AT)touch $@ diff --git a/depends/README.md b/depends/README.md index 79865ff011..c12ea8bcb3 100644 --- a/depends/README.md +++ b/depends/README.md @@ -91,6 +91,7 @@ The following can be set when running make: make FOO=bar NO_ZMQ: Don't download/build/cache packages needed for enabling zeromq NO_WALLET: Don't download/build/cache libs needed to enable the wallet NO_UPNP: Don't download/build/cache packages needed for enabling upnp + MULTIPROCESS: build libmultiprocess (experimental, requires cmake) DEBUG: disable some optimizations and enable more runtime checking HOST_ID_SALT: Optional salt to use when generating host package ids BUILD_ID_SALT: Optional salt to use when generating build package ids diff --git a/depends/config.site.in b/depends/config.site.in index fb9bf713cc..103aa0d2ff 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -16,6 +16,9 @@ fi if test -z $with_qt_bindir && test -z "@no_qt@"; then with_qt_bindir=$depends_prefix/native/bin fi +if test -z $with_mpgen && test -n "@multiprocess@"; then + with_mpgen=$depends_prefix/native +fi if test -z $with_qrencode && test -n "@no_qr@"; then with_qrencode=no @@ -25,6 +28,10 @@ if test -z $enable_wallet && test -n "@no_wallet@"; then enable_wallet=no fi +if test -z $enable_multiprocess && test -n "@multiprocess@"; then + enable_multiprocess=yes +fi + if test -z $with_miniupnpc && test -n "@no_upnp@"; then with_miniupnpc=no fi diff --git a/depends/funcs.mk b/depends/funcs.mk index a4434b5167..135ebba9f8 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -130,11 +130,11 @@ $(1)_config_env+=$($(1)_config_env_$(host_arch)_$(host_os)) $($(1)_config_env_$( $(1)_config_env+=PKG_CONFIG_LIBDIR=$($($(1)_type)_prefix)/lib/pkgconfig $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig +$(1)_config_env+=CMAKE_MODULE_PATH=$($($(1)_type)_prefix)/lib/cmake $(1)_config_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_build_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_stage_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_autoconf=./configure --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" - ifneq ($($(1)_nm),) $(1)_autoconf += NM="$$($(1)_nm)" endif @@ -156,6 +156,15 @@ endif ifneq ($($(1)_ldflags),) $(1)_autoconf += LDFLAGS="$$($(1)_ldflags)" endif + +$(1)_cmake=cmake -DCMAKE_INSTALL_PREFIX=$($($(1)_type)_prefix) +ifneq ($($(1)_type),build) +ifneq ($(host),$(build)) +$(1)_cmake += -DCMAKE_SYSTEM_NAME=$($(host_os)_cmake_system) -DCMAKE_SYSROOT=$(host_prefix) +$(1)_cmake += -DCMAKE_C_COMPILER_TARGET=$(host) -DCMAKE_C_COMPILER=$(firstword $($($(1)_type)_CC)) -DCMAKE_C_FLAGS="$(wordlist 2,1000,$($($(1)_type)_CC))" +$(1)_cmake += -DCMAKE_CXX_COMPILER_TARGET=$(host) -DCMAKE_CXX_COMPILER=$(firstword $($($(1)_type)_CXX)) -DCMAKE_CXX_FLAGS="$(wordlist 2,1000,$($($(1)_type)_CXX))" +endif +endif endef define int_add_cmds diff --git a/depends/hosts/android.mk b/depends/hosts/android.mk index 969ec2a1cb..eabd84bbbe 100644 --- a/depends/hosts/android.mk +++ b/depends/hosts/android.mk @@ -9,3 +9,4 @@ android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++ android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ranlib endif +android_cmake_system=Android diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 1bc4fb8189..82e086a326 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -14,3 +14,4 @@ darwin_debug_CFLAGS=-O1 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) darwin_native_toolchain=native_cctools +darwin_cmake_system=Darwin diff --git a/depends/hosts/default.mk b/depends/hosts/default.mk index 144e5f88b7..258619a9d0 100644 --- a/depends/hosts/default.mk +++ b/depends/hosts/default.mk @@ -13,9 +13,18 @@ default_host_OTOOL = $(host_toolchain)otool default_host_NM = $(host_toolchain)nm define add_host_tool_func +ifneq ($(filter $(origin $1),undefined default),) +# Do not consider the well-known var $1 if it is undefined or is taking a value +# that is predefined by "make" (e.g. the make variable "CC" has a predefined +# value of "cc") $(host_os)_$1?=$$(default_host_$1) $(host_arch)_$(host_os)_$1?=$$($(host_os)_$1) $(host_arch)_$(host_os)_$(release_type)_$1?=$$($(host_os)_$1) +else +$(host_os)_$1=$(or $($1),$($(host_os)_$1),$(default_host_$1)) +$(host_arch)_$(host_os)_$1=$(or $($1),$($(host_arch)_$(host_os)_$1),$$($(host_os)_$1)) +$(host_arch)_$(host_os)_$(release_type)_$1=$(or $($1),$($(host_arch)_$(host_os)_$(release_type)_$1),$$($(host_os)_$1)) +endif host_$1=$$($(host_arch)_$(host_os)_$1) endef diff --git a/depends/hosts/linux.mk b/depends/hosts/linux.mk index b13a0f1ad7..8ab448ce5f 100644 --- a/depends/hosts/linux.mk +++ b/depends/hosts/linux.mk @@ -29,3 +29,4 @@ i686_linux_CXX=$(default_host_CXX) -m32 x86_64_linux_CC=$(default_host_CC) -m64 x86_64_linux_CXX=$(default_host_CXX) -m64 endif +linux_cmake_system=Linux diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index dbfb62fdcf..be5fec570c 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -8,3 +8,5 @@ mingw32_debug_CFLAGS=-O1 mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS) mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC + +mingw_cmake_system=Windows diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index cbe4fe4d97..3a7e605b4f 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,7 +1,7 @@ package=boost $(package)_version=1_70_0 $(package)_download_path=https://dl.bintray.com/boostorg/release/1.70.0/source/ -$(package)_file_name=$(package)_$($(package)_version).tar.bz2 +$(package)_file_name=boost_$($(package)_version).tar.bz2 $(package)_sha256_hash=430ae8354789de4fd19ee52f3b1f739e1fba576f0aded0897c3c2bc00fb38778 define $(package)_set_vars @@ -9,9 +9,9 @@ $(package)_config_opts_release=variant=release $(package)_config_opts_debug=variant=debug $(package)_config_opts=--layout=tagged --build-type=complete --user-config=user-config.jam $(package)_config_opts+=threading=multi link=static -sNO_BZIP2=1 -sNO_ZLIB=1 -$(package)_config_opts_linux=threadapi=pthread runtime-link=shared -$(package)_config_opts_darwin=--toolset=clang-darwin runtime-link=shared -$(package)_config_opts_mingw32=binary-format=pe target-os=windows threadapi=win32 runtime-link=static +$(package)_config_opts_linux=target-os=linux threadapi=pthread runtime-link=shared +$(package)_config_opts_darwin=target-os=darwin runtime-link=shared +$(package)_config_opts_mingw32=target-os=windows binary-format=pe threadapi=win32 runtime-link=static $(package)_config_opts_x86_64_mingw32=address-model=64 $(package)_config_opts_i686_mingw32=address-model=32 $(package)_config_opts_i686_linux=address-model=32 architecture=x86 @@ -20,8 +20,11 @@ $(package)_config_opts_aarch64_android=address-model=64 $(package)_config_opts_x86_64_android=address-model=64 $(package)_config_opts_armv7a_android=address-model=32 $(package)_toolset_$(host_os)=gcc +$(package)_toolset_darwin=clang +ifneq (,$(findstring clang,$($(package)_cxx))) + $(package)_toolset_$(host_os)=clang +endif $(package)_archiver_$(host_os)=$($(package)_ar) -$(package)_toolset_darwin=clang-darwin $(package)_config_libraries=filesystem,system,thread,test $(package)_cxxflags=-std=c++11 -fvisibility=hidden $(package)_cxxflags_linux=-fPIC @@ -33,13 +36,13 @@ define $(package)_preprocess_cmds endef define $(package)_config_cmds - ./bootstrap.sh --without-icu --with-libraries=$($(package)_config_libraries) + ./bootstrap.sh --without-icu --with-libraries=$($(package)_config_libraries) --with-toolset=$($(package)_toolset_$(host_os)) endef define $(package)_build_cmds - ./b2 -d2 -j2 -d1 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) stage + ./b2 -d2 -j2 -d1 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) stage endef define $(package)_stage_cmds - ./b2 -d0 -j4 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) install + ./b2 -d0 -j4 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) install endef diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk new file mode 100644 index 0000000000..abeb26545f --- /dev/null +++ b/depends/packages/capnp.mk @@ -0,0 +1,18 @@ +package=capnp +$(package)_version=$(native_$(package)_version) +$(package)_download_path=$(native_$(package)_download_path) +$(package)_file_name=$(native_$(package)_file_name) +$(package)_sha256_hash=$(native_$(package)_sha256_hash) +$(package)_dependencies=native_$(package) + +define $(package)_config_cmds + $($(package)_autoconf) --with-external-capnp +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk new file mode 100644 index 0000000000..3e5cf5f160 --- /dev/null +++ b/depends/packages/libmultiprocess.mk @@ -0,0 +1,18 @@ +package=libmultiprocess +$(package)_version=$(native_$(package)_version) +$(package)_download_path=$(native_$(package)_download_path) +$(package)_file_name=$(native_$(package)_file_name) +$(package)_sha256_hash=$(native_$(package)_sha256_hash) +$(package)_dependencies=native_$(package) boost capnp + +define $(package)_config_cmds + $($(package)_cmake) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/depends/packages/native_capnp.mk b/depends/packages/native_capnp.mk new file mode 100644 index 0000000000..ed5a6deee2 --- /dev/null +++ b/depends/packages/native_capnp.mk @@ -0,0 +1,18 @@ +package=native_capnp +$(package)_version=0.7.0 +$(package)_download_path=https://capnproto.org/ +$(package)_download_file=capnproto-c++-$($(package)_version).tar.gz +$(package)_file_name=capnproto-cxx-$($(package)_version).tar.gz +$(package)_sha256_hash=c9a4c0bd88123064d483ab46ecee777f14d933359e23bff6fb4f4dbd28b4cd41 + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk new file mode 100644 index 0000000000..c50fdc3f6b --- /dev/null +++ b/depends/packages/native_libmultiprocess.mk @@ -0,0 +1,18 @@ +package=native_libmultiprocess +$(package)_version=5741d750a04e644a03336090d8979c6d033e32c0 +$(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive +$(package)_file_name=$($(package)_version).tar.gz +$(package)_sha256_hash=ac848db49a6ed53e423c62d54bd87f1f08cbb0326254a8667e10bbfe5bf032a4 +$(package)_dependencies=native_capnp + +define $(package)_config_cmds + $($(package)_cmake) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 42dbaa77a6..8fe2c771c9 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -16,6 +16,9 @@ zmq_packages=zeromq upnp_packages=miniupnpc +multiprocess_packages = libmultiprocess capnp +multiprocess_native_packages = native_libmultiprocess native_capnp + darwin_native_packages = native_biplist native_ds_store native_mac_alias ifneq ($(build_os),darwin) diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 366b1d0c42..631851855a 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -10,6 +10,7 @@ $(package)_build_subdir=qtbase $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch fix_android_qmake_conf.patch fix_android_jni_static.patch +# Update OSX_QT_TRANSLATIONS when this is updated $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) $(package)_qttranslations_sha256_hash=fb5a47799754af73d3bf501fe513342cfe2fc37f64e80df5533f6110e804220c diff --git a/doc/build-windows.md b/doc/build-windows.md index bbff638b90..d3dc467f19 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -9,7 +9,7 @@ The options known to work for building Bitcoin Core on Windows are: and is the platform used to build the Bitcoin Core Windows release binaries. * On Windows, using [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about) and the Mingw-w64 cross compiler tool chain. -* On Windows, using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). +* On Windows, using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). See [README.md](/build_msvc/README.md). Other options which may work, but which have not been extensively tested are (please contribute instructions): diff --git a/doc/multiprocess.md b/doc/multiprocess.md new file mode 100644 index 0000000000..471d8561f7 --- /dev/null +++ b/doc/multiprocess.md @@ -0,0 +1,35 @@ +# Multiprocess Bitcoin + +On unix systems, the `--enable-multiprocess` build option can be passed to `./configure` to build new `bitcoin-node`, `bitcoin-wallet`, and `bitcoin-gui` executables alongside existing `bitcoind` and `bitcoin-qt` executables. + +`bitcoin-node` is a drop-in replacement for `bitcoind`, and `bitcoin-gui` is a drop-in replacement for `bitcoin-qt`, and there are no differences in use or external behavior between the new and old executables. But internally (after [#10102](https://github.com/bitcoin/bitcoin/pull/10102)), `bitcoin-gui` will spawn a `bitcoin-node` process to run P2P and RPC code, communicating with it across a socket pair, and `bitcoin-node` will spawn `bitcoin-wallet` to run wallet code, also communicating over a socket pair. This will let node, wallet, and GUI code run in separate address spaces for better isolation, and allow future improvements like being able to start and stop components independently on different machines and environments. + +## Next steps + +Specific next steps after [#10102](https://github.com/bitcoin/bitcoin/pull/10102) will be: + +- [ ] Adding `-ipcbind` and `-ipcconnect` options to `bitcoin-node`, `bitcoin-wallet`, and `bitcoin-gui` executables so they can listen and connect to TCP ports and unix socket paths. This will allow separate processes to be started and stopped any time and connect to each other. +- [ ] Adding `-server` and `-rpcbind` options to the `bitcoin-wallet` executable so wallet processes can handle RPC requests directly without going through the node. +- [ ] Supporting windows, not just unix systems. The existing socket code is already cross-platform, so the only windows-specific code that needs to be written is code spawning a process and passing a socket descriptor. This can be implemented with `CreateProcess` and `WSADuplicateSocket`. Example: https://memset.wordpress.com/2010/10/13/win32-api-passing-socket-with-ipc-method/. +- [ ] Adding sandbox features, restricting subprocess access to resources and data. See [https://eklitzke.org/multiprocess-bitcoin](https://eklitzke.org/multiprocess-bitcoin). + +## Debugging + +After [#10102](https://github.com/bitcoin/bitcoin/pull/10102), the `-debug=ipc` command line option can be used to see requests and responses between processes. + +## Installation + +The multiprocess feature requires [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) as dependencies. A simple way to get starting using it without installing these dependencies manually is to use the [depends system](../depends) with the `MULTIPROCESS=1` [dependency option](../depends#dependency-options) passed to make: + +``` +cd <BITCOIN_SOURCE_DIRECTORY> +make -C depends NO_QT=1 MULTIPROCESS=1 +./configure --prefix=$PWD/depends/x86_64-pc-linux-gnu +make +src/bitcoin-node -regtest -printtoconsole -debug=ipc +BITCOIND=bitcoin-node test/functional/test_runner.py +``` + +The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option). + +Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess#installation) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general. diff --git a/doc/productivity.md b/doc/productivity.md index 1bf3d9afb5..555f0afe3c 100644 --- a/doc/productivity.md +++ b/doc/productivity.md @@ -12,7 +12,7 @@ Table of Contents * [Multiple working directories with `git worktrees`](#multiple-working-directories-with-git-worktrees) * [Interactive "dummy rebases" for fixups and execs with `git merge-base`](#interactive-dummy-rebases-for-fixups-and-execs-with-git-merge-base) * [Writing code](#writing-code) - * [Format C/C++/Protobuf diffs with `clang-format-diff.py`](#format-ccprotobuf-diffs-with-clang-format-diffpy) + * [Format C/C++ diffs with `clang-format-diff.py`](#format-cc-diffs-with-clang-format-diffpy) * [Format Python diffs with `yapf-diff.py`](#format-python-diffs-with-yapf-diffpy) * [Rebasing/Merging code](#rebasingmerging-code) * [More conflict context with `merge.conflictstyle diff3`](#more-conflict-context-with-mergeconflictstyle-diff3) @@ -118,13 +118,13 @@ You can also set up [upstream refspecs](#reference-prs-easily-with-refspecs) to Writing code ------------ -### Format C/C++/Protobuf diffs with `clang-format-diff.py` +### Format C/C++ diffs with `clang-format-diff.py` See [contrib/devtools/README.md](/contrib/devtools/README.md#clang-format-diff.py). ### Format Python diffs with `yapf-diff.py` -Usage is exactly the same as [`clang-format-diff.py`](#format-ccprotobuf-diffs-with-clang-format-diffpy). You can get it [here](https://github.com/MarcoFalke/yapf-diff). +Usage is exactly the same as [`clang-format-diff.py`](#format-cc-diffs-with-clang-format-diffpy). You can get it [here](https://github.com/MarcoFalke/yapf-diff). Rebasing/Merging code ------------- diff --git a/doc/reduce-traffic.md b/doc/reduce-traffic.md index e39e43df7a..ce77a00dd5 100644 --- a/doc/reduce-traffic.md +++ b/doc/reduce-traffic.md @@ -23,7 +23,7 @@ longer serving historic blocks (blocks older than one week). Keep in mind that new nodes require other nodes that are willing to serve historic blocks. -Whitelisted peers will never be disconnected, although their traffic counts for +Peers with the `noban` permission will never be disconnected, although their traffic counts for calculating the target. ## 2. Disable "listening" (`-listen=0`) diff --git a/doc/release-notes-16528.md b/doc/release-notes-16528.md new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/doc/release-notes-16528.md diff --git a/doc/release-notes-18918.md b/doc/release-notes-18918.md new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/doc/release-notes-18918.md diff --git a/doc/release-notes.md b/doc/release-notes.md index 0d668a6302..d9d0ecd631 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -34,40 +34,55 @@ How to Upgrade ============== If you are running an older version, shut it down. Wait until it has completely -shut down (which might take a few minutes for older versions), then run the +shut down (which might take a few minutes in some cases), then run the installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) or `bitcoind`/`bitcoin-qt` (on Linux). Upgrading directly from a version of Bitcoin Core that has reached its EOL is -possible, but it might take some time if the datadir needs to be migrated. Old +possible, but it might take some time if the data directory needs to be migrated. Old wallet versions of Bitcoin Core are generally supported. Compatibility ============== -Bitcoin Core is supported and extensively tested on operating systems using -the Linux kernel, macOS 10.12+, and Windows 7 and newer. It is not recommended -to use Bitcoin Core on unsupported systems. - -Bitcoin Core should also work on most other Unix-like systems but is not -as frequently tested on them. +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no longer supported. Additionally, Bitcoin Core does not yet change appearance when macOS "dark mode" is activated. -In addition to previously supported CPU platforms, this release's pre-compiled -distribution provides binaries for the RISC-V platform. - Notable changes =============== P2P and network changes ----------------------- +- The mempool now tracks whether transactions submitted via the wallet or RPCs + have been successfully broadcast. Every 10-15 minutes, the node will try to + announce unbroadcast transactions until a peer requests it via a `getdata` + message or the transaction is removed from the mempool for other reasons. + The node will not track the broadcast status of transactions submitted to the + node using P2P relay. This version reduces the initial broadcast guarantees + for wallet transactions submitted via P2P to a node running the wallet. (#18038) + Updated RPCs ------------ +- `getmempoolinfo` now returns an additional `unbroadcastcount` field. The + mempool tracks locally submitted transactions until their initial broadcast + is acknowledged by a peer. This field returns the count of transactions + waiting for acknowledgement. + +- Mempool RPCs such as `getmempoolentry` and `getrawmempool` with + `verbose=true` now return an additional `unbroadcast` field. This indicates + whether initial broadcast of the transaction has been acknowledged by a + peer. `getmempoolancestors` and `getmempooldescendants` are also updated. + + Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section below. New RPCs @@ -79,6 +94,9 @@ Build System Updated settings ---------------- +- The `-debug=db` logging category, which was deprecated in 0.20 and replaced by + `-debug=walletdb` to distinguish it from `coindb`, has been removed. (#19202) + Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. New settings @@ -87,7 +105,139 @@ New settings Wallet ------ -#### Wallet RPC changes +- To improve wallet privacy, the frequency of wallet rebroadcast attempts is + reduced from approximately once every 15 minutes to once every 12-36 hours. + To maintain a similar level of guarantee for initial broadcast of wallet + transactions, the mempool tracks these transactions as a part of the newly + introduced unbroadcast set. See the "P2P and network changes" section for + more information on the unbroadcast set. (#18038) + +- The wallet can create a transaction without change even when the keypool is + empty. Previously it failed. (#17219) + +- The `-salvagewallet` startup option has been removed. A new `salvage` command + has been added to the `bitcoin-wallet` tool which performs the salvage + operations that `-salvagewallet` did. (#18918) + +### Experimental Descriptor Wallets + +Please note that Descriptor Wallets are still experimental and not all expected functionality +is available. Additionally there may be some bugs and current functions may change in the future. +Bugs and missing functionality can be reported to the [issue tracker](https://github.com/bitcoin/bitcoin/issues). + +0.21 introduces a new type of wallet - Descriptor Wallets. Descriptor Wallets store +scriptPubKey information using descriptors. This is in contrast to the Legacy Wallet +structure where keys are used to generate scriptPubKeys and addresses. Because of this +shift to being script based instead of key based, many of the confusing things that Legacy +Wallets do are not possible with Descriptor Wallets. Descriptor Wallets use a definition +of "mine" for scripts which is simpler and more intuitive than that used by Legacy Wallets. +Descriptor Wallets also uses different semantics for watch-only things and imports. + +As Descriptor Wallets are a new type of wallet, their introduction does not affect existing wallets. +Users who already have a Bitcoin Core wallet can continue to use it as they did before without +any change in behavior. Newly created Legacy Wallets (which is the default type of wallet) will +behave as they did in previous versions of Bitcoin Core. + +The differences between Descriptor Wallets and Legacy Wallets are largely limited to non user facing +things. They are intended to behave similarly except for the import/export and watchonly functionality +as described below. + +#### Creating Descriptor Wallets + +Descriptor Wallets are not created by default. They must be explicitly created using the +`createwallet` RPC or via the GUI. A `descriptors` option has been added to `createwallet`. +Setting `descriptors` to `true` will create a Descriptor Wallet instead of a Legacy Wallet. + +In the GUI, a checkbox has been added to the Create Wallet Dialog to indicate that a +Descriptor Wallet should be created. + +Without those options being set, a Legacy Wallet will be created instead. Additionally the +Default Wallet created upon first startup of Bitcoin Core will be a Legacy Wallet. + +#### `IsMine` Semantics + +`IsMine` refers to the function used to determine whether a script belongs to the wallet. +This is used to determine whether an output belongs to the wallet. `IsMine` in Legacy Wallets +returns true if the wallet would be able to sign an input that spends an output with that script. +Since keys can be involved in a variety of different scripts, this definition for `IsMine` can +lead to many unexpected scripts being considered part of the wallet. + +With Descriptor Wallets, descriptors explicitly specify the set of scripts that are owned by +the wallet. Since descriptors are deterministic and easily enumerable, users will know exactly +what scripts the wallet will consider to belong to it. Additionally the implementation of `IsMine` +in Descriptor Wallets is far simpler than for Legacy Wallets. Notably, in Legacy Wallets, `IsMine` +allowed for users to take one type of address (e.g. P2PKH), mutate it into another address type +(e.g. P2WPKH), and the wallet would still detect outputs sending to the new address type +even without that address being requested from the wallet. Descriptor Wallets does not +allow for this and will only watch for the addresses that were explicitly requested from the wallet. + +These changes to `IsMine` will make it easier to reason about what scripts the wallet will +actually be watching for in outputs. However for the vast majority of users, this change is +largely transparent and will not have noticeable effect. + +#### Imports and Exports + +In Legacy Wallets, raw scripts and keys could be imported to the wallet. Those imported scripts +and keys are treated separately from the keys generated by the wallet. This complicates the `IsMine` +logic as it has to distinguish between spendable and watchonly. + +Descriptor Wallets handle importing scripts and keys differently. Only complete descriptors can be +imported. These descriptors are then added to the wallet as if it were a descriptor generated by +the wallet itself. This simplifies the `IsMine` logic so that it no longer has to distinguish +between spendable and watchonly. As such, the watchonly model for Descriptor Wallets is also +different and described in more detail in the next section. + +To import into a Descriptor Wallet, a new `importdescriptors` RPC has been added that uses a syntax +similar to that of `importmulti`. + +As Legacy Wallets and Descriptor Wallets use different mechanisms for storing and importing scripts and keys +the existing import RPCs have been disabled for descriptor wallets. +New export RPCs for Descriptor Wallets have not yet been added. + +The following RPCs are disabled for Descriptor Wallets: + +* importprivkey +* importpubkey +* importaddress +* importwallet +* dumpprivkey +* dumpwallet +* importmulti +* addmultisigaddress +* sethdseed + +#### Watchonly Wallets + +A Legacy Wallet contains both private keys and scripts that were being watched. +Those watched scripts would not contribute to your normal balance. In order to see the watchonly +balance and to use watchonly things in transactions, an `include_watchonly` option was added +to many RPCs that would allow users to do that. However it is easy to forget to include this option. + +Descriptor Wallets move to a per-wallet watchonly model. Instead an entire wallet is considered to be +watchonly depending on whether it was created with private keys disabled. This eliminates the need +to distinguish between things that are watchonly and things that are not within a wallet itself. + +This change does have a caveat. If a Descriptor Wallet with private keys *enabled* has +a multiple key descriptor without all of the private keys (e.g. `multi(...)` with only one private key), +then the wallet will fail to sign and broadcast transactions. Such wallets would need to use the PSBT +workflow but the typical GUI Send, `sendtoaddress`, etc. workflows would still be available, just +non-functional. + +This issue is worsened if the wallet contains both single key (e.g. `wpkh(...)`) descriptors and such +multiple key descriptors as some transactions could be signed and broadast and others not. This is +due to some transactions containing only single key inputs, while others would contain both single +key and multiple key inputs, depending on which are available and how the coin selection algorithm +selects inputs. However this is not considered to be a supported use case; multisigs +should be in their own wallets which do not already have descriptors. Although users cannot export +descriptors with private keys for now as explained earlier. + +#### BIP 44/49/84 Support + +The change to using descriptors changes the default derivation paths used by Bitcoin Core +to adhere to BIP 44/49/84. Descriptors with different derivation paths can be imported without +issue. + +### Wallet RPC changes - The `upgradewallet` RPC replaces the `-upgradewallet` command line option. (#15761) diff --git a/doc/release-notes/release-notes-0.20.0.md b/doc/release-notes/release-notes-0.20.0.md new file mode 100644 index 0000000000..6fc1606b37 --- /dev/null +++ b/doc/release-notes/release-notes-0.20.0.md @@ -0,0 +1,987 @@ +0.20.0 Release Notes +==================== + +Bitcoin Core version 0.20.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.20.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. + +Known Bugs +========== + +The process for generating the source code release ("tarball") has changed in an +effort to make it more complete, however, there are a few regressions in +this release: + +- The generated `configure` script is currently missing, and you will need to + install autotools and run `./autogen.sh` before you can run + `./configure`. This is the same as when checking out from git. + +- Instead of running `make` simply, you should instead run + `BITCOIN_GENBUILD_NO_GIT=1 make`. + +Notable changes +=============== + +P2P and network changes +----------------------- + +#### Removal of BIP61 reject network messages from Bitcoin Core + +The `-enablebip61` command line option to enable BIP61 has been removed. +(#17004) + +This feature has been disabled by default since Bitcoin Core version 0.18.0. +Nodes on the network can not generally be trusted to send valid messages +(including reject messages), so this should only ever be used when +connected to a trusted node. Please use the alternatives recommended +below if you rely on this removed feature: + +- Testing or debugging of implementations of the Bitcoin P2P network protocol + should be done by inspecting the log messages that are produced by a recent + version of Bitcoin Core. Bitcoin Core logs debug messages + (`-debug=<category>`) to a stream (`-printtoconsole`) or to a file + (`-debuglogfile=<debug.log>`). + +- Testing the validity of a block can be achieved by specific RPCs: + + - `submitblock` + + - `getblocktemplate` with `'mode'` set to `'proposal'` for blocks with + potentially invalid POW + +- Testing the validity of a transaction can be achieved by specific RPCs: + + - `sendrawtransaction` + + - `testmempoolaccept` + +- Wallets should not assume a transaction has propagated to the network + just because there are no reject messages. Instead, listen for the + transaction to be announced by other peers on the network. Wallets + should not assume a lack of reject messages means a transaction pays + an appropriate fee. Instead, set fees using fee estimation and use + replace-by-fee to increase a transaction's fee if it hasn't confirmed + within the desired amount of time. + +The removal of BIP61 reject message support also has the following minor RPC +and logging implications: + +- `testmempoolaccept` and `sendrawtransaction` no longer return the P2P reject + code when a transaction is not accepted to the mempool. They still return the + verbal reject reason. + +- Log messages that previously reported the reject code when a transaction was + not accepted to the mempool now no longer report the reject code. The reason + for rejection is still reported. + +Updated RPCs +------------ + +- The RPCs which accept descriptors now accept the new `sortedmulti(...)` descriptor + type which supports multisig scripts where the public keys are sorted + lexicographically in the resulting script. (#17056) + +- The `walletprocesspsbt` and `walletcreatefundedpsbt` RPCs now include + BIP32 derivation paths by default for public keys if we know them. + This can be disabled by setting the `bip32derivs` parameter to + `false`. (#17264) + +- The `bumpfee` RPC's parameter `totalFee`, which was deprecated in + 0.19, has been removed. (#18312) + +- The `bumpfee` RPC will return a PSBT when used with wallets that have + private keys disabled. (#16373) + +- The `getpeerinfo` RPC now includes a `mapped_as` field to indicate the + mapped Autonomous System used for diversifying peer selection. See the + `-asmap` configuration option described below in _New Settings_. (#16702) + +- The `createmultisig` and `addmultisigaddress` RPCs now return an + output script descriptor for the newly created address. (#18032) + +Build System +------------ + +- OpenSSL is no longer used by Bitcoin Core. (#17265) + +- BIP70 support has been fully removed from Bitcoin Core. The + `--enable-bip70` option remains, but it will throw an error during configure. + (#17165) + +- glibc 2.17 or greater is now required to run the release binaries. This + retains compatibility with RHEL 7, CentOS 7, Debian 8 and Ubuntu 14.04 LTS. (#17538) + +- The source code archives that are provided with gitian builds no longer contain + any autotools artifacts. Therefore, to build from such source, a user + should run the `./autogen.sh` script from the root of the unpacked archive. + This implies that `autotools` and other required packages are installed on the + user's system. (#18331) + +New settings +------------ + +- New `rpcwhitelist` and `rpcwhitelistdefault` configuration parameters + allow giving certain RPC users permissions to only some RPC calls. + (#12763) + +- A new `-asmap` configuration option has been added to diversify the + node's network connections by mapping IP addresses Autonomous System + Numbers (ASNs) and then limiting the number of connections made to any + single ASN. See [issue #16599](https://github.com/bitcoin/bitcoin/issues/16599), + [PR #16702](https://github.com/bitcoin/bitcoin/pull/16702), and the + `bitcoind help` for more information. This option is experimental and + subject to removal or breaking changes in future releases, so the + legacy /16 prefix mapping of IP addresses remains the default. (#16702) + +Updated settings +---------------- + +- All custom settings configured when Bitcoin Core starts are now + written to the `debug.log` file to assist troubleshooting. (#16115) + +- Importing blocks upon startup via the `bootstrap.dat` file no longer + occurs by default. The file must now be specified with + `-loadblock=<file>`. (#17044) + +- The `-debug=db` logging category has been renamed to + `-debug=walletdb` to distinguish it from `coindb`. The `-debug=db` + option has been deprecated and will be removed in the next major + release. (#17410) + +- The `-walletnotify` configuration parameter will now replace any `%w` + in its argument with the name of the wallet generating the + notification. This is not supported on Windows. (#13339) + +Removed settings +---------------- + +- The `-whitelistforcerelay` configuration parameter has been removed after + it was discovered that it was rendered ineffective in version 0.13 and + hasn't actually been supported for almost four years. (#17985) + +GUI changes +----------- + +- The "Start Bitcoin Core on system login" option has been removed on macOS. + (#17567) + +- In the Peers window, the details for a peer now displays a `Mapped AS` + field to indicate the mapped Autonomous System used for diversifying + peer selection. See the `-asmap` configuration option in _New + Settings_, above. (#18402) + +- A "known bug" [announced](https://bitcoincore.org/en/releases/0.18.0/#wallet-gui) + in the release notes of version 0.18 has been fixed. The issue + affected anyone who simultaneously used multiple Bitcoin Core wallets + and the GUI coin control feature. (#18894) + +- For watch-only wallets, creating a new transaction in the Send screen + or fee bumping an existing transaction in the Transactions screen will + automatically copy a Partially-Signed Bitcoin Transaction (PSBT) to + the system clipboard. This can then be pasted into an external + program such as [HWI](https://github.com/bitcoin-core/HWI) for + signing. Future versions of Bitcoin Core should support a GUI option + for finalizing and broadcasting PSBTs, but for now the debug console + may be used with the `finalizepsbt` and `sendrawtransaction` RPCs. + (#16944, #17492) + +Wallet +------ + +- The wallet now by default uses bech32 addresses when using RPC, and + creates native segwit change outputs. (#16884) + +- The way that output trust was computed has been fixed, which affects + confirmed/unconfirmed balance status and coin selection. (#16766) + +- The `gettransaction`, `listtransactions` and `listsinceblock` RPC + responses now also include the height of the block that contains the + wallet transaction, if any. (#17437) + +- The `getaddressinfo` RPC has had its `label` field deprecated + (re-enable for this release using the configuration parameter + `-deprecatedrpc=label`). The `labels` field is altered from returning + JSON objects to returning a JSON array of label names (re-enable + previous behavior for this release using the configuration parameter + `-deprecatedrpc=labelspurpose`). Backwards compatibility using the + deprecated configuration parameters is expected to be dropped in the + 0.21 release. (#17585, #17578) + +Documentation changes +--------------------- + +- Bitcoin Core's automatically-generated source code documentation is + now available at https://doxygen.bitcoincore.org. (#17596) + +Low-level changes +================= + +Utilities +--------- + +- The `bitcoin-cli` utility used with the `-getinfo` parameter now + returns a `headers` field with the number of downloaded block headers + on the best headers chain (similar to the `blocks` field that is also + returned) and a `verificationprogress` field that estimates how much + of the best block chain has been synced by the local node. The + information returned no longer includes the `protocolversion`, + `walletversion`, and `keypoololdest` fields. (#17302, #17650) + +- The `bitcoin-cli` utility now accepts a `-stdinwalletpassphrase` + parameter that can be used when calling the `walletpassphrase` and + `walletpassphrasechange` RPCs to read the passphrase from standard + input without echoing it to the terminal, improving security against + anyone who can look at your screen. The existing `-stdinrpcpass` + parameter is also updated to not echo the passphrase. (#13716) + +Command line +------------ + +- Command line options prefixed with main/test/regtest network names like + `-main.port=8333` `-test.server=1` previously were allowed but ignored. Now + they trigger "Invalid parameter" errors on startup. (#17482) + +New RPCs +-------- + +- The `dumptxoutset` RPC outputs a serialized snapshot of the current + UTXO set. A script is provided in the `contrib/devtools` directory + for generating a snapshot of the UTXO set at a particular block + height. (#16899) + +- The `generatetodescriptor` RPC allows testers using regtest mode to + generate blocks that pay an arbitrary output script descriptor. + (#16943) + +Updated RPCs +------------ + +- The `verifychain` RPC default values are now static instead of + depending on the command line options or configuration file + (`-checklevel`, and `-checkblocks`). Users can pass in the RPC + arguments explicitly when they don't want to rely on the default + values. (#18541) + +- The `getblockchaininfo` RPC's `verificationprogress` field will no + longer report values higher than 1. Previously it would occasionally + report the chain was more than 100% verified. (#17328) + +Tests +----- + +- It is now an error to use an unqualified `walletdir=path` setting in + the config file if running on testnet or regtest networks. The setting + now needs to be qualified as `chain.walletdir=path` or placed in the + appropriate `[chain]` section. (#17447) + +- `-fallbackfee` was 0 (disabled) by default for the main chain, but + 0.0002 by default for the test chains. Now it is 0 by default for all + chains. Testnet and regtest users will have to add + `fallbackfee=0.0002` to their configuration if they weren't setting it + and they want it to keep working like before. (#16524) + +Build system +------------ + +- Support is provided for building with the Android Native Development + Kit (NDK). (#16110) + +0.20.0 change log +================= + +### Mining +- #18742 miner: Avoid stack-use-after-return in validationinterface (MarcoFalke) + +### Block and transaction handling +- #15283 log: Fix UB with bench on genesis block (instagibbs) +- #16507 feefilter: Compute the absolute fee rather than stored rate (instagibbs) +- #16688 log: Add validation interface logging (jkczyz) +- #16805 log: Add timing information to FlushStateToDisk() (jamesob) +- #16902 O(1) `OP_IF/NOTIF/ELSE/ENDIF` script implementation (sipa) +- #16945 introduce CChainState::GetCoinsCacheSizeState (jamesob) +- #16974 Walk pindexBestHeader back to ChainActive().Tip() if it is invalid (TheBlueMatt) +- #17004 Remove REJECT code from CValidationState (jnewbery) +- #17080 Explain why `fCheckDuplicateInputs` can not be skipped and remove it (MarcoFalke) +- #17328 GuessVerificationProgress: cap the ratio to 1 (darosior) +- #17399 Templatize ValidationState instead of subclassing (jkczyz) +- #17407 node: Add reference to mempool in NodeContext (MarcoFalke) +- #17708 prevector: Avoid misaligned member accesses (ajtowns) +- #17850,#17896,#17957,#18021,#18021,#18112 Serialization improvements (sipa) +- #17925 Improve UpdateTransactionsFromBlock with Epochs (JeremyRubin) +- #18002 Abstract out script execution out of `VerifyWitnessProgram()` (sipa) +- #18388 Make VerifyWitnessProgram use a Span stack (sipa) +- #18433 serialization: prevent int overflow for big Coin::nHeight (pierreN) +- #18500 chainparams: Bump assumed valid hash (MarcoFalke) +- #18551 Do not clear validationinterface entries being executed (sipa) + +### P2P protocol and network code +- #15437 Remove BIP61 reject messages (MarcoFalke) +- #16702 Supply and use asmap to improve IP bucketing in addrman (naumenkogs) +- #16851 Continue relaying transactions after they expire from mapRelay (ajtowns) +- #17164 Avoid allocating memory for addrKnown where we don't need it (naumenkogs) +- #17243 tools: add PoissonNextSend method that returns mockable time (amitiuttarwar) +- #17251 SocketHandler logs peer id for close and disconnect (Sjors) +- #17573 Seed RNG with precision timestamps on receipt of net messages (TheBlueMatt) +- #17624 Fix an uninitialized read in ProcessMessage(…, "tx", …) when receiving a transaction we already have (practicalswift) +- #17754 Don't allow resolving of std::string with embedded NUL characters. Add tests (practicalswift) +- #17758 Fix CNetAddr::IsRFC2544 comment + tests (tynes) +- #17812 config, net, test: Asmap feature refinements and functional tests (jonatack) +- #17951 Use rolling bloom filter of recent block txs for AlreadyHave() check (sdaftuar) +- #17985 Remove forcerelay of rejected txs (MarcoFalke) +- #18023 Fix some asmap issues (sipa) +- #18054 Reference instead of copy in BlockConnected range loop (jonatack) +- #18376 Fix use-after-free in tests (vasild) +- #18454 Make addr relay mockable, add test (MarcoFalke) +- #18458 Add missing `cs_vNodes` lock (MarcoFalke) +- #18506 Hardcoded seeds update for 0.20 (laanwj) +- #18808 Drop unknown types in getdata (jnewbery) +- #18962 Only send a getheaders for one block in an INV (jnewbery) + +### Wallet +- #13339 Replace %w by wallet name in -walletnotify script (promag) +- #15931 Remove GetDepthInMainChain dependency on locked chain interface (ariard) +- #16373 bumpfee: Return PSBT when wallet has privkeys disabled (instagibbs) +- #16524 Disable -fallbackfee by default (jtimon) +- #16766 Make IsTrusted scan parents recursively (JeremyRubin) +- #16884 Change default address type to bech32 (instagibbs) +- #16911 Only check the hash of transactions loaded from disk (achow101) +- #16923 Handle duplicate fileid exception (promag) +- #17056 descriptors: Introduce sortedmulti descriptor (achow101) +- #17070 Avoid showing GUI popups on RPC errors (MarcoFalke) +- #17138 Remove wallet access to some node arguments (jnewbery) +- #17237 LearnRelatedScripts only if KeepDestination (promag) +- #17260 Split some CWallet functions into new LegacyScriptPubKeyMan (achow101) +- #17261 Make ScriptPubKeyMan an actual interface and the wallet to have multiple (achow101) +- #17290 Enable BnB coin selection for preset inputs and subtract fee from outputs (achow101) +- #17373 Various fixes and cleanup to keypool handling in LegacyScriptPubKeyMan and CWallet (achow101) +- #17410 Rename `db` log category to `walletdb` (like `coindb`) (laanwj) +- #17444 Avoid showing GUI popups on RPC errors (take 2) (MarcoFalke) +- #17447 Make -walletdir network only (promag) +- #17537 Cleanup and move opportunistic and superfluous TopUp()s (achow101) +- #17553 Remove out of date comments for CalculateMaximumSignedTxSize (instagibbs) +- #17568 Fix when sufficient preset inputs and subtractFeeFromOutputs (achow101) +- #17677 Activate watchonly wallet behavior for LegacySPKM only (instagibbs) +- #17719 Document better -keypool as a look-ahead safety mechanism (ariard) +- #17843 Reset reused transactions cache (fjahr) +- #17889 Improve CWallet:MarkDestinationsDirty (promag) +- #18034 Get the OutputType for a descriptor (achow101) +- #18067 Improve LegacyScriptPubKeyMan::CanProvide script recognition (ryanofsky) +- #18115 Pass in transactions and messages for signing instead of exporting the private keys (achow101) +- #18192,#18546 Bugfix: Wallet: Safely deal with change in the address book (luke-jr) +- #18204 descriptors: Improve descriptor cache and cache xpubs (achow101) +- #18274 rpc/wallet: Initialize nFeeRequired to avoid using garbage value on failure (kallewoof) +- #18312 Remove deprecated fee bumping by totalFee (jonatack) +- #18338 Fix wallet unload race condition (promag) + +### RPC and other APIs +- #12763 Add RPC Whitelist Feature from #12248 (JeremyRubin) +- #13716 cli: `-stdinwalletpassphrase` and non-echo stdin passwords (kallewoof) +- #16689 Add missing fields to wallet rpc help output (ariard) +- #16821 Fix bug where duplicate PSBT keys are accepted (erasmospunk) +- #16899 UTXO snapshot creation (dumptxoutset) +- #17156 psbt: Check that various indexes and amounts are within bounds (achow101) +- #17264 Set default bip32derivs to true for psbt methods (Sjors) +- #17283 improve getaddressinfo test coverage, help, code docs (jonatack) +- #17302 cli: Add "headers" and "verificationprogress" to -getinfo (laanwj) +- #17318 replace asserts in RPC code with `CHECK_NONFATAL` and add linter (adamjonas) +- #17437 Expose block height of wallet transactions (promag) +- #17519 Remove unused `COINBASE_FLAGS` (narula) +- #17578 Simplify getaddressinfo labels, deprecate previous behavior (jonatack) +- #17585 deprecate getaddressinfo label (jonatack) +- #17746 Remove vector copy from listtransactions (promag) +- #17809 Auto-format RPCResult (MarcoFalke) +- #18032 Output a descriptor in createmultisig and addmultisigaddress (achow101) +- #18122 Update validateaddress RPCExamples to bech32 (theStack) +- #18208 Change RPCExamples to bech32 (yusufsahinhamza) +- #18268 Remove redundant types from descriptions (docallag) +- #18346 Document an RPCResult for all calls; Enforce at compile time (MarcoFalke) +- #18396 Add missing HelpExampleRpc for getblockfilter (theStack) +- #18398 Fix broken RPCExamples for waitforblock(height) (theStack) +- #18444 Remove final comma for last entry of fixed-size arrays/objects in RPCResult (luke-jr) +- #18459 Remove unused getbalances() code (jonatack) +- #18484 Correctly compute redeemScript from witnessScript for signrawtransaction (achow101) +- #18487 Fix rpcRunLater race in walletpassphrase (promag) +- #18499 Make rpc documentation not depend on call-time rpc args (MarcoFalke) +- #18532 Avoid initialization-order-fiasco on static CRPCCommand tables (MarcoFalke) +- #18541 Make verifychain default values static, not depend on global args (MarcoFalke) +- #18809 Do not advertise dumptxoutset as a way to flush the chainstate (MarcoFalke) +- #18814 Relock wallet only if most recent callback (promag) + +### GUI +- #15023 Restore RPC Console to non-wallet tray icon menu (luke-jr) +- #15084 Don't disable the sync overlay when wallet is disabled (benthecarman) +- #15098 Show addresses for "SendToSelf" transactions (hebasto) +- #15756 Add shortcuts for tab tools (promag) +- #16944 create PSBT with watch-only wallet (Sjors) +- #16964 Change sendcoins dialogue Yes to Send (instagibbs) +- #17068 Always generate `bitcoinstrings.cpp` on `make translate` (D4nte) +- #17096 Rename debug window (Zero-1729) +- #17105 Make RPCConsole::TabTypes an enum class (promag) +- #17125 Add toolTip and placeholderText to sign message fields (dannmat) +- #17165 Remove BIP70 support (fanquake) +- #17180 Improved tooltip for send amount field (JeremyCrookshank) +- #17186 Add placeholder text to the sign message field (Danny-Scott) +- #17195 Send amount placeholder value (JeremyCrookshank) +- #17226 Fix payAmount tooltip in SendCoinsEntry (promag) +- #17360 Cleaning up hide button tool tip (Danny-Scott) +- #17446 Changed tooltip for 'Label' & 'Message' text fields to be more clear (dannmat) +- #17453 Fix intro dialog labels when the prune button is toggled (hebasto) +- #17474 Bugfix: GUI: Recognise `NETWORK_LIMITED` in formatServicesStr (luke-jr) +- #17492 Bump fee returns PSBT on clipboard for watchonly-only wallets (instagibbs) +- #17567 Remove macOS start on login code (fanquake) +- #17587 Show watch-only balance in send screen (Sjors) +- #17694 Disable 3rd-party tx-urls when wallet disabled (brakmic) +- #17696 Force set nPruneSize in QSettings after the intro dialog (hebasto) +- #17702 Move static placeholder texts to forms (laanwj) +- #17826 Log Qt related info (hebasto) +- #17886 Restore English translation option (achow101) +- #17906 Set CConnman byte counters earlier to avoid uninitialized reads (ryanofsky) +- #17935 Hide HD & encryption icons when no wallet loaded (brakmic) +- #17998 Shortcut to close ModalOverlay (emilengler) +- #18007 Bugfix: GUI: Hide the HD/encrypt icons earlier so they get re-shown if another wallet is open (luke-jr) +- #18060 Drop PeerTableModel dependency to ClientModel (promag) +- #18062 Fix unintialized WalletView::progressDialog (promag) +- #18091 Pass clientmodel changes from walletframe to walletviews (jonasschnelli) +- #18101 Fix deprecated QCharRef usage (hebasto) +- #18121 Throttle GUI update pace when -reindex (hebasto) +- #18123 Fix race in WalletModel::pollBalanceChanged (ryanofsky) +- #18160 Avoid Wallet::GetBalance in WalletModel::pollBalanceChanged (promag) +- #18360 Bump transifex slug and update English translations for 0.20 (laanwj) +- #18402 Display mapped AS in peers info window (jonatack) +- #18492 Translations update pre-branch (laanwj) +- #18549 Fix Window -> Minimize menu item (hebasto) +- #18578 Fix leak in CoinControlDialog::updateView (promag) +- #18894 Fix manual coin control with multiple wallets loaded (promag) + +### Build system +- #16667 Remove mingw linker workaround from win gitian descriptor (fanquake) +- #16669 Use new fork of osslsigncode for windows gitian signing (fanquake) +- #16949 Only pass --disable-dependency-tracking to packages that understand it (fanquake) +- #17008 Bump libevent to 2.1.11 in depends (stefanwouldgo) +- #17029 gitian: Various improvements for windows descriptor (dongcarl) +- #17033 Disable _FORTIFY_SOURCE when enable-debug (achow101) +- #17057 Switch to upstream libdmg-hfsplus (fanquake) +- #17066 Remove workaround for ancient libtool (hebasto) +- #17074 Added double quotes (mztriz) +- #17087 Add variable printing target to Makefiles (dongcarl) +- #17118 depends macOS: point --sysroot to SDK (Sjors) +- #17231 Fix boost mac cross build with clang 9+ (theuni) +- #17265 Remove OpenSSL (fanquake) +- #17284 Update retry to current version (RandyMcMillan) +- #17308 nsis: Write to correct filename in first place (dongcarl) +- #17324,#18099 Update univalue subtree (MarcoFalke) +- #17398 Update leveldb to 1.22+ (laanwj) +- #17409 Avoid hardcoded libfaketime dir in gitian (MarcoFalke) +- #17466 Fix C{,XX} pickup (dongcarl) +- #17483 Set gitian arch back to amd64 (MarcoFalke) +- #17486 Make Travis catch unused variables (Sjors) +- #17538 Bump minimum libc to 2.17 for release binaries (fanquake) +- #17542 Create test utility library from src/test/util/ (brakmic) +- #17545 Remove libanl.so.1 from ALLOWED_LIBRARIES (fanquake) +- #17547 Fix configure report about qr (hebasto) +- #17569 Allow export of environ symbols and work around rv64 toolchain issue (laanwj) +- #17647 lcov: filter depends from coverage reports (nijynot) +- #17658 Add ability to skip building qrencode (fanquake) +- #17678 Support for S390X and POWER targets (MarcoFalke) +- #17682 util: Update tinyformat to upstream (laanwj) +- #17698 Don't configure `xcb_proto` (fanquake) +- #17730 Remove Qt networking features (fanquake) +- #17738 Remove linking librt for backwards compatibility (fanquake) +- #17740 Remove configure checks for win libraries we don't link against (fanquake) +- #17741 Included `test_bitcoin-qt` in msvc build (sipsorcery) +- #17756 Remove `WINDOWS_BITS` from build system (fanquake) +- #17769 Set `AC_PREREQ` to 2.69 (fanquake) +- #17880 Add -Wdate-time to Werror flags (fanquake) +- #17910 Remove double `LIBBITCOIN_SERVER` linking (fanquake) +- #17928 Consistent use of package variable (Bushstar) +- #17933 guix: Pin Guix using `guix time-machine` (dongcarl) +- #17948 pass -fno-ident in Windows gitian descriptor (fanquake) +- #18003 Remove --large-address-aware linker flag (fanquake) +- #18004 Don't embed a build-id when building libdmg-hfsplus (fanquake) +- #18051 Fix behavior when `ALLOW_HOST_PACKAGES` unset (hebasto) +- #18059 Add missing attributes to Win installer (fanquake) +- #18104 Skip i686 build by default in guix and gitian (MarcoFalke) +- #18107 Add `cov_fuzz` target (MarcoFalke) +- #18135 Add --enable-determinism configure flag (fanquake) +- #18145 Add Wreturn-type to Werror flags, check on more Travis machines (Sjors) +- #18264 Remove Boost Chrono (fanquake) +- #18290 Set minimum Automake version to 1.13 (hebasto) +- #18320 guix: Remove now-unnecessary gcc make flag (dongcarl) +- #18331 Use git archive as source tarball (hebasto) +- #18397 Fix libevent linking for `bench_bitcoin` binary (hebasto) +- #18426 scripts: `Previous_release`: improve behaviour on failed download (theStack) +- #18429 Remove double `LIBBITCOIN_SERVER` from bench-Makefile (brakmic) +- #18528 Create `test_fuzz` library from src/test/fuzz/fuzz.cpp (brakmic) +- #18558 Fix boost detection for arch armv7l (hebasto) +- #18598 gitian: Add missing automake package to gitian-win-signer.yml (achow101) +- #18676 Check libevent minimum version in configure script (hebasto) +- #18945 Ensure source tarball has leading directory name (laanwj) + +### Platform support +- #16110 Add Android NDK support (icota) +- #16392 macOS toolchain update (fanquake) +- #16569 Increase init file stop timeout (setpill) +- #17151 Remove OpenSSL PRNG seeding (Windows, Qt only) (fanquake) +- #17365 Update README.md with working Android targets and API levels (icota) +- #17521 Only use D-Bus with Qt on linux (fanquake) +- #17550 Set minimum supported macOS to 10.12 (fanquake) +- #17592 Appveyor install libevent[thread] vcpkg (sipsorcery) +- #17660 Remove deprecated key from macOS Info.plist (fanquake) +- #17663 Pass `-dead_strip_dylibs` to ld on macOS (fanquake) +- #17676 Don't use OpenGL in Qt on macOS (fanquake) +- #17686 Add `-bind_at_load` to macOS hardened LDFLAGS (fanquake) +- #17787 scripts: Add macho pie check to security-check.py (fanquake) +- #17800 random: don't special case clock usage on macOS (fanquake) +- #17863 scripts: Add macho dylib checks to symbol-check.py (fanquake) +- #17899 msvc: Ignore msvc linker warning and update to msvc build instructions (sipsorcery) +- #17916 windows: Enable heap terminate-on-corruption (fanquake) +- #18082 logging: Enable `thread_local` usage on macos (fanquake) +- #18108 Fix `.gitignore` policy in `build_msvc` directory (hebasto) +- #18295 scripts: Add macho lazy bindings check to security-check.py (fanquake) +- #18358 util: Fix compilation with mingw-w64 7.0.0 (fanquake) +- #18359 Fix sysctl() detection on macOS (fanquake) +- #18364 random: remove getentropy() fallback for macOS < 10.12 (fanquake) +- #18395 scripts: Add pe dylib checking to symbol-check.py (fanquake) +- #18415 scripts: Add macho tests to test-security-check.py (fanquake) +- #18425 releases: Update with new Windows code signing certificate (achow101) +- #18702 Fix ASLR for bitcoin-cli on Windows (fanquake) + +### Tests and QA +- #12134 Build previous releases and run functional tests (Sjors) +- #13693 Add coverage to estimaterawfee and estimatesmartfee (Empact) +- #13728 lint: Run the ci lint stage on mac (Empact) +- #15443 Add getdescriptorinfo functional test (promag) +- #15888 Add `wallet_implicitsegwit` to test the ability to transform keys between address types (luke-jr) +- #16540 Add `ASSERT_DEBUG_LOG` to unit test framework (MarcoFalke) +- #16597 travis: Run full test suite on native macos (Sjors) +- #16681 Use self.chain instead of 'regtest' in all current tests (jtimon) +- #16786 add unit test for wallet watch-only methods involving PubKeys (theStack) +- #16943 Add generatetodescriptor RPC (MarcoFalke) +- #16973 Fix `combine_logs.py` for AppVeyor build (mzumsande) +- #16975 Show debug log on unit test failure (MarcoFalke) +- #16978 Seed test RNG context for each test case, print seed (MarcoFalke) +- #17009, #17018, #17050, #17051, #17071, #17076, #17083, #17093, #17109, #17113, #17136, #17229, #17291, #17357, #17771, #17777, #17917, #17926, #17972, #17989, #17996, #18009, #18029, #18047, #18126, #18176, #18206, #18353, #18363, #18407, #18417, #18423, #18445, #18455, #18565 Add fuzzing harnesses (practicalswift) +- #17011 ci: Use busybox utils for one build (MarcoFalke) +- #17030 Fix Python Docstring to include all Args (jbampton) +- #17041 ci: Run tests on arm (MarcoFalke) +- #17069 Pass fuzzing inputs as constant references (practicalswift) +- #17091 Add test for loadblock option and linearize scripts (fjahr) +- #17108 fix "tx-size-small" errors after default address change (theStack) +- #17121 Speed up `wallet_backup` by whitelisting peers (immediate tx relay) (theStack) +- #17124 Speed up `wallet_address_types` by whitelisting peers (immediate tx relay) (theStack) +- #17140 Fix bug in `blockfilter_index_tests` (jimpo) +- #17199 use default address type (bech32) for `wallet_bumpfee` tests (theStack) +- #17205 ci: Enable address sanitizer (asan) stack-use-after-return checking (practicalswift) +- #17206 Add testcase to simulate bitcoin schema in leveldb (adamjonas) +- #17209 Remove no longer needed UBSan suppressions (issues fixed). Add documentation (practicalswift) +- #17220 Add unit testing for the CompressScript function (adamjonas) +- #17225 Test serialisation as part of deserialisation fuzzing. Test round-trip equality where possible (practicalswift) +- #17228 Add RegTestingSetup to `setup_common` (MarcoFalke) +- #17233 travis: Run unit and functional tests on native arm (MarcoFalke) +- #17235 Skip unnecessary fuzzer initialisation. Hold ECCVerifyHandle only when needed (practicalswift) +- #17240 ci: Disable functional tests on mac host (MarcoFalke) +- #17254 Fix `script_p2sh_tests` `OP_PUSHBACK2/4` missing (adamjonas) +- #17267 bench: Fix negative values and zero for -evals flag (nijynot) +- #17275 pubkey: Assert CPubKey's ECCVerifyHandle precondition (practicalswift) +- #17288 Added TestWrapper class for interactive Python environments (jachiang) +- #17292 Add new mempool benchmarks for a complex pool (JeremyRubin) +- #17299 add reason checks for non-standard txs in `test_IsStandard` (theStack) +- #17322 Fix input size assertion in `wallet_bumpfee.py` (instagibbs) +- #17327 Add `rpc_fundrawtransaction` logging (jonatack) +- #17330 Add `shrinkdebugfile=0` to regtest bitcoin.conf (sdaftuar) +- #17340 Speed up fundrawtransaction test (jnewbery) +- #17345 Do not instantiate CAddrDB for static call CAddrDB::Read() (hebasto) +- #17362 Speed up `wallet_avoidreuse`, add logging (jonatack) +- #17363 add "diamond" unit test to MempoolAncestryTests (theStack) +- #17366 Reset global args between test suites (MarcoFalke) +- #17367 ci: Run non-cross-compile builds natively (MarcoFalke) +- #17378 TestShell: Fix typos & implement cleanups (jachiang) +- #17384 Create new test library (MarcoFalke) +- #17387 `wallet_importmulti`: use addresses of the same type as being imported (achow101) +- #17388 Add missing newline in `util_ChainMerge` test (ryanofsky) +- #17390 Add `util_ArgParsing` test (ryanofsky) +- #17420 travis: Rework `cache_err_msg` (MarcoFalke) +- #17423 ci: Make ci system read-only on the git work tree (MarcoFalke) +- #17435 check custom ancestor limit in `mempool_packages.py` (theStack) +- #17455 Update valgrind suppressions (practicalswift) +- #17461 Check custom descendant limit in `mempool_packages.py` (theStack) +- #17469 Remove fragile `assert_memory_usage_stable` (MarcoFalke) +- #17470 ci: Use clang-8 for fuzzing to run on aarch64 ci systems (MarcoFalke) +- #17480 Add unit test for non-standard txs with too large scriptSig (theStack) +- #17497 Skip tests when utils haven't been compiled (fanquake) +- #17502 Add unit test for non-standard bare multisig txs (theStack) +- #17511 Add bounds checks before base58 decoding (sipa) +- #17517 ci: Bump to clang-8 for asan build to avoid segfaults on ppc64le (MarcoFalke) +- #17522 Wait until mempool is loaded in `wallet_abandonconflict` (MarcoFalke) +- #17532 Add functional test for non-standard txs with too large scriptSig (theStack) +- #17541 Add functional test for non-standard bare multisig txs (theStack) +- #17555 Add unit test for non-standard txs with wrong nVersion (dspicher) +- #17571 Add `libtest_util` library to msvc build configuration (sipsorcery) +- #17591 ci: Add big endian platform - s390x (elichai) +- #17593 Move more utility functions into test utility library (mzumsande) +- #17633 Add option --valgrind to run the functional tests under Valgrind (practicalswift) +- #17635 ci: Add centos 7 build (hebasto) +- #17641 Add unit test for leveldb creation with unicode path (sipsorcery) +- #17674 Add initialization order fiasco detection in Travis (practicalswift) +- #17675 Enable tests which are incorrectly skipped when running `test_runner.py --usecli` (practicalswift) +- #17685 Fix bug in the descriptor parsing fuzzing harness (`descriptor_parse`) (practicalswift) +- #17705 re-enable CLI test support by using EncodeDecimal in json.dumps() (fanquake) +- #17720 add unit test for non-standard "scriptsig-not-pushonly" txs (theStack) +- #17767 ci: Fix qemu issues (MarcoFalke) +- #17793 ci: Update github actions ci vcpkg cache on msbuild update (hebasto) +- #17806 Change filemode of `rpc_whitelist.py` (emilengler) +- #17849 ci: Fix brew python link (hebasto) +- #17851 Add `std::to_string` to list of locale dependent functions (practicalswift) +- #17893 Fix double-negative arg test (hebasto) +- #17900 ci: Combine 32-bit build with centos 7 build (theStack) +- #17921 Test `OP_CSV` empty stack fail in `feature_csv_activation.py` (theStack) +- #17931 Fix `p2p_invalid_messages` failing in Python 3.8 because of warning (elichai) +- #17947 add unit test for non-standard txs with too large tx size (theStack) +- #17959 Check specific reject reasons in `feature_csv_activation.py` (theStack) +- #17984 Add p2p test for forcerelay permission (MarcoFalke) +- #18001 Updated appveyor job to checkout a specific vcpkg commit ID (sipsorcery) +- #18008 fix fuzzing using libFuzzer on macOS (fanquake) +- #18013 bench: Fix benchmarks filters (elichai) +- #18018 reset fIsBareMultisigStd after bare-multisig tests (fanquake) +- #18022 Fix appveyor `test_bitcoin` build of `*.raw` (MarcoFalke) +- #18037 util: Allow scheduler to be mocked (amitiuttarwar) +- #18056 ci: Check for submodules (emilengler) +- #18069 Replace 'regtest' leftovers by self.chain (theStack) +- #18081 Set a name for CI Docker containers (fanquake) +- #18109 Avoid hitting some known minor tinyformat issues when fuzzing strprintf(…) (practicalswift) +- #18155 Add harness which fuzzes EvalScript and VerifyScript using a fuzzed signature checker (practicalswift) +- #18159 Add --valgrind option to `test/fuzz/test_runner.py` for running fuzzing test cases under valgrind (practicalswift) +- #18166 ci: Run fuzz testing test cases (bitcoin-core/qa-assets) under valgrind to catch memory errors (practicalswift) +- #18172 Transaction expiry from mempool (0xB10C) +- #18181 Remove incorrect assumptions in `validation_flush_tests` (MarcoFalke) +- #18183 Set `catch_system_errors=no` on boost unit tests (MarcoFalke) +- #18195 Add `cost_of_change` parameter assertions to `bnb_search_test` (yancyribbens) +- #18209 Reduce unneeded whitelist permissions in tests (MarcoFalke) +- #18211 Disable mockforward scheduler unit test for now (MarcoFalke) +- #18213 Fix race in `p2p_segwit` (MarcoFalke) +- #18224 Make AnalyzePSBT next role calculation simple, correct (instagibbs) +- #18228 Add missing syncwithvalidationinterfacequeue (MarcoFalke) +- #18247 Wait for both veracks in `add_p2p_connection` (MarcoFalke) +- #18249 Bump timeouts to accomodate really slow disks (MarcoFalke) +- #18255 Add `bad-txns-*-toolarge` test cases to `invalid_txs` (MarcoFalke) +- #18263 rpc: change setmocktime check to use IsMockableChain (gzhao408) +- #18285 Check that `wait_until` returns if time point is in the past (MarcoFalke) +- #18286 Add locale fuzzer to `FUZZERS_MISSING_CORPORA` (practicalswift) +- #18292 fuzz: Add `assert(script == decompressed_script)` (MarcoFalke) +- #18299 Update `FUZZERS_MISSING_CORPORA` to enable regression fuzzing for all harnesses in master (practicalswift) +- #18300 fuzz: Add option to merge input dir to test runner (MarcoFalke) +- #18305 Explain why test logging should be used (MarcoFalke) +- #18306 Add logging to `wallet_listsinceblock.py` (jonatack) +- #18311 Bumpfee test fix (instagibbs) +- #18314 Add deserialization fuzzing of SnapshotMetadata (`utxo_snapshot`) (practicalswift) +- #18319 fuzz: Add missing `ECC_Start` to `key_io` test (MarcoFalke) +- #18334 Add basic test for BIP 37 (MarcoFalke) +- #18350 Fix mining to an invalid target + ensure that a new block has the correct hash internally (TheQuantumPhysicist) +- #18378 Bugfix & simplify bn2vch using `int.to_bytes` (sipa) +- #18393 Don't assume presence of `__builtin_mul_overflow(…)` in `MultiplicationOverflow(…)` fuzzing harness (practicalswift) +- #18406 add executable flag for `rpc_estimatefee.py` (theStack) +- #18420 listsinceblock block height checks (jonatack) +- #18430 ci: Only clone bitcoin-core/qa-assets when fuzzing (MarcoFalke) +- #18438 ci: Use homebrew addon on native macos (hebasto) +- #18447 Add coverage for script parse error in ParseScript (pierreN) +- #18472 Remove unsafe `BOOST_TEST_MESSAGE` (MarcoFalke) +- #18474 check that peer is connected when calling sync_* (MarcoFalke) +- #18477 ci: Use focal for fuzzers (MarcoFalke) +- #18481 add BIP37 'filterclear' test to p2p_filter.py (theStack) +- #18496 Remove redundant `sync_with_ping` after `add_p2p_connection` (jonatack) +- #18509 fuzz: Avoid running over all inputs after merging them (MarcoFalke) +- #18510 fuzz: Add CScriptNum::getint coverage (MarcoFalke) +- #18514 remove rapidcheck integration and tests (fanquake) +- #18515 Add BIP37 remote crash bug [CVE-2013-5700] test to `p2p_filter.py` (theStack) +- #18516 relax bumpfee `dust_to_fee` txsize an extra vbyte (jonatack) +- #18518 fuzz: Extend descriptor fuzz test (MarcoFalke) +- #18519 fuzz: Extend script fuzz test (MarcoFalke) +- #18521 fuzz: Add `process_messages` harness (MarcoFalke) +- #18529 Add fuzzer version of randomized prevector test (sipa) +- #18534 skip backwards compat tests if not compiled with wallet (fanquake) +- #18540 `wallet_bumpfee` assertion fixup (jonatack) +- #18543 Use one node to avoid a race due to missing sync in `rpc_signrawtransaction` (MarcoFalke) +- #18561 Properly raise FailedToStartError when rpc shutdown before warmup finished (MarcoFalke) +- #18562 ci: Run unit tests sequential once (MarcoFalke) +- #18563 Fix `unregister_all_during_call` cleanup (ryanofsky) +- #18566 Set `-use_value_profile=1` when merging fuzz inputs (MarcoFalke) +- #18757 Remove enumeration of expected deserialization exceptions in ProcessMessage(…) fuzzer (practicalswift) +- #18878 Add test for conflicted wallet tx notifications (ryanofsky) +- #18975 Remove const to work around compiler error on xenial (laanwj) + +### Documentation +- #16947 Doxygen-friendly script/descriptor.h comments (ch4ot1c) +- #16983 Add detailed info about Bitcoin Core files (hebasto) +- #16986 Doxygen-friendly CuckooCache comments (ch4ot1c) +- #17022 move-only: Steps for "before major release branch-off" (MarcoFalke) +- #17026 Update bips.md for default bech32 addresses in 0.20.0 (MarcoFalke) +- #17081 Fix Makefile target in benchmarking.md (theStack) +- #17102 Add missing indexes/blockfilter/basic to doc/files.md (MarcoFalke) +- #17119 Fix broken bitcoin-cli examples (andrewtoth) +- #17134 Add switch on enum example to developer notes (hebasto) +- #17142 Update macdeploy README to include all files produced by `make deploy` (za-kk) +- #17146 github: Add warning for bug reports (laanwj) +- #17157 Added instructions for how to add an upsteam to forked repo (dannmat) +- #17159 Add a note about backporting (carnhofdaki) +- #17169 Correct function name in ReportHardwareRand() (fanquake) +- #17177 Describe log files + consistent paths in test READMEs (fjahr) +- #17239 Changed miniupnp links to https (sandakersmann) +- #17281 Add developer note on `c_str()` (laanwj) +- #17285 Bip70 removal follow-up (fjahr) +- #17286 Fix help-debug -checkpoints (ariard) +- #17309 update MSVC instructions to remove Qt OpenSSL linking (fanquake) +- #17339 Add template for good first issues (michaelfolkson) +- #17351 Fix some misspellings (RandyMcMillan) +- #17353 Add ShellCheck to lint tests dependencies (hebasto) +- #17370 Update doc/bips.md with recent changes in master (MarcoFalke) +- #17393 Added regtest config for linearize script (gr0kchain) +- #17411 Add some better examples for scripted diff (laanwj) +- #17503 Remove bitness from bitcoin-qt help message and manpage (laanwj) +- #17539 Update and improve Developer Notes (hebasto) +- #17561 Changed MiniUPnPc link to https in dependencies.md (sandakersmann) +- #17596 Change doxygen URL to doxygen.bitcoincore.org (laanwj) +- #17598 Update release process with latest changes (MarcoFalke) +- #17617 Unify unix epoch time descriptions (jonatack) +- #17637 script: Add keyserver to verify-commits readme (emilengler) +- #17648 Rename wallet-tool references to bitcoin-wallet (hel-o) +- #17688 Add "ci" prefix to CONTRIBUTING.md (hebasto) +- #17751 Use recommended shebang approach in documentation code block (hackerrdave) +- #17752 Fix directory path for secp256k1 subtree in developer-notes (hackerrdave) +- #17772 Mention PR Club in CONTRIBUTING.md (emilengler) +- #17804 Misc RPC help fixes (MarcoFalke) +- #17819 Developer notes guideline on RPCExamples addresses (jonatack) +- #17825 Update dependencies.md (hebasto) +- #17873 Add to Doxygen documentation guidelines (jonatack) +- #17907 Fix improper Doxygen inline comments (Empact) +- #17942 Improve fuzzing docs for macOS users (fjahr) +- #17945 Fix doxygen errors (Empact) +- #18025 Add missing supported rpcs to doc/descriptors.md (andrewtoth) +- #18070 Add note about `brew doctor` (givanse) +- #18125 Remove PPA note from release-process.md (fanquake) +- #18170 Minor grammatical changes and flow improvements (travinkeith) +- #18212 Add missing step in win deployment instructions (dangershony) +- #18219 Add warning against wallet.dat re-use (corollari) +- #18253 Correct spelling errors in comments (Empact) +- #18278 interfaces: Describe and follow some code conventions (ryanofsky) +- #18283 Explain rebase policy in CONTRIBUTING.md (MarcoFalke) +- #18340 Mention MAKE=gmake workaround when building on a BSD (fanquake) +- #18341 Replace remaining literal BTC with `CURRENCY_UNIT` (domob1812) +- #18342 Add fuzzing quickstart guides for libFuzzer and afl-fuzz (practicalswift) +- #18344 Fix nit in getblockchaininfo (stevenroose) +- #18379 Comment fix merkle.cpp (4d55397500) +- #18382 note the costs of fetching all pull requests (vasild) +- #18391 Update init and reduce-traffic docs for -blocksonly (glowang) +- #18464 Block-relay-only vs blocksonly (MarcoFalke) +- #18486 Explain new test logging (MarcoFalke) +- #18505 Update webchat URLs in README.md (SuriyaaKudoIsc) +- #18513 Fix git add argument (HashUnlimited) +- #18577 Correct scripted-diff example link (yahiheb) +- #18589 Fix naming of macOS SDK and clarify version (achow101) + +### Miscellaneous +- #15600 lockedpool: When possible, use madvise to avoid including sensitive information in core dumps (luke-jr) +- #15934 Merge settings one place instead of five places (ryanofsky) +- #16115 On bitcoind startup, write config args to debug.log (LarryRuane) +- #16117 util: Replace boost sleep with std sleep (MarcoFalke) +- #16161 util: Fix compilation errors in support/lockedpool.cpp (jkczyz) +- #16802 scripts: In linearize, search for next position of magic bytes rather than fail (takinbo) +- #16889 Add some general std::vector utility functions (sipa) +- #17049 contrib: Bump gitian descriptors for 0.20 (MarcoFalke) +- #17052 scripts: Update `copyright_header` script to include additional files (GChuf) +- #17059 util: Simplify path argument for cblocktreedb ctor (hebasto) +- #17191 random: Remove call to `RAND_screen()` (Windows only) (fanquake) +- #17192 util: Add `check_nonfatal` and use it in src/rpc (MarcoFalke) +- #17218 Replace the LogPrint function with a macro (jkczyz) +- #17266 util: Rename decodedumptime to parseiso8601datetime (elichai) +- #17270 Feed environment data into RNG initializers (sipa) +- #17282 contrib: Remove accounts from bash completion (fanquake) +- #17293 Add assertion to randrange that input is not 0 (JeremyRubin) +- #17325 log: Fix log message for -par=1 (hebasto) +- #17329 linter: Strip trailing / in path for git-subtree-check (jnewbery) +- #17336 scripts: Search for first block file for linearize-data with some block files pruned (Rjected) +- #17361 scripts: Lint gitian descriptors with shellcheck (hebasto) +- #17482 util: Disallow network-qualified command line options (ryanofsky) +- #17507 random: mark RandAddPeriodic and SeedPeriodic as noexcept (fanquake) +- #17527 Fix CPUID subleaf iteration (sipa) +- #17604 util: Make schedulebatchpriority advisory only (fanquake) +- #17650 util: Remove unwanted fields from bitcoin-cli -getinfo (malevolent) +- #17671 script: Fixed wget call in gitian-build.py (willyko) +- #17699 Make env data logging optional (sipa) +- #17721 util: Don't allow base58 decoding of non-base58 strings. add base58 tests (practicalswift) +- #17750 util: Change getwarnings parameter to bool (jnewbery) +- #17753 util: Don't allow base32/64-decoding or parsemoney(…) on strings with embedded nul characters. add tests (practicalswift) +- #17823 scripts: Read suspicious hosts from a file instead of hardcoding (sanjaykdragon) +- #18162 util: Avoid potential uninitialized read in `formatiso8601datetime(int64_t)` by checking `gmtime_s`/`gmtime_r` return value (practicalswift) +- #18167 Fix a violation of C++ standard rules where unions are used for type-punning (TheQuantumPhysicist) +- #18225 util: Fail to parse empty string in parsemoney (MarcoFalke) +- #18270 util: Fail to parse whitespace-only strings in parsemoney(…) (instead of parsing as zero) (practicalswift) +- #18316 util: Helpexamplerpc formatting (jonatack) +- #18357 Fix missing header in sync.h (promag) +- #18412 script: Fix `script_err_sig_pushonly` error string (theStack) +- #18416 util: Limit decimal range of numbers parsescript accepts (pierreN) +- #18503 init: Replace `URL_WEBSITE` with `PACKAGE_URL` (MarcoFalke) +- #18526 Remove PID file at the very end (hebasto) +- #18553 Avoid non-trivial global constants in SHA-NI code (sipa) +- #18665 Do not expose and consider `-logthreadnames` when it does not work (hebasto) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- 251 +- 4d55397500 +- Aaron Clauson +- Adam Jonas +- Albert +- Amiti Uttarwar +- Andrew Chow +- Andrew Toth +- Anthony Towns +- Antoine Riard +- Ava Barron +- Ben Carman +- Ben Woosley +- Block Mechanic +- Brian Solon +- Bushstar +- Carl Dong +- Carnhof Daki +- Cory Fields +- Daki Carnhof +- Dan Gershony +- Daniel Kraft +- dannmat +- Danny-Scott +- darosior +- David O'Callaghan +- Dominik Spicher +- Elichai Turkel +- Emil Engler +- emu +- Fabian Jahr +- fanquake +- Filip Gospodinov +- Franck Royer +- Gastón I. Silva +- gchuf +- Gleb Naumenko +- Gloria Zhao +- glowang +- Gr0kchain +- Gregory Sanders +- hackerrdave +- Harris +- hel0 +- Hennadii Stepanov +- ianliu +- Igor Cota +- James Chiang +- James O'Beirne +- Jan Beich +- Jan Sarenik +- Jeffrey Czyz +- Jeremy Rubin +- JeremyCrookshank +- Jim Posen +- John Bampton +- John L. Jegutanis +- John Newbery +- Jon Atack +- Jon Layton +- Jonas Schnelli +- João Barbosa +- Jorge Timón +- Karl-Johan Alm +- kodslav +- Larry Ruane +- Luke Dashjr +- malevolent +- MapleLaker +- marcaiaf +- MarcoFalke +- Marius Kjærstad +- Mark Erhardt +- Mark Tyneway +- Martin Erlandsson +- Martin Zumsande +- Matt Corallo +- Matt Ward +- Michael Folkson +- Michael Polzer +- Micky Yun Chan +- Neha Narula +- nijynot +- naumenkogs +- NullFunctor +- Peter Bushnell +- pierrenn +- Pieter Wuille +- practicalswift +- randymcmillan +- Rjected +- Russell Yanofsky +- Samer Afach +- Samuel Dobson +- Sanjay K +- Sebastian Falbesoner +- setpill +- Sjors Provoost +- Stefan Richter +- stefanwouldgo +- Steven Roose +- Suhas Daftuar +- Suriyaa Sundararuban +- TheCharlatan +- Tim Akinbo +- Travin Keith +- tryphe +- Vasil Dimov +- Willy Ko +- Wilson Ccasihue S +- Wladimir J. van der Laan +- Yahia Chiheb +- Yancy Ribbens +- Yusuf Sahin HAMZA +- Zakk +- Zero + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/translation_strings_policy.md b/doc/translation_strings_policy.md index 634aca3559..1931302dda 100644 --- a/doc/translation_strings_policy.md +++ b/doc/translation_strings_policy.md @@ -23,7 +23,8 @@ On a high level, these strings are to be translated: ### GUI strings -Anything that appears to the user in the GUI is to be translated. This includes labels, menu items, button texts, tooltips and window titles. +Do not translate technical or extremely rare errors. +Anything else that appears to the user in the GUI is to be translated. This includes labels, menu items, button texts, tooltips and window titles. This includes messages passed to the GUI through the UI interface through `InitMessage`, `ThreadSafeMessageBox` or `ShowProgress`. General recommendations diff --git a/doc/zmq.md b/doc/zmq.md index a309abd0cc..3a1194de1c 100644 --- a/doc/zmq.md +++ b/doc/zmq.md @@ -96,7 +96,7 @@ ZeroMQ endpoint specifiers for TCP (and others) are documented in the Client side, then, the ZeroMQ subscriber socket must have the ZMQ_SUBSCRIBE option set to one or either of these prefixes (for instance, just `hash`); without doing so will result in no messages -arriving. Please see `contrib/zmq/zmq_sub.py` for a working example. +arriving. Please see [`contrib/zmq/zmq_sub.py`](/contrib/zmq/zmq_sub.py) for a working example. ## Remarks diff --git a/share/genbuild.sh b/share/genbuild.sh index 81fa2ed5d0..1153df933f 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -18,8 +18,8 @@ else exit 1 fi -DESC="" -SUFFIX="" +GIT_TAG="" +GIT_COMMIT="" if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ]; then # clean 'dirty' status of touched files that haven't been modified git diff >/dev/null 2>/dev/null @@ -27,18 +27,18 @@ if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$ # if latest commit is tagged and not dirty, then override using the tag name RAWDESC=$(git describe --abbrev=0 2>/dev/null) if [ "$(git rev-parse HEAD)" = "$(git rev-list -1 $RAWDESC 2>/dev/null)" ]; then - git diff-index --quiet HEAD -- && DESC=$RAWDESC + git diff-index --quiet HEAD -- && GIT_TAG=$RAWDESC fi # otherwise generate suffix from git, i.e. string like "59887e8-dirty" - SUFFIX=$(git rev-parse --short HEAD) - git diff-index --quiet HEAD -- || SUFFIX="$SUFFIX-dirty" + GIT_COMMIT=$(git rev-parse --short HEAD) + git diff-index --quiet HEAD -- || GIT_COMMIT="$GIT_COMMIT-dirty" fi -if [ -n "$DESC" ]; then - NEWINFO="#define BUILD_DESC \"$DESC\"" -elif [ -n "$SUFFIX" ]; then - NEWINFO="#define BUILD_SUFFIX $SUFFIX" +if [ -n "$GIT_TAG" ]; then + NEWINFO="#define BUILD_GIT_TAG \"$GIT_TAG\"" +elif [ -n "$GIT_COMMIT" ]; then + NEWINFO="#define BUILD_GIT_COMMIT \"$GIT_COMMIT\"" else NEWINFO="// No build information available" fi diff --git a/src/Makefile.am b/src/Makefile.am index 627df97cad..7a280a67a7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -85,6 +85,10 @@ if BUILD_BITCOIND bin_PROGRAMS += bitcoind endif +if BUILD_BITCOIN_NODE + bin_PROGRAMS += bitcoin-node +endif + if BUILD_BITCOIN_CLI bin_PROGRAMS += bitcoin-cli endif @@ -223,6 +227,7 @@ BITCOIN_CORE_H = \ util/message.h \ util/moneystr.h \ util/rbf.h \ + util/ref.h \ util/settings.h \ util/string.h \ util/threadnames.h \ @@ -236,6 +241,7 @@ BITCOIN_CORE_H = \ versionbitsinfo.h \ walletinitinterface.h \ wallet/coincontrol.h \ + wallet/context.h \ wallet/crypter.h \ wallet/db.h \ wallet/feebumper.h \ @@ -243,6 +249,7 @@ BITCOIN_CORE_H = \ wallet/ismine.h \ wallet/load.h \ wallet/rpcwallet.h \ + wallet/salvage.h \ wallet/scriptpubkeyman.h \ wallet/wallet.h \ wallet/walletdb.h \ @@ -344,6 +351,7 @@ libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ interfaces/wallet.cpp \ wallet/coincontrol.cpp \ + wallet/context.cpp \ wallet/crypter.cpp \ wallet/db.cpp \ wallet/feebumper.cpp \ @@ -351,6 +359,7 @@ libbitcoin_wallet_a_SOURCES = \ wallet/load.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ + wallet/salvage.cpp \ wallet/scriptpubkeyman.cpp \ wallet/wallet.cpp \ wallet/walletdb.cpp \ @@ -496,7 +505,6 @@ libbitcoin_util_a_SOURCES = \ support/lockedpool.cpp \ chainparamsbase.cpp \ clientversion.cpp \ - compat/glibc_sanity_fdelt.cpp \ compat/glibc_sanity.cpp \ compat/glibcxx_sanity.cpp \ compat/strnlen.cpp \ @@ -547,22 +555,21 @@ libbitcoin_cli_a_SOURCES = \ nodist_libbitcoin_util_a_SOURCES = $(srcdir)/obj/build.h # -# bitcoind binary # -bitcoind_SOURCES = bitcoind.cpp -bitcoind_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -bitcoind_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -bitcoind_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +# bitcoind & bitcoin-node binaries # +bitcoin_daemon_sources = bitcoind.cpp +bitcoin_bin_cppflags = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +bitcoin_bin_cxxflags = $(AM_CXXFLAGS) $(PIE_FLAGS) +bitcoin_bin_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) if TARGET_WINDOWS -bitcoind_SOURCES += bitcoind-res.rc +bitcoin_daemon_sources += bitcoind-res.rc endif -bitcoind_LDADD = \ - $(LIBBITCOIN_SERVER) \ +bitcoin_bin_ldadd = \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ - $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) \ + $(LIBUNIVALUE) \ $(LIBBITCOIN_ZMQ) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ @@ -571,7 +578,19 @@ bitcoind_LDADD = \ $(LIBMEMENV) \ $(LIBSECP256K1) -bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) +bitcoin_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) + +bitcoind_SOURCES = $(bitcoin_daemon_sources) +bitcoind_CPPFLAGS = $(bitcoin_bin_cppflags) +bitcoind_CXXFLAGS = $(bitcoin_bin_cxxflags) +bitcoind_LDFLAGS = $(bitcoin_bin_ldflags) +bitcoind_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) + +bitcoin_node_SOURCES = $(bitcoin_daemon_sources) +bitcoin_node_CPPFLAGS = $(bitcoin_bin_cppflags) +bitcoin_node_CXXFLAGS = $(bitcoin_bin_cxxflags) +bitcoin_node_LDFLAGS = $(bitcoin_bin_ldflags) +bitcoin_node_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) # bitcoin-cli binary # bitcoin_cli_SOURCES = bitcoin-cli.cpp @@ -615,29 +634,14 @@ bitcoin_tx_LDADD += $(BOOST_LIBS) # bitcoin-wallet binary # bitcoin_wallet_SOURCES = bitcoin-wallet.cpp -bitcoin_wallet_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -bitcoin_wallet_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -bitcoin_wallet_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +bitcoin_wallet_CPPFLAGS = $(bitcoin_bin_cppflags) +bitcoin_wallet_CXXFLAGS = $(bitcoin_bin_cxxflags) +bitcoin_wallet_LDFLAGS = $(bitcoin_bin_ldflags) +bitcoin_wallet_LDADD = $(LIBBITCOIN_WALLET_TOOL) $(bitcoin_bin_ldadd) if TARGET_WINDOWS bitcoin_wallet_SOURCES += bitcoin-wallet-res.rc endif - -bitcoin_wallet_LDADD = \ - $(LIBBITCOIN_WALLET_TOOL) \ - $(LIBBITCOIN_WALLET) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_ZMQ) \ - $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ - $(LIBMEMENV) \ - $(LIBSECP256K1) \ - $(LIBUNIVALUE) - -bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(ZMQ_LIBS) # # bitcoinconsensus library # diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 766c0fca54..93b5156af3 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -29,6 +29,7 @@ bench_bench_bitcoin_SOURCES = \ bench/crypto_hash.cpp \ bench/ccoins_caching.cpp \ bench/gcs_filter.cpp \ + bench/hashpadding.cpp \ bench/merkle_root.cpp \ bench/mempool_eviction.cpp \ bench/mempool_stress.cpp \ diff --git a/src/Makefile.leveldb.include b/src/Makefile.leveldb.include index 79ff72ca8d..8a28f4f249 100644 --- a/src/Makefile.leveldb.include +++ b/src/Makefile.leveldb.include @@ -36,7 +36,7 @@ LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_POSIX endif leveldb_libleveldb_a_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS) -leveldb_libleveldb_a_CXXFLAGS = $(filter-out -Wconditional-uninitialized -Werror=conditional-uninitialized, $(AM_CXXFLAGS)) $(PIE_FLAGS) +leveldb_libleveldb_a_CXXFLAGS = $(filter-out -Wconditional-uninitialized -Werror=conditional-uninitialized -Wsuggest-override -Werror=suggest-override, $(AM_CXXFLAGS)) $(PIE_FLAGS) leveldb_libleveldb_a_SOURCES= leveldb_libleveldb_a_SOURCES += leveldb/port/port_stdcxx.h diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index cf09eee2cb..13bfea7646 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -3,6 +3,11 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. bin_PROGRAMS += qt/bitcoin-qt + +if BUILD_BITCOIN_GUI + bin_PROGRAMS += bitcoin-gui +endif + EXTRA_LIBRARIES += qt/libbitcoinqt.a # bitcoin qt core # @@ -294,29 +299,43 @@ QT_FORMS_H=$(join $(dir $(QT_FORMS_UI)),$(addprefix ui_, $(notdir $(QT_FORMS_UI: # Most files will depend on the forms and moc files as includes. Generate them # before anything else. $(QT_MOC): $(QT_FORMS_H) -$(qt_libbitcoinqt_a_OBJECTS) $(qt_bitcoin_qt_OBJECTS) : | $(QT_MOC) +$(qt_libbitcoinqt_a_OBJECTS) $(qt_bitcoin_qt_OBJECTS) $(bitcoin_gui_OBJECTS) : | $(QT_MOC) -# bitcoin-qt binary # -qt_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ +# bitcoin-qt and bitcoin-gui binaries # +bitcoin_qt_cppflags = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ $(QT_INCLUDES) $(QR_CFLAGS) -qt_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) +bitcoin_qt_cxxflags = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) -qt_bitcoin_qt_SOURCES = qt/main.cpp +bitcoin_qt_sources = qt/main.cpp if TARGET_WINDOWS - qt_bitcoin_qt_SOURCES += $(BITCOIN_RC) + bitcoin_qt_sources += $(BITCOIN_RC) endif -qt_bitcoin_qt_LDADD = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER) +bitcoin_qt_ldadd = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER) if ENABLE_WALLET -qt_bitcoin_qt_LDADD += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET) +bitcoin_qt_ldadd += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET) endif if ENABLE_ZMQ -qt_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) +bitcoin_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ +bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) -qt_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -qt_bitcoin_qt_LIBTOOLFLAGS = $(AM_LIBTOOLFLAGS) --tag CXX +bitcoin_qt_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX + +qt_bitcoin_qt_CPPFLAGS = $(bitcoin_qt_cppflags) +qt_bitcoin_qt_CXXFLAGS = $(bitcoin_qt_cxxflags) +qt_bitcoin_qt_SOURCES = $(bitcoin_qt_sources) +qt_bitcoin_qt_LDADD = $(bitcoin_qt_ldadd) +qt_bitcoin_qt_LDFLAGS = $(bitcoin_qt_ldflags) +qt_bitcoin_qt_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) + +bitcoin_gui_CPPFLAGS = $(bitcoin_qt_cppflags) +bitcoin_gui_CXXFLAGS = $(bitcoin_qt_cxxflags) +bitcoin_gui_SOURCES = $(bitcoin_qt_sources) +bitcoin_gui_LDADD = $(bitcoin_qt_ldadd) +bitcoin_gui_LDFLAGS = $(bitcoin_qt_ldflags) +bitcoin_gui_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) #locale/foo.ts -> locale/foo.qm QT_QM=$(QT_TS:.ts=.qm) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 3a0d4fdc15..7909cb4a0f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -31,6 +31,7 @@ FUZZ_TARGETS = \ test/fuzz/chain \ test/fuzz/checkqueue \ test/fuzz/coins_deserialize \ + test/fuzz/coins_view \ test/fuzz/cuckoocache \ test/fuzz/decode_tx \ test/fuzz/descriptor_parse \ @@ -109,9 +110,14 @@ FUZZ_TARGETS = \ test/fuzz/rbf \ test/fuzz/rolling_bloom_filter \ test/fuzz/script \ + test/fuzz/script_bitcoin_consensus \ + test/fuzz/script_descriptor_cache \ test/fuzz/script_deserialize \ test/fuzz/script_flags \ + test/fuzz/script_interpreter \ test/fuzz/script_ops \ + test/fuzz/script_sigcache \ + test/fuzz/script_sign \ test/fuzz/scriptnum_ops \ test/fuzz/service_deserialize \ test/fuzz/signature_checker \ @@ -229,6 +235,7 @@ BITCOIN_TESTS =\ test/prevector_tests.cpp \ test/raii_event_tests.cpp \ test/random_tests.cpp \ + test/ref_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ @@ -465,6 +472,12 @@ test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_coins_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_coins_view_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_coins_view_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_coins_view_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_coins_view_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_coins_view_SOURCES = test/fuzz/coins_view.cpp + test_fuzz_cuckoocache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_cuckoocache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_cuckoocache_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -933,6 +946,18 @@ test_fuzz_script_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_script_SOURCES = test/fuzz/script.cpp +test_fuzz_script_bitcoin_consensus_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_script_bitcoin_consensus_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_script_bitcoin_consensus_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_script_bitcoin_consensus_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_bitcoin_consensus_SOURCES = test/fuzz/script_bitcoin_consensus.cpp + +test_fuzz_script_descriptor_cache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_script_descriptor_cache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_script_descriptor_cache_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_script_descriptor_cache_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_descriptor_cache_SOURCES = test/fuzz/script_descriptor_cache.cpp + test_fuzz_script_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSCRIPT_DESERIALIZE=1 test_fuzz_script_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_script_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -945,12 +970,30 @@ test_fuzz_script_flags_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_script_flags_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_script_flags_SOURCES = test/fuzz/script_flags.cpp +test_fuzz_script_interpreter_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_script_interpreter_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_script_interpreter_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_script_interpreter_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_interpreter_SOURCES = test/fuzz/script_interpreter.cpp + test_fuzz_script_ops_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_ops_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_script_ops_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_script_ops_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_script_ops_SOURCES = test/fuzz/script_ops.cpp +test_fuzz_script_sigcache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_script_sigcache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_script_sigcache_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_script_sigcache_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_sigcache_SOURCES = test/fuzz/script_sigcache.cpp + +test_fuzz_script_sign_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_script_sign_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_script_sign_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_script_sign_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_sign_SOURCES = test/fuzz/script_sign.cpp + test_fuzz_scriptnum_ops_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_scriptnum_ops_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_scriptnum_ops_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/bench/hashpadding.cpp b/src/bench/hashpadding.cpp new file mode 100644 index 0000000000..985be8bdba --- /dev/null +++ b/src/bench/hashpadding.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2015-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <hash.h> +#include <random.h> +#include <uint256.h> + + +static void PrePadded(benchmark::State& state) +{ + + CSHA256 hasher; + + // Setup the salted hasher + uint256 nonce = GetRandHash(); + hasher.Write(nonce.begin(), 32); + hasher.Write(nonce.begin(), 32); + uint256 data = GetRandHash(); + while (state.KeepRunning()) { + unsigned char out[32]; + CSHA256 h = hasher; + h.Write(data.begin(), 32); + h.Finalize(out); + } +} + +BENCHMARK(PrePadded, 10000); + +static void RegularPadded(benchmark::State& state) +{ + CSHA256 hasher; + + // Setup the salted hasher + uint256 nonce = GetRandHash(); + uint256 data = GetRandHash(); + while (state.KeepRunning()) { + unsigned char out[32]; + CSHA256 h = hasher; + h.Write(nonce.begin(), 32); + h.Write(data.begin(), 32); + h.Finalize(out); + } +} + +BENCHMARK(RegularPadded, 10000); diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index 00e5d7e7a0..42b351a72d 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -20,9 +20,7 @@ struct nontrivial_t { int x; nontrivial_t() :x(-1) {} - ADD_SERIALIZE_METHODS - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) {READWRITE(x);} + SERIALIZE_METHODS(nontrivial_t, obj) { READWRITE(obj.x); } }; static_assert(!IS_TRIVIALLY_CONSTRUCTIBLE<nontrivial_t>::value, "expected nontrivial_t to not be trivially constructible"); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index cdaabd6fab..8d85789b4e 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -9,6 +9,7 @@ #include <chainparamsbase.h> #include <clientversion.h> +#include <optional.h> #include <rpc/client.h> #include <rpc/protocol.h> #include <rpc/request.h> @@ -20,6 +21,7 @@ #include <functional> #include <memory> #include <stdio.h> +#include <string> #include <tuple> #include <event2/buffer.h> @@ -157,7 +159,7 @@ struct HTTPReply std::string body; }; -static const char *http_errorstring(int code) +static std::string http_errorstring(int code) { switch(code) { #if LIBEVENT_VERSION_NUMBER >= 0x02010300 @@ -250,7 +252,7 @@ public: UniValue ProcessReply(const UniValue &batch_in) override { UniValue result(UniValue::VOBJ); - std::vector<UniValue> batch = JSONRPCProcessBatchReply(batch_in, batch_in.size()); + const std::vector<UniValue> batch = JSONRPCProcessBatchReply(batch_in); // Errors in getnetworkinfo() and getblockchaininfo() are fatal, pass them on; // getwalletinfo() and getbalances() are allowed to fail if there is no wallet. if (!batch[ID_NETWORKINFO]["error"].isNull()) { @@ -304,7 +306,7 @@ public: } }; -static UniValue CallRPC(BaseRequestHandler *rh, const std::string& strMethod, const std::vector<std::string>& args) +static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, const std::vector<std::string>& args, const Optional<std::string>& rpcwallet = {}) { std::string host; // In preference order, we choose the following for the port: @@ -369,14 +371,12 @@ static UniValue CallRPC(BaseRequestHandler *rh, const std::string& strMethod, co // check if we should use a special wallet endpoint std::string endpoint = "/"; - if (!gArgs.GetArgs("-rpcwallet").empty()) { - std::string walletName = gArgs.GetArg("-rpcwallet", ""); - char *encodedURI = evhttp_uriencode(walletName.data(), walletName.size(), false); + if (rpcwallet) { + char* encodedURI = evhttp_uriencode(rpcwallet->data(), rpcwallet->size(), false); if (encodedURI) { - endpoint = "/wallet/"+ std::string(encodedURI); + endpoint = "/wallet/" + std::string(encodedURI); free(encodedURI); - } - else { + } else { throw CConnectionFailed("uri-encode failed"); } } @@ -418,6 +418,65 @@ static UniValue CallRPC(BaseRequestHandler *rh, const std::string& strMethod, co return reply; } +/** + * ConnectAndCallRPC wraps CallRPC with -rpcwait and an exception handler. + * + * @param[in] rh Pointer to RequestHandler. + * @param[in] strMethod Reference to const string method to forward to CallRPC. + * @param[in] rpcwallet Reference to const optional string wallet name to forward to CallRPC. + * @returns the RPC response as a UniValue object. + * @throws a CConnectionFailed std::runtime_error if connection failed or RPC server still in warmup. + */ +static UniValue ConnectAndCallRPC(BaseRequestHandler* rh, const std::string& strMethod, const std::vector<std::string>& args, const Optional<std::string>& rpcwallet = {}) +{ + UniValue response(UniValue::VOBJ); + // Execute and handle connection failures with -rpcwait. + const bool fWait = gArgs.GetBoolArg("-rpcwait", false); + do { + try { + response = CallRPC(rh, strMethod, args, rpcwallet); + if (fWait) { + const UniValue& error = find_value(response, "error"); + if (!error.isNull() && error["code"].get_int() == RPC_IN_WARMUP) { + throw CConnectionFailed("server in warmup"); + } + } + break; // Connection succeeded, no need to retry. + } catch (const CConnectionFailed&) { + if (fWait) { + UninterruptibleSleep(std::chrono::milliseconds{1000}); + } else { + throw; + } + } + } while (fWait); + return response; +} + +/** + * GetWalletBalances calls listwallets; if more than one wallet is loaded, it then + * fetches mine.trusted balances for each loaded wallet and pushes them to `result`. + * + * @param result Reference to UniValue object the wallet names and balances are pushed to. + */ +static void GetWalletBalances(UniValue& result) +{ + std::unique_ptr<BaseRequestHandler> rh{MakeUnique<DefaultRequestHandler>()}; + const UniValue listwallets = ConnectAndCallRPC(rh.get(), "listwallets", /* args=*/{}); + if (!find_value(listwallets, "error").isNull()) return; + const UniValue& wallets = find_value(listwallets, "result"); + if (wallets.size() <= 1) return; + + UniValue balances(UniValue::VOBJ); + for (const UniValue& wallet : wallets.getValues()) { + const std::string wallet_name = wallet.get_str(); + const UniValue getbalances = ConnectAndCallRPC(rh.get(), "getbalances", /* args=*/{}, wallet_name); + const UniValue& balance = find_value(getbalances, "result")["mine"]["trusted"]; + balances.pushKV(wallet_name, balance); + } + result.pushKV("balances", balances); +} + static int CommandLineRPC(int argc, char *argv[]) { std::string strPrint; @@ -474,9 +533,8 @@ static int CommandLineRPC(int argc, char *argv[]) } std::unique_ptr<BaseRequestHandler> rh; std::string method; - if (gArgs.GetBoolArg("-getinfo", false)) { + if (gArgs.IsArgSet("-getinfo")) { rh.reset(new GetinfoRequestHandler()); - method = ""; } else { rh.reset(new DefaultRequestHandler()); if (args.size() < 1) { @@ -485,62 +543,46 @@ static int CommandLineRPC(int argc, char *argv[]) method = args[0]; args.erase(args.begin()); // Remove trailing method name from arguments vector } - - // Execute and handle connection failures with -rpcwait - const bool fWait = gArgs.GetBoolArg("-rpcwait", false); - do { - try { - const UniValue reply = CallRPC(rh.get(), method, args); - - // Parse reply - const UniValue& result = find_value(reply, "result"); - const UniValue& error = find_value(reply, "error"); - - if (!error.isNull()) { - // Error - int code = error["code"].get_int(); - if (fWait && code == RPC_IN_WARMUP) - throw CConnectionFailed("server in warmup"); - strPrint = "error: " + error.write(); - nRet = abs(code); - if (error.isObject()) - { - UniValue errCode = find_value(error, "code"); - UniValue errMsg = find_value(error, "message"); - strPrint = errCode.isNull() ? "" : "error code: "+errCode.getValStr()+"\n"; - - if (errMsg.isStr()) - strPrint += "error message:\n"+errMsg.get_str(); - - if (errCode.isNum() && errCode.get_int() == RPC_WALLET_NOT_SPECIFIED) { - strPrint += "\nTry adding \"-rpcwallet=<filename>\" option to bitcoin-cli command line."; - } - } - } else { - // Result - if (result.isNull()) - strPrint = ""; - else if (result.isStr()) - strPrint = result.get_str(); - else - strPrint = result.write(2); + Optional<std::string> wallet_name{}; + if (gArgs.IsArgSet("-rpcwallet")) wallet_name = gArgs.GetArg("-rpcwallet", ""); + const UniValue reply = ConnectAndCallRPC(rh.get(), method, args, wallet_name); + + // Parse reply + UniValue result = find_value(reply, "result"); + const UniValue& error = find_value(reply, "error"); + if (!error.isNull()) { + // Error + strPrint = "error: " + error.write(); + nRet = abs(error["code"].get_int()); + if (error.isObject()) { + const UniValue& errCode = find_value(error, "code"); + const UniValue& errMsg = find_value(error, "message"); + strPrint = errCode.isNull() ? "" : ("error code: " + errCode.getValStr() + "\n"); + + if (errMsg.isStr()) { + strPrint += ("error message:\n" + errMsg.get_str()); + } + if (errCode.isNum() && errCode.get_int() == RPC_WALLET_NOT_SPECIFIED) { + strPrint += "\nTry adding \"-rpcwallet=<filename>\" option to bitcoin-cli command line."; } - // Connection succeeded, no need to retry. - break; } - catch (const CConnectionFailed&) { - if (fWait) - UninterruptibleSleep(std::chrono::milliseconds{1000}); - else - throw; + } else { + if (gArgs.IsArgSet("-getinfo") && !gArgs.IsArgSet("-rpcwallet")) { + GetWalletBalances(result); // fetch multiwallet balances and append to result } - } while (fWait); - } - catch (const std::exception& e) { + // Result + if (result.isNull()) { + strPrint = ""; + } else if (result.isStr()) { + strPrint = result.get_str(); + } else { + strPrint = result.write(2); + } + } + } catch (const std::exception& e) { strPrint = std::string("error: ") + e.what(); nRet = EXIT_FAILURE; - } - catch (...) { + } catch (...) { PrintExceptionContinue(nullptr, "CommandLineRPC()"); throw; } diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 7f9439788a..b420463c00 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -31,6 +31,7 @@ static void SetupWalletToolArgs() gArgs.AddArg("info", "Get wallet info", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); gArgs.AddArg("create", "Create new wallet file", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); + gArgs.AddArg("salvage", "Attempt to recover private keys from a corrupt wallet", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); } static bool WalletAppInit(int argc, char* argv[]) diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 2aa416ca44..b8e8717896 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -16,6 +16,7 @@ #include <noui.h> #include <shutdown.h> #include <ui_interface.h> +#include <util/ref.h> #include <util/strencodings.h> #include <util/system.h> #include <util/threadnames.h> @@ -56,7 +57,7 @@ static bool AppInit(int argc, char* argv[]) SetupServerArgs(node); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { - return InitError(strprintf("Error parsing command line arguments: %s\n", error)); + return InitError(Untranslated(strprintf("Error parsing command line arguments: %s\n", error))); } // Process help and version before taking care about datadir @@ -77,25 +78,26 @@ static bool AppInit(int argc, char* argv[]) return true; } + util::Ref context{node}; try { if (!CheckDataDirOption()) { - return InitError(strprintf("Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", ""))); + return InitError(Untranslated(strprintf("Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")))); } if (!gArgs.ReadConfigFiles(error, true)) { - return InitError(strprintf("Error reading configuration file: %s\n", error)); + return InitError(Untranslated(strprintf("Error reading configuration file: %s\n", error))); } // Check for -chain, -testnet or -regtest parameter (Params() calls are only valid after this clause) try { SelectParams(gArgs.GetChainName()); } catch (const std::exception& e) { - return InitError(strprintf("%s\n", e.what())); + return InitError(Untranslated(strprintf("%s\n", e.what()))); } // Error out when loose non-argument tokens are encountered on command line for (int i = 1; i < argc; i++) { if (!IsSwitchChar(argv[i][0])) { - return InitError(strprintf("Command line contains unexpected token '%s', see bitcoind -h for a list of options.\n", argv[i])); + return InitError(Untranslated(strprintf("Command line contains unexpected token '%s', see bitcoind -h for a list of options.\n", argv[i]))); } } @@ -130,13 +132,13 @@ static bool AppInit(int argc, char* argv[]) // Daemonize if (daemon(1, 0)) { // don't chdir (1), do close FDs (0) - return InitError(strprintf("daemon() failed: %s\n", strerror(errno))); + return InitError(Untranslated(strprintf("daemon() failed: %s\n", strerror(errno)))); } #if defined(MAC_OSX) #pragma GCC diagnostic pop #endif #else - return InitError("-daemon is not supported on this operating system\n"); + return InitError(Untranslated("-daemon is not supported on this operating system\n")); #endif // HAVE_DECL_DAEMON } // Lock data directory after daemonization @@ -145,7 +147,7 @@ static bool AppInit(int argc, char* argv[]) // If locking the data directory failed, exit immediately return false; } - fRet = AppInitMain(node); + fRet = AppInitMain(context, node); } catch (const std::exception& e) { PrintExceptionContinue(&e, "AppInit()"); diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 263d863cfa..a47709cd82 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -105,13 +105,12 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c std::vector<bool> have_txn(txn_available.size()); { LOCK(pool->cs); - const std::vector<std::pair<uint256, CTxMemPool::txiter> >& vTxHashes = pool->vTxHashes; - for (size_t i = 0; i < vTxHashes.size(); i++) { - uint64_t shortid = cmpctblock.GetShortID(vTxHashes[i].first); + for (size_t i = 0; i < pool->vTxHashes.size(); i++) { + uint64_t shortid = cmpctblock.GetShortID(pool->vTxHashes[i].first); std::unordered_map<uint64_t, uint16_t>::iterator idit = shorttxids.find(shortid); if (idit != shorttxids.end()) { if (!have_txn[idit->second]) { - txn_available[idit->second] = vTxHashes[i].second->GetSharedTx(); + txn_available[idit->second] = pool->vTxHashes[i].second->GetSharedTx(); have_txn[idit->second] = true; mempool_count++; } else { diff --git a/src/blockencodings.h b/src/blockencodings.h index 377ac3a1a6..326db1b4a7 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -92,12 +92,13 @@ private: friend class PartiallyDownloadedBlock; - static const int SHORTTXIDS_LENGTH = 6; protected: std::vector<uint64_t> shorttxids; std::vector<PrefilledTransaction> prefilledtxn; public: + static constexpr int SHORTTXIDS_LENGTH = 6; + CBlockHeader header; // Dummy for deserialization @@ -125,7 +126,7 @@ class PartiallyDownloadedBlock { protected: std::vector<CTransactionRef> txn_available; size_t prefilled_count = 0, mempool_count = 0, extra_count = 0; - CTxMemPool* pool; + const CTxMemPool* pool; public: CBlockHeader header; explicit PartiallyDownloadedBlock(CTxMemPool* poolIn) : pool(poolIn) {} diff --git a/src/blockfilter.h b/src/blockfilter.h index ff8744b217..96cefbf3b2 100644 --- a/src/blockfilter.h +++ b/src/blockfilter.h @@ -144,8 +144,8 @@ public: template <typename Stream> void Serialize(Stream& s) const { - s << m_block_hash - << static_cast<uint8_t>(m_filter_type) + s << static_cast<uint8_t>(m_filter_type) + << m_block_hash << m_filter.GetEncoded(); } @@ -154,8 +154,8 @@ public: std::vector<unsigned char> encoded_filter; uint8_t filter_type; - s >> m_block_hash - >> filter_type + s >> filter_type + >> m_block_hash >> encoded_filter; m_filter_type = static_cast<BlockFilterType>(filter_type); diff --git a/src/bloom.h b/src/bloom.h index 9173b80d66..9307257852 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -64,15 +64,7 @@ public: CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweak, unsigned char nFlagsIn); CBloomFilter() : nHashFuncs(0), nTweak(0), nFlags(0) {} - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(vData); - READWRITE(nHashFuncs); - READWRITE(nTweak); - READWRITE(nFlags); - } + SERIALIZE_METHODS(CBloomFilter, obj) { READWRITE(obj.vData, obj.nHashFuncs, obj.nTweak, obj.nFlags); } void insert(const std::vector<unsigned char>& vKey); void insert(const COutPoint& outpoint); diff --git a/src/clientversion.cpp b/src/clientversion.cpp index 5d9eaea6d0..993967a180 100644 --- a/src/clientversion.cpp +++ b/src/clientversion.cpp @@ -14,59 +14,34 @@ */ const std::string CLIENT_NAME("Satoshi"); -/** - * Client version number - */ -#define CLIENT_VERSION_SUFFIX "" - - -/** - * The following part of the code determines the CLIENT_BUILD variable. - * Several mechanisms are used for this: - * * first, if HAVE_BUILD_INFO is defined, include build.h, a file that is - * generated by the build environment, possibly containing the output - * of git-describe in a macro called BUILD_DESC - * * secondly, if this is an exported version of the code, GIT_ARCHIVE will - * be defined (automatically using the export-subst git attribute), and - * GIT_COMMIT will contain the commit id. - * * then, three options exist for determining CLIENT_BUILD: - * * if BUILD_DESC is defined, use that literally (output of git-describe) - * * if not, but GIT_COMMIT is defined, use v[maj].[min].[rev].[build]-g[commit] - * * otherwise, use v[maj].[min].[rev].[build]-unk - * finally CLIENT_VERSION_SUFFIX is added - */ -//! First, include build.h if requested #ifdef HAVE_BUILD_INFO #include <obj/build.h> +// The <obj/build.h>, which is generated by the build environment (share/genbuild.sh), +// could contain only one line of the following: +// - "#define BUILD_GIT_TAG ...", if the top commit is tagged +// - "#define BUILD_GIT_COMMIT ...", if the top commit is not tagged +// - "// No build information available", if proper git information is not available #endif -//! git will put "#define GIT_ARCHIVE 1" on the next line inside archives. $Format:%n#define GIT_ARCHIVE 1$ -#ifdef GIT_ARCHIVE -#define GIT_COMMIT_ID "$Format:%H$" -#define GIT_COMMIT_DATE "$Format:%cD$" -#endif - -#define BUILD_DESC_WITH_SUFFIX(maj, min, rev, build, suffix) \ - "v" DO_STRINGIZE(maj) "." DO_STRINGIZE(min) "." DO_STRINGIZE(rev) "." DO_STRINGIZE(build) "-" DO_STRINGIZE(suffix) - -#define BUILD_DESC_FROM_COMMIT(maj, min, rev, build, commit) \ - "v" DO_STRINGIZE(maj) "." DO_STRINGIZE(min) "." DO_STRINGIZE(rev) "." DO_STRINGIZE(build) "-g" commit +//! git will put "#define GIT_COMMIT_ID ..." on the next line inside archives. $Format:%n#define GIT_COMMIT_ID "%H"$ -#define BUILD_DESC_FROM_UNKNOWN(maj, min, rev, build) \ - "v" DO_STRINGIZE(maj) "." DO_STRINGIZE(min) "." DO_STRINGIZE(rev) "." DO_STRINGIZE(build) "-unk" - -#ifndef BUILD_DESC -#ifdef BUILD_SUFFIX -#define BUILD_DESC BUILD_DESC_WITH_SUFFIX(CLIENT_VERSION_MAJOR, CLIENT_VERSION_MINOR, CLIENT_VERSION_REVISION, CLIENT_VERSION_BUILD, BUILD_SUFFIX) -#elif defined(GIT_COMMIT_ID) -#define BUILD_DESC BUILD_DESC_FROM_COMMIT(CLIENT_VERSION_MAJOR, CLIENT_VERSION_MINOR, CLIENT_VERSION_REVISION, CLIENT_VERSION_BUILD, GIT_COMMIT_ID) +#ifdef BUILD_GIT_TAG + #define BUILD_DESC BUILD_GIT_TAG + #define BUILD_SUFFIX "" #else -#define BUILD_DESC BUILD_DESC_FROM_UNKNOWN(CLIENT_VERSION_MAJOR, CLIENT_VERSION_MINOR, CLIENT_VERSION_REVISION, CLIENT_VERSION_BUILD) -#endif + #define BUILD_DESC "v" STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) \ + "." STRINGIZE(CLIENT_VERSION_REVISION) "." STRINGIZE(CLIENT_VERSION_BUILD) + #ifdef BUILD_GIT_COMMIT + #define BUILD_SUFFIX "-" BUILD_GIT_COMMIT + #elif defined(GIT_COMMIT_ID) + #define BUILD_SUFFIX "-g" GIT_COMMIT_ID + #else + #define BUILD_SUFFIX "-unk" + #endif #endif -const std::string CLIENT_BUILD(BUILD_DESC CLIENT_VERSION_SUFFIX); +const std::string CLIENT_BUILD(BUILD_DESC BUILD_SUFFIX); static std::string FormatVersion(int nVersion) { diff --git a/src/compat/assumptions.h b/src/compat/assumptions.h index 6e7b4d3ded..4b0b224c69 100644 --- a/src/compat/assumptions.h +++ b/src/compat/assumptions.h @@ -50,6 +50,7 @@ static_assert(sizeof(double) == 8, "64-bit double assumed"); // code. static_assert(sizeof(short) == 2, "16-bit short assumed"); static_assert(sizeof(int) == 4, "32-bit int assumed"); +static_assert(sizeof(unsigned) == 4, "32-bit unsigned assumed"); // Assumption: We assume size_t to be 32-bit or 64-bit. // Example(s): size_t assumed to be at least 32-bit in ecdsa_signature_parse_der_lax(...). diff --git a/src/compat/glibc_compat.cpp b/src/compat/glibc_compat.cpp index 4de4fd7f45..d17de33e86 100644 --- a/src/compat/glibc_compat.cpp +++ b/src/compat/glibc_compat.cpp @@ -9,10 +9,6 @@ #include <cstddef> #include <cstdint> -#if defined(HAVE_SYS_SELECT_H) -#include <sys/select.h> -#endif - // Prior to GLIBC_2.14, memcpy was aliased to memmove. extern "C" void* memmove(void* a, const void* b, size_t c); extern "C" void* memcpy(void* a, const void* b, size_t c) @@ -20,15 +16,6 @@ extern "C" void* memcpy(void* a, const void* b, size_t c) return memmove(a, b, c); } -extern "C" void __chk_fail(void) __attribute__((__noreturn__)); -extern "C" FDELT_TYPE __fdelt_warn(FDELT_TYPE a) -{ - if (a >= FD_SETSIZE) - __chk_fail(); - return a / __NFDBITS; -} -extern "C" FDELT_TYPE __fdelt_chk(FDELT_TYPE) __attribute__((weak, alias("__fdelt_warn"))); - #if defined(__i386__) || defined(__arm__) extern "C" int64_t __udivmoddi4(uint64_t u, uint64_t v, uint64_t* rp); diff --git a/src/compat/glibc_sanity.cpp b/src/compat/glibc_sanity.cpp index cc74f28899..0367b9a53f 100644 --- a/src/compat/glibc_sanity.cpp +++ b/src/compat/glibc_sanity.cpp @@ -8,10 +8,6 @@ #include <cstddef> -#if defined(HAVE_SYS_SELECT_H) -bool sanity_test_fdelt(); -#endif - extern "C" void* memcpy(void* a, const void* b, size_t c); void* memcpy_int(void* a, const void* b, size_t c) { @@ -45,9 +41,5 @@ bool sanity_test_memcpy() bool glibc_sanity_test() { -#if defined(HAVE_SYS_SELECT_H) - if (!sanity_test_fdelt()) - return false; -#endif return sanity_test_memcpy<1025>(); } diff --git a/src/compat/glibc_sanity_fdelt.cpp b/src/compat/glibc_sanity_fdelt.cpp deleted file mode 100644 index 87140d0c71..0000000000 --- a/src/compat/glibc_sanity_fdelt.cpp +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#if defined(HAVE_SYS_SELECT_H) -#ifdef HAVE_CSTRING_DEPENDENT_FD_ZERO -#include <cstring> -#endif -#include <sys/select.h> - -// trigger: Call FD_SET to trigger __fdelt_chk. FORTIFY_SOURCE must be defined -// as >0 and optimizations must be set to at least -O2. -// test: Add a file descriptor to an empty fd_set. Verify that it has been -// correctly added. -bool sanity_test_fdelt() -{ - fd_set fds; - FD_ZERO(&fds); - FD_SET(0, &fds); - return FD_ISSET(0, &fds); -} -#endif diff --git a/src/core_read.cpp b/src/core_read.cpp index df78c319ee..1c0a8a096d 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -19,6 +19,7 @@ #include <boost/algorithm/string/split.hpp> #include <algorithm> +#include <string> CScript ParseScript(const std::string& s) { @@ -34,10 +35,9 @@ CScript ParseScript(const std::string& s) if (op < OP_NOP && op != OP_RESERVED) continue; - const char* name = GetOpName(static_cast<opcodetype>(op)); - if (strcmp(name, "OP_UNKNOWN") == 0) + std::string strName = GetOpName(static_cast<opcodetype>(op)); + if (strName == "OP_UNKNOWN") continue; - std::string strName(name); mapOpNames[strName] = static_cast<opcodetype>(op); // Convenience: OP_ADD and just ADD are both recognized: boost::algorithm::replace_first(strName, "OP_", ""); diff --git a/src/flatfile.h b/src/flatfile.h index 60b3503cc3..04f6373a24 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -16,13 +16,7 @@ struct FlatFilePos int nFile; unsigned int nPos; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT_MODE(nFile, VarIntMode::NONNEGATIVE_SIGNED)); - READWRITE(VARINT(nPos)); - } + SERIALIZE_METHODS(FlatFilePos, obj) { READWRITE(VARINT_MODE(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED), VARINT(obj.nPos)); } FlatFilePos() : nFile(-1), nPos(0) {} diff --git a/src/fs.cpp b/src/fs.cpp index 066c6c10d3..e68c97b3ca 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -6,6 +6,9 @@ #ifndef WIN32 #include <fcntl.h> +#include <string> +#include <sys/file.h> +#include <sys/utsname.h> #else #ifndef NOMINMAX #define NOMINMAX @@ -47,20 +50,38 @@ FileLock::~FileLock() } } +static bool IsWSL() +{ + struct utsname uname_data; + return uname(&uname_data) == 0 && std::string(uname_data.version).find("Microsoft") != std::string::npos; +} + bool FileLock::TryLock() { if (fd == -1) { return false; } - struct flock lock; - lock.l_type = F_WRLCK; - lock.l_whence = SEEK_SET; - lock.l_start = 0; - lock.l_len = 0; - if (fcntl(fd, F_SETLK, &lock) == -1) { - reason = GetErrorReason(); - return false; + + // Exclusive file locking is broken on WSL using fcntl (issue #18622) + // This workaround can be removed once the bug on WSL is fixed + static const bool is_wsl = IsWSL(); + if (is_wsl) { + if (flock(fd, LOCK_EX | LOCK_NB) == -1) { + reason = GetErrorReason(); + return false; + } + } else { + struct flock lock; + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + if (fcntl(fd, F_SETLK, &lock) == -1) { + reason = GetErrorReason(); + return false; + } } + return true; } #else diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 60c4d06f12..f1b9997371 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -9,7 +9,6 @@ #include <httpserver.h> #include <rpc/protocol.h> #include <rpc/server.h> -#include <ui_interface.h> #include <util/strencodings.h> #include <util/system.h> #include <util/translation.h> @@ -151,7 +150,7 @@ static bool RPCAuthorized(const std::string& strAuth, std::string& strAuthUserna return multiUserAuthorized(strUserPass); } -static bool HTTPReq_JSONRPC(HTTPRequest* req, const std::string &) +static bool HTTPReq_JSONRPC(const util::Ref& context, HTTPRequest* req) { // JSONRPC handles only POST if (req->GetRequestMethod() != HTTPRequest::POST) { @@ -166,7 +165,7 @@ static bool HTTPReq_JSONRPC(HTTPRequest* req, const std::string &) return false; } - JSONRPCRequest jreq; + JSONRPCRequest jreq(context); jreq.peerAddr = req->GetPeer().ToString(); if (!RPCAuthorized(authHeader.second, jreq.authUser)) { LogPrintf("ThreadRPCServer incorrect password attempt from %s\n", jreq.peerAddr); @@ -249,11 +248,8 @@ static bool InitRPCAuthentication() { if (gArgs.GetArg("-rpcpassword", "") == "") { - LogPrintf("No rpcpassword set - using random cookie authentication.\n"); + LogPrintf("Using random cookie authentication.\n"); if (!GenerateAuthCookie(&strRPCUserColonPass)) { - uiInterface.ThreadSafeMessageBox( - _("Error: A fatal internal error occurred, see debug.log for details").translated, // Same message as AbortNode - "", CClientUIInterface::MSG_ERROR); return false; } } else { @@ -288,15 +284,16 @@ static bool InitRPCAuthentication() return true; } -bool StartHTTPRPC() +bool StartHTTPRPC(const util::Ref& context) { LogPrint(BCLog::RPC, "Starting HTTP RPC server\n"); if (!InitRPCAuthentication()) return false; - RegisterHTTPHandler("/", true, HTTPReq_JSONRPC); + auto handle_rpc = [&context](HTTPRequest* req, const std::string&) { return HTTPReq_JSONRPC(context, req); }; + RegisterHTTPHandler("/", true, handle_rpc); if (g_wallet_init_interface.HasWalletSupport()) { - RegisterHTTPHandler("/wallet/", false, HTTPReq_JSONRPC); + RegisterHTTPHandler("/wallet/", false, handle_rpc); } struct event_base* eventBase = EventBase(); assert(eventBase); diff --git a/src/httprpc.h b/src/httprpc.h index 99e4d59b8a..a6a38fc95a 100644 --- a/src/httprpc.h +++ b/src/httprpc.h @@ -5,11 +5,14 @@ #ifndef BITCOIN_HTTPRPC_H #define BITCOIN_HTTPRPC_H +namespace util { +class Ref; +} // namespace util /** Start HTTP RPC subsystem. * Precondition; HTTP and RPC has been started. */ -bool StartHTTPRPC(); +bool StartHTTPRPC(const util::Ref& context); /** Interrupt HTTP RPC subsystem. */ void InterruptHTTPRPC(); @@ -21,7 +24,7 @@ void StopHTTPRPC(); /** Start HTTP REST subsystem. * Precondition; HTTP and RPC has been started. */ -void StartREST(); +void StartREST(const util::Ref& context); /** Interrupt RPC REST subsystem. */ void InterruptREST(); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 176284d103..5e78fd1d71 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -6,14 +6,15 @@ #include <chainparamsbase.h> #include <compat.h> -#include <util/threadnames.h> -#include <util/system.h> -#include <util/strencodings.h> #include <netbase.h> #include <rpc/protocol.h> // For HTTP status codes #include <shutdown.h> #include <sync.h> #include <ui_interface.h> +#include <util/strencodings.h> +#include <util/system.h> +#include <util/threadnames.h> +#include <util/translation.h> #include <deque> #include <memory> @@ -175,7 +176,7 @@ static bool InitHTTPAllowList() LookupSubNet(strAllow, subnet); if (!subnet.IsValid()) { uiInterface.ThreadSafeMessageBox( - strprintf("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24).", strAllow), + strprintf(Untranslated("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24)."), strAllow), "", CClientUIInterface::MSG_ERROR); return false; } @@ -420,7 +421,7 @@ bool UpdateHTTPServerLogging(bool enable) { #endif } -static std::thread threadHTTP; +static std::thread g_thread_http; static std::vector<std::thread> g_thread_http_workers; void StartHTTPServer() @@ -428,7 +429,7 @@ void StartHTTPServer() LogPrint(BCLog::HTTP, "Starting HTTP server\n"); int rpcThreads = std::max((long)gArgs.GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); LogPrintf("HTTP: starting %d worker threads\n", rpcThreads); - threadHTTP = std::thread(ThreadHTTP, eventBase); + g_thread_http = std::thread(ThreadHTTP, eventBase); for (int i = 0; i < rpcThreads; i++) { g_thread_http_workers.emplace_back(HTTPWorkQueueRun, workQueue, i); @@ -466,7 +467,7 @@ void StopHTTPServer() boundSockets.clear(); if (eventBase) { LogPrint(BCLog::HTTP, "Waiting for HTTP event thread to exit\n"); - threadHTTP.join(); + if (g_thread_http.joinable()) g_thread_http.join(); } if (eventHTTP) { evhttp_free(eventHTTP); diff --git a/src/index/base.cpp b/src/index/base.cpp index 7bff463f5b..74ea421e13 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -8,6 +8,7 @@ #include <tinyformat.h> #include <ui_interface.h> #include <util/system.h> +#include <util/translation.h> #include <validation.h> #include <warnings.h> @@ -23,7 +24,7 @@ static void FatalError(const char* fmt, const Args&... args) SetMiscWarning(strMessage); LogPrintf("*** %s\n", strMessage); uiInterface.ThreadSafeMessageBox( - "Error: A fatal internal error occurred, see debug.log for details", + Untranslated("Error: A fatal internal error occurred, see debug.log for details"), "", CClientUIInterface::MSG_ERROR); StartShutdown(); } diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index c3ce8d7af0..65a5f03a8e 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -31,6 +31,12 @@ constexpr char DB_FILTER_POS = 'P'; constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000; // 16 MiB /** The pre-allocation chunk size for fltr?????.dat files */ constexpr unsigned int FLTR_FILE_CHUNK_SIZE = 0x100000; // 1 MiB +/** Maximum size of the cfheaders cache + * We have a limit to prevent a bug in filling this cache + * potentially turning into an OOM. At 2000 entries, this cache + * is big enough for a 2,000,000 length block chain, which + * we should be enough until ~2047. */ +constexpr size_t CF_HEADERS_CACHE_MAX_SZ{2000}; namespace { @@ -39,14 +45,7 @@ struct DBVal { uint256 header; FlatFilePos pos; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(hash); - READWRITE(header); - READWRITE(pos); - } + SERIALIZE_METHODS(DBVal, obj) { READWRITE(obj.hash, obj.header, obj.pos); } }; struct DBHeightKey { @@ -78,17 +77,14 @@ struct DBHashKey { explicit DBHashKey(const uint256& hash_in) : hash(hash_in) {} - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { + SERIALIZE_METHODS(DBHashKey, obj) { char prefix = DB_BLOCK_HASH; READWRITE(prefix); if (prefix != DB_BLOCK_HASH) { throw std::ios_base::failure("Invalid format for block filter index DB hash key"); } - READWRITE(hash); + READWRITE(obj.hash); } }; @@ -387,13 +383,32 @@ bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& return ReadFilterFromDisk(entry.pos, filter_out); } -bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const +bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) { + LOCK(m_cs_headers_cache); + + bool is_checkpoint{block_index->nHeight % CFCHECKPT_INTERVAL == 0}; + + if (is_checkpoint) { + // Try to find the block in the headers cache if this is a checkpoint height. + auto header = m_headers_cache.find(block_index->GetBlockHash()); + if (header != m_headers_cache.end()) { + header_out = header->second; + return true; + } + } + DBVal entry; if (!LookupOne(*m_db, block_index, entry)) { return false; } + if (is_checkpoint && + m_headers_cache.size() < CF_HEADERS_CACHE_MAX_SZ) { + // Add to the headers cache if this is a checkpoint height. + m_headers_cache.emplace(block_index->GetBlockHash(), entry.header); + } + header_out = entry.header; return true; } diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index 436d52515f..317f8c0e40 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -10,6 +10,14 @@ #include <flatfile.h> #include <index/base.h> +/** Interval between compact filter checkpoints. See BIP 157. */ +static constexpr int CFCHECKPT_INTERVAL = 1000; + +struct FilterHeaderHasher +{ + size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } +}; + /** * BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of * blocks by height. An index is constructed for each supported filter type with its own database @@ -30,6 +38,10 @@ private: bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const; size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter); + Mutex m_cs_headers_cache; + /** cache of block hash to filter header, to avoid disk access when responding to getcfcheckpt. */ + std::unordered_map<uint256, uint256, FilterHeaderHasher> m_headers_cache GUARDED_BY(m_cs_headers_cache); + protected: bool Init() override; @@ -54,7 +66,7 @@ public: bool LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const; /** Get a single filter header by block. */ - bool LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const; + bool LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out); /** Get a range of filters between two heights on a chain. */ bool LookupFilterRange(int start_height, const CBlockIndex* stop_index, diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 5bbe6ad1df..59d1888fff 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -9,8 +9,6 @@ #include <util/translation.h> #include <validation.h> -#include <boost/thread.hpp> - constexpr char DB_BEST_BLOCK = 'B'; constexpr char DB_TXINDEX = 't'; constexpr char DB_TXINDEX_BLOCK = 'T'; @@ -21,12 +19,10 @@ struct CDiskTxPos : public FlatFilePos { unsigned int nTxOffset; // after header - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(FlatFilePos, *this); - READWRITE(VARINT(nTxOffset)); + SERIALIZE_METHODS(CDiskTxPos, obj) + { + READWRITEAS(FlatFilePos, obj); + READWRITE(VARINT(obj.nTxOffset)); } CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { @@ -152,7 +148,6 @@ bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& bool interrupted = false; std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator()); for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) { - boost::this_thread::interruption_point(); if (ShutdownRequested()) { interrupted = true; break; diff --git a/src/init.cpp b/src/init.cpp index 3e2ee4d425..56d75c90c7 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -18,6 +18,7 @@ #include <compat/sanity.h> #include <consensus/validation.h> #include <fs.h> +#include <hash.h> #include <httprpc.h> #include <httpserver.h> #include <index/blockfilterindex.h> @@ -42,6 +43,7 @@ #include <script/sigcache.h> #include <script/standard.h> #include <shutdown.h> +#include <sync.h> #include <timedata.h> #include <torcontrol.h> #include <txdb.h> @@ -49,19 +51,19 @@ #include <ui_interface.h> #include <util/asmap.h> #include <util/moneystr.h> +#include <util/string.h> #include <util/system.h> #include <util/threadnames.h> #include <util/translation.h> #include <validation.h> -#include <hash.h> - #include <validationinterface.h> #include <walletinitinterface.h> +#include <functional> +#include <set> #include <stdint.h> #include <stdio.h> -#include <set> #ifndef WIN32 #include <attributes.h> @@ -70,11 +72,9 @@ #include <sys/stat.h> #endif -#include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/replace.hpp> -#include <boost/algorithm/string/split.hpp> #include <boost/signals2/signal.hpp> -#include <boost/thread.hpp> +#include <boost/thread/thread.hpp> #if ENABLE_ZMQ #include <zmq/zmqabstractnotifier.h> @@ -121,7 +121,7 @@ NODISCARD static bool CreatePidFile() #endif return true; } else { - return InitError(strprintf(_("Unable to create the PID file '%s': %s").translated, GetPidFile().string(), std::strerror(errno))); + return InitError(strprintf(_("Unable to create the PID file '%s': %s"), GetPidFile().string(), std::strerror(errno))); } } @@ -172,11 +172,10 @@ void Interrupt(NodeContext& node) void Shutdown(NodeContext& node) { + static Mutex g_shutdown_mutex; + TRY_LOCK(g_shutdown_mutex, lock_shutdown); + if (!lock_shutdown) return; LogPrintf("%s: In progress...\n", __func__); - static RecursiveMutex cs_Shutdown; - TRY_LOCK(cs_Shutdown, lockShutdown); - if (!lockShutdown) - return; /// Note: Shutdown() must be able to handle cases in which initialization failed part of the way, /// for example if the data directory was found to be locked. @@ -243,9 +242,9 @@ void Shutdown(NodeContext& node) } // FlushStateToDisk generates a ChainStateFlushed callback, which we should avoid missing - { + if (node.chainman) { LOCK(cs_main); - for (CChainState* chainstate : g_chainman.GetAll()) { + for (CChainState* chainstate : node.chainman->GetAll()) { if (chainstate->CanFlushToDisk()) { chainstate->ForceFlushStateToDisk(); } @@ -270,9 +269,9 @@ void Shutdown(NodeContext& node) // up with our current chain to avoid any strange pruning edge cases and make // next startup faster by avoiding rescan. - { + if (node.chainman) { LOCK(cs_main); - for (CChainState* chainstate : g_chainman.GetAll()) { + for (CChainState* chainstate : node.chainman->GetAll()) { if (chainstate->CanFlushToDisk()) { chainstate->ForceFlushStateToDisk(); chainstate->ResetCoinsViews(); @@ -298,7 +297,8 @@ void Shutdown(NodeContext& node) globalVerifyHandle.reset(); ECC_Stop(); node.args = nullptr; - if (node.mempool) node.mempool = nullptr; + node.mempool = nullptr; + node.chainman = nullptr; node.scheduler.reset(); try { @@ -350,13 +350,13 @@ static void registerSignalHandler(int signal, void(*handler)(int)) static boost::signals2::connection rpc_notify_block_change_connection; static void OnRPCStarted() { - rpc_notify_block_change_connection = uiInterface.NotifyBlockTip_connect(&RPCNotifyBlockChange); + rpc_notify_block_change_connection = uiInterface.NotifyBlockTip_connect(std::bind(RPCNotifyBlockChange, std::placeholders::_2)); } static void OnRPCStopped() { rpc_notify_block_change_connection.disconnect(); - RPCNotifyBlockChange(false, nullptr); + RPCNotifyBlockChange(nullptr); g_best_block_cv.notify_all(); LogPrint(BCLog::RPC, "RPC stopped.\n"); } @@ -442,10 +442,11 @@ void SetupServerArgs(NodeContext& node) gArgs.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by peers forward or backward by this amount. (default: %u seconds)", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h), 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h). Limit does not apply to peers with 'noban' permission. 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor hidden services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-onlynet=<net>", "Make outgoing connections only through network <net> (ipv4, ipv6 or onion). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-permitbaremultisig", strprintf("Relay non-P2SH multisig (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-port=<port>", strprintf("Listen for connections on <port> (default: %u, testnet: %u, regtest: %u)", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-proxy=<ip:port>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -465,11 +466,7 @@ void SetupServerArgs(NodeContext& node) hidden_args.emplace_back("-upnp"); #endif gArgs.AddArg("-whitebind=<[permissions@]addr>", "Bind to given address and whitelist peers connecting to it. " - "Use [host]:port notation for IPv6. Allowed permissions are bloomfilter (allow requesting BIP37 filtered blocks and transactions), " - "noban (do not ban for misbehavior), " - "forcerelay (relay transactions that are already in the mempool; implies relay), " - "relay (relay even in -blocksonly mode), " - "and mempool (allow requesting BIP35 mempool contents). " + "Use [host]:port notation for IPv6. Allowed permissions: " + Join(NET_PERMISSIONS_DOC, ", ") + ". " "Specify multiple permissions separated by commas (default: noban,mempool,relay). Can be specified multiple times.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-whitelist=<[permissions@]IP address or network>", "Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) or " @@ -499,14 +496,7 @@ void SetupServerArgs(NodeContext& node) #endif gArgs.AddArg("-checkblocks=<n>", strprintf("How many blocks to check at startup (default: %u, 0 = all)", DEFAULT_CHECKBLOCKS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - gArgs.AddArg("-checklevel=<n>", strprintf("How thorough the block verification of -checkblocks is: " - "level 0 reads the blocks from disk, " - "level 1 verifies block validity, " - "level 2 verifies undo data, " - "level 3 checks disconnection of tip blocks, " - "and level 4 tries to reconnect the blocks, " - "each level includes the checks of the previous levels " - "(0-4, default: %u)", DEFAULT_CHECKLEVEL), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + gArgs.AddArg("-checklevel=<n>", strprintf("How thorough the block verification of -checkblocks is: %s (0-4, default: %u)", Join(CHECKLEVEL_DOC, ", "), DEFAULT_CHECKLEVEL), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, chainstate, and other validation data structures occasionally. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-checkpoints", strprintf("Enable rejection of any forks from the known historical chain until block 295000 (default: %u)", DEFAULT_CHECKPOINTS_ENABLED), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -603,9 +593,9 @@ std::string LicenseInfo() } #if HAVE_SYSTEM -static void BlockNotifyCallback(bool initialSync, const CBlockIndex *pBlockIndex) +static void BlockNotifyCallback(SynchronizationState sync_state, const CBlockIndex* pBlockIndex) { - if (initialSync || !pBlockIndex) + if (sync_state != SynchronizationState::POST_INIT || !pBlockIndex) return; std::string strCmd = gArgs.GetArg("-blocknotify", ""); @@ -621,7 +611,7 @@ static bool fHaveGenesis = false; static Mutex g_genesis_wait_mutex; static std::condition_variable g_genesis_wait_cv; -static void BlockNotifyGenesisWait(bool, const CBlockIndex *pBlockIndex) +static void BlockNotifyGenesisWait(const CBlockIndex* pBlockIndex) { if (pBlockIndex != nullptr) { { @@ -687,7 +677,7 @@ static void CleanupBlockRevFiles() } } -static void ThreadImport(std::vector<fs::path> vImportFiles) +static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles) { const CChainParams& chainparams = Params(); util::ThreadRename("loadblk"); @@ -739,9 +729,9 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) // scan for better chains in the block chain database, that are not yet connected in the active best chain // We can't hold cs_main during ActivateBestChain even though we're accessing - // the g_chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve + // the chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve // the relevant pointers before the ABC call. - for (CChainState* chainstate : WITH_LOCK(::cs_main, return g_chainman.GetAll())) { + for (CChainState* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { BlockValidationState state; if (!chainstate->ActivateBestChain(state, chainparams, nullptr)) { LogPrintf("Failed to connect best block (%s)\n", state.ToString()); @@ -768,32 +758,30 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) */ static bool InitSanityCheck() { - if(!ECC_InitSanityCheck()) { - InitError("Elliptic curve cryptography sanity check failure. Aborting."); - return false; + if (!ECC_InitSanityCheck()) { + return InitError(Untranslated("Elliptic curve cryptography sanity check failure. Aborting.")); } if (!glibc_sanity_test() || !glibcxx_sanity_test()) return false; if (!Random_SanityCheck()) { - InitError("OS cryptographic RNG sanity check failure. Aborting."); - return false; + return InitError(Untranslated("OS cryptographic RNG sanity check failure. Aborting.")); } return true; } -static bool AppInitServers() +static bool AppInitServers(const util::Ref& context) { RPCServer::OnStarted(&OnRPCStarted); RPCServer::OnStopped(&OnRPCStopped); if (!InitHTTPServer()) return false; StartRPC(); - if (!StartHTTPRPC()) + if (!StartHTTPRPC(context)) return false; - if (gArgs.GetBoolArg("-rest", DEFAULT_REST_ENABLE)) StartREST(); + if (gArgs.GetBoolArg("-rest", DEFAULT_REST_ENABLE)) StartREST(context); StartHTTPServer(); return true; } @@ -929,8 +917,9 @@ bool AppInitBasicSetup() HeapSetInformation(nullptr, HeapEnableTerminationOnCorruption, nullptr, 0); #endif - if (!SetupNetworking()) - return InitError("Initializing networking failed"); + if (!SetupNetworking()) { + return InitError(Untranslated("Initializing networking failed.")); + } #ifndef WIN32 if (!gArgs.GetBoolArg("-sysperms", false)) { @@ -962,21 +951,31 @@ bool AppInitParameterInteraction() // also see: InitParameterInteraction() - // Warn if network-specific options (-addnode, -connect, etc) are + // Error if network-specific options (-addnode, -connect, etc) are // specified in default section of config file, but not overridden // on the command line or in this network's section of the config file. std::string network = gArgs.GetChainName(); + bilingual_str errors; for (const auto& arg : gArgs.GetUnsuitableSectionOnlyArgs()) { - return InitError(strprintf(_("Config setting for %s only applied on %s network when in [%s] section.").translated, arg, network, network)); + errors += strprintf(_("Config setting for %s only applied on %s network when in [%s] section.") + Untranslated("\n"), arg, network, network); + } + + if (!errors.empty()) { + return InitError(errors); } // Warn if unrecognized section name are present in the config file. + bilingual_str warnings; for (const auto& section : gArgs.GetUnrecognizedSections()) { - InitWarning(strprintf("%s:%i " + _("Section [%s] is not recognized.").translated, section.m_file, section.m_line, section.m_name)); + warnings += strprintf(Untranslated("%s:%i ") + _("Section [%s] is not recognized.") + Untranslated("\n"), section.m_file, section.m_line, section.m_name); + } + + if (!warnings.empty()) { + InitWarning(warnings); } if (!fs::is_directory(GetBlocksDir())) { - return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist.").translated, gArgs.GetArg("-blocksdir", ""))); + return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), gArgs.GetArg("-blocksdir", ""))); } // parse and validate enabled filter types @@ -988,25 +987,32 @@ bool AppInitParameterInteraction() for (const auto& name : names) { BlockFilterType filter_type; if (!BlockFilterTypeByName(name, filter_type)) { - return InitError(strprintf(_("Unknown -blockfilterindex value %s.").translated, name)); + return InitError(strprintf(_("Unknown -blockfilterindex value %s."), name)); } g_enabled_filter_types.insert(filter_type); } } + // Basic filters are the only supported filters. The basic filters index must be enabled + // to serve compact filters + if (gArgs.GetBoolArg("-peerblockfilters", DEFAULT_PEERBLOCKFILTERS) && + g_enabled_filter_types.count(BlockFilterType::BASIC) != 1) { + return InitError(_("Cannot set -peerblockfilters without -blockfilterindex.")); + } + // if using block pruning, then disallow txindex if (gArgs.GetArg("-prune", 0)) { if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) - return InitError(_("Prune mode is incompatible with -txindex.").translated); + return InitError(_("Prune mode is incompatible with -txindex.")); if (!g_enabled_filter_types.empty()) { - return InitError(_("Prune mode is incompatible with -blockfilterindex.").translated); + return InitError(_("Prune mode is incompatible with -blockfilterindex.")); } } // -bind and -whitebind can't be set when not listening size_t nUserBind = gArgs.GetArgs("-bind").size() + gArgs.GetArgs("-whitebind").size(); if (nUserBind != 0 && !gArgs.GetBoolArg("-listen", DEFAULT_LISTEN)) { - return InitError("Cannot set -bind or -whitebind together with -listen=0"); + return InitError(Untranslated("Cannot set -bind or -whitebind together with -listen=0")); } // Make sure enough file descriptors are available @@ -1024,11 +1030,11 @@ bool AppInitParameterInteraction() #endif nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS), 0); if (nFD < MIN_CORE_FILEDESCRIPTORS) - return InitError(_("Not enough file descriptors available.").translated); + return InitError(_("Not enough file descriptors available.")); nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS, nMaxConnections); if (nMaxConnections < nUserMaxConnections) - InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations.").translated, nUserMaxConnections, nMaxConnections)); + InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); // ********************************************************* Step 3: parameter-to-internal-flags if (gArgs.IsArgSet("-debug")) { @@ -1039,7 +1045,7 @@ bool AppInitParameterInteraction() [](std::string cat){return cat == "0" || cat == "none";})) { for (const auto& cat : categories) { if (!LogInstance().EnableCategory(cat)) { - InitWarning(strprintf(_("Unsupported logging category %s=%s.").translated, "-debug", cat)); + InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debug", cat)); } } } @@ -1048,7 +1054,7 @@ bool AppInitParameterInteraction() // Now remove the logging categories which were explicitly excluded for (const std::string& cat : gArgs.GetArgs("-debugexclude")) { if (!LogInstance().DisableCategory(cat)) { - InitWarning(strprintf(_("Unsupported logging category %s=%s.").translated, "-debugexclude", cat)); + InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); } } @@ -1069,7 +1075,7 @@ bool AppInitParameterInteraction() if (gArgs.IsArgSet("-minimumchainwork")) { const std::string minChainWorkStr = gArgs.GetArg("-minimumchainwork", ""); if (!IsHexNumber(minChainWorkStr)) { - return InitError(strprintf("Invalid non-hex (%s) minimum chain work value specified", minChainWorkStr)); + return InitError(strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), minChainWorkStr)); } nMinimumChainWork = UintToArith256(uint256S(minChainWorkStr)); } else { @@ -1084,21 +1090,21 @@ bool AppInitParameterInteraction() int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t nMempoolSizeMin = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; if (nMempoolSizeMax < 0 || nMempoolSizeMax < nMempoolSizeMin) - return InitError(strprintf(_("-maxmempool must be at least %d MB").translated, std::ceil(nMempoolSizeMin / 1000000.0))); + return InitError(strprintf(_("-maxmempool must be at least %d MB"), std::ceil(nMempoolSizeMin / 1000000.0))); // incremental relay fee sets the minimum feerate increase necessary for BIP 125 replacement in the mempool // and the amount the mempool min fee increases above the feerate of txs evicted due to mempool limiting. if (gArgs.IsArgSet("-incrementalrelayfee")) { CAmount n = 0; if (!ParseMoney(gArgs.GetArg("-incrementalrelayfee", ""), n)) - return InitError(AmountErrMsg("incrementalrelayfee", gArgs.GetArg("-incrementalrelayfee", "")).translated); + return InitError(AmountErrMsg("incrementalrelayfee", gArgs.GetArg("-incrementalrelayfee", ""))); incrementalRelayFee = CFeeRate(n); } // block pruning; get the amount of disk space (in MiB) to allot for block & undo files int64_t nPruneArg = gArgs.GetArg("-prune", 0); if (nPruneArg < 0) { - return InitError(_("Prune cannot be configured with a negative value.").translated); + return InitError(_("Prune cannot be configured with a negative value.")); } nPruneTarget = (uint64_t) nPruneArg * 1024 * 1024; if (nPruneArg == 1) { // manual pruning: -prune=1 @@ -1107,7 +1113,7 @@ bool AppInitParameterInteraction() fPruneMode = true; } else if (nPruneTarget) { if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) { - return InitError(strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number.").translated, MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)); + return InitError(strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)); } LogPrintf("Prune configured to target %u MiB on disk for block and undo files.\n", nPruneTarget / 1024 / 1024); fPruneMode = true; @@ -1120,13 +1126,13 @@ bool AppInitParameterInteraction() peer_connect_timeout = gArgs.GetArg("-peertimeout", DEFAULT_PEER_CONNECT_TIMEOUT); if (peer_connect_timeout <= 0) { - return InitError("peertimeout cannot be configured with a negative value."); + return InitError(Untranslated("peertimeout cannot be configured with a negative value.")); } if (gArgs.IsArgSet("-minrelaytxfee")) { CAmount n = 0; if (!ParseMoney(gArgs.GetArg("-minrelaytxfee", ""), n)) { - return InitError(AmountErrMsg("minrelaytxfee", gArgs.GetArg("-minrelaytxfee", "")).translated); + return InitError(AmountErrMsg("minrelaytxfee", gArgs.GetArg("-minrelaytxfee", ""))); } // High fee check is done afterward in CWallet::CreateWalletFromFile() ::minRelayTxFee = CFeeRate(n); @@ -1142,7 +1148,7 @@ bool AppInitParameterInteraction() { CAmount n = 0; if (!ParseMoney(gArgs.GetArg("-blockmintxfee", ""), n)) - return InitError(AmountErrMsg("blockmintxfee", gArgs.GetArg("-blockmintxfee", "")).translated); + return InitError(AmountErrMsg("blockmintxfee", gArgs.GetArg("-blockmintxfee", ""))); } // Feerate used to define dust. Shouldn't be changed lightly as old @@ -1151,13 +1157,13 @@ bool AppInitParameterInteraction() { CAmount n = 0; if (!ParseMoney(gArgs.GetArg("-dustrelayfee", ""), n)) - return InitError(AmountErrMsg("dustrelayfee", gArgs.GetArg("-dustrelayfee", "")).translated); + return InitError(AmountErrMsg("dustrelayfee", gArgs.GetArg("-dustrelayfee", ""))); dustRelayFee = CFeeRate(n); } fRequireStandard = !gArgs.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (!chainparams.IsTestChain() && !fRequireStandard) { - return InitError(strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.NetworkIDString())); + return InitError(strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.NetworkIDString())); } nBytesPerSigOp = gArgs.GetArg("-bytespersigop", nBytesPerSigOp); @@ -1174,10 +1180,10 @@ bool AppInitParameterInteraction() nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); if (gArgs.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0) - return InitError("rpcserialversion must be non-negative."); + return InitError(Untranslated("rpcserialversion must be non-negative.")); if (gArgs.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) - return InitError("unknown rpcserialversion requested."); + return InitError(Untranslated("Unknown rpcserialversion requested.")); nMaxTipAge = gArgs.GetArg("-maxtipage", DEFAULT_MAX_TIP_AGE); @@ -1189,10 +1195,10 @@ static bool LockDataDirectory(bool probeOnly) // Make sure only a single Bitcoin process is using the data directory. fs::path datadir = GetDataDir(); if (!DirIsWritable(datadir)) { - return InitError(strprintf(_("Cannot write to data directory '%s'; check permissions.").translated, datadir.string())); + return InitError(strprintf(_("Cannot write to data directory '%s'; check permissions."), datadir.string())); } if (!LockDirectory(datadir, ".lock", probeOnly)) { - return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running.").translated, datadir.string(), PACKAGE_NAME)); + return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running."), datadir.string(), PACKAGE_NAME)); } return true; } @@ -1210,7 +1216,7 @@ bool AppInitSanityChecks() // Sanity check if (!InitSanityCheck()) - return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down.").translated, PACKAGE_NAME)); + return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down."), PACKAGE_NAME)); // Probe the data directory lock to give an early error message, if possible // We cannot hold the data directory lock here, as the forking for daemon() hasn't yet happened, @@ -1230,7 +1236,7 @@ bool AppInitLockDataDirectory() return true; } -bool AppInitMain(NodeContext& node) +bool AppInitMain(const util::Ref& context, NodeContext& node) { const CChainParams& chainparams = Params(); // ********************************************************* Step 4a: application initialization @@ -1246,7 +1252,7 @@ bool AppInitMain(NodeContext& node) } } if (!LogInstance().StartLogging()) { - return InitError(strprintf("Could not open debug log file %s", + return InitError(strprintf(Untranslated("Could not open debug log file %s"), LogInstance().m_file_path.string())); } @@ -1261,7 +1267,7 @@ bool AppInitMain(NodeContext& node) LogPrintf("Config file: %s\n", config_file_path.string()); } else if (gArgs.IsArgSet("-conf")) { // Warn if no conf file exists at path provided by user - InitWarning(strprintf(_("The specified config file %s does not exist\n").translated, config_file_path.string())); + InitWarning(strprintf(_("The specified config file %s does not exist\n"), config_file_path.string())); } else { // Not categorizing as "Warning" because it's the default behavior LogPrintf("Config file: %s (not found, skipping)\n", config_file_path.string()); @@ -1332,7 +1338,6 @@ bool AppInitMain(NodeContext& node) for (const auto& client : node.chain_clients) { client->registerRpcs(); } - g_rpc_node = &node; #if ENABLE_ZMQ RegisterZMQRPCCommands(tableRPC); #endif @@ -1345,8 +1350,8 @@ bool AppInitMain(NodeContext& node) if (gArgs.GetBoolArg("-server", false)) { uiInterface.InitMessage_connect(SetRPCWarmupStatus); - if (!AppInitServers()) - return InitError(_("Unable to start HTTP server. See debug log for details.").translated); + if (!AppInitServers(context)) + return InitError(_("Unable to start HTTP server. See debug log for details.")); } // ********************************************************* Step 5: verify wallet database integrity @@ -1370,20 +1375,23 @@ bool AppInitMain(NodeContext& node) // which are all started after this, may use it from the node context. assert(!node.mempool); node.mempool = &::mempool; + assert(!node.chainman); + node.chainman = &g_chainman; + ChainstateManager& chainman = EnsureChainman(node); - node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), *node.scheduler, *node.mempool)); + node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), *node.scheduler, *node.chainman, *node.mempool)); RegisterValidationInterface(node.peer_logic.get()); // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; for (const std::string& cmt : gArgs.GetArgs("-uacomment")) { if (cmt != SanitizeString(cmt, SAFE_CHARS_UA_COMMENT)) - return InitError(strprintf(_("User Agent comment (%s) contains unsafe characters.").translated, cmt)); + return InitError(strprintf(_("User Agent comment (%s) contains unsafe characters."), cmt)); uacomments.push_back(cmt); } strSubVersion = FormatSubVersion(CLIENT_NAME, CLIENT_VERSION, uacomments); if (strSubVersion.size() > MAX_SUBVERSION_LENGTH) { - return InitError(strprintf(_("Total length of network version string (%i) exceeds maximum length (%i). Reduce the number or size of uacomments.").translated, + return InitError(strprintf(_("Total length of network version string (%i) exceeds maximum length (%i). Reduce the number or size of uacomments."), strSubVersion.size(), MAX_SUBVERSION_LENGTH)); } @@ -1392,7 +1400,7 @@ bool AppInitMain(NodeContext& node) for (const std::string& snet : gArgs.GetArgs("-onlynet")) { enum Network net = ParseNetwork(snet); if (net == NET_UNROUTABLE) - return InitError(strprintf(_("Unknown network specified in -onlynet: '%s'").translated, snet)); + return InitError(strprintf(_("Unknown network specified in -onlynet: '%s'"), snet)); nets.insert(net); } for (int n = 0; n < NET_MAX; n++) { @@ -1413,12 +1421,12 @@ bool AppInitMain(NodeContext& node) if (proxyArg != "" && proxyArg != "0") { CService proxyAddr; if (!Lookup(proxyArg, proxyAddr, 9050, fNameLookup)) { - return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'").translated, proxyArg)); + return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); } proxyType addrProxy = proxyType(proxyAddr, proxyRandomize); if (!addrProxy.IsValid()) - return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'").translated, proxyArg)); + return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); SetProxy(NET_IPV4, addrProxy); SetProxy(NET_IPV6, addrProxy); @@ -1437,11 +1445,11 @@ bool AppInitMain(NodeContext& node) } else { CService onionProxy; if (!Lookup(onionArg, onionProxy, 9050, fNameLookup)) { - return InitError(strprintf(_("Invalid -onion address or hostname: '%s'").translated, onionArg)); + return InitError(strprintf(_("Invalid -onion address or hostname: '%s'"), onionArg)); } proxyType addrOnion = proxyType(onionProxy, proxyRandomize); if (!addrOnion.IsValid()) - return InitError(strprintf(_("Invalid -onion address or hostname: '%s'").translated, onionArg)); + return InitError(strprintf(_("Invalid -onion address or hostname: '%s'"), onionArg)); SetProxy(NET_ONION, addrOnion); SetReachable(NET_ONION, true); } @@ -1457,7 +1465,7 @@ bool AppInitMain(NodeContext& node) if (Lookup(strAddr, addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) AddLocal(addrLocal, LOCAL_MANUAL); else - return InitError(ResolveErrMsg("externalip", strAddr)); + return InitError(Untranslated(ResolveErrMsg("externalip", strAddr))); } // Read asmap file if configured @@ -1470,12 +1478,12 @@ bool AppInitMain(NodeContext& node) asmap_path = GetDataDir() / asmap_path; } if (!fs::exists(asmap_path)) { - InitError(strprintf(_("Could not find asmap file %s").translated, asmap_path)); + InitError(strprintf(_("Could not find asmap file %s"), asmap_path)); return false; } std::vector<bool> asmap = CAddrMan::DecodeAsmap(asmap_path); if (asmap.size() == 0) { - InitError(strprintf(_("Could not parse asmap file %s").translated, asmap_path)); + InitError(strprintf(_("Could not parse asmap file %s"), asmap_path)); return false; } const uint256 asmap_version = SerializeHash(asmap); @@ -1542,7 +1550,7 @@ bool AppInitMain(NodeContext& node) auto is_coinsview_empty = [&](CChainState* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { return fReset || fReindexChainState || chainstate->CoinsTip().GetBestBlock().IsNull(); }; - std::string strLoadError; + bilingual_str strLoadError; uiInterface.InitMessage(_("Loading block index...").translated); @@ -1550,7 +1558,7 @@ bool AppInitMain(NodeContext& node) const int64_t load_block_index_start_time = GetTimeMillis(); try { LOCK(cs_main); - g_chainman.InitializeChainstate(); + chainman.InitializeChainstate(); UnloadBlockIndex(); // new CBlockTreeDB tries to delete the existing file, which @@ -1571,9 +1579,9 @@ bool AppInitMain(NodeContext& node) // block file from disk. // Note that it also sets fReindex based on the disk flag! // From here on out fReindex and fReset mean something different! - if (!LoadBlockIndex(chainparams)) { + if (!chainman.LoadBlockIndex(chainparams)) { if (ShutdownRequested()) break; - strLoadError = _("Error loading block database").translated; + strLoadError = _("Error loading block database"); break; } @@ -1581,13 +1589,13 @@ bool AppInitMain(NodeContext& node) // (we're likely using a testnet datadir, or the other way around). if (!::BlockIndex().empty() && !LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) { - return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?").translated); + return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); } // Check for changed -prune state. What we are concerned about is a user who has pruned blocks // in the past, but is now trying to run unpruned. if (fHavePruned && !fPruneMode) { - strLoadError = _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain").translated; + strLoadError = _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain"); break; } @@ -1596,7 +1604,7 @@ bool AppInitMain(NodeContext& node) // (otherwise we use the one already on disk). // This is called again in ThreadImport after the reindex completes. if (!fReindex && !LoadGenesisBlock(chainparams)) { - strLoadError = _("Error initializing block database").translated; + strLoadError = _("Error initializing block database"); break; } @@ -1605,7 +1613,7 @@ bool AppInitMain(NodeContext& node) bool failed_chainstate_init = false; - for (CChainState* chainstate : g_chainman.GetAll()) { + for (CChainState* chainstate : chainman.GetAll()) { LogPrintf("Initializing chainstate %s\n", chainstate->ToString()); chainstate->InitCoinsDB( /* cache_size_bytes */ nCoinDBCache, @@ -1614,21 +1622,21 @@ bool AppInitMain(NodeContext& node) chainstate->CoinsErrorCatcher().AddReadErrCallback([]() { uiInterface.ThreadSafeMessageBox( - _("Error reading from database, shutting down.").translated, + _("Error reading from database, shutting down."), "", CClientUIInterface::MSG_ERROR); }); // If necessary, upgrade from older database format. // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate if (!chainstate->CoinsDB().Upgrade()) { - strLoadError = _("Error upgrading chainstate database").translated; + strLoadError = _("Error upgrading chainstate database"); failed_chainstate_init = true; break; } // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate if (!chainstate->ReplayBlocks(chainparams)) { - strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated; + strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate."); failed_chainstate_init = true; break; } @@ -1640,7 +1648,7 @@ bool AppInitMain(NodeContext& node) if (!is_coinsview_empty(chainstate)) { // LoadChainTip initializes the chain based on CoinsTip()'s best block if (!chainstate->LoadChainTip(chainparams)) { - strLoadError = _("Error initializing block database").translated; + strLoadError = _("Error initializing block database"); failed_chainstate_init = true; break; // out of the per-chainstate loop } @@ -1653,14 +1661,14 @@ bool AppInitMain(NodeContext& node) } } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); - strLoadError = _("Error opening block database").translated; + strLoadError = _("Error opening block database"); break; } bool failed_rewind{false}; // Can't hold cs_main while calling RewindBlockIndex, so retrieve the relevant // chainstates beforehand. - for (CChainState* chainstate : WITH_LOCK(::cs_main, return g_chainman.GetAll())) { + for (CChainState* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { if (!fReset) { // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. // It both disconnects blocks based on the chainstate, and drops block data in @@ -1669,7 +1677,7 @@ bool AppInitMain(NodeContext& node) if (!chainstate->RewindBlockIndex(chainparams)) { strLoadError = _( "Unable to rewind the database to a pre-fork state. " - "You will need to redownload the blockchain").translated; + "You will need to redownload the blockchain"); failed_rewind = true; break; // out of the per-chainstate loop } @@ -1685,7 +1693,7 @@ bool AppInitMain(NodeContext& node) try { LOCK(cs_main); - for (CChainState* chainstate : g_chainman.GetAll()) { + for (CChainState* chainstate : chainman.GetAll()) { if (!is_coinsview_empty(chainstate)) { uiInterface.InitMessage(_("Verifying blocks...").translated); if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { @@ -1694,11 +1702,11 @@ bool AppInitMain(NodeContext& node) } const CBlockIndex* tip = chainstate->m_chain.Tip(); - RPCNotifyBlockChange(true, tip); + RPCNotifyBlockChange(tip); if (tip && tip->nTime > GetAdjustedTime() + 2 * 60 * 60) { strLoadError = _("The block database contains a block which appears to be from the future. " "This may be due to your computer's date and time being set incorrectly. " - "Only rebuild the block database if you are sure that your computer's date and time are correct").translated; + "Only rebuild the block database if you are sure that your computer's date and time are correct"); failed_verification = true; break; } @@ -1710,7 +1718,7 @@ bool AppInitMain(NodeContext& node) chainparams, &chainstate->CoinsDB(), gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { - strLoadError = _("Corrupted block database detected").translated; + strLoadError = _("Corrupted block database detected"); failed_verification = true; break; } @@ -1718,7 +1726,7 @@ bool AppInitMain(NodeContext& node) } } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); - strLoadError = _("Error opening block database").translated; + strLoadError = _("Error opening block database"); failed_verification = true; break; } @@ -1733,8 +1741,8 @@ bool AppInitMain(NodeContext& node) // first suggest a reindex if (!fReset) { bool fRet = uiInterface.ThreadSafeQuestion( - strLoadError + ".\n\n" + _("Do you want to rebuild the block database now?").translated, - strLoadError + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", + strLoadError + Untranslated(".\n\n") + _("Do you want to rebuild the block database now?"), + strLoadError.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); if (fRet) { fReindex = true; @@ -1791,7 +1799,7 @@ bool AppInitMain(NodeContext& node) nLocalServices = ServiceFlags(nLocalServices & ~NODE_NETWORK); if (!fReindex) { LOCK(cs_main); - for (CChainState* chainstate : g_chainman.GetAll()) { + for (CChainState* chainstate : chainman.GetAll()) { uiInterface.InitMessage(_("Pruning blockstore...").translated); chainstate->PruneAndFlush(); } @@ -1807,11 +1815,11 @@ bool AppInitMain(NodeContext& node) // ********************************************************* Step 11: import blocks if (!CheckDiskSpace(GetDataDir())) { - InitError(strprintf(_("Error: Disk space is low for %s").translated, GetDataDir())); + InitError(strprintf(_("Error: Disk space is low for %s"), GetDataDir())); return false; } if (!CheckDiskSpace(GetBlocksDir())) { - InitError(strprintf(_("Error: Disk space is low for %s").translated, GetBlocksDir())); + InitError(strprintf(_("Error: Disk space is low for %s"), GetBlocksDir())); return false; } @@ -1819,7 +1827,7 @@ bool AppInitMain(NodeContext& node) // No locking, as this happens before any background thread is started. boost::signals2::connection block_notify_genesis_wait_connection; if (::ChainActive().Tip() == nullptr) { - block_notify_genesis_wait_connection = uiInterface.NotifyBlockTip_connect(BlockNotifyGenesisWait); + block_notify_genesis_wait_connection = uiInterface.NotifyBlockTip_connect(std::bind(BlockNotifyGenesisWait, std::placeholders::_2)); } else { fHaveGenesis = true; } @@ -1834,7 +1842,7 @@ bool AppInitMain(NodeContext& node) vImportFiles.push_back(strFile); } - threadGroup.create_thread(std::bind(&ThreadImport, vImportFiles)); + threadGroup.create_thread([=, &chainman] { ThreadImport(chainman, vImportFiles); }); // Wait for genesis block to be processed { @@ -1880,7 +1888,7 @@ bool AppInitMain(NodeContext& node) connOptions.m_max_outbound_full_relay = std::min(MAX_OUTBOUND_FULL_RELAY_CONNECTIONS, connOptions.nMaxConnections); connOptions.m_max_outbound_block_relay = std::min(MAX_BLOCKS_ONLY_CONNECTIONS, connOptions.nMaxConnections-connOptions.m_max_outbound_full_relay); connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; - connOptions.nMaxFeeler = 1; + connOptions.nMaxFeeler = MAX_FEELER_CONNECTIONS; connOptions.nBestHeight = chain_active_height; connOptions.uiInterface = &uiInterface; connOptions.m_banman = node.banman.get(); @@ -1896,21 +1904,21 @@ bool AppInitMain(NodeContext& node) for (const std::string& strBind : gArgs.GetArgs("-bind")) { CService addrBind; if (!Lookup(strBind, addrBind, GetListenPort(), false)) { - return InitError(ResolveErrMsg("bind", strBind)); + return InitError(Untranslated(ResolveErrMsg("bind", strBind))); } connOptions.vBinds.push_back(addrBind); } for (const std::string& strBind : gArgs.GetArgs("-whitebind")) { NetWhitebindPermissions whitebind; std::string error; - if (!NetWhitebindPermissions::TryParse(strBind, whitebind, error)) return InitError(error); + if (!NetWhitebindPermissions::TryParse(strBind, whitebind, error)) return InitError(Untranslated(error)); connOptions.vWhiteBinds.push_back(whitebind); } for (const auto& net : gArgs.GetArgs("-whitelist")) { NetWhitelistPermissions subnet; std::string error; - if (!NetWhitelistPermissions::TryParse(net, subnet, error)) return InitError(error); + if (!NetWhitelistPermissions::TryParse(net, subnet, error)) return InitError(Untranslated(error)); connOptions.vWhitelistedRange.push_back(subnet); } diff --git a/src/init.h b/src/init.h index ef568b6f38..33fe96e8ea 100644 --- a/src/init.h +++ b/src/init.h @@ -14,6 +14,9 @@ struct NodeContext; namespace boost { class thread_group; } // namespace boost +namespace util { +class Ref; +} // namespace util /** Interrupt threads */ void Interrupt(NodeContext& node); @@ -51,7 +54,7 @@ bool AppInitLockDataDirectory(); * @note This should only be done after daemonization. Call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read, AppInitLockDataDirectory should have been called. */ -bool AppInitMain(NodeContext& node); +bool AppInitMain(const util::Ref& context, NodeContext& node); /** * Register all arguments with the ArgsManager diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index c5262e4bc0..d1e04b114d 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -63,9 +63,9 @@ public: { m_notifications->transactionAddedToMempool(tx); } - void TransactionRemovedFromMempool(const CTransactionRef& tx) override + void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) override { - m_notifications->transactionRemovedFromMempool(tx); + m_notifications->transactionRemovedFromMempool(tx, reason); } void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* index) override { @@ -127,7 +127,7 @@ public: ::tableRPC.appendCommand(m_command.name, &m_command); } - void disconnect() override final + void disconnect() final { if (m_wrapped_command) { m_wrapped_command = nullptr; @@ -344,8 +344,8 @@ public: bool shutdownRequested() override { return ShutdownRequested(); } int64_t getAdjustedTime() override { return GetAdjustedTime(); } void initMessage(const std::string& message) override { ::uiInterface.InitMessage(message); } - void initWarning(const std::string& message) override { InitWarning(message); } - void initError(const std::string& message) override { InitError(message); } + void initWarning(const bilingual_str& message) override { InitWarning(message); } + void initError(const bilingual_str& message) override { InitError(message); } void showProgress(const std::string& title, int progress, bool resume_possible) override { ::uiInterface.ShowProgress(title, progress, resume_possible); diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index e33fe54ac8..61d7ddb934 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -20,7 +20,9 @@ class CRPCCommand; class CScheduler; class Coin; class uint256; +enum class MemPoolRemovalReason; enum class RBFTransactionState; +struct bilingual_str; struct CBlockLocator; struct FeeCalculation; struct NodeContext; @@ -224,10 +226,10 @@ public: virtual void initMessage(const std::string& message) = 0; //! Send init warning. - virtual void initWarning(const std::string& message) = 0; + virtual void initWarning(const bilingual_str& message) = 0; //! Send init error. - virtual void initError(const std::string& message) = 0; + virtual void initError(const bilingual_str& message) = 0; //! Send progress indicator. virtual void showProgress(const std::string& title, int progress, bool resume_possible) = 0; @@ -238,7 +240,7 @@ public: public: virtual ~Notifications() {} virtual void transactionAddedToMempool(const CTransactionRef& tx) {} - virtual void transactionRemovedFromMempool(const CTransactionRef& ptx) {} + virtual void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) {} virtual void blockConnected(const CBlock& block, int height) {} virtual void blockDisconnected(const CBlock& block, int height) {} virtual void updatedBlockTip() {} diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 8564819e6a..bd1b36fea5 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -27,7 +27,9 @@ #include <sync.h> #include <txmempool.h> #include <ui_interface.h> +#include <util/ref.h> #include <util/system.h> +#include <util/translation.h> #include <validation.h> #include <warnings.h> @@ -54,7 +56,7 @@ namespace { class NodeImpl : public Node { public: - void initError(const std::string& message) override { InitError(message); } + void initError(const std::string& message) override { InitError(Untranslated(message)); } bool parseParameters(int argc, const char* const argv[], std::string& error) override { return gArgs.ParseParameters(argc, argv, error); @@ -79,14 +81,22 @@ public: bool appInitMain() override { m_context.chain = MakeChain(m_context); - return AppInitMain(m_context); + return AppInitMain(m_context_ref, m_context); } void appShutdown() override { Interrupt(m_context); Shutdown(m_context); } - void startShutdown() override { StartShutdown(); } + void startShutdown() override + { + StartShutdown(); + // Stop RPC for clean shutdown if any of waitfor* commands is executed. + if (gArgs.GetBoolArg("-server", false)) { + InterruptRPC(); + StopRPC(); + } + } bool shutdownRequested() override { return ShutdownRequested(); } void mapPort(bool use_upnp) override { @@ -185,6 +195,11 @@ public: LOCK(::cs_main); return ::ChainActive().Height(); } + uint256 getBestBlockHash() override + { + const CBlockIndex* tip = WITH_LOCK(::cs_main, return ::ChainActive().Tip()); + return tip ? tip->GetBlockHash() : Params().GenesisBlock().GetHash(); + } int64_t getLastBlockTime() override { LOCK(::cs_main); @@ -224,7 +239,7 @@ public: CFeeRate getDustRelayFee() override { return ::dustRelayFee; } UniValue executeRpc(const std::string& command, const UniValue& params, const std::string& uri) override { - JSONRPCRequest req; + JSONRPCRequest req(m_context_ref); req.params = params; req.strMethod = command; req.URI = uri; @@ -307,21 +322,22 @@ public: } std::unique_ptr<Handler> handleNotifyBlockTip(NotifyBlockTipFn fn) override { - return MakeHandler(::uiInterface.NotifyBlockTip_connect([fn](bool initial_download, const CBlockIndex* block) { - fn(initial_download, block->nHeight, block->GetBlockTime(), + return MakeHandler(::uiInterface.NotifyBlockTip_connect([fn](SynchronizationState sync_state, const CBlockIndex* block) { + fn(sync_state, BlockTip{block->nHeight, block->GetBlockTime(), block->GetBlockHash()}, GuessVerificationProgress(Params().TxData(), block)); })); } std::unique_ptr<Handler> handleNotifyHeaderTip(NotifyHeaderTipFn fn) override { return MakeHandler( - ::uiInterface.NotifyHeaderTip_connect([fn](bool initial_download, const CBlockIndex* block) { - fn(initial_download, block->nHeight, block->GetBlockTime(), + ::uiInterface.NotifyHeaderTip_connect([fn](SynchronizationState sync_state, const CBlockIndex* block) { + fn(sync_state, BlockTip{block->nHeight, block->GetBlockTime(), block->GetBlockHash()}, /* verification progress is unused when a header was received */ 0); })); } NodeContext* context() override { return &m_context; } NodeContext m_context; + util::Ref m_context_ref{m_context}; }; } // namespace diff --git a/src/interfaces/node.h b/src/interfaces/node.h index db9b42b293..0b7fb6736a 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -27,6 +27,7 @@ class Coin; class RPCTimerInterface; class UniValue; class proxyType; +enum class SynchronizationState; enum class WalletCreationStatus; struct CNodeStateStats; struct NodeContext; @@ -35,6 +36,7 @@ struct bilingual_str; namespace interfaces { class Handler; class Wallet; +struct BlockTip; //! Top-level interface for a bitcoin node (bitcoind process). class Node @@ -148,6 +150,9 @@ public: //! Get num blocks. virtual int getNumBlocks() = 0; + //! Get best block hash. + virtual uint256 getBestBlockHash() = 0; + //! Get last block time. virtual int64_t getLastBlockTime() = 0; @@ -213,11 +218,11 @@ public: //! Register handler for message box messages. using MessageBoxFn = - std::function<bool(const std::string& message, const std::string& caption, unsigned int style)>; + std::function<bool(const bilingual_str& message, const std::string& caption, unsigned int style)>; virtual std::unique_ptr<Handler> handleMessageBox(MessageBoxFn fn) = 0; //! Register handler for question messages. - using QuestionFn = std::function<bool(const std::string& message, + using QuestionFn = std::function<bool(const bilingual_str& message, const std::string& non_interactive_message, const std::string& caption, unsigned int style)>; @@ -249,12 +254,12 @@ public: //! Register handler for block tip messages. using NotifyBlockTipFn = - std::function<void(bool initial_download, int height, int64_t block_time, double verification_progress)>; + std::function<void(SynchronizationState, interfaces::BlockTip tip, double verification_progress)>; virtual std::unique_ptr<Handler> handleNotifyBlockTip(NotifyBlockTipFn fn) = 0; //! Register handler for header tip messages. using NotifyHeaderTipFn = - std::function<void(bool initial_download, int height, int64_t block_time, double verification_progress)>; + std::function<void(SynchronizationState, interfaces::BlockTip tip, double verification_progress)>; virtual std::unique_ptr<Handler> handleNotifyHeaderTip(NotifyHeaderTipFn fn) = 0; //! Return pointer to internal chain interface, useful for testing. @@ -264,6 +269,13 @@ public: //! Return implementation of Node interface. std::unique_ptr<Node> MakeNode(); +//! Block tip (could be a header or not, depends on the subscribed signal). +struct BlockTip { + int block_height; + int64_t block_time; + uint256 block_hash; +}; + } // namespace interfaces #endif // BITCOIN_INTERFACES_NODE_H diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 13b034936b..397403d308 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -9,13 +9,16 @@ #include <interfaces/handler.h> #include <policy/fees.h> #include <primitives/transaction.h> +#include <rpc/server.h> #include <script/standard.h> #include <support/allocators/secure.h> #include <sync.h> #include <ui_interface.h> #include <uint256.h> #include <util/check.h> +#include <util/ref.h> #include <util/system.h> +#include <wallet/context.h> #include <wallet/feebumper.h> #include <wallet/fees.h> #include <wallet/ismine.h> @@ -351,14 +354,13 @@ public: } return result; } - bool tryGetBalances(WalletBalances& balances, int& num_blocks, bool force, int cached_num_blocks) override + bool tryGetBalances(WalletBalances& balances, uint256& block_hash) override { TRY_LOCK(m_wallet->cs_wallet, locked_wallet); if (!locked_wallet) { return false; } - num_blocks = m_wallet->GetLastBlockHeight(); - if (!force && num_blocks == cached_num_blocks) return false; + block_hash = m_wallet->GetLastBlockHash(); balances = getBalances(); return true; } @@ -482,16 +484,21 @@ class WalletClientImpl : public ChainClient { public: WalletClientImpl(Chain& chain, std::vector<std::string> wallet_filenames) - : m_chain(chain), m_wallet_filenames(std::move(wallet_filenames)) + : m_wallet_filenames(std::move(wallet_filenames)) { + m_context.chain = &chain; } void registerRpcs() override { - g_rpc_chain = &m_chain; - return RegisterWalletRPCCommands(m_chain, m_rpc_handlers); + for (const CRPCCommand& command : GetWalletRPCCommands()) { + m_rpc_commands.emplace_back(command.category, command.name, [this, &command](const JSONRPCRequest& request, UniValue& result, bool last_handler) { + return command.actor({request, m_context}, result, last_handler); + }, command.argNames, command.unique_id); + m_rpc_handlers.emplace_back(m_context.chain->handleRpc(m_rpc_commands.back())); + } } - bool verify() override { return VerifyWallets(m_chain, m_wallet_filenames); } - bool load() override { return LoadWallets(m_chain, m_wallet_filenames); } + bool verify() override { return VerifyWallets(*m_context.chain, m_wallet_filenames); } + bool load() override { return LoadWallets(*m_context.chain, m_wallet_filenames); } void start(CScheduler& scheduler) override { return StartWallets(scheduler); } void flush() override { return FlushWallets(); } void stop() override { return StopWallets(); } @@ -506,9 +513,10 @@ public: } ~WalletClientImpl() override { UnloadWallets(); } - Chain& m_chain; + WalletContext m_context; std::vector<std::string> m_wallet_filenames; std::vector<std::unique_ptr<Handler>> m_rpc_handlers; + std::list<CRPCCommand> m_rpc_commands; }; } // namespace diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index f35335c69f..67569a3e55 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -202,11 +202,8 @@ public: //! Get balances. virtual WalletBalances getBalances() = 0; - //! Get balances if possible without waiting for chain and wallet locks. - virtual bool tryGetBalances(WalletBalances& balances, - int& num_blocks, - bool force, - int cached_num_blocks) = 0; + //! Get balances if possible without blocking. + virtual bool tryGetBalances(WalletBalances& balances, uint256& block_hash) = 0; //! Get balance. virtual CAmount getBalance() = 0; diff --git a/src/logging.cpp b/src/logging.cpp index eb9da06d9b..35e0754f20 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -22,8 +22,8 @@ BCLog::Logger& LogInstance() * access the logger. When the shutdown sequence is fully audited and tested, * explicit destruction of these objects can be implemented by changing this * from a raw pointer to a std::unique_ptr. - * Since the destructor is never called, the logger and all its members must - * have a trivial destructor. + * Since the ~Logger() destructor is never called, the Logger class and all + * its subclasses must have implicitly-defined destructors. * * This method of initialization was originally introduced in * ee3374234c60aba2cc4c5cd5cac1c0aefc2d817c. @@ -41,7 +41,7 @@ static int FileWriteStr(const std::string &str, FILE *fp) bool BCLog::Logger::StartLogging() { - std::lock_guard<std::mutex> scoped_lock(m_cs); + StdLockGuard scoped_lock(m_cs); assert(m_buffering); assert(m_fileout == nullptr); @@ -80,7 +80,7 @@ bool BCLog::Logger::StartLogging() void BCLog::Logger::DisconnectTestLogger() { - std::lock_guard<std::mutex> scoped_lock(m_cs); + StdLockGuard scoped_lock(m_cs); m_buffering = true; if (m_fileout != nullptr) fclose(m_fileout); m_fileout = nullptr; @@ -95,15 +95,7 @@ void BCLog::Logger::EnableCategory(BCLog::LogFlags flag) bool BCLog::Logger::EnableCategory(const std::string& str) { BCLog::LogFlags flag; - if (!GetLogCategory(flag, str)) { - if (str == "db") { - // DEPRECATION: Added in 0.20, should start returning an error in 0.21 - LogPrintf("Warning: logging category 'db' is deprecated, use 'walletdb' instead\n"); - EnableCategory(BCLog::WALLETDB); - return true; - } - return false; - } + if (!GetLogCategory(flag, str)) return false; EnableCategory(flag); return true; } @@ -246,7 +238,7 @@ namespace BCLog { void BCLog::Logger::LogPrintStr(const std::string& str) { - std::lock_guard<std::mutex> scoped_lock(m_cs); + StdLockGuard scoped_lock(m_cs); std::string str_prefixed = LogEscapeMessage(str); if (m_log_threadnames && m_started_new_line) { diff --git a/src/logging.h b/src/logging.h index ab07010316..7e646ef67a 100644 --- a/src/logging.h +++ b/src/logging.h @@ -8,6 +8,7 @@ #include <fs.h> #include <tinyformat.h> +#include <threadsafety.h> #include <util/string.h> #include <atomic> @@ -61,10 +62,11 @@ namespace BCLog { class Logger { private: - mutable std::mutex m_cs; // Can not use Mutex from sync.h because in debug mode it would cause a deadlock when a potential deadlock was detected - FILE* m_fileout = nullptr; // GUARDED_BY(m_cs) - std::list<std::string> m_msgs_before_open; // GUARDED_BY(m_cs) - bool m_buffering{true}; //!< Buffer messages before logging can be started. GUARDED_BY(m_cs) + mutable StdMutex m_cs; // Can not use Mutex from sync.h because in debug mode it would cause a deadlock when a potential deadlock was detected + + FILE* m_fileout GUARDED_BY(m_cs) = nullptr; + std::list<std::string> m_msgs_before_open GUARDED_BY(m_cs); + bool m_buffering GUARDED_BY(m_cs) = true; //!< Buffer messages before logging can be started. /** * m_started_new_line is a state variable that will suppress printing of @@ -79,7 +81,7 @@ namespace BCLog { std::string LogTimestampStr(const std::string& str); /** Slots that connect to the print signal */ - std::list<std::function<void(const std::string&)>> m_print_callbacks /* GUARDED_BY(m_cs) */ {}; + std::list<std::function<void(const std::string&)>> m_print_callbacks GUARDED_BY(m_cs) {}; public: bool m_print_to_console = false; @@ -98,14 +100,14 @@ namespace BCLog { /** Returns whether logs will be written to any output */ bool Enabled() const { - std::lock_guard<std::mutex> scoped_lock(m_cs); + StdLockGuard scoped_lock(m_cs); return m_buffering || m_print_to_console || m_print_to_file || !m_print_callbacks.empty(); } /** Connect a slot to the print signal and return the connection */ std::list<std::function<void(const std::string&)>>::iterator PushBackCallback(std::function<void(const std::string&)> fun) { - std::lock_guard<std::mutex> scoped_lock(m_cs); + StdLockGuard scoped_lock(m_cs); m_print_callbacks.push_back(std::move(fun)); return --m_print_callbacks.end(); } @@ -113,7 +115,7 @@ namespace BCLog { /** Delete a connection */ void DeleteCallback(std::list<std::function<void(const std::string&)>>::iterator it) { - std::lock_guard<std::mutex> scoped_lock(m_cs); + StdLockGuard scoped_lock(m_cs); m_print_callbacks.erase(it); } diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index 4ac6219886..8072b12119 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -9,6 +9,24 @@ #include <consensus/consensus.h> +std::vector<unsigned char> BitsToBytes(const std::vector<bool>& bits) +{ + std::vector<unsigned char> ret((bits.size() + 7) / 8); + for (unsigned int p = 0; p < bits.size(); p++) { + ret[p / 8] |= bits[p] << (p % 8); + } + return ret; +} + +std::vector<bool> BytesToBits(const std::vector<unsigned char>& bytes) +{ + std::vector<bool> ret(bytes.size() * 8); + for (unsigned int p = 0; p < ret.size(); p++) { + ret[p] = (bytes[p / 8] & (1 << (p % 8))) != 0; + } + return ret; +} + CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter* filter, const std::set<uint256>* txids) { header = block.GetBlockHeader(); diff --git a/src/merkleblock.h b/src/merkleblock.h index e641c8aa94..b2d2828784 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -13,6 +13,10 @@ #include <vector> +// Helper functions for serialization. +std::vector<unsigned char> BitsToBytes(const std::vector<bool>& bits); +std::vector<bool> BytesToBits(const std::vector<unsigned char>& bytes); + /** Data structure that represents a partial merkle tree. * * It represents a subset of the txid's of a known block, in a way that @@ -81,27 +85,14 @@ protected: public: - /** serialization implementation */ - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nTransactions); - READWRITE(vHash); - std::vector<unsigned char> vBytes; - if (ser_action.ForRead()) { - READWRITE(vBytes); - CPartialMerkleTree &us = *(const_cast<CPartialMerkleTree*>(this)); - us.vBits.resize(vBytes.size() * 8); - for (unsigned int p = 0; p < us.vBits.size(); p++) - us.vBits[p] = (vBytes[p / 8] & (1 << (p % 8))) != 0; - us.fBad = false; - } else { - vBytes.resize((vBits.size()+7)/8); - for (unsigned int p = 0; p < vBits.size(); p++) - vBytes[p / 8] |= vBits[p] << (p % 8); - READWRITE(vBytes); - } + SERIALIZE_METHODS(CPartialMerkleTree, obj) + { + READWRITE(obj.nTransactions, obj.vHash); + std::vector<unsigned char> bytes; + SER_WRITE(obj, bytes = BitsToBytes(obj.vBits)); + READWRITE(bytes); + SER_READ(obj, obj.vBits = BytesToBits(bytes)); + SER_READ(obj, obj.fBad = false); } /** Construct a partial merkle tree from a list of transaction ids, and a mask that selects a subset of them */ @@ -157,13 +148,7 @@ public: CMerkleBlock() {} - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(header); - READWRITE(txn); - } + SERIALIZE_METHODS(CMerkleBlock, obj) { READWRITE(obj.header, obj.txn); } private: // Combined constructor to consolidate code diff --git a/src/net.cpp b/src/net.cpp index dcc613ba88..f3b7e8dab1 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -16,6 +16,7 @@ #include <crypto/sha256.h> #include <netbase.h> #include <net_permissions.h> +#include <protocol.h> #include <random.h> #include <scheduler.h> #include <ui_interface.h> @@ -51,6 +52,19 @@ static constexpr std::chrono::minutes DUMP_PEERS_INTERVAL{15}; /** Number of DNS seeds to query when the number of connections is low. */ static constexpr int DNSSEEDS_TO_QUERY_AT_ONCE = 3; +/** How long to delay before querying DNS seeds + * + * If we have more than THRESHOLD entries in addrman, then it's likely + * that we got those addresses from having previously connected to the P2P + * network, and that we'll be able to successfully reconnect to the P2P + * network via contacting one of them. So if that's the case, spend a + * little longer trying to connect to known peers before querying the + * DNS seeds. + */ +static constexpr std::chrono::seconds DNSSEEDS_DELAY_FEW_PEERS{11}; +static constexpr std::chrono::minutes DNSSEEDS_DELAY_MANY_PEERS{5}; +static constexpr int DNSSEEDS_DELAY_PEER_THRESHOLD = 1000; // "many" vs "few" peers + // We add a random period time (0 to 1 seconds) to feeler connections to prevent synchronization. #define FEELER_SLEEP_WINDOW 1 @@ -631,14 +645,14 @@ int CNode::GetSendVersion() const int V1TransportDeserializer::readHeader(const char *pch, unsigned int nBytes) { // copy data to temporary parsing buffer - unsigned int nRemaining = 24 - nHdrPos; + unsigned int nRemaining = CMessageHeader::HEADER_SIZE - nHdrPos; unsigned int nCopy = std::min(nRemaining, nBytes); memcpy(&hdrbuf[nHdrPos], pch, nCopy); nHdrPos += nCopy; // if header incomplete, exit - if (nHdrPos < 24) + if (nHdrPos < CMessageHeader::HEADER_SIZE) return nCopy; // deserialize to CMessageHeader @@ -1454,7 +1468,7 @@ void CConnman::ThreadSocketHandler() void CConnman::WakeMessageHandler() { { - std::lock_guard<std::mutex> lock(mutexMsgProc); + LOCK(mutexMsgProc); fMsgProcWake = true; } condMsgProc.notify_one(); @@ -1586,31 +1600,72 @@ void CConnman::ThreadDNSAddressSeed() if (gArgs.GetBoolArg("-forcednsseed", DEFAULT_FORCEDNSSEED)) { // When -forcednsseed is provided, query all. seeds_right_now = seeds.size(); + } else if (addrman.size() == 0) { + // If we have no known peers, query all. + // This will occur on the first run, or if peers.dat has been + // deleted. + seeds_right_now = seeds.size(); } + // goal: only query DNS seed if address need is acute + // * If we have a reasonable number of peers in addrman, spend + // some time trying them first. This improves user privacy by + // creating fewer identifying DNS requests, reduces trust by + // giving seeds less influence on the network topology, and + // reduces traffic to the seeds. + // * When querying DNS seeds query a few at once, this ensures + // that we don't give DNS seeds the ability to eclipse nodes + // that query them. + // * If we continue having problems, eventually query all the + // DNS seeds, and if that fails too, also try the fixed seeds. + // (done in ThreadOpenConnections) + const std::chrono::seconds seeds_wait_time = (addrman.size() >= DNSSEEDS_DELAY_PEER_THRESHOLD ? DNSSEEDS_DELAY_MANY_PEERS : DNSSEEDS_DELAY_FEW_PEERS); + for (const std::string& seed : seeds) { - // goal: only query DNS seed if address need is acute - // Avoiding DNS seeds when we don't need them improves user privacy by - // creating fewer identifying DNS requests, reduces trust by giving seeds - // less influence on the network topology, and reduces traffic to the seeds. - if (addrman.size() > 0 && seeds_right_now == 0) { - if (!interruptNet.sleep_for(std::chrono::seconds(11))) return; + if (seeds_right_now == 0) { + seeds_right_now += DNSSEEDS_TO_QUERY_AT_ONCE; - LOCK(cs_vNodes); - int nRelevant = 0; - for (const CNode* pnode : vNodes) { - nRelevant += pnode->fSuccessfullyConnected && !pnode->fFeeler && !pnode->fOneShot && !pnode->m_manual_connection && !pnode->fInbound; - } - if (nRelevant >= 2) { - LogPrintf("P2P peers available. Skipped DNS seeding.\n"); - return; + if (addrman.size() > 0) { + LogPrintf("Waiting %d seconds before querying DNS seeds.\n", seeds_wait_time.count()); + std::chrono::seconds to_wait = seeds_wait_time; + while (to_wait.count() > 0) { + // if sleeping for the MANY_PEERS interval, wake up + // early to see if we have enough peers and can stop + // this thread entirely freeing up its resources + std::chrono::seconds w = std::min(DNSSEEDS_DELAY_FEW_PEERS, to_wait); + if (!interruptNet.sleep_for(w)) return; + to_wait -= w; + + int nRelevant = 0; + { + LOCK(cs_vNodes); + for (const CNode* pnode : vNodes) { + nRelevant += pnode->fSuccessfullyConnected && !pnode->fFeeler && !pnode->fOneShot && !pnode->m_manual_connection && !pnode->fInbound; + } + } + if (nRelevant >= 2) { + if (found > 0) { + LogPrintf("%d addresses found from DNS seeds\n", found); + LogPrintf("P2P peers available. Finished DNS seeding.\n"); + } else { + LogPrintf("P2P peers available. Skipped DNS seeding.\n"); + } + return; + } + } } - seeds_right_now += DNSSEEDS_TO_QUERY_AT_ONCE; } - if (interruptNet) { - return; + if (interruptNet) return; + + // hold off on querying seeds if P2P network deactivated + if (!fNetworkActive) { + LogPrintf("Waiting for network to be reactivated before querying DNS seeds.\n"); + do { + if (!interruptNet.sleep_for(std::chrono::seconds{1})) return; + } while (!fNetworkActive); } + LogPrintf("Loading addresses from DNS seed %s\n", seed); if (HaveNameProxy()) { AddOneShot(seed); @@ -1754,6 +1809,9 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) return; // Add seed nodes if DNS seeds are all down (an infrastructure attack?). + // Note that we only do this if we started with an empty peers.dat, + // (in which case we will query DNS seeds immediately) *and* the DNS + // seeds have not returned any results. if (addrman.size() == 0 && (GetTime() - nStart > 60)) { static bool done = false; if (!done) { @@ -2057,7 +2115,7 @@ void CConnman::ThreadMessageHandler() WAIT_LOCK(mutexMsgProc, lock); if (!fMoreWork) { - condMsgProc.wait_until(lock, std::chrono::steady_clock::now() + std::chrono::milliseconds(100), [this] { return fMsgProcWake; }); + condMsgProc.wait_until(lock, std::chrono::steady_clock::now() + std::chrono::milliseconds(100), [this]() EXCLUSIVE_LOCKS_REQUIRED(mutexMsgProc) { return fMsgProcWake; }); } fMsgProcWake = false; } @@ -2068,9 +2126,8 @@ void CConnman::ThreadMessageHandler() -bool CConnman::BindListenPort(const CService& addrBind, std::string& strError, NetPermissionFlags permissions) +bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, NetPermissionFlags permissions) { - strError = ""; int nOne = 1; // Create socket for listening for incoming connections @@ -2078,16 +2135,16 @@ bool CConnman::BindListenPort(const CService& addrBind, std::string& strError, N socklen_t len = sizeof(sockaddr); if (!addrBind.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { - strError = strprintf("Error: Bind address family for %s not supported", addrBind.ToString()); - LogPrintf("%s\n", strError); + strError = strprintf(Untranslated("Error: Bind address family for %s not supported"), addrBind.ToString()); + LogPrintf("%s\n", strError.original); return false; } SOCKET hListenSocket = CreateSocket(addrBind); if (hListenSocket == INVALID_SOCKET) { - strError = strprintf("Error: Couldn't open socket for incoming connections (socket returned error %s)", NetworkErrorString(WSAGetLastError())); - LogPrintf("%s\n", strError); + strError = strprintf(Untranslated("Error: Couldn't open socket for incoming connections (socket returned error %s)"), NetworkErrorString(WSAGetLastError())); + LogPrintf("%s\n", strError.original); return false; } @@ -2111,10 +2168,10 @@ bool CConnman::BindListenPort(const CService& addrBind, std::string& strError, N { int nErr = WSAGetLastError(); if (nErr == WSAEADDRINUSE) - strError = strprintf(_("Unable to bind to %s on this computer. %s is probably already running.").translated, addrBind.ToString(), PACKAGE_NAME); + strError = strprintf(_("Unable to bind to %s on this computer. %s is probably already running."), addrBind.ToString(), PACKAGE_NAME); else - strError = strprintf(_("Unable to bind to %s on this computer (bind returned error %s)").translated, addrBind.ToString(), NetworkErrorString(nErr)); - LogPrintf("%s\n", strError); + strError = strprintf(_("Unable to bind to %s on this computer (bind returned error %s)"), addrBind.ToString(), NetworkErrorString(nErr)); + LogPrintf("%s\n", strError.original); CloseSocket(hListenSocket); return false; } @@ -2123,8 +2180,8 @@ bool CConnman::BindListenPort(const CService& addrBind, std::string& strError, N // Listen for incoming connections if (listen(hListenSocket, SOMAXCONN) == SOCKET_ERROR) { - strError = strprintf(_("Error: Listening for incoming connections failed (listen returned error %s)").translated, NetworkErrorString(WSAGetLastError())); - LogPrintf("%s\n", strError); + strError = strprintf(_("Error: Listening for incoming connections failed (listen returned error %s)"), NetworkErrorString(WSAGetLastError())); + LogPrintf("%s\n", strError.original); CloseSocket(hListenSocket); return false; } @@ -2218,7 +2275,7 @@ NodeId CConnman::GetNewNodeId() bool CConnman::Bind(const CService &addr, unsigned int flags, NetPermissionFlags permissions) { if (!(flags & BF_EXPLICIT) && !IsReachable(addr)) return false; - std::string strError; + bilingual_str strError; if (!BindListenPort(addr, strError, permissions)) { if ((flags & BF_REPORT_ERROR) && clientInterface) { clientInterface->ThreadSafeMessageBox(strError, "", CClientUIInterface::MSG_ERROR); @@ -2265,7 +2322,7 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) if (fListen && !InitBinds(connOptions.vBinds, connOptions.vWhiteBinds)) { if (clientInterface) { clientInterface->ThreadSafeMessageBox( - _("Failed to listen on any port. Use -listen=0 if you want this.").translated, + _("Failed to listen on any port. Use -listen=0 if you want this."), "", CClientUIInterface::MSG_ERROR); } return false; @@ -2331,7 +2388,7 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) if (connOptions.m_use_addrman_outgoing && !connOptions.m_specified_outgoing.empty()) { if (clientInterface) { clientInterface->ThreadSafeMessageBox( - _("Cannot provide specific connections and have addrman find outgoing connections at the same.").translated, + _("Cannot provide specific connections and have addrman find outgoing connections at the same."), "", CClientUIInterface::MSG_ERROR); } return false; @@ -2366,7 +2423,7 @@ static CNetCleanup instance_of_cnetcleanup; void CConnman::Interrupt() { { - std::lock_guard<std::mutex> lock(mutexMsgProc); + LOCK(mutexMsgProc); flagInterruptMsgProc = true; } condMsgProc.notify_all(); @@ -2582,7 +2639,7 @@ void CConnman::RecordBytesSent(uint64_t bytes) nMaxOutboundTotalBytesSentInCycle = 0; } - // TODO, exclude whitebind peers + // TODO, exclude peers with noban permission nMaxOutboundTotalBytesSentInCycle += bytes; } @@ -21,8 +21,8 @@ #include <random.h> #include <streams.h> #include <sync.h> -#include <uint256.h> #include <threadinterrupt.h> +#include <uint256.h> #include <atomic> #include <deque> @@ -39,6 +39,7 @@ class CScheduler; class CNode; class BanMan; +struct bilingual_str; /** Default for -whitelistrelay. */ static const bool DEFAULT_WHITELISTRELAY = true; @@ -61,6 +62,8 @@ static const int MAX_OUTBOUND_FULL_RELAY_CONNECTIONS = 8; static const int MAX_ADDNODE_CONNECTIONS = 8; /** Maximum number of block-relay-only outgoing connections */ static const int MAX_BLOCKS_ONLY_CONNECTIONS = 2; +/** Maximum number of feeler connections */ +static const int MAX_FEELER_CONNECTIONS = 1; /** -listen default */ static const bool DEFAULT_LISTEN = true; /** -upnp default */ @@ -334,7 +337,7 @@ private: NetPermissionFlags m_permissions; }; - bool BindListenPort(const CService& bindAddr, std::string& strError, NetPermissionFlags permissions); + bool BindListenPort(const CService& bindAddr, bilingual_str& strError, NetPermissionFlags permissions); bool Bind(const CService& addr, unsigned int flags, NetPermissionFlags permissions); bool InitBinds(const std::vector<CService>& binds, const std::vector<NetWhitebindPermissions>& whiteBinds); void ThreadOpenAddedConnections(); @@ -451,7 +454,7 @@ private: const uint64_t nSeed0, nSeed1; /** flag for waking the message processor. */ - bool fMsgProcWake; + bool fMsgProcWake GUARDED_BY(mutexMsgProc); std::condition_variable condMsgProc; Mutex mutexMsgProc; diff --git a/src/net_permissions.cpp b/src/net_permissions.cpp index 22fa5ee73b..d76a512a6c 100644 --- a/src/net_permissions.cpp +++ b/src/net_permissions.cpp @@ -8,6 +8,14 @@ #include <util/system.h> #include <util/translation.h> +const std::vector<std::string> NET_PERMISSIONS_DOC{ + "bloomfilter (allow requesting BIP37 filtered blocks and transactions)", + "noban (do not ban for misbehavior)", + "forcerelay (relay transactions that are already in the mempool; implies relay)", + "relay (relay even in -blocksonly mode)", + "mempool (allow requesting BIP35 mempool contents)", +}; + // The parse the following format "perm1,perm2@xxxxxx" bool TryParsePermissionFlags(const std::string str, NetPermissionFlags& output, size_t& readen, std::string& error) { diff --git a/src/net_permissions.h b/src/net_permissions.h index 962a2159fc..d35c5ee0cd 100644 --- a/src/net_permissions.h +++ b/src/net_permissions.h @@ -2,12 +2,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <netaddress.h> + #include <string> #include <vector> -#include <netaddress.h> #ifndef BITCOIN_NET_PERMISSIONS_H #define BITCOIN_NET_PERMISSIONS_H + +extern const std::vector<std::string> NET_PERMISSIONS_DOC; + enum NetPermissionFlags { PF_NONE = 0, @@ -27,6 +31,7 @@ enum NetPermissionFlags PF_ISIMPLICIT = (1U << 31), PF_ALL = PF_BLOOMFILTER | PF_FORCERELAY | PF_RELAY | PF_NOBAN | PF_MEMPOOL, }; + class NetPermissions { public: @@ -45,6 +50,7 @@ public: flags = static_cast<NetPermissionFlags>(flags & ~f); } }; + class NetWhitebindPermissions : public NetPermissions { public: diff --git a/src/net_processing.cpp b/src/net_processing.cpp index a4d4953281..d48745aef2 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -8,9 +8,11 @@ #include <addrman.h> #include <banman.h> #include <blockencodings.h> +#include <blockfilter.h> #include <chainparams.h> #include <consensus/validation.h> #include <hash.h> +#include <index/blockfilterindex.h> #include <validation.h> #include <merkleblock.h> #include <netmessagemaker.h> @@ -127,6 +129,10 @@ static constexpr unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_ static constexpr unsigned int AVG_FEEFILTER_BROADCAST_INTERVAL = 10 * 60; /** Maximum feefilter broadcast delay after significant change. */ static constexpr unsigned int MAX_FEEFILTER_CHANGE_DELAY = 5 * 60; +/** Maximum number of compact filters that may be requested with one getcfilters. See BIP 157. */ +static constexpr uint32_t MAX_GETCFILTERS_SIZE = 1000; +/** Maximum number of cf hashes that may be requested with one getcfheaders. See BIP 157. */ +static constexpr uint32_t MAX_GETCFHEADERS_SIZE = 2000; struct COrphanTx { // When modifying, adapt the copy of this definition in tests/DoS_tests. @@ -435,32 +441,32 @@ static CNodeState *State(NodeId pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return &it->second; } -static void UpdatePreferredDownload(CNode* node, CNodeState* state) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static void UpdatePreferredDownload(const CNode& node, CNodeState* state) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { nPreferredDownload -= state->fPreferredDownload; // Whether this node should be marked as a preferred download node. - state->fPreferredDownload = (!node->fInbound || node->HasPermission(PF_NOBAN)) && !node->fOneShot && !node->fClient; + state->fPreferredDownload = (!node.fInbound || node.HasPermission(PF_NOBAN)) && !node.fOneShot && !node.fClient; nPreferredDownload += state->fPreferredDownload; } -static void PushNodeVersion(CNode *pnode, CConnman* connman, int64_t nTime) +static void PushNodeVersion(CNode& pnode, CConnman* connman, int64_t nTime) { // Note that pnode->GetLocalServices() is a reflection of the local // services we were offering when the CNode object was created for this // peer. - ServiceFlags nLocalNodeServices = pnode->GetLocalServices(); - uint64_t nonce = pnode->GetLocalNonce(); - int nNodeStartingHeight = pnode->GetMyStartingHeight(); - NodeId nodeid = pnode->GetId(); - CAddress addr = pnode->addr; + ServiceFlags nLocalNodeServices = pnode.GetLocalServices(); + uint64_t nonce = pnode.GetLocalNonce(); + int nNodeStartingHeight = pnode.GetMyStartingHeight(); + NodeId nodeid = pnode.GetId(); + CAddress addr = pnode.addr; CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService(), addr.nServices)); CAddress addrMe = CAddress(CService(), nLocalNodeServices); - connman->PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, - nonce, strSubVersion, nNodeStartingHeight, ::g_relay_txes && pnode->m_tx_relay != nullptr)); + connman->PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, + nonce, strSubVersion, nNodeStartingHeight, ::g_relay_txes && pnode.m_tx_relay != nullptr)); if (fLogIPs) { LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); @@ -793,9 +799,9 @@ void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) // Returns true for outbound peers, excluding manual connections, feelers, and // one-shots. -static bool IsOutboundDisconnectionCandidate(const CNode *node) +static bool IsOutboundDisconnectionCandidate(const CNode& node) { - return !(node->fInbound || node->m_manual_connection || node->fFeeler || node->fOneShot); + return !(node.fInbound || node.m_manual_connection || node.fFeeler || node.fOneShot); } void PeerLogicValidation::InitializeNode(CNode *pnode) { @@ -807,7 +813,7 @@ void PeerLogicValidation::InitializeNode(CNode *pnode) { mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName), pnode->fInbound, pnode->m_manual_connection)); } if(!pnode->fInbound) - PushNodeVersion(pnode, connman, GetTime()); + PushNodeVersion(*pnode, connman, GetTime()); } void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const @@ -815,10 +821,16 @@ void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const std::set<uint256> unbroadcast_txids = m_mempool.GetUnbroadcastTxs(); for (const uint256& txid : unbroadcast_txids) { - RelayTransaction(txid, *connman); + // Sanity check: all unbroadcast txns should exist in the mempool + if (m_mempool.exists(txid)) { + RelayTransaction(txid, *connman); + } else { + m_mempool.RemoveUnbroadcastTx(txid, true); + } } - // schedule next run for 10-15 minutes in the future + // Schedule next run for 10-15 minutes in the future. + // We add randomness on every cycle to avoid the possibility of P2P fingerprinting. const std::chrono::milliseconds delta = std::chrono::minutes{10} + GetRandMillis(std::chrono::minutes{5}); scheduler.scheduleFromNow([&] { ReattemptInitialBroadcast(scheduler); }, delta); } @@ -1146,9 +1158,10 @@ static bool BlockRequestAllowed(const CBlockIndex* pindex, const Consensus::Para (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, consensusParams) < STALE_RELAY_AGE_LIMIT); } -PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CScheduler& scheduler, CTxMemPool& pool) +PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool) : connman(connmanIn), m_banman(banman), + m_chainman(chainman), m_mempool(pool), m_stale_tip_check_time(0) { @@ -1449,7 +1462,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& connman.ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); } -void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, const CInv& inv, CConnman* connman) +void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, const CInv& inv, CConnman* connman) { bool send = false; std::shared_ptr<const CBlock> a_recent_block; @@ -1491,28 +1504,30 @@ void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, c if (pindex) { send = BlockRequestAllowed(pindex, consensusParams); if (!send) { - LogPrint(BCLog::NET, "%s: ignoring request from peer=%i for old block that isn't in the main chain\n", __func__, pfrom->GetId()); + LogPrint(BCLog::NET, "%s: ignoring request from peer=%i for old block that isn't in the main chain\n", __func__, pfrom.GetId()); } } - const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); + const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); // disconnect node in case we have reached the outbound limit for serving historical blocks - // never disconnect whitelisted nodes - if (send && connman->OutboundTargetReached(true) && ( ((pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - pindex->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.type == MSG_FILTERED_BLOCK) && !pfrom->HasPermission(PF_NOBAN)) - { - LogPrint(BCLog::NET, "historical block serving limit reached, disconnect peer=%d\n", pfrom->GetId()); + if (send && + connman->OutboundTargetReached(true) && + (((pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - pindex->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.type == MSG_FILTERED_BLOCK) && + !pfrom.HasPermission(PF_NOBAN) // never disconnect nodes with the noban permission + ) { + LogPrint(BCLog::NET, "historical block serving limit reached, disconnect peer=%d\n", pfrom.GetId()); //disconnect node - pfrom->fDisconnect = true; + pfrom.fDisconnect = true; send = false; } // Avoid leaking prune-height by never sending blocks below the NODE_NETWORK_LIMITED threshold - if (send && !pfrom->HasPermission(PF_NOBAN) && ( - (((pfrom->GetLocalServices() & NODE_NETWORK_LIMITED) == NODE_NETWORK_LIMITED) && ((pfrom->GetLocalServices() & NODE_NETWORK) != NODE_NETWORK) && (::ChainActive().Tip()->nHeight - pindex->nHeight > (int)NODE_NETWORK_LIMITED_MIN_BLOCKS + 2 /* add two blocks buffer extension for possible races */) ) + if (send && !pfrom.HasPermission(PF_NOBAN) && ( + (((pfrom.GetLocalServices() & NODE_NETWORK_LIMITED) == NODE_NETWORK_LIMITED) && ((pfrom.GetLocalServices() & NODE_NETWORK) != NODE_NETWORK) && (::ChainActive().Tip()->nHeight - pindex->nHeight > (int)NODE_NETWORK_LIMITED_MIN_BLOCKS + 2 /* add two blocks buffer extension for possible races */) ) )) { - LogPrint(BCLog::NET, "Ignore block request below NODE_NETWORK_LIMITED threshold from peer=%d\n", pfrom->GetId()); + LogPrint(BCLog::NET, "Ignore block request below NODE_NETWORK_LIMITED threshold from peer=%d\n", pfrom.GetId()); //disconnect node and prevent it from stalling (would otherwise wait for the missing block) - pfrom->fDisconnect = true; + pfrom.fDisconnect = true; send = false; } // Pruned nodes may have deleted the block, so check whether @@ -1529,7 +1544,7 @@ void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, c if (!ReadRawBlockFromDisk(block_data, pindex, chainparams.MessageStart())) { assert(!"cannot load block from disk"); } - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, MakeSpan(block_data))); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, MakeSpan(block_data))); // Don't set pblock as we've sent the block } else { // Send block from disk @@ -1540,22 +1555,22 @@ void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, c } if (pblock) { if (inv.type == MSG_BLOCK) - connman->PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); + connman->PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_WITNESS_BLOCK) - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_FILTERED_BLOCK) { bool sendMerkleBlock = false; CMerkleBlock merkleBlock; - if (pfrom->m_tx_relay != nullptr) { - LOCK(pfrom->m_tx_relay->cs_filter); - if (pfrom->m_tx_relay->pfilter) { + if (pfrom.m_tx_relay != nullptr) { + LOCK(pfrom.m_tx_relay->cs_filter); + if (pfrom.m_tx_relay->pfilter) { sendMerkleBlock = true; - merkleBlock = CMerkleBlock(*pblock, *pfrom->m_tx_relay->pfilter); + merkleBlock = CMerkleBlock(*pblock, *pfrom.m_tx_relay->pfilter); } } if (sendMerkleBlock) { - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see // This avoids hurting performance by pointlessly requiring a round-trip // Note that there is currently no way for a node to request any single transactions we didn't send here - @@ -1564,7 +1579,7 @@ void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, c // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; for (PairType& pair : merkleBlock.vMatchedTxn) - connman->PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); + connman->PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); } // else // no response @@ -1575,110 +1590,118 @@ void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, c // they won't have a useful mempool to match against a compact block, // and we don't feel like constructing the object for them, so // instead we respond with the full, non-compact block. - bool fPeerWantsWitness = State(pfrom->GetId())->fWantsCmpctWitness; + bool fPeerWantsWitness = State(pfrom.GetId())->fWantsCmpctWitness; int nSendFlags = fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; if (CanDirectFetch(consensusParams) && pindex->nHeight >= ::ChainActive().Height() - MAX_CMPCTBLOCK_DEPTH) { if ((fPeerWantsWitness || !fWitnessesPresentInARecentCompactBlock) && a_recent_compact_block && a_recent_compact_block->header.GetHash() == pindex->GetBlockHash()) { - connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); + connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); } else { CBlockHeaderAndShortTxIDs cmpctblock(*pblock, fPeerWantsWitness); - connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); } } else { - connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, *pblock)); + connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, *pblock)); } } } // Trigger the peer node to send a getblocks request for the next batch of inventory - if (inv.hash == pfrom->hashContinue) + if (inv.hash == pfrom.hashContinue) { // Bypass PushInventory, this must send even if redundant, // and we want it right after the last block so they don't // wait for other stuff first. std::vector<CInv> vInv; vInv.push_back(CInv(MSG_BLOCK, ::ChainActive().Tip()->GetBlockHash())); - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::INV, vInv)); - pfrom->hashContinue.SetNull(); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::INV, vInv)); + pfrom.hashContinue.SetNull(); } } } -void static ProcessGetData(CNode* pfrom, const CChainParams& chainparams, CConnman* connman, CTxMemPool& mempool, const std::atomic<bool>& interruptMsgProc) LOCKS_EXCLUDED(cs_main) +//! Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). +CTransactionRef static FindTxForGetData(CNode& peer, const uint256& txid, const std::chrono::seconds mempool_req, const std::chrono::seconds longlived_mempool_time) LOCKS_EXCLUDED(cs_main) { - AssertLockNotHeld(cs_main); + // Check if the requested transaction is so recent that we're just + // about to announce it to the peer; if so, they certainly shouldn't + // know we already have it. + { + LOCK(peer.m_tx_relay->cs_tx_inventory); + if (peer.m_tx_relay->setInventoryTxToSend.count(txid)) return {}; + } - std::deque<CInv>::iterator it = pfrom->vRecvGetData.begin(); - std::vector<CInv> vNotFound; - const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); + { + LOCK(cs_main); + // Look up transaction in relay pool + auto mi = mapRelay.find(txid); + if (mi != mapRelay.end()) return mi->second; + } - // Note that if we receive a getdata for a MSG_TX or MSG_WITNESS_TX from a - // block-relay-only outbound peer, we will stop processing further getdata - // messages from this peer (likely resulting in our peer eventually - // disconnecting us). - if (pfrom->m_tx_relay != nullptr) { - // mempool entries added before this time have likely expired from mapRelay - const std::chrono::seconds longlived_mempool_time = GetTime<std::chrono::seconds>() - RELAY_TX_CACHE_TIME; - const std::chrono::seconds mempool_req = pfrom->m_tx_relay->m_last_mempool_req.load(); + auto txinfo = mempool.info(txid); + if (txinfo.tx) { + // To protect privacy, do not answer getdata using the mempool when + // that TX couldn't have been INVed in reply to a MEMPOOL request, + // or when it's too recent to have expired from mapRelay. + if ((mempool_req.count() && txinfo.m_time <= mempool_req) || txinfo.m_time <= longlived_mempool_time) { + return txinfo.tx; + } + } - LOCK(cs_main); + return {}; +} - while (it != pfrom->vRecvGetData.end() && (it->type == MSG_TX || it->type == MSG_WITNESS_TX)) { - if (interruptMsgProc) - return; - // Don't bother if send buffer is too full to respond anyway - if (pfrom->fPauseSend) - break; +void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnman* connman, CTxMemPool& mempool, const std::atomic<bool>& interruptMsgProc) LOCKS_EXCLUDED(cs_main) +{ + AssertLockNotHeld(cs_main); - const CInv &inv = *it; - it++; + std::deque<CInv>::iterator it = pfrom.vRecvGetData.begin(); + std::vector<CInv> vNotFound; + const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); + + // mempool entries added before this time have likely expired from mapRelay + const std::chrono::seconds longlived_mempool_time = GetTime<std::chrono::seconds>() - RELAY_TX_CACHE_TIME; + // Get last mempool request time + const std::chrono::seconds mempool_req = pfrom.m_tx_relay != nullptr ? pfrom.m_tx_relay->m_last_mempool_req.load() + : std::chrono::seconds::min(); + + // Process as many TX items from the front of the getdata queue as + // possible, since they're common and it's efficient to batch process + // them. + while (it != pfrom.vRecvGetData.end() && (it->type == MSG_TX || it->type == MSG_WITNESS_TX)) { + if (interruptMsgProc) return; + // The send buffer provides backpressure. If there's no space in + // the buffer, pause processing until the next call. + if (pfrom.fPauseSend) break; + + const CInv &inv = *it++; + + if (pfrom.m_tx_relay == nullptr) { + // Ignore GETDATA requests for transactions from blocks-only peers. + continue; + } - // Send stream from relay memory - bool push = false; - auto mi = mapRelay.find(inv.hash); + CTransactionRef tx = FindTxForGetData(pfrom, inv.hash, mempool_req, longlived_mempool_time); + if (tx) { int nSendFlags = (inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); - if (mi != mapRelay.end()) { - connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *mi->second)); - push = true; - } else { - auto txinfo = mempool.info(inv.hash); - // To protect privacy, do not answer getdata using the mempool when - // that TX couldn't have been INVed in reply to a MEMPOOL request, - // or when it's too recent to have expired from mapRelay. - if (txinfo.tx && ( - (mempool_req.count() && txinfo.m_time <= mempool_req) - || (txinfo.m_time <= longlived_mempool_time))) - { - connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *txinfo.tx)); - push = true; - } - } - - if (push) { - // We interpret fulfilling a GETDATA for a transaction as a - // successful initial broadcast and remove it from our - // unbroadcast set. - mempool.RemoveUnbroadcastTx(inv.hash); - } else { - vNotFound.push_back(inv); - } + connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); + mempool.RemoveUnbroadcastTx(inv.hash); + } else { + vNotFound.push_back(inv); } - } // release cs_main + } - if (it != pfrom->vRecvGetData.end() && !pfrom->fPauseSend) { - const CInv &inv = *it; + // Only process one BLOCK item per call, since they're uncommon and can be + // expensive to process. + if (it != pfrom.vRecvGetData.end() && !pfrom.fPauseSend) { + const CInv &inv = *it++; if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) { - it++; ProcessGetBlockData(pfrom, chainparams, inv, connman); } + // else: If the first item on the queue is an unknown type, we erase it + // and continue processing the queue on the next call. } - // Unknown types in the GetData stay in vRecvGetData and block any future - // message from this peer, see vRecvGetData check in ProcessMessages(). - // Depending on future p2p changes, we might either drop unknown getdata on - // the floor or disconnect the peer. - - pfrom->vRecvGetData.erase(pfrom->vRecvGetData.begin(), it); + pfrom.vRecvGetData.erase(pfrom.vRecvGetData.begin(), it); if (!vNotFound.empty()) { // Let the peer know that we didn't find what it asked for, so it doesn't @@ -1695,37 +1718,37 @@ void static ProcessGetData(CNode* pfrom, const CChainParams& chainparams, CConnm // In normal operation, we often send NOTFOUND messages for parents of // transactions that we relay; if a peer is missing a parent, they may // assume we have them and request the parents from us. - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); } } -static uint32_t GetFetchFlags(CNode* pfrom) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { +static uint32_t GetFetchFlags(const CNode& pfrom) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { uint32_t nFetchFlags = 0; - if ((pfrom->GetLocalServices() & NODE_WITNESS) && State(pfrom->GetId())->fHaveWitness) { + if ((pfrom.GetLocalServices() & NODE_WITNESS) && State(pfrom.GetId())->fHaveWitness) { nFetchFlags |= MSG_WITNESS_FLAG; } return nFetchFlags; } -inline void static SendBlockTransactions(const CBlock& block, const BlockTransactionsRequest& req, CNode* pfrom, CConnman* connman) { +inline void static SendBlockTransactions(const CBlock& block, const BlockTransactionsRequest& req, CNode& pfrom, CConnman* connman) { BlockTransactions resp(req); for (size_t i = 0; i < req.indexes.size(); i++) { if (req.indexes[i] >= block.vtx.size()) { LOCK(cs_main); - Misbehaving(pfrom->GetId(), 100, strprintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->GetId())); + Misbehaving(pfrom.GetId(), 100, strprintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom.GetId())); return; } resp.txn[i] = block.vtx[req.indexes[i]]; } LOCK(cs_main); - const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); - int nSendFlags = State(pfrom->GetId())->fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; - connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); + const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); + int nSendFlags = State(pfrom.GetId())->fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; + connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); } -bool static ProcessHeadersMessage(CNode* pfrom, CConnman* connman, CTxMemPool& mempool, const std::vector<CBlockHeader>& headers, const CChainParams& chainparams, bool via_compact_block) +bool static ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateManager& chainman, CTxMemPool& mempool, const std::vector<CBlockHeader>& headers, const CChainParams& chainparams, bool via_compact_block) { - const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); + const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); size_t nCount = headers.size(); if (nCount == 0) { @@ -1737,7 +1760,7 @@ bool static ProcessHeadersMessage(CNode* pfrom, CConnman* connman, CTxMemPool& m const CBlockIndex *pindexLast = nullptr; { LOCK(cs_main); - CNodeState *nodestate = State(pfrom->GetId()); + CNodeState *nodestate = State(pfrom.GetId()); // If this looks like it could be a block announcement (nCount < // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that @@ -1749,19 +1772,19 @@ bool static ProcessHeadersMessage(CNode* pfrom, CConnman* connman, CTxMemPool& m // nUnconnectingHeaders gets reset back to 0. if (!LookupBlockIndex(headers[0].hashPrevBlock) && nCount < MAX_BLOCKS_TO_ANNOUNCE) { nodestate->nUnconnectingHeaders++; - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), pindexBestHeader->nHeight, - pfrom->GetId(), nodestate->nUnconnectingHeaders); + pfrom.GetId(), nodestate->nUnconnectingHeaders); // Set hashLastUnknownBlock for this peer, so that if we // eventually get the headers - even from a different peer - // we can use this peer to download. - UpdateBlockAvailability(pfrom->GetId(), headers.back().GetHash()); + UpdateBlockAvailability(pfrom.GetId(), headers.back().GetHash()); if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { - Misbehaving(pfrom->GetId(), 20); + Misbehaving(pfrom.GetId(), 20); } return true; } @@ -1769,7 +1792,7 @@ bool static ProcessHeadersMessage(CNode* pfrom, CConnman* connman, CTxMemPool& m uint256 hashLastBlock; for (const CBlockHeader& header : headers) { if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { - Misbehaving(pfrom->GetId(), 20, "non-continuous headers sequence"); + Misbehaving(pfrom.GetId(), 20, "non-continuous headers sequence"); return false; } hashLastBlock = header.GetHash(); @@ -1783,23 +1806,23 @@ bool static ProcessHeadersMessage(CNode* pfrom, CConnman* connman, CTxMemPool& m } BlockValidationState state; - if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { + if (!chainman.ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { if (state.IsInvalid()) { - MaybePunishNodeForBlock(pfrom->GetId(), state, via_compact_block, "invalid header received"); + MaybePunishNodeForBlock(pfrom.GetId(), state, via_compact_block, "invalid header received"); return false; } } { LOCK(cs_main); - CNodeState *nodestate = State(pfrom->GetId()); + CNodeState *nodestate = State(pfrom.GetId()); if (nodestate->nUnconnectingHeaders > 0) { - LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->GetId(), nodestate->nUnconnectingHeaders); + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom.GetId(), nodestate->nUnconnectingHeaders); } nodestate->nUnconnectingHeaders = 0; assert(pindexLast); - UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); + UpdateBlockAvailability(pfrom.GetId(), pindexLast->GetBlockHash()); // From here, pindexBestKnownBlock should be guaranteed to be non-null, // because it is set in UpdateBlockAvailability. Some nullptr checks @@ -1813,8 +1836,8 @@ bool static ProcessHeadersMessage(CNode* pfrom, CConnman* connman, CTxMemPool& m // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of ::ChainActive().Tip or pindexBestHeader, continue // from there instead. - LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexLast), uint256())); + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom.GetId(), pfrom.nStartingHeight); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexLast), uint256())); } bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); @@ -1827,7 +1850,7 @@ bool static ProcessHeadersMessage(CNode* pfrom, CConnman* connman, CTxMemPool& m while (pindexWalk && !::ChainActive().Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && !mapBlocksInFlight.count(pindexWalk->GetBlockHash()) && - (!IsWitnessEnabled(pindexWalk->pprev, chainparams.GetConsensus()) || State(pfrom->GetId())->fHaveWitness)) { + (!IsWitnessEnabled(pindexWalk->pprev, chainparams.GetConsensus()) || State(pfrom.GetId())->fHaveWitness)) { // We don't have this block, and it's not yet in flight. vToFetch.push_back(pindexWalk); } @@ -1851,9 +1874,9 @@ bool static ProcessHeadersMessage(CNode* pfrom, CConnman* connman, CTxMemPool& m } uint32_t nFetchFlags = GetFetchFlags(pfrom); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(mempool, pfrom->GetId(), pindex->GetBlockHash(), pindex); + MarkBlockAsInFlight(mempool, pfrom.GetId(), pindex->GetBlockHash(), pindex); LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom->GetId()); + pindex->GetBlockHash().ToString(), pfrom.GetId()); } if (vGetData.size() > 1) { LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", @@ -1864,7 +1887,7 @@ bool static ProcessHeadersMessage(CNode* pfrom, CConnman* connman, CTxMemPool& m // In any case, we want to download using a compact block, not a regular one vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); } - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); } } } @@ -1883,19 +1906,19 @@ bool static ProcessHeadersMessage(CNode* pfrom, CConnman* connman, CTxMemPool& m // nMinimumChainWork, even if a peer has a chain past our tip, // as an anti-DoS measure. if (IsOutboundDisconnectionCandidate(pfrom)) { - LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom->GetId()); - pfrom->fDisconnect = true; + LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom.GetId()); + pfrom.fDisconnect = true; } } } - if (!pfrom->fDisconnect && IsOutboundDisconnectionCandidate(pfrom) && nodestate->pindexBestKnownBlock != nullptr && pfrom->m_tx_relay != nullptr) { + if (!pfrom.fDisconnect && IsOutboundDisconnectionCandidate(pfrom) && nodestate->pindexBestKnownBlock != nullptr && pfrom.m_tx_relay != nullptr) { // If this is an outbound full-relay peer, check to see if we should protect // it from the bad/lagging chain logic. // Note that block-relay-only peers are already implicitly protected, so we // only consider setting m_protect for the full-relay peers. if (g_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= ::ChainActive().Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { - LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom->GetId()); + LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom.GetId()); nodestate->m_chain_sync.m_protect = true; ++g_outbound_peers_with_protect_from_disconnect; } @@ -1965,9 +1988,226 @@ void static ProcessOrphanTx(CConnman* connman, CTxMemPool& mempool, std::set<uin } } -bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) +/** + * Validation logic for compact filters request handling. + * + * May disconnect from the peer in the case of a bad request. + * + * @param[in] pfrom The peer that we received the request from + * @param[in] chain_params Chain parameters + * @param[in] filter_type The filter type the request is for. Must be basic filters. + * @param[in] start_height The start height for the request + * @param[in] stop_hash The stop_hash for the request + * @param[in] max_height_diff The maximum number of items permitted to request, as specified in BIP 157 + * @param[out] stop_index The CBlockIndex for the stop_hash block, if the request can be serviced. + * @param[out] filter_index The filter index, if the request can be serviced. + * @return True if the request can be serviced. + */ +static bool PrepareBlockFilterRequest(CNode& pfrom, const CChainParams& chain_params, + BlockFilterType filter_type, uint32_t start_height, + const uint256& stop_hash, uint32_t max_height_diff, + const CBlockIndex*& stop_index, + BlockFilterIndex*& filter_index) +{ + const bool supported_filter_type = + (filter_type == BlockFilterType::BASIC && + gArgs.GetBoolArg("-peerblockfilters", DEFAULT_PEERBLOCKFILTERS)); + if (!supported_filter_type) { + LogPrint(BCLog::NET, "peer %d requested unsupported block filter type: %d\n", + pfrom.GetId(), static_cast<uint8_t>(filter_type)); + pfrom.fDisconnect = true; + return false; + } + + { + LOCK(cs_main); + stop_index = LookupBlockIndex(stop_hash); + + // Check that the stop block exists and the peer would be allowed to fetch it. + if (!stop_index || !BlockRequestAllowed(stop_index, chain_params.GetConsensus())) { + LogPrint(BCLog::NET, "peer %d requested invalid block hash: %s\n", + pfrom.GetId(), stop_hash.ToString()); + pfrom.fDisconnect = true; + return false; + } + } + + uint32_t stop_height = stop_index->nHeight; + if (start_height > stop_height) { + LogPrint(BCLog::NET, "peer %d sent invalid getcfilters/getcfheaders with " /* Continued */ + "start height %d and stop height %d\n", + pfrom.GetId(), start_height, stop_height); + pfrom.fDisconnect = true; + return false; + } + if (stop_height - start_height >= max_height_diff) { + LogPrint(BCLog::NET, "peer %d requested too many cfilters/cfheaders: %d / %d\n", + pfrom.GetId(), stop_height - start_height + 1, max_height_diff); + pfrom.fDisconnect = true; + return false; + } + + filter_index = GetBlockFilterIndex(filter_type); + if (!filter_index) { + LogPrint(BCLog::NET, "Filter index for supported type %s not found\n", BlockFilterTypeName(filter_type)); + return false; + } + + return true; +} + +/** + * Handle a cfilters request. + * + * May disconnect from the peer in the case of a bad request. + * + * @param[in] pfrom The peer that we received the request from + * @param[in] vRecv The raw message received + * @param[in] chain_params Chain parameters + * @param[in] connman Pointer to the connection manager + */ +static void ProcessGetCFilters(CNode& pfrom, CDataStream& vRecv, const CChainParams& chain_params, + CConnman& connman) { - LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(msg_type), vRecv.size(), pfrom->GetId()); + uint8_t filter_type_ser; + uint32_t start_height; + uint256 stop_hash; + + vRecv >> filter_type_ser >> start_height >> stop_hash; + + const BlockFilterType filter_type = static_cast<BlockFilterType>(filter_type_ser); + + const CBlockIndex* stop_index; + BlockFilterIndex* filter_index; + if (!PrepareBlockFilterRequest(pfrom, chain_params, filter_type, start_height, stop_hash, + MAX_GETCFILTERS_SIZE, stop_index, filter_index)) { + return; + } + + std::vector<BlockFilter> filters; + + if (!filter_index->LookupFilterRange(start_height, stop_index, filters)) { + LogPrint(BCLog::NET, "Failed to find block filter in index: filter_type=%s, start_height=%d, stop_hash=%s\n", + BlockFilterTypeName(filter_type), start_height, stop_hash.ToString()); + return; + } + + for (const auto& filter : filters) { + CSerializedNetMsg msg = CNetMsgMaker(pfrom.GetSendVersion()) + .Make(NetMsgType::CFILTER, filter); + connman.PushMessage(&pfrom, std::move(msg)); + } +} + +/** + * Handle a cfheaders request. + * + * May disconnect from the peer in the case of a bad request. + * + * @param[in] pfrom The peer that we received the request from + * @param[in] vRecv The raw message received + * @param[in] chain_params Chain parameters + * @param[in] connman Pointer to the connection manager + */ +static void ProcessGetCFHeaders(CNode& pfrom, CDataStream& vRecv, const CChainParams& chain_params, + CConnman& connman) +{ + uint8_t filter_type_ser; + uint32_t start_height; + uint256 stop_hash; + + vRecv >> filter_type_ser >> start_height >> stop_hash; + + const BlockFilterType filter_type = static_cast<BlockFilterType>(filter_type_ser); + + const CBlockIndex* stop_index; + BlockFilterIndex* filter_index; + if (!PrepareBlockFilterRequest(pfrom, chain_params, filter_type, start_height, stop_hash, + MAX_GETCFHEADERS_SIZE, stop_index, filter_index)) { + return; + } + + uint256 prev_header; + if (start_height > 0) { + const CBlockIndex* const prev_block = + stop_index->GetAncestor(static_cast<int>(start_height - 1)); + if (!filter_index->LookupFilterHeader(prev_block, prev_header)) { + LogPrint(BCLog::NET, "Failed to find block filter header in index: filter_type=%s, block_hash=%s\n", + BlockFilterTypeName(filter_type), prev_block->GetBlockHash().ToString()); + return; + } + } + + std::vector<uint256> filter_hashes; + if (!filter_index->LookupFilterHashRange(start_height, stop_index, filter_hashes)) { + LogPrint(BCLog::NET, "Failed to find block filter hashes in index: filter_type=%s, start_height=%d, stop_hash=%s\n", + BlockFilterTypeName(filter_type), start_height, stop_hash.ToString()); + return; + } + + CSerializedNetMsg msg = CNetMsgMaker(pfrom.GetSendVersion()) + .Make(NetMsgType::CFHEADERS, + filter_type_ser, + stop_index->GetBlockHash(), + prev_header, + filter_hashes); + connman.PushMessage(&pfrom, std::move(msg)); +} + +/** + * Handle a getcfcheckpt request. + * + * May disconnect from the peer in the case of a bad request. + * + * @param[in] pfrom The peer that we received the request from + * @param[in] vRecv The raw message received + * @param[in] chain_params Chain parameters + * @param[in] connman Pointer to the connection manager + */ +static void ProcessGetCFCheckPt(CNode& pfrom, CDataStream& vRecv, const CChainParams& chain_params, + CConnman& connman) +{ + uint8_t filter_type_ser; + uint256 stop_hash; + + vRecv >> filter_type_ser >> stop_hash; + + const BlockFilterType filter_type = static_cast<BlockFilterType>(filter_type_ser); + + const CBlockIndex* stop_index; + BlockFilterIndex* filter_index; + if (!PrepareBlockFilterRequest(pfrom, chain_params, filter_type, /*start_height=*/0, stop_hash, + /*max_height_diff=*/std::numeric_limits<uint32_t>::max(), + stop_index, filter_index)) { + return; + } + + std::vector<uint256> headers(stop_index->nHeight / CFCHECKPT_INTERVAL); + + // Populate headers. + const CBlockIndex* block_index = stop_index; + for (int i = headers.size() - 1; i >= 0; i--) { + int height = (i + 1) * CFCHECKPT_INTERVAL; + block_index = block_index->GetAncestor(height); + + if (!filter_index->LookupFilterHeader(block_index, headers[i])) { + LogPrint(BCLog::NET, "Failed to find block filter header in index: filter_type=%s, block_hash=%s\n", + BlockFilterTypeName(filter_type), block_index->GetBlockHash().ToString()); + return; + } + } + + CSerializedNetMsg msg = CNetMsgMaker(pfrom.GetSendVersion()) + .Make(NetMsgType::CFCHECKPT, + filter_type_ser, + stop_index->GetBlockHash(), + headers); + connman.PushMessage(&pfrom, std::move(msg)); +} + +bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, ChainstateManager& chainman, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) +{ + LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(msg_type), vRecv.size(), pfrom.GetId()); if (gArgs.IsArgSet("-dropmessagestest") && GetRand(gArgs.GetArg("-dropmessagestest", 0)) == 0) { LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); @@ -1975,26 +2215,26 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec } - if (!(pfrom->GetLocalServices() & NODE_BLOOM) && + if (!(pfrom.GetLocalServices() & NODE_BLOOM) && (msg_type == NetMsgType::FILTERLOAD || msg_type == NetMsgType::FILTERADD)) { - if (pfrom->nVersion >= NO_BLOOM_VERSION) { + if (pfrom.nVersion >= NO_BLOOM_VERSION) { LOCK(cs_main); - Misbehaving(pfrom->GetId(), 100); + Misbehaving(pfrom.GetId(), 100); return false; } else { - pfrom->fDisconnect = true; + pfrom.fDisconnect = true; return false; } } if (msg_type == NetMsgType::VERSION) { // Each connection can only send one version message - if (pfrom->nVersion != 0) + if (pfrom.nVersion != 0) { LOCK(cs_main); - Misbehaving(pfrom->GetId(), 1); + Misbehaving(pfrom.GetId(), 1); return false; } @@ -2013,21 +2253,21 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec vRecv >> nVersion >> nServiceInt >> nTime >> addrMe; nSendVersion = std::min(nVersion, PROTOCOL_VERSION); nServices = ServiceFlags(nServiceInt); - if (!pfrom->fInbound) + if (!pfrom.fInbound) { - connman->SetServices(pfrom->addr, nServices); + connman->SetServices(pfrom.addr, nServices); } - if (!pfrom->fInbound && !pfrom->fFeeler && !pfrom->m_manual_connection && !HasAllDesirableServiceFlags(nServices)) + if (!pfrom.fInbound && !pfrom.fFeeler && !pfrom.m_manual_connection && !HasAllDesirableServiceFlags(nServices)) { - LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, GetDesirableServiceFlags(nServices)); - pfrom->fDisconnect = true; + LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom.GetId(), nServices, GetDesirableServiceFlags(nServices)); + pfrom.fDisconnect = true; return false; } if (nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version - LogPrint(BCLog::NET, "peer=%d using obsolete version %i; disconnecting\n", pfrom->GetId(), nVersion); - pfrom->fDisconnect = true; + LogPrint(BCLog::NET, "peer=%d using obsolete version %i; disconnecting\n", pfrom.GetId(), nVersion); + pfrom.fDisconnect = true; return false; } @@ -2044,145 +2284,145 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec if (!vRecv.empty()) vRecv >> fRelay; // Disconnect if we connected to ourself - if (pfrom->fInbound && !connman->CheckIncomingNonce(nNonce)) + if (pfrom.fInbound && !connman->CheckIncomingNonce(nNonce)) { - LogPrintf("connected to self at %s, disconnecting\n", pfrom->addr.ToString()); - pfrom->fDisconnect = true; + LogPrintf("connected to self at %s, disconnecting\n", pfrom.addr.ToString()); + pfrom.fDisconnect = true; return true; } - if (pfrom->fInbound && addrMe.IsRoutable()) + if (pfrom.fInbound && addrMe.IsRoutable()) { SeenLocal(addrMe); } // Be shy and don't send version until we hear - if (pfrom->fInbound) + if (pfrom.fInbound) PushNodeVersion(pfrom, connman, GetAdjustedTime()); - connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); + connman->PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); - pfrom->nServices = nServices; - pfrom->SetAddrLocal(addrMe); + pfrom.nServices = nServices; + pfrom.SetAddrLocal(addrMe); { - LOCK(pfrom->cs_SubVer); - pfrom->cleanSubVer = cleanSubVer; + LOCK(pfrom.cs_SubVer); + pfrom.cleanSubVer = cleanSubVer; } - pfrom->nStartingHeight = nStartingHeight; + pfrom.nStartingHeight = nStartingHeight; // set nodes not relaying blocks and tx and not serving (parts) of the historical blockchain as "clients" - pfrom->fClient = (!(nServices & NODE_NETWORK) && !(nServices & NODE_NETWORK_LIMITED)); + pfrom.fClient = (!(nServices & NODE_NETWORK) && !(nServices & NODE_NETWORK_LIMITED)); // set nodes not capable of serving the complete blockchain history as "limited nodes" - pfrom->m_limited_node = (!(nServices & NODE_NETWORK) && (nServices & NODE_NETWORK_LIMITED)); + pfrom.m_limited_node = (!(nServices & NODE_NETWORK) && (nServices & NODE_NETWORK_LIMITED)); - if (pfrom->m_tx_relay != nullptr) { - LOCK(pfrom->m_tx_relay->cs_filter); - pfrom->m_tx_relay->fRelayTxes = fRelay; // set to true after we get the first filter* message + if (pfrom.m_tx_relay != nullptr) { + LOCK(pfrom.m_tx_relay->cs_filter); + pfrom.m_tx_relay->fRelayTxes = fRelay; // set to true after we get the first filter* message } // Change version - pfrom->SetSendVersion(nSendVersion); - pfrom->nVersion = nVersion; + pfrom.SetSendVersion(nSendVersion); + pfrom.nVersion = nVersion; if((nServices & NODE_WITNESS)) { LOCK(cs_main); - State(pfrom->GetId())->fHaveWitness = true; + State(pfrom.GetId())->fHaveWitness = true; } // Potentially mark this peer as a preferred download peer. { LOCK(cs_main); - UpdatePreferredDownload(pfrom, State(pfrom->GetId())); + UpdatePreferredDownload(pfrom, State(pfrom.GetId())); } - if (!pfrom->fInbound && pfrom->IsAddrRelayPeer()) + if (!pfrom.fInbound && pfrom.IsAddrRelayPeer()) { // Advertise our address if (fListen && !::ChainstateActive().IsInitialBlockDownload()) { - CAddress addr = GetLocalAddress(&pfrom->addr, pfrom->GetLocalServices()); + CAddress addr = GetLocalAddress(&pfrom.addr, pfrom.GetLocalServices()); FastRandomContext insecure_rand; if (addr.IsRoutable()) { LogPrint(BCLog::NET, "ProcessMessages: advertising address %s\n", addr.ToString()); - pfrom->PushAddress(addr, insecure_rand); - } else if (IsPeerAddrLocalGood(pfrom)) { + pfrom.PushAddress(addr, insecure_rand); + } else if (IsPeerAddrLocalGood(&pfrom)) { addr.SetIP(addrMe); LogPrint(BCLog::NET, "ProcessMessages: advertising address %s\n", addr.ToString()); - pfrom->PushAddress(addr, insecure_rand); + pfrom.PushAddress(addr, insecure_rand); } } // Get recent addresses - if (pfrom->fOneShot || pfrom->nVersion >= CADDR_TIME_VERSION || connman->GetAddressCount() < 1000) + if (pfrom.fOneShot || pfrom.nVersion >= CADDR_TIME_VERSION || connman->GetAddressCount() < 1000) { - connman->PushMessage(pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); - pfrom->fGetAddr = true; + connman->PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); + pfrom.fGetAddr = true; } - connman->MarkAddressGood(pfrom->addr); + connman->MarkAddressGood(pfrom.addr); } std::string remoteAddr; if (fLogIPs) - remoteAddr = ", peeraddr=" + pfrom->addr.ToString(); + remoteAddr = ", peeraddr=" + pfrom.addr.ToString(); LogPrint(BCLog::NET, "receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", - cleanSubVer, pfrom->nVersion, - pfrom->nStartingHeight, addrMe.ToString(), pfrom->GetId(), + cleanSubVer, pfrom.nVersion, + pfrom.nStartingHeight, addrMe.ToString(), pfrom.GetId(), remoteAddr); int64_t nTimeOffset = nTime - GetTime(); - pfrom->nTimeOffset = nTimeOffset; - AddTimeData(pfrom->addr, nTimeOffset); + pfrom.nTimeOffset = nTimeOffset; + AddTimeData(pfrom.addr, nTimeOffset); // If the peer is old enough to have the old alert system, send it the final alert. - if (pfrom->nVersion <= 70012) { + if (pfrom.nVersion <= 70012) { CDataStream finalAlert(ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50"), SER_NETWORK, PROTOCOL_VERSION); - connman->PushMessage(pfrom, CNetMsgMaker(nSendVersion).Make("alert", finalAlert)); + connman->PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make("alert", finalAlert)); } // Feeler connections exist only to verify if address is online. - if (pfrom->fFeeler) { - assert(pfrom->fInbound == false); - pfrom->fDisconnect = true; + if (pfrom.fFeeler) { + assert(pfrom.fInbound == false); + pfrom.fDisconnect = true; } return true; } - if (pfrom->nVersion == 0) { + if (pfrom.nVersion == 0) { // Must have a version message before anything else LOCK(cs_main); - Misbehaving(pfrom->GetId(), 1); + Misbehaving(pfrom.GetId(), 1); return false; } // At this point, the outgoing message serialization version can't change. - const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); + const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); if (msg_type == NetMsgType::VERACK) { - pfrom->SetRecvVersion(std::min(pfrom->nVersion.load(), PROTOCOL_VERSION)); + pfrom.SetRecvVersion(std::min(pfrom.nVersion.load(), PROTOCOL_VERSION)); - if (!pfrom->fInbound) { + if (!pfrom.fInbound) { // Mark this node as currently connected, so we update its timestamp later. LOCK(cs_main); - State(pfrom->GetId())->fCurrentlyConnected = true; + State(pfrom.GetId())->fCurrentlyConnected = true; LogPrintf("New outbound peer connected: version: %d, blocks=%d, peer=%d%s (%s)\n", - pfrom->nVersion.load(), pfrom->nStartingHeight, - pfrom->GetId(), (fLogIPs ? strprintf(", peeraddr=%s", pfrom->addr.ToString()) : ""), - pfrom->m_tx_relay == nullptr ? "block-relay" : "full-relay"); + pfrom.nVersion.load(), pfrom.nStartingHeight, + pfrom.GetId(), (fLogIPs ? strprintf(", peeraddr=%s", pfrom.addr.ToString()) : ""), + pfrom.m_tx_relay == nullptr ? "block-relay" : "full-relay"); } - if (pfrom->nVersion >= SENDHEADERS_VERSION) { + if (pfrom.nVersion >= SENDHEADERS_VERSION) { // Tell our peer we prefer to receive headers rather than inv's // We send this to non-NODE NETWORK peers as well, because even // non-NODE NETWORK peers can announce blocks (such as pruning // nodes) - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); } - if (pfrom->nVersion >= SHORT_IDS_BLOCKS_VERSION) { + if (pfrom.nVersion >= SHORT_IDS_BLOCKS_VERSION) { // Tell our peer we are willing to provide version 1 or 2 cmpctblocks // However, we do not request new block announcements using // cmpctblock messages. @@ -2190,19 +2430,19 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // they may wish to request compact blocks from us bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 2; - if (pfrom->GetLocalServices() & NODE_WITNESS) - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + if (pfrom.GetLocalServices() & NODE_WITNESS) + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); nCMPCTBLOCKVersion = 1; - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); } - pfrom->fSuccessfullyConnected = true; + pfrom.fSuccessfullyConnected = true; return true; } - if (!pfrom->fSuccessfullyConnected) { + if (!pfrom.fSuccessfullyConnected) { // Must have a verack message before anything else LOCK(cs_main); - Misbehaving(pfrom->GetId(), 1); + Misbehaving(pfrom.GetId(), 1); return false; } @@ -2211,15 +2451,15 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec vRecv >> vAddr; // Don't want addr from older versions unless seeding - if (pfrom->nVersion < CADDR_TIME_VERSION && connman->GetAddressCount() > 1000) + if (pfrom.nVersion < CADDR_TIME_VERSION && connman->GetAddressCount() > 1000) return true; - if (!pfrom->IsAddrRelayPeer()) { + if (!pfrom.IsAddrRelayPeer()) { return true; } if (vAddr.size() > 1000) { LOCK(cs_main); - Misbehaving(pfrom->GetId(), 20, strprintf("message addr size() = %u", vAddr.size())); + Misbehaving(pfrom.GetId(), 20, strprintf("message addr size() = %u", vAddr.size())); return false; } @@ -2240,10 +2480,10 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; - pfrom->AddAddressKnown(addr); + pfrom.AddAddressKnown(addr); if (banman->IsBanned(addr)) continue; // Do not process banned addresses beyond remembering we received them bool fReachable = IsReachable(addr); - if (addr.nTime > nSince && !pfrom->fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) + if (addr.nTime > nSince && !pfrom.fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { // Relay to a limited number of other nodes RelayAddress(addr, fReachable, *connman); @@ -2252,17 +2492,17 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec if (fReachable) vAddrOk.push_back(addr); } - connman->AddNewAddresses(vAddrOk, pfrom->addr, 2 * 60 * 60); + connman->AddNewAddresses(vAddrOk, pfrom.addr, 2 * 60 * 60); if (vAddr.size() < 1000) - pfrom->fGetAddr = false; - if (pfrom->fOneShot) - pfrom->fDisconnect = true; + pfrom.fGetAddr = false; + if (pfrom.fOneShot) + pfrom.fDisconnect = true; return true; } if (msg_type == NetMsgType::SENDHEADERS) { LOCK(cs_main); - State(pfrom->GetId())->fPreferHeaders = true; + State(pfrom.GetId())->fPreferHeaders = true; return true; } @@ -2270,20 +2510,20 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 0; vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion; - if (nCMPCTBLOCKVersion == 1 || ((pfrom->GetLocalServices() & NODE_WITNESS) && nCMPCTBLOCKVersion == 2)) { + if (nCMPCTBLOCKVersion == 1 || ((pfrom.GetLocalServices() & NODE_WITNESS) && nCMPCTBLOCKVersion == 2)) { LOCK(cs_main); // fProvidesHeaderAndIDs is used to "lock in" version of compact blocks we send (fWantsCmpctWitness) - if (!State(pfrom->GetId())->fProvidesHeaderAndIDs) { - State(pfrom->GetId())->fProvidesHeaderAndIDs = true; - State(pfrom->GetId())->fWantsCmpctWitness = nCMPCTBLOCKVersion == 2; + if (!State(pfrom.GetId())->fProvidesHeaderAndIDs) { + State(pfrom.GetId())->fProvidesHeaderAndIDs = true; + State(pfrom.GetId())->fWantsCmpctWitness = nCMPCTBLOCKVersion == 2; } - if (State(pfrom->GetId())->fWantsCmpctWitness == (nCMPCTBLOCKVersion == 2)) // ignore later version announces - State(pfrom->GetId())->fPreferHeaderAndIDs = fAnnounceUsingCMPCTBLOCK; - if (!State(pfrom->GetId())->fSupportsDesiredCmpctVersion) { - if (pfrom->GetLocalServices() & NODE_WITNESS) - State(pfrom->GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 2); + if (State(pfrom.GetId())->fWantsCmpctWitness == (nCMPCTBLOCKVersion == 2)) // ignore later version announces + State(pfrom.GetId())->fPreferHeaderAndIDs = fAnnounceUsingCMPCTBLOCK; + if (!State(pfrom.GetId())->fSupportsDesiredCmpctVersion) { + if (pfrom.GetLocalServices() & NODE_WITNESS) + State(pfrom.GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 2); else - State(pfrom->GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 1); + State(pfrom.GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 1); } } return true; @@ -2295,22 +2535,23 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec if (vInv.size() > MAX_INV_SZ) { LOCK(cs_main); - Misbehaving(pfrom->GetId(), 20, strprintf("message inv size() = %u", vInv.size())); + Misbehaving(pfrom.GetId(), 20, strprintf("message inv size() = %u", vInv.size())); return false; } // We won't accept tx inv's if we're in blocks-only mode, or this is a // block-relay-only peer - bool fBlocksOnly = !g_relay_txes || (pfrom->m_tx_relay == nullptr); + bool fBlocksOnly = !g_relay_txes || (pfrom.m_tx_relay == nullptr); // Allow whitelisted peers to send data other than blocks in blocks only mode if whitelistrelay is true - if (pfrom->HasPermission(PF_RELAY)) + if (pfrom.HasPermission(PF_RELAY)) fBlocksOnly = false; LOCK(cs_main); uint32_t nFetchFlags = GetFetchFlags(pfrom); const auto current_time = GetTime<std::chrono::microseconds>(); + uint256* best_block{nullptr}; for (CInv &inv : vInv) { @@ -2318,36 +2559,39 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec return true; bool fAlreadyHave = AlreadyHave(inv, mempool); - LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->GetId()); + LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); if (inv.type == MSG_TX) { inv.type |= nFetchFlags; } if (inv.type == MSG_BLOCK) { - UpdateBlockAvailability(pfrom->GetId(), inv.hash); + UpdateBlockAvailability(pfrom.GetId(), inv.hash); if (!fAlreadyHave && !fImporting && !fReindex && !mapBlocksInFlight.count(inv.hash)) { - // We used to request the full block here, but since headers-announcements are now the - // primary method of announcement on the network, and since, in the case that a node - // fell back to inv we probably have a reorg which we should get the headers for first, - // we now only provide a getheaders response here. When we receive the headers, we will - // then ask for the blocks we need. - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), inv.hash)); - LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->GetId()); + // Headers-first is the primary method of announcement on + // the network. If a node fell back to sending blocks by inv, + // it's probably for a re-org. The final block hash + // provided should be the highest, so send a getheaders and + // then fetch the blocks we need to catch up. + best_block = &inv.hash; } - } - else - { - pfrom->AddInventoryKnown(inv); + } else { + pfrom.AddInventoryKnown(inv); if (fBlocksOnly) { - LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom->GetId()); - pfrom->fDisconnect = true; + LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); + pfrom.fDisconnect = true; return true; } else if (!fAlreadyHave && !fImporting && !fReindex && !::ChainstateActive().IsInitialBlockDownload()) { - RequestTx(State(pfrom->GetId()), inv.hash, current_time); + RequestTx(State(pfrom.GetId()), inv.hash, current_time); } } } + + if (best_block != nullptr) { + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), *best_block)); + LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, best_block->ToString(), pfrom.GetId()); + } + return true; } @@ -2357,17 +2601,17 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec if (vInv.size() > MAX_INV_SZ) { LOCK(cs_main); - Misbehaving(pfrom->GetId(), 20, strprintf("message getdata size() = %u", vInv.size())); + Misbehaving(pfrom.GetId(), 20, strprintf("message getdata size() = %u", vInv.size())); return false; } - LogPrint(BCLog::NET, "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->GetId()); + LogPrint(BCLog::NET, "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom.GetId()); if (vInv.size() > 0) { - LogPrint(BCLog::NET, "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->GetId()); + LogPrint(BCLog::NET, "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom.GetId()); } - pfrom->vRecvGetData.insert(pfrom->vRecvGetData.end(), vInv.begin(), vInv.end()); + pfrom.vRecvGetData.insert(pfrom.vRecvGetData.end(), vInv.begin(), vInv.end()); ProcessGetData(pfrom, chainparams, connman, mempool, interruptMsgProc); return true; } @@ -2378,8 +2622,8 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec vRecv >> locator >> hashStop; if (locator.vHave.size() > MAX_LOCATOR_SZ) { - LogPrint(BCLog::NET, "getblocks locator size %lld > %d, disconnect peer=%d\n", locator.vHave.size(), MAX_LOCATOR_SZ, pfrom->GetId()); - pfrom->fDisconnect = true; + LogPrint(BCLog::NET, "getblocks locator size %lld > %d, disconnect peer=%d\n", locator.vHave.size(), MAX_LOCATOR_SZ, pfrom.GetId()); + pfrom.fDisconnect = true; return true; } @@ -2411,7 +2655,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec if (pindex) pindex = ::ChainActive().Next(pindex); int nLimit = 500; - LogPrint(BCLog::NET, "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->GetId()); + LogPrint(BCLog::NET, "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom.GetId()); for (; pindex; pindex = ::ChainActive().Next(pindex)) { if (pindex->GetBlockHash() == hashStop) @@ -2427,13 +2671,13 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec LogPrint(BCLog::NET, " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); break; } - pfrom->PushInventory(CInv(MSG_BLOCK, pindex->GetBlockHash())); + pfrom.PushInventory(CInv(MSG_BLOCK, pindex->GetBlockHash())); if (--nLimit <= 0) { // When this block is requested, we'll send an inv that'll // trigger the peer to getblocks the next batch of inventory. LogPrint(BCLog::NET, " getblocks stopping at limit %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); - pfrom->hashContinue = pindex->GetBlockHash(); + pfrom.hashContinue = pindex->GetBlockHash(); break; } } @@ -2460,7 +2704,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec const CBlockIndex* pindex = LookupBlockIndex(req.blockhash); if (!pindex || !(pindex->nStatus & BLOCK_HAVE_DATA)) { - LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block we don't have\n", pfrom->GetId()); + LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block we don't have\n", pfrom.GetId()); return true; } @@ -2472,11 +2716,11 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // might maliciously send lots of getblocktxn requests to trigger // expensive disk reads, because it will require the peer to // actually receive all the data read from disk over the network. - LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block > %i deep\n", pfrom->GetId(), MAX_BLOCKTXN_DEPTH); + LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block > %i deep\n", pfrom.GetId(), MAX_BLOCKTXN_DEPTH); CInv inv; - inv.type = State(pfrom->GetId())->fWantsCmpctWitness ? MSG_WITNESS_BLOCK : MSG_BLOCK; + inv.type = State(pfrom.GetId())->fWantsCmpctWitness ? MSG_WITNESS_BLOCK : MSG_BLOCK; inv.hash = req.blockhash; - pfrom->vRecvGetData.push_back(inv); + pfrom.vRecvGetData.push_back(inv); // The message processing loop will go around again (without pausing) and we'll respond then (without cs_main) return true; } @@ -2495,18 +2739,18 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec vRecv >> locator >> hashStop; if (locator.vHave.size() > MAX_LOCATOR_SZ) { - LogPrint(BCLog::NET, "getheaders locator size %lld > %d, disconnect peer=%d\n", locator.vHave.size(), MAX_LOCATOR_SZ, pfrom->GetId()); - pfrom->fDisconnect = true; + LogPrint(BCLog::NET, "getheaders locator size %lld > %d, disconnect peer=%d\n", locator.vHave.size(), MAX_LOCATOR_SZ, pfrom.GetId()); + pfrom.fDisconnect = true; return true; } LOCK(cs_main); - if (::ChainstateActive().IsInitialBlockDownload() && !pfrom->HasPermission(PF_NOBAN)) { - LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->GetId()); + if (::ChainstateActive().IsInitialBlockDownload() && !pfrom.HasPermission(PF_NOBAN)) { + LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom.GetId()); return true; } - CNodeState *nodestate = State(pfrom->GetId()); + CNodeState *nodestate = State(pfrom.GetId()); const CBlockIndex* pindex = nullptr; if (locator.IsNull()) { @@ -2517,7 +2761,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec } if (!BlockRequestAllowed(pindex, chainparams.GetConsensus())) { - LogPrint(BCLog::NET, "%s: ignoring request from peer=%i for old block header that isn't in the main chain\n", __func__, pfrom->GetId()); + LogPrint(BCLog::NET, "%s: ignoring request from peer=%i for old block header that isn't in the main chain\n", __func__, pfrom.GetId()); return true; } } @@ -2532,7 +2776,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // we must use CBlocks, as CBlockHeaders won't include the 0x00 nTx count at the end std::vector<CBlock> vHeaders; int nLimit = MAX_HEADERS_RESULTS; - LogPrint(BCLog::NET, "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->GetId()); + LogPrint(BCLog::NET, "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom.GetId()); for (; pindex; pindex = ::ChainActive().Next(pindex)) { vHeaders.push_back(pindex->GetBlockHeader()); @@ -2552,18 +2796,18 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // will re-announce the new block via headers (or compact blocks again) // in the SendMessages logic. nodestate->pindexBestHeaderSent = pindex ? pindex : ::ChainActive().Tip(); - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); return true; } if (msg_type == NetMsgType::TX) { // Stop processing the transaction early if - // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off - // or if this peer is supposed to be a block-relay-only peer - if ((!g_relay_txes && !pfrom->HasPermission(PF_RELAY)) || (pfrom->m_tx_relay == nullptr)) + // 1) We are in blocks only mode and peer has no relay permission + // 2) This peer is a block-relay-only peer + if ((!g_relay_txes && !pfrom.HasPermission(PF_RELAY)) || (pfrom.m_tx_relay == nullptr)) { - LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom->GetId()); - pfrom->fDisconnect = true; + LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom.GetId()); + pfrom.fDisconnect = true; return true; } @@ -2572,13 +2816,13 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec const CTransaction& tx = *ptx; CInv inv(MSG_TX, tx.GetHash()); - pfrom->AddInventoryKnown(inv); + pfrom.AddInventoryKnown(inv); LOCK2(cs_main, g_cs_orphans); TxValidationState state; - CNodeState* nodestate = State(pfrom->GetId()); + CNodeState* nodestate = State(pfrom.GetId()); nodestate->m_tx_download.m_tx_announced.erase(inv.hash); nodestate->m_tx_download.m_tx_in_flight.erase(inv.hash); EraseTxRequest(inv.hash); @@ -2593,20 +2837,20 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(inv.hash, i)); if (it_by_prev != mapOrphanTransactionsByPrev.end()) { for (const auto& elem : it_by_prev->second) { - pfrom->orphan_work_set.insert(elem->first); + pfrom.orphan_work_set.insert(elem->first); } } } - pfrom->nLastTXTime = GetTime(); + pfrom.nLastTXTime = GetTime(); LogPrint(BCLog::MEMPOOL, "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", - pfrom->GetId(), + pfrom.GetId(), tx.GetHash().ToString(), mempool.size(), mempool.DynamicMemoryUsage() / 1000); // Recursively process any orphan transactions that depended on this one - ProcessOrphanTx(connman, mempool, pfrom->orphan_work_set, lRemovedTxn); + ProcessOrphanTx(connman, mempool, pfrom.orphan_work_set, lRemovedTxn); } else if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { @@ -2623,10 +2867,10 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec for (const CTxIn& txin : tx.vin) { CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); - pfrom->AddInventoryKnown(_inv); - if (!AlreadyHave(_inv, mempool)) RequestTx(State(pfrom->GetId()), _inv.hash, current_time); + pfrom.AddInventoryKnown(_inv); + if (!AlreadyHave(_inv, mempool)) RequestTx(State(pfrom.GetId()), _inv.hash, current_time); } - AddOrphanTx(ptx, pfrom->GetId()); + AddOrphanTx(ptx, pfrom.GetId()); // DoS prevention: do not allow mapOrphanTransactions to grow unbounded (see CVE-2012-3789) unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, gArgs.GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); @@ -2654,15 +2898,15 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec AddToCompactExtraTransactions(ptx); } - if (pfrom->HasPermission(PF_FORCERELAY)) { + if (pfrom.HasPermission(PF_FORCERELAY)) { // Always relay transactions received from whitelisted peers, even // if they were already in the mempool, // allowing the node to function as a gateway for // nodes hidden behind it. if (!mempool.exists(tx.GetHash())) { - LogPrintf("Not relaying non-mempool transaction %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->GetId()); + LogPrintf("Not relaying non-mempool transaction %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); } else { - LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->GetId()); + LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); RelayTransaction(tx.GetHash(), *connman); } } @@ -2691,9 +2935,9 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec if (state.IsInvalid()) { LogPrint(BCLog::MEMPOOLREJ, "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), - pfrom->GetId(), + pfrom.GetId(), state.ToString()); - MaybePunishNodeForTx(pfrom->GetId(), state); + MaybePunishNodeForTx(pfrom.GetId(), state); } return true; } @@ -2702,7 +2946,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec { // Ignore cmpctblock received while importing if (fImporting || fReindex) { - LogPrint(BCLog::NET, "Unexpected cmpctblock message received from peer %d\n", pfrom->GetId()); + LogPrint(BCLog::NET, "Unexpected cmpctblock message received from peer %d\n", pfrom.GetId()); return true; } @@ -2717,7 +2961,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec if (!LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers if (!::ChainstateActive().IsInitialBlockDownload()) - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); return true; } @@ -2728,9 +2972,9 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec const CBlockIndex *pindex = nullptr; BlockValidationState state; - if (!ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { + if (!chainman.ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { if (state.IsInvalid()) { - MaybePunishNodeForBlock(pfrom->GetId(), state, /*via_compact_block*/ true, "invalid header via cmpctblock"); + MaybePunishNodeForBlock(pfrom.GetId(), state, /*via_compact_block*/ true, "invalid header via cmpctblock"); return true; } } @@ -2755,9 +2999,9 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec LOCK2(cs_main, g_cs_orphans); // If AcceptBlockHeader returned true, it set pindex assert(pindex); - UpdateBlockAvailability(pfrom->GetId(), pindex->GetBlockHash()); + UpdateBlockAvailability(pfrom.GetId(), pindex->GetBlockHash()); - CNodeState *nodestate = State(pfrom->GetId()); + CNodeState *nodestate = State(pfrom.GetId()); // If this was a new header with more work than our tip, update the // peer's last block announcement time @@ -2778,7 +3022,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // so we just grab the block via normal getdata std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } return true; } @@ -2797,9 +3041,9 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // possibilities in compact block processing... if (pindex->nHeight <= ::ChainActive().Height() + 2) { if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || - (fAlreadyInFlight && blockInFlightIt->second.first == pfrom->GetId())) { + (fAlreadyInFlight && blockInFlightIt->second.first == pfrom.GetId())) { std::list<QueuedBlock>::iterator* queuedBlockIt = nullptr; - if (!MarkBlockAsInFlight(mempool, pfrom->GetId(), pindex->GetBlockHash(), pindex, &queuedBlockIt)) { + if (!MarkBlockAsInFlight(mempool, pfrom.GetId(), pindex->GetBlockHash(), pindex, &queuedBlockIt)) { if (!(*queuedBlockIt)->partialBlock) (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); else { @@ -2813,13 +3057,13 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec ReadStatus status = partialBlock.InitData(cmpctblock, vExtraTxnForCompact); if (status == READ_STATUS_INVALID) { MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist - Misbehaving(pfrom->GetId(), 100, strprintf("Peer %d sent us invalid compact block\n", pfrom->GetId())); + Misbehaving(pfrom.GetId(), 100, strprintf("Peer %d sent us invalid compact block\n", pfrom.GetId())); return true; } else if (status == READ_STATUS_FAILED) { // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } @@ -2836,7 +3080,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec fProcessBLOCKTXN = true; } else { req.blockhash = pindex->GetBlockHash(); - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); } } else { // This block is either already in flight from a different @@ -2862,7 +3106,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } else { // If this was an announce-cmpctblock, we want the same treatment as a header message @@ -2872,7 +3116,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec } // cs_main if (fProcessBLOCKTXN) - return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, mempool, connman, banman, interruptMsgProc); + return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, chainman, mempool, connman, banman, interruptMsgProc); if (fRevertToHeaderProcessing) { // Headers received from HB compact block peers are permitted to be @@ -2880,7 +3124,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // the peer if the header turns out to be for an invalid block. // Note that if a peer tries to build on an invalid chain, that // will be detected and the peer will be banned. - return ProcessHeadersMessage(pfrom, connman, mempool, {cmpctblock.header}, chainparams, /*via_compact_block=*/true); + return ProcessHeadersMessage(pfrom, connman, chainman, mempool, {cmpctblock.header}, chainparams, /*via_compact_block=*/true); } if (fBlockReconstructed) { @@ -2888,7 +3132,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // block that is in flight from some other peer. { LOCK(cs_main); - mapBlockSource.emplace(pblock->GetHash(), std::make_pair(pfrom->GetId(), false)); + mapBlockSource.emplace(pblock->GetHash(), std::make_pair(pfrom.GetId(), false)); } bool fNewBlock = false; // Setting fForceProcessing to true means that we bypass some of @@ -2900,9 +3144,9 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // we have a chain with at least nMinimumChainWork), and we ignore // compact blocks with less work than our tip, it is safe to treat // reconstructed compact blocks as having been requested. - ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); + chainman.ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { - pfrom->nLastBlockTime = GetTime(); + pfrom.nLastBlockTime = GetTime(); } else { LOCK(cs_main); mapBlockSource.erase(pblock->GetHash()); @@ -2923,7 +3167,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec { // Ignore blocktxn received while importing if (fImporting || fReindex) { - LogPrint(BCLog::NET, "Unexpected blocktxn message received from peer %d\n", pfrom->GetId()); + LogPrint(BCLog::NET, "Unexpected blocktxn message received from peer %d\n", pfrom.GetId()); return true; } @@ -2937,8 +3181,8 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || - it->second.first != pfrom->GetId()) { - LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->GetId()); + it->second.first != pfrom.GetId()) { + LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom.GetId()); return true; } @@ -2946,13 +3190,13 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec ReadStatus status = partialBlock.FillBlock(*pblock, resp.txn); if (status == READ_STATUS_INVALID) { MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist - Misbehaving(pfrom->GetId(), 100, strprintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->GetId())); + Misbehaving(pfrom.GetId(), 100, strprintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom.GetId())); return true; } else if (status == READ_STATUS_FAILED) { // Might have collided, fall back to getdata now :( std::vector<CInv> invs; invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom), resp.blockhash)); - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { // Block is either okay, or possibly we received // READ_STATUS_CHECKBLOCK_FAILED. @@ -2979,7 +3223,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // BIP 152 permits peers to relay compact blocks after validating // the header only; we should not punish peers if the block turns // out to be invalid. - mapBlockSource.emplace(resp.blockhash, std::make_pair(pfrom->GetId(), false)); + mapBlockSource.emplace(resp.blockhash, std::make_pair(pfrom.GetId(), false)); } } // Don't hold cs_main when we call into ProcessNewBlock if (fBlockRead) { @@ -2990,9 +3234,9 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // disk-space attacks), but this should be safe due to the // protections in the compact block handler -- see related comment // in compact block optimistic reconstruction handling. - ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); + chainman.ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { - pfrom->nLastBlockTime = GetTime(); + pfrom.nLastBlockTime = GetTime(); } else { LOCK(cs_main); mapBlockSource.erase(pblock->GetHash()); @@ -3005,7 +3249,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec { // Ignore headers received while importing if (fImporting || fReindex) { - LogPrint(BCLog::NET, "Unexpected headers message received from peer %d\n", pfrom->GetId()); + LogPrint(BCLog::NET, "Unexpected headers message received from peer %d\n", pfrom.GetId()); return true; } @@ -3015,7 +3259,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec unsigned int nCount = ReadCompactSize(vRecv); if (nCount > MAX_HEADERS_RESULTS) { LOCK(cs_main); - Misbehaving(pfrom->GetId(), 20, strprintf("headers message size = %u", nCount)); + Misbehaving(pfrom.GetId(), 20, strprintf("headers message size = %u", nCount)); return false; } headers.resize(nCount); @@ -3024,21 +3268,21 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } - return ProcessHeadersMessage(pfrom, connman, mempool, headers, chainparams, /*via_compact_block=*/false); + return ProcessHeadersMessage(pfrom, connman, chainman, mempool, headers, chainparams, /*via_compact_block=*/false); } if (msg_type == NetMsgType::BLOCK) { // Ignore block received while importing if (fImporting || fReindex) { - LogPrint(BCLog::NET, "Unexpected block message received from peer %d\n", pfrom->GetId()); + LogPrint(BCLog::NET, "Unexpected block message received from peer %d\n", pfrom.GetId()); return true; } std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); vRecv >> *pblock; - LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->GetId()); + LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom.GetId()); bool forceProcessing = false; const uint256 hash(pblock->GetHash()); @@ -3050,12 +3294,12 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // mapBlockSource is only used for punishing peers and setting // which peers send us compact blocks, so the race between here and // cs_main in ProcessNewBlock is fine. - mapBlockSource.emplace(hash, std::make_pair(pfrom->GetId(), true)); + mapBlockSource.emplace(hash, std::make_pair(pfrom.GetId(), true)); } bool fNewBlock = false; - ProcessNewBlock(chainparams, pblock, forceProcessing, &fNewBlock); + chainman.ProcessNewBlock(chainparams, pblock, forceProcessing, &fNewBlock); if (fNewBlock) { - pfrom->nLastBlockTime = GetTime(); + pfrom.nLastBlockTime = GetTime(); } else { LOCK(cs_main); mapBlockSource.erase(pblock->GetHash()); @@ -3069,64 +3313,64 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // to users' AddrMan and later request them by sending getaddr messages. // Making nodes which are behind NAT and can only make outgoing connections ignore // the getaddr message mitigates the attack. - if (!pfrom->fInbound) { - LogPrint(BCLog::NET, "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->GetId()); + if (!pfrom.fInbound) { + LogPrint(BCLog::NET, "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom.GetId()); return true; } - if (!pfrom->IsAddrRelayPeer()) { - LogPrint(BCLog::NET, "Ignoring \"getaddr\" from block-relay-only connection. peer=%d\n", pfrom->GetId()); + if (!pfrom.IsAddrRelayPeer()) { + LogPrint(BCLog::NET, "Ignoring \"getaddr\" from block-relay-only connection. peer=%d\n", pfrom.GetId()); return true; } // Only send one GetAddr response per connection to reduce resource waste // and discourage addr stamping of INV announcements. - if (pfrom->fSentAddr) { - LogPrint(BCLog::NET, "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->GetId()); + if (pfrom.fSentAddr) { + LogPrint(BCLog::NET, "Ignoring repeated \"getaddr\". peer=%d\n", pfrom.GetId()); return true; } - pfrom->fSentAddr = true; + pfrom.fSentAddr = true; - pfrom->vAddrToSend.clear(); + pfrom.vAddrToSend.clear(); std::vector<CAddress> vAddr = connman->GetAddresses(); FastRandomContext insecure_rand; for (const CAddress &addr : vAddr) { if (!banman->IsBanned(addr)) { - pfrom->PushAddress(addr, insecure_rand); + pfrom.PushAddress(addr, insecure_rand); } } return true; } if (msg_type == NetMsgType::MEMPOOL) { - if (!(pfrom->GetLocalServices() & NODE_BLOOM) && !pfrom->HasPermission(PF_MEMPOOL)) + if (!(pfrom.GetLocalServices() & NODE_BLOOM) && !pfrom.HasPermission(PF_MEMPOOL)) { - if (!pfrom->HasPermission(PF_NOBAN)) + if (!pfrom.HasPermission(PF_NOBAN)) { - LogPrint(BCLog::NET, "mempool request with bloom filters disabled, disconnect peer=%d\n", pfrom->GetId()); - pfrom->fDisconnect = true; + LogPrint(BCLog::NET, "mempool request with bloom filters disabled, disconnect peer=%d\n", pfrom.GetId()); + pfrom.fDisconnect = true; } return true; } - if (connman->OutboundTargetReached(false) && !pfrom->HasPermission(PF_MEMPOOL)) + if (connman->OutboundTargetReached(false) && !pfrom.HasPermission(PF_MEMPOOL)) { - if (!pfrom->HasPermission(PF_NOBAN)) + if (!pfrom.HasPermission(PF_NOBAN)) { - LogPrint(BCLog::NET, "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom->GetId()); - pfrom->fDisconnect = true; + LogPrint(BCLog::NET, "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom.GetId()); + pfrom.fDisconnect = true; } return true; } - if (pfrom->m_tx_relay != nullptr) { - LOCK(pfrom->m_tx_relay->cs_tx_inventory); - pfrom->m_tx_relay->fSendMempool = true; + if (pfrom.m_tx_relay != nullptr) { + LOCK(pfrom.m_tx_relay->cs_tx_inventory); + pfrom.m_tx_relay->fSendMempool = true; } return true; } if (msg_type == NetMsgType::PING) { - if (pfrom->nVersion > BIP0031_VERSION) + if (pfrom.nVersion > BIP0031_VERSION) { uint64_t nonce = 0; vRecv >> nonce; @@ -3141,7 +3385,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // it, if the remote node sends a ping once per second and this node takes 5 // seconds to respond to each, the 5th ping the remote sends would appear to // return very quickly. - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); + connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); } return true; } @@ -3157,15 +3401,15 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec vRecv >> nonce; // Only process pong message if there is an outstanding ping (old ping without nonce should never pong) - if (pfrom->nPingNonceSent != 0) { - if (nonce == pfrom->nPingNonceSent) { + if (pfrom.nPingNonceSent != 0) { + if (nonce == pfrom.nPingNonceSent) { // Matching pong received, this ping is no longer outstanding bPingFinished = true; - int64_t pingUsecTime = pingUsecEnd - pfrom->nPingUsecStart; + int64_t pingUsecTime = pingUsecEnd - pfrom.nPingUsecStart; if (pingUsecTime > 0) { // Successful ping time measurement, replace previous - pfrom->nPingUsecTime = pingUsecTime; - pfrom->nMinPingUsecTime = std::min(pfrom->nMinPingUsecTime.load(), pingUsecTime); + pfrom.nPingUsecTime = pingUsecTime; + pfrom.nMinPingUsecTime = std::min(pfrom.nMinPingUsecTime.load(), pingUsecTime); } else { // This should never happen sProblem = "Timing mishap"; @@ -3190,14 +3434,14 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec if (!(sProblem.empty())) { LogPrint(BCLog::NET, "pong peer=%d: %s, %x expected, %x received, %u bytes\n", - pfrom->GetId(), + pfrom.GetId(), sProblem, - pfrom->nPingNonceSent, + pfrom.nPingNonceSent, nonce, nAvail); } if (bPingFinished) { - pfrom->nPingNonceSent = 0; + pfrom.nPingNonceSent = 0; } return true; } @@ -3210,13 +3454,13 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec { // There is no excuse for sending a too-large filter LOCK(cs_main); - Misbehaving(pfrom->GetId(), 100); + Misbehaving(pfrom.GetId(), 100); } - else if (pfrom->m_tx_relay != nullptr) + else if (pfrom.m_tx_relay != nullptr) { - LOCK(pfrom->m_tx_relay->cs_filter); - pfrom->m_tx_relay->pfilter.reset(new CBloomFilter(filter)); - pfrom->m_tx_relay->fRelayTxes = true; + LOCK(pfrom.m_tx_relay->cs_filter); + pfrom.m_tx_relay->pfilter.reset(new CBloomFilter(filter)); + pfrom.m_tx_relay->fRelayTxes = true; } return true; } @@ -3230,30 +3474,30 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec bool bad = false; if (vData.size() > MAX_SCRIPT_ELEMENT_SIZE) { bad = true; - } else if (pfrom->m_tx_relay != nullptr) { - LOCK(pfrom->m_tx_relay->cs_filter); - if (pfrom->m_tx_relay->pfilter) { - pfrom->m_tx_relay->pfilter->insert(vData); + } else if (pfrom.m_tx_relay != nullptr) { + LOCK(pfrom.m_tx_relay->cs_filter); + if (pfrom.m_tx_relay->pfilter) { + pfrom.m_tx_relay->pfilter->insert(vData); } else { bad = true; } } if (bad) { LOCK(cs_main); - Misbehaving(pfrom->GetId(), 100); + Misbehaving(pfrom.GetId(), 100); } return true; } if (msg_type == NetMsgType::FILTERCLEAR) { - if (pfrom->m_tx_relay == nullptr) { + if (pfrom.m_tx_relay == nullptr) { return true; } - LOCK(pfrom->m_tx_relay->cs_filter); - if (pfrom->GetLocalServices() & NODE_BLOOM) { - pfrom->m_tx_relay->pfilter = nullptr; + LOCK(pfrom.m_tx_relay->cs_filter); + if (pfrom.GetLocalServices() & NODE_BLOOM) { + pfrom.m_tx_relay->pfilter = nullptr; } - pfrom->m_tx_relay->fRelayTxes = true; + pfrom.m_tx_relay->fRelayTxes = true; return true; } @@ -3261,19 +3505,34 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec CAmount newFeeFilter = 0; vRecv >> newFeeFilter; if (MoneyRange(newFeeFilter)) { - if (pfrom->m_tx_relay != nullptr) { - LOCK(pfrom->m_tx_relay->cs_feeFilter); - pfrom->m_tx_relay->minFeeFilter = newFeeFilter; + if (pfrom.m_tx_relay != nullptr) { + LOCK(pfrom.m_tx_relay->cs_feeFilter); + pfrom.m_tx_relay->minFeeFilter = newFeeFilter; } - LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->GetId()); + LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom.GetId()); } return true; } + if (msg_type == NetMsgType::GETCFILTERS) { + ProcessGetCFilters(pfrom, vRecv, chainparams, *connman); + return true; + } + + if (msg_type == NetMsgType::GETCFHEADERS) { + ProcessGetCFHeaders(pfrom, vRecv, chainparams, *connman); + return true; + } + + if (msg_type == NetMsgType::GETCFCHECKPT) { + ProcessGetCFCheckPt(pfrom, vRecv, chainparams, *connman); + return true; + } + if (msg_type == NetMsgType::NOTFOUND) { // Remove the NOTFOUND transactions from the peer LOCK(cs_main); - CNodeState *state = State(pfrom->GetId()); + CNodeState *state = State(pfrom.GetId()); std::vector<CInv> vInv; vRecv >> vInv; if (vInv.size() <= MAX_PEER_TX_IN_FLIGHT + MAX_BLOCKS_IN_TRANSIT_PER_PEER) { @@ -3296,31 +3555,31 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec } // Ignore unknown commands for extensibility - LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(msg_type), pfrom->GetId()); + LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(msg_type), pfrom.GetId()); return true; } -bool PeerLogicValidation::CheckIfBanned(CNode* pnode) +bool PeerLogicValidation::CheckIfBanned(CNode& pnode) { AssertLockHeld(cs_main); - CNodeState &state = *State(pnode->GetId()); + CNodeState &state = *State(pnode.GetId()); if (state.fShouldBan) { state.fShouldBan = false; - if (pnode->HasPermission(PF_NOBAN)) - LogPrintf("Warning: not punishing whitelisted peer %s!\n", pnode->addr.ToString()); - else if (pnode->m_manual_connection) - LogPrintf("Warning: not punishing manually-connected peer %s!\n", pnode->addr.ToString()); - else if (pnode->addr.IsLocal()) { + if (pnode.HasPermission(PF_NOBAN)) + LogPrintf("Warning: not punishing whitelisted peer %s!\n", pnode.addr.ToString()); + else if (pnode.m_manual_connection) + LogPrintf("Warning: not punishing manually-connected peer %s!\n", pnode.addr.ToString()); + else if (pnode.addr.IsLocal()) { // Disconnect but don't ban _this_ local node - LogPrintf("Warning: disconnecting but not banning local peer %s!\n", pnode->addr.ToString()); - pnode->fDisconnect = true; + LogPrintf("Warning: disconnecting but not banning local peer %s!\n", pnode.addr.ToString()); + pnode.fDisconnect = true; } else { // Disconnect and ban all nodes sharing the address if (m_banman) { - m_banman->Ban(pnode->addr, BanReasonNodeMisbehaving); + m_banman->Ban(pnode.addr, BanReasonNodeMisbehaving); } - connman->DisconnectNode(pnode->addr); + connman->DisconnectNode(pnode.addr); } return true; } @@ -3341,7 +3600,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter bool fMoreWork = false; if (!pfrom->vRecvGetData.empty()) - ProcessGetData(pfrom, chainparams, connman, m_mempool, interruptMsgProc); + ProcessGetData(*pfrom, chainparams, connman, m_mempool, interruptMsgProc); if (!pfrom->orphan_work_set.empty()) { std::list<CTransactionRef> removed_txn; @@ -3409,7 +3668,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter bool fRet = false; try { - fRet = ProcessMessage(pfrom, msg_type, vRecv, msg.m_time, chainparams, m_mempool, connman, m_banman, interruptMsgProc); + fRet = ProcessMessage(*pfrom, msg_type, vRecv, msg.m_time, chainparams, m_chainman, m_mempool, connman, m_banman, interruptMsgProc); if (interruptMsgProc) return false; if (!pfrom->vRecvGetData.empty()) @@ -3425,17 +3684,17 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter } LOCK(cs_main); - CheckIfBanned(pfrom); + CheckIfBanned(*pfrom); return fMoreWork; } -void PeerLogicValidation::ConsiderEviction(CNode *pto, int64_t time_in_seconds) +void PeerLogicValidation::ConsiderEviction(CNode& pto, int64_t time_in_seconds) { AssertLockHeld(cs_main); - CNodeState &state = *State(pto->GetId()); - const CNetMsgMaker msgMaker(pto->GetSendVersion()); + CNodeState &state = *State(pto.GetId()); + const CNetMsgMaker msgMaker(pto.GetSendVersion()); if (!state.m_chain_sync.m_protect && IsOutboundDisconnectionCandidate(pto) && state.fSyncStarted) { // This is an outbound peer subject to disconnection if they don't @@ -3464,12 +3723,12 @@ void PeerLogicValidation::ConsiderEviction(CNode *pto, int64_t time_in_seconds) // message to give the peer a chance to update us. if (state.m_chain_sync.m_sent_getheaders) { // They've run out of time to catch up! - LogPrintf("Disconnecting outbound peer %d for old chain, best known block = %s\n", pto->GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>"); - pto->fDisconnect = true; + LogPrintf("Disconnecting outbound peer %d for old chain, best known block = %s\n", pto.GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>"); + pto.fDisconnect = true; } else { assert(state.m_chain_sync.m_work_header); - LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto->GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); - connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); + LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto.GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); + connman->PushMessage(&pto, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); state.m_chain_sync.m_sent_getheaders = true; constexpr int64_t HEADERS_RESPONSE_TIME = 120; // 2 minutes // Bump the timeout to allow a response, which could clear the timeout @@ -3499,7 +3758,7 @@ void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) AssertLockHeld(cs_main); // Ignore non-outbound peers, or nodes marked for disconnect already - if (!IsOutboundDisconnectionCandidate(pnode) || pnode->fDisconnect) return; + if (!IsOutboundDisconnectionCandidate(*pnode) || pnode->fDisconnect) return; CNodeState *state = State(pnode->GetId()); if (state == nullptr) return; // shouldn't be possible, but just in case // Don't evict our protected peers @@ -3628,7 +3887,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) if (!lockMain) return true; - if (CheckIfBanned(pto)) return true; + if (CheckIfBanned(*pto)) return true; CNodeState &state = *State(pto->GetId()); @@ -4039,7 +4298,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // Check that outbound peers have reasonable chains // GetTime() is used by this anti-DoS logic so we can test this using mocktime - ConsiderEviction(pto, GetTime()); + ConsiderEviction(*pto, GetTime()); // // Message: getdata (blocks) @@ -4050,7 +4309,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) NodeId staller = -1; FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller, consensusParams); for (const CBlockIndex *pindex : vToDownload) { - uint32_t nFetchFlags = GetFetchFlags(pto); + uint32_t nFetchFlags = GetFetchFlags(*pto); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); MarkBlockAsInFlight(m_mempool, pto->GetId(), pindex->GetBlockHash(), pindex); LogPrint(BCLog::NET, "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), @@ -4094,7 +4353,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // Erase this entry from tx_process_time (it may be added back for // processing at a later time, see below) tx_process_time.erase(tx_process_time.begin()); - CInv inv(MSG_TX | GetFetchFlags(pto), txid); + CInv inv(MSG_TX | GetFetchFlags(*pto), txid); if (!AlreadyHave(inv, m_mempool)) { // If this transaction was last requested more than 1 minute ago, // then request. diff --git a/src/net_processing.h b/src/net_processing.h index a85d5e7c70..19beca0cc4 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -12,6 +12,7 @@ #include <validationinterface.h> class CTxMemPool; +class ChainstateManager; extern RecursiveMutex cs_main; extern RecursiveMutex g_cs_orphans; @@ -21,17 +22,19 @@ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; /** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; static const bool DEFAULT_PEERBLOOMFILTERS = false; +static const bool DEFAULT_PEERBLOCKFILTERS = false; class PeerLogicValidation final : public CValidationInterface, public NetEventsInterface { private: CConnman* const connman; BanMan* const m_banman; + ChainstateManager& m_chainman; CTxMemPool& m_mempool; - bool CheckIfBanned(CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool CheckIfBanned(CNode& pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main); public: - PeerLogicValidation(CConnman* connman, BanMan* banman, CScheduler& scheduler, CTxMemPool& pool); + PeerLogicValidation(CConnman* connman, BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool); /** * Overridden from CValidationInterface. @@ -71,7 +74,7 @@ public: bool SendMessages(CNode* pto) override EXCLUSIVE_LOCKS_REQUIRED(pto->cs_sendProcessing); /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ - void ConsiderEviction(CNode *pto, int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void ConsiderEviction(CNode& pto, int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Evict extra outbound peers. If we think our tip may be stale, connect to an extra outbound */ void CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams); /** If we have extra outbound peers, try to disconnect the one with the oldest block announcement */ diff --git a/src/netaddress.h b/src/netaddress.h index d8f19deffe..e640c07d32 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -99,12 +99,7 @@ class CNetAddr friend bool operator!=(const CNetAddr& a, const CNetAddr& b) { return !(a == b); } friend bool operator<(const CNetAddr& a, const CNetAddr& b); - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(ip); - } + SERIALIZE_METHODS(CNetAddr, obj) { READWRITE(obj.ip); } friend class CSubNet; }; @@ -136,14 +131,7 @@ class CSubNet friend bool operator!=(const CSubNet& a, const CSubNet& b) { return !(a == b); } friend bool operator<(const CSubNet& a, const CSubNet& b); - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(network); - READWRITE(netmask); - READWRITE(valid); - } + SERIALIZE_METHODS(CSubNet, obj) { READWRITE(obj.network, obj.netmask, obj.valid); } }; /** A combination of a network address (CNetAddr) and a (TCP) port */ @@ -171,13 +159,7 @@ class CService : public CNetAddr CService(const struct in6_addr& ipv6Addr, unsigned short port); explicit CService(const struct sockaddr_in6& addr); - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(ip); - READWRITE(WrapBigEndian(port)); - } + SERIALIZE_METHODS(CService, obj) { READWRITE(obj.ip, Using<BigEndianFormatter<2>>(obj.port)); } }; bool SanityCheckASMap(const std::vector<bool>& asmap); diff --git a/src/netbase.cpp b/src/netbase.cpp index a70179cb16..9fe03c6a24 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -28,9 +28,9 @@ #endif // Settings -static RecursiveMutex cs_proxyInfos; -static proxyType proxyInfo[NET_MAX] GUARDED_BY(cs_proxyInfos); -static proxyType nameProxy GUARDED_BY(cs_proxyInfos); +static Mutex g_proxyinfo_mutex; +static proxyType proxyInfo[NET_MAX] GUARDED_BY(g_proxyinfo_mutex); +static proxyType nameProxy GUARDED_BY(g_proxyinfo_mutex); int nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; bool fNameLookup = DEFAULT_NAME_LOOKUP; @@ -711,14 +711,14 @@ bool SetProxy(enum Network net, const proxyType &addrProxy) { assert(net >= 0 && net < NET_MAX); if (!addrProxy.IsValid()) return false; - LOCK(cs_proxyInfos); + LOCK(g_proxyinfo_mutex); proxyInfo[net] = addrProxy; return true; } bool GetProxy(enum Network net, proxyType &proxyInfoOut) { assert(net >= 0 && net < NET_MAX); - LOCK(cs_proxyInfos); + LOCK(g_proxyinfo_mutex); if (!proxyInfo[net].IsValid()) return false; proxyInfoOut = proxyInfo[net]; @@ -744,13 +744,13 @@ bool GetProxy(enum Network net, proxyType &proxyInfoOut) { bool SetNameProxy(const proxyType &addrProxy) { if (!addrProxy.IsValid()) return false; - LOCK(cs_proxyInfos); + LOCK(g_proxyinfo_mutex); nameProxy = addrProxy; return true; } bool GetNameProxy(proxyType &nameProxyOut) { - LOCK(cs_proxyInfos); + LOCK(g_proxyinfo_mutex); if(!nameProxy.IsValid()) return false; nameProxyOut = nameProxy; @@ -758,12 +758,12 @@ bool GetNameProxy(proxyType &nameProxyOut) { } bool HaveNameProxy() { - LOCK(cs_proxyInfos); + LOCK(g_proxyinfo_mutex); return nameProxy.IsValid(); } bool IsProxy(const CNetAddr &addr) { - LOCK(cs_proxyInfos); + LOCK(g_proxyinfo_mutex); for (int i = 0; i < NET_MAX; i++) { if (addr == static_cast<CNetAddr>(proxyInfo[i].proxy)) return true; diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index ec52a08ace..e3c4c828b6 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -33,7 +33,7 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, } //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) +bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void()>& interruption_point) { stats = CCoinsStats(); std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); @@ -49,6 +49,7 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) uint256 prevkey; std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { + interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { diff --git a/src/node/coinstats.h b/src/node/coinstats.h index a19af0fd1b..d9cdaa3036 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -10,6 +10,7 @@ #include <uint256.h> #include <cstdint> +#include <functional> class CCoinsView; @@ -29,6 +30,6 @@ struct CCoinsStats }; //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats); +bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void()>& interruption_point = {}); #endif // BITCOIN_NODE_COINSTATS_H diff --git a/src/node/context.h b/src/node/context.h index 566ff170be..c45d9e6689 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_NODE_CONTEXT_H #define BITCOIN_NODE_CONTEXT_H +#include <cassert> #include <memory> #include <vector> @@ -13,6 +14,7 @@ class BanMan; class CConnman; class CScheduler; class CTxMemPool; +class ChainstateManager; class PeerLogicValidation; namespace interfaces { class Chain; @@ -33,6 +35,7 @@ struct NodeContext { std::unique_ptr<CConnman> connman; CTxMemPool* mempool{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<PeerLogicValidation> peer_logic; + ChainstateManager* chainman{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<BanMan> banman; ArgsManager* args{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<interfaces::Chain> chain; @@ -46,4 +49,10 @@ struct NodeContext { ~NodeContext(); }; +inline ChainstateManager& EnsureChainman(const NodeContext& node) +{ + assert(node.chainman); + return *node.chainman; +} + #endif // BITCOIN_NODE_CONTEXT_H diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 702a0cbe53..c8b4d60fd0 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -35,16 +35,7 @@ public: m_coins_count(coins_count), m_nchaintx(nchaintx) { } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(m_base_blockhash); - READWRITE(m_coins_count); - READWRITE(m_nchaintx); - } - + SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count, obj.m_nchaintx); } }; #endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/noui.cpp b/src/noui.cpp index 11cfe7f94d..ddb3a50ff7 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -5,8 +5,9 @@ #include <noui.h> +#include <logging.h> #include <ui_interface.h> -#include <util/system.h> +#include <util/translation.h> #include <string> @@ -18,7 +19,7 @@ boost::signals2::connection noui_ThreadSafeMessageBoxConn; boost::signals2::connection noui_ThreadSafeQuestionConn; boost::signals2::connection noui_InitMessageConn; -bool noui_ThreadSafeMessageBox(const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeMessageBox(const bilingual_str& message, const std::string& caption, unsigned int style) { bool fSecure = style & CClientUIInterface::SECURE; style &= ~CClientUIInterface::SECURE; @@ -43,15 +44,15 @@ bool noui_ThreadSafeMessageBox(const std::string& message, const std::string& ca } if (!fSecure) { - LogPrintf("%s%s\n", strCaption, message); + LogPrintf("%s%s\n", strCaption, message.original); } - tfm::format(std::cerr, "%s%s\n", strCaption, message); + tfm::format(std::cerr, "%s%s\n", strCaption, message.original); return false; } -bool noui_ThreadSafeQuestion(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeQuestion(const bilingual_str& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) { - return noui_ThreadSafeMessageBox(message, caption, style); + return noui_ThreadSafeMessageBox(Untranslated(message), caption, style); } void noui_InitMessage(const std::string& message) @@ -66,13 +67,13 @@ void noui_connect() noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessage); } -bool noui_ThreadSafeMessageBoxRedirect(const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeMessageBoxRedirect(const bilingual_str& message, const std::string& caption, unsigned int style) { - LogPrintf("%s: %s\n", caption, message); + LogPrintf("%s: %s\n", caption, message.original); return false; } -bool noui_ThreadSafeQuestionRedirect(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeQuestionRedirect(const bilingual_str& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) { LogPrintf("%s: %s\n", caption, message); return false; diff --git a/src/noui.h b/src/noui.h index 5e5767b453..8ec5708328 100644 --- a/src/noui.h +++ b/src/noui.h @@ -7,10 +7,12 @@ #include <string> +struct bilingual_str; + /** Non-GUI handler, which logs and prints messages. */ -bool noui_ThreadSafeMessageBox(const std::string& message, const std::string& caption, unsigned int style); +bool noui_ThreadSafeMessageBox(const bilingual_str& message, const std::string& caption, unsigned int style); /** Non-GUI handler, which logs and prints questions. */ -bool noui_ThreadSafeQuestion(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style); +bool noui_ThreadSafeQuestion(const bilingual_str& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style); /** Non-GUI handler, which only logs a message. */ void noui_InitMessage(const std::string& message); diff --git a/src/policy/feerate.h b/src/policy/feerate.h index c040867965..61fa80c130 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -48,12 +48,7 @@ public: CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } std::string ToString() const; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nSatoshisPerK); - } + SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); } }; #endif // BITCOIN_POLICY_FEERATE_H diff --git a/src/primitives/block.h b/src/primitives/block.h index 750d42efbc..fd8fc8b868 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -33,17 +33,7 @@ public: SetNull(); } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(this->nVersion); - READWRITE(hashPrevBlock); - READWRITE(hashMerkleRoot); - READWRITE(nTime); - READWRITE(nBits); - READWRITE(nNonce); - } + SERIALIZE_METHODS(CBlockHeader, obj) { READWRITE(obj.nVersion, obj.hashPrevBlock, obj.hashMerkleRoot, obj.nTime, obj.nBits, obj.nNonce); } void SetNull() { @@ -89,12 +79,10 @@ public: *(static_cast<CBlockHeader*>(this)) = header; } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CBlockHeader, *this); - READWRITE(vtx); + SERIALIZE_METHODS(CBlock, obj) + { + READWRITEAS(CBlockHeader, obj); + READWRITE(obj.vtx); } void SetNull() @@ -131,14 +119,12 @@ struct CBlockLocator explicit CBlockLocator(const std::vector<uint256>& vHaveIn) : vHave(vHaveIn) {} - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { + SERIALIZE_METHODS(CBlockLocator, obj) + { int nVersion = s.GetVersion(); if (!(s.GetType() & SER_GETHASH)) READWRITE(nVersion); - READWRITE(vHave); + READWRITE(obj.vHave); } void SetNull() diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 58b3e8aedc..4514db578a 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -26,13 +26,7 @@ public: COutPoint(): n(NULL_INDEX) { } COutPoint(const uint256& hashIn, uint32_t nIn): hash(hashIn), n(nIn) { } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(hash); - READWRITE(n); - } + SERIALIZE_METHODS(COutPoint, obj) { READWRITE(obj.hash, obj.n); } void SetNull() { hash.SetNull(); n = NULL_INDEX; } bool IsNull() const { return (hash.IsNull() && n == NULL_INDEX); } @@ -103,14 +97,7 @@ public: explicit CTxIn(COutPoint prevoutIn, CScript scriptSigIn=CScript(), uint32_t nSequenceIn=SEQUENCE_FINAL); CTxIn(uint256 hashPrevTx, uint32_t nOut, CScript scriptSigIn=CScript(), uint32_t nSequenceIn=SEQUENCE_FINAL); - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(prevout); - READWRITE(scriptSig); - READWRITE(nSequence); - } + SERIALIZE_METHODS(CTxIn, obj) { READWRITE(obj.prevout, obj.scriptSig, obj.nSequence); } friend bool operator==(const CTxIn& a, const CTxIn& b) { @@ -143,13 +130,7 @@ public: CTxOut(const CAmount& nValueIn, CScript scriptPubKeyIn); - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nValue); - READWRITE(scriptPubKey); - } + SERIALIZE_METHODS(CTxOut, obj) { READWRITE(obj.nValue, obj.scriptPubKey); } void SetNull() { diff --git a/src/protocol.cpp b/src/protocol.cpp index a3e844e35b..2dfe4bee74 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -5,8 +5,8 @@ #include <protocol.h> -#include <util/system.h> #include <util/strencodings.h> +#include <util/system.h> #ifndef WIN32 # include <arpa/inet.h> @@ -40,6 +40,12 @@ const char *SENDCMPCT="sendcmpct"; const char *CMPCTBLOCK="cmpctblock"; const char *GETBLOCKTXN="getblocktxn"; const char *BLOCKTXN="blocktxn"; +const char *GETCFILTERS="getcfilters"; +const char *CFILTER="cfilter"; +const char *GETCFHEADERS="getcfheaders"; +const char *CFHEADERS="cfheaders"; +const char *GETCFCHECKPT="getcfcheckpt"; +const char *CFCHECKPT="cfcheckpt"; } // namespace NetMsgType /** All known message types. Keep this in the same order as the list of @@ -71,6 +77,12 @@ const static std::string allNetMessageTypes[] = { NetMsgType::CMPCTBLOCK, NetMsgType::GETBLOCKTXN, NetMsgType::BLOCKTXN, + NetMsgType::GETCFILTERS, + NetMsgType::CFILTER, + NetMsgType::GETCFHEADERS, + NetMsgType::CFHEADERS, + NetMsgType::GETCFCHECKPT, + NetMsgType::CFCHECKPT, }; const static std::vector<std::string> allNetMessageTypesVec(allNetMessageTypes, allNetMessageTypes+ARRAYLEN(allNetMessageTypes)); @@ -143,24 +155,6 @@ void SetServiceFlagsIBDCache(bool state) { g_initial_block_download_completed = state; } - -CAddress::CAddress() : CService() -{ - Init(); -} - -CAddress::CAddress(CService ipIn, ServiceFlags nServicesIn) : CService(ipIn) -{ - Init(); - nServices = nServicesIn; -} - -void CAddress::Init() -{ - nServices = NODE_NONE; - nTime = 100000000; -} - CInv::CInv() { type = 0; @@ -204,3 +198,42 @@ const std::vector<std::string> &getAllNetMessageTypes() { return allNetMessageTypesVec; } + +/** + * Convert a service flag (NODE_*) to a human readable string. + * It supports unknown service flags which will be returned as "UNKNOWN[...]". + * @param[in] bit the service flag is calculated as (1 << bit) + */ +static std::string serviceFlagToStr(size_t bit) +{ + const uint64_t service_flag = 1ULL << bit; + switch ((ServiceFlags)service_flag) { + case NODE_NONE: abort(); // impossible + case NODE_NETWORK: return "NETWORK"; + case NODE_GETUTXO: return "GETUTXO"; + case NODE_BLOOM: return "BLOOM"; + case NODE_WITNESS: return "WITNESS"; + case NODE_NETWORK_LIMITED: return "NETWORK_LIMITED"; + // Not using default, so we get warned when a case is missing + } + + std::ostringstream stream; + stream.imbue(std::locale::classic()); + stream << "UNKNOWN["; + stream << "2^" << bit; + stream << "]"; + return stream.str(); +} + +std::vector<std::string> serviceFlagsToStr(uint64_t flags) +{ + std::vector<std::string> str_flags; + + for (size_t i = 0; i < sizeof(flags) * 8; ++i) { + if (flags & (1ULL << i)) { + str_flags.emplace_back(serviceFlagToStr(i)); + } + } + + return str_flags; +} diff --git a/src/protocol.h b/src/protocol.h index 6639ae2aac..985f44640b 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -46,16 +46,7 @@ public: std::string GetCommand() const; bool IsValid(const MessageStartChars& messageStart) const; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(pchMessageStart); - READWRITE(pchCommand); - READWRITE(nMessageSize); - READWRITE(pchChecksum); - } + SERIALIZE_METHODS(CMessageHeader, obj) { READWRITE(obj.pchMessageStart, obj.pchCommand, obj.nMessageSize, obj.pchChecksum); } char pchMessageStart[MESSAGE_START_SIZE]; char pchCommand[COMMAND_SIZE]; @@ -74,100 +65,100 @@ namespace NetMsgType { * receiving node at the beginning of a connection. * @see https://bitcoin.org/en/developer-reference#version */ -extern const char *VERSION; +extern const char* VERSION; /** * The verack message acknowledges a previously-received version message, * informing the connecting node that it can begin to send other messages. * @see https://bitcoin.org/en/developer-reference#verack */ -extern const char *VERACK; +extern const char* VERACK; /** * The addr (IP address) message relays connection information for peers on the * network. * @see https://bitcoin.org/en/developer-reference#addr */ -extern const char *ADDR; +extern const char* ADDR; /** * The inv message (inventory message) transmits one or more inventories of * objects known to the transmitting peer. * @see https://bitcoin.org/en/developer-reference#inv */ -extern const char *INV; +extern const char* INV; /** * The getdata message requests one or more data objects from another node. * @see https://bitcoin.org/en/developer-reference#getdata */ -extern const char *GETDATA; +extern const char* GETDATA; /** * The merkleblock message is a reply to a getdata message which requested a * block using the inventory type MSG_MERKLEBLOCK. * @since protocol version 70001 as described by BIP37. * @see https://bitcoin.org/en/developer-reference#merkleblock */ -extern const char *MERKLEBLOCK; +extern const char* MERKLEBLOCK; /** * The getblocks message requests an inv message that provides block header * hashes starting from a particular point in the block chain. * @see https://bitcoin.org/en/developer-reference#getblocks */ -extern const char *GETBLOCKS; +extern const char* GETBLOCKS; /** * The getheaders message requests a headers message that provides block * headers starting from a particular point in the block chain. * @since protocol version 31800. * @see https://bitcoin.org/en/developer-reference#getheaders */ -extern const char *GETHEADERS; +extern const char* GETHEADERS; /** * The tx message transmits a single transaction. * @see https://bitcoin.org/en/developer-reference#tx */ -extern const char *TX; +extern const char* TX; /** * The headers message sends one or more block headers to a node which * previously requested certain headers with a getheaders message. * @since protocol version 31800. * @see https://bitcoin.org/en/developer-reference#headers */ -extern const char *HEADERS; +extern const char* HEADERS; /** * The block message transmits a single serialized block. * @see https://bitcoin.org/en/developer-reference#block */ -extern const char *BLOCK; +extern const char* BLOCK; /** * The getaddr message requests an addr message from the receiving node, * preferably one with lots of IP addresses of other receiving nodes. * @see https://bitcoin.org/en/developer-reference#getaddr */ -extern const char *GETADDR; +extern const char* GETADDR; /** * The mempool message requests the TXIDs of transactions that the receiving * node has verified as valid but which have not yet appeared in a block. * @since protocol version 60002. * @see https://bitcoin.org/en/developer-reference#mempool */ -extern const char *MEMPOOL; +extern const char* MEMPOOL; /** * The ping message is sent periodically to help confirm that the receiving * peer is still connected. * @see https://bitcoin.org/en/developer-reference#ping */ -extern const char *PING; +extern const char* PING; /** * The pong message replies to a ping message, proving to the pinging node that * the ponging node is still alive. * @since protocol version 60001 as described by BIP31. * @see https://bitcoin.org/en/developer-reference#pong */ -extern const char *PONG; +extern const char* PONG; /** * The notfound message is a reply to a getdata message which requested an * object the receiving node does not have available for relay. * @since protocol version 70001. * @see https://bitcoin.org/en/developer-reference#notfound */ -extern const char *NOTFOUND; +extern const char* NOTFOUND; /** * The filterload message tells the receiving peer to filter all relayed * transactions and requested merkle blocks through the provided filter. @@ -176,7 +167,7 @@ extern const char *NOTFOUND; * 70011 as described by BIP111. * @see https://bitcoin.org/en/developer-reference#filterload */ -extern const char *FILTERLOAD; +extern const char* FILTERLOAD; /** * The filteradd message tells the receiving peer to add a single element to a * previously-set bloom filter, such as a new public key. @@ -185,7 +176,7 @@ extern const char *FILTERLOAD; * 70011 as described by BIP111. * @see https://bitcoin.org/en/developer-reference#filteradd */ -extern const char *FILTERADD; +extern const char* FILTERADD; /** * The filterclear message tells the receiving peer to remove a previously-set * bloom filter. @@ -194,20 +185,20 @@ extern const char *FILTERADD; * 70011 as described by BIP111. * @see https://bitcoin.org/en/developer-reference#filterclear */ -extern const char *FILTERCLEAR; +extern const char* FILTERCLEAR; /** * Indicates that a node prefers to receive new block announcements via a * "headers" message rather than an "inv". * @since protocol version 70012 as described by BIP130. * @see https://bitcoin.org/en/developer-reference#sendheaders */ -extern const char *SENDHEADERS; +extern const char* SENDHEADERS; /** * The feefilter message tells the receiving peer not to inv us any txs * which do not meet the specified min fee rate. * @since protocol version 70013 as described by BIP133 */ -extern const char *FEEFILTER; +extern const char* FEEFILTER; /** * Contains a 1-byte bool and 8-byte LE version number. * Indicates that a node is willing to provide blocks via "cmpctblock" messages. @@ -215,33 +206,69 @@ extern const char *FEEFILTER; * "cmpctblock" message rather than an "inv", depending on message contents. * @since protocol version 70014 as described by BIP 152 */ -extern const char *SENDCMPCT; +extern const char* SENDCMPCT; /** * Contains a CBlockHeaderAndShortTxIDs object - providing a header and * list of "short txids". * @since protocol version 70014 as described by BIP 152 */ -extern const char *CMPCTBLOCK; +extern const char* CMPCTBLOCK; /** * Contains a BlockTransactionsRequest * Peer should respond with "blocktxn" message. * @since protocol version 70014 as described by BIP 152 */ -extern const char *GETBLOCKTXN; +extern const char* GETBLOCKTXN; /** * Contains a BlockTransactions. * Sent in response to a "getblocktxn" message. * @since protocol version 70014 as described by BIP 152 */ -extern const char *BLOCKTXN; -}; +extern const char* BLOCKTXN; +/** + * getcfilters requests compact filters for a range of blocks. + * Only available with service bit NODE_COMPACT_FILTERS as described by + * BIP 157 & 158. + */ +extern const char* GETCFILTERS; +/** + * cfilter is a response to a getcfilters request containing a single compact + * filter. + */ +extern const char* CFILTER; +/** + * getcfheaders requests a compact filter header and the filter hashes for a + * range of blocks, which can then be used to reconstruct the filter headers + * for those blocks. + * Only available with service bit NODE_COMPACT_FILTERS as described by + * BIP 157 & 158. + */ +extern const char* GETCFHEADERS; +/** + * cfheaders is a response to a getcfheaders request containing a filter header + * and a vector of filter hashes for each subsequent block in the requested range. + */ +extern const char* CFHEADERS; +/** + * getcfcheckpt requests evenly spaced compact filter headers, enabling + * parallelized download and validation of the headers between them. + * Only available with service bit NODE_COMPACT_FILTERS as described by + * BIP 157 & 158. + */ +extern const char* GETCFCHECKPT; +/** + * cfcheckpt is a response to a getcfcheckpt request containing a vector of + * evenly spaced filter headers for blocks on the requested chain. + */ +extern const char* CFCHECKPT; +}; // namespace NetMsgType /* Get a vector of all valid message types (see above) */ -const std::vector<std::string> &getAllNetMessageTypes(); +const std::vector<std::string>& getAllNetMessageTypes(); /** nServices flags */ enum ServiceFlags : uint64_t { - // NOTE: When adding here, be sure to update qt/guiutil.cpp's formatServicesStr too + // NOTE: When adding here, be sure to update serviceFlagToStr too // Nothing NODE_NONE = 0, // NODE_NETWORK means that the node is capable of serving the complete block chain. It is currently @@ -273,6 +300,13 @@ enum ServiceFlags : uint64_t { }; /** + * Convert service flags (a bitmask of NODE_*) to human readable strings. + * It supports unknown service flags which will be returned as "UNKNOWN[...]". + * @param[in] flags multiple NODE_* bitwise-OR-ed together + */ +std::vector<std::string> serviceFlagsToStr(uint64_t flags); + +/** * Gets the set of service flags which are "desirable" for a given peer. * * These are the flags which are required for a peer to support for them @@ -306,7 +340,8 @@ void SetServiceFlagsIBDCache(bool status); * == GetDesirableServiceFlags(services), ie determines whether the given * set of service flags are sufficient for a peer to be "relevant". */ -static inline bool HasAllDesirableServiceFlags(ServiceFlags services) { +static inline bool HasAllDesirableServiceFlags(ServiceFlags services) +{ return !(GetDesirableServiceFlags(services) & (~services)); } @@ -314,62 +349,55 @@ static inline bool HasAllDesirableServiceFlags(ServiceFlags services) { * Checks if a peer with the given service flags may be capable of having a * robust address-storage DB. */ -static inline bool MayHaveUsefulAddressDB(ServiceFlags services) { +static inline bool MayHaveUsefulAddressDB(ServiceFlags services) +{ return (services & NODE_NETWORK) || (services & NODE_NETWORK_LIMITED); } /** A CService with information about it as peer */ class CAddress : public CService { -public: - CAddress(); - explicit CAddress(CService ipIn, ServiceFlags nServicesIn); + static constexpr uint32_t TIME_INIT{100000000}; - void Init(); - - ADD_SERIALIZE_METHODS; +public: + CAddress() : CService{} {}; + explicit CAddress(CService ipIn, ServiceFlags nServicesIn) : CService{ipIn}, nServices{nServicesIn} {}; - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CAddress, obj) { - if (ser_action.ForRead()) - Init(); + SER_READ(obj, obj.nTime = TIME_INIT); int nVersion = s.GetVersion(); - if (s.GetType() & SER_DISK) + if (s.GetType() & SER_DISK) { READWRITE(nVersion); + } if ((s.GetType() & SER_DISK) || - (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) - READWRITE(nTime); - uint64_t nServicesInt = nServices; - READWRITE(nServicesInt); - nServices = static_cast<ServiceFlags>(nServicesInt); - READWRITEAS(CService, *this); + (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) { + READWRITE(obj.nTime); + } + READWRITE(Using<CustomUintFormatter<8>>(obj.nServices)); + READWRITEAS(CService, obj); } - // TODO: make private (improves encapsulation) -public: - ServiceFlags nServices; - + ServiceFlags nServices{NODE_NONE}; // disk and network only - unsigned int nTime; + uint32_t nTime{TIME_INIT}; }; /** getdata message type flags */ const uint32_t MSG_WITNESS_FLAG = 1 << 30; -const uint32_t MSG_TYPE_MASK = 0xffffffff >> 2; +const uint32_t MSG_TYPE_MASK = 0xffffffff >> 2; /** getdata / inv message types. * These numbers are defined by the protocol. When adding a new value, be sure * to mention it in the respective BIP. */ -enum GetDataMsg -{ +enum GetDataMsg { UNDEFINED = 0, MSG_TX = 1, MSG_BLOCK = 2, // The following can only occur in getdata. Invs always use TX or BLOCK. - MSG_FILTERED_BLOCK = 3, //!< Defined in BIP37 - MSG_CMPCT_BLOCK = 4, //!< Defined in BIP152 + MSG_FILTERED_BLOCK = 3, //!< Defined in BIP37 + MSG_CMPCT_BLOCK = 4, //!< Defined in BIP152 MSG_WITNESS_BLOCK = MSG_BLOCK | MSG_WITNESS_FLAG, //!< Defined in BIP144 MSG_WITNESS_TX = MSG_TX | MSG_WITNESS_FLAG, //!< Defined in BIP144 MSG_FILTERED_WITNESS_BLOCK = MSG_FILTERED_BLOCK | MSG_WITNESS_FLAG, @@ -382,21 +410,13 @@ public: CInv(); CInv(int typeIn, const uint256& hashIn); - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(type); - READWRITE(hash); - } + SERIALIZE_METHODS(CInv, obj) { READWRITE(obj.type, obj.hash); } friend bool operator<(const CInv& a, const CInv& b); std::string GetCommand() const; std::string ToString() const; -public: int type; uint256 hash; }; diff --git a/src/psbt.h b/src/psbt.h index af57994f3a..888e0fd119 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -41,7 +41,7 @@ static constexpr uint8_t PSBT_OUT_BIP32_DERIVATION = 0x02; static constexpr uint8_t PSBT_SEPARATOR = 0x00; // BIP 174 does not specify a maximum file size, but we set a limit anyway -// to prevent reading a stream indefinately and running out of memory. +// to prevent reading a stream indefinitely and running out of memory. const std::streamsize MAX_FILE_SIZE_PSBT = 100000000; // 100 MiB /** A structure for PSBTs which contain per-input information */ diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index f48f28d03a..aa4ec04497 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -35,7 +35,7 @@ public: } protected: - bool filterAcceptsRow(int row, const QModelIndex& parent) const + bool filterAcceptsRow(int row, const QModelIndex& parent) const override { auto model = sourceModel(); auto label = model->index(row, AddressTableModel::Label, parent); @@ -106,7 +106,7 @@ AddressBookPage::AddressBookPage(const PlatformStyle *platformStyle, Mode _mode, ui->newAddress->setVisible(true); break; case ReceivingTab: - ui->labelExplanation->setText(tr("These are your Bitcoin addresses for receiving payments. Use the 'Create new receiving address' button in the receive tab to create new addresses.")); + ui->labelExplanation->setText(tr("These are your Bitcoin addresses for receiving payments. Use the 'Create new receiving address' button in the receive tab to create new addresses.\nSigning is only possible with addresses of the type 'legacy'.")); ui->deleteAddress->setVisible(false); ui->newAddress->setVisible(false); break; diff --git a/src/qt/addressbookpage.h b/src/qt/addressbookpage.h index 6394d26801..3d303a6f68 100644 --- a/src/qt/addressbookpage.h +++ b/src/qt/addressbookpage.h @@ -45,7 +45,7 @@ public: const QString &getReturnValue() const { return returnValue; } public Q_SLOTS: - void done(int retval); + void done(int retval) override; private: Ui::AddressBookPage *ui; diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 8110f4e895..665c8e6053 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -11,6 +11,7 @@ #include <wallet/wallet.h> #include <algorithm> +#include <typeinfo> #include <QFont> #include <QDebug> @@ -75,12 +76,14 @@ public: explicit AddressTablePriv(AddressTableModel *_parent): parent(_parent) {} - void refreshAddressTable(interfaces::Wallet& wallet) + void refreshAddressTable(interfaces::Wallet& wallet, bool pk_hash_only = false) { cachedAddressTable.clear(); { for (const auto& address : wallet.getAddresses()) { + if (pk_hash_only && address.dest.type() != typeid(PKHash)) + continue; AddressTableEntry::Type addressType = translateTransactionType( QString::fromStdString(address.purpose), address.is_mine); cachedAddressTable.append(AddressTableEntry(addressType, @@ -159,12 +162,12 @@ public: } }; -AddressTableModel::AddressTableModel(WalletModel *parent) : +AddressTableModel::AddressTableModel(WalletModel *parent, bool pk_hash_only) : QAbstractTableModel(parent), walletModel(parent) { columns << tr("Label") << tr("Address"); priv = new AddressTablePriv(this); - priv->refreshAddressTable(parent->wallet()); + priv->refreshAddressTable(parent->wallet(), pk_hash_only); } AddressTableModel::~AddressTableModel() diff --git a/src/qt/addresstablemodel.h b/src/qt/addresstablemodel.h index 035f3e0571..73316cadc4 100644 --- a/src/qt/addresstablemodel.h +++ b/src/qt/addresstablemodel.h @@ -25,7 +25,7 @@ class AddressTableModel : public QAbstractTableModel Q_OBJECT public: - explicit AddressTableModel(WalletModel *parent = nullptr); + explicit AddressTableModel(WalletModel *parent = nullptr, bool pk_hash_only = false); ~AddressTableModel(); enum ColumnIndex { @@ -52,14 +52,14 @@ public: /** @name Methods overridden from QAbstractTableModel @{*/ - int rowCount(const QModelIndex &parent) const; - int columnCount(const QModelIndex &parent) const; - QVariant data(const QModelIndex &index, int role) const; - bool setData(const QModelIndex &index, const QVariant &value, int role); - QVariant headerData(int section, Qt::Orientation orientation, int role) const; - QModelIndex index(int row, int column, const QModelIndex &parent) const; - bool removeRows(int row, int count, const QModelIndex &parent = QModelIndex()); - Qt::ItemFlags flags(const QModelIndex &index) const; + int rowCount(const QModelIndex &parent) const override; + int columnCount(const QModelIndex &parent) const override; + QVariant data(const QModelIndex &index, int role) const override; + bool setData(const QModelIndex &index, const QVariant &value, int role) override; + QVariant headerData(int section, Qt::Orientation orientation, int role) const override; + QModelIndex index(int row, int column, const QModelIndex &parent) const override; + bool removeRows(int row, int count, const QModelIndex &parent = QModelIndex()) override; + Qt::ItemFlags flags(const QModelIndex &index) const override; /*@}*/ /* Add an address to the model. diff --git a/src/qt/askpassphrasedialog.h b/src/qt/askpassphrasedialog.h index 20fc5045ae..9557e72936 100644 --- a/src/qt/askpassphrasedialog.h +++ b/src/qt/askpassphrasedialog.h @@ -32,7 +32,7 @@ public: explicit AskPassphraseDialog(Mode mode, QWidget *parent, SecureString* passphrase_out = nullptr); ~AskPassphraseDialog(); - void accept(); + void accept() override; void setModel(WalletModel *model); @@ -49,8 +49,8 @@ private Q_SLOTS: void toggleShowPassword(bool); protected: - bool event(QEvent *event); - bool eventFilter(QObject *object, QEvent *event); + bool event(QEvent *event) override; + bool eventFilter(QObject *object, QEvent *event) override; }; #endif // BITCOIN_QT_ASKPASSPHRASEDIALOG_H diff --git a/src/qt/bantablemodel.h b/src/qt/bantablemodel.h index 88d5b5811b..57f559fc14 100644 --- a/src/qt/bantablemodel.h +++ b/src/qt/bantablemodel.h @@ -56,16 +56,17 @@ public: /** @name Methods overridden from QAbstractTableModel @{*/ - int rowCount(const QModelIndex &parent) const; - int columnCount(const QModelIndex &parent) const; - QVariant data(const QModelIndex &index, int role) const; - QVariant headerData(int section, Qt::Orientation orientation, int role) const; - QModelIndex index(int row, int column, const QModelIndex &parent) const; - Qt::ItemFlags flags(const QModelIndex &index) const; - void sort(int column, Qt::SortOrder order); - bool shouldShow(); + int rowCount(const QModelIndex &parent) const override; + int columnCount(const QModelIndex &parent) const override; + QVariant data(const QModelIndex &index, int role) const override; + QVariant headerData(int section, Qt::Orientation orientation, int role) const override; + QModelIndex index(int row, int column, const QModelIndex &parent) const override; + Qt::ItemFlags flags(const QModelIndex &index) const override; + void sort(int column, Qt::SortOrder order) override; /*@}*/ + bool shouldShow(); + public Q_SLOTS: void refresh(); diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 8939b566f7..f8cdb5df23 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -34,6 +34,7 @@ #include <uint256.h> #include <util/system.h> #include <util/threadnames.h> +#include <validation.h> #include <memory> @@ -61,8 +62,26 @@ Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin); // Declare meta types used for QMetaObject::invokeMethod Q_DECLARE_METATYPE(bool*) Q_DECLARE_METATYPE(CAmount) +Q_DECLARE_METATYPE(SynchronizationState) Q_DECLARE_METATYPE(uint256) +static void RegisterMetaTypes() +{ + // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection + qRegisterMetaType<bool*>(); + qRegisterMetaType<SynchronizationState>(); + #ifdef ENABLE_WALLET + qRegisterMetaType<WalletModel*>(); + #endif + // Register typedefs (see http://qt-project.org/doc/qt-5/qmetatype.html#qRegisterMetaType) + // IMPORTANT: if CAmount is no longer a typedef use the normal variant above (see https://doc.qt.io/qt-5/qmetatype.html#qRegisterMetaType-1) + qRegisterMetaType<CAmount>("CAmount"); + qRegisterMetaType<size_t>("size_t"); + + qRegisterMetaType<std::function<void()>>("std::function<void()>"); + qRegisterMetaType<QMessageBox::Icon>("QMessageBox::Icon"); +} + static QString GetLangTerritory() { QSettings settings; @@ -182,6 +201,7 @@ BitcoinApplication::BitcoinApplication(interfaces::Node& node): returnValue(0), platformStyle(nullptr) { + RegisterMetaTypes(); setQuitOnLastWindowClosed(false); } @@ -210,8 +230,6 @@ BitcoinApplication::~BitcoinApplication() delete window; window = nullptr; - delete optionsModel; - optionsModel = nullptr; delete platformStyle; platformStyle = nullptr; } @@ -225,7 +243,7 @@ void BitcoinApplication::createPaymentServer() void BitcoinApplication::createOptionsModel(bool resetSettings) { - optionsModel = new OptionsModel(m_node, nullptr, resetSettings); + optionsModel = new OptionsModel(m_node, this, resetSettings); } void BitcoinApplication::createWindow(const NetworkStyle *networkStyle) @@ -433,19 +451,6 @@ int GuiMain(int argc, char* argv[]) BitcoinApplication app(*node); - // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection - qRegisterMetaType<bool*>(); -#ifdef ENABLE_WALLET - qRegisterMetaType<WalletModel*>(); -#endif - // Register typedefs (see http://qt-project.org/doc/qt-5/qmetatype.html#qRegisterMetaType) - // IMPORTANT: if CAmount is no longer a typedef use the normal variant above (see https://doc.qt.io/qt-5/qmetatype.html#qRegisterMetaType-1) - qRegisterMetaType<CAmount>("CAmount"); - qRegisterMetaType<size_t>("size_t"); - - qRegisterMetaType<std::function<void()>>("std::function<void()>"); - qRegisterMetaType<QMessageBox::Icon>("QMessageBox::Icon"); - /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these // Command-line options take precedence: node->setupServerArgs(); diff --git a/src/qt/bitcoinaddressvalidator.h b/src/qt/bitcoinaddressvalidator.h index 30d4a26d0e..52c06828a3 100644 --- a/src/qt/bitcoinaddressvalidator.h +++ b/src/qt/bitcoinaddressvalidator.h @@ -17,7 +17,7 @@ class BitcoinAddressEntryValidator : public QValidator public: explicit BitcoinAddressEntryValidator(QObject *parent); - State validate(QString &input, int &pos) const; + State validate(QString &input, int &pos) const override; }; /** Bitcoin address widget validator, checks for a valid bitcoin address. @@ -29,7 +29,7 @@ class BitcoinAddressCheckValidator : public QValidator public: explicit BitcoinAddressCheckValidator(QObject *parent); - State validate(QString &input, int &pos) const; + State validate(QString &input, int &pos) const override; }; #endif // BITCOIN_QT_BITCOINADDRESSVALIDATOR_H diff --git a/src/qt/bitcoinamountfield.cpp b/src/qt/bitcoinamountfield.cpp index 7acc82370f..4c57f1e352 100644 --- a/src/qt/bitcoinamountfield.cpp +++ b/src/qt/bitcoinamountfield.cpp @@ -31,7 +31,7 @@ public: connect(lineEdit(), &QLineEdit::textEdited, this, &AmountSpinBox::valueChanged); } - QValidator::State validate(QString &text, int &pos) const + QValidator::State validate(QString &text, int &pos) const override { if(text.isEmpty()) return QValidator::Intermediate; @@ -41,7 +41,7 @@ public: return valid ? QValidator::Intermediate : QValidator::Invalid; } - void fixup(QString &input) const + void fixup(QString &input) const override { bool valid; CAmount val; @@ -87,7 +87,7 @@ public: m_max_amount = value; } - void stepBy(int steps) + void stepBy(int steps) override { bool valid = false; CAmount val = value(&valid); @@ -114,7 +114,7 @@ public: singleStep = step; } - QSize minimumSizeHint() const + QSize minimumSizeHint() const override { if(cachedMinimumSizeHint.isEmpty()) { @@ -175,7 +175,7 @@ private: } protected: - bool event(QEvent *event) + bool event(QEvent *event) override { if (event->type() == QEvent::KeyPress || event->type() == QEvent::KeyRelease) { @@ -190,7 +190,7 @@ protected: return QAbstractSpinBox::event(event); } - StepEnabled stepEnabled() const + StepEnabled stepEnabled() const override { if (isReadOnly()) // Disable steps when AmountSpinBox is read-only return StepNone; diff --git a/src/qt/bitcoinamountfield.h b/src/qt/bitcoinamountfield.h index 2db6b65f2c..d3e61aac29 100644 --- a/src/qt/bitcoinamountfield.h +++ b/src/qt/bitcoinamountfield.h @@ -70,7 +70,7 @@ Q_SIGNALS: protected: /** Intercept focus-in event and ',' key presses */ - bool eventFilter(QObject *object, QEvent *event); + bool eventFilter(QObject *object, QEvent *event) override; private: AmountSpinBox *amount; diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index cf2de1a417..195d80d47c 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -36,6 +36,8 @@ #include <interfaces/node.h> #include <ui_interface.h> #include <util/system.h> +#include <util/translation.h> +#include <validation.h> #include <QAction> #include <QApplication> @@ -348,10 +350,18 @@ void BitcoinGUI::createActions() m_create_wallet_action->setEnabled(false); m_create_wallet_action->setStatusTip(tr("Create a new wallet")); + m_close_all_wallets_action = new QAction(tr("Close All Wallets..."), this); + m_close_all_wallets_action->setStatusTip(tr("Close all wallets")); + showHelpMessageAction = new QAction(tr("&Command-line options"), this); showHelpMessageAction->setMenuRole(QAction::NoRole); showHelpMessageAction->setStatusTip(tr("Show the %1 help message to get a list with possible Bitcoin command-line options").arg(PACKAGE_NAME)); + m_mask_values_action = new QAction(tr("&Mask values"), this); + m_mask_values_action->setShortcut(QKeySequence(Qt::CTRL + Qt::SHIFT + Qt::Key_M)); + m_mask_values_action->setStatusTip(tr("Mask the values in the Overview tab")); + m_mask_values_action->setCheckable(true); + connect(quitAction, &QAction::triggered, qApp, QApplication::quit); connect(aboutAction, &QAction::triggered, this, &BitcoinGUI::aboutClicked); connect(aboutQtAction, &QAction::triggered, qApp, QApplication::aboutQt); @@ -414,6 +424,10 @@ void BitcoinGUI::createActions() connect(activity, &CreateWalletActivity::finished, activity, &QObject::deleteLater); activity->create(); }); + connect(m_close_all_wallets_action, &QAction::triggered, [this] { + m_wallet_controller->closeAllWallets(this); + }); + connect(m_mask_values_action, &QAction::toggled, this, &BitcoinGUI::setPrivacy); } #endif // ENABLE_WALLET @@ -438,6 +452,7 @@ void BitcoinGUI::createMenuBar() file->addAction(m_create_wallet_action); file->addAction(m_open_wallet_action); file->addAction(m_close_wallet_action); + file->addAction(m_close_all_wallets_action); file->addSeparator(); file->addAction(openAction); file->addAction(backupWalletAction); @@ -454,6 +469,8 @@ void BitcoinGUI::createMenuBar() settings->addAction(encryptWalletAction); settings->addAction(changePassphraseAction); settings->addSeparator(); + settings->addAction(m_mask_values_action); + settings->addSeparator(); } settings->addAction(optionsAction); @@ -566,7 +583,7 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) connect(_clientModel, &ClientModel::networkActiveChanged, this, &BitcoinGUI::setNetworkActive); modalOverlay->setKnownBestHeight(_clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(_clientModel->getHeaderTipTime())); - setNumBlocks(m_node.getNumBlocks(), QDateTime::fromTime_t(m_node.getLastBlockTime()), m_node.getVerificationProgress(), false); + setNumBlocks(m_node.getNumBlocks(), QDateTime::fromTime_t(m_node.getLastBlockTime()), m_node.getVerificationProgress(), false, SynchronizationState::INIT_DOWNLOAD); connect(_clientModel, &ClientModel::numBlocksChanged, this, &BitcoinGUI::setNumBlocks); // Receive and report messages from client model @@ -716,6 +733,7 @@ void BitcoinGUI::setWalletActionsEnabled(bool enabled) usedReceivingAddressesAction->setEnabled(enabled); openAction->setEnabled(enabled); m_close_wallet_action->setEnabled(enabled); + m_close_all_wallets_action->setEnabled(enabled); } void BitcoinGUI::createTrayIcon() @@ -925,11 +943,15 @@ void BitcoinGUI::openOptionsDialogWithTab(OptionsDialog::Tab tab) dlg.exec(); } -void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool header) +void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool header, SynchronizationState sync_state) { // Disabling macOS App Nap on initial sync, disk and reindex operations. #ifdef Q_OS_MAC - (m_node.isInitialBlockDownload() || m_node.getReindex() || m_node.getImporting()) ? m_app_nap_inhibitor->disableAppNap() : m_app_nap_inhibitor->enableAppNap(); + if (sync_state == SynchronizationState::POST_INIT) { + m_app_nap_inhibitor->enableAppNap(); + } else { + m_app_nap_inhibitor->disableAppNap(); + } #endif if (modalOverlay) @@ -1039,7 +1061,7 @@ void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVer progressBar->setToolTip(tooltip); } -void BitcoinGUI::message(const QString& title, QString message, unsigned int style, bool* ret) +void BitcoinGUI::message(const QString& title, QString message, unsigned int style, bool* ret, const QString& detailed_message) { // Default title. On macOS, the window title is ignored (as required by the macOS Guidelines). QString strTitle{PACKAGE_NAME}; @@ -1093,6 +1115,7 @@ void BitcoinGUI::message(const QString& title, QString message, unsigned int sty showNormalIfMinimized(); QMessageBox mBox(static_cast<QMessageBox::Icon>(nMBoxIcon), strTitle, message, buttons, this); mBox.setTextFormat(Qt::PlainText); + mBox.setDetailedText(detailed_message); int r = mBox.exec(); if (ret != nullptr) *ret = r == QMessageBox::Ok; @@ -1244,7 +1267,7 @@ void BitcoinGUI::setEncryptionStatus(int status) labelWalletEncryptionIcon->setToolTip(tr("Wallet is <b>encrypted</b> and currently <b>unlocked</b>")); encryptWalletAction->setChecked(true); changePassphraseAction->setEnabled(true); - encryptWalletAction->setEnabled(false); // TODO: decrypt currently not supported + encryptWalletAction->setEnabled(false); break; case WalletModel::Locked: labelWalletEncryptionIcon->show(); @@ -1252,7 +1275,7 @@ void BitcoinGUI::setEncryptionStatus(int status) labelWalletEncryptionIcon->setToolTip(tr("Wallet is <b>encrypted</b> and currently <b>locked</b>")); encryptWalletAction->setChecked(true); changePassphraseAction->setEnabled(true); - encryptWalletAction->setEnabled(false); // TODO: decrypt currently not supported + encryptWalletAction->setEnabled(false); break; } } @@ -1368,20 +1391,27 @@ void BitcoinGUI::showModalOverlay() modalOverlay->toggleVisibility(); } -static bool ThreadSafeMessageBox(BitcoinGUI* gui, const std::string& message, const std::string& caption, unsigned int style) +static bool ThreadSafeMessageBox(BitcoinGUI* gui, const bilingual_str& message, const std::string& caption, unsigned int style) { bool modal = (style & CClientUIInterface::MODAL); // The SECURE flag has no effect in the Qt GUI. // bool secure = (style & CClientUIInterface::SECURE); style &= ~CClientUIInterface::SECURE; bool ret = false; + + QString detailed_message; // This is original message, in English, for googling and referencing. + if (message.original != message.translated) { + detailed_message = BitcoinGUI::tr("Original message:") + "\n" + QString::fromStdString(message.original); + } + // In case of modal message, use blocking connection to wait for user to click a button bool invoked = QMetaObject::invokeMethod(gui, "message", modal ? GUIUtil::blockingGUIThreadConnection() : Qt::QueuedConnection, Q_ARG(QString, QString::fromStdString(caption)), - Q_ARG(QString, QString::fromStdString(message)), + Q_ARG(QString, QString::fromStdString(message.translated)), Q_ARG(unsigned int, style), - Q_ARG(bool*, &ret)); + Q_ARG(bool*, &ret), + Q_ARG(QString, detailed_message)); assert(invoked); return ret; } @@ -1400,6 +1430,12 @@ void BitcoinGUI::unsubscribeFromCoreSignals() m_handler_question->disconnect(); } +bool BitcoinGUI::isPrivacyModeActivated() const +{ + assert(m_mask_values_action); + return m_mask_values_action->isChecked(); +} + UnitDisplayStatusBarControl::UnitDisplayStatusBarControl(const PlatformStyle *platformStyle) : optionsModel(nullptr), menu(nullptr) diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 70366e12d1..b009e279b6 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -38,6 +38,7 @@ class WalletFrame; class WalletModel; class HelpMessageDialog; class ModalOverlay; +enum class SynchronizationState; namespace interfaces { class Handler; @@ -98,13 +99,15 @@ public: /** Disconnect core signals from GUI client */ void unsubscribeFromCoreSignals(); + bool isPrivacyModeActivated() const; + protected: - void changeEvent(QEvent *e); - void closeEvent(QCloseEvent *event); - void showEvent(QShowEvent *event); - void dragEnterEvent(QDragEnterEvent *event); - void dropEvent(QDropEvent *event); - bool eventFilter(QObject *object, QEvent *event); + void changeEvent(QEvent *e) override; + void closeEvent(QCloseEvent *event) override; + void showEvent(QShowEvent *event) override; + void dragEnterEvent(QDragEnterEvent *event) override; + void dropEvent(QDropEvent *event) override; + bool eventFilter(QObject *object, QEvent *event) override; private: interfaces::Node& m_node; @@ -152,8 +155,10 @@ private: QAction* m_open_wallet_action{nullptr}; QMenu* m_open_wallet_menu{nullptr}; QAction* m_close_wallet_action{nullptr}; + QAction* m_close_all_wallets_action{nullptr}; QAction* m_wallet_selector_label_action = nullptr; QAction* m_wallet_selector_action = nullptr; + QAction* m_mask_values_action{nullptr}; QLabel *m_wallet_selector_label = nullptr; QComboBox* m_wallet_selector = nullptr; @@ -206,6 +211,7 @@ Q_SIGNALS: void receivedURI(const QString &uri); /** Signal raised when RPC console shown */ void consoleShown(RPCConsole* console); + void setPrivacy(bool privacy); public Q_SLOTS: /** Set number of connections shown in the UI */ @@ -213,16 +219,17 @@ public Q_SLOTS: /** Set network state shown in the UI */ void setNetworkActive(bool networkActive); /** Set number of blocks and last block date shown in the UI */ - void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers); + void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers, SynchronizationState sync_state); /** Notify the user of an event from the core network or transaction handling code. - @param[in] title the message box / notification title - @param[in] message the displayed text - @param[in] style modality and style definitions (icon and used buttons - buttons only for message boxes) - @see CClientUIInterface::MessageBoxFlags - @param[in] ret pointer to a bool that will be modified to whether Ok was clicked (modal only) + @param[in] title the message box / notification title + @param[in] message the displayed text + @param[in] style modality and style definitions (icon and used buttons - buttons only for message boxes) + @see CClientUIInterface::MessageBoxFlags + @param[in] ret pointer to a bool that will be modified to whether Ok was clicked (modal only) + @param[in] detailed_message the text to be displayed in the details area */ - void message(const QString& title, QString message, unsigned int style, bool* ret = nullptr); + void message(const QString& title, QString message, unsigned int style, bool* ret = nullptr, const QString& detailed_message = QString()); #ifdef ENABLE_WALLET void setCurrentWallet(WalletModel* wallet_model); @@ -324,7 +331,7 @@ public: protected: /** So that it responds to left-button clicks */ - void mousePressEvent(QMouseEvent *event); + void mousePressEvent(QMouseEvent *event) override; private: OptionsModel *optionsModel; diff --git a/src/qt/bitcoinunits.cpp b/src/qt/bitcoinunits.cpp index d9711af123..318a6dcbfd 100644 --- a/src/qt/bitcoinunits.cpp +++ b/src/qt/bitcoinunits.cpp @@ -6,6 +6,8 @@ #include <QStringList> +#include <cassert> + BitcoinUnits::BitcoinUnits(QObject *parent): QAbstractListModel(parent), unitlist(availableUnits()) @@ -94,7 +96,7 @@ int BitcoinUnits::decimals(int unit) } } -QString BitcoinUnits::format(int unit, const CAmount& nIn, bool fPlus, SeparatorStyle separators) +QString BitcoinUnits::format(int unit, const CAmount& nIn, bool fPlus, SeparatorStyle separators, bool justify) { // Note: not using straight sprintf here because we do NOT want // localized number formatting. @@ -106,6 +108,7 @@ QString BitcoinUnits::format(int unit, const CAmount& nIn, bool fPlus, Separator qint64 n_abs = (n > 0 ? n : -n); qint64 quotient = n_abs / coin; QString quotient_str = QString::number(quotient); + if (justify) quotient_str = quotient_str.rightJustified(16 - num_decimals, ' '); // Use SI-style thin space separators as these are locale independent and can't be // confused with the decimal marker. @@ -150,6 +153,17 @@ QString BitcoinUnits::formatHtmlWithUnit(int unit, const CAmount& amount, bool p return QString("<span style='white-space: nowrap;'>%1</span>").arg(str); } +QString BitcoinUnits::formatWithPrivacy(int unit, const CAmount& amount, SeparatorStyle separators, bool privacy) +{ + assert(amount >= 0); + QString value; + if (privacy) { + value = format(unit, 0, false, separators, true).replace('0', '#'); + } else { + value = format(unit, amount, false, separators, true); + } + return value + QString(" ") + shortName(unit); +} bool BitcoinUnits::parse(int unit, const QString &value, CAmount *val_out) { diff --git a/src/qt/bitcoinunits.h b/src/qt/bitcoinunits.h index 4c8a889965..dac5484393 100644 --- a/src/qt/bitcoinunits.h +++ b/src/qt/bitcoinunits.h @@ -72,11 +72,13 @@ public: //! Number of decimals left static int decimals(int unit); //! Format as string - static QString format(int unit, const CAmount& amount, bool plussign=false, SeparatorStyle separators=separatorStandard); + static QString format(int unit, const CAmount& amount, bool plussign = false, SeparatorStyle separators = separatorStandard, bool justify = false); //! Format as string (with unit) static QString formatWithUnit(int unit, const CAmount& amount, bool plussign=false, SeparatorStyle separators=separatorStandard); //! Format as HTML string (with unit) static QString formatHtmlWithUnit(int unit, const CAmount& amount, bool plussign=false, SeparatorStyle separators=separatorStandard); + //! Format as string (with unit) of fixed length to preserve privacy, if it is set. + static QString formatWithPrivacy(int unit, const CAmount& amount, SeparatorStyle separators, bool privacy); //! Parse string to coin amount static bool parse(int unit, const QString &value, CAmount *val_out); //! Gets title for amount column including current display unit if optionsModel reference available */ @@ -90,8 +92,8 @@ public: /** Unit identifier */ UnitRole = Qt::UserRole }; - int rowCount(const QModelIndex &parent) const; - QVariant data(const QModelIndex &index, int role) const; + int rowCount(const QModelIndex &parent) const override; + QVariant data(const QModelIndex &index, int role) const override; ///@} static QString removeSpaces(QString text) diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index b94fcc9865..c1d358f7f6 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -15,6 +15,7 @@ #include <net.h> #include <netbase.h> #include <util/system.h> +#include <validation.h> #include <stdint.h> @@ -113,6 +114,29 @@ int ClientModel::getNumBlocks() const return m_cached_num_blocks; } +uint256 ClientModel::getBestBlockHash() +{ + uint256 tip{WITH_LOCK(m_cached_tip_mutex, return m_cached_tip_blocks)}; + + if (!tip.IsNull()) { + return tip; + } + + // Lock order must be: first `cs_main`, then `m_cached_tip_mutex`. + // The following will lock `cs_main` (and release it), so we must not + // own `m_cached_tip_mutex` here. + tip = m_node.getBestBlockHash(); + + LOCK(m_cached_tip_mutex); + // We checked that `m_cached_tip_blocks` is not null above, but then we + // released the mutex `m_cached_tip_mutex`, so it could have changed in the + // meantime. Thus, check again. + if (m_cached_tip_blocks.IsNull()) { + m_cached_tip_blocks = tip; + } + return m_cached_tip_blocks; +} + void ClientModel::updateNumConnections(int numConnections) { Q_EMIT numConnectionsChanged(numConnections); @@ -234,36 +258,33 @@ static void BannedListChanged(ClientModel *clientmodel) assert(invoked); } -static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, int height, int64_t blockTime, double verificationProgress, bool fHeader) +static void BlockTipChanged(ClientModel* clientmodel, SynchronizationState sync_state, interfaces::BlockTip tip, double verificationProgress, bool fHeader) { - // lock free async UI updates in case we have a new block tip - // during initial sync, only update the UI if the last update - // was > 250ms (MODEL_UPDATE_DELAY) ago - int64_t now = 0; - if (initialSync) - now = GetTimeMillis(); - - int64_t& nLastUpdateNotification = fHeader ? nLastHeaderTipUpdateNotification : nLastBlockTipUpdateNotification; - if (fHeader) { // cache best headers time and height to reduce future cs_main locks - clientmodel->cachedBestHeaderHeight = height; - clientmodel->cachedBestHeaderTime = blockTime; + clientmodel->cachedBestHeaderHeight = tip.block_height; + clientmodel->cachedBestHeaderTime = tip.block_time; } else { - clientmodel->m_cached_num_blocks = height; + clientmodel->m_cached_num_blocks = tip.block_height; + WITH_LOCK(clientmodel->m_cached_tip_mutex, clientmodel->m_cached_tip_blocks = tip.block_hash;); } - // During initial sync, block notifications, and header notifications from reindexing are both throttled. - if (!initialSync || (fHeader && !clientmodel->node().getReindex()) || now - nLastUpdateNotification > MODEL_UPDATE_DELAY) { - //pass an async signal to the UI thread - bool invoked = QMetaObject::invokeMethod(clientmodel, "numBlocksChanged", Qt::QueuedConnection, - Q_ARG(int, height), - Q_ARG(QDateTime, QDateTime::fromTime_t(blockTime)), - Q_ARG(double, verificationProgress), - Q_ARG(bool, fHeader)); - assert(invoked); - nLastUpdateNotification = now; + // Throttle GUI notifications about (a) blocks during initial sync, and (b) both blocks and headers during reindex. + const bool throttle = (sync_state != SynchronizationState::POST_INIT && !fHeader) || sync_state == SynchronizationState::INIT_REINDEX; + const int64_t now = throttle ? GetTimeMillis() : 0; + int64_t& nLastUpdateNotification = fHeader ? nLastHeaderTipUpdateNotification : nLastBlockTipUpdateNotification; + if (throttle && now < nLastUpdateNotification + MODEL_UPDATE_DELAY) { + return; } + + bool invoked = QMetaObject::invokeMethod(clientmodel, "numBlocksChanged", Qt::QueuedConnection, + Q_ARG(int, tip.block_height), + Q_ARG(QDateTime, QDateTime::fromTime_t(tip.block_time)), + Q_ARG(double, verificationProgress), + Q_ARG(bool, fHeader), + Q_ARG(SynchronizationState, sync_state)); + assert(invoked); + nLastUpdateNotification = now; } void ClientModel::subscribeToCoreSignals() @@ -274,8 +295,8 @@ void ClientModel::subscribeToCoreSignals() m_handler_notify_network_active_changed = m_node.handleNotifyNetworkActiveChanged(std::bind(NotifyNetworkActiveChanged, this, std::placeholders::_1)); m_handler_notify_alert_changed = m_node.handleNotifyAlertChanged(std::bind(NotifyAlertChanged, this)); m_handler_banned_list_changed = m_node.handleBannedListChanged(std::bind(BannedListChanged, this)); - m_handler_notify_block_tip = m_node.handleNotifyBlockTip(std::bind(BlockTipChanged, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4, false)); - m_handler_notify_header_tip = m_node.handleNotifyHeaderTip(std::bind(BlockTipChanged, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4, true)); + m_handler_notify_block_tip = m_node.handleNotifyBlockTip(std::bind(BlockTipChanged, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, false)); + m_handler_notify_header_tip = m_node.handleNotifyHeaderTip(std::bind(BlockTipChanged, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, true)); } void ClientModel::unsubscribeFromCoreSignals() diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 7ac4120a8f..7f12cce1d9 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -10,12 +10,14 @@ #include <atomic> #include <memory> +#include <sync.h> +#include <uint256.h> class BanTableModel; +class CBlockIndex; class OptionsModel; class PeerTableModel; - -class CBlockIndex; +enum class SynchronizationState; namespace interfaces { class Handler; @@ -57,6 +59,7 @@ public: //! Return number of connections, default is in- and outbound (total) int getNumConnections(unsigned int flags = CONNECTIONS_ALL) const; int getNumBlocks() const; + uint256 getBestBlockHash(); int getHeaderTipHeight() const; int64_t getHeaderTipTime() const; @@ -74,11 +77,14 @@ public: bool getProxyInfo(std::string& ip_port) const; - // caches for the best header, number of blocks + // caches for the best header: hash, number of blocks and block time mutable std::atomic<int> cachedBestHeaderHeight; mutable std::atomic<int64_t> cachedBestHeaderTime; mutable std::atomic<int> m_cached_num_blocks{-1}; + Mutex m_cached_tip_mutex; + uint256 m_cached_tip_blocks GUARDED_BY(m_cached_tip_mutex){}; + private: interfaces::Node& m_node; std::unique_ptr<interfaces::Handler> m_handler_show_progress; @@ -100,7 +106,7 @@ private: Q_SIGNALS: void numConnectionsChanged(int count); - void numBlocksChanged(int count, const QDateTime& blockDate, double nVerificationProgress, bool header); + void numBlocksChanged(int count, const QDateTime& blockDate, double nVerificationProgress, bool header, SynchronizationState sync_state); void mempoolSizeChanged(long count, size_t mempoolSizeInBytes); void networkActiveChanged(bool networkActive); void alertsChanged(const QString &warnings); diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index b46f416b6c..db77c17df0 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -41,10 +41,11 @@ bool CCoinControlWidgetItem::operator<(const QTreeWidgetItem &other) const { return QTreeWidgetItem::operator<(other); } -CoinControlDialog::CoinControlDialog(const PlatformStyle *_platformStyle, QWidget *parent) : +CoinControlDialog::CoinControlDialog(CCoinControl& coin_control, WalletModel* _model, const PlatformStyle *_platformStyle, QWidget *parent) : QDialog(parent), ui(new Ui::CoinControlDialog), - model(nullptr), + m_coin_control(coin_control), + model(_model), platformStyle(_platformStyle) { ui->setupUi(this); @@ -136,6 +137,13 @@ CoinControlDialog::CoinControlDialog(const PlatformStyle *_platformStyle, QWidge sortView(settings.value("nCoinControlSortColumn").toInt(), (static_cast<Qt::SortOrder>(settings.value("nCoinControlSortOrder").toInt()))); GUIUtil::handleCloseWindowShortcut(this); + + if(_model->getOptionsModel() && _model->getAddressTableModel()) + { + updateView(); + updateLabelLocked(); + CoinControlDialog::updateLabels(m_coin_control, _model, this); + } } CoinControlDialog::~CoinControlDialog() @@ -148,18 +156,6 @@ CoinControlDialog::~CoinControlDialog() delete ui; } -void CoinControlDialog::setModel(WalletModel *_model) -{ - this->model = _model; - - if(_model && _model->getOptionsModel() && _model->getAddressTableModel()) - { - updateView(); - updateLabelLocked(); - CoinControlDialog::updateLabels(_model, this); - } -} - // ok button void CoinControlDialog::buttonBoxClicked(QAbstractButton* button) { @@ -185,8 +181,8 @@ void CoinControlDialog::buttonSelectAllClicked() ui->treeWidget->topLevelItem(i)->setCheckState(COLUMN_CHECKBOX, state); ui->treeWidget->setEnabled(true); if (state == Qt::Unchecked) - coinControl()->UnSelectAll(); // just to be sure - CoinControlDialog::updateLabels(model, this); + m_coin_control.UnSelectAll(); // just to be sure + CoinControlDialog::updateLabels(m_coin_control, model, this); } // context menu @@ -371,15 +367,15 @@ void CoinControlDialog::viewItemChanged(QTreeWidgetItem* item, int column) COutPoint outpt(uint256S(item->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), item->data(COLUMN_ADDRESS, VOutRole).toUInt()); if (item->checkState(COLUMN_CHECKBOX) == Qt::Unchecked) - coinControl()->UnSelect(outpt); + m_coin_control.UnSelect(outpt); else if (item->isDisabled()) // locked (this happens if "check all" through parent node) item->setCheckState(COLUMN_CHECKBOX, Qt::Unchecked); else - coinControl()->Select(outpt); + m_coin_control.Select(outpt); // selection changed -> update labels if (ui->treeWidget->isEnabled()) // do not update on every click for (un)select all - CoinControlDialog::updateLabels(model, this); + CoinControlDialog::updateLabels(m_coin_control, model, this); } } @@ -396,7 +392,7 @@ void CoinControlDialog::updateLabelLocked() else ui->labelLocked->setVisible(false); } -void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) +void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel *model, QDialog* dialog) { if (!model) return; @@ -428,7 +424,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) bool fWitness = false; std::vector<COutPoint> vCoinControl; - coinControl()->ListSelected(vCoinControl); + m_coin_control.ListSelected(vCoinControl); size_t i = 0; for (const auto& out : model->wallet().getCoins(vCoinControl)) { @@ -439,7 +435,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) const COutPoint& outpt = vCoinControl[i++]; if (out.is_spent) { - coinControl()->UnSelect(outpt); + m_coin_control.UnSelect(outpt); continue; } @@ -492,7 +488,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) nBytes -= 34; // Fee - nPayFee = model->wallet().getMinimumFee(nBytes, *coinControl(), nullptr /* returned_target */, nullptr /* reason */); + nPayFee = model->wallet().getMinimumFee(nBytes, m_coin_control, nullptr /* returned_target */, nullptr /* reason */); if (nPayAmount > 0) { @@ -584,12 +580,6 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) label->setVisible(nChange < 0); } -CCoinControl* CoinControlDialog::coinControl() -{ - static CCoinControl coin_control; - return &coin_control; -} - void CoinControlDialog::updateView() { if (!model || !model->getOptionsModel() || !model->getAddressTableModel()) @@ -606,8 +596,7 @@ void CoinControlDialog::updateView() int nDisplayUnit = model->getOptionsModel()->getDisplayUnit(); for (const auto& coins : model->wallet().listCoins()) { - CCoinControlWidgetItem *itemWalletAddress = new CCoinControlWidgetItem(); - itemWalletAddress->setCheckState(COLUMN_CHECKBOX, Qt::Unchecked); + CCoinControlWidgetItem* itemWalletAddress{nullptr}; QString sWalletAddress = QString::fromStdString(EncodeDestination(coins.first)); QString sWalletLabel = model->getAddressTableModel()->labelForAddress(sWalletAddress); if (sWalletLabel.isEmpty()) @@ -616,7 +605,7 @@ void CoinControlDialog::updateView() if (treeMode) { // wallet address - ui->treeWidget->addTopLevelItem(itemWalletAddress); + itemWalletAddress = new CCoinControlWidgetItem(ui->treeWidget); itemWalletAddress->setFlags(flgTristate); itemWalletAddress->setCheckState(COLUMN_CHECKBOX, Qt::Unchecked); @@ -690,13 +679,13 @@ void CoinControlDialog::updateView() // disable locked coins if (model->wallet().isLockedCoin(output)) { - coinControl()->UnSelect(output); // just to be sure + m_coin_control.UnSelect(output); // just to be sure itemOutput->setDisabled(true); itemOutput->setIcon(COLUMN_CHECKBOX, platformStyle->SingleColorIcon(":/icons/lock_closed")); } // set checkbox - if (coinControl()->IsSelected(output)) + if (m_coin_control.IsSelected(output)) itemOutput->setCheckState(COLUMN_CHECKBOX, Qt::Checked); } diff --git a/src/qt/coincontroldialog.h b/src/qt/coincontroldialog.h index efc06a7656..3de7fd6d54 100644 --- a/src/qt/coincontroldialog.h +++ b/src/qt/coincontroldialog.h @@ -31,10 +31,9 @@ class CCoinControlWidgetItem : public QTreeWidgetItem { public: explicit CCoinControlWidgetItem(QTreeWidget *parent, int type = Type) : QTreeWidgetItem(parent, type) {} - explicit CCoinControlWidgetItem(int type = Type) : QTreeWidgetItem(type) {} explicit CCoinControlWidgetItem(QTreeWidgetItem *parent, int type = Type) : QTreeWidgetItem(parent, type) {} - bool operator<(const QTreeWidgetItem &other) const; + bool operator<(const QTreeWidgetItem &other) const override; }; @@ -43,20 +42,18 @@ class CoinControlDialog : public QDialog Q_OBJECT public: - explicit CoinControlDialog(const PlatformStyle *platformStyle, QWidget *parent = nullptr); + explicit CoinControlDialog(CCoinControl& coin_control, WalletModel* model, const PlatformStyle *platformStyle, QWidget *parent = nullptr); ~CoinControlDialog(); - void setModel(WalletModel *model); - // static because also called from sendcoinsdialog - static void updateLabels(WalletModel*, QDialog*); + static void updateLabels(CCoinControl& m_coin_control, WalletModel*, QDialog*); static QList<CAmount> payAmounts; - static CCoinControl *coinControl(); static bool fSubtractFeeFromAmount; private: Ui::CoinControlDialog *ui; + CCoinControl& m_coin_control; WalletModel *model; int sortColumn; Qt::SortOrder sortOrder; diff --git a/src/qt/coincontroltreewidget.h b/src/qt/coincontroltreewidget.h index 86af555385..ac03a409c1 100644 --- a/src/qt/coincontroltreewidget.h +++ b/src/qt/coincontroltreewidget.h @@ -16,7 +16,7 @@ public: explicit CoinControlTreeWidget(QWidget *parent = nullptr); protected: - virtual void keyPressEvent(QKeyEvent *event); + virtual void keyPressEvent(QKeyEvent *event) override; }; #endif // BITCOIN_QT_COINCONTROLTREEWIDGET_H diff --git a/src/qt/editaddressdialog.h b/src/qt/editaddressdialog.h index 4d0e0709be..3be63156fd 100644 --- a/src/qt/editaddressdialog.h +++ b/src/qt/editaddressdialog.h @@ -40,7 +40,7 @@ public: void setAddress(const QString &address); public Q_SLOTS: - void accept(); + void accept() override; private: bool saveCurrentRow(); diff --git a/src/qt/forms/overviewpage.ui b/src/qt/forms/overviewpage.ui index 710801ee96..4d3f90c484 100644 --- a/src/qt/forms/overviewpage.ui +++ b/src/qt/forms/overviewpage.ui @@ -6,8 +6,8 @@ <rect> <x>0</x> <y>0</y> - <width>596</width> - <height>342</height> + <width>798</width> + <height>318</height> </rect> </property> <property name="windowTitle"> @@ -118,6 +118,7 @@ <widget class="QLabel" name="labelWatchPending"> <property name="font"> <font> + <family>Monospace</family> <weight>75</weight> <bold>true</bold> </font> @@ -129,7 +130,7 @@ <string>Unconfirmed transactions to watch-only addresses</string> </property> <property name="text"> - <string notr="true">0.000 000 00 BTC</string> + <string notr="true">0.00000000 BTC</string> </property> <property name="alignment"> <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> @@ -143,6 +144,7 @@ <widget class="QLabel" name="labelUnconfirmed"> <property name="font"> <font> + <family>Monospace</family> <weight>75</weight> <bold>true</bold> </font> @@ -154,7 +156,7 @@ <string>Total of transactions that have yet to be confirmed, and do not yet count toward the spendable balance</string> </property> <property name="text"> - <string notr="true">0.000 000 00 BTC</string> + <string notr="true">0.00000000 BTC</string> </property> <property name="alignment"> <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> @@ -168,6 +170,7 @@ <widget class="QLabel" name="labelWatchImmature"> <property name="font"> <font> + <family>Monospace</family> <weight>75</weight> <bold>true</bold> </font> @@ -179,7 +182,7 @@ <string>Mined balance in watch-only addresses that has not yet matured</string> </property> <property name="text"> - <string notr="true">0.000 000 00 BTC</string> + <string notr="true">0.00000000 BTC</string> </property> <property name="alignment"> <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> @@ -226,6 +229,7 @@ <widget class="QLabel" name="labelImmature"> <property name="font"> <font> + <family>Monospace</family> <weight>75</weight> <bold>true</bold> </font> @@ -237,7 +241,7 @@ <string>Mined balance that has not yet matured</string> </property> <property name="text"> - <string notr="true">0.000 000 00 BTC</string> + <string notr="true">0.00000000 BTC</string> </property> <property name="alignment"> <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> @@ -271,6 +275,7 @@ <widget class="QLabel" name="labelTotal"> <property name="font"> <font> + <family>Monospace</family> <weight>75</weight> <bold>true</bold> </font> @@ -282,7 +287,7 @@ <string>Your current total balance</string> </property> <property name="text"> - <string notr="true">0.000 000 00 BTC</string> + <string notr="true">21 000 000.00000000 BTC</string> </property> <property name="alignment"> <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> @@ -296,6 +301,7 @@ <widget class="QLabel" name="labelWatchTotal"> <property name="font"> <font> + <family>Monospace</family> <weight>75</weight> <bold>true</bold> </font> @@ -307,7 +313,7 @@ <string>Current total balance in watch-only addresses</string> </property> <property name="text"> - <string notr="true">0.000 000 00 BTC</string> + <string notr="true">21 000 000.00000000 BTC</string> </property> <property name="alignment"> <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> @@ -338,6 +344,7 @@ <widget class="QLabel" name="labelBalance"> <property name="font"> <font> + <family>Monospace</family> <weight>75</weight> <bold>true</bold> </font> @@ -349,7 +356,7 @@ <string>Your current spendable balance</string> </property> <property name="text"> - <string notr="true">0.000 000 00 BTC</string> + <string notr="true">21 000 000.00000000 BTC</string> </property> <property name="alignment"> <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> @@ -363,6 +370,7 @@ <widget class="QLabel" name="labelWatchAvailable"> <property name="font"> <font> + <family>Monospace</family> <weight>75</weight> <bold>true</bold> </font> @@ -374,7 +382,7 @@ <string>Your current balance in watch-only addresses</string> </property> <property name="text"> - <string notr="true">0.000 000 00 BTC</string> + <string notr="true">21 000 000.00000000 BTC</string> </property> <property name="alignment"> <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> diff --git a/src/qt/forms/receiverequestdialog.ui b/src/qt/forms/receiverequestdialog.ui index 9f896ee3b1..f6d4723465 100644 --- a/src/qt/forms/receiverequestdialog.ui +++ b/src/qt/forms/receiverequestdialog.ui @@ -6,68 +6,233 @@ <rect> <x>0</x> <y>0</y> - <width>487</width> - <height>597</height> + <width>413</width> + <height>229</height> </rect> </property> - <layout class="QVBoxLayout" name="verticalLayout_3"> - <item> - <widget class="QRImageWidget" name="lblQRCode"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Expanding" vsizetype="Fixed"> - <horstretch>0</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> - <property name="minimumSize"> - <size> - <width>300</width> - <height>320</height> - </size> - </property> - <property name="toolTip"> - <string>QR Code</string> + <property name="windowTitle"> + <string>Request payment to ...</string> + </property> + <layout class="QGridLayout" name="gridLayout" columnstretch="0,1"> + <property name="sizeConstraint"> + <enum>QLayout::SetFixedSize</enum> + </property> + <item row="0" column="0" colspan="2" alignment="Qt::AlignHCenter"> + <widget class="QRImageWidget" name="qr_code"> + <property name="text"> + <string notr="true">QR image</string> + </property> + <property name="textInteractionFlags"> + <set>Qt::NoTextInteraction</set> + </property> + </widget> + </item> + <item row="1" column="0" colspan="2"> + <widget class="QLabel" name="payment_header"> + <property name="font"> + <font> + <weight>75</weight> + <bold>true</bold> + </font> + </property> + <property name="text"> + <string>Payment information</string> + </property> + <property name="textInteractionFlags"> + <set>Qt::NoTextInteraction</set> + </property> + </widget> + </item> + <item row="2" column="0" alignment="Qt::AlignRight|Qt::AlignTop"> + <widget class="QLabel" name="uri_tag"> + <property name="font"> + <font> + <weight>75</weight> + <bold>true</bold> + </font> + </property> + <property name="text"> + <string notr="true">URI:</string> </property> <property name="textFormat"> <enum>Qt::PlainText</enum> </property> - <property name="alignment"> - <set>Qt::AlignCenter</set> + <property name="textInteractionFlags"> + <set>Qt::NoTextInteraction</set> + </property> + </widget> + </item> + <item row="2" column="1" alignment="Qt::AlignTop"> + <widget class="QLabel" name="uri_content"> + <property name="text"> + <string notr="true">bitcoin:BC1...</string> + </property> + <property name="textFormat"> + <enum>Qt::RichText</enum> </property> <property name="wordWrap"> <bool>true</bool> </property> + <property name="textInteractionFlags"> + <set>Qt::TextSelectableByMouse</set> + </property> + </widget> + </item> + <item row="3" column="0" alignment="Qt::AlignRight|Qt::AlignTop"> + <widget class="QLabel" name="address_tag"> + <property name="font"> + <font> + <weight>75</weight> + <bold>true</bold> + </font> + </property> + <property name="text"> + <string>Address:</string> + </property> + <property name="textInteractionFlags"> + <set>Qt::NoTextInteraction</set> + </property> </widget> </item> - <item> - <widget class="QTextEdit" name="outUri"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Expanding" vsizetype="Expanding"> - <horstretch>0</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> - <property name="minimumSize"> - <size> - <width>0</width> - <height>50</height> - </size> - </property> - <property name="frameShape"> - <enum>QFrame::NoFrame</enum> - </property> - <property name="frameShadow"> - <enum>QFrame::Plain</enum> - </property> - <property name="tabChangesFocus"> + <item row="3" column="1" alignment="Qt::AlignTop"> + <widget class="QLabel" name="address_content"> + <property name="text"> + <string notr="true">bc1...</string> + </property> + <property name="textFormat"> + <enum>Qt::PlainText</enum> + </property> + <property name="textInteractionFlags"> + <set>Qt::TextSelectableByMouse</set> + </property> + </widget> + </item> + <item row="4" column="0" alignment="Qt::AlignRight|Qt::AlignTop"> + <widget class="QLabel" name="amount_tag"> + <property name="font"> + <font> + <weight>75</weight> + <bold>true</bold> + </font> + </property> + <property name="text"> + <string>Amount:</string> + </property> + <property name="textInteractionFlags"> + <set>Qt::NoTextInteraction</set> + </property> + </widget> + </item> + <item row="4" column="1" alignment="Qt::AlignTop"> + <widget class="QLabel" name="amount_content"> + <property name="text"> + <string notr="true">0.00000000 BTC</string> + </property> + <property name="textFormat"> + <enum>Qt::PlainText</enum> + </property> + <property name="textInteractionFlags"> + <set>Qt::TextSelectableByMouse</set> + </property> + </widget> + </item> + <item row="5" column="0" alignment="Qt::AlignRight|Qt::AlignTop"> + <widget class="QLabel" name="label_tag"> + <property name="font"> + <font> + <weight>75</weight> + <bold>true</bold> + </font> + </property> + <property name="text"> + <string>Label:</string> + </property> + <property name="textInteractionFlags"> + <set>Qt::NoTextInteraction</set> + </property> + </widget> + </item> + <item row="5" column="1" alignment="Qt::AlignTop"> + <widget class="QLabel" name="label_content"> + <property name="text"> + <string notr="true">label content</string> + </property> + <property name="textFormat"> + <enum>Qt::PlainText</enum> + </property> + <property name="wordWrap"> + <bool>true</bool> + </property> + <property name="textInteractionFlags"> + <set>Qt::TextSelectableByMouse</set> + </property> + </widget> + </item> + <item row="6" column="0" alignment="Qt::AlignRight|Qt::AlignTop"> + <widget class="QLabel" name="message_tag"> + <property name="font"> + <font> + <weight>75</weight> + <bold>true</bold> + </font> + </property> + <property name="text"> + <string>Message:</string> + </property> + <property name="textInteractionFlags"> + <set>Qt::NoTextInteraction</set> + </property> + </widget> + </item> + <item row="6" column="1" alignment="Qt::AlignTop"> + <widget class="QLabel" name="message_content"> + <property name="text"> + <string notr="true">message content</string> + </property> + <property name="textFormat"> + <enum>Qt::PlainText</enum> + </property> + <property name="wordWrap"> + <bool>true</bool> + </property> + <property name="textInteractionFlags"> + <set>Qt::TextSelectableByMouse</set> + </property> + </widget> + </item> + <item row="7" column="0" alignment="Qt::AlignRight|Qt::AlignTop"> + <widget class="QLabel" name="wallet_tag"> + <property name="font"> + <font> + <weight>75</weight> + <bold>true</bold> + </font> + </property> + <property name="text"> + <string>Wallet:</string> + </property> + <property name="textInteractionFlags"> + <set>Qt::NoTextInteraction</set> + </property> + </widget> + </item> + <item row="7" column="1" alignment="Qt::AlignTop"> + <widget class="QLabel" name="wallet_content"> + <property name="text"> + <string notr="true">wallet name</string> + </property> + <property name="textFormat"> + <enum>Qt::PlainText</enum> + </property> + <property name="wordWrap"> <bool>true</bool> </property> <property name="textInteractionFlags"> - <set>Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse</set> + <set>Qt::TextSelectableByMouse</set> </property> </widget> </item> - <item> + <item row="8" column="0" colspan="2"> <layout class="QHBoxLayout" name="horizontalLayout"> <item> <widget class="QPushButton" name="btnCopyURI"> @@ -114,8 +279,11 @@ </item> <item> <widget class="QDialogButtonBox" name="buttonBox"> + <property name="focusPolicy"> + <enum>Qt::StrongFocus</enum> + </property> <property name="standardButtons"> - <set>QDialogButtonBox::Close</set> + <set>QDialogButtonBox::Ok</set> </property> </widget> </item> @@ -130,37 +298,27 @@ <header>qt/qrimagewidget.h</header> </customwidget> </customwidgets> + <tabstops> + <tabstop>buttonBox</tabstop> + <tabstop>btnCopyURI</tabstop> + <tabstop>btnCopyAddress</tabstop> + <tabstop>btnSaveAs</tabstop> + </tabstops> <resources/> <connections> <connection> <sender>buttonBox</sender> - <signal>rejected()</signal> - <receiver>ReceiveRequestDialog</receiver> - <slot>reject()</slot> - <hints> - <hint type="sourcelabel"> - <x>452</x> - <y>573</y> - </hint> - <hint type="destinationlabel"> - <x>243</x> - <y>298</y> - </hint> - </hints> - </connection> - <connection> - <sender>buttonBox</sender> <signal>accepted()</signal> <receiver>ReceiveRequestDialog</receiver> <slot>accept()</slot> <hints> <hint type="sourcelabel"> - <x>452</x> - <y>573</y> + <x>135</x> + <y>230</y> </hint> <hint type="destinationlabel"> - <x>243</x> - <y>298</y> + <x>135</x> + <y>126</y> </hint> </hints> </connection> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index af86fe5d27..ce44d4f3a5 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -48,13 +48,14 @@ #include <QKeyEvent> #include <QLineEdit> #include <QList> +#include <QMenu> #include <QMouseEvent> #include <QProgressDialog> #include <QScreen> #include <QSettings> +#include <QShortcut> #include <QSize> #include <QString> -#include <QShortcut> #include <QTextDocument> // for Qt::mightBeRichText #include <QThread> #include <QUrlQuery> @@ -231,7 +232,7 @@ QString HtmlEscape(const std::string& str, bool fMultiLine) return HtmlEscape(QString::fromStdString(str), fMultiLine); } -void copyEntryData(QAbstractItemView *view, int column, int role) +void copyEntryData(const QAbstractItemView *view, int column, int role) { if(!view || !view->selectionModel()) return; @@ -244,13 +245,20 @@ void copyEntryData(QAbstractItemView *view, int column, int role) } } -QList<QModelIndex> getEntryData(QAbstractItemView *view, int column) +QList<QModelIndex> getEntryData(const QAbstractItemView *view, int column) { if(!view || !view->selectionModel()) return QList<QModelIndex>(); return view->selectionModel()->selectedRows(column); } +bool hasEntryData(const QAbstractItemView *view, int column, int role) +{ + QModelIndexList selection = getEntryData(view, column); + if (selection.isEmpty()) return false; + return !selection.at(0).data(role).toString().isEmpty(); +} + QString getDefaultDataDirectory() { return boostPathToQString(GetDefaultDataDir()); @@ -743,34 +751,12 @@ QString formatDurationStr(int secs) return strList.join(" "); } -QString serviceFlagToStr(const quint64 mask, const int bit) -{ - switch (ServiceFlags(mask)) { - case NODE_NONE: abort(); // impossible - case NODE_NETWORK: return "NETWORK"; - case NODE_GETUTXO: return "GETUTXO"; - case NODE_BLOOM: return "BLOOM"; - case NODE_WITNESS: return "WITNESS"; - case NODE_NETWORK_LIMITED: return "NETWORK_LIMITED"; - // Not using default, so we get warned when a case is missing - } - if (bit < 8) { - return QString("%1[%2]").arg("UNKNOWN").arg(mask); - } else { - return QString("%1[2^%2]").arg("UNKNOWN").arg(bit); - } -} - QString formatServicesStr(quint64 mask) { QStringList strList; - for (int i = 0; i < 64; i++) { - uint64_t check = 1LL << i; - if (mask & check) - { - strList.append(serviceFlagToStr(check, i)); - } + for (const auto& flag : serviceFlagsToStr(mask)) { + strList.append(QString::fromStdString(flag)); } if (strList.size()) @@ -910,4 +896,11 @@ void LogQtInfo() } } +void PopupMenu(QMenu* menu, const QPoint& point, QAction* at_action) +{ + // The qminimal plugin does not provide window system integration. + if (QApplication::platformName() == "minimal") return; + menu->popup(point, at_action); +} + } // namespace GUIUtil diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 0d9293fbad..8741d90102 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -28,9 +28,12 @@ namespace interfaces QT_BEGIN_NAMESPACE class QAbstractItemView; +class QAction; class QDateTime; class QFont; class QLineEdit; +class QMenu; +class QPoint; class QProgressDialog; class QUrl; class QWidget; @@ -68,14 +71,21 @@ namespace GUIUtil @param[in] role Data role to extract from the model @see TransactionView::copyLabel, TransactionView::copyAmount, TransactionView::copyAddress */ - void copyEntryData(QAbstractItemView *view, int column, int role=Qt::EditRole); + void copyEntryData(const QAbstractItemView *view, int column, int role=Qt::EditRole); /** Return a field of the currently selected entry as a QString. Does nothing if nothing is selected. @param[in] column Data column to extract from the model @see TransactionView::copyLabel, TransactionView::copyAmount, TransactionView::copyAddress */ - QList<QModelIndex> getEntryData(QAbstractItemView *view, int column); + QList<QModelIndex> getEntryData(const QAbstractItemView *view, int column); + + /** Returns true if the specified field of the currently selected view entry is not empty. + @param[in] column Data column to extract from the model + @param[in] role Data role to extract from the model + @see TransactionView::contextualMenu + */ + bool hasEntryData(const QAbstractItemView *view, int column, int role); void setClipboard(const QString& str); @@ -145,7 +155,7 @@ namespace GUIUtil explicit ToolTipToRichTextFilter(int size_threshold, QObject *parent = nullptr); protected: - bool eventFilter(QObject *obj, QEvent *evt); + bool eventFilter(QObject *obj, QEvent *evt) override; private: int size_threshold; @@ -227,7 +237,7 @@ namespace GUIUtil */ void clicked(const QPoint& point); protected: - void mouseReleaseEvent(QMouseEvent *event); + void mouseReleaseEvent(QMouseEvent *event) override; }; class ClickableProgressBar : public QProgressBar @@ -240,7 +250,7 @@ namespace GUIUtil */ void clicked(const QPoint& point); protected: - void mouseReleaseEvent(QMouseEvent *event); + void mouseReleaseEvent(QMouseEvent *event) override; }; typedef ClickableProgressBar ProgressBar; @@ -255,7 +265,7 @@ namespace GUIUtil void keyEscapePressed(); private: - bool eventFilter(QObject *object, QEvent *event); + bool eventFilter(QObject *object, QEvent *event) override; }; // Fix known bugs in QProgressDialog class. @@ -273,6 +283,11 @@ namespace GUIUtil * Writes to debug.log short info about the used Qt and the host system. */ void LogQtInfo(); + + /** + * Call QMenu::popup() only on supported QT_QPA_PLATFORM. + */ + void PopupMenu(QMenu* menu, const QPoint& point, QAction* at_action = nullptr); } // namespace GUIUtil #endif // BITCOIN_QT_GUIUTIL_H diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index 49a1992803..0ba1beaf3e 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -5,12 +5,12 @@ #include <qt/modaloverlay.h> #include <qt/forms/ui_modaloverlay.h> -#include <qt/guiutil.h> - #include <chainparams.h> +#include <qt/guiutil.h> -#include <QResizeEvent> +#include <QEasingCurve> #include <QPropertyAnimation> +#include <QResizeEvent> ModalOverlay::ModalOverlay(bool enable_wallet, QWidget *parent) : QWidget(parent), @@ -33,6 +33,11 @@ userClosed(false) ui->infoText->setVisible(false); ui->infoTextStrong->setText(tr("%1 is currently syncing. It will download headers and blocks from peers and validate them until reaching the tip of the block chain.").arg(PACKAGE_NAME)); } + + m_animation.setTargetObject(this); + m_animation.setPropertyName("pos"); + m_animation.setDuration(300 /* ms */); + m_animation.setEasingCurve(QEasingCurve::OutQuad); } ModalOverlay::~ModalOverlay() @@ -48,6 +53,9 @@ bool ModalOverlay::eventFilter(QObject * obj, QEvent * ev) { if (!layerIsVisible) setGeometry(0, height(), width(), height()); + if (m_animation.endValue().toPoint().y() > 0) { + m_animation.setEndValue(QPoint(0, height())); + } } else if (ev->type() == QEvent::ChildAdded) { raise(); @@ -166,14 +174,10 @@ void ModalOverlay::showHide(bool hide, bool userRequested) if (!isVisible() && !hide) setVisible(true); - setGeometry(0, hide ? 0 : height(), width(), height()); - - QPropertyAnimation* animation = new QPropertyAnimation(this, "pos"); - animation->setDuration(300); - animation->setStartValue(QPoint(0, hide ? 0 : this->height())); - animation->setEndValue(QPoint(0, hide ? this->height() : 0)); - animation->setEasingCurve(QEasingCurve::OutQuad); - animation->start(QAbstractAnimation::DeleteWhenStopped); + m_animation.setStartValue(QPoint(0, hide ? 0 : height())); + // The eventFilter() updates the endValue if it is required for QEvent::Resize. + m_animation.setEndValue(QPoint(0, hide ? height() : 0)); + m_animation.start(QAbstractAnimation::KeepWhenStopped); layerIsVisible = !hide; } diff --git a/src/qt/modaloverlay.h b/src/qt/modaloverlay.h index 076ec30b58..1d84046d3d 100644 --- a/src/qt/modaloverlay.h +++ b/src/qt/modaloverlay.h @@ -6,6 +6,7 @@ #define BITCOIN_QT_MODALOVERLAY_H #include <QDateTime> +#include <QPropertyAnimation> #include <QWidget> //! The required delta of headers to the estimated number of available headers until we show the IBD progress @@ -35,8 +36,8 @@ public Q_SLOTS: bool isLayerVisible() const { return layerIsVisible; } protected: - bool eventFilter(QObject * obj, QEvent * ev); - bool event(QEvent* ev); + bool eventFilter(QObject * obj, QEvent * ev) override; + bool event(QEvent* ev) override; private: Ui::ModalOverlay *ui; @@ -45,6 +46,7 @@ private: QVector<QPair<qint64, double> > blockProcessTime; bool layerIsVisible; bool userClosed; + QPropertyAnimation m_animation; void UpdateHeaderSyncLabel(); }; diff --git a/src/qt/openuridialog.h b/src/qt/openuridialog.h index 4b610f74d7..667af2ec75 100644 --- a/src/qt/openuridialog.h +++ b/src/qt/openuridialog.h @@ -22,7 +22,7 @@ public: QString getURI(); protected Q_SLOTS: - void accept(); + void accept() override; private: Ui::OpenURIDialog *ui; diff --git a/src/qt/optionsdialog.h b/src/qt/optionsdialog.h index 9bc1c8ae4f..568c8b6fd0 100644 --- a/src/qt/optionsdialog.h +++ b/src/qt/optionsdialog.h @@ -28,7 +28,7 @@ class ProxyAddressValidator : public QValidator public: explicit ProxyAddressValidator(QObject *parent); - State validate(QString &input, int &pos) const; + State validate(QString &input, int &pos) const override; }; /** Preferences dialog. */ diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index b3260349e7..6ca5ac9d75 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -68,9 +68,9 @@ public: void Init(bool resetSettings = false); void Reset(); - int rowCount(const QModelIndex & parent = QModelIndex()) const; - QVariant data(const QModelIndex & index, int role = Qt::DisplayRole) const; - bool setData(const QModelIndex & index, const QVariant & value, int role = Qt::EditRole); + int rowCount(const QModelIndex & parent = QModelIndex()) const override; + QVariant data(const QModelIndex & index, int role = Qt::DisplayRole) const override; + bool setData(const QModelIndex & index, const QVariant & value, int role = Qt::EditRole) override; /** Updates current unit in memory, settings and emits displayUnitChanged(newUnit) signal */ void setDisplayUnit(const QVariant &value); diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index e0ae1f9e92..0af70f2735 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -16,7 +16,9 @@ #include <qt/walletmodel.h> #include <QAbstractItemDelegate> +#include <QApplication> #include <QPainter> +#include <QStatusTipEvent> #define DECORATION_SIZE 54 #define NUM_ITEMS 5 @@ -35,7 +37,7 @@ public: } inline void paint(QPainter *painter, const QStyleOptionViewItem &option, - const QModelIndex &index ) const + const QModelIndex &index ) const override { painter->save(); @@ -99,7 +101,7 @@ public: painter->restore(); } - inline QSize sizeHint(const QStyleOptionViewItem &option, const QModelIndex &index) const + inline QSize sizeHint(const QStyleOptionViewItem &option, const QModelIndex &index) const override { return QSize(DECORATION_SIZE, DECORATION_SIZE); } @@ -152,6 +154,21 @@ void OverviewPage::handleOutOfSyncWarningClicks() Q_EMIT outOfSyncWarningClicked(); } +void OverviewPage::setPrivacy(bool privacy) +{ + m_privacy = privacy; + if (m_balances.balance != -1) { + setBalance(m_balances); + } + + ui->listTransactions->setVisible(!m_privacy); + + const QString status_tip = m_privacy ? tr("Privacy mode activated for the Overview tab. To unmask the values, uncheck Settings->Mask values.") : ""; + setStatusTip(status_tip); + QStatusTipEvent event(status_tip); + QApplication::sendEvent(this, &event); +} + OverviewPage::~OverviewPage() { delete ui; @@ -163,25 +180,25 @@ void OverviewPage::setBalance(const interfaces::WalletBalances& balances) m_balances = balances; if (walletModel->wallet().isLegacy()) { if (walletModel->wallet().privateKeysDisabled()) { - ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelBalance->setText(BitcoinUnits::formatWithPrivacy(unit, balances.watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelUnconfirmed->setText(BitcoinUnits::formatWithPrivacy(unit, balances.unconfirmed_watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelImmature->setText(BitcoinUnits::formatWithPrivacy(unit, balances.immature_watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelTotal->setText(BitcoinUnits::formatWithPrivacy(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); } else { - ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.balance, false, BitcoinUnits::separatorAlways)); - ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_balance, false, BitcoinUnits::separatorAlways)); - ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_balance, false, BitcoinUnits::separatorAlways)); - ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchAvailable->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchPending->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelBalance->setText(BitcoinUnits::formatWithPrivacy(unit, balances.balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelUnconfirmed->setText(BitcoinUnits::formatWithPrivacy(unit, balances.unconfirmed_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelImmature->setText(BitcoinUnits::formatWithPrivacy(unit, balances.immature_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelTotal->setText(BitcoinUnits::formatWithPrivacy(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelWatchAvailable->setText(BitcoinUnits::formatWithPrivacy(unit, balances.watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelWatchPending->setText(BitcoinUnits::formatWithPrivacy(unit, balances.unconfirmed_watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelWatchImmature->setText(BitcoinUnits::formatWithPrivacy(unit, balances.immature_watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelWatchTotal->setText(BitcoinUnits::formatWithPrivacy(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); } } else { - ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.balance, false, BitcoinUnits::separatorAlways)); - ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_balance, false, BitcoinUnits::separatorAlways)); - ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_balance, false, BitcoinUnits::separatorAlways)); - ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, false, BitcoinUnits::separatorAlways)); + ui->labelBalance->setText(BitcoinUnits::formatWithPrivacy(unit, balances.balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelUnconfirmed->setText(BitcoinUnits::formatWithPrivacy(unit, balances.unconfirmed_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelImmature->setText(BitcoinUnits::formatWithPrivacy(unit, balances.immature_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelTotal->setText(BitcoinUnits::formatWithPrivacy(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, BitcoinUnits::separatorAlways, m_privacy)); } // only show immature (newly mined) balance if it's non-zero, so as not to complicate things // for the non-mining users diff --git a/src/qt/overviewpage.h b/src/qt/overviewpage.h index 00ba7ad4ce..4cf673b6a6 100644 --- a/src/qt/overviewpage.h +++ b/src/qt/overviewpage.h @@ -39,6 +39,7 @@ public: public Q_SLOTS: void setBalance(const interfaces::WalletBalances& balances); + void setPrivacy(bool privacy); Q_SIGNALS: void transactionClicked(const QModelIndex &index); @@ -49,6 +50,7 @@ private: ClientModel *clientModel; WalletModel *walletModel; interfaces::WalletBalances m_balances; + bool m_privacy{false}; TxViewDelegate *txdelegate; std::unique_ptr<TransactionFilterProxy> filter; diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index aa9a7327ba..154f4a7ea6 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -98,7 +98,7 @@ private Q_SLOTS: protected: // Constructor registers this on the parent QApplication to // receive QEvent::FileOpen and QEvent:Drop events - bool eventFilter(QObject *object, QEvent *event); + bool eventFilter(QObject *object, QEvent *event) override; private: bool saveURIs; // true during startup diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index 0e20b1d9dc..99de772ac0 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -68,13 +68,13 @@ public: /** @name Methods overridden from QAbstractTableModel @{*/ - int rowCount(const QModelIndex &parent) const; - int columnCount(const QModelIndex &parent) const; - QVariant data(const QModelIndex &index, int role) const; - QVariant headerData(int section, Qt::Orientation orientation, int role) const; - QModelIndex index(int row, int column, const QModelIndex &parent) const; - Qt::ItemFlags flags(const QModelIndex &index) const; - void sort(int column, Qt::SortOrder order); + int rowCount(const QModelIndex &parent) const override; + int columnCount(const QModelIndex &parent) const override; + QVariant data(const QModelIndex &index, int role) const override; + QVariant headerData(int section, Qt::Orientation orientation, int role) const override; + QModelIndex index(int row, int column, const QModelIndex &parent) const override; + Qt::ItemFlags flags(const QModelIndex &index) const override; + void sort(int column, Qt::SortOrder order) override; /*@}*/ public Q_SLOTS: diff --git a/src/qt/qrimagewidget.h b/src/qt/qrimagewidget.h index 345bb64092..cca598c2ce 100644 --- a/src/qt/qrimagewidget.h +++ b/src/qt/qrimagewidget.h @@ -35,8 +35,8 @@ public Q_SLOTS: void copyImage(); protected: - virtual void mousePressEvent(QMouseEvent *event); - virtual void contextMenuEvent(QContextMenuEvent *event); + virtual void mousePressEvent(QMouseEvent *event) override; + virtual void contextMenuEvent(QContextMenuEvent *event) override; private: QMenu *contextMenu; diff --git a/src/qt/qvalidatedlineedit.h b/src/qt/qvalidatedlineedit.h index 815d5e0040..2c72b2ecda 100644 --- a/src/qt/qvalidatedlineedit.h +++ b/src/qt/qvalidatedlineedit.h @@ -21,8 +21,8 @@ public: bool isValid(); protected: - void focusInEvent(QFocusEvent *evt); - void focusOutEvent(QFocusEvent *evt); + void focusInEvent(QFocusEvent *evt) override; + void focusOutEvent(QFocusEvent *evt) override; private: bool valid; diff --git a/src/qt/receivecoinsdialog.h b/src/qt/receivecoinsdialog.h index 5bca2f46e2..2f48cd58f0 100644 --- a/src/qt/receivecoinsdialog.h +++ b/src/qt/receivecoinsdialog.h @@ -46,11 +46,11 @@ public: public Q_SLOTS: void clear(); - void reject(); - void accept(); + void reject() override; + void accept() override; protected: - virtual void keyPressEvent(QKeyEvent *event); + virtual void keyPressEvent(QKeyEvent *event) override; private: Ui::ReceiveCoinsDialog *ui; @@ -61,7 +61,7 @@ private: QModelIndex selectedRow(); void copyColumnToClipboard(int column); - virtual void resizeEvent(QResizeEvent *event); + virtual void resizeEvent(QResizeEvent *event) override; private Q_SLOTS: void on_receiveButton_clicked(); diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index 30bd5c6a5a..d385c42821 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -8,10 +8,11 @@ #include <qt/bitcoinunits.h> #include <qt/guiutil.h> #include <qt/optionsmodel.h> +#include <qt/qrimagewidget.h> #include <qt/walletmodel.h> -#include <QClipboard> -#include <QPixmap> +#include <QDialog> +#include <QString> #if defined(HAVE_CONFIG_H) #include <config/bitcoin-config.h> /* for USE_QRCODE */ @@ -23,14 +24,6 @@ ReceiveRequestDialog::ReceiveRequestDialog(QWidget *parent) : model(nullptr) { ui->setupUi(this); - -#ifndef USE_QRCODE - ui->btnSaveAs->setVisible(false); - ui->lblQRCode->setVisible(false); -#endif - - connect(ui->btnSaveAs, &QPushButton::clicked, ui->lblQRCode, &QRImageWidget::saveImage); - GUIUtil::handleCloseWindowShortcut(this); } @@ -44,7 +37,7 @@ void ReceiveRequestDialog::setModel(WalletModel *_model) this->model = _model; if (_model) - connect(_model->getOptionsModel(), &OptionsModel::displayUnitChanged, this, &ReceiveRequestDialog::update); + connect(_model->getOptionsModel(), &OptionsModel::displayUnitChanged, this, &ReceiveRequestDialog::updateDisplayUnit); // update the display unit if necessary update(); @@ -53,40 +46,55 @@ void ReceiveRequestDialog::setModel(WalletModel *_model) void ReceiveRequestDialog::setInfo(const SendCoinsRecipient &_info) { this->info = _info; - update(); -} + setWindowTitle(tr("Request payment to %1").arg(info.label.isEmpty() ? info.address : info.label)); + QString uri = GUIUtil::formatBitcoinURI(info); -void ReceiveRequestDialog::update() -{ - if(!model) - return; - QString target = info.label; - if(target.isEmpty()) - target = info.address; - setWindowTitle(tr("Request payment to %1").arg(target)); +#ifdef USE_QRCODE + if (ui->qr_code->setQR(uri, info.address)) { + connect(ui->btnSaveAs, &QPushButton::clicked, ui->qr_code, &QRImageWidget::saveImage); + } else { + ui->btnSaveAs->setEnabled(false); + } +#else + ui->btnSaveAs->hide(); + ui->qr_code->hide(); +#endif - QString uri = GUIUtil::formatBitcoinURI(info); - ui->btnSaveAs->setEnabled(false); - QString html; - html += "<html><font face='verdana, arial, helvetica, sans-serif'>"; - html += "<b>"+tr("Payment information")+"</b><br>"; - html += "<b>"+tr("URI")+"</b>: "; - html += "<a href=\""+uri+"\">" + GUIUtil::HtmlEscape(uri) + "</a><br>"; - html += "<b>"+tr("Address")+"</b>: " + GUIUtil::HtmlEscape(info.address) + "<br>"; - if(info.amount) - html += "<b>"+tr("Amount")+"</b>: " + BitcoinUnits::formatHtmlWithUnit(model->getOptionsModel()->getDisplayUnit(), info.amount) + "<br>"; - if(!info.label.isEmpty()) - html += "<b>"+tr("Label")+"</b>: " + GUIUtil::HtmlEscape(info.label) + "<br>"; - if(!info.message.isEmpty()) - html += "<b>"+tr("Message")+"</b>: " + GUIUtil::HtmlEscape(info.message) + "<br>"; - if(model->isMultiwallet()) { - html += "<b>"+tr("Wallet")+"</b>: " + GUIUtil::HtmlEscape(model->getWalletName()) + "<br>"; + ui->uri_content->setText("<a href=\"" + uri + "\">" + GUIUtil::HtmlEscape(uri) + "</a>"); + ui->address_content->setText(info.address); + + if (!info.amount) { + ui->amount_tag->hide(); + ui->amount_content->hide(); + } // Amount is set in updateDisplayUnit() slot. + updateDisplayUnit(); + + if (!info.label.isEmpty()) { + ui->label_content->setText(info.label); + } else { + ui->label_tag->hide(); + ui->label_content->hide(); } - ui->outUri->setText(html); - if (ui->lblQRCode->setQR(uri, info.address)) { - ui->btnSaveAs->setEnabled(true); + if (!info.message.isEmpty()) { + ui->message_content->setText(info.message); + } else { + ui->message_tag->hide(); + ui->message_content->hide(); } + + if (!model->getWalletName().isEmpty()) { + ui->wallet_content->setText(model->getWalletName()); + } else { + ui->wallet_tag->hide(); + ui->wallet_content->hide(); + } +} + +void ReceiveRequestDialog::updateDisplayUnit() +{ + if (!model) return; + ui->amount_content->setText(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), info.amount)); } void ReceiveRequestDialog::on_btnCopyURI_clicked() diff --git a/src/qt/receiverequestdialog.h b/src/qt/receiverequestdialog.h index 40e3d5ffa8..846478643d 100644 --- a/src/qt/receiverequestdialog.h +++ b/src/qt/receiverequestdialog.h @@ -29,8 +29,7 @@ public: private Q_SLOTS: void on_btnCopyURI_clicked(); void on_btnCopyAddress_clicked(); - - void update(); + void updateDisplayUnit(); private: Ui::ReceiveRequestDialog *ui; diff --git a/src/qt/recentrequeststablemodel.h b/src/qt/recentrequeststablemodel.h index 5e7f6acdc8..c0bd3461bb 100644 --- a/src/qt/recentrequeststablemodel.h +++ b/src/qt/recentrequeststablemodel.h @@ -24,19 +24,11 @@ public: QDateTime date; SendCoinsRecipient recipient; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - unsigned int nDate = date.toTime_t(); - - READWRITE(this->nVersion); - READWRITE(id); - READWRITE(nDate); - READWRITE(recipient); - - if (ser_action.ForRead()) - date = QDateTime::fromTime_t(nDate); + SERIALIZE_METHODS(RecentRequestEntry, obj) { + unsigned int date_timet; + SER_WRITE(obj, date_timet = obj.date.toTime_t()); + READWRITE(obj.nVersion, obj.id, date_timet, obj.recipient); + SER_READ(obj, obj.date = QDateTime::fromTime_t(date_timet)); } }; @@ -73,14 +65,15 @@ public: /** @name Methods overridden from QAbstractTableModel @{*/ - int rowCount(const QModelIndex &parent) const; - int columnCount(const QModelIndex &parent) const; - QVariant data(const QModelIndex &index, int role) const; - bool setData(const QModelIndex &index, const QVariant &value, int role); - QVariant headerData(int section, Qt::Orientation orientation, int role) const; - QModelIndex index(int row, int column, const QModelIndex &parent = QModelIndex()) const; - bool removeRows(int row, int count, const QModelIndex &parent = QModelIndex()); - Qt::ItemFlags flags(const QModelIndex &index) const; + int rowCount(const QModelIndex &parent) const override; + int columnCount(const QModelIndex &parent) const override; + QVariant data(const QModelIndex &index, int role) const override; + bool setData(const QModelIndex &index, const QVariant &value, int role) override; + QVariant headerData(int section, Qt::Orientation orientation, int role) const override; + QModelIndex index(int row, int column, const QModelIndex &parent = QModelIndex()) const override; + bool removeRows(int row, int count, const QModelIndex &parent = QModelIndex()) override; + Qt::ItemFlags flags(const QModelIndex &index) const override; + void sort(int column, Qt::SortOrder order = Qt::AscendingOrder) override; /*@}*/ const RecentRequestEntry &entry(int row) const { return list[row]; } @@ -89,7 +82,6 @@ public: void addNewRequest(RecentRequestEntry &recipient); public Q_SLOTS: - void sort(int column, Qt::SortOrder order = Qt::AscendingOrder); void updateDisplayUnit(); private: diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index bfe316bdcd..0f89d4e6fe 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -24,23 +24,22 @@ #include <univalue.h> #ifdef ENABLE_WALLET -#include <db_cxx.h> +#include <wallet/db.h> #include <wallet/wallet.h> #endif +#include <QFont> #include <QKeyEvent> #include <QMenu> #include <QMessageBox> -#include <QScrollBar> #include <QScreen> +#include <QScrollBar> #include <QSettings> +#include <QString> +#include <QStringList> #include <QTime> #include <QTimer> -#include <QStringList> -// TODO: add a scrollback limit, as there is currently none -// TODO: make it possible to filter out categories (esp debug messages when implemented) -// TODO: receive errors and debug messages through ClientModel const int CONSOLE_HISTORY = 50; const int INITIAL_TRAFFIC_GRAPH_MINS = 30; @@ -115,8 +114,8 @@ class QtRPCTimerInterface: public RPCTimerInterface { public: ~QtRPCTimerInterface() {} - const char *Name() { return "Qt"; } - RPCTimerBase* NewTimer(std::function<void()>& func, int64_t millis) + const char *Name() override { return "Qt"; } + RPCTimerBase* NewTimer(std::function<void()>& func, int64_t millis) override { return new QtRPCTimerBase(func, millis); } @@ -480,7 +479,7 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty // set library version labels #ifdef ENABLE_WALLET - ui->berkeleyDBVersion->setText(DbEnv::version(nullptr, nullptr, nullptr)); + ui->berkeleyDBVersion->setText(QString::fromStdString(BerkeleyDatabaseVersion())); #else ui->label_berkeleyDBVersion->hide(); ui->berkeleyDBVersion->hide(); @@ -496,7 +495,7 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty ui->detailWidget->hide(); ui->peerHeading->setText(tr("Select a peer to view detailed information.")); - consoleFontSize = settings.value(fontSizeSettingsKey, QFontInfo(QFont()).pointSize()).toInt(); + consoleFontSize = settings.value(fontSizeSettingsKey, QFont().pointSize()).toInt(); clear(); GUIUtil::handleCloseWindowShortcut(this); diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index f586d04022..de8e37cca2 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -71,8 +71,8 @@ public: QKeySequence tabShortcut(TabTypes tab_type) const; protected: - virtual bool eventFilter(QObject* obj, QEvent *event); - void keyPressEvent(QKeyEvent *); + virtual bool eventFilter(QObject* obj, QEvent *event) override; + void keyPressEvent(QKeyEvent *) override; private Q_SLOTS: void on_lineEdit_returnPressed(); @@ -83,9 +83,9 @@ private Q_SLOTS: void on_sldGraphRange_valueChanged(int value); /** update traffic statistics */ void updateTrafficStats(quint64 totalBytesIn, quint64 totalBytesOut); - void resizeEvent(QResizeEvent *event); - void showEvent(QShowEvent *event); - void hideEvent(QHideEvent *event); + void resizeEvent(QResizeEvent *event) override; + void showEvent(QShowEvent *event) override; + void hideEvent(QHideEvent *event) override; /** Show custom context menu on Peers tab */ void showPeersTableContextMenu(const QPoint& point); /** Show custom context menu on Bans tab */ diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 7b389a48d6..9e23fe78d8 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -57,6 +57,7 @@ SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *p ui(new Ui::SendCoinsDialog), clientModel(nullptr), model(nullptr), + m_coin_control(new CCoinControl), fNewRecipientAllowed(true), fFeeMinimized(true), platformStyle(_platformStyle) @@ -259,14 +260,9 @@ bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informa m_current_transaction = MakeUnique<WalletModelTransaction>(recipients); WalletModel::SendCoinsReturn prepareStatus; - // Always use a CCoinControl instance, use the CoinControlDialog instance if CoinControl has been enabled - CCoinControl ctrl; - if (model->getOptionsModel()->getCoinControlFeatures()) - ctrl = *CoinControlDialog::coinControl(); + updateCoinControlState(*m_coin_control); - updateCoinControlState(ctrl); - - prepareStatus = model->prepareTransaction(*m_current_transaction, ctrl); + prepareStatus = model->prepareTransaction(*m_current_transaction, *m_coin_control); // process prepareStatus and on error generate message shown to user processSendCoinsReturn(prepareStatus, @@ -454,7 +450,7 @@ void SendCoinsDialog::on_sendButton_clicked() } if (!send_failure) { accept(); - CoinControlDialog::coinControl()->UnSelectAll(); + m_coin_control->UnSelectAll(); coinControlUpdateLabels(); } fNewRecipientAllowed = true; @@ -466,7 +462,7 @@ void SendCoinsDialog::clear() m_current_transaction.reset(); // Clear coin control settings - CoinControlDialog::coinControl()->UnSelectAll(); + m_coin_control->UnSelectAll(); ui->checkBoxCoinControlChange->setChecked(false); ui->lineEditCoinControlChange->clear(); coinControlUpdateLabels(); @@ -689,17 +685,11 @@ void SendCoinsDialog::on_buttonMinimizeFee_clicked() void SendCoinsDialog::useAvailableBalance(SendCoinsEntry* entry) { - // Get CCoinControl instance if CoinControl is enabled or create a new one. - CCoinControl coin_control; - if (model->getOptionsModel()->getCoinControlFeatures()) { - coin_control = *CoinControlDialog::coinControl(); - } - // Include watch-only for wallets without private key - coin_control.fAllowWatchOnly = model->wallet().privateKeysDisabled(); + m_coin_control->fAllowWatchOnly = model->wallet().privateKeysDisabled(); // Calculate available amount to send. - CAmount amount = model->wallet().getAvailableBalance(coin_control); + CAmount amount = model->wallet().getAvailableBalance(*m_coin_control); for (int i = 0; i < ui->entries->count(); ++i) { SendCoinsEntry* e = qobject_cast<SendCoinsEntry*>(ui->entries->itemAt(i)->widget()); if (e && !e->isHidden() && e != entry) { @@ -758,12 +748,11 @@ void SendCoinsDialog::updateSmartFeeLabel() { if(!model || !model->getOptionsModel()) return; - CCoinControl coin_control; - updateCoinControlState(coin_control); - coin_control.m_feerate.reset(); // Explicitly use only fee estimation rate for smart fee labels + updateCoinControlState(*m_coin_control); + m_coin_control->m_feerate.reset(); // Explicitly use only fee estimation rate for smart fee labels int returned_target; FeeReason reason; - CFeeRate feeRate = CFeeRate(model->wallet().getMinimumFee(1000, coin_control, &returned_target, &reason)); + CFeeRate feeRate = CFeeRate(model->wallet().getMinimumFee(1000, *m_coin_control, &returned_target, &reason)); ui->labelSmartFee->setText(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), feeRate.GetFeePerK()) + "/kB"); @@ -834,7 +823,7 @@ void SendCoinsDialog::coinControlFeatureChanged(bool checked) ui->frameCoinControl->setVisible(checked); if (!checked && model) // coin control features disabled - CoinControlDialog::coinControl()->SetNull(); + m_coin_control->SetNull(); coinControlUpdateLabels(); } @@ -842,8 +831,7 @@ void SendCoinsDialog::coinControlFeatureChanged(bool checked) // Coin Control: button inputs -> show actual coin control dialog void SendCoinsDialog::coinControlButtonClicked() { - CoinControlDialog dlg(platformStyle); - dlg.setModel(model); + CoinControlDialog dlg(*m_coin_control, model, platformStyle); dlg.exec(); coinControlUpdateLabels(); } @@ -853,7 +841,7 @@ void SendCoinsDialog::coinControlChangeChecked(int state) { if (state == Qt::Unchecked) { - CoinControlDialog::coinControl()->destChange = CNoDestination(); + m_coin_control->destChange = CNoDestination(); ui->labelCoinControlChangeLabel->clear(); } else @@ -869,7 +857,7 @@ void SendCoinsDialog::coinControlChangeEdited(const QString& text) if (model && model->getAddressTableModel()) { // Default to no change address until verified - CoinControlDialog::coinControl()->destChange = CNoDestination(); + m_coin_control->destChange = CNoDestination(); ui->labelCoinControlChangeLabel->setStyleSheet("QLabel{color:red;}"); const CTxDestination dest = DecodeDestination(text.toStdString()); @@ -892,7 +880,7 @@ void SendCoinsDialog::coinControlChangeEdited(const QString& text) QMessageBox::Yes | QMessageBox::Cancel, QMessageBox::Cancel); if(btnRetVal == QMessageBox::Yes) - CoinControlDialog::coinControl()->destChange = dest; + m_coin_control->destChange = dest; else { ui->lineEditCoinControlChange->setText(""); @@ -911,7 +899,7 @@ void SendCoinsDialog::coinControlChangeEdited(const QString& text) else ui->labelCoinControlChangeLabel->setText(tr("(no label)")); - CoinControlDialog::coinControl()->destChange = dest; + m_coin_control->destChange = dest; } } } @@ -923,7 +911,7 @@ void SendCoinsDialog::coinControlUpdateLabels() if (!model || !model->getOptionsModel()) return; - updateCoinControlState(*CoinControlDialog::coinControl()); + updateCoinControlState(*m_coin_control); // set pay amounts CoinControlDialog::payAmounts.clear(); @@ -941,10 +929,10 @@ void SendCoinsDialog::coinControlUpdateLabels() } } - if (CoinControlDialog::coinControl()->HasSelected()) + if (m_coin_control->HasSelected()) { // actual coin control calculation - CoinControlDialog::updateLabels(model, this); + CoinControlDialog::updateLabels(*m_coin_control, model, this); // show coin control stats ui->labelCoinControlAutomaticallySelected->hide(); diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 36bc2a846b..6961aa7821 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -12,6 +12,7 @@ #include <QString> #include <QTimer> +class CCoinControl; class ClientModel; class PlatformStyle; class SendCoinsEntry; @@ -47,8 +48,8 @@ public: public Q_SLOTS: void clear(); - void reject(); - void accept(); + void reject() override; + void accept() override; SendCoinsEntry *addEntry(); void updateTabsAndLabels(); void setBalance(const interfaces::WalletBalances& balances); @@ -60,6 +61,7 @@ private: Ui::SendCoinsDialog *ui; ClientModel *clientModel; WalletModel *model; + std::unique_ptr<CCoinControl> m_coin_control; std::unique_ptr<WalletModelTransaction> m_current_transaction; bool fNewRecipientAllowed; bool fFeeMinimized; @@ -112,7 +114,7 @@ class SendConfirmationDialog : public QMessageBox public: SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text = "", const QString& detailed_text = "", int secDelay = SEND_CONFIRM_DELAY, const QString& confirmText = "Send", QWidget* parent = nullptr); - int exec(); + int exec() override; private Q_SLOTS: void countDown(); diff --git a/src/qt/sendcoinsrecipient.h b/src/qt/sendcoinsrecipient.h index 12279fab64..6619faf417 100644 --- a/src/qt/sendcoinsrecipient.h +++ b/src/qt/sendcoinsrecipient.h @@ -44,30 +44,21 @@ public: static const int CURRENT_VERSION = 1; int nVersion; - ADD_SERIALIZE_METHODS; + SERIALIZE_METHODS(SendCoinsRecipient, obj) + { + std::string address_str, label_str, message_str, auth_merchant_str; - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - std::string sAddress = address.toStdString(); - std::string sLabel = label.toStdString(); - std::string sMessage = message.toStdString(); - std::string sAuthenticatedMerchant = authenticatedMerchant.toStdString(); + SER_WRITE(obj, address_str = obj.address.toStdString()); + SER_WRITE(obj, label_str = obj.label.toStdString()); + SER_WRITE(obj, message_str = obj.message.toStdString()); + SER_WRITE(obj, auth_merchant_str = obj.authenticatedMerchant.toStdString()); - READWRITE(this->nVersion); - READWRITE(sAddress); - READWRITE(sLabel); - READWRITE(amount); - READWRITE(sMessage); - READWRITE(sPaymentRequest); - READWRITE(sAuthenticatedMerchant); + READWRITE(obj.nVersion, address_str, label_str, obj.amount, message_str, obj.sPaymentRequest, auth_merchant_str); - if (ser_action.ForRead()) - { - address = QString::fromStdString(sAddress); - label = QString::fromStdString(sLabel); - message = QString::fromStdString(sMessage); - authenticatedMerchant = QString::fromStdString(sAuthenticatedMerchant); - } + SER_READ(obj, obj.address = QString::fromStdString(address_str)); + SER_READ(obj, obj.label = QString::fromStdString(label_str)); + SER_READ(obj, obj.message = QString::fromStdString(message_str)); + SER_READ(obj, obj.authenticatedMerchant = QString::fromStdString(auth_merchant_str)); } }; diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 5cfbb67449..4835dd7954 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -91,6 +91,7 @@ void SignVerifyMessageDialog::on_addressBookButton_SM_clicked() { if (model && model->getAddressTableModel()) { + model->refresh(/* pk_hash_only */ true); AddressBookPage dlg(platformStyle, AddressBookPage::ForSelection, AddressBookPage::ReceivingTab, this); dlg.setModel(model->getAddressTableModel()); if (dlg.exec()) diff --git a/src/qt/signverifymessagedialog.h b/src/qt/signverifymessagedialog.h index d2e04cd4fe..d33a2d038d 100644 --- a/src/qt/signverifymessagedialog.h +++ b/src/qt/signverifymessagedialog.h @@ -30,7 +30,7 @@ public: void showTab_VM(bool fShow); protected: - bool eventFilter(QObject *object, QEvent *event); + bool eventFilter(QObject *object, QEvent *event) override; private: Ui::SignVerifyMessageDialog *ui; diff --git a/src/qt/splashscreen.h b/src/qt/splashscreen.h index f99dd0c701..3158524117 100644 --- a/src/qt/splashscreen.h +++ b/src/qt/splashscreen.h @@ -32,8 +32,8 @@ public: ~SplashScreen(); protected: - void paintEvent(QPaintEvent *event); - void closeEvent(QCloseEvent *event); + void paintEvent(QPaintEvent *event) override; + void closeEvent(QCloseEvent *event) override; public Q_SLOTS: /** Hide the splash screen window and schedule the splash screen object for deletion */ @@ -43,7 +43,7 @@ public Q_SLOTS: void showMessage(const QString &message, int alignment, const QColor &color); protected: - bool eventFilter(QObject * obj, QEvent * ev); + bool eventFilter(QObject * obj, QEvent * ev) override; private: /** Connect core signals to splash screen */ diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 2ee9ae0d86..8da0250e57 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -201,7 +201,7 @@ void TestGUI(interfaces::Node& node) OverviewPage overviewPage(platformStyle.get()); overviewPage.setWalletModel(&walletModel); QLabel* balanceLabel = overviewPage.findChild<QLabel*>("labelBalance"); - QString balanceText = balanceLabel->text(); + QString balanceText = balanceLabel->text().trimmed(); int unit = walletModel.getOptionsModel()->getDisplayUnit(); CAmount balance = walletModel.wallet().getBalance(); QString balanceComparison = BitcoinUnits::formatWithUnit(unit, balance, false, BitcoinUnits::separatorAlways); @@ -229,15 +229,23 @@ void TestGUI(interfaces::Node& node) for (QWidget* widget : QApplication::topLevelWidgets()) { if (widget->inherits("ReceiveRequestDialog")) { ReceiveRequestDialog* receiveRequestDialog = qobject_cast<ReceiveRequestDialog*>(widget); - QTextEdit* rlist = receiveRequestDialog->QObject::findChild<QTextEdit*>("outUri"); - QString paymentText = rlist->toPlainText(); - QStringList paymentTextList = paymentText.split('\n'); - QCOMPARE(paymentTextList.at(0), QString("Payment information")); - QVERIFY(paymentTextList.at(1).indexOf(QString("URI: bitcoin:")) != -1); - QVERIFY(paymentTextList.at(2).indexOf(QString("Address:")) != -1); - QCOMPARE(paymentTextList.at(3), QString("Amount: 0.00000001 ") + QString::fromStdString(CURRENCY_UNIT)); - QCOMPARE(paymentTextList.at(4), QString("Label: TEST_LABEL_1")); - QCOMPARE(paymentTextList.at(5), QString("Message: TEST_MESSAGE_1")); + QCOMPARE(receiveRequestDialog->QObject::findChild<QLabel*>("payment_header")->text(), QString("Payment information")); + QCOMPARE(receiveRequestDialog->QObject::findChild<QLabel*>("uri_tag")->text(), QString("URI:")); + QString uri = receiveRequestDialog->QObject::findChild<QLabel*>("uri_content")->text(); + QCOMPARE(uri.count("bitcoin:"), 2); + QCOMPARE(receiveRequestDialog->QObject::findChild<QLabel*>("address_tag")->text(), QString("Address:")); + + QCOMPARE(uri.count("amount=0.00000001"), 2); + QCOMPARE(receiveRequestDialog->QObject::findChild<QLabel*>("amount_tag")->text(), QString("Amount:")); + QCOMPARE(receiveRequestDialog->QObject::findChild<QLabel*>("amount_content")->text(), QString("0.00000001 ") + QString::fromStdString(CURRENCY_UNIT)); + + QCOMPARE(uri.count("label=TEST_LABEL_1"), 2); + QCOMPARE(receiveRequestDialog->QObject::findChild<QLabel*>("label_tag")->text(), QString("Label:")); + QCOMPARE(receiveRequestDialog->QObject::findChild<QLabel*>("label_content")->text(), QString("TEST_LABEL_1")); + + QCOMPARE(uri.count("message=TEST_MESSAGE_1"), 2); + QCOMPARE(receiveRequestDialog->QObject::findChild<QLabel*>("message_tag")->text(), QString("Message:")); + QCOMPARE(receiveRequestDialog->QObject::findChild<QLabel*>("message_content")->text(), QString("TEST_MESSAGE_1")); } } diff --git a/src/qt/trafficgraphwidget.cpp b/src/qt/trafficgraphwidget.cpp index 757648f485..6428fc4daf 100644 --- a/src/qt/trafficgraphwidget.cpp +++ b/src/qt/trafficgraphwidget.cpp @@ -7,6 +7,7 @@ #include <qt/clientmodel.h> #include <QPainter> +#include <QPainterPath> #include <QColor> #include <QTimer> diff --git a/src/qt/trafficgraphwidget.h b/src/qt/trafficgraphwidget.h index 7e8bfb2337..2d8c825815 100644 --- a/src/qt/trafficgraphwidget.h +++ b/src/qt/trafficgraphwidget.h @@ -25,7 +25,7 @@ public: int getGraphRangeMins() const; protected: - void paintEvent(QPaintEvent *); + void paintEvent(QPaintEvent *) override; public Q_SLOTS: void updateRates(); diff --git a/src/qt/transactionfilterproxy.h b/src/qt/transactionfilterproxy.h index 685f8d3a26..d6bb84f7e6 100644 --- a/src/qt/transactionfilterproxy.h +++ b/src/qt/transactionfilterproxy.h @@ -49,10 +49,10 @@ public: /** Set whether to show conflicted transactions. */ void setShowInactive(bool showInactive); - int rowCount(const QModelIndex &parent = QModelIndex()) const; + int rowCount(const QModelIndex &parent = QModelIndex()) const override; protected: - bool filterAcceptsRow(int source_row, const QModelIndex & source_parent) const; + bool filterAcceptsRow(int source_row, const QModelIndex & source_parent) const override; private: QDateTime dateFrom; diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index a32d218fc9..01dff8069c 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -162,7 +162,7 @@ QList<TransactionRecord> TransactionRecord::decomposeTransaction(const interface return parts; } -void TransactionRecord::updateStatus(const interfaces::WalletTxStatus& wtx, int numBlocks, int64_t block_time) +void TransactionRecord::updateStatus(const interfaces::WalletTxStatus& wtx, const uint256& block_hash, int numBlocks, int64_t block_time) { // Determine transaction status @@ -174,7 +174,7 @@ void TransactionRecord::updateStatus(const interfaces::WalletTxStatus& wtx, int idx); status.countsForBalance = wtx.is_trusted && !(wtx.blocks_to_maturity > 0); status.depth = wtx.depth_in_main_chain; - status.cur_num_blocks = numBlocks; + status.m_cur_block_hash = block_hash; const bool up_to_date = ((int64_t)QDateTime::currentMSecsSinceEpoch() / 1000 - block_time < MAX_BLOCK_TIME_GAP); if (up_to_date && !wtx.is_final) { @@ -233,9 +233,9 @@ void TransactionRecord::updateStatus(const interfaces::WalletTxStatus& wtx, int status.needsUpdate = false; } -bool TransactionRecord::statusUpdateNeeded(int numBlocks) const +bool TransactionRecord::statusUpdateNeeded(const uint256& block_hash) const { - return status.cur_num_blocks != numBlocks || status.needsUpdate; + return status.m_cur_block_hash != block_hash || status.needsUpdate; } QString TransactionRecord::getTxHash() const diff --git a/src/qt/transactionrecord.h b/src/qt/transactionrecord.h index 3f64cefd09..c983c527c0 100644 --- a/src/qt/transactionrecord.h +++ b/src/qt/transactionrecord.h @@ -23,9 +23,8 @@ struct WalletTxStatus; class TransactionStatus { public: - TransactionStatus(): - countsForBalance(false), sortKey(""), - matures_in(0), status(Unconfirmed), depth(0), open_for(0), cur_num_blocks(-1) + TransactionStatus() : countsForBalance(false), sortKey(""), + matures_in(0), status(Unconfirmed), depth(0), open_for(0) { } enum Status { @@ -61,8 +60,8 @@ public: finalization */ /**@}*/ - /** Current number of blocks (to know whether cached status is still valid) */ - int cur_num_blocks; + /** Current block hash (to know whether cached status is still valid) */ + uint256 m_cur_block_hash{}; bool needsUpdate; }; @@ -138,11 +137,11 @@ public: /** Update status from core wallet tx. */ - void updateStatus(const interfaces::WalletTxStatus& wtx, int numBlocks, int64_t block_time); + void updateStatus(const interfaces::WalletTxStatus& wtx, const uint256& block_hash, int numBlocks, int64_t block_time); /** Return whether a status update is needed. */ - bool statusUpdateNeeded(int numBlocks) const; + bool statusUpdateNeeded(const uint256& block_hash) const; }; #endif // BITCOIN_QT_TRANSACTIONRECORD_H diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 18554aef1f..04eb1ae706 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -176,7 +176,7 @@ public: return cachedWallet.size(); } - TransactionRecord *index(interfaces::Wallet& wallet, const int cur_num_blocks, const int idx) + TransactionRecord* index(interfaces::Wallet& wallet, const uint256& cur_block_hash, const int idx) { if(idx >= 0 && idx < cachedWallet.size()) { @@ -192,8 +192,8 @@ public: interfaces::WalletTxStatus wtx; int numBlocks; int64_t block_time; - if (rec->statusUpdateNeeded(cur_num_blocks) && wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time)) { - rec->updateStatus(wtx, numBlocks, block_time); + if (rec->statusUpdateNeeded(cur_block_hash) && wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time)) { + rec->updateStatus(wtx, cur_block_hash, numBlocks, block_time); } return rec; } @@ -618,7 +618,7 @@ QVariant TransactionTableModel::data(const QModelIndex &index, int role) const return details; } case ConfirmedRole: - return rec->status.countsForBalance; + return rec->status.status == TransactionStatus::Status::Confirming || rec->status.status == TransactionStatus::Status::Confirmed; case FormattedAmountRole: // Used for copy/export, so don't include separators return formatTxAmount(rec, false, BitcoinUnits::separatorNever); @@ -664,7 +664,7 @@ QVariant TransactionTableModel::headerData(int section, Qt::Orientation orientat QModelIndex TransactionTableModel::index(int row, int column, const QModelIndex &parent) const { Q_UNUSED(parent); - TransactionRecord *data = priv->index(walletModel->wallet(), walletModel->clientModel().getNumBlocks(), row); + TransactionRecord* data = priv->index(walletModel->wallet(), walletModel->clientModel().getBestBlockHash(), row); if(data) { return createIndex(row, column, data); diff --git a/src/qt/transactiontablemodel.h b/src/qt/transactiontablemodel.h index 7a7d98962b..f06f0ea15f 100644 --- a/src/qt/transactiontablemodel.h +++ b/src/qt/transactiontablemodel.h @@ -76,11 +76,11 @@ public: RawDecorationRole, }; - int rowCount(const QModelIndex &parent) const; - int columnCount(const QModelIndex &parent) const; - QVariant data(const QModelIndex &index, int role) const; - QVariant headerData(int section, Qt::Orientation orientation, int role) const; - QModelIndex index(int row, int column, const QModelIndex & parent = QModelIndex()) const; + int rowCount(const QModelIndex &parent) const override; + int columnCount(const QModelIndex &parent) const override; + QVariant data(const QModelIndex &index, int role) const override; + QVariant headerData(int section, Qt::Orientation orientation, int role) const override; + QModelIndex index(int row, int column, const QModelIndex & parent = QModelIndex()) const override; bool processingQueuedTransactions() const { return fProcessingQueuedTransactions; } private: diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 3c638fb358..3df81807f0 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -8,6 +8,7 @@ #include <qt/bitcoinunits.h> #include <qt/csvmodelwriter.h> #include <qt/editaddressdialog.h> +#include <qt/guiutil.h> #include <qt/optionsmodel.h> #include <qt/platformstyle.h> #include <qt/transactiondescdialog.h> @@ -36,8 +37,7 @@ #include <QVBoxLayout> TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *parent) : - QWidget(parent), model(nullptr), transactionProxyModel(nullptr), - transactionView(nullptr), abandonAction(nullptr), bumpFeeAction(nullptr), columnResizingFixer(nullptr) + QWidget(parent) { // Build filter row setContentsMargins(0,0,0,0); @@ -152,8 +152,8 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa abandonAction = new QAction(tr("Abandon transaction"), this); bumpFeeAction = new QAction(tr("Increase transaction fee"), this); bumpFeeAction->setObjectName("bumpFeeAction"); - QAction *copyAddressAction = new QAction(tr("Copy address"), this); - QAction *copyLabelAction = new QAction(tr("Copy label"), this); + copyAddressAction = new QAction(tr("Copy address"), this); + copyLabelAction = new QAction(tr("Copy label"), this); QAction *copyAmountAction = new QAction(tr("Copy amount"), this); QAction *copyTxIDAction = new QAction(tr("Copy transaction ID"), this); QAction *copyTxHexAction = new QAction(tr("Copy raw transaction"), this); @@ -395,10 +395,11 @@ void TransactionView::contextualMenu(const QPoint &point) hash.SetHex(selection.at(0).data(TransactionTableModel::TxHashRole).toString().toStdString()); abandonAction->setEnabled(model->wallet().transactionCanBeAbandoned(hash)); bumpFeeAction->setEnabled(model->wallet().transactionCanBeBumped(hash)); + copyAddressAction->setEnabled(GUIUtil::hasEntryData(transactionView, 0, TransactionTableModel::AddressRole)); + copyLabelAction->setEnabled(GUIUtil::hasEntryData(transactionView, 0, TransactionTableModel::LabelRole)); - if(index.isValid()) - { - contextMenu->popup(transactionView->viewport()->mapToGlobal(point)); + if (index.isValid()) { + GUIUtil::PopupMenu(contextMenu, transactionView->viewport()->mapToGlobal(point)); } } diff --git a/src/qt/transactionview.h b/src/qt/transactionview.h index eca5656077..9ce7f4ad97 100644 --- a/src/qt/transactionview.h +++ b/src/qt/transactionview.h @@ -60,9 +60,9 @@ public: }; private: - WalletModel *model; - TransactionFilterProxy *transactionProxyModel; - QTableView *transactionView; + WalletModel *model{nullptr}; + TransactionFilterProxy *transactionProxyModel{nullptr}; + QTableView *transactionView{nullptr}; QComboBox *dateWidget; QComboBox *typeWidget; @@ -75,16 +75,18 @@ private: QFrame *dateRangeWidget; QDateTimeEdit *dateFrom; QDateTimeEdit *dateTo; - QAction *abandonAction; - QAction *bumpFeeAction; + QAction *abandonAction{nullptr}; + QAction *bumpFeeAction{nullptr}; + QAction *copyAddressAction{nullptr}; + QAction *copyLabelAction{nullptr}; QWidget *createDateRangeWidget(); - GUIUtil::TableViewLastColumnResizingFixer *columnResizingFixer; + GUIUtil::TableViewLastColumnResizingFixer *columnResizingFixer{nullptr}; - virtual void resizeEvent(QResizeEvent* event); + virtual void resizeEvent(QResizeEvent* event) override; - bool eventFilter(QObject *obj, QEvent *event); + bool eventFilter(QObject *obj, QEvent *event) override; private Q_SLOTS: void contextualMenu(const QPoint &); diff --git a/src/qt/utilitydialog.h b/src/qt/utilitydialog.h index b6a42d3d9d..425b468f40 100644 --- a/src/qt/utilitydialog.h +++ b/src/qt/utilitydialog.h @@ -51,7 +51,7 @@ public: static QWidget* showShutdownWindow(QMainWindow* window); protected: - void closeEvent(QCloseEvent *event); + void closeEvent(QCloseEvent *event) override; }; #endif // BITCOIN_QT_UTILITYDIALOG_H diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index 7cde3ca30b..f1438e3439 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -92,6 +92,23 @@ void WalletController::closeWallet(WalletModel* wallet_model, QWidget* parent) removeAndDeleteWallet(wallet_model); } +void WalletController::closeAllWallets(QWidget* parent) +{ + QMessageBox::StandardButton button = QMessageBox::question(parent, tr("Close all wallets"), + tr("Are you sure you wish to close all wallets?"), + QMessageBox::Yes|QMessageBox::Cancel, + QMessageBox::Yes); + if (button != QMessageBox::Yes) return; + + QMutexLocker locker(&m_mutex); + for (WalletModel* wallet_model : m_wallets) { + wallet_model->wallet().remove(); + Q_EMIT walletRemoved(wallet_model); + delete wallet_model; + } + m_wallets.clear(); +} + WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wallet> wallet) { QMutexLocker locker(&m_mutex); @@ -248,7 +265,7 @@ void CreateWalletActivity::finish() if (!m_error_message.original.empty()) { QMessageBox::critical(m_parent_widget, tr("Create wallet failed"), QString::fromStdString(m_error_message.translated)); } else if (!m_warning_message.empty()) { - QMessageBox::warning(m_parent_widget, tr("Create wallet warning"), QString::fromStdString(Join(m_warning_message, "\n", OpTranslated))); + QMessageBox::warning(m_parent_widget, tr("Create wallet warning"), QString::fromStdString(Join(m_warning_message, Untranslated("\n")).translated)); } if (m_wallet_model) Q_EMIT created(m_wallet_model); @@ -289,7 +306,7 @@ void OpenWalletActivity::finish() if (!m_error_message.original.empty()) { QMessageBox::critical(m_parent_widget, tr("Open wallet failed"), QString::fromStdString(m_error_message.translated)); } else if (!m_warning_message.empty()) { - QMessageBox::warning(m_parent_widget, tr("Open wallet warning"), QString::fromStdString(Join(m_warning_message, "\n", OpTranslated))); + QMessageBox::warning(m_parent_widget, tr("Open wallet warning"), QString::fromStdString(Join(m_warning_message, Untranslated("\n")).translated)); } if (m_wallet_model) Q_EMIT opened(m_wallet_model); diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index 24dd83adf7..f7e366878d 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -62,6 +62,7 @@ public: std::map<std::string, bool> listWalletDir() const; void closeWallet(WalletModel* wallet_model, QWidget* parent = nullptr); + void closeAllWallets(QWidget* parent = nullptr); Q_SIGNALS: void walletAdded(WalletModel* wallet_model); diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 02a9583ae9..5e68ee4f93 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -53,6 +53,7 @@ bool WalletFrame::addWallet(WalletModel *walletModel) walletView->setClientModel(clientModel); walletView->setWalletModel(walletModel); walletView->showOutOfSyncWarning(bOutOfSync); + walletView->setPrivacy(gui->isPrivacyModeActivated()); WalletView* current_wallet_view = currentWalletView(); if (current_wallet_view) { @@ -73,6 +74,7 @@ bool WalletFrame::addWallet(WalletModel *walletModel) connect(walletView, &WalletView::encryptionStatusChanged, gui, &BitcoinGUI::updateWalletStatus); connect(walletView, &WalletView::incomingTransaction, gui, &BitcoinGUI::incomingTransaction); connect(walletView, &WalletView::hdEnabledStatusChanged, gui, &BitcoinGUI::updateWalletStatus); + connect(gui, &BitcoinGUI::setPrivacy, walletView, &WalletView::setPrivacy); return true; } diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 70ee7f4917..671b5e1ce6 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -39,14 +39,14 @@ WalletModel::WalletModel(std::unique_ptr<interfaces::Wallet> wallet, ClientModel& client_model, const PlatformStyle *platformStyle, QObject *parent) : QObject(parent), m_wallet(std::move(wallet)), - m_client_model(client_model), + m_client_model(&client_model), m_node(client_model.node()), optionsModel(client_model.getOptionsModel()), addressTableModel(nullptr), transactionTableModel(nullptr), recentRequestsTableModel(nullptr), cachedEncryptionStatus(Unencrypted), - cachedNumBlocks(0) + timer(new QTimer(this)) { fHaveWatchOnly = m_wallet->haveWatchOnly(); addressTableModel = new AddressTableModel(this); @@ -64,11 +64,16 @@ WalletModel::~WalletModel() void WalletModel::startPollBalance() { // This timer will be fired repeatedly to update the balance - QTimer* timer = new QTimer(this); connect(timer, &QTimer::timeout, this, &WalletModel::pollBalanceChanged); timer->start(MODEL_UPDATE_DELAY); } +void WalletModel::setClientModel(ClientModel* client_model) +{ + m_client_model = client_model; + if (!m_client_model) timer->stop(); +} + void WalletModel::updateStatus() { EncryptionStatus newEncryptionStatus = getEncryptionStatus(); @@ -80,24 +85,30 @@ void WalletModel::updateStatus() void WalletModel::pollBalanceChanged() { + // Avoid recomputing wallet balances unless a TransactionChanged or + // BlockTip notification was received. + if (!fForceCheckBalanceChanged && m_cached_last_update_tip == m_client_model->getBestBlockHash()) return; + // Try to get balances and return early if locks can't be acquired. This // avoids the GUI from getting stuck on periodical polls if the core is // holding the locks for a longer time - for example, during a wallet // rescan. interfaces::WalletBalances new_balances; - int numBlocks = -1; - if (!m_wallet->tryGetBalances(new_balances, numBlocks, fForceCheckBalanceChanged, cachedNumBlocks)) { + uint256 block_hash; + if (!m_wallet->tryGetBalances(new_balances, block_hash)) { return; } - fForceCheckBalanceChanged = false; + if (fForceCheckBalanceChanged || block_hash != m_cached_last_update_tip) { + fForceCheckBalanceChanged = false; - // Balance and number of transactions might have changed - cachedNumBlocks = numBlocks; + // Balance and number of transactions might have changed + m_cached_last_update_tip = block_hash; - checkBalanceChanged(new_balances); - if(transactionTableModel) - transactionTableModel->updateConfirmations(); + checkBalanceChanged(new_balances); + if(transactionTableModel) + transactionTableModel->updateConfirmations(); + } } void WalletModel::checkBalanceChanged(const interfaces::WalletBalances& new_balances) @@ -304,16 +315,10 @@ WalletModel::EncryptionStatus WalletModel::getEncryptionStatus() const bool WalletModel::setWalletEncrypted(bool encrypted, const SecureString &passphrase) { - if(encrypted) - { - // Encrypt + if (encrypted) { return m_wallet->encryptWallet(passphrase); } - else - { - // Decrypt -- TODO; not supported yet - return false; - } + return false; } bool WalletModel::setWalletLocked(bool locked, const SecureString &passPhrase) @@ -578,3 +583,8 @@ bool WalletModel::isMultiwallet() { return m_node.getWallets().size() > 1; } + +void WalletModel::refresh(bool pk_hash_only) +{ + addressTableModel = new AddressTableModel(this, pk_hash_only); +} diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 07004b7c6b..38e8a14556 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -144,7 +144,8 @@ public: interfaces::Node& node() const { return m_node; } interfaces::Wallet& wallet() const { return *m_wallet; } - ClientModel& clientModel() const { return m_client_model; } + ClientModel& clientModel() const { return *m_client_model; } + void setClientModel(ClientModel* client_model); QString getWalletName() const; QString getDisplayName() const; @@ -152,6 +153,8 @@ public: bool isMultiwallet(); AddressTableModel* getAddressTableModel() const { return addressTableModel; } + + void refresh(bool pk_hash_only = false); private: std::unique_ptr<interfaces::Wallet> m_wallet; std::unique_ptr<interfaces::Handler> m_handler_unload; @@ -161,7 +164,7 @@ private: std::unique_ptr<interfaces::Handler> m_handler_show_progress; std::unique_ptr<interfaces::Handler> m_handler_watch_only_changed; std::unique_ptr<interfaces::Handler> m_handler_can_get_addrs_changed; - ClientModel& m_client_model; + ClientModel* m_client_model; interfaces::Node& m_node; bool fHaveWatchOnly; @@ -178,7 +181,10 @@ private: // Cache some values to be able to detect changes interfaces::WalletBalances m_cached_balances; EncryptionStatus cachedEncryptionStatus; - int cachedNumBlocks; + QTimer* timer; + + // Block hash denoting when the last balance update was done. + uint256 m_cached_last_update_tip{}; void subscribeToCoreSignals(); void unsubscribeFromCoreSignals(); diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index 5d9b420df7..861d1c5f4a 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -85,6 +85,8 @@ WalletView::WalletView(const PlatformStyle *_platformStyle, QWidget *parent): connect(sendCoinsPage, &SendCoinsDialog::message, this, &WalletView::message); // Pass through messages from transactionView connect(transactionView, &TransactionView::message, this, &WalletView::message); + + connect(this, &WalletView::setPrivacy, overviewPage, &OverviewPage::setPrivacy); } WalletView::~WalletView() @@ -97,6 +99,7 @@ void WalletView::setClientModel(ClientModel *_clientModel) overviewPage->setClientModel(_clientModel); sendCoinsPage->setClientModel(_clientModel); + if (walletModel) walletModel->setClientModel(_clientModel); } void WalletView::setWalletModel(WalletModel *_walletModel) @@ -255,7 +258,7 @@ void WalletView::gotoLoadPSBT() TransactionError result = BroadcastTransaction(*clientModel->node().context(), tx, err_string, DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK(), /* relay */ true, /* wait_callback */ false); if (result == TransactionError::OK) { - Q_EMIT message(tr("Success"), tr("Broadcasted transaction sucessfully."), CClientUIInterface::MSG_INFORMATION | CClientUIInterface::MODAL); + Q_EMIT message(tr("Success"), tr("Broadcasted transaction successfully."), CClientUIInterface::MSG_INFORMATION | CClientUIInterface::MODAL); } else { Q_EMIT message(tr("Error"), QString::fromStdString(err_string), CClientUIInterface::MSG_ERROR); } diff --git a/src/qt/walletview.h b/src/qt/walletview.h index 11f894e7f8..fd09456baa 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -115,6 +115,7 @@ public Q_SLOTS: void requestedSyncWarningInfo(); Q_SIGNALS: + void setPrivacy(bool privacy); void transactionClicked(); void coinsSent(); /** Fired when a message should be reported to the user */ diff --git a/src/random.cpp b/src/random.cpp index 5b8782d1ce..9c9a35709a 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -14,16 +14,14 @@ #include <wincrypt.h> #endif #include <logging.h> // for LogPrintf() +#include <randomenv.h> +#include <support/allocators/secure.h> #include <sync.h> // for Mutex #include <util/time.h> // for GetTimeMicros() #include <stdlib.h> #include <thread> -#include <randomenv.h> - -#include <support/allocators/secure.h> - #ifndef WIN32 #include <fcntl.h> #include <sys/time.h> @@ -590,16 +588,6 @@ uint64_t GetRand(uint64_t nMax) noexcept return FastRandomContext(g_mock_deterministic_tests).randrange(nMax); } -std::chrono::microseconds GetRandMicros(std::chrono::microseconds duration_max) noexcept -{ - return std::chrono::microseconds{GetRand(duration_max.count())}; -} - -std::chrono::milliseconds GetRandMillis(std::chrono::milliseconds duration_max) noexcept -{ - return std::chrono::milliseconds{GetRand(duration_max.count())}; -} - int GetRandInt(int nMax) noexcept { return GetRand(nMax); diff --git a/src/random.h b/src/random.h index 690125079b..0c6dc24983 100644 --- a/src/random.h +++ b/src/random.h @@ -67,9 +67,21 @@ * Thread-safe. */ void GetRandBytes(unsigned char* buf, int num) noexcept; +/** Generate a uniform random integer in the range [0..range). Precondition: range > 0 */ uint64_t GetRand(uint64_t nMax) noexcept; -std::chrono::microseconds GetRandMicros(std::chrono::microseconds duration_max) noexcept; -std::chrono::milliseconds GetRandMillis(std::chrono::milliseconds duration_max) noexcept; +/** Generate a uniform random duration in the range [0..max). Precondition: max.count() > 0 */ +template <typename D> +D GetRandomDuration(typename std::common_type<D>::type max) noexcept +// Having the compiler infer the template argument from the function argument +// is dangerous, because the desired return value generally has a different +// type than the function argument. So std::common_type is used to force the +// call site to specify the type of the return value. +{ + assert(max.count() > 0); + return D{GetRand(max.count())}; +}; +constexpr auto GetRandMicros = GetRandomDuration<std::chrono::microseconds>; +constexpr auto GetRandMillis = GetRandomDuration<std::chrono::milliseconds>; int GetRandInt(int nMax) noexcept; uint256 GetRandHash() noexcept; diff --git a/src/rest.cpp b/src/rest.cpp index 5f99e26bad..cde8b472d3 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -18,6 +18,7 @@ #include <sync.h> #include <txmempool.h> #include <util/check.h> +#include <util/ref.h> #include <util/strencodings.h> #include <validation.h> #include <version.h> @@ -49,18 +50,13 @@ struct CCoin { uint32_t nHeight; CTxOut out; - ADD_SERIALIZE_METHODS; - CCoin() : nHeight(0) {} explicit CCoin(Coin&& in) : nHeight(in.nHeight), out(std::move(in.out)) {} - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CCoin, obj) { uint32_t nTxVerDummy = 0; - READWRITE(nTxVerDummy); - READWRITE(nHeight); - READWRITE(out); + READWRITE(nTxVerDummy, obj.nHeight, obj.out); } }; @@ -80,13 +76,14 @@ static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string me * @param[in] req the HTTP request * return pointer to the mempool or nullptr if no mempool found */ -static CTxMemPool* GetMemPool(HTTPRequest* req) +static CTxMemPool* GetMemPool(const util::Ref& context, HTTPRequest* req) { - if (!g_rpc_node || !g_rpc_node->mempool) { + NodeContext* node = context.Has<NodeContext>() ? &context.Get<NodeContext>() : nullptr; + if (!node || !node->mempool) { RESTERR(req, HTTP_NOT_FOUND, "Mempool disabled or instance not found"); return nullptr; } - return g_rpc_node->mempool; + return node->mempool; } static RetFormat ParseDataFormat(std::string& param, const std::string& strReq) @@ -134,7 +131,8 @@ static bool CheckWarmup(HTTPRequest* req) return true; } -static bool rest_headers(HTTPRequest* req, +static bool rest_headers(const util::Ref& context, + HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) @@ -275,12 +273,12 @@ static bool rest_block(HTTPRequest* req, } } -static bool rest_block_extended(HTTPRequest* req, const std::string& strURIPart) +static bool rest_block_extended(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { return rest_block(req, strURIPart, true); } -static bool rest_block_notxdetails(HTTPRequest* req, const std::string& strURIPart) +static bool rest_block_notxdetails(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { return rest_block(req, strURIPart, false); } @@ -288,7 +286,7 @@ static bool rest_block_notxdetails(HTTPRequest* req, const std::string& strURIPa // A bit of a hack - dependency on a function defined in rpc/blockchain.cpp UniValue getblockchaininfo(const JSONRPCRequest& request); -static bool rest_chaininfo(HTTPRequest* req, const std::string& strURIPart) +static bool rest_chaininfo(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; @@ -297,7 +295,7 @@ static bool rest_chaininfo(HTTPRequest* req, const std::string& strURIPart) switch (rf) { case RetFormat::JSON: { - JSONRPCRequest jsonRequest; + JSONRPCRequest jsonRequest(context); jsonRequest.params = UniValue(UniValue::VARR); UniValue chainInfoObject = getblockchaininfo(jsonRequest); std::string strJSON = chainInfoObject.write() + "\n"; @@ -311,11 +309,11 @@ static bool rest_chaininfo(HTTPRequest* req, const std::string& strURIPart) } } -static bool rest_mempool_info(HTTPRequest* req, const std::string& strURIPart) +static bool rest_mempool_info(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; - const CTxMemPool* mempool = GetMemPool(req); + const CTxMemPool* mempool = GetMemPool(context, req); if (!mempool) return false; std::string param; const RetFormat rf = ParseDataFormat(param, strURIPart); @@ -335,10 +333,10 @@ static bool rest_mempool_info(HTTPRequest* req, const std::string& strURIPart) } } -static bool rest_mempool_contents(HTTPRequest* req, const std::string& strURIPart) +static bool rest_mempool_contents(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; - const CTxMemPool* mempool = GetMemPool(req); + const CTxMemPool* mempool = GetMemPool(context, req); if (!mempool) return false; std::string param; const RetFormat rf = ParseDataFormat(param, strURIPart); @@ -358,7 +356,7 @@ static bool rest_mempool_contents(HTTPRequest* req, const std::string& strURIPar } } -static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) +static bool rest_tx(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; @@ -414,7 +412,7 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) } } -static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) +static bool rest_getutxos(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; @@ -523,7 +521,7 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) }; if (fCheckMemPool) { - const CTxMemPool* mempool = GetMemPool(req); + const CTxMemPool* mempool = GetMemPool(context, req); if (!mempool) return false; // use db+mempool as cache backend in case user likes to query mempool LOCK2(cs_main, mempool->cs); @@ -600,7 +598,7 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) } } -static bool rest_blockhash_by_height(HTTPRequest* req, +static bool rest_blockhash_by_height(const util::Ref& context, HTTPRequest* req, const std::string& str_uri_part) { if (!CheckWarmup(req)) return false; @@ -648,7 +646,7 @@ static bool rest_blockhash_by_height(HTTPRequest* req, static const struct { const char* prefix; - bool (*handler)(HTTPRequest* req, const std::string& strReq); + bool (*handler)(const util::Ref& context, HTTPRequest* req, const std::string& strReq); } uri_prefixes[] = { {"/rest/tx/", rest_tx}, {"/rest/block/notxdetails/", rest_block_notxdetails}, @@ -661,10 +659,12 @@ static const struct { {"/rest/blockhashbyheight/", rest_blockhash_by_height}, }; -void StartREST() +void StartREST(const util::Ref& context) { - for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) - RegisterHTTPHandler(uri_prefixes[i].prefix, false, uri_prefixes[i].handler); + for (const auto& up : uri_prefixes) { + auto handler = [&context, up](HTTPRequest* req, const std::string& prefix) { return up.handler(context, req, prefix); }; + RegisterHTTPHandler(up.prefix, false, handler); + } } void InterruptREST() diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f7ccbae706..eedb1c3d71 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -29,6 +29,7 @@ #include <txdb.h> #include <txmempool.h> #include <undo.h> +#include <util/ref.h> #include <util/strencodings.h> #include <util/system.h> #include <validation.h> @@ -51,15 +52,29 @@ struct CUpdatedBlock static Mutex cs_blockchange; static std::condition_variable cond_blockchange; -static CUpdatedBlock latestblock; +static CUpdatedBlock latestblock GUARDED_BY(cs_blockchange); -CTxMemPool& EnsureMemPool() +NodeContext& EnsureNodeContext(const util::Ref& context) { - CHECK_NONFATAL(g_rpc_node); - if (!g_rpc_node->mempool) { + if (!context.Has<NodeContext>()) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Node context not found"); + } + return context.Get<NodeContext>(); +} + +CTxMemPool& EnsureMemPool(const util::Ref& context) +{ + NodeContext& node = EnsureNodeContext(context); + if (!node.mempool) { throw JSONRPCError(RPC_CLIENT_MEMPOOL_DISABLED, "Mempool disabled or instance not found"); } - return *g_rpc_node->mempool; + return *node.mempool; +} + +ChainstateManager& EnsureChainman(const util::Ref& context) +{ + NodeContext& node = EnsureNodeContext(context); + return EnsureChainman(node); } /* Calculate the difficulty for a given block index. @@ -205,10 +220,10 @@ static UniValue getbestblockhash(const JSONRPCRequest& request) return ::ChainActive().Tip()->GetBlockHash().GetHex(); } -void RPCNotifyBlockChange(bool ibd, const CBlockIndex * pindex) +void RPCNotifyBlockChange(const CBlockIndex* pindex) { if(pindex) { - std::lock_guard<std::mutex> lock(cs_blockchange); + LOCK(cs_blockchange); latestblock.hash = pindex->GetBlockHash(); latestblock.height = pindex->nHeight; } @@ -243,9 +258,9 @@ static UniValue waitfornewblock(const JSONRPCRequest& request) WAIT_LOCK(cs_blockchange, lock); block = latestblock; if(timeout) - cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&block]{return latestblock.height != block.height || latestblock.hash != block.hash || !IsRPCRunning(); }); + cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&block]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height != block.height || latestblock.hash != block.hash || !IsRPCRunning(); }); else - cond_blockchange.wait(lock, [&block]{return latestblock.height != block.height || latestblock.hash != block.hash || !IsRPCRunning(); }); + cond_blockchange.wait(lock, [&block]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height != block.height || latestblock.hash != block.hash || !IsRPCRunning(); }); block = latestblock; } UniValue ret(UniValue::VOBJ); @@ -285,9 +300,9 @@ static UniValue waitforblock(const JSONRPCRequest& request) { WAIT_LOCK(cs_blockchange, lock); if(timeout) - cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&hash]{return latestblock.hash == hash || !IsRPCRunning();}); + cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&hash]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.hash == hash || !IsRPCRunning();}); else - cond_blockchange.wait(lock, [&hash]{return latestblock.hash == hash || !IsRPCRunning(); }); + cond_blockchange.wait(lock, [&hash]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.hash == hash || !IsRPCRunning(); }); block = latestblock; } @@ -329,9 +344,9 @@ static UniValue waitforblockheight(const JSONRPCRequest& request) { WAIT_LOCK(cs_blockchange, lock); if(timeout) - cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&height]{return latestblock.height >= height || !IsRPCRunning();}); + cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&height]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height >= height || !IsRPCRunning();}); else - cond_blockchange.wait(lock, [&height]{return latestblock.height >= height || !IsRPCRunning(); }); + cond_blockchange.wait(lock, [&height]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height >= height || !IsRPCRunning(); }); block = latestblock; } UniValue ret(UniValue::VOBJ); @@ -399,6 +414,7 @@ static std::vector<RPCResult> MempoolEntryDescription() { return { RPCResult{RPCResult::Type::ARR, "spentby", "unconfirmed transactions spending outputs from this transaction", {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "child transaction id"}}}, RPCResult{RPCResult::Type::BOOL, "bip125-replaceable", "Whether this transaction could be replaced due to BIP125 (replace-by-fee)"}, + RPCResult{RPCResult::Type::BOOL, "unbroadcast", "Whether this transaction is currently unbroadcast (initial broadcast not yet acknowledged by any peers)"}, };} static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPoolEntry& e) EXCLUSIVE_LOCKS_REQUIRED(pool.cs) @@ -460,6 +476,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool } info.pushKV("bip125-replaceable", rbfStatus); + info.pushKV("unbroadcast", pool.IsUnbroadcastTx(tx.GetHash())); } UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose) @@ -519,7 +536,7 @@ static UniValue getrawmempool(const JSONRPCRequest& request) if (!request.params[0].isNull()) fVerbose = request.params[0].get_bool(); - return MempoolToJSON(EnsureMemPool(), fVerbose); + return MempoolToJSON(EnsureMemPool(request.context), fVerbose); } static UniValue getmempoolancestors(const JSONRPCRequest& request) @@ -549,7 +566,7 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request) uint256 hash = ParseHashV(request.params[0], "parameter 1"); - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); LOCK(mempool.cs); CTxMemPool::txiter it = mempool.mapTx.find(hash); @@ -612,7 +629,7 @@ static UniValue getmempooldescendants(const JSONRPCRequest& request) uint256 hash = ParseHashV(request.params[0], "parameter 1"); - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); LOCK(mempool.cs); CTxMemPool::txiter it = mempool.mapTx.find(hash); @@ -662,7 +679,7 @@ static UniValue getmempoolentry(const JSONRPCRequest& request) uint256 hash = ParseHashV(request.params[0], "parameter 1"); - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); LOCK(mempool.cs); CTxMemPool::txiter it = mempool.mapTx.find(hash); @@ -979,7 +996,7 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) ::ChainstateActive().ForceFlushStateToDisk(); CCoinsView* coins_view = WITH_LOCK(cs_main, return &ChainstateActive().CoinsDB()); - if (GetUTXOStats(coins_view, stats)) { + if (GetUTXOStats(coins_view, stats, RpcInterruptionPoint)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); ret.pushKV("transactions", (int64_t)stats.nTransactions); @@ -1045,7 +1062,7 @@ UniValue gettxout(const JSONRPCRequest& request) CCoinsViewCache* coins_view = &::ChainstateActive().CoinsTip(); if (fMempool) { - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); LOCK(mempool.cs); CCoinsViewMemPool view(coins_view, mempool); if (!view.GetCoin(out, coin) || mempool.isSpent(out)) { @@ -1078,7 +1095,8 @@ static UniValue verifychain(const JSONRPCRequest& request) RPCHelpMan{"verifychain", "\nVerifies blockchain database.\n", { - {"checklevel", RPCArg::Type::NUM, /* default */ strprintf("%d, range=0-4", DEFAULT_CHECKLEVEL), "How thorough the block verification is."}, + {"checklevel", RPCArg::Type::NUM, /* default */ strprintf("%d, range=0-4", DEFAULT_CHECKLEVEL), + strprintf("How thorough the block verification is:\n - %s", Join(CHECKLEVEL_DOC, "\n- "))}, {"nblocks", RPCArg::Type::NUM, /* default */ strprintf("%d, 0=all", DEFAULT_CHECKBLOCKS), "The number of blocks to check."}, }, RPCResult{ @@ -1389,7 +1407,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) ret.pushKV("maxmempool", (int64_t) maxmempool); ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); ret.pushKV("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); - + ret.pushKV("unbroadcastcount", uint64_t{pool.GetUnbroadcastTxs().size()}); return ret; } @@ -1408,6 +1426,7 @@ static UniValue getmempoolinfo(const JSONRPCRequest& request) {RPCResult::Type::NUM, "maxmempool", "Maximum memory usage for the mempool"}, {RPCResult::Type::STR_AMOUNT, "mempoolminfee", "Minimum fee rate in " + CURRENCY_UNIT + "/kB for tx to be accepted. Is the maximum of minrelaytxfee and minimum mempool fee"}, {RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"}, + {RPCResult::Type::NUM, "unbroadcastcount", "Current number of transactions that haven't passed initial broadcast yet"} }}, RPCExamples{ HelpExampleCli("getmempoolinfo", "") @@ -1415,7 +1434,7 @@ static UniValue getmempoolinfo(const JSONRPCRequest& request) }, }.Check(request); - return MempoolInfoToJSON(EnsureMemPool()); + return MempoolInfoToJSON(EnsureMemPool(request.context)); } static UniValue preciousblock(const JSONRPCRequest& request) @@ -1934,7 +1953,7 @@ static UniValue savemempool(const JSONRPCRequest& request) }, }.Check(request); - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); if (!mempool.IsLoaded()) { throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); @@ -1956,6 +1975,7 @@ bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& Coin coin; if (!cursor->GetKey(key) || !cursor->GetValue(coin)) return false; if (++count % 8192 == 0) { + RpcInterruptionPoint(); if (should_abort) { // allow to abort the scan via the abort reference return false; @@ -1976,7 +1996,6 @@ bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& } /** RAII object to prevent concurrency issue when scanning the txout set */ -static std::mutex g_utxosetscan; static std::atomic<int> g_scan_progress; static std::atomic<bool> g_scan_in_progress; static std::atomic<bool> g_should_abort_scan; @@ -1989,18 +2008,15 @@ public: bool reserve() { CHECK_NONFATAL(!m_could_reserve); - std::lock_guard<std::mutex> lock(g_utxosetscan); - if (g_scan_in_progress) { + if (g_scan_in_progress.exchange(true)) { return false; } - g_scan_in_progress = true; m_could_reserve = true; return true; } ~CoinsViewScanReserver() { if (m_could_reserve) { - std::lock_guard<std::mutex> lock(g_utxosetscan); g_scan_in_progress = false; } } @@ -2299,7 +2315,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) ::ChainstateActive().ForceFlushStateToDisk(); - if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats)) { + if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats, RpcInterruptionPoint)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } @@ -2317,9 +2333,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) unsigned int iter{0}; while (pcursor->Valid()) { - if (iter % 5000 == 0 && !IsRPCRunning()) { - throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); - } + if (iter % 5000 == 0) RpcInterruptionPoint(); ++iter; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { afile << key; @@ -2385,5 +2399,3 @@ static const CRPCCommand commands[] = for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) t.appendCommand(commands[vcidx].name, &commands[vcidx]); } - -NodeContext* g_rpc_node = nullptr; diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index a02e5fae0e..5c9a43b13e 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -16,8 +16,12 @@ extern RecursiveMutex cs_main; class CBlock; class CBlockIndex; class CTxMemPool; +class ChainstateManager; class UniValue; struct NodeContext; +namespace util { +class Ref; +} // namespace util static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; @@ -30,7 +34,7 @@ static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; double GetDifficulty(const CBlockIndex* blockindex); /** Callback for when block tip changed. */ -void RPCNotifyBlockChange(bool ibd, const CBlockIndex *); +void RPCNotifyBlockChange(const CBlockIndex*); /** Block description to JSON */ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails = false) LOCKS_EXCLUDED(cs_main); @@ -47,11 +51,8 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex /** Used by getblockstats to get feerates at different percentiles by weight */ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], std::vector<std::pair<CAmount, int64_t>>& scores, int64_t total_weight); -//! Pointer to node state that needs to be declared as a global to be accessible -//! RPC methods. Due to limitations of the RPC framework, there's currently no -//! direct way to pass in state to RPC methods without globals. -extern NodeContext* g_rpc_node; - -CTxMemPool& EnsureMemPool(); +NodeContext& EnsureNodeContext(const util::Ref& context); +CTxMemPool& EnsureMemPool(const util::Ref& context); +ChainstateManager& EnsureChainman(const util::Ref& context); #endif diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 05d3fd6afb..3db0cb04ed 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -101,7 +101,7 @@ static UniValue getnetworkhashps(const JSONRPCRequest& request) return GetNetworkHashPS(!request.params[0].isNull() ? request.params[0].get_int() : 120, !request.params[1].isNull() ? request.params[1].get_int() : -1); } -static bool GenerateBlock(CBlock& block, uint64_t& max_tries, unsigned int& extra_nonce, uint256& block_hash) +static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& max_tries, unsigned int& extra_nonce, uint256& block_hash) { block_hash.SetNull(); @@ -124,14 +124,15 @@ static bool GenerateBlock(CBlock& block, uint64_t& max_tries, unsigned int& extr } std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); - if (!ProcessNewBlock(chainparams, shared_pblock, true, nullptr)) + if (!chainman.ProcessNewBlock(chainparams, shared_pblock, true, nullptr)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); + } block_hash = block.GetHash(); return true; } -static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) +static UniValue generateBlocks(ChainstateManager& chainman, const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) { int nHeightEnd = 0; int nHeight = 0; @@ -151,7 +152,7 @@ static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbas CBlock *pblock = &pblocktemplate->block; uint256 block_hash; - if (!GenerateBlock(*pblock, nMaxTries, nExtraNonce, block_hash)) { + if (!GenerateBlock(chainman, *pblock, nMaxTries, nExtraNonce, block_hash)) { break; } @@ -178,7 +179,7 @@ static bool getScriptFromDescriptor(const std::string& descriptor, CScript& scri throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); } - // Combo desriptors can have 2 or 4 scripts, so we can't just check scripts.size() == 1 + // Combo descriptors can have 2 or 4 scripts, so we can't just check scripts.size() == 1 CHECK_NONFATAL(scripts.size() > 0 && scripts.size() <= 4); if (scripts.size() == 1) { @@ -227,9 +228,10 @@ static UniValue generatetodescriptor(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); } - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); + ChainstateManager& chainman = EnsureChainman(request.context); - return generateBlocks(mempool, coinbase_script, num_blocks, max_tries); + return generateBlocks(chainman, mempool, coinbase_script, num_blocks, max_tries); } static UniValue generatetoaddress(const JSONRPCRequest& request) @@ -265,11 +267,12 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Error: Invalid address"); } - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); + ChainstateManager& chainman = EnsureChainman(request.context); CScript coinbase_script = GetScriptForDestination(destination); - return generateBlocks(mempool, coinbase_script, nGenerate, nMaxTries); + return generateBlocks(chainman, mempool, coinbase_script, nGenerate, nMaxTries); } static UniValue generateblock(const JSONRPCRequest& request) @@ -311,7 +314,7 @@ static UniValue generateblock(const JSONRPCRequest& request) coinbase_script = GetScriptForDestination(destination); } - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); std::vector<CTransactionRef> txs; const auto raw_txs_or_txids = request.params[1].get_array(); @@ -370,7 +373,7 @@ static UniValue generateblock(const JSONRPCRequest& request) uint64_t max_tries{1000000}; unsigned int extra_nonce{0}; - if (!GenerateBlock(block, max_tries, extra_nonce, block_hash) || block_hash.IsNull()) { + if (!GenerateBlock(EnsureChainman(request.context), block, max_tries, extra_nonce, block_hash) || block_hash.IsNull()) { throw JSONRPCError(RPC_MISC_ERROR, "Failed to make block."); } @@ -403,7 +406,7 @@ static UniValue getmininginfo(const JSONRPCRequest& request) }.Check(request); LOCK(cs_main); - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); UniValue obj(UniValue::VOBJ); obj.pushKV("blocks", (int)::ChainActive().Height()); @@ -449,7 +452,7 @@ static UniValue prioritisetransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "Priority is no longer supported, dummy argument to prioritisetransaction must be 0."); } - EnsureMemPool().PrioritiseTransaction(hash, nAmount); + EnsureMemPool(request.context).PrioritiseTransaction(hash, nAmount); return true; } @@ -635,17 +638,18 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) if (strMode != "template") throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid mode"); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - if (g_rpc_node->connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) + if (node.connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); if (::ChainstateActive().IsInitialBlockDownload()) throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); static unsigned int nTransactionsUpdatedLast; - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); if (!lpval.isNull()) { @@ -787,6 +791,8 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) result.pushKV("capabilities", aCaps); UniValue aRules(UniValue::VARR); + aRules.push_back("csv"); + if (!fPreSegWit) aRules.push_back("!segwit"); UniValue vbavailable(UniValue::VOBJ); for (int j = 0; j < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++j) { Consensus::DeploymentPos pos = Consensus::DeploymentPos(j); @@ -874,7 +880,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) return result; } -class submitblock_StateCatcher : public CValidationInterface +class submitblock_StateCatcher final : public CValidationInterface { public: uint256 hash; @@ -942,17 +948,17 @@ static UniValue submitblock(const JSONRPCRequest& request) } bool new_block; - submitblock_StateCatcher sc(block.GetHash()); - RegisterValidationInterface(&sc); - bool accepted = ProcessNewBlock(Params(), blockptr, /* fForceProcessing */ true, /* fNewBlock */ &new_block); - UnregisterValidationInterface(&sc); + auto sc = std::make_shared<submitblock_StateCatcher>(block.GetHash()); + RegisterSharedValidationInterface(sc); + bool accepted = EnsureChainman(request.context).ProcessNewBlock(Params(), blockptr, /* fForceProcessing */ true, /* fNewBlock */ &new_block); + UnregisterSharedValidationInterface(sc); if (!new_block && accepted) { return "duplicate"; } - if (!sc.found) { + if (!sc->found) { return "inconclusive"; } - return BIP22ValidationResult(sc.state); + return BIP22ValidationResult(sc->state); } static UniValue submitheader(const JSONRPCRequest& request) @@ -983,7 +989,7 @@ static UniValue submitheader(const JSONRPCRequest& request) } BlockValidationState state; - ProcessNewBlockHeaders({h}, state, Params()); + EnsureChainman(request.context).ProcessNewBlockHeaders({h}, state, Params()); if (state.IsValid()) return NullUniValue; if (state.IsError()) { throw JSONRPCError(RPC_VERIFY_ERROR, state.ToString()); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index f3c5fed858..ce98a7c937 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -15,6 +15,7 @@ #include <script/descriptor.h> #include <util/check.h> #include <util/message.h> // For MessageSign(), MessageVerify() +#include <util/ref.h> #include <util/strencodings.h> #include <util/system.h> @@ -366,8 +367,8 @@ static UniValue setmocktime(const JSONRPCRequest& request) RPCTypeCheck(request.params, {UniValue::VNUM}); int64_t time = request.params[0].get_int64(); SetMockTime(time); - if (g_rpc_node) { - for (const auto& chain_client : g_rpc_node->chain_clients) { + if (request.context.Has<NodeContext>()) { + for (const auto& chain_client : request.context.Get<NodeContext>().chain_clients) { chain_client->setMockTime(time); } } @@ -398,9 +399,10 @@ static UniValue mockscheduler(const JSONRPCRequest& request) } // protect against null pointer dereference - CHECK_NONFATAL(g_rpc_node); - CHECK_NONFATAL(g_rpc_node->scheduler); - g_rpc_node->scheduler->MockForward(std::chrono::seconds(delta_seconds)); + CHECK_NONFATAL(request.context.Has<NodeContext>()); + NodeContext& node = request.context.Get<NodeContext>(); + CHECK_NONFATAL(node.scheduler); + node.scheduler->MockForward(std::chrono::seconds(delta_seconds)); return NullUniValue; } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index d6d15f8b56..e29aa03695 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -42,10 +42,11 @@ static UniValue getconnectioncount(const JSONRPCRequest& request) }, }.Check(request); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - return (int)g_rpc_node->connman->GetNodeCount(CConnman::CONNECTIONS_ALL); + return (int)node.connman->GetNodeCount(CConnman::CONNECTIONS_ALL); } static UniValue ping(const JSONRPCRequest& request) @@ -62,11 +63,12 @@ static UniValue ping(const JSONRPCRequest& request) }, }.Check(request); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); // Request that each node send a ping during next message processing pass - g_rpc_node->connman->ForEachNode([](CNode* pnode) { + node.connman->ForEachNode([](CNode* pnode) { pnode->fPingQueued = true; }); return NullUniValue; @@ -139,11 +141,12 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) }, }.Check(request); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); std::vector<CNodeStats> vstats; - g_rpc_node->connman->GetNodeStats(vstats); + node.connman->GetNodeStats(vstats); UniValue ret(UniValue::VARR); @@ -248,7 +251,8 @@ static UniValue addnode(const JSONRPCRequest& request) }, }.ToString()); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); std::string strNode = request.params[0].get_str(); @@ -256,18 +260,18 @@ static UniValue addnode(const JSONRPCRequest& request) if (strCommand == "onetry") { CAddress addr; - g_rpc_node->connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), false, false, true); + node.connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), false, false, true); return NullUniValue; } if (strCommand == "add") { - if(!g_rpc_node->connman->AddNode(strNode)) + if(!node.connman->AddNode(strNode)) throw JSONRPCError(RPC_CLIENT_NODE_ALREADY_ADDED, "Error: Node already added"); } else if(strCommand == "remove") { - if(!g_rpc_node->connman->RemoveAddedNode(strNode)) + if(!node.connman->RemoveAddedNode(strNode)) throw JSONRPCError(RPC_CLIENT_NODE_NOT_ADDED, "Error: Node has not been added."); } @@ -293,7 +297,8 @@ static UniValue disconnectnode(const JSONRPCRequest& request) }, }.Check(request); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); bool success; @@ -302,11 +307,11 @@ static UniValue disconnectnode(const JSONRPCRequest& request) if (!address_arg.isNull() && id_arg.isNull()) { /* handle disconnect-by-address */ - success = g_rpc_node->connman->DisconnectNode(address_arg.get_str()); + success = node.connman->DisconnectNode(address_arg.get_str()); } else if (!id_arg.isNull() && (address_arg.isNull() || (address_arg.isStr() && address_arg.get_str().empty()))) { /* handle disconnect-by-id */ NodeId nodeid = (NodeId) id_arg.get_int64(); - success = g_rpc_node->connman->DisconnectNode(nodeid); + success = node.connman->DisconnectNode(nodeid); } else { throw JSONRPCError(RPC_INVALID_PARAMS, "Only one of address and nodeid should be provided."); } @@ -350,10 +355,11 @@ static UniValue getaddednodeinfo(const JSONRPCRequest& request) }, }.Check(request); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - std::vector<AddedNodeInfo> vInfo = g_rpc_node->connman->GetAddedNodeInfo(); + std::vector<AddedNodeInfo> vInfo = node.connman->GetAddedNodeInfo(); if (!request.params[0].isNull()) { bool found = false; @@ -417,21 +423,22 @@ static UniValue getnettotals(const JSONRPCRequest& request) + HelpExampleRpc("getnettotals", "") }, }.Check(request); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); UniValue obj(UniValue::VOBJ); - obj.pushKV("totalbytesrecv", g_rpc_node->connman->GetTotalBytesRecv()); - obj.pushKV("totalbytessent", g_rpc_node->connman->GetTotalBytesSent()); + obj.pushKV("totalbytesrecv", node.connman->GetTotalBytesRecv()); + obj.pushKV("totalbytessent", node.connman->GetTotalBytesSent()); obj.pushKV("timemillis", GetTimeMillis()); UniValue outboundLimit(UniValue::VOBJ); - outboundLimit.pushKV("timeframe", g_rpc_node->connman->GetMaxOutboundTimeframe()); - outboundLimit.pushKV("target", g_rpc_node->connman->GetMaxOutboundTarget()); - outboundLimit.pushKV("target_reached", g_rpc_node->connman->OutboundTargetReached(false)); - outboundLimit.pushKV("serve_historical_blocks", !g_rpc_node->connman->OutboundTargetReached(true)); - outboundLimit.pushKV("bytes_left_in_cycle", g_rpc_node->connman->GetOutboundTargetBytesLeft()); - outboundLimit.pushKV("time_left_in_cycle", g_rpc_node->connman->GetMaxOutboundTimeLeftInCycle()); + outboundLimit.pushKV("timeframe", node.connman->GetMaxOutboundTimeframe()); + outboundLimit.pushKV("target", node.connman->GetMaxOutboundTarget()); + outboundLimit.pushKV("target_reached", node.connman->OutboundTargetReached(false)); + outboundLimit.pushKV("serve_historical_blocks", !node.connman->OutboundTargetReached(true)); + outboundLimit.pushKV("bytes_left_in_cycle", node.connman->GetOutboundTargetBytesLeft()); + outboundLimit.pushKV("time_left_in_cycle", node.connman->GetMaxOutboundTimeLeftInCycle()); obj.pushKV("uploadtarget", outboundLimit); return obj; } @@ -513,16 +520,17 @@ static UniValue getnetworkinfo(const JSONRPCRequest& request) obj.pushKV("version", CLIENT_VERSION); obj.pushKV("subversion", strSubVersion); obj.pushKV("protocolversion",PROTOCOL_VERSION); - if (g_rpc_node->connman) { - ServiceFlags services = g_rpc_node->connman->GetLocalServices(); + NodeContext& node = EnsureNodeContext(request.context); + if (node.connman) { + ServiceFlags services = node.connman->GetLocalServices(); obj.pushKV("localservices", strprintf("%016x", services)); obj.pushKV("localservicesnames", GetServicesNames(services)); } obj.pushKV("localrelay", g_relay_txes); obj.pushKV("timeoffset", GetTimeOffset()); - if (g_rpc_node->connman) { - obj.pushKV("networkactive", g_rpc_node->connman->GetNetworkActive()); - obj.pushKV("connections", (int)g_rpc_node->connman->GetNodeCount(CConnman::CONNECTIONS_ALL)); + if (node.connman) { + obj.pushKV("networkactive", node.connman->GetNetworkActive()); + obj.pushKV("connections", (int)node.connman->GetNodeCount(CConnman::CONNECTIONS_ALL)); } obj.pushKV("networks", GetNetworksInfo()); obj.pushKV("relayfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); @@ -567,7 +575,8 @@ static UniValue setban(const JSONRPCRequest& request) if (request.fHelp || !help.IsValidNumArgs(request.params.size()) || (strCommand != "add" && strCommand != "remove")) { throw std::runtime_error(help.ToString()); } - if (!g_rpc_node->banman) { + NodeContext& node = EnsureNodeContext(request.context); + if (!node.banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } @@ -591,7 +600,7 @@ static UniValue setban(const JSONRPCRequest& request) if (strCommand == "add") { - if (isSubnet ? g_rpc_node->banman->IsBanned(subNet) : g_rpc_node->banman->IsBanned(netAddr)) { + if (isSubnet ? node.banman->IsBanned(subNet) : node.banman->IsBanned(netAddr)) { throw JSONRPCError(RPC_CLIENT_NODE_ALREADY_ADDED, "Error: IP/Subnet already banned"); } @@ -604,20 +613,20 @@ static UniValue setban(const JSONRPCRequest& request) absolute = true; if (isSubnet) { - g_rpc_node->banman->Ban(subNet, BanReasonManuallyAdded, banTime, absolute); - if (g_rpc_node->connman) { - g_rpc_node->connman->DisconnectNode(subNet); + node.banman->Ban(subNet, BanReasonManuallyAdded, banTime, absolute); + if (node.connman) { + node.connman->DisconnectNode(subNet); } } else { - g_rpc_node->banman->Ban(netAddr, BanReasonManuallyAdded, banTime, absolute); - if (g_rpc_node->connman) { - g_rpc_node->connman->DisconnectNode(netAddr); + node.banman->Ban(netAddr, BanReasonManuallyAdded, banTime, absolute); + if (node.connman) { + node.connman->DisconnectNode(netAddr); } } } else if(strCommand == "remove") { - if (!( isSubnet ? g_rpc_node->banman->Unban(subNet) : g_rpc_node->banman->Unban(netAddr) )) { + if (!( isSubnet ? node.banman->Unban(subNet) : node.banman->Unban(netAddr) )) { throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Unban failed. Requested address/subnet was not previously banned."); } } @@ -645,12 +654,13 @@ static UniValue listbanned(const JSONRPCRequest& request) }, }.Check(request); - if(!g_rpc_node->banman) { + NodeContext& node = EnsureNodeContext(request.context); + if(!node.banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } banmap_t banMap; - g_rpc_node->banman->GetBanned(banMap); + node.banman->GetBanned(banMap); UniValue bannedAddresses(UniValue::VARR); for (const auto& entry : banMap) @@ -679,11 +689,12 @@ static UniValue clearbanned(const JSONRPCRequest& request) + HelpExampleRpc("clearbanned", "") }, }.Check(request); - if (!g_rpc_node->banman) { + NodeContext& node = EnsureNodeContext(request.context); + if (!node.banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } - g_rpc_node->banman->ClearBanned(); + node.banman->ClearBanned(); return NullUniValue; } @@ -699,13 +710,14 @@ static UniValue setnetworkactive(const JSONRPCRequest& request) RPCExamples{""}, }.Check(request); - if (!g_rpc_node->connman) { + NodeContext& node = EnsureNodeContext(request.context); + if (!node.connman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } - g_rpc_node->connman->SetNetworkActive(request.params[0].get_bool()); + node.connman->SetNetworkActive(request.params[0].get_bool()); - return g_rpc_node->connman->GetNetworkActive(); + return node.connman->GetNetworkActive(); } static UniValue getnodeaddresses(const JSONRPCRequest& request) @@ -732,7 +744,8 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) + HelpExampleRpc("getnodeaddresses", "8") }, }.Check(request); - if (!g_rpc_node->connman) { + NodeContext& node = EnsureNodeContext(request.context); + if (!node.connman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } @@ -744,7 +757,7 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) } } // returns a shuffled list of CAddress - std::vector<CAddress> vAddr = g_rpc_node->connman->GetAddresses(); + std::vector<CAddress> vAddr = node.connman->GetAddresses(); UniValue ret(UniValue::VARR); int address_return_count = std::min<int>(count, vAddr.size()); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 063ee1697c..e14217c307 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -653,7 +653,7 @@ static UniValue combinerawtransaction(const JSONRPCRequest& request) CCoinsView viewDummy; CCoinsViewCache view(&viewDummy); { - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); LOCK(cs_main); LOCK(mempool.cs); CCoinsViewCache &viewChain = ::ChainstateActive().CoinsTip(); @@ -778,7 +778,8 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) for (const CTxIn& txin : mtx.vin) { coins[txin.prevout]; // Create empty map entry keyed by prevout. } - FindCoins(*g_rpc_node, coins); + NodeContext& node = EnsureNodeContext(request.context); + FindCoins(node, coins); // Parse the prevtxs array ParsePrevouts(request.params[2], &keystore, coins); @@ -837,7 +838,8 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) std::string err_string; AssertLockNotHeld(cs_main); - const TransactionError err = BroadcastTransaction(*g_rpc_node, tx, err_string, max_raw_tx_fee, /*relay*/ true, /*wait_callback*/ true); + NodeContext& node = EnsureNodeContext(request.context); + const TransactionError err = BroadcastTransaction(node, tx, err_string, max_raw_tx_fee, /*relay*/ true, /*wait_callback*/ true); if (TransactionError::OK != err) { throw JSONRPCTransactionError(err, err_string); } @@ -904,7 +906,7 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) DEFAULT_MAX_RAW_TX_FEE_RATE : CFeeRate(AmountFromValue(request.params[1])); - CTxMemPool& mempool = EnsureMemPool(); + CTxMemPool& mempool = EnsureMemPool(request.context); int64_t virtual_size = GetVirtualTransactionSize(*tx); CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); @@ -1555,7 +1557,7 @@ UniValue utxoupdatepsbt(const JSONRPCRequest& request) CCoinsView viewDummy; CCoinsViewCache view(&viewDummy); { - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); LOCK2(cs_main, mempool.cs); CCoinsViewCache &viewChain = ::ChainstateActive().CoinsTip(); CCoinsViewMemPool viewMempool(&viewChain, mempool); diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index 56cac6661e..7fef45f50e 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -130,20 +130,20 @@ void DeleteAuthCookie() } } -std::vector<UniValue> JSONRPCProcessBatchReply(const UniValue &in, size_t num) +std::vector<UniValue> JSONRPCProcessBatchReply(const UniValue& in) { if (!in.isArray()) { throw std::runtime_error("Batch must be an array"); } + const size_t num {in.size()}; std::vector<UniValue> batch(num); - for (size_t i=0; i<in.size(); ++i) { - const UniValue &rec = in[i]; + for (const UniValue& rec : in.getValues()) { if (!rec.isObject()) { - throw std::runtime_error("Batch member must be object"); + throw std::runtime_error("Batch member must be an object"); } size_t id = rec["id"].get_int(); if (id >= num) { - throw std::runtime_error("Batch member id larger than size"); + throw std::runtime_error("Batch member id is larger than batch size"); } batch[id] = rec; } diff --git a/src/rpc/request.h b/src/rpc/request.h index 99eb4f9354..4761e9e371 100644 --- a/src/rpc/request.h +++ b/src/rpc/request.h @@ -10,6 +10,10 @@ #include <univalue.h> +namespace util { +class Ref; +} // namespace util + UniValue JSONRPCRequestObj(const std::string& strMethod, const UniValue& params, const UniValue& id); UniValue JSONRPCReplyObj(const UniValue& result, const UniValue& error, const UniValue& id); std::string JSONRPCReply(const UniValue& result, const UniValue& error, const UniValue& id); @@ -22,7 +26,7 @@ bool GetAuthCookie(std::string *cookie_out); /** Delete RPC authentication cookie from disk */ void DeleteAuthCookie(); /** Parse JSON-RPC batch reply into a vector */ -std::vector<UniValue> JSONRPCProcessBatchReply(const UniValue &in, size_t num); +std::vector<UniValue> JSONRPCProcessBatchReply(const UniValue& in); class JSONRPCRequest { @@ -34,8 +38,19 @@ public: std::string URI; std::string authUser; std::string peerAddr; + const util::Ref& context; + + JSONRPCRequest(const util::Ref& context) : id(NullUniValue), params(NullUniValue), fHelp(false), context(context) {} + + //! Initializes request information from another request object and the + //! given context. The implementation should be updated if any members are + //! added or removed above. + JSONRPCRequest(const JSONRPCRequest& other, const util::Ref& context) + : id(other.id), strMethod(other.strMethod), params(other.params), fHelp(other.fHelp), URI(other.URI), + authUser(other.authUser), peerAddr(other.peerAddr), context(context) + { + } - JSONRPCRequest() : id(NullUniValue), params(NullUniValue), fHelp(false) {} void parse(const UniValue& valRequest); }; diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index e2618c16da..844f62cbc6 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -11,11 +11,13 @@ #include <util/strencodings.h> #include <util/system.h> -#include <boost/signals2/signal.hpp> #include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/split.hpp> +#include <boost/signals2/signal.hpp> +#include <cassert> #include <memory> // for unique_ptr +#include <mutex> #include <unordered_map> static RecursiveMutex cs_rpcWarmup; @@ -25,7 +27,8 @@ static std::string rpcWarmupStatus GUARDED_BY(cs_rpcWarmup) = "RPC server starte /* Timer-creating functions */ static RPCTimerInterface* timerInterface = nullptr; /* Map of name to timer. */ -static std::map<std::string, std::unique_ptr<RPCTimerBase> > deadlineTimers; +static Mutex g_deadline_timers_mutex; +static std::map<std::string, std::unique_ptr<RPCTimerBase> > deadlineTimers GUARDED_BY(g_deadline_timers_mutex); static bool ExecuteCommand(const CRPCCommand& command, const JSONRPCRequest& request, UniValue& result, bool last_handler); struct RPCCommandExecutionInfo @@ -290,17 +293,26 @@ void StartRPC() void InterruptRPC() { - LogPrint(BCLog::RPC, "Interrupting RPC\n"); - // Interrupt e.g. running longpolls - g_rpc_running = false; + static std::once_flag g_rpc_interrupt_flag; + // This function could be called twice if the GUI has been started with -server=1. + std::call_once(g_rpc_interrupt_flag, []() { + LogPrint(BCLog::RPC, "Interrupting RPC\n"); + // Interrupt e.g. running longpolls + g_rpc_running = false; + }); } void StopRPC() { - LogPrint(BCLog::RPC, "Stopping RPC\n"); - deadlineTimers.clear(); - DeleteAuthCookie(); - g_rpcSignals.Stopped(); + static std::once_flag g_rpc_stop_flag; + // This function could be called twice if the GUI has been started with -server=1. + assert(!g_rpc_running); + std::call_once(g_rpc_stop_flag, []() { + LogPrint(BCLog::RPC, "Stopping RPC\n"); + WITH_LOCK(g_deadline_timers_mutex, deadlineTimers.clear()); + DeleteAuthCookie(); + g_rpcSignals.Stopped(); + }); } bool IsRPCRunning() @@ -308,6 +320,11 @@ bool IsRPCRunning() return g_rpc_running; } +void RpcInterruptionPoint() +{ + if (!IsRPCRunning()) throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); +} + void SetRPCWarmupStatus(const std::string& newStatus) { LOCK(cs_rpcWarmup); @@ -486,6 +503,7 @@ void RPCRunLater(const std::string& name, std::function<void()> func, int64_t nS { if (!timerInterface) throw JSONRPCError(RPC_INTERNAL_ERROR, "No timer handler registered for RPC"); + LOCK(g_deadline_timers_mutex); deadlineTimers.erase(name); LogPrint(BCLog::RPC, "queue run of timer %s in %i seconds (using %s)\n", name, nSeconds, timerInterface->Name()); deadlineTimers.emplace(name, std::unique_ptr<RPCTimerBase>(timerInterface->NewTimer(func, nSeconds*1000))); diff --git a/src/rpc/server.h b/src/rpc/server.h index c91bf1f613..d7a04ff6e8 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -9,10 +9,10 @@ #include <amount.h> #include <rpc/request.h> +#include <functional> #include <map> #include <stdint.h> #include <string> -#include <functional> #include <univalue.h> @@ -29,6 +29,9 @@ namespace RPCServer /** Query whether RPC is running */ bool IsRPCRunning(); +/** Throw JSONRPCError if RPC is not running */ +void RpcInterruptionPoint(); + /** * Set the RPC warmup status. When this is done, all RPC calls will error out * immediately with RPC_IN_WARMUP. diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 860fa198d5..39bf05fbbd 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -843,16 +843,9 @@ UniValue GetServicesNames(ServiceFlags services) { UniValue servicesNames(UniValue::VARR); - if (services & NODE_NETWORK) - servicesNames.push_back("NETWORK"); - if (services & NODE_GETUTXO) - servicesNames.push_back("GETUTXO"); - if (services & NODE_BLOOM) - servicesNames.push_back("BLOOM"); - if (services & NODE_WITNESS) - servicesNames.push_back("WITNESS"); - if (services & NODE_NETWORK_LIMITED) - servicesNames.push_back("NETWORK_LIMITED"); + for (const auto& flag : serviceFlagsToStr(services)) { + servicesNames.push_back(flag); + } return servicesNames; } diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 5279f40506..ed0175bb10 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -481,7 +481,7 @@ public: return AddChecksum(ret); } - bool ToPrivateString(const SigningProvider& arg, std::string& out) const override final + bool ToPrivateString(const SigningProvider& arg, std::string& out) const final { bool ret = ToStringHelper(&arg, out, true); out = AddChecksum(out); diff --git a/src/script/keyorigin.h b/src/script/keyorigin.h index 467605ce46..a318ff0f9d 100644 --- a/src/script/keyorigin.h +++ b/src/script/keyorigin.h @@ -18,13 +18,7 @@ struct KeyOriginInfo return std::equal(std::begin(a.fingerprint), std::end(a.fingerprint), std::begin(b.fingerprint)) && a.path == b.path; } - ADD_SERIALIZE_METHODS; - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(fingerprint); - READWRITE(path); - } + SERIALIZE_METHODS(KeyOriginInfo, obj) { READWRITE(obj.fingerprint, obj.path); } void clear() { diff --git a/src/script/script.cpp b/src/script/script.cpp index ae0de1d24e..92c6fe7785 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -7,7 +7,9 @@ #include <util/strencodings.h> -const char* GetOpName(opcodetype opcode) +#include <string> + +std::string GetOpName(opcodetype opcode) { switch (opcode) { diff --git a/src/script/script.h b/src/script/script.h index 773ffbb985..c1f2b66921 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -193,7 +193,7 @@ enum opcodetype // Maximum value that an opcode can be static const unsigned int MAX_OPCODE = OP_NOP10; -const char* GetOpName(opcodetype opcode); +std::string GetOpName(opcodetype opcode); class scriptnum_error : public std::runtime_error { @@ -412,12 +412,7 @@ public: CScript(std::vector<unsigned char>::const_iterator pbegin, std::vector<unsigned char>::const_iterator pend) : CScriptBase(pbegin, pend) { } CScript(const unsigned char* pbegin, const unsigned char* pend) : CScriptBase(pbegin, pend) { } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CScriptBase, *this); - } + SERIALIZE_METHODS(CScript, obj) { READWRITEAS(CScriptBase, obj); } explicit CScript(int64_t b) { operator<<(b); } explicit CScript(opcodetype b) { operator<<(b); } diff --git a/src/script/script_error.cpp b/src/script/script_error.cpp index 57e8fee539..69e14803f1 100644 --- a/src/script/script_error.cpp +++ b/src/script/script_error.cpp @@ -5,7 +5,9 @@ #include <script/script_error.h> -const char* ScriptErrorString(const ScriptError serror) +#include <string> + +std::string ScriptErrorString(const ScriptError serror) { switch (serror) { diff --git a/src/script/script_error.h b/src/script/script_error.h index 400f63ff0f..2978c147e1 100644 --- a/src/script/script_error.h +++ b/src/script/script_error.h @@ -6,6 +6,8 @@ #ifndef BITCOIN_SCRIPT_SCRIPT_ERROR_H #define BITCOIN_SCRIPT_SCRIPT_ERROR_H +#include <string> + typedef enum ScriptError_t { SCRIPT_ERR_OK = 0, @@ -73,6 +75,6 @@ typedef enum ScriptError_t #define SCRIPT_ERR_LAST SCRIPT_ERR_ERROR_COUNT -const char* ScriptErrorString(const ScriptError error); +std::string ScriptErrorString(const ScriptError error); #endif // BITCOIN_SCRIPT_SCRIPT_ERROR_H diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index e7b6df3ce8..aaecab1ef2 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -11,7 +11,7 @@ #include <util/system.h> #include <cuckoocache.h> -#include <boost/thread.hpp> +#include <boost/thread/shared_mutex.hpp> namespace { /** @@ -23,7 +23,7 @@ class CSignatureCache { private: //! Entries are SHA256(nonce || signature hash || public key || signature): - uint256 nonce; + CSHA256 m_salted_hasher; typedef CuckooCache::cache<uint256, SignatureCacheHasher> map_type; map_type setValid; boost::shared_mutex cs_sigcache; @@ -31,13 +31,19 @@ private: public: CSignatureCache() { - GetRandBytes(nonce.begin(), 32); + uint256 nonce = GetRandHash(); + // We want the nonce to be 64 bytes long to force the hasher to process + // this chunk, which makes later hash computations more efficient. We + // just write our 32-byte entropy twice to fill the 64 bytes. + m_salted_hasher.Write(nonce.begin(), 32); + m_salted_hasher.Write(nonce.begin(), 32); } void ComputeEntry(uint256& entry, const uint256 &hash, const std::vector<unsigned char>& vchSig, const CPubKey& pubkey) { - CSHA256().Write(nonce.begin(), 32).Write(hash.begin(), 32).Write(&pubkey[0], pubkey.size()).Write(&vchSig[0], vchSig.size()).Finalize(entry.begin()); + CSHA256 hasher = m_salted_hasher; + hasher.Write(hash.begin(), 32).Write(&pubkey[0], pubkey.size()).Write(&vchSig[0], vchSig.size()).Finalize(entry.begin()); } bool diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 7d89a336fb..c90c2c24a0 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -9,6 +9,8 @@ #include <pubkey.h> #include <script/script.h> +#include <string> + typedef std::vector<unsigned char> valtype; bool fAcceptDatacarrier = DEFAULT_ACCEPT_DATACARRIER; @@ -25,7 +27,7 @@ WitnessV0ScriptHash::WitnessV0ScriptHash(const CScript& in) CSHA256().Write(in.data(), in.size()).Finalize(begin()); } -const char* GetTxnOutputType(txnouttype t) +std::string GetTxnOutputType(txnouttype t) { switch (t) { @@ -39,7 +41,7 @@ const char* GetTxnOutputType(txnouttype t) case TX_WITNESS_V0_SCRIPTHASH: return "witness_v0_scripthash"; case TX_WITNESS_UNKNOWN: return "witness_unknown"; } - return nullptr; + assert(false); } static bool MatchPayToPubkey(const CScript& script, valtype& pubkey) diff --git a/src/script/standard.h b/src/script/standard.h index 49a45f3eba..2929425670 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -11,6 +11,8 @@ #include <boost/variant.hpp> +#include <string> + static const bool DEFAULT_ACCEPT_DATACARRIER = true; @@ -44,8 +46,7 @@ extern unsigned nMaxDatacarrierBytes; /** * Mandatory script verification flags that all new blocks must comply with for * them to be valid. (but old blocks may not comply with) Currently just P2SH, - * but in the future other flags may be added, such as a soft-fork to enforce - * strict DER encoding. + * but in the future other flags may be added. * * Failing one of these tests may trigger a DoS ban - see CheckInputScripts() for * details. @@ -146,7 +147,7 @@ typedef boost::variant<CNoDestination, PKHash, ScriptHash, WitnessV0ScriptHash, bool IsValidDestination(const CTxDestination& dest); /** Get the name of a txnouttype as a C string, or nullptr if unknown. */ -const char* GetTxnOutputType(txnouttype t); +std::string GetTxnOutputType(txnouttype t); /** * Parse a scriptPubKey and identify script type for standard scripts. If diff --git a/src/serialize.h b/src/serialize.h index fe53eeed31..71c2cfa164 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -43,26 +43,6 @@ static const unsigned int MAX_VECTOR_ALLOCATE = 5000000; struct deserialize_type {}; constexpr deserialize_type deserialize {}; -/** - * Used to bypass the rule against non-const reference to temporary - * where it makes sense with wrappers. - */ -template<typename T> -inline T& REF(const T& val) -{ - return const_cast<T&>(val); -} - -/** - * Used to acquire a non-const pointer "this" to generate bodies - * of const serialization operations from a template - */ -template<typename T> -inline T* NCONST_PTR(const T* val) -{ - return const_cast<T*>(val); -} - //! Safely convert odd char pointer types to standard ones. inline char* CharCast(char* c) { return c; } inline char* CharCast(unsigned char* c) { return (char*)c; } @@ -190,22 +170,8 @@ template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; } #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__)) #define READWRITEAS(type, obj) (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper<type>(obj))) - -/** - * Implement three methods for serializable objects. These are actually wrappers over - * "SerializationOp" template, which implements the body of each class' serialization - * code. Adding "ADD_SERIALIZE_METHODS" in the body of the class causes these wrappers to be - * added as members. - */ -#define ADD_SERIALIZE_METHODS \ - template<typename Stream> \ - void Serialize(Stream& s) const { \ - NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize()); \ - } \ - template<typename Stream> \ - void Unserialize(Stream& s) { \ - SerializationOp(s, CSerActionUnserialize()); \ - } +#define SER_READ(obj, code) ::SerRead(s, ser_action, obj, [&](Stream& s, typename std::remove_const<Type>::type& obj) { code; }) +#define SER_WRITE(obj, code) ::SerWrite(s, ser_action, obj, [&](Stream& s, const Type& obj) { code; }) /** * Implement the Ser and Unser methods needed for implementing a formatter (see Using below). @@ -501,7 +467,7 @@ static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T& #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj) #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj) #define COMPACTSIZE(obj) Using<CompactSizeFormatter>(obj) -#define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) +#define LIMITED_STRING(obj,n) Using<LimitedStringFormatter<n>>(obj) /** Serialization wrapper class for integers in VarInt format. */ template<VarIntMode Mode> @@ -518,7 +484,16 @@ struct VarIntFormatter } }; -template<int Bytes> +/** Serialization wrapper class for custom integers and enums. + * + * It permits specifying the serialized size (1 to 8 bytes) and endianness. + * + * Use the big endian mode for values that are stored in memory in native + * byte order, but serialized in big endian notation. This is only intended + * to implement serializers that are compatible with existing formats, and + * its use is not recommended for new data structures. + */ +template<int Bytes, bool BigEndian = false> struct CustomUintFormatter { static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range"); @@ -527,52 +502,31 @@ struct CustomUintFormatter template <typename Stream, typename I> void Ser(Stream& s, I v) { if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range"); - uint64_t raw = htole64(v); - s.write((const char*)&raw, Bytes); + if (BigEndian) { + uint64_t raw = htobe64(v); + s.write(((const char*)&raw) + 8 - Bytes, Bytes); + } else { + uint64_t raw = htole64(v); + s.write((const char*)&raw, Bytes); + } } template <typename Stream, typename I> void Unser(Stream& s, I& v) { - static_assert(std::numeric_limits<I>::max() >= MAX && std::numeric_limits<I>::min() <= 0, "CustomUintFormatter type too small"); + using U = typename std::conditional<std::is_enum<I>::value, std::underlying_type<I>, std::common_type<I>>::type::type; + static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small"); uint64_t raw = 0; - s.read((char*)&raw, Bytes); - v = le64toh(raw); + if (BigEndian) { + s.read(((char*)&raw) + 8 - Bytes, Bytes); + v = static_cast<I>(be64toh(raw)); + } else { + s.read((char*)&raw, Bytes); + v = static_cast<I>(le64toh(raw)); + } } }; -/** Serialization wrapper class for big-endian integers. - * - * Use this wrapper around integer types that are stored in memory in native - * byte order, but serialized in big endian notation. This is only intended - * to implement serializers that are compatible with existing formats, and - * its use is not recommended for new data structures. - * - * Only 16-bit types are supported for now. - */ -template<typename I> -class BigEndian -{ -protected: - I& m_val; -public: - explicit BigEndian(I& val) : m_val(val) - { - static_assert(std::is_unsigned<I>::value, "BigEndian type must be unsigned integer"); - static_assert(sizeof(I) == 2 && std::numeric_limits<I>::min() == 0 && std::numeric_limits<I>::max() == std::numeric_limits<uint16_t>::max(), "Unsupported BigEndian size"); - } - - template<typename Stream> - void Serialize(Stream& s) const - { - ser_writedata16be(s, m_val); - } - - template<typename Stream> - void Unserialize(Stream& s) - { - m_val = ser_readdata16be(s); - } -}; +template<int Bytes> using BigEndianFormatter = CustomUintFormatter<Bytes, true>; /** Formatter for integers in CompactSize format. */ struct CompactSizeFormatter @@ -598,37 +552,26 @@ struct CompactSizeFormatter }; template<size_t Limit> -class LimitedString +struct LimitedStringFormatter { -protected: - std::string& string; -public: - explicit LimitedString(std::string& _string) : string(_string) {} - template<typename Stream> - void Unserialize(Stream& s) + void Unser(Stream& s, std::string& v) { size_t size = ReadCompactSize(s); if (size > Limit) { throw std::ios_base::failure("String length limit exceeded"); } - string.resize(size); - if (size != 0) - s.read((char*)string.data(), size); + v.resize(size); + if (size != 0) s.read((char*)v.data(), size); } template<typename Stream> - void Serialize(Stream& s) const + void Ser(Stream& s, const std::string& v) { - WriteCompactSize(s, string.size()); - if (!string.empty()) - s.write((char*)string.data(), string.size()); + s << v; } }; -template<typename I> -BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); } - /** Formatter to serialize/deserialize vector elements using another formatter * * Example: @@ -1025,7 +968,7 @@ void Unserialize(Stream& is, std::shared_ptr<const T>& p) /** - * Support for ADD_SERIALIZE_METHODS and READWRITE macro + * Support for SERIALIZE_METHODS and READWRITE macro. */ struct CSerActionSerialize { @@ -1124,6 +1067,28 @@ inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&& ::UnserializeMany(s, args...); } +template<typename Stream, typename Type, typename Fn> +inline void SerRead(Stream& s, CSerActionSerialize ser_action, Type&&, Fn&&) +{ +} + +template<typename Stream, typename Type, typename Fn> +inline void SerRead(Stream& s, CSerActionUnserialize ser_action, Type&& obj, Fn&& fn) +{ + fn(s, std::forward<Type>(obj)); +} + +template<typename Stream, typename Type, typename Fn> +inline void SerWrite(Stream& s, CSerActionSerialize ser_action, Type&& obj, Fn&& fn) +{ + fn(s, std::forward<Type>(obj)); +} + +template<typename Stream, typename Type, typename Fn> +inline void SerWrite(Stream& s, CSerActionUnserialize ser_action, Type&&, Fn&&) +{ +} + template<typename I> inline void WriteVarInt(CSizeComputer &s, I n) { diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index f17b539e09..b4f392116c 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -29,7 +29,6 @@ #endif LockedPoolManager* LockedPoolManager::_instance = nullptr; -std::once_flag LockedPoolManager::init_flag; /*******************************************************************************/ // Utilities diff --git a/src/support/lockedpool.h b/src/support/lockedpool.h index de668f0773..b9e2e99d1a 100644 --- a/src/support/lockedpool.h +++ b/src/support/lockedpool.h @@ -221,7 +221,8 @@ public: /** Return the current instance, or create it once */ static LockedPoolManager& Instance() { - std::call_once(LockedPoolManager::init_flag, LockedPoolManager::CreateInstance); + static std::once_flag init_flag; + std::call_once(init_flag, LockedPoolManager::CreateInstance); return *LockedPoolManager::_instance; } @@ -234,7 +235,6 @@ private: static bool LockingFailed(); static LockedPoolManager* _instance; - static std::once_flag init_flag; }; #endif // BITCOIN_SUPPORT_LOCKEDPOOL_H diff --git a/src/sync.cpp b/src/sync.cpp index b86c57e498..9abdedbed4 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -7,15 +7,19 @@ #endif #include <sync.h> -#include <tinyformat.h> #include <logging.h> +#include <tinyformat.h> #include <util/strencodings.h> #include <util/threadnames.h> #include <map> #include <set> #include <system_error> +#include <thread> +#include <unordered_map> +#include <utility> +#include <vector> #ifdef DEBUG_LOCKCONTENTION #if !defined(HAVE_THREAD_LOCAL) @@ -73,35 +77,35 @@ private: int sourceLine; }; -typedef std::vector<std::pair<void*, CLockLocation> > LockStack; -typedef std::map<std::pair<void*, void*>, LockStack> LockOrders; -typedef std::set<std::pair<void*, void*> > InvLockOrders; +using LockStackItem = std::pair<void*, CLockLocation>; +using LockStack = std::vector<LockStackItem>; +using LockStacks = std::unordered_map<std::thread::id, LockStack>; -struct LockData { - // Very ugly hack: as the global constructs and destructors run single - // threaded, we use this boolean to know whether LockData still exists, - // as DeleteLock can get called by global RecursiveMutex destructors - // after LockData disappears. - bool available; - LockData() : available(true) {} - ~LockData() { available = false; } +using LockPair = std::pair<void*, void*>; +using LockOrders = std::map<LockPair, LockStack>; +using InvLockOrders = std::set<LockPair>; +struct LockData { + LockStacks m_lock_stacks; LockOrders lockorders; InvLockOrders invlockorders; std::mutex dd_mutex; }; + LockData& GetLockData() { - static LockData lockdata; - return lockdata; + // This approach guarantees that the object is not destroyed until after its last use. + // The operating system automatically reclaims all the memory in a program's heap when that program exits. + // Since the ~LockData() destructor is never called, the LockData class and all + // its subclasses must have implicitly-defined destructors. + static LockData& lock_data = *new LockData(); + return lock_data; } -static thread_local LockStack g_lockstack; - -static void potential_deadlock_detected(const std::pair<void*, void*>& mismatch, const LockStack& s1, const LockStack& s2) +static void potential_deadlock_detected(const LockPair& mismatch, const LockStack& s1, const LockStack& s2) { LogPrintf("POTENTIAL DEADLOCK DETECTED\n"); LogPrintf("Previous lock order was:\n"); - for (const std::pair<void*, CLockLocation> & i : s2) { + for (const LockStackItem& i : s2) { if (i.first == mismatch.first) { LogPrintf(" (1)"); /* Continued */ } @@ -111,7 +115,7 @@ static void potential_deadlock_detected(const std::pair<void*, void*>& mismatch, LogPrintf(" %s\n", i.second.ToString()); } LogPrintf("Current lock order is:\n"); - for (const std::pair<void*, CLockLocation> & i : s1) { + for (const LockStackItem& i : s1) { if (i.first == mismatch.first) { LogPrintf(" (1)"); /* Continued */ } @@ -132,18 +136,18 @@ static void push_lock(void* c, const CLockLocation& locklocation) LockData& lockdata = GetLockData(); std::lock_guard<std::mutex> lock(lockdata.dd_mutex); - g_lockstack.push_back(std::make_pair(c, locklocation)); - - for (const std::pair<void*, CLockLocation>& i : g_lockstack) { + LockStack& lock_stack = lockdata.m_lock_stacks[std::this_thread::get_id()]; + lock_stack.emplace_back(c, locklocation); + for (const LockStackItem& i : lock_stack) { if (i.first == c) break; - std::pair<void*, void*> p1 = std::make_pair(i.first, c); + const LockPair p1 = std::make_pair(i.first, c); if (lockdata.lockorders.count(p1)) continue; - lockdata.lockorders.emplace(p1, g_lockstack); + lockdata.lockorders.emplace(p1, lock_stack); - std::pair<void*, void*> p2 = std::make_pair(c, i.first); + const LockPair p2 = std::make_pair(c, i.first); lockdata.invlockorders.insert(p2); if (lockdata.lockorders.count(p2)) potential_deadlock_detected(p1, lockdata.lockorders[p2], lockdata.lockorders[p1]); @@ -152,7 +156,14 @@ static void push_lock(void* c, const CLockLocation& locklocation) static void pop_lock() { - g_lockstack.pop_back(); + LockData& lockdata = GetLockData(); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); + + LockStack& lock_stack = lockdata.m_lock_stacks[std::this_thread::get_id()]; + lock_stack.pop_back(); + if (lock_stack.empty()) { + lockdata.m_lock_stacks.erase(std::this_thread::get_id()); + } } void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry) @@ -162,11 +173,17 @@ void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line) { - if (!g_lockstack.empty()) { - const auto& lastlock = g_lockstack.back(); - if (lastlock.first == cs) { - lockname = lastlock.second.Name(); - return; + { + LockData& lockdata = GetLockData(); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); + + const LockStack& lock_stack = lockdata.m_lock_stacks[std::this_thread::get_id()]; + if (!lock_stack.empty()) { + const auto& lastlock = lock_stack.back(); + if (lastlock.first == cs) { + lockname = lastlock.second.Name(); + return; + } } } throw std::system_error(EPERM, std::generic_category(), strprintf("%s:%s %s was not most recent critical section locked", file, line, guardname)); @@ -179,49 +196,60 @@ void LeaveCritical() std::string LocksHeld() { + LockData& lockdata = GetLockData(); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); + + const LockStack& lock_stack = lockdata.m_lock_stacks[std::this_thread::get_id()]; std::string result; - for (const std::pair<void*, CLockLocation>& i : g_lockstack) + for (const LockStackItem& i : lock_stack) result += i.second.ToString() + std::string("\n"); return result; } -void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) +static bool LockHeld(void* mutex) +{ + LockData& lockdata = GetLockData(); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); + + const LockStack& lock_stack = lockdata.m_lock_stacks[std::this_thread::get_id()]; + for (const LockStackItem& i : lock_stack) { + if (i.first == mutex) return true; + } + + return false; +} + +template <typename MutexType> +void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) { - for (const std::pair<void*, CLockLocation>& i : g_lockstack) - if (i.first == cs) - return; + if (LockHeld(cs)) return; tfm::format(std::cerr, "Assertion failed: lock %s not held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld()); abort(); } +template void AssertLockHeldInternal(const char*, const char*, int, Mutex*); +template void AssertLockHeldInternal(const char*, const char*, int, RecursiveMutex*); void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) { - for (const std::pair<void*, CLockLocation>& i : g_lockstack) { - if (i.first == cs) { - tfm::format(std::cerr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld()); - abort(); - } - } + if (!LockHeld(cs)) return; + tfm::format(std::cerr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld()); + abort(); } void DeleteLock(void* cs) { LockData& lockdata = GetLockData(); - if (!lockdata.available) { - // We're already shutting down. - return; - } std::lock_guard<std::mutex> lock(lockdata.dd_mutex); - std::pair<void*, void*> item = std::make_pair(cs, nullptr); + const LockPair item = std::make_pair(cs, nullptr); LockOrders::iterator it = lockdata.lockorders.lower_bound(item); while (it != lockdata.lockorders.end() && it->first.first == cs) { - std::pair<void*, void*> invitem = std::make_pair(it->first.second, it->first.first); + const LockPair invitem = std::make_pair(it->first.second, it->first.first); lockdata.invlockorders.erase(invitem); lockdata.lockorders.erase(it++); } InvLockOrders::iterator invit = lockdata.invlockorders.lower_bound(item); while (invit != lockdata.invlockorders.end() && invit->first == cs) { - std::pair<void*, void*> invinvitem = std::make_pair(invit->second, invit->first); + const LockPair invinvitem = std::make_pair(invit->second, invit->first); lockdata.lockorders.erase(invinvitem); lockdata.invlockorders.erase(invit++); } diff --git a/src/sync.h b/src/sync.h index 0c6f0ef0a7..60e5a87aec 100644 --- a/src/sync.h +++ b/src/sync.h @@ -52,7 +52,8 @@ void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs void LeaveCritical(); void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line); std::string LocksHeld(); -void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) ASSERT_EXCLUSIVE_LOCK(cs); +template <typename MutexType> +void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) ASSERT_EXCLUSIVE_LOCK(cs); void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs); void DeleteLock(void* cs); @@ -66,7 +67,8 @@ extern bool g_debug_lockorder_abort; void static inline EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false) {} void static inline LeaveCritical() {} void static inline CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line) {} -void static inline AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) ASSERT_EXCLUSIVE_LOCK(cs) {} +template <typename MutexType> +void static inline AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) ASSERT_EXCLUSIVE_LOCK(cs) {} void static inline AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) {} void static inline DeleteLock(void* cs) {} #endif diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 8694891a51..14cf1a4a76 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -132,24 +132,7 @@ public: return base.GetShortID(txhash); } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(header); - READWRITE(nonce); - size_t shorttxids_size = shorttxids.size(); - READWRITE(VARINT(shorttxids_size)); - shorttxids.resize(shorttxids_size); - for (size_t i = 0; i < shorttxids.size(); i++) { - uint32_t lsb = shorttxids[i] & 0xffffffff; - uint16_t msb = (shorttxids[i] >> 32) & 0xffff; - READWRITE(lsb); - READWRITE(msb); - shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb); - } - READWRITE(prefilledtxn); - } + SERIALIZE_METHODS(TestHeaderAndShortIDs, obj) { READWRITE(obj.header, obj.nonce, Using<VectorFormatter<CustomUintFormatter<CBlockHeaderAndShortTxIDs::SHORTTXIDS_LENGTH>>>(obj.shorttxids), obj.prefilledtxn); } }; BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index ff01f730a3..7dff2e6e86 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -43,8 +43,8 @@ static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex BOOST_CHECK(filter_index.LookupFilterHashRange(block_index->nHeight, block_index, filter_hashes)); - BOOST_CHECK_EQUAL(filters.size(), 1); - BOOST_CHECK_EQUAL(filter_hashes.size(), 1); + BOOST_CHECK_EQUAL(filters.size(), 1U); + BOOST_CHECK_EQUAL(filter_hashes.size(), 1U); BOOST_CHECK_EQUAL(filter.GetHash(), expected_filter.GetHash()); BOOST_CHECK_EQUAL(filter_header, expected_filter.ComputeHeader(last_header)); @@ -94,7 +94,7 @@ bool BuildChainTestingSetup::BuildChain(const CBlockIndex* pindex, CBlockHeader header = block->GetBlockHeader(); BlockValidationState state; - if (!ProcessNewBlockHeaders({header}, state, Params(), &pindex)) { + if (!EnsureChainman(m_node).ProcessNewBlockHeaders({header}, state, Params(), &pindex)) { return false; } } @@ -171,7 +171,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) uint256 chainA_last_header = last_header; for (size_t i = 0; i < 2; i++) { const auto& block = chainA[i]; - BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); } for (size_t i = 0; i < 2; i++) { const auto& block = chainA[i]; @@ -189,7 +189,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) uint256 chainB_last_header = last_header; for (size_t i = 0; i < 3; i++) { const auto& block = chainB[i]; - BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); } for (size_t i = 0; i < 3; i++) { const auto& block = chainB[i]; @@ -220,7 +220,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) // Reorg back to chain A. for (size_t i = 2; i < 4; i++) { const auto& block = chainA[i]; - BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); } // Check that chain A and B blocks can be retrieved. @@ -255,8 +255,9 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) BOOST_CHECK(filter_index.LookupFilterRange(0, tip, filters)); BOOST_CHECK(filter_index.LookupFilterHashRange(0, tip, filter_hashes)); - BOOST_CHECK_EQUAL(filters.size(), tip->nHeight + 1); - BOOST_CHECK_EQUAL(filter_hashes.size(), tip->nHeight + 1); + assert(tip->nHeight >= 0); + BOOST_CHECK_EQUAL(filters.size(), tip->nHeight + 1U); + BOOST_CHECK_EQUAL(filter_hashes.size(), tip->nHeight + 1U); filters.clear(); filter_hashes.clear(); diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp index e69503ef35..178c261365 100644 --- a/src/test/blockfilter_tests.cpp +++ b/src/test/blockfilter_tests.cpp @@ -42,14 +42,14 @@ BOOST_AUTO_TEST_CASE(gcsfilter_test) BOOST_AUTO_TEST_CASE(gcsfilter_default_constructor) { GCSFilter filter; - BOOST_CHECK_EQUAL(filter.GetN(), 0); - BOOST_CHECK_EQUAL(filter.GetEncoded().size(), 1); + BOOST_CHECK_EQUAL(filter.GetN(), 0U); + BOOST_CHECK_EQUAL(filter.GetEncoded().size(), 1U); const GCSFilter::Params& params = filter.GetParams(); - BOOST_CHECK_EQUAL(params.m_siphash_k0, 0); - BOOST_CHECK_EQUAL(params.m_siphash_k1, 0); + BOOST_CHECK_EQUAL(params.m_siphash_k0, 0U); + BOOST_CHECK_EQUAL(params.m_siphash_k1, 0U); BOOST_CHECK_EQUAL(params.m_P, 0); - BOOST_CHECK_EQUAL(params.m_M, 1); + BOOST_CHECK_EQUAL(params.m_M, 1U); } BOOST_AUTO_TEST_CASE(blockfilter_basic_test) diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index a75d9fc9ec..736c260eeb 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -488,7 +488,7 @@ BOOST_AUTO_TEST_CASE(rolling_bloom) ++nHits; } // Expect about 100 hits - BOOST_CHECK_EQUAL(nHits, 75); + BOOST_CHECK_EQUAL(nHits, 75U); BOOST_CHECK(rb1.contains(data[DATASIZE-1])); rb1.reset(); @@ -516,7 +516,7 @@ BOOST_AUTO_TEST_CASE(rolling_bloom) ++nHits; } // Expect about 5 false positives - BOOST_CHECK_EQUAL(nHits, 6); + BOOST_CHECK_EQUAL(nHits, 6U); // last-1000-entry, 0.01% false positive: CRollingBloomFilter rb2(1000, 0.001); diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 0565982215..8348810ac1 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -3,13 +3,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <checkqueue.h> +#include <sync.h> #include <test/util/setup_common.h> #include <util/memory.h> #include <util/system.h> #include <util/time.h> #include <boost/test/unit_test.hpp> -#include <boost/thread.hpp> +#include <boost/thread/thread.hpp> #include <atomic> #include <condition_variable> @@ -57,14 +58,14 @@ struct FailingCheck { }; struct UniqueCheck { - static std::mutex m; - static std::unordered_multiset<size_t> results; + static Mutex m; + static std::unordered_multiset<size_t> results GUARDED_BY(m); size_t check_id; UniqueCheck(size_t check_id_in) : check_id(check_id_in){}; UniqueCheck() : check_id(0){}; bool operator()() { - std::lock_guard<std::mutex> l(m); + LOCK(m); results.insert(check_id); return true; } @@ -127,7 +128,7 @@ struct FrozenCleanupCheck { std::mutex FrozenCleanupCheck::m{}; std::atomic<uint64_t> FrozenCleanupCheck::nFrozen{0}; std::condition_variable FrozenCleanupCheck::cv{}; -std::mutex UniqueCheck::m; +Mutex UniqueCheck::m; std::unordered_multiset<size_t> UniqueCheck::results; std::atomic<size_t> FakeCheckCheckCompletion::n_calls{0}; std::atomic<size_t> MemoryCheck::fake_allocated_memory{0}; @@ -290,11 +291,15 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) control.Add(vChecks); } } - bool r = true; - BOOST_REQUIRE_EQUAL(UniqueCheck::results.size(), COUNT); - for (size_t i = 0; i < COUNT; ++i) - r = r && UniqueCheck::results.count(i) == 1; - BOOST_REQUIRE(r); + { + LOCK(UniqueCheck::m); + bool r = true; + BOOST_REQUIRE_EQUAL(UniqueCheck::results.size(), COUNT); + for (size_t i = 0; i < COUNT; ++i) { + r = r && UniqueCheck::results.count(i) == 1; + } + BOOST_REQUIRE(r); + } tg.interrupt_all(); tg.join_all(); } diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index df1a119d79..4ddbc8338e 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -70,14 +70,14 @@ BOOST_AUTO_TEST_CASE(compress_script_to_ckey_id) CPubKey pubkey = key.GetPubKey(); CScript script = CScript() << OP_DUP << OP_HASH160 << ToByteVector(pubkey.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; - BOOST_CHECK_EQUAL(script.size(), 25); + BOOST_CHECK_EQUAL(script.size(), 25U); std::vector<unsigned char> out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); // Check compressed script - BOOST_CHECK_EQUAL(out.size(), 21); + BOOST_CHECK_EQUAL(out.size(), 21U); BOOST_CHECK_EQUAL(out[0], 0x00); BOOST_CHECK_EQUAL(memcmp(&out[1], &script[3], 20), 0); // compare the 20 relevant chars of the CKeyId in the script } @@ -87,14 +87,14 @@ BOOST_AUTO_TEST_CASE(compress_script_to_cscript_id) // case CScriptID CScript script, redeemScript; script << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; - BOOST_CHECK_EQUAL(script.size(), 23); + BOOST_CHECK_EQUAL(script.size(), 23U); std::vector<unsigned char> out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); // Check compressed script - BOOST_CHECK_EQUAL(out.size(), 21); + BOOST_CHECK_EQUAL(out.size(), 21U); BOOST_CHECK_EQUAL(out[0], 0x01); BOOST_CHECK_EQUAL(memcmp(&out[1], &script[2], 20), 0); // compare the 20 relevant chars of the CScriptId in the script } @@ -105,14 +105,14 @@ BOOST_AUTO_TEST_CASE(compress_script_to_compressed_pubkey_id) key.MakeNewKey(true); // case compressed PubKeyID CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // COMPRESSED_PUBLIC_KEY_SIZE (33) - BOOST_CHECK_EQUAL(script.size(), 35); + BOOST_CHECK_EQUAL(script.size(), 35U); std::vector<unsigned char> out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); // Check compressed script - BOOST_CHECK_EQUAL(out.size(), 33); + BOOST_CHECK_EQUAL(out.size(), 33U); BOOST_CHECK_EQUAL(memcmp(&out[0], &script[1], 1), 0); BOOST_CHECK_EQUAL(memcmp(&out[1], &script[2], 32), 0); // compare the 32 chars of the compressed CPubKey } @@ -122,14 +122,14 @@ BOOST_AUTO_TEST_CASE(compress_script_to_uncompressed_pubkey_id) CKey key; key.MakeNewKey(false); // case uncompressed PubKeyID CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // PUBLIC_KEY_SIZE (65) - BOOST_CHECK_EQUAL(script.size(), 67); // 1 char code + 65 char pubkey + OP_CHECKSIG + BOOST_CHECK_EQUAL(script.size(), 67U); // 1 char code + 65 char pubkey + OP_CHECKSIG std::vector<unsigned char> out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); // Check compressed script - BOOST_CHECK_EQUAL(out.size(), 33); + BOOST_CHECK_EQUAL(out.size(), 33U); BOOST_CHECK_EQUAL(memcmp(&out[1], &script[2], 32), 0); // first 32 chars of CPubKey are copied into out[1:] BOOST_CHECK_EQUAL(out[0], 0x04 | (script[65] & 0x01)); // least significant bit (lsb) of last char of pubkey is mapped into out[0] } diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index c378546e8b..3d802cbeb3 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -331,24 +331,26 @@ struct StringContentsSerializer { } StringContentsSerializer& operator+=(const StringContentsSerializer& s) { return *this += s.str; } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - if (ser_action.ForRead()) { - str.clear(); - char c = 0; - while (true) { - try { - READWRITE(c); - str.push_back(c); - } catch (const std::ios_base::failure&) { - break; - } + template<typename Stream> + void Serialize(Stream& s) const + { + for (size_t i = 0; i < str.size(); i++) { + s << str[i]; + } + } + + template<typename Stream> + void Unserialize(Stream& s) + { + str.clear(); + char c = 0; + while (true) { + try { + s >> c; + str.push_back(c); + } catch (const std::ios_base::failure&) { + break; } - } else { - for (size_t i = 0; i < str.size(); i++) - READWRITE(str[i]); } } }; diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 6314c1a42f..348b170536 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -78,7 +78,7 @@ BOOST_FIXTURE_TEST_SUITE(denialofservice_tests, TestingSetup) BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), nullptr, *m_node.scheduler, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); // Mock an outbound peer CAddress addr1(ip(0xa0b0c001), NODE_NONE); @@ -148,14 +148,14 @@ static void AddRandomOutboundPeer(std::vector<CNode *> &vNodes, PeerLogicValidat BOOST_AUTO_TEST_CASE(stale_tip_peer_management) { auto connman = MakeUnique<CConnmanTest>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), nullptr, *m_node.scheduler, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); const Consensus::Params& consensusParams = Params().GetConsensus(); - constexpr int max_outbound_full_relay = 8; + constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; CConnman::Options options; - options.nMaxConnections = 125; + options.nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; options.m_max_outbound_full_relay = max_outbound_full_relay; - options.nMaxFeeler = 1; + options.nMaxFeeler = MAX_FEELER_CONNECTIONS; connman->Init(options); std::vector<CNode *> vNodes; @@ -221,7 +221,7 @@ BOOST_AUTO_TEST_CASE(DoS_banning) { auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); banman->ClearBanned(); CAddress addr1(ip(0xa0b0c001), NODE_NONE); @@ -276,7 +276,7 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) { auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); banman->ClearBanned(); gArgs.ForceSetArg("-banscore", "111"); // because 11 is my favorite number @@ -323,7 +323,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) { auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); banman->ClearBanned(); int64_t nStartTime = GetTime(); diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 121b80ab2d..5f9a78ceb2 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -232,7 +232,7 @@ void DoCheck(const std::string& prv, const std::string& pub, int flags, const st std::vector<CScript> spks_inferred; FlatSigningProvider provider_inferred; BOOST_CHECK(inferred->Expand(0, provider_inferred, spks_inferred, provider_inferred)); - BOOST_CHECK_EQUAL(spks_inferred.size(), 1); + BOOST_CHECK_EQUAL(spks_inferred.size(), 1U); BOOST_CHECK(spks_inferred[0] == spks[n]); BOOST_CHECK_EQUAL(IsSolvable(provider_inferred, spks_inferred[0]), !(flags & UNSOLVABLE)); BOOST_CHECK(provider_inferred.origins == script_provider.origins); diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp index 9bb0b3ef02..be7484cd0b 100644 --- a/src/test/flatfile_tests.cpp +++ b/src/test/flatfile_tests.cpp @@ -93,16 +93,16 @@ BOOST_AUTO_TEST_CASE(flatfile_allocate) bool out_of_space; - BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 0), 1, out_of_space), 100); - BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 0))), 100); + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 0), 1, out_of_space), 100U); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 0))), 100U); BOOST_CHECK(!out_of_space); - BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 1, out_of_space), 0); - BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 100); + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 1, out_of_space), 0U); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 100U); BOOST_CHECK(!out_of_space); - BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 2, out_of_space), 101); - BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 200); + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 2, out_of_space), 101U); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 200U); BOOST_CHECK(!out_of_space); } @@ -116,11 +116,11 @@ BOOST_AUTO_TEST_CASE(flatfile_flush) // Flush without finalize should not truncate file. seq.Flush(FlatFilePos(0, 1)); - BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 100); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 100U); // Flush with finalize should truncate file. seq.Flush(FlatFilePos(0, 1), true); - BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 1); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 1U); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/addrdb.cpp b/src/test/fuzz/addrdb.cpp index f21ff3fac3..524cea83fe 100644 --- a/src/test/fuzz/addrdb.cpp +++ b/src/test/fuzz/addrdb.cpp @@ -3,13 +3,13 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrdb.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <cassert> #include <cstdint> +#include <optional> #include <string> #include <vector> @@ -30,7 +30,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) })}; break; case 2: { - const Optional<CBanEntry> ban_entry = ConsumeDeserializable<CBanEntry>(fuzzed_data_provider); + const std::optional<CBanEntry> ban_entry = ConsumeDeserializable<CBanEntry>(fuzzed_data_provider); if (ban_entry) { return *ban_entry; } diff --git a/src/test/fuzz/asmap.cpp b/src/test/fuzz/asmap.cpp index ea56277eac..40ca01bd9f 100644 --- a/src/test/fuzz/asmap.cpp +++ b/src/test/fuzz/asmap.cpp @@ -23,8 +23,8 @@ static const std::vector<bool> IPV4_PREFIX_ASMAP = { true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00 true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00 true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00 - true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, // Match 0xFF - true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true // Match 0xFF + true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, // Match 0xFF + true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true // Match 0xFF }; void test_one_input(const std::vector<uint8_t>& buffer) diff --git a/src/test/fuzz/asmap_direct.cpp b/src/test/fuzz/asmap_direct.cpp index 790f80237d..2d21eff9d6 100644 --- a/src/test/fuzz/asmap_direct.cpp +++ b/src/test/fuzz/asmap_direct.cpp @@ -2,10 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/asmap.h> #include <test/fuzz/fuzz.h> +#include <util/asmap.h> #include <cstdint> +#include <optional> #include <vector> #include <assert.h> @@ -13,20 +14,19 @@ void test_one_input(const std::vector<uint8_t>& buffer) { // Encoding: [asmap using 1 bit / byte] 0xFF [addr using 1 bit / byte] - bool have_sep = false; - size_t sep_pos; + std::optional<size_t> sep_pos_opt; for (size_t pos = 0; pos < buffer.size(); ++pos) { uint8_t x = buffer[pos]; if ((x & 0xFE) == 0) continue; if (x == 0xFF) { - if (have_sep) return; - have_sep = true; - sep_pos = pos; + if (sep_pos_opt) return; + sep_pos_opt = pos; } else { return; } } - if (!have_sep) return; // Needs exactly 1 separator + if (!sep_pos_opt) return; // Needs exactly 1 separator + const size_t sep_pos{sep_pos_opt.value()}; if (buffer.size() - sep_pos - 1 > 128) return; // At most 128 bits in IP address // Checks on asmap @@ -34,7 +34,9 @@ void test_one_input(const std::vector<uint8_t>& buffer) if (SanityCheckASMap(asmap, buffer.size() - 1 - sep_pos)) { // Verify that for valid asmaps, no prefix (except up to 7 zero padding bits) is valid. std::vector<bool> asmap_prefix = asmap; - while (!asmap_prefix.empty() && asmap_prefix.size() + 7 > asmap.size() && asmap_prefix.back() == false) asmap_prefix.pop_back(); + while (!asmap_prefix.empty() && asmap_prefix.size() + 7 > asmap.size() && asmap_prefix.back() == false) { + asmap_prefix.pop_back(); + } while (!asmap_prefix.empty()) { asmap_prefix.pop_back(); assert(!SanityCheckASMap(asmap_prefix, buffer.size() - 1 - sep_pos)); diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index f30fa03e0b..91bd34a251 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -38,12 +38,17 @@ void test_one_input(const std::vector<uint8_t>& buffer) const Consensus::Params& consensus_params = Params().GetConsensus(); BlockValidationState validation_state_pow_and_merkle; const bool valid_incl_pow_and_merkle = CheckBlock(block, validation_state_pow_and_merkle, consensus_params, /* fCheckPOW= */ true, /* fCheckMerkleRoot= */ true); + assert(validation_state_pow_and_merkle.IsValid() || validation_state_pow_and_merkle.IsInvalid() || validation_state_pow_and_merkle.IsError()); + (void)validation_state_pow_and_merkle.Error(""); BlockValidationState validation_state_pow; const bool valid_incl_pow = CheckBlock(block, validation_state_pow, consensus_params, /* fCheckPOW= */ true, /* fCheckMerkleRoot= */ false); + assert(validation_state_pow.IsValid() || validation_state_pow.IsInvalid() || validation_state_pow.IsError()); BlockValidationState validation_state_merkle; const bool valid_incl_merkle = CheckBlock(block, validation_state_merkle, consensus_params, /* fCheckPOW= */ false, /* fCheckMerkleRoot= */ true); + assert(validation_state_merkle.IsValid() || validation_state_merkle.IsInvalid() || validation_state_merkle.IsError()); BlockValidationState validation_state_none; const bool valid_incl_none = CheckBlock(block, validation_state_none, consensus_params, /* fCheckPOW= */ false, /* fCheckMerkleRoot= */ false); + assert(validation_state_none.IsValid() || validation_state_none.IsInvalid() || validation_state_none.IsError()); if (valid_incl_pow_and_merkle) { assert(valid_incl_pow && valid_incl_merkle && valid_incl_none); } else if (valid_incl_merkle || valid_incl_pow) { diff --git a/src/test/fuzz/block_header.cpp b/src/test/fuzz/block_header.cpp index 92dcccc0e1..09c2b4a951 100644 --- a/src/test/fuzz/block_header.cpp +++ b/src/test/fuzz/block_header.cpp @@ -2,7 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <optional.h> #include <primitives/block.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> @@ -11,13 +10,14 @@ #include <cassert> #include <cstdint> +#include <optional> #include <string> #include <vector> void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const Optional<CBlockHeader> block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); + const std::optional<CBlockHeader> block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); if (!block_header) { return; } @@ -38,4 +38,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) block.SetNull(); assert(block.GetBlockHeader().GetHash() == mut_block_header.GetHash()); } + { + std::optional<CBlockLocator> block_locator = ConsumeDeserializable<CBlockLocator>(fuzzed_data_provider); + if (block_locator) { + (void)block_locator->IsNull(); + block_locator->SetNull(); + assert(block_locator->IsNull()); + } + } } diff --git a/src/test/fuzz/blockfilter.cpp b/src/test/fuzz/blockfilter.cpp index be9320dcbf..7232325a20 100644 --- a/src/test/fuzz/blockfilter.cpp +++ b/src/test/fuzz/blockfilter.cpp @@ -3,19 +3,19 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <blockfilter.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <cstdint> +#include <optional> #include <string> #include <vector> void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const Optional<BlockFilter> block_filter = ConsumeDeserializable<BlockFilter>(fuzzed_data_provider); + const std::optional<BlockFilter> block_filter = ConsumeDeserializable<BlockFilter>(fuzzed_data_provider); if (!block_filter) { return; } diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index 7039bf16c1..d955c71bc9 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bloom.h> -#include <optional.h> #include <primitives/transaction.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> @@ -12,6 +11,7 @@ #include <cassert> #include <cstdint> +#include <optional> #include <string> #include <vector> @@ -35,7 +35,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) break; } case 1: { - const Optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + const std::optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (!out_point) { break; } @@ -46,7 +46,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) break; } case 2: { - const Optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); + const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); if (!u256) { break; } @@ -57,7 +57,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) break; } case 3: { - const Optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mut_tx) { break; } diff --git a/src/test/fuzz/chain.cpp b/src/test/fuzz/chain.cpp index b322516cc7..47c71850ce 100644 --- a/src/test/fuzz/chain.cpp +++ b/src/test/fuzz/chain.cpp @@ -3,18 +3,18 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chain.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <cstdint> +#include <optional> #include <vector> void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - Optional<CDiskBlockIndex> disk_block_index = ConsumeDeserializable<CDiskBlockIndex>(fuzzed_data_provider); + std::optional<CDiskBlockIndex> disk_block_index = ConsumeDeserializable<CDiskBlockIndex>(fuzzed_data_provider); if (!disk_block_index) { return; } diff --git a/src/test/fuzz/checkqueue.cpp b/src/test/fuzz/checkqueue.cpp index 2ed097b827..c69043bb6b 100644 --- a/src/test/fuzz/checkqueue.cpp +++ b/src/test/fuzz/checkqueue.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <checkqueue.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp new file mode 100644 index 0000000000..52dd62a145 --- /dev/null +++ b/src/test/fuzz/coins_view.cpp @@ -0,0 +1,294 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <amount.h> +#include <chainparams.h> +#include <chainparamsbase.h> +#include <coins.h> +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <key.h> +#include <node/coinstats.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <pubkey.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <validation.h> + +#include <cstdint> +#include <limits> +#include <optional> +#include <string> +#include <vector> + +namespace { +const Coin EMPTY_COIN{}; + +bool operator==(const Coin& a, const Coin& b) +{ + if (a.IsSpent() && b.IsSpent()) return true; + return a.fCoinBase == b.fCoinBase && a.nHeight == b.nHeight && a.out == b.out; +} +} // namespace + +void initialize() +{ + static const ECCVerifyHandle ecc_verify_handle; + ECC_Start(); + SelectParams(CBaseChainParams::REGTEST); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + CCoinsView backend_coins_view; + CCoinsViewCache coins_view_cache{&backend_coins_view}; + COutPoint random_out_point; + Coin random_coin; + CMutableTransaction random_mutable_transaction; + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 9)) { + case 0: { + if (random_coin.IsSpent()) { + break; + } + Coin coin = random_coin; + bool expected_code_path = false; + const bool possible_overwrite = fuzzed_data_provider.ConsumeBool(); + try { + coins_view_cache.AddCoin(random_out_point, std::move(coin), possible_overwrite); + expected_code_path = true; + } catch (const std::logic_error& e) { + if (e.what() == std::string{"Attempted to overwrite an unspent coin (when possible_overwrite is false)"}) { + assert(!possible_overwrite); + expected_code_path = true; + } + } + assert(expected_code_path); + break; + } + case 1: { + (void)coins_view_cache.Flush(); + break; + } + case 2: { + coins_view_cache.SetBestBlock(ConsumeUInt256(fuzzed_data_provider)); + break; + } + case 3: { + Coin move_to; + (void)coins_view_cache.SpendCoin(random_out_point, fuzzed_data_provider.ConsumeBool() ? &move_to : nullptr); + break; + } + case 4: { + coins_view_cache.Uncache(random_out_point); + break; + } + case 5: { + if (fuzzed_data_provider.ConsumeBool()) { + backend_coins_view = CCoinsView{}; + } + coins_view_cache.SetBackend(backend_coins_view); + break; + } + case 6: { + const std::optional<COutPoint> opt_out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + if (!opt_out_point) { + break; + } + random_out_point = *opt_out_point; + break; + } + case 7: { + const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); + if (!opt_coin) { + break; + } + random_coin = *opt_coin; + break; + } + case 8: { + const std::optional<CMutableTransaction> opt_mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!opt_mutable_transaction) { + break; + } + random_mutable_transaction = *opt_mutable_transaction; + break; + } + case 9: { + CCoinsMap coins_map; + while (fuzzed_data_provider.ConsumeBool()) { + CCoinsCacheEntry coins_cache_entry; + coins_cache_entry.flags = fuzzed_data_provider.ConsumeIntegral<unsigned char>(); + if (fuzzed_data_provider.ConsumeBool()) { + coins_cache_entry.coin = random_coin; + } else { + const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); + if (!opt_coin) { + break; + } + coins_cache_entry.coin = *opt_coin; + } + coins_map.emplace(random_out_point, std::move(coins_cache_entry)); + } + bool expected_code_path = false; + try { + coins_view_cache.BatchWrite(coins_map, fuzzed_data_provider.ConsumeBool() ? ConsumeUInt256(fuzzed_data_provider) : coins_view_cache.GetBestBlock()); + expected_code_path = true; + } catch (const std::logic_error& e) { + if (e.what() == std::string{"FRESH flag misapplied to coin that exists in parent cache"}) { + expected_code_path = true; + } + } + assert(expected_code_path); + break; + } + } + } + + { + const Coin& coin_using_access_coin = coins_view_cache.AccessCoin(random_out_point); + const bool exists_using_access_coin = !(coin_using_access_coin == EMPTY_COIN); + const bool exists_using_have_coin = coins_view_cache.HaveCoin(random_out_point); + const bool exists_using_have_coin_in_cache = coins_view_cache.HaveCoinInCache(random_out_point); + Coin coin_using_get_coin; + const bool exists_using_get_coin = coins_view_cache.GetCoin(random_out_point, coin_using_get_coin); + if (exists_using_get_coin) { + assert(coin_using_get_coin == coin_using_access_coin); + } + assert((exists_using_access_coin && exists_using_have_coin_in_cache && exists_using_have_coin && exists_using_get_coin) || + (!exists_using_access_coin && !exists_using_have_coin_in_cache && !exists_using_have_coin && !exists_using_get_coin)); + const bool exists_using_have_coin_in_backend = backend_coins_view.HaveCoin(random_out_point); + if (exists_using_have_coin_in_backend) { + assert(exists_using_have_coin); + } + Coin coin_using_backend_get_coin; + if (backend_coins_view.GetCoin(random_out_point, coin_using_backend_get_coin)) { + assert(exists_using_have_coin_in_backend); + assert(coin_using_get_coin == coin_using_backend_get_coin); + } else { + assert(!exists_using_have_coin_in_backend); + } + } + + { + bool expected_code_path = false; + try { + (void)coins_view_cache.Cursor(); + } catch (const std::logic_error&) { + expected_code_path = true; + } + assert(expected_code_path); + (void)coins_view_cache.DynamicMemoryUsage(); + (void)coins_view_cache.EstimateSize(); + (void)coins_view_cache.GetBestBlock(); + (void)coins_view_cache.GetCacheSize(); + (void)coins_view_cache.GetHeadBlocks(); + (void)coins_view_cache.HaveInputs(CTransaction{random_mutable_transaction}); + } + + { + const CCoinsViewCursor* coins_view_cursor = backend_coins_view.Cursor(); + assert(coins_view_cursor == nullptr); + (void)backend_coins_view.EstimateSize(); + (void)backend_coins_view.GetBestBlock(); + (void)backend_coins_view.GetHeadBlocks(); + } + + if (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 6)) { + case 0: { + const CTransaction transaction{random_mutable_transaction}; + bool is_spent = false; + for (const CTxOut& tx_out : transaction.vout) { + if (Coin{tx_out, 0, transaction.IsCoinBase()}.IsSpent()) { + is_spent = true; + } + } + if (is_spent) { + // Avoid: + // coins.cpp:69: void CCoinsViewCache::AddCoin(const COutPoint &, Coin &&, bool): Assertion `!coin.IsSpent()' failed. + break; + } + bool expected_code_path = false; + const int height = fuzzed_data_provider.ConsumeIntegral<int>(); + const bool possible_overwrite = fuzzed_data_provider.ConsumeBool(); + try { + AddCoins(coins_view_cache, transaction, height, possible_overwrite); + expected_code_path = true; + } catch (const std::logic_error& e) { + if (e.what() == std::string{"Attempted to overwrite an unspent coin (when possible_overwrite is false)"}) { + assert(!possible_overwrite); + expected_code_path = true; + } + } + assert(expected_code_path); + break; + } + case 1: { + (void)AreInputsStandard(CTransaction{random_mutable_transaction}, coins_view_cache); + break; + } + case 2: { + TxValidationState state; + CAmount tx_fee_out; + const CTransaction transaction{random_mutable_transaction}; + if (ContainsSpentInput(transaction, coins_view_cache)) { + // Avoid: + // consensus/tx_verify.cpp:171: bool Consensus::CheckTxInputs(const CTransaction &, TxValidationState &, const CCoinsViewCache &, int, CAmount &): Assertion `!coin.IsSpent()' failed. + break; + } + try { + (void)Consensus::CheckTxInputs(transaction, state, coins_view_cache, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, std::numeric_limits<int>::max()), tx_fee_out); + assert(MoneyRange(tx_fee_out)); + } catch (const std::runtime_error&) { + } + break; + } + case 3: { + const CTransaction transaction{random_mutable_transaction}; + if (ContainsSpentInput(transaction, coins_view_cache)) { + // Avoid: + // consensus/tx_verify.cpp:130: unsigned int GetP2SHSigOpCount(const CTransaction &, const CCoinsViewCache &): Assertion `!coin.IsSpent()' failed. + break; + } + (void)GetP2SHSigOpCount(transaction, coins_view_cache); + break; + } + case 4: { + const CTransaction transaction{random_mutable_transaction}; + if (ContainsSpentInput(transaction, coins_view_cache)) { + // Avoid: + // consensus/tx_verify.cpp:130: unsigned int GetP2SHSigOpCount(const CTransaction &, const CCoinsViewCache &): Assertion `!coin.IsSpent()' failed. + break; + } + const int flags = fuzzed_data_provider.ConsumeIntegral<int>(); + if (!transaction.vin.empty() && (flags & SCRIPT_VERIFY_WITNESS) != 0 && (flags & SCRIPT_VERIFY_P2SH) == 0) { + // Avoid: + // script/interpreter.cpp:1705: size_t CountWitnessSigOps(const CScript &, const CScript &, const CScriptWitness *, unsigned int): Assertion `(flags & SCRIPT_VERIFY_P2SH) != 0' failed. + break; + } + (void)GetTransactionSigOpCost(transaction, coins_view_cache, flags); + break; + } + case 5: { + CCoinsStats stats; + bool expected_code_path = false; + try { + (void)GetUTXOStats(&coins_view_cache, stats); + } catch (const std::logic_error&) { + expected_code_path = true; + } + assert(expected_code_path); + break; + } + case 6: { + (void)IsWitnessStandard(CTransaction{random_mutable_transaction}, coins_view_cache); + break; + } + } + } +} diff --git a/src/test/fuzz/cuckoocache.cpp b/src/test/fuzz/cuckoocache.cpp index f674efe1b1..5b45aa79d8 100644 --- a/src/test/fuzz/cuckoocache.cpp +++ b/src/test/fuzz/cuckoocache.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <cuckoocache.h> -#include <optional.h> #include <script/sigcache.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp index f29acace23..ce8700befa 100644 --- a/src/test/fuzz/fees.cpp +++ b/src/test/fuzz/fees.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <amount.h> -#include <optional.h> #include <policy/fees.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> diff --git a/src/test/fuzz/flatfile.cpp b/src/test/fuzz/flatfile.cpp index a55de77df7..95dabb8bab 100644 --- a/src/test/fuzz/flatfile.cpp +++ b/src/test/fuzz/flatfile.cpp @@ -3,24 +3,24 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <flatfile.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <cassert> #include <cstdint> +#include <optional> #include <string> #include <vector> void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - Optional<FlatFilePos> flat_file_pos = ConsumeDeserializable<FlatFilePos>(fuzzed_data_provider); + std::optional<FlatFilePos> flat_file_pos = ConsumeDeserializable<FlatFilePos>(fuzzed_data_provider); if (!flat_file_pos) { return; } - Optional<FlatFilePos> another_flat_file_pos = ConsumeDeserializable<FlatFilePos>(fuzzed_data_provider); + std::optional<FlatFilePos> another_flat_file_pos = ConsumeDeserializable<FlatFilePos>(fuzzed_data_provider); if (another_flat_file_pos) { assert((*flat_file_pos == *another_flat_file_pos) != (*flat_file_pos != *another_flat_file_pos)); } diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index 6e2188fe86..82e1d55c0b 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -19,8 +19,6 @@ static bool read_stdin(std::vector<uint8_t>& data) ssize_t length = 0; while ((length = read(STDIN_FILENO, buffer, 1024)) > 0) { data.insert(data.end(), buffer, buffer + length); - - if (data.size() > (1 << 20)) return false; } return length == 0; } diff --git a/src/test/fuzz/golomb_rice.cpp b/src/test/fuzz/golomb_rice.cpp index 3e20416116..a9f450b0c4 100644 --- a/src/test/fuzz/golomb_rice.cpp +++ b/src/test/fuzz/golomb_rice.cpp @@ -5,8 +5,8 @@ #include <blockfilter.h> #include <serialize.h> #include <streams.h> -#include <test/fuzz/fuzz.h> #include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <util/bytevectorhash.h> #include <util/golombrice.h> diff --git a/src/test/fuzz/hex.cpp b/src/test/fuzz/hex.cpp index 5fed17c17c..6a8699fd0f 100644 --- a/src/test/fuzz/hex.cpp +++ b/src/test/fuzz/hex.cpp @@ -16,7 +16,8 @@ #include <string> #include <vector> -void initialize() { +void initialize() +{ static const ECCVerifyHandle verify_handle; } diff --git a/src/test/fuzz/merkleblock.cpp b/src/test/fuzz/merkleblock.cpp index eb8fa1d421..c44e334272 100644 --- a/src/test/fuzz/merkleblock.cpp +++ b/src/test/fuzz/merkleblock.cpp @@ -3,20 +3,20 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <merkleblock.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <uint256.h> #include <cstdint> +#include <optional> #include <string> #include <vector> void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - Optional<CPartialMerkleTree> partial_merkle_tree = ConsumeDeserializable<CPartialMerkleTree>(fuzzed_data_provider); + std::optional<CPartialMerkleTree> partial_merkle_tree = ConsumeDeserializable<CPartialMerkleTree>(fuzzed_data_provider); if (!partial_merkle_tree) { return; } diff --git a/src/test/fuzz/message.cpp b/src/test/fuzz/message.cpp index dfa98a812b..fa0322a391 100644 --- a/src/test/fuzz/message.cpp +++ b/src/test/fuzz/message.cpp @@ -4,7 +4,6 @@ #include <chainparams.h> #include <key_io.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> diff --git a/src/test/fuzz/net_permissions.cpp b/src/test/fuzz/net_permissions.cpp index bfc5d21427..c071283467 100644 --- a/src/test/fuzz/net_permissions.cpp +++ b/src/test/fuzz/net_permissions.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <net_permissions.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 201f49c87b..1cbf9b347f 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -2,7 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <optional.h> #include <policy/fees.h> #include <primitives/transaction.h> #include <test/fuzz/FuzzedDataProvider.h> @@ -11,6 +10,7 @@ #include <txmempool.h> #include <cstdint> +#include <optional> #include <string> #include <vector> @@ -21,7 +21,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) while (fuzzed_data_provider.ConsumeBool()) { switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 3)) { case 0: { - const Optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mtx) { break; } @@ -35,7 +35,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) case 1: { std::vector<CTxMemPoolEntry> mempool_entries; while (fuzzed_data_provider.ConsumeBool()) { - const Optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mtx) { break; } diff --git a/src/test/fuzz/pow.cpp b/src/test/fuzz/pow.cpp index 0343d33401..b7fc72373d 100644 --- a/src/test/fuzz/pow.cpp +++ b/src/test/fuzz/pow.cpp @@ -4,7 +4,6 @@ #include <chain.h> #include <chainparams.h> -#include <optional.h> #include <pow.h> #include <primitives/block.h> #include <test/fuzz/FuzzedDataProvider.h> @@ -12,6 +11,7 @@ #include <test/fuzz/util.h> #include <cstdint> +#include <optional> #include <string> #include <vector> @@ -28,7 +28,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) const uint32_t fixed_time = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const uint32_t fixed_bits = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); while (fuzzed_data_provider.remaining_bytes() > 0) { - const Optional<CBlockHeader> block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); + const std::optional<CBlockHeader> block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); if (!block_header) { continue; } @@ -72,7 +72,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) } } { - const Optional<uint256> hash = ConsumeDeserializable<uint256>(fuzzed_data_provider); + const std::optional<uint256> hash = ConsumeDeserializable<uint256>(fuzzed_data_provider); if (hash) { (void)CheckProofOfWork(*hash, fuzzed_data_provider.ConsumeIntegral<unsigned int>(), consensus_params); } diff --git a/src/test/fuzz/prevector.cpp b/src/test/fuzz/prevector.cpp index 64920f4af5..626e187cbd 100644 --- a/src/test/fuzz/prevector.cpp +++ b/src/test/fuzz/prevector.cpp @@ -14,8 +14,9 @@ namespace { -template<unsigned int N, typename T> -class prevector_tester { +template <unsigned int N, typename T> +class prevector_tester +{ typedef std::vector<T> realtype; realtype real_vector; realtype real_vector_alt; @@ -27,35 +28,36 @@ class prevector_tester { typedef typename pretype::size_type Size; public: - void test() const { + void test() const + { const pretype& const_pre_vector = pre_vector; assert(real_vector.size() == pre_vector.size()); assert(real_vector.empty() == pre_vector.empty()); for (Size s = 0; s < real_vector.size(); s++) { - assert(real_vector[s] == pre_vector[s]); - assert(&(pre_vector[s]) == &(pre_vector.begin()[s])); - assert(&(pre_vector[s]) == &*(pre_vector.begin() + s)); - assert(&(pre_vector[s]) == &*((pre_vector.end() + s) - real_vector.size())); + assert(real_vector[s] == pre_vector[s]); + assert(&(pre_vector[s]) == &(pre_vector.begin()[s])); + assert(&(pre_vector[s]) == &*(pre_vector.begin() + s)); + assert(&(pre_vector[s]) == &*((pre_vector.end() + s) - real_vector.size())); } // assert(realtype(pre_vector) == real_vector); assert(pretype(real_vector.begin(), real_vector.end()) == pre_vector); assert(pretype(pre_vector.begin(), pre_vector.end()) == pre_vector); size_t pos = 0; for (const T& v : pre_vector) { - assert(v == real_vector[pos]); - ++pos; + assert(v == real_vector[pos]); + ++pos; } for (const T& v : reverse_iterate(pre_vector)) { - --pos; - assert(v == real_vector[pos]); + --pos; + assert(v == real_vector[pos]); } for (const T& v : const_pre_vector) { - assert(v == real_vector[pos]); - ++pos; + assert(v == real_vector[pos]); + ++pos; } for (const T& v : reverse_iterate(const_pre_vector)) { - --pos; - assert(v == real_vector[pos]); + --pos; + assert(v == real_vector[pos]); } CDataStream ss1(SER_DISK, 0); CDataStream ss2(SER_DISK, 0); @@ -67,101 +69,120 @@ public: } } - void resize(Size s) { + void resize(Size s) + { real_vector.resize(s); assert(real_vector.size() == s); pre_vector.resize(s); assert(pre_vector.size() == s); } - void reserve(Size s) { + void reserve(Size s) + { real_vector.reserve(s); assert(real_vector.capacity() >= s); pre_vector.reserve(s); assert(pre_vector.capacity() >= s); } - void insert(Size position, const T& value) { + void insert(Size position, const T& value) + { real_vector.insert(real_vector.begin() + position, value); pre_vector.insert(pre_vector.begin() + position, value); } - void insert(Size position, Size count, const T& value) { + void insert(Size position, Size count, const T& value) + { real_vector.insert(real_vector.begin() + position, count, value); pre_vector.insert(pre_vector.begin() + position, count, value); } - template<typename I> - void insert_range(Size position, I first, I last) { + template <typename I> + void insert_range(Size position, I first, I last) + { real_vector.insert(real_vector.begin() + position, first, last); pre_vector.insert(pre_vector.begin() + position, first, last); } - void erase(Size position) { + void erase(Size position) + { real_vector.erase(real_vector.begin() + position); pre_vector.erase(pre_vector.begin() + position); } - void erase(Size first, Size last) { + void erase(Size first, Size last) + { real_vector.erase(real_vector.begin() + first, real_vector.begin() + last); pre_vector.erase(pre_vector.begin() + first, pre_vector.begin() + last); } - void update(Size pos, const T& value) { + void update(Size pos, const T& value) + { real_vector[pos] = value; pre_vector[pos] = value; } - void push_back(const T& value) { + void push_back(const T& value) + { real_vector.push_back(value); pre_vector.push_back(value); } - void pop_back() { + void pop_back() + { real_vector.pop_back(); pre_vector.pop_back(); } - void clear() { + void clear() + { real_vector.clear(); pre_vector.clear(); } - void assign(Size n, const T& value) { + void assign(Size n, const T& value) + { real_vector.assign(n, value); pre_vector.assign(n, value); } - Size size() const { + Size size() const + { return real_vector.size(); } - Size capacity() const { + Size capacity() const + { return pre_vector.capacity(); } - void shrink_to_fit() { + void shrink_to_fit() + { pre_vector.shrink_to_fit(); } - void swap() { + void swap() + { real_vector.swap(real_vector_alt); pre_vector.swap(pre_vector_alt); } - void move() { + void move() + { real_vector = std::move(real_vector_alt); real_vector_alt.clear(); pre_vector = std::move(pre_vector_alt); pre_vector_alt.clear(); } - void copy() { + void copy() + { real_vector = real_vector_alt; pre_vector = pre_vector_alt; } - void resize_uninitialized(realtype values) { + void resize_uninitialized(realtype values) + { size_t r = values.size(); size_t s = real_vector.size() / 2; if (real_vector.capacity() < s + r) { @@ -181,7 +202,7 @@ public: } }; -} +} // namespace void test_one_input(const std::vector<uint8_t>& buffer) { diff --git a/src/test/fuzz/primitives_transaction.cpp b/src/test/fuzz/primitives_transaction.cpp index 2e5ba6bdb0..4a0f920f58 100644 --- a/src/test/fuzz/primitives_transaction.cpp +++ b/src/test/fuzz/primitives_transaction.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <optional.h> #include <primitives/transaction.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <cstdint> +#include <optional> #include <string> #include <vector> @@ -16,7 +16,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const CScript script = ConsumeScript(fuzzed_data_provider); - const Optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + const std::optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (out_point) { const CTxIn tx_in{*out_point, script, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; (void)tx_in; @@ -24,8 +24,8 @@ void test_one_input(const std::vector<uint8_t>& buffer) const CTxOut tx_out_1{ConsumeMoney(fuzzed_data_provider), script}; const CTxOut tx_out_2{ConsumeMoney(fuzzed_data_provider), ConsumeScript(fuzzed_data_provider)}; assert((tx_out_1 == tx_out_2) != (tx_out_1 != tx_out_2)); - const Optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); - const Optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (mutable_tx_1 && mutable_tx_2) { const CTransaction tx_1{*mutable_tx_1}; const CTransaction tx_2{*mutable_tx_2}; diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index c03365199a..211a84b5f2 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -14,6 +14,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/util/mining.h> +#include <test/util/net.h> #include <test/util/setup_common.h> #include <util/memory.h> #include <validationinterface.h> @@ -29,7 +30,7 @@ #include <string> #include <vector> -bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc); +bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, ChainstateManager& chainman, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc); namespace { @@ -63,19 +64,23 @@ void initialize() void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + ConnmanTestMsg& connman = *(ConnmanTestMsg*)g_setup->m_node.connman.get(); const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; if (!LIMIT_TO_MESSAGE_TYPE.empty() && random_message_type != LIMIT_TO_MESSAGE_TYPE) { return; } CDataStream random_bytes_data_stream{fuzzed_data_provider.ConsumeRemainingBytes<unsigned char>(), SER_NETWORK, PROTOCOL_VERSION}; - CNode p2p_node{0, ServiceFlags(NODE_NETWORK | NODE_WITNESS | NODE_BLOOM), 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, false}; + CNode& p2p_node = *MakeUnique<CNode>(0, ServiceFlags(NODE_NETWORK | NODE_WITNESS | NODE_BLOOM), 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, false).release(); p2p_node.fSuccessfullyConnected = true; p2p_node.nVersion = PROTOCOL_VERSION; p2p_node.SetSendVersion(PROTOCOL_VERSION); + connman.AddTestNode(p2p_node); g_setup->m_node.peer_logic->InitializeNode(&p2p_node); try { - (void)ProcessMessage(&p2p_node, random_message_type, random_bytes_data_stream, GetTimeMillis(), Params(), *g_setup->m_node.mempool, g_setup->m_node.connman.get(), g_setup->m_node.banman.get(), std::atomic<bool>{false}); + (void)ProcessMessage(p2p_node, random_message_type, random_bytes_data_stream, GetTimeMillis(), Params(), *g_setup->m_node.chainman, *g_setup->m_node.mempool, g_setup->m_node.connman.get(), g_setup->m_node.banman.get(), std::atomic<bool>{false}); } catch (const std::ios_base::failure&) { } SyncWithValidationInterfaceQueue(); + LOCK2(::cs_main, g_cs_orphans); // See init.cpp for rationale for implicit locking order requirement + g_setup->m_node.connman->StopNodes(); } diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index bcbf65bdca..ad6c115a49 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -75,6 +75,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) } catch (const std::ios_base::failure&) { } } - connman.ClearTestNodes(); SyncWithValidationInterfaceQueue(); + LOCK2(::cs_main, g_cs_orphans); // See init.cpp for rationale for implicit locking order requirement + g_setup->m_node.connman->StopNodes(); } diff --git a/src/test/fuzz/protocol.cpp b/src/test/fuzz/protocol.cpp index 954471de6c..78df0f89e7 100644 --- a/src/test/fuzz/protocol.cpp +++ b/src/test/fuzz/protocol.cpp @@ -2,20 +2,20 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <optional.h> #include <protocol.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <cstdint> +#include <optional> #include <stdexcept> #include <vector> void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const Optional<CInv> inv = ConsumeDeserializable<CInv>(fuzzed_data_provider); + const std::optional<CInv> inv = ConsumeDeserializable<CInv>(fuzzed_data_provider); if (!inv) { return; } @@ -24,7 +24,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) } catch (const std::out_of_range&) { } (void)inv->ToString(); - const Optional<CInv> another_inv = ConsumeDeserializable<CInv>(fuzzed_data_provider); + const std::optional<CInv> another_inv = ConsumeDeserializable<CInv>(fuzzed_data_provider); if (!another_inv) { return; } diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index eb54b05df9..1fd88a5f7b 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -2,7 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <optional.h> #include <policy/rbf.h> #include <primitives/transaction.h> #include <sync.h> @@ -12,19 +11,20 @@ #include <txmempool.h> #include <cstdint> +#include <optional> #include <string> #include <vector> void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - Optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mtx) { return; } CTxMemPool pool; while (fuzzed_data_provider.ConsumeBool()) { - const Optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!another_mtx) { break; } diff --git a/src/test/fuzz/rolling_bloom_filter.cpp b/src/test/fuzz/rolling_bloom_filter.cpp index 3b37321977..623b8cff3a 100644 --- a/src/test/fuzz/rolling_bloom_filter.cpp +++ b/src/test/fuzz/rolling_bloom_filter.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bloom.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> @@ -11,6 +10,7 @@ #include <cassert> #include <cstdint> +#include <optional> #include <string> #include <vector> @@ -32,7 +32,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) break; } case 1: { - const Optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); + const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); if (!u256) { break; } diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index de82122dd6..933cf9049d 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -11,6 +11,7 @@ #include <script/descriptor.h> #include <script/interpreter.h> #include <script/script.h> +#include <script/script_error.h> #include <script/sign.h> #include <script/signingprovider.h> #include <script/standard.h> @@ -21,6 +22,13 @@ #include <univalue.h> #include <util/memory.h> +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <optional> +#include <string> +#include <vector> + void initialize() { // Fuzzers using pubkey must hold an ECCVerifyHandle. @@ -32,7 +40,7 @@ void initialize() void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const Optional<CScript> script_opt = ConsumeDeserializable<CScript>(fuzzed_data_provider); + const std::optional<CScript> script_opt = ConsumeDeserializable<CScript>(fuzzed_data_provider); if (!script_opt) return; const CScript script{*script_opt}; @@ -101,7 +109,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) } } - const Optional<CScript> other_script = ConsumeDeserializable<CScript>(fuzzed_data_provider); + const std::optional<CScript> other_script = ConsumeDeserializable<CScript>(fuzzed_data_provider); if (other_script) { { CScript script_mut{script}; @@ -119,4 +127,40 @@ void test_one_input(const std::vector<uint8_t>& buffer) wit.SetNull(); } } + + (void)GetOpName(ConsumeOpcodeType(fuzzed_data_provider)); + (void)ScriptErrorString(static_cast<ScriptError>(fuzzed_data_provider.ConsumeIntegralInRange<int>(0, SCRIPT_ERR_ERROR_COUNT))); + + { + const std::vector<uint8_t> bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); + CScript append_script{bytes.begin(), bytes.end()}; + append_script << fuzzed_data_provider.ConsumeIntegral<int64_t>(); + append_script << ConsumeOpcodeType(fuzzed_data_provider); + append_script << CScriptNum{fuzzed_data_provider.ConsumeIntegral<int64_t>()}; + append_script << ConsumeRandomLengthByteVector(fuzzed_data_provider); + } + + { + WitnessUnknown witness_unknown_1{}; + witness_unknown_1.version = fuzzed_data_provider.ConsumeIntegral<int>(); + const std::vector<uint8_t> witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); + witness_unknown_1.length = witness_unknown_program_1.size(); + std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown_1.program); + + WitnessUnknown witness_unknown_2{}; + witness_unknown_2.version = fuzzed_data_provider.ConsumeIntegral<int>(); + const std::vector<uint8_t> witness_unknown_program_2 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); + witness_unknown_2.length = witness_unknown_program_2.size(); + std::copy(witness_unknown_program_2.begin(), witness_unknown_program_2.end(), witness_unknown_2.program); + + (void)(witness_unknown_1 == witness_unknown_2); + (void)(witness_unknown_1 < witness_unknown_2); + } + + { + const CTxDestination tx_destination_1 = ConsumeTxDestination(fuzzed_data_provider); + const CTxDestination tx_destination_2 = ConsumeTxDestination(fuzzed_data_provider); + (void)(tx_destination_1 == tx_destination_2); + (void)(tx_destination_1 < tx_destination_2); + } } diff --git a/src/test/fuzz/script_bitcoin_consensus.cpp b/src/test/fuzz/script_bitcoin_consensus.cpp new file mode 100644 index 0000000000..22f4b4f44a --- /dev/null +++ b/src/test/fuzz/script_bitcoin_consensus.cpp @@ -0,0 +1,31 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <script/bitcoinconsensus.h> +#include <script/interpreter.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const std::vector<uint8_t> random_bytes_1 = ConsumeRandomLengthByteVector(fuzzed_data_provider); + const std::vector<uint8_t> random_bytes_2 = ConsumeRandomLengthByteVector(fuzzed_data_provider); + const CAmount money = ConsumeMoney(fuzzed_data_provider); + bitcoinconsensus_error err; + bitcoinconsensus_error* err_p = fuzzed_data_provider.ConsumeBool() ? &err : nullptr; + const unsigned int n_in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + const unsigned int flags = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + assert(bitcoinconsensus_version() == BITCOINCONSENSUS_API_VER); + if ((flags & SCRIPT_VERIFY_WITNESS) != 0 && (flags & SCRIPT_VERIFY_P2SH) == 0) { + return; + } + (void)bitcoinconsensus_verify_script(random_bytes_1.data(), random_bytes_1.size(), random_bytes_2.data(), random_bytes_2.size(), n_in, flags, err_p); + (void)bitcoinconsensus_verify_script_with_amount(random_bytes_1.data(), random_bytes_1.size(), money, random_bytes_2.data(), random_bytes_2.size(), n_in, flags, err_p); +} diff --git a/src/test/fuzz/script_descriptor_cache.cpp b/src/test/fuzz/script_descriptor_cache.cpp new file mode 100644 index 0000000000..4bfe61cec7 --- /dev/null +++ b/src/test/fuzz/script_descriptor_cache.cpp @@ -0,0 +1,42 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <optional.h> +#include <pubkey.h> +#include <script/descriptor.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + DescriptorCache descriptor_cache; + while (fuzzed_data_provider.ConsumeBool()) { + const std::vector<uint8_t> code = fuzzed_data_provider.ConsumeBytes<uint8_t>(BIP32_EXTKEY_SIZE); + if (code.size() == BIP32_EXTKEY_SIZE) { + CExtPubKey xpub; + xpub.Decode(code.data()); + const uint32_t key_exp_pos = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); + CExtPubKey xpub_fetched; + if (fuzzed_data_provider.ConsumeBool()) { + (void)descriptor_cache.GetCachedParentExtPubKey(key_exp_pos, xpub_fetched); + descriptor_cache.CacheParentExtPubKey(key_exp_pos, xpub); + assert(descriptor_cache.GetCachedParentExtPubKey(key_exp_pos, xpub_fetched)); + } else { + const uint32_t der_index = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); + (void)descriptor_cache.GetCachedDerivedExtPubKey(key_exp_pos, der_index, xpub_fetched); + descriptor_cache.CacheDerivedExtPubKey(key_exp_pos, der_index, xpub); + assert(descriptor_cache.GetCachedDerivedExtPubKey(key_exp_pos, der_index, xpub_fetched)); + } + assert(xpub == xpub_fetched); + } + (void)descriptor_cache.GetCachedParentExtPubKeys(); + (void)descriptor_cache.GetCachedDerivedExtPubKeys(); + } +} diff --git a/src/test/fuzz/script_interpreter.cpp b/src/test/fuzz/script_interpreter.cpp new file mode 100644 index 0000000000..26d5732f24 --- /dev/null +++ b/src/test/fuzz/script_interpreter.cpp @@ -0,0 +1,41 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <primitives/transaction.h> +#include <script/interpreter.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <optional> +#include <string> +#include <vector> + +bool CastToBool(const std::vector<unsigned char>& vch); + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + { + const CScript script_code = ConsumeScript(fuzzed_data_provider); + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (mtx) { + const CTransaction tx_to{*mtx}; + const unsigned int in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + if (in < tx_to.vin.size()) { + (void)SignatureHash(script_code, tx_to, in, fuzzed_data_provider.ConsumeIntegral<int>(), ConsumeMoney(fuzzed_data_provider), fuzzed_data_provider.PickValueInArray({SigVersion::BASE, SigVersion::WITNESS_V0}), nullptr); + const std::optional<CMutableTransaction> mtx_precomputed = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (mtx_precomputed) { + const CTransaction tx_precomputed{*mtx_precomputed}; + const PrecomputedTransactionData precomputed_transaction_data{tx_precomputed}; + (void)SignatureHash(script_code, tx_to, in, fuzzed_data_provider.ConsumeIntegral<int>(), ConsumeMoney(fuzzed_data_provider), fuzzed_data_provider.PickValueInArray({SigVersion::BASE, SigVersion::WITNESS_V0}), &precomputed_transaction_data); + } + } + } + } + { + (void)CastToBool(ConsumeRandomLengthByteVector(fuzzed_data_provider)); + } +} diff --git a/src/test/fuzz/script_sigcache.cpp b/src/test/fuzz/script_sigcache.cpp new file mode 100644 index 0000000000..434a47b702 --- /dev/null +++ b/src/test/fuzz/script_sigcache.cpp @@ -0,0 +1,45 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <chainparamsbase.h> +#include <key.h> +#include <pubkey.h> +#include <script/sigcache.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <optional> +#include <string> +#include <vector> + +void initialize() +{ + static const ECCVerifyHandle ecc_verify_handle; + ECC_Start(); + SelectParams(CBaseChainParams::REGTEST); + InitSignatureCache(); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const CTransaction tx = mutable_transaction ? CTransaction{*mutable_transaction} : CTransaction{}; + const unsigned int n_in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + const CAmount amount = ConsumeMoney(fuzzed_data_provider); + const bool store = fuzzed_data_provider.ConsumeBool(); + PrecomputedTransactionData tx_data; + CachingTransactionSignatureChecker caching_transaction_signature_checker{mutable_transaction ? &tx : nullptr, n_in, amount, store, tx_data}; + const std::optional<CPubKey> pub_key = ConsumeDeserializable<CPubKey>(fuzzed_data_provider); + if (pub_key) { + const std::vector<uint8_t> random_bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); + if (!random_bytes.empty()) { + (void)caching_transaction_signature_checker.VerifySignature(random_bytes, *pub_key, ConsumeUInt256(fuzzed_data_provider)); + } + } +} diff --git a/src/test/fuzz/script_sign.cpp b/src/test/fuzz/script_sign.cpp new file mode 100644 index 0000000000..c626f950e7 --- /dev/null +++ b/src/test/fuzz/script_sign.cpp @@ -0,0 +1,149 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <chainparamsbase.h> +#include <key.h> +#include <pubkey.h> +#include <script/keyorigin.h> +#include <script/sign.h> +#include <script/signingprovider.h> +#include <streams.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cassert> +#include <cstdint> +#include <iostream> +#include <map> +#include <optional> +#include <string> +#include <vector> + +void initialize() +{ + static const ECCVerifyHandle ecc_verify_handle; + ECC_Start(); + SelectParams(CBaseChainParams::REGTEST); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const std::vector<uint8_t> key = ConsumeRandomLengthByteVector(fuzzed_data_provider, 128); + + { + CDataStream random_data_stream = ConsumeDataStream(fuzzed_data_provider); + std::map<CPubKey, KeyOriginInfo> hd_keypaths; + try { + DeserializeHDKeypaths(random_data_stream, key, hd_keypaths); + } catch (const std::ios_base::failure&) { + } + CDataStream serialized{SER_NETWORK, PROTOCOL_VERSION}; + SerializeHDKeypaths(serialized, hd_keypaths, fuzzed_data_provider.ConsumeIntegral<uint8_t>()); + } + + { + std::map<CPubKey, KeyOriginInfo> hd_keypaths; + while (fuzzed_data_provider.ConsumeBool()) { + const std::optional<CPubKey> pub_key = ConsumeDeserializable<CPubKey>(fuzzed_data_provider); + if (!pub_key) { + break; + } + const std::optional<KeyOriginInfo> key_origin_info = ConsumeDeserializable<KeyOriginInfo>(fuzzed_data_provider); + if (!key_origin_info) { + break; + } + hd_keypaths[*pub_key] = *key_origin_info; + } + CDataStream serialized{SER_NETWORK, PROTOCOL_VERSION}; + try { + SerializeHDKeypaths(serialized, hd_keypaths, fuzzed_data_provider.ConsumeIntegral<uint8_t>()); + } catch (const std::ios_base::failure&) { + } + std::map<CPubKey, KeyOriginInfo> deserialized_hd_keypaths; + try { + DeserializeHDKeypaths(serialized, key, hd_keypaths); + } catch (const std::ios_base::failure&) { + } + assert(hd_keypaths.size() >= deserialized_hd_keypaths.size()); + } + + { + SignatureData signature_data_1{ConsumeScript(fuzzed_data_provider)}; + SignatureData signature_data_2{ConsumeScript(fuzzed_data_provider)}; + signature_data_1.MergeSignatureData(signature_data_2); + } + + FillableSigningProvider provider; + CKey k; + const std::vector<uint8_t> key_data = ConsumeRandomLengthByteVector(fuzzed_data_provider); + k.Set(key_data.begin(), key_data.end(), fuzzed_data_provider.ConsumeBool()); + if (k.IsValid()) { + provider.AddKey(k); + } + + { + const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CTxOut> tx_out = ConsumeDeserializable<CTxOut>(fuzzed_data_provider); + const unsigned int n_in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + if (mutable_transaction && tx_out && mutable_transaction->vin.size() > n_in) { + SignatureData signature_data_1 = DataFromTransaction(*mutable_transaction, n_in, *tx_out); + CTxIn input; + UpdateInput(input, signature_data_1); + const CScript script = ConsumeScript(fuzzed_data_provider); + SignatureData signature_data_2{script}; + signature_data_1.MergeSignatureData(signature_data_2); + } + if (mutable_transaction) { + CTransaction tx_from{*mutable_transaction}; + CMutableTransaction tx_to; + const std::optional<CMutableTransaction> opt_tx_to = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (opt_tx_to) { + tx_to = *opt_tx_to; + } + CMutableTransaction script_tx_to = tx_to; + CMutableTransaction sign_transaction_tx_to = tx_to; + if (n_in < tx_to.vin.size() && tx_to.vin[n_in].prevout.n < tx_from.vout.size()) { + (void)SignSignature(provider, tx_from, tx_to, n_in, fuzzed_data_provider.ConsumeIntegral<int>()); + } + if (n_in < script_tx_to.vin.size()) { + (void)SignSignature(provider, ConsumeScript(fuzzed_data_provider), script_tx_to, n_in, ConsumeMoney(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<int>()); + MutableTransactionSignatureCreator signature_creator{&tx_to, n_in, ConsumeMoney(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<int>()}; + std::vector<unsigned char> vch_sig; + CKeyID address; + if (fuzzed_data_provider.ConsumeBool()) { + if (k.IsValid()) { + address = k.GetPubKey().GetID(); + } + } else { + address = CKeyID{ConsumeUInt160(fuzzed_data_provider)}; + } + (void)signature_creator.CreateSig(provider, vch_sig, address, ConsumeScript(fuzzed_data_provider), fuzzed_data_provider.PickValueInArray({SigVersion::BASE, SigVersion::WITNESS_V0})); + } + std::map<COutPoint, Coin> coins; + while (fuzzed_data_provider.ConsumeBool()) { + const std::optional<COutPoint> outpoint = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + if (!outpoint) { + break; + } + const std::optional<Coin> coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); + if (!coin) { + break; + } + coins[*outpoint] = *coin; + } + std::map<int, std::string> input_errors; + (void)SignTransaction(sign_transaction_tx_to, &provider, coins, fuzzed_data_provider.ConsumeIntegral<int>(), input_errors); + } + } + + { + SignatureData signature_data_1; + (void)ProduceSignature(provider, DUMMY_SIGNATURE_CREATOR, ConsumeScript(fuzzed_data_provider), signature_data_1); + SignatureData signature_data_2; + (void)ProduceSignature(provider, DUMMY_MAXIMUM_SIGNATURE_CREATOR, ConsumeScript(fuzzed_data_provider), signature_data_2); + } +} diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 49bee0e81f..271062dc95 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -12,6 +12,7 @@ #include <rpc/server.h> #include <rpc/util.h> #include <script/descriptor.h> +#include <script/script.h> #include <serialize.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> @@ -89,11 +90,15 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)urlDecode(random_string_1); (void)ValidAsCString(random_string_1); (void)_(random_string_1.c_str()); + try { + throw scriptnum_error{random_string_1}; + } catch (const std::runtime_error&) { + } { CDataStream data_stream{SER_NETWORK, INIT_PROTO_VERSION}; std::string s; - LimitedString<10> limited_string = LIMITED_STRING(s, 10); + auto limited_string = LIMITED_STRING(s, 10); data_stream << random_string_1; try { data_stream >> limited_string; @@ -108,7 +113,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) } { CDataStream data_stream{SER_NETWORK, INIT_PROTO_VERSION}; - const LimitedString<10> limited_string = LIMITED_STRING(random_string_1, 10); + const auto limited_string = LIMITED_STRING(random_string_1, 10); data_stream << limited_string; std::string deserialized_string; data_stream >> deserialized_string; @@ -119,4 +124,10 @@ void test_one_input(const std::vector<uint8_t>& buffer) int64_t amount_out; (void)ParseFixedPoint(random_string_1, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 1024), &amount_out); } + { + (void)Untranslated(random_string_1); + const bilingual_str bs1{random_string_1, random_string_2}; + const bilingual_str bs2{random_string_2, random_string_1}; + (void)(bs1 + bs2); + } } diff --git a/src/test/fuzz/strprintf.cpp b/src/test/fuzz/strprintf.cpp index d5be1070bd..29064bc45c 100644 --- a/src/test/fuzz/strprintf.cpp +++ b/src/test/fuzz/strprintf.cpp @@ -6,6 +6,7 @@ #include <test/fuzz/fuzz.h> #include <tinyformat.h> #include <util/strencodings.h> +#include <util/translation.h> #include <algorithm> #include <cstdint> @@ -16,6 +17,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::string format_string = fuzzed_data_provider.ConsumeRandomLengthString(64); + const bilingual_str bilingual_string{format_string, format_string}; const int digits_in_format_specifier = std::count_if(format_string.begin(), format_string.end(), IsDigit); @@ -47,50 +49,62 @@ void test_one_input(const std::vector<uint8_t>& buffer) try { (void)strprintf(format_string, (signed char*)nullptr); + (void)tinyformat::format(bilingual_string, (signed char*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (unsigned char*)nullptr); + (void)tinyformat::format(bilingual_string, (unsigned char*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (void*)nullptr); + (void)tinyformat::format(bilingual_string, (void*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (bool*)nullptr); + (void)tinyformat::format(bilingual_string, (bool*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (float*)nullptr); + (void)tinyformat::format(bilingual_string, (float*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (double*)nullptr); + (void)tinyformat::format(bilingual_string, (double*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (int16_t*)nullptr); + (void)tinyformat::format(bilingual_string, (int16_t*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (uint16_t*)nullptr); + (void)tinyformat::format(bilingual_string, (uint16_t*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (int32_t*)nullptr); + (void)tinyformat::format(bilingual_string, (int32_t*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (uint32_t*)nullptr); + (void)tinyformat::format(bilingual_string, (uint32_t*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (int64_t*)nullptr); + (void)tinyformat::format(bilingual_string, (int64_t*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (uint64_t*)nullptr); + (void)tinyformat::format(bilingual_string, (uint64_t*)nullptr); } catch (const tinyformat::format_error&) { } @@ -98,21 +112,27 @@ void test_one_input(const std::vector<uint8_t>& buffer) switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 5)) { case 0: (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32)); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeRandomLengthString(32)); break; case 1: (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32).c_str()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeRandomLengthString(32).c_str()); break; case 2: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<signed char>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<signed char>()); break; case 3: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<unsigned char>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<unsigned char>()); break; case 4: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<char>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<char>()); break; case 5: (void)strprintf(format_string, fuzzed_data_provider.ConsumeBool()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeBool()); break; } } catch (const tinyformat::format_error&) { @@ -138,27 +158,35 @@ void test_one_input(const std::vector<uint8_t>& buffer) switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 7)) { case 0: (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<float>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeFloatingPoint<float>()); break; case 1: (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<double>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeFloatingPoint<double>()); break; case 2: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int16_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<int16_t>()); break; case 3: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint16_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<uint16_t>()); break; case 4: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int32_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<int32_t>()); break; case 5: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint32_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<uint32_t>()); break; case 6: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int64_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<int64_t>()); break; case 7: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<uint64_t>()); break; } } catch (const tinyformat::format_error&) { diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp index 7f378c2b13..01b523cee4 100644 --- a/src/test/fuzz/system.cpp +++ b/src/test/fuzz/system.cpp @@ -85,7 +85,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) case 7: { const std::vector<std::string> random_arguments = ConsumeRandomLengthStringVector(fuzzed_data_provider); std::vector<const char*> argv; - argv.resize(random_arguments.size()); + argv.reserve(random_arguments.size()); for (const std::string& random_argument : random_arguments) { argv.push_back(random_argument.c_str()); } diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 501bb1de5a..f26878a704 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -8,10 +8,11 @@ #include <amount.h> #include <arith_uint256.h> #include <attributes.h> +#include <coins.h> #include <consensus/consensus.h> -#include <optional.h> #include <primitives/transaction.h> #include <script/script.h> +#include <script/standard.h> #include <serialize.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> @@ -20,7 +21,9 @@ #include <uint256.h> #include <version.h> +#include <algorithm> #include <cstdint> +#include <optional> #include <string> #include <vector> @@ -30,6 +33,11 @@ NODISCARD inline std::vector<uint8_t> ConsumeRandomLengthByteVector(FuzzedDataPr return {s.begin(), s.end()}; } +NODISCARD inline CDataStream ConsumeDataStream(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept +{ + return {ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length), SER_NETWORK, INIT_PROTO_VERSION}; +} + NODISCARD inline std::vector<std::string> ConsumeRandomLengthStringVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_vector_size = 16, const size_t max_string_length = 16) noexcept { const size_t n_elements = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, max_vector_size); @@ -52,7 +60,7 @@ NODISCARD inline std::vector<T> ConsumeRandomLengthIntegralVector(FuzzedDataProv } template <typename T> -NODISCARD inline Optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept +NODISCARD inline std::optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept { const std::vector<uint8_t> buffer = ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length); CDataStream ds{buffer, SER_NETWORK, INIT_PROTO_VERSION}; @@ -60,7 +68,7 @@ NODISCARD inline Optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzzed_da try { ds >> obj; } catch (const std::ios_base::failure&) { - return nullopt; + return std::nullopt; } return obj; } @@ -86,10 +94,19 @@ NODISCARD inline CScriptNum ConsumeScriptNum(FuzzedDataProvider& fuzzed_data_pro return CScriptNum{fuzzed_data_provider.ConsumeIntegral<int64_t>()}; } +NODISCARD inline uint160 ConsumeUInt160(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + const std::vector<uint8_t> v160 = fuzzed_data_provider.ConsumeBytes<uint8_t>(160 / 8); + if (v160.size() != 160 / 8) { + return {}; + } + return uint160{v160}; +} + NODISCARD inline uint256 ConsumeUInt256(FuzzedDataProvider& fuzzed_data_provider) noexcept { - const std::vector<unsigned char> v256 = fuzzed_data_provider.ConsumeBytes<unsigned char>(sizeof(uint256)); - if (v256.size() != sizeof(uint256)) { + const std::vector<uint8_t> v256 = fuzzed_data_provider.ConsumeBytes<uint8_t>(256 / 8); + if (v256.size() != 256 / 8) { return {}; } return uint256{v256}; @@ -115,6 +132,43 @@ NODISCARD inline CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzze return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; } +NODISCARD inline CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + CTxDestination tx_destination; + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 5)) { + case 0: { + tx_destination = CNoDestination{}; + break; + } + case 1: { + tx_destination = PKHash{ConsumeUInt160(fuzzed_data_provider)}; + break; + } + case 2: { + tx_destination = ScriptHash{ConsumeUInt160(fuzzed_data_provider)}; + break; + } + case 3: { + tx_destination = WitnessV0ScriptHash{ConsumeUInt256(fuzzed_data_provider)}; + break; + } + case 4: { + tx_destination = WitnessV0KeyHash{ConsumeUInt160(fuzzed_data_provider)}; + break; + } + case 5: { + WitnessUnknown witness_unknown{}; + witness_unknown.version = fuzzed_data_provider.ConsumeIntegral<int>(); + const std::vector<uint8_t> witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); + witness_unknown.length = witness_unknown_program_1.size(); + std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown.program); + tx_destination = witness_unknown; + break; + } + } + return tx_destination; +} + template <typename T> NODISCARD bool MultiplicationOverflow(const T i, const T j) noexcept { @@ -149,4 +203,15 @@ NODISCARD bool AdditionOverflow(const T i, const T j) noexcept return std::numeric_limits<T>::max() - i < j; } +NODISCARD inline bool ContainsSpentInput(const CTransaction& tx, const CCoinsViewCache& inputs) noexcept +{ + for (const CTxIn& tx_in : tx.vin) { + const Coin& coin = inputs.AccessCoin(tx_in.prevout); + if (coin.IsSpent()) { + return true; + } + } + return false; +} + #endif // BITCOIN_TEST_FUZZ_UTIL_H diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 512e48f8e5..45c9b90ee9 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -13,9 +13,18 @@ #include <boost/algorithm/string.hpp> #include <boost/test/unit_test.hpp> -BOOST_FIXTURE_TEST_SUITE(getarg_tests, BasicTestingSetup) +namespace getarg_tests{ + class LocalTestingSetup : BasicTestingSetup { + protected: + void SetupArgs(const std::vector<std::pair<std::string, unsigned int>>& args); + void ResetArgs(const std::string& strArg); + ArgsManager m_args; + }; +} + +BOOST_FIXTURE_TEST_SUITE(getarg_tests, LocalTestingSetup) -static void ResetArgs(const std::string& strArg) +void LocalTestingSetup :: ResetArgs(const std::string& strArg) { std::vector<std::string> vecArg; if (strArg.size()) @@ -30,14 +39,14 @@ static void ResetArgs(const std::string& strArg) vecChar.push_back(s.c_str()); std::string error; - BOOST_CHECK(gArgs.ParseParameters(vecChar.size(), vecChar.data(), error)); + BOOST_CHECK(m_args.ParseParameters(vecChar.size(), vecChar.data(), error)); } -static void SetupArgs(const std::vector<std::pair<std::string, unsigned int>>& args) +void LocalTestingSetup :: SetupArgs(const std::vector<std::pair<std::string, unsigned int>>& args) { - gArgs.ClearArgs(); + m_args.ClearArgs(); for (const auto& arg : args) { - gArgs.AddArg(arg.first, "", arg.second, OptionsCategory::OPTIONS); + m_args.AddArg(arg.first, "", arg.second, OptionsCategory::OPTIONS); } } @@ -46,52 +55,52 @@ BOOST_AUTO_TEST_CASE(boolarg) const auto foo = std::make_pair("-foo", ArgsManager::ALLOW_ANY); SetupArgs({foo}); ResetArgs("-foo"); - BOOST_CHECK(gArgs.GetBoolArg("-foo", false)); - BOOST_CHECK(gArgs.GetBoolArg("-foo", true)); + BOOST_CHECK(m_args.GetBoolArg("-foo", false)); + BOOST_CHECK(m_args.GetBoolArg("-foo", true)); - BOOST_CHECK(!gArgs.GetBoolArg("-fo", false)); - BOOST_CHECK(gArgs.GetBoolArg("-fo", true)); + BOOST_CHECK(!m_args.GetBoolArg("-fo", false)); + BOOST_CHECK(m_args.GetBoolArg("-fo", true)); - BOOST_CHECK(!gArgs.GetBoolArg("-fooo", false)); - BOOST_CHECK(gArgs.GetBoolArg("-fooo", true)); + BOOST_CHECK(!m_args.GetBoolArg("-fooo", false)); + BOOST_CHECK(m_args.GetBoolArg("-fooo", true)); ResetArgs("-foo=0"); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", false)); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", true)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", false)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", true)); ResetArgs("-foo=1"); - BOOST_CHECK(gArgs.GetBoolArg("-foo", false)); - BOOST_CHECK(gArgs.GetBoolArg("-foo", true)); + BOOST_CHECK(m_args.GetBoolArg("-foo", false)); + BOOST_CHECK(m_args.GetBoolArg("-foo", true)); // New 0.6 feature: auto-map -nosomething to !-something: ResetArgs("-nofoo"); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", false)); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", true)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", false)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", true)); ResetArgs("-nofoo=1"); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", false)); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", true)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", false)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", true)); ResetArgs("-foo -nofoo"); // -nofoo should win - BOOST_CHECK(!gArgs.GetBoolArg("-foo", false)); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", true)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", false)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", true)); ResetArgs("-foo=1 -nofoo=1"); // -nofoo should win - BOOST_CHECK(!gArgs.GetBoolArg("-foo", false)); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", true)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", false)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", true)); ResetArgs("-foo=0 -nofoo=0"); // -nofoo=0 should win - BOOST_CHECK(gArgs.GetBoolArg("-foo", false)); - BOOST_CHECK(gArgs.GetBoolArg("-foo", true)); + BOOST_CHECK(m_args.GetBoolArg("-foo", false)); + BOOST_CHECK(m_args.GetBoolArg("-foo", true)); // New 0.6 feature: treat -- same as -: ResetArgs("--foo=1"); - BOOST_CHECK(gArgs.GetBoolArg("-foo", false)); - BOOST_CHECK(gArgs.GetBoolArg("-foo", true)); + BOOST_CHECK(m_args.GetBoolArg("-foo", false)); + BOOST_CHECK(m_args.GetBoolArg("-foo", true)); ResetArgs("--nofoo=1"); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", false)); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", true)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", false)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", true)); } @@ -101,24 +110,24 @@ BOOST_AUTO_TEST_CASE(stringarg) const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_ANY); SetupArgs({foo, bar}); ResetArgs(""); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", ""), ""); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", "eleven"), "eleven"); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", ""), ""); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", "eleven"), "eleven"); ResetArgs("-foo -bar"); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", ""), ""); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", "eleven"), ""); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", ""), ""); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", "eleven"), ""); ResetArgs("-foo="); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", ""), ""); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", "eleven"), ""); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", ""), ""); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", "eleven"), ""); ResetArgs("-foo=11"); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", ""), "11"); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", "eleven"), "11"); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", ""), "11"); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", "eleven"), "11"); ResetArgs("-foo=eleven"); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", ""), "eleven"); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", "eleven"), "eleven"); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", ""), "eleven"); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", "eleven"), "eleven"); } @@ -128,20 +137,20 @@ BOOST_AUTO_TEST_CASE(intarg) const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_ANY); SetupArgs({foo, bar}); ResetArgs(""); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", 11), 11); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", 0), 0); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", 11), 11); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", 0), 0); ResetArgs("-foo -bar"); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", 11), 0); - BOOST_CHECK_EQUAL(gArgs.GetArg("-bar", 11), 0); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", 11), 0); + BOOST_CHECK_EQUAL(m_args.GetArg("-bar", 11), 0); ResetArgs("-foo=11 -bar=12"); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", 0), 11); - BOOST_CHECK_EQUAL(gArgs.GetArg("-bar", 11), 12); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", 0), 11); + BOOST_CHECK_EQUAL(m_args.GetArg("-bar", 11), 12); ResetArgs("-foo=NaN -bar=NotANumber"); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", 1), 0); - BOOST_CHECK_EQUAL(gArgs.GetArg("-bar", 11), 0); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", 1), 0); + BOOST_CHECK_EQUAL(m_args.GetArg("-bar", 11), 0); } BOOST_AUTO_TEST_CASE(doubledash) @@ -150,11 +159,11 @@ BOOST_AUTO_TEST_CASE(doubledash) const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_ANY); SetupArgs({foo, bar}); ResetArgs("--foo"); - BOOST_CHECK_EQUAL(gArgs.GetBoolArg("-foo", false), true); + BOOST_CHECK_EQUAL(m_args.GetBoolArg("-foo", false), true); ResetArgs("--foo=verbose --bar=1"); - BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", ""), "verbose"); - BOOST_CHECK_EQUAL(gArgs.GetArg("-bar", 0), 1); + BOOST_CHECK_EQUAL(m_args.GetArg("-foo", ""), "verbose"); + BOOST_CHECK_EQUAL(m_args.GetArg("-bar", 0), 1); } BOOST_AUTO_TEST_CASE(boolargno) @@ -163,24 +172,24 @@ BOOST_AUTO_TEST_CASE(boolargno) const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_ANY); SetupArgs({foo, bar}); ResetArgs("-nofoo"); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", true)); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", false)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", true)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", false)); ResetArgs("-nofoo=1"); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", true)); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", false)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", true)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", false)); ResetArgs("-nofoo=0"); - BOOST_CHECK(gArgs.GetBoolArg("-foo", true)); - BOOST_CHECK(gArgs.GetBoolArg("-foo", false)); + BOOST_CHECK(m_args.GetBoolArg("-foo", true)); + BOOST_CHECK(m_args.GetBoolArg("-foo", false)); ResetArgs("-foo --nofoo"); // --nofoo should win - BOOST_CHECK(!gArgs.GetBoolArg("-foo", true)); - BOOST_CHECK(!gArgs.GetBoolArg("-foo", false)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", true)); + BOOST_CHECK(!m_args.GetBoolArg("-foo", false)); ResetArgs("-nofoo -foo"); // foo always wins: - BOOST_CHECK(gArgs.GetBoolArg("-foo", true)); - BOOST_CHECK(gArgs.GetBoolArg("-foo", false)); + BOOST_CHECK(m_args.GetBoolArg("-foo", true)); + BOOST_CHECK(m_args.GetBoolArg("-foo", false)); } BOOST_AUTO_TEST_CASE(logargs) @@ -200,7 +209,7 @@ BOOST_AUTO_TEST_CASE(logargs) }); // Log the arguments - gArgs.LogArgs(); + m_args.LogArgs(); LogInstance().DeleteCallback(print_connection); // Check that what should appear does, and what shouldn't doesn't. diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 9f3ca87206..57eee94330 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -253,7 +253,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) pblock->nNonce = blockinfo[i].nonce; } std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); - BOOST_CHECK(ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); + BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); pblock->hashPrevBlock = pblock->GetHash(); } diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index ec6a290334..2e1972cc3f 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -393,7 +393,7 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) BOOST_CHECK(NetWhitelistPermissions::TryParse("bloom,forcerelay,noban,relay,mempool@1.2.3.4/32", whitelistPermissions, error)); const auto strings = NetPermissions::ToStrings(PF_ALL); - BOOST_CHECK_EQUAL(strings.size(), 5); + BOOST_CHECK_EQUAL(strings.size(), 5U); BOOST_CHECK(std::find(strings.begin(), strings.end(), "bloomfilter") != strings.end()); BOOST_CHECK(std::find(strings.begin(), strings.end(), "forcerelay") != strings.end()); BOOST_CHECK(std::find(strings.begin(), strings.end(), "relay") != strings.end()); diff --git a/src/test/raii_event_tests.cpp b/src/test/raii_event_tests.cpp index 04bf7c20c1..8c2712f764 100644 --- a/src/test/raii_event_tests.cpp +++ b/src/test/raii_event_tests.cpp @@ -4,9 +4,6 @@ #include <event2/event.h> -#ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED -// It would probably be ideal to define dummy test(s) that report skipped, but boost::test doesn't seem to make that practical (at least not in versions available with common distros) - #include <map> #include <stdlib.h> @@ -16,6 +13,10 @@ #include <boost/test/unit_test.hpp> +BOOST_FIXTURE_TEST_SUITE(raii_event_tests, BasicTestingSetup) + +#ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED + static std::map<void*, short> tags; static std::map<void*, uint16_t> orders; static uint16_t tagSequence = 0; @@ -34,8 +35,6 @@ static void tag_free(void* mem) { free(mem); } -BOOST_FIXTURE_TEST_SUITE(raii_event_tests, BasicTestingSetup) - BOOST_AUTO_TEST_CASE(raii_event_creation) { event_set_mem_functions(tag_malloc, realloc, tag_free); @@ -87,6 +86,14 @@ BOOST_AUTO_TEST_CASE(raii_event_order) event_set_mem_functions(malloc, realloc, free); } -BOOST_AUTO_TEST_SUITE_END() +#else + +BOOST_AUTO_TEST_CASE(raii_event_tests_SKIPPED) +{ + // It would probably be ideal to report skipped, but boost::test doesn't seem to make that practical (at least not in versions available with common distros) + BOOST_TEST_MESSAGE("Skipping raii_event_tess: libevent doesn't support event_set_mem_functions"); +} #endif // EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index ca3b92f2e1..978a7bee4d 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -28,6 +28,8 @@ BOOST_AUTO_TEST_CASE(fastrandom_tests) for (int i = 10; i > 0; --i) { BOOST_CHECK_EQUAL(GetRand(std::numeric_limits<uint64_t>::max()), uint64_t{10393729187455219830U}); BOOST_CHECK_EQUAL(GetRandInt(std::numeric_limits<int>::max()), int{769702006}); + BOOST_CHECK_EQUAL(GetRandMicros(std::chrono::hours{1}).count(), 2917185654); + BOOST_CHECK_EQUAL(GetRandMillis(std::chrono::hours{1}).count(), 2144374); } BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); @@ -47,6 +49,8 @@ BOOST_AUTO_TEST_CASE(fastrandom_tests) for (int i = 10; i > 0; --i) { BOOST_CHECK(GetRand(std::numeric_limits<uint64_t>::max()) != uint64_t{10393729187455219830U}); BOOST_CHECK(GetRandInt(std::numeric_limits<int>::max()) != int{769702006}); + BOOST_CHECK(GetRandMicros(std::chrono::hours{1}) != std::chrono::microseconds{2917185654}); + BOOST_CHECK(GetRandMillis(std::chrono::hours{1}) != std::chrono::milliseconds{2144374}); } { FastRandomContext ctx3, ctx4; @@ -87,7 +91,7 @@ BOOST_AUTO_TEST_CASE(stdrandom_test) BOOST_CHECK(x >= 3); BOOST_CHECK(x <= 9); - std::vector<int> test{1,2,3,4,5,6,7,8,9,10}; + std::vector<int> test{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; std::shuffle(test.begin(), test.end(), ctx); for (int j = 1; j <= 10; ++j) { BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); @@ -97,7 +101,6 @@ BOOST_AUTO_TEST_CASE(stdrandom_test) BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); } } - } /** Test that Shuffle reaches every permutation with equal probability. */ @@ -127,7 +130,7 @@ BOOST_AUTO_TEST_CASE(shuffle_stat_test) } BOOST_CHECK(chi_score > 58.1411); // 99.9999% confidence interval BOOST_CHECK(chi_score < 210.275); - BOOST_CHECK_EQUAL(sum, 12000); + BOOST_CHECK_EQUAL(sum, 12000U); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/ref_tests.cpp b/src/test/ref_tests.cpp new file mode 100644 index 0000000000..0ec0799fbc --- /dev/null +++ b/src/test/ref_tests.cpp @@ -0,0 +1,33 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/ref.h> + +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(ref_tests) + +BOOST_AUTO_TEST_CASE(ref_test) +{ + util::Ref ref; + BOOST_CHECK(!ref.Has<int>()); + BOOST_CHECK_THROW(ref.Get<int>(), NonFatalCheckError); + int value = 5; + ref.Set(value); + BOOST_CHECK(ref.Has<int>()); + BOOST_CHECK_EQUAL(ref.Get<int>(), 5); + ++ref.Get<int>(); + BOOST_CHECK_EQUAL(ref.Get<int>(), 6); + BOOST_CHECK_EQUAL(value, 6); + ++value; + BOOST_CHECK_EQUAL(value, 7); + BOOST_CHECK_EQUAL(ref.Get<int>(), 7); + BOOST_CHECK(!ref.Has<bool>()); + BOOST_CHECK_THROW(ref.Get<bool>(), NonFatalCheckError); + ref.Clear(); + BOOST_CHECK(!ref.Has<int>()); + BOOST_CHECK_THROW(ref.Get<int>(), NonFatalCheckError); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index d9c66f1c19..b54cbb3f00 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -10,6 +10,7 @@ #include <interfaces/chain.h> #include <node/context.h> #include <test/util/setup_common.h> +#include <util/ref.h> #include <util/time.h> #include <boost/algorithm/string.hpp> @@ -19,13 +20,20 @@ #include <rpc/blockchain.h> -UniValue CallRPC(std::string args) +class RPCTestingSetup : public TestingSetup +{ +public: + UniValue CallRPC(std::string args); +}; + +UniValue RPCTestingSetup::CallRPC(std::string args) { std::vector<std::string> vArgs; boost::split(vArgs, args, boost::is_any_of(" \t")); std::string strMethod = vArgs[0]; vArgs.erase(vArgs.begin()); - JSONRPCRequest request; + util::Ref context{m_node}; + JSONRPCRequest request(context); request.strMethod = strMethod; request.params = RPCConvertValues(strMethod, vArgs); request.fHelp = false; @@ -40,7 +48,7 @@ UniValue CallRPC(std::string args) } -BOOST_FIXTURE_TEST_SUITE(rpc_tests, TestingSetup) +BOOST_FIXTURE_TEST_SUITE(rpc_tests, RPCTestingSetup) BOOST_AUTO_TEST_CASE(rpc_rawparams) { diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 1395a7f38c..fcee6a9b9d 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -7,7 +7,7 @@ #include <util/time.h> #include <boost/test/unit_test.hpp> -#include <boost/thread.hpp> +#include <boost/thread/thread.hpp> #include <mutex> diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 56454f61f3..cb3ae290d1 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -102,7 +102,7 @@ static ScriptErrorDesc script_errors[]={ {SCRIPT_ERR_SIG_FINDANDDELETE, "SIG_FINDANDDELETE"}, }; -static const char *FormatScriptError(ScriptError_t err) +static std::string FormatScriptError(ScriptError_t err) { for (unsigned int i=0; i<ARRAYLEN(script_errors); ++i) if (script_errors[i].err == err) @@ -134,7 +134,7 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript CMutableTransaction tx = BuildSpendingTransaction(scriptSig, scriptWitness, txCredit); CMutableTransaction tx2 = tx; BOOST_CHECK_MESSAGE(VerifyScript(scriptSig, scriptPubKey, &scriptWitness, flags, MutableTransactionSignatureChecker(&tx, 0, txCredit.vout[0].nValue), &err) == expect, message); - BOOST_CHECK_MESSAGE(err == scriptError, std::string(FormatScriptError(err)) + " where " + std::string(FormatScriptError((ScriptError_t)scriptError)) + " expected: " + message); + BOOST_CHECK_MESSAGE(err == scriptError, FormatScriptError(err) + " where " + FormatScriptError((ScriptError_t)scriptError) + " expected: " + message); // Verify that removing flags from a passing test or adding flags to a failing test does not change the result. for (int i = 0; i < 16; ++i) { diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 9a6c721ab8..c2328f931c 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -29,15 +29,13 @@ public: memcpy(charstrval, charstrvalin, sizeof(charstrval)); } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(intval); - READWRITE(boolval); - READWRITE(stringval); - READWRITE(charstrval); - READWRITE(txval); + SERIALIZE_METHODS(CSerializeMethodsTestSingle, obj) + { + READWRITE(obj.intval); + READWRITE(obj.boolval); + READWRITE(obj.stringval); + READWRITE(obj.charstrval); + READWRITE(obj.txval); } bool operator==(const CSerializeMethodsTestSingle& rhs) @@ -54,11 +52,10 @@ class CSerializeMethodsTestMany : public CSerializeMethodsTestSingle { public: using CSerializeMethodsTestSingle::CSerializeMethodsTestSingle; - ADD_SERIALIZE_METHODS; - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(intval, boolval, stringval, charstrval, txval); + SERIALIZE_METHODS(CSerializeMethodsTestMany, obj) + { + READWRITE(obj.intval, obj.boolval, obj.stringval, obj.charstrval, obj.txval); } }; diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 177d8fda73..c509a252e0 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -72,28 +72,28 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) std::vector<unsigned char> vch = {1, 255, 3, 4, 5, 6}; VectorReader reader(SER_NETWORK, INIT_PROTO_VERSION, vch, 0); - BOOST_CHECK_EQUAL(reader.size(), 6); + BOOST_CHECK_EQUAL(reader.size(), 6U); BOOST_CHECK(!reader.empty()); // Read a single byte as an unsigned char. unsigned char a; reader >> a; BOOST_CHECK_EQUAL(a, 1); - BOOST_CHECK_EQUAL(reader.size(), 5); + BOOST_CHECK_EQUAL(reader.size(), 5U); BOOST_CHECK(!reader.empty()); // Read a single byte as a signed char. signed char b; reader >> b; BOOST_CHECK_EQUAL(b, -1); - BOOST_CHECK_EQUAL(reader.size(), 4); + BOOST_CHECK_EQUAL(reader.size(), 4U); BOOST_CHECK(!reader.empty()); // Read a 4 bytes as an unsigned int. unsigned int c; reader >> c; - BOOST_CHECK_EQUAL(c, 100992003); // 3,4,5,6 in little-endian base-256 - BOOST_CHECK_EQUAL(reader.size(), 0); + BOOST_CHECK_EQUAL(c, 100992003U); // 3,4,5,6 in little-endian base-256 + BOOST_CHECK_EQUAL(reader.size(), 0U); BOOST_CHECK(reader.empty()); // Reading after end of byte vector throws an error. @@ -104,7 +104,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) VectorReader new_reader(SER_NETWORK, INIT_PROTO_VERSION, vch, 0); new_reader >> d; BOOST_CHECK_EQUAL(d, 67370753); // 1,255,3,4 in little-endian base-256 - BOOST_CHECK_EQUAL(new_reader.size(), 2); + BOOST_CHECK_EQUAL(new_reader.size(), 2U); BOOST_CHECK(!new_reader.empty()); // Reading after end of byte vector throws an error even if the reader is @@ -136,14 +136,14 @@ BOOST_AUTO_TEST_CASE(bitstream_reader_writer) BOOST_CHECK_EQUAL(serialized_int2, (uint16_t)0x1072); // NOTE: Serialized as LE BitStreamReader<CDataStream> bit_reader(data_copy); - BOOST_CHECK_EQUAL(bit_reader.Read(1), 0); - BOOST_CHECK_EQUAL(bit_reader.Read(2), 2); - BOOST_CHECK_EQUAL(bit_reader.Read(3), 6); - BOOST_CHECK_EQUAL(bit_reader.Read(4), 11); - BOOST_CHECK_EQUAL(bit_reader.Read(5), 1); - BOOST_CHECK_EQUAL(bit_reader.Read(6), 32); - BOOST_CHECK_EQUAL(bit_reader.Read(7), 7); - BOOST_CHECK_EQUAL(bit_reader.Read(16), 30497); + BOOST_CHECK_EQUAL(bit_reader.Read(1), 0U); + BOOST_CHECK_EQUAL(bit_reader.Read(2), 2U); + BOOST_CHECK_EQUAL(bit_reader.Read(3), 6U); + BOOST_CHECK_EQUAL(bit_reader.Read(4), 11U); + BOOST_CHECK_EQUAL(bit_reader.Read(5), 1U); + BOOST_CHECK_EQUAL(bit_reader.Read(6), 32U); + BOOST_CHECK_EQUAL(bit_reader.Read(7), 7U); + BOOST_CHECK_EQUAL(bit_reader.Read(16), 30497U); BOOST_CHECK_THROW(bit_reader.Read(8), std::ios_base::failure); } @@ -236,7 +236,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_CHECK_EQUAL(i, 1); // After reading bytes 0 and 1, we're positioned at 2. - BOOST_CHECK_EQUAL(bf.GetPos(), 2); + BOOST_CHECK_EQUAL(bf.GetPos(), 2U); // Rewind to offset 0, ok (within the 10 byte window). BOOST_CHECK(bf.SetPos(0)); @@ -263,18 +263,18 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) // The default argument removes the limit completely. BOOST_CHECK(bf.SetLimit()); // The read position should still be at 3 (no change). - BOOST_CHECK_EQUAL(bf.GetPos(), 3); + BOOST_CHECK_EQUAL(bf.GetPos(), 3U); // Read from current offset, 3, forward until position 10. for (uint8_t j = 3; j < 10; ++j) { bf >> i; BOOST_CHECK_EQUAL(i, j); } - BOOST_CHECK_EQUAL(bf.GetPos(), 10); + BOOST_CHECK_EQUAL(bf.GetPos(), 10U); // We're guaranteed (just barely) to be able to rewind to zero. BOOST_CHECK(bf.SetPos(0)); - BOOST_CHECK_EQUAL(bf.GetPos(), 0); + BOOST_CHECK_EQUAL(bf.GetPos(), 0U); bf >> i; BOOST_CHECK_EQUAL(i, 0); @@ -284,12 +284,12 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_CHECK(bf.SetPos(10)); bf >> i; BOOST_CHECK_EQUAL(i, 10); - BOOST_CHECK_EQUAL(bf.GetPos(), 11); + BOOST_CHECK_EQUAL(bf.GetPos(), 11U); // Now it's only guaranteed that we can rewind to offset 1 // (current read position, 11, minus rewind amount, 10). BOOST_CHECK(bf.SetPos(1)); - BOOST_CHECK_EQUAL(bf.GetPos(), 1); + BOOST_CHECK_EQUAL(bf.GetPos(), 1U); bf >> i; BOOST_CHECK_EQUAL(i, 1); @@ -303,7 +303,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_CHECK_EQUAL(a[j], 11 + j); } } - BOOST_CHECK_EQUAL(bf.GetPos(), 40); + BOOST_CHECK_EQUAL(bf.GetPos(), 40U); // We've read the entire file, the next read should throw. try { @@ -317,11 +317,11 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_CHECK(bf.eof()); // Still at offset 40, we can go back 10, to 30. - BOOST_CHECK_EQUAL(bf.GetPos(), 40); + BOOST_CHECK_EQUAL(bf.GetPos(), 40U); BOOST_CHECK(bf.SetPos(30)); bf >> i; BOOST_CHECK_EQUAL(i, 30); - BOOST_CHECK_EQUAL(bf.GetPos(), 31); + BOOST_CHECK_EQUAL(bf.GetPos(), 31U); // We're too far to rewind to position zero. BOOST_CHECK(!bf.SetPos(0)); diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index 1df6844062..dac7f1a07b 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -31,7 +31,7 @@ CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) assert(block->nNonce); } - bool processed{ProcessNewBlock(Params(), block, true, nullptr)}; + bool processed{EnsureChainman(node).ProcessNewBlock(Params(), block, true, nullptr)}; assert(processed); return CTxIn{block->vtx[0]->GetHash(), 0}; diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index bf0afc4171..3b7a7c8d12 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -123,7 +123,6 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const const CChainParams& chainparams = Params(); // Ideally we'd move all the RPC tests to the functional testing framework // instead of unit tests, but for now we need these here. - g_rpc_node = &m_node; RegisterAllCoreRPCCommands(tableRPC); m_node.scheduler = MakeUnique<CScheduler>(); @@ -131,11 +130,12 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const // We have to run a scheduler thread to prevent ActivateBestChain // from blocking due to queue overrun. threadGroup.create_thread([&]{ m_node.scheduler->serviceQueue(); }); - GetMainSignals().RegisterBackgroundSignalScheduler(*g_rpc_node->scheduler); + GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - g_chainman.InitializeChainstate(); + m_node.chainman = &::g_chainman; + m_node.chainman->InitializeChainstate(); ::ChainstateActive().InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); assert(!::ChainstateActive().CanFlushToDisk()); @@ -161,7 +161,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const m_node.mempool->setSanityCheck(1.0); m_node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. - m_node.peer_logic = MakeUnique<PeerLogicValidation>(m_node.connman.get(), m_node.banman.get(), *m_node.scheduler, *m_node.mempool); + m_node.peer_logic = MakeUnique<PeerLogicValidation>(m_node.connman.get(), m_node.banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); { CConnman::Options options; options.m_msgproc = m_node.peer_logic.get(); @@ -176,14 +176,14 @@ TestingSetup::~TestingSetup() threadGroup.join_all(); GetMainSignals().FlushBackgroundCallbacks(); GetMainSignals().UnregisterBackgroundSignalScheduler(); - g_rpc_node = nullptr; m_node.connman.reset(); m_node.banman.reset(); m_node.args = nullptr; m_node.mempool = nullptr; m_node.scheduler.reset(); UnloadBlockIndex(); - g_chainman.Reset(); + m_node.chainman->Reset(); + m_node.chainman = nullptr; pblocktree.reset(); } @@ -228,7 +228,7 @@ CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransa while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); - ProcessNewBlock(chainparams, shared_pblock, true, nullptr); + EnsureChainman(m_node).ProcessNewBlock(chainparams, shared_pblock, true, nullptr); CBlock result = block; return result; diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 2477f9ad06..d5cda8a95b 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -17,7 +17,7 @@ #include <type_traits> -#include <boost/thread.hpp> +#include <boost/thread/thread.hpp> /** This is connected to the logger. Can be used to redirect logs to any other log */ extern const std::function<void(const std::string&)> G_TEST_LOG_FUN; diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 45b7fd4932..cf26ca3adb 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -240,9 +240,9 @@ public: BOOST_CHECK_EQUAL(test.GetSetting("-value").write(), expect.setting.write()); auto settings_list = test.GetSettingsList("-value"); if (expect.setting.isNull() || expect.setting.isFalse()) { - BOOST_CHECK_EQUAL(settings_list.size(), 0); + BOOST_CHECK_EQUAL(settings_list.size(), 0U); } else { - BOOST_CHECK_EQUAL(settings_list.size(), 1); + BOOST_CHECK_EQUAL(settings_list.size(), 1U); BOOST_CHECK_EQUAL(settings_list[0].write(), expect.setting.write()); } @@ -1911,7 +1911,7 @@ BOOST_AUTO_TEST_CASE(test_spanparsing) input = "xxx"; results = Split(MakeSpan(input), 'x'); - BOOST_CHECK_EQUAL(results.size(), 4); + BOOST_CHECK_EQUAL(results.size(), 4U); BOOST_CHECK_EQUAL(SpanToStr(results[0]), ""); BOOST_CHECK_EQUAL(SpanToStr(results[1]), ""); BOOST_CHECK_EQUAL(SpanToStr(results[2]), ""); @@ -1919,19 +1919,19 @@ BOOST_AUTO_TEST_CASE(test_spanparsing) input = "one#two#three"; results = Split(MakeSpan(input), '-'); - BOOST_CHECK_EQUAL(results.size(), 1); + BOOST_CHECK_EQUAL(results.size(), 1U); BOOST_CHECK_EQUAL(SpanToStr(results[0]), "one#two#three"); input = "one#two#three"; results = Split(MakeSpan(input), '#'); - BOOST_CHECK_EQUAL(results.size(), 3); + BOOST_CHECK_EQUAL(results.size(), 3U); BOOST_CHECK_EQUAL(SpanToStr(results[0]), "one"); BOOST_CHECK_EQUAL(SpanToStr(results[1]), "two"); BOOST_CHECK_EQUAL(SpanToStr(results[2]), "three"); input = "*foo*bar*"; results = Split(MakeSpan(input), '*'); - BOOST_CHECK_EQUAL(results.size(), 4); + BOOST_CHECK_EQUAL(results.size(), 4U); BOOST_CHECK_EQUAL(SpanToStr(results[0]), ""); BOOST_CHECK_EQUAL(SpanToStr(results[1]), "foo"); BOOST_CHECK_EQUAL(SpanToStr(results[2]), "bar"); @@ -1990,24 +1990,24 @@ BOOST_AUTO_TEST_CASE(test_tracked_vector) BOOST_CHECK(t3.origin == &t3); auto v1 = Vector(t1); - BOOST_CHECK_EQUAL(v1.size(), 1); + BOOST_CHECK_EQUAL(v1.size(), 1U); BOOST_CHECK(v1[0].origin == &t1); BOOST_CHECK_EQUAL(v1[0].copies, 1); auto v2 = Vector(std::move(t2)); - BOOST_CHECK_EQUAL(v2.size(), 1); + BOOST_CHECK_EQUAL(v2.size(), 1U); BOOST_CHECK(v2[0].origin == &t2); BOOST_CHECK_EQUAL(v2[0].copies, 0); auto v3 = Vector(t1, std::move(t2)); - BOOST_CHECK_EQUAL(v3.size(), 2); + BOOST_CHECK_EQUAL(v3.size(), 2U); BOOST_CHECK(v3[0].origin == &t1); BOOST_CHECK(v3[1].origin == &t2); BOOST_CHECK_EQUAL(v3[0].copies, 1); BOOST_CHECK_EQUAL(v3[1].copies, 0); auto v4 = Vector(std::move(v3[0]), v3[1], std::move(t3)); - BOOST_CHECK_EQUAL(v4.size(), 3); + BOOST_CHECK_EQUAL(v4.size(), 3U); BOOST_CHECK(v4[0].origin == &t1); BOOST_CHECK(v4[1].origin == &t2); BOOST_CHECK(v4[2].origin == &t3); @@ -2016,7 +2016,7 @@ BOOST_AUTO_TEST_CASE(test_tracked_vector) BOOST_CHECK_EQUAL(v4[2].copies, 0); auto v5 = Cat(v1, v4); - BOOST_CHECK_EQUAL(v5.size(), 4); + BOOST_CHECK_EQUAL(v5.size(), 4U); BOOST_CHECK(v5[0].origin == &t1); BOOST_CHECK(v5[1].origin == &t1); BOOST_CHECK(v5[2].origin == &t2); @@ -2027,7 +2027,7 @@ BOOST_AUTO_TEST_CASE(test_tracked_vector) BOOST_CHECK_EQUAL(v5[3].copies, 1); auto v6 = Cat(std::move(v1), v3); - BOOST_CHECK_EQUAL(v6.size(), 3); + BOOST_CHECK_EQUAL(v6.size(), 3U); BOOST_CHECK(v6[0].origin == &t1); BOOST_CHECK(v6[1].origin == &t1); BOOST_CHECK(v6[2].origin == &t2); @@ -2036,7 +2036,7 @@ BOOST_AUTO_TEST_CASE(test_tracked_vector) BOOST_CHECK_EQUAL(v6[2].copies, 1); auto v7 = Cat(v2, std::move(v4)); - BOOST_CHECK_EQUAL(v7.size(), 4); + BOOST_CHECK_EQUAL(v7.size(), 4U); BOOST_CHECK(v7[0].origin == &t2); BOOST_CHECK(v7[1].origin == &t1); BOOST_CHECK(v7[2].origin == &t2); @@ -2047,7 +2047,7 @@ BOOST_AUTO_TEST_CASE(test_tracked_vector) BOOST_CHECK_EQUAL(v7[3].copies, 0); auto v8 = Cat(std::move(v2), std::move(v3)); - BOOST_CHECK_EQUAL(v8.size(), 3); + BOOST_CHECK_EQUAL(v8.size(), 3U); BOOST_CHECK(v8[0].origin == &t2); BOOST_CHECK(v8[1].origin == &t1); BOOST_CHECK(v8[2].origin == &t2); diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index f226caf717..4dcc080b2d 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -62,7 +62,7 @@ BOOST_AUTO_TEST_CASE(util_threadnames_test_rename_threaded) std::set<std::string> names = RenameEnMasse(100); - BOOST_CHECK_EQUAL(names.size(), 100); + BOOST_CHECK_EQUAL(names.size(), 100U); // Names "test_thread.[n]" should exist for n = [0, 99] for (int i = 0; i < 100; ++i) { diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index c345f1eafb..45e0c5484e 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -32,7 +32,7 @@ struct MinerTestingSetup : public RegTestingSetup { BOOST_FIXTURE_TEST_SUITE(validation_block_tests, MinerTestingSetup) -struct TestSubscriber : public CValidationInterface { +struct TestSubscriber final : public CValidationInterface { uint256 m_expected_tip; explicit TestSubscriber(uint256 tip) : m_expected_tip(tip) {} @@ -163,10 +163,10 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) std::transform(blocks.begin(), blocks.end(), std::back_inserter(headers), [](std::shared_ptr<const CBlock> b) { return b->GetBlockHeader(); }); // Process all the headers so we understand the toplogy of the chain - BOOST_CHECK(ProcessNewBlockHeaders(headers, state, Params())); + BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlockHeaders(headers, state, Params())); // Connect the genesis block and drain any outstanding events - BOOST_CHECK(ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored)); + BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored)); SyncWithValidationInterfaceQueue(); // subscribe to events (this subscriber will validate event ordering) @@ -175,26 +175,26 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) LOCK(cs_main); initial_tip = ::ChainActive().Tip(); } - TestSubscriber sub(initial_tip->GetBlockHash()); - RegisterValidationInterface(&sub); + auto sub = std::make_shared<TestSubscriber>(initial_tip->GetBlockHash()); + RegisterSharedValidationInterface(sub); // create a bunch of threads that repeatedly process a block generated above at random // this will create parallelism and randomness inside validation - the ValidationInterface // will subscribe to events generated during block validation and assert on ordering invariance std::vector<std::thread> threads; for (int i = 0; i < 10; i++) { - threads.emplace_back([&blocks]() { + threads.emplace_back([&]() { bool ignored; FastRandomContext insecure; for (int i = 0; i < 1000; i++) { auto block = blocks[insecure.randrange(blocks.size() - 1)]; - ProcessNewBlock(Params(), block, true, &ignored); + EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, &ignored); } // to make sure that eventually we process the full chain - do it here for (auto block : blocks) { if (block->vtx.size() == 1) { - bool processed = ProcessNewBlock(Params(), block, true, &ignored); + bool processed = EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, &ignored); assert(processed); } } @@ -204,14 +204,12 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) for (auto& t : threads) { t.join(); } - while (GetMainSignals().CallbacksPending() > 0) { - UninterruptibleSleep(std::chrono::milliseconds{100}); - } + SyncWithValidationInterfaceQueue(); - UnregisterValidationInterface(&sub); + UnregisterSharedValidationInterface(sub); LOCK(cs_main); - BOOST_CHECK_EQUAL(sub.m_expected_tip, ::ChainActive().Tip()->GetBlockHash()); + BOOST_CHECK_EQUAL(sub->m_expected_tip, ::ChainActive().Tip()->GetBlockHash()); } /** @@ -234,8 +232,8 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) BOOST_AUTO_TEST_CASE(mempool_locks_reorg) { bool ignored; - auto ProcessBlock = [&ignored](std::shared_ptr<const CBlock> block) -> bool { - return ProcessNewBlock(Params(), block, /* fForceProcessing */ true, /* fNewBlock */ &ignored); + auto ProcessBlock = [&](std::shared_ptr<const CBlock> block) -> bool { + return EnsureChainman(m_node).ProcessNewBlock(Params(), block, /* fForceProcessing */ true, /* fNewBlock */ &ignored); }; // Process all mined blocks diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index 388a2dbd13..a863e3a4d5 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -46,7 +46,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) // (prevector<28, unsigned char>) when assigned 56 bytes of data per above. // // See also: Coin::DynamicMemoryUsage(). - constexpr int COIN_SIZE = is_64_bit ? 80 : 64; + constexpr unsigned int COIN_SIZE = is_64_bit ? 80 : 64; auto print_view_mem_usage = [](CCoinsViewCache& view) { BOOST_TEST_MESSAGE("CCoinsViewCache memory usage: " << view.DynamicMemoryUsage()); @@ -79,7 +79,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) } print_view_mem_usage(view); - BOOST_CHECK_EQUAL(view.DynamicMemoryUsage(), is_64_bit ? 32 : 16); + BOOST_CHECK_EQUAL(view.DynamicMemoryUsage(), is_64_bit ? 32U : 16U); // We should be able to add COINS_UNTIL_CRITICAL coins to the cache before going CRITICAL. // This is contingent not only on the dynamic memory usage of the Coins diff --git a/src/test/validationinterface_tests.cpp b/src/test/validationinterface_tests.cpp index 208be92852..ceba689e52 100644 --- a/src/test/validationinterface_tests.cpp +++ b/src/test/validationinterface_tests.cpp @@ -12,6 +12,40 @@ BOOST_FIXTURE_TEST_SUITE(validationinterface_tests, TestingSetup) +struct TestSubscriberNoop final : public CValidationInterface { + void BlockChecked(const CBlock&, const BlockValidationState&) override {} +}; + +BOOST_AUTO_TEST_CASE(unregister_validation_interface_race) +{ + std::atomic<bool> generate{true}; + + // Start thread to generate notifications + std::thread gen{[&] { + const CBlock block_dummy; + BlockValidationState state_dummy; + while (generate) { + GetMainSignals().BlockChecked(block_dummy, state_dummy); + } + }}; + + // Start thread to consume notifications + std::thread sub{[&] { + // keep going for about 1 sec, which is 250k iterations + for (int i = 0; i < 250000; i++) { + auto sub = std::make_shared<TestSubscriberNoop>(); + RegisterSharedValidationInterface(sub); + UnregisterSharedValidationInterface(sub); + } + // tell the other thread we are done + generate = false; + }}; + + gen.join(); + sub.join(); + BOOST_CHECK(!generate); +} + class TestInterface : public CValidationInterface { public: diff --git a/src/threadsafety.h b/src/threadsafety.h index bb988dfdfd..942aa3fdcd 100644 --- a/src/threadsafety.h +++ b/src/threadsafety.h @@ -6,6 +6,8 @@ #ifndef BITCOIN_THREADSAFETY_H #define BITCOIN_THREADSAFETY_H +#include <mutex> + #ifdef __clang__ // TL;DR Add GUARDED_BY(mutex) to member variables. The others are // rarely necessary. Ex: int nFoo GUARDED_BY(cs_foo); @@ -54,4 +56,19 @@ #define ASSERT_EXCLUSIVE_LOCK(...) #endif // __GNUC__ +// StdMutex provides an annotated version of std::mutex for us, +// and should only be used when sync.h Mutex/LOCK/etc are not usable. +class LOCKABLE StdMutex : public std::mutex +{ +}; + +// StdLockGuard provides an annotated version of std::lock_guard for us, +// and should only be used when sync.h Mutex/LOCK/etc are not usable. +class SCOPED_LOCKABLE StdLockGuard : public std::lock_guard<StdMutex> +{ +public: + explicit StdLockGuard(StdMutex& cs) EXCLUSIVE_LOCK_FUNCTION(cs) : std::lock_guard<StdMutex>(cs) {} + ~StdLockGuard() UNLOCK_FUNCTION() {} +}; + #endif // BITCOIN_THREADSAFETY_H diff --git a/src/timedata.cpp b/src/timedata.cpp index 942b3cb919..c67a3c96e8 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -15,9 +15,8 @@ #include <util/translation.h> #include <warnings.h> - -static RecursiveMutex cs_nTimeOffset; -static int64_t nTimeOffset GUARDED_BY(cs_nTimeOffset) = 0; +static Mutex g_timeoffset_mutex; +static int64_t nTimeOffset GUARDED_BY(g_timeoffset_mutex) = 0; /** * "Never go to sea with two chronometers; take one or three." @@ -28,7 +27,7 @@ static int64_t nTimeOffset GUARDED_BY(cs_nTimeOffset) = 0; */ int64_t GetTimeOffset() { - LOCK(cs_nTimeOffset); + LOCK(g_timeoffset_mutex); return nTimeOffset; } @@ -46,7 +45,7 @@ static int64_t abs64(int64_t n) void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) { - LOCK(cs_nTimeOffset); + LOCK(g_timeoffset_mutex); // Ignore duplicates static std::set<CNetAddr> setKnown; if (setKnown.size() == BITCOIN_TIMEDATA_MAX_SAMPLES) @@ -57,7 +56,7 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) // Add data static CMedianFilter<int64_t> vTimeOffsets(BITCOIN_TIMEDATA_MAX_SAMPLES, 0); vTimeOffsets.input(nOffsetSample); - LogPrint(BCLog::NET,"added time data, samples %d, offset %+d (%+d minutes)\n", vTimeOffsets.size(), nOffsetSample, nOffsetSample/60); + LogPrint(BCLog::NET, "added time data, samples %d, offset %+d (%+d minutes)\n", vTimeOffsets.size(), nOffsetSample, nOffsetSample / 60); // There is a known issue here (see issue #4521): // @@ -76,33 +75,27 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) // So we should hold off on fixing this and clean it up as part of // a timing cleanup that strengthens it in a number of other ways. // - if (vTimeOffsets.size() >= 5 && vTimeOffsets.size() % 2 == 1) - { + if (vTimeOffsets.size() >= 5 && vTimeOffsets.size() % 2 == 1) { int64_t nMedian = vTimeOffsets.median(); std::vector<int64_t> vSorted = vTimeOffsets.sorted(); // Only let other nodes change our time by so much - if (abs64(nMedian) <= std::max<int64_t>(0, gArgs.GetArg("-maxtimeadjustment", DEFAULT_MAX_TIME_ADJUSTMENT))) - { + if (abs64(nMedian) <= std::max<int64_t>(0, gArgs.GetArg("-maxtimeadjustment", DEFAULT_MAX_TIME_ADJUSTMENT))) { nTimeOffset = nMedian; - } - else - { + } else { nTimeOffset = 0; static bool fDone; - if (!fDone) - { + if (!fDone) { // If nobody has a time different than ours but within 5 minutes of ours, give a warning bool fMatch = false; - for (const int64_t nOffset : vSorted) - if (nOffset != 0 && abs64(nOffset) < 5 * 60) - fMatch = true; + for (const int64_t nOffset : vSorted) { + if (nOffset != 0 && abs64(nOffset) < 5 * 60) fMatch = true; + } - if (!fMatch) - { + if (!fMatch) { fDone = true; - std::string strMessage = strprintf(_("Please check that your computer's date and time are correct! If your clock is wrong, %s will not work properly.").translated, PACKAGE_NAME); - SetMiscWarning(strMessage); + bilingual_str strMessage = strprintf(_("Please check that your computer's date and time are correct! If your clock is wrong, %s will not work properly."), PACKAGE_NAME); + SetMiscWarning(strMessage.translated); uiInterface.ThreadSafeMessageBox(strMessage, "", CClientUIInterface::MSG_WARNING); } } @@ -113,8 +106,7 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) LogPrint(BCLog::NET, "%+d ", n); /* Continued */ } LogPrint(BCLog::NET, "| "); /* Continued */ - - LogPrint(BCLog::NET, "nTimeOffset = %+d (%+d minutes)\n", nTimeOffset, nTimeOffset/60); + LogPrint(BCLog::NET, "nTimeOffset = %+d (%+d minutes)\n", nTimeOffset, nTimeOffset / 60); } } } diff --git a/src/txdb.cpp b/src/txdb.cpp index 071aa1336b..6f652c1375 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -16,8 +16,6 @@ #include <stdint.h> -#include <boost/thread.hpp> - static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; @@ -36,19 +34,7 @@ struct CoinEntry { char key; explicit CoinEntry(const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)), key(DB_COIN) {} - template<typename Stream> - void Serialize(Stream &s) const { - s << key; - s << outpoint->hash; - s << VARINT(outpoint->n); - } - - template<typename Stream> - void Unserialize(Stream& s) { - s >> key; - s >> outpoint->hash; - s >> VARINT(outpoint->n); - } + SERIALIZE_METHODS(CoinEntry, obj) { READWRITE(obj.key, obj.outpoint->hash, VARINT(obj.outpoint->n)); } }; } @@ -254,7 +240,6 @@ bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, // Load m_block_index while (pcursor->Valid()) { - boost::this_thread::interruption_point(); if (ShutdownRequested()) return false; std::pair<char, uint256> key; if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) { @@ -366,7 +351,6 @@ bool CCoinsViewDB::Upgrade() { std::pair<unsigned char, uint256> key; std::pair<unsigned char, uint256> prev_key = {DB_COINS, uint256()}; while (pcursor->Valid()) { - boost::this_thread::interruption_point(); if (ShutdownRequested()) { break; } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index c5c0208d8f..7d8eb8a323 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -410,7 +410,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) // for any reason except being included in a block. Clients interested // in transactions included in blocks can subscribe to the BlockConnected // notification. - GetMainSignals().TransactionRemovedFromMempool(it->GetSharedTx()); + GetMainSignals().TransactionRemovedFromMempool(it->GetSharedTx(), reason); } const uint256 hash = it->GetTx().GetHash(); diff --git a/src/txmempool.h b/src/txmempool.h index 4bee78b8d6..583f7614b7 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -704,18 +704,27 @@ public: /** Adds a transaction to the unbroadcast set */ void AddUnbroadcastTx(const uint256& txid) { LOCK(cs); - m_unbroadcast_txids.insert(txid); + // Sanity Check: the transaction should also be in the mempool + if (exists(txid)) { + m_unbroadcast_txids.insert(txid); + } } /** Removes a transaction from the unbroadcast set */ void RemoveUnbroadcastTx(const uint256& txid, const bool unchecked = false); /** Returns transactions in unbroadcast set */ - const std::set<uint256> GetUnbroadcastTxs() const { + std::set<uint256> GetUnbroadcastTxs() const { LOCK(cs); return m_unbroadcast_txids; } + /** Returns whether a txid is in the unbroadcast set */ + bool IsUnbroadcastTx(const uint256& txid) const { + LOCK(cs); + return (m_unbroadcast_txids.count(txid) != 0); + } + private: /** UpdateForDescendants is used by UpdateTransactionsFromBlock to update * the descendants for a single transaction that has been added to the diff --git a/src/ui_interface.cpp b/src/ui_interface.cpp index 85bb746d19..15795bd67f 100644 --- a/src/ui_interface.cpp +++ b/src/ui_interface.cpp @@ -4,6 +4,8 @@ #include <ui_interface.h> +#include <util/translation.h> + #include <boost/signals2/last_value.hpp> #include <boost/signals2/signal.hpp> @@ -40,25 +42,24 @@ ADD_SIGNALS_IMPL_WRAPPER(NotifyBlockTip); ADD_SIGNALS_IMPL_WRAPPER(NotifyHeaderTip); ADD_SIGNALS_IMPL_WRAPPER(BannedListChanged); -bool CClientUIInterface::ThreadSafeMessageBox(const std::string& message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeMessageBox(message, caption, style); } -bool CClientUIInterface::ThreadSafeQuestion(const std::string& message, const std::string& non_interactive_message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeQuestion(message, non_interactive_message, caption, style); } +bool CClientUIInterface::ThreadSafeMessageBox(const bilingual_str& message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeMessageBox(message, caption, style); } +bool CClientUIInterface::ThreadSafeQuestion(const bilingual_str& message, const std::string& non_interactive_message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeQuestion(message, non_interactive_message, caption, style); } void CClientUIInterface::InitMessage(const std::string& message) { return g_ui_signals.InitMessage(message); } void CClientUIInterface::NotifyNumConnectionsChanged(int newNumConnections) { return g_ui_signals.NotifyNumConnectionsChanged(newNumConnections); } void CClientUIInterface::NotifyNetworkActiveChanged(bool networkActive) { return g_ui_signals.NotifyNetworkActiveChanged(networkActive); } void CClientUIInterface::NotifyAlertChanged() { return g_ui_signals.NotifyAlertChanged(); } void CClientUIInterface::ShowProgress(const std::string& title, int nProgress, bool resume_possible) { return g_ui_signals.ShowProgress(title, nProgress, resume_possible); } -void CClientUIInterface::NotifyBlockTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyBlockTip(b, i); } -void CClientUIInterface::NotifyHeaderTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyHeaderTip(b, i); } +void CClientUIInterface::NotifyBlockTip(SynchronizationState s, const CBlockIndex* i) { return g_ui_signals.NotifyBlockTip(s, i); } +void CClientUIInterface::NotifyHeaderTip(SynchronizationState s, const CBlockIndex* i) { return g_ui_signals.NotifyHeaderTip(s, i); } void CClientUIInterface::BannedListChanged() { return g_ui_signals.BannedListChanged(); } - -bool InitError(const std::string& str) +bool InitError(const bilingual_str& str) { uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_ERROR); return false; } -void InitWarning(const std::string& str) +void InitWarning(const bilingual_str& str) { uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_WARNING); } diff --git a/src/ui_interface.h b/src/ui_interface.h index b402177b85..d45811178f 100644 --- a/src/ui_interface.h +++ b/src/ui_interface.h @@ -11,6 +11,9 @@ #include <string> class CBlockIndex; +enum class SynchronizationState; +struct bilingual_str; + namespace boost { namespace signals2 { class connection; @@ -82,10 +85,10 @@ public: boost::signals2::connection signal_name##_connect(std::function<signal_name##Sig> fn); /** Show message box. */ - ADD_SIGNALS_DECL_WRAPPER(ThreadSafeMessageBox, bool, const std::string& message, const std::string& caption, unsigned int style); + ADD_SIGNALS_DECL_WRAPPER(ThreadSafeMessageBox, bool, const bilingual_str& message, const std::string& caption, unsigned int style); /** If possible, ask the user a question. If not, falls back to ThreadSafeMessageBox(noninteractive_message, caption, style) and returns false. */ - ADD_SIGNALS_DECL_WRAPPER(ThreadSafeQuestion, bool, const std::string& message, const std::string& noninteractive_message, const std::string& caption, unsigned int style); + ADD_SIGNALS_DECL_WRAPPER(ThreadSafeQuestion, bool, const bilingual_str& message, const std::string& noninteractive_message, const std::string& caption, unsigned int style); /** Progress message during initialization. */ ADD_SIGNALS_DECL_WRAPPER(InitMessage, void, const std::string& message); @@ -108,20 +111,20 @@ public: ADD_SIGNALS_DECL_WRAPPER(ShowProgress, void, const std::string& title, int nProgress, bool resume_possible); /** New block has been accepted */ - ADD_SIGNALS_DECL_WRAPPER(NotifyBlockTip, void, bool, const CBlockIndex*); + ADD_SIGNALS_DECL_WRAPPER(NotifyBlockTip, void, SynchronizationState, const CBlockIndex*); /** Best header has changed */ - ADD_SIGNALS_DECL_WRAPPER(NotifyHeaderTip, void, bool, const CBlockIndex*); + ADD_SIGNALS_DECL_WRAPPER(NotifyHeaderTip, void, SynchronizationState, const CBlockIndex*); /** Banlist did change. */ ADD_SIGNALS_DECL_WRAPPER(BannedListChanged, void, void); }; /** Show warning message **/ -void InitWarning(const std::string& str); +void InitWarning(const bilingual_str& str); /** Show error message **/ -bool InitError(const std::string& str); +bool InitError(const bilingual_str& str); extern CClientUIInterface uiInterface; diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index b4090482b9..bd77d74218 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -93,7 +93,8 @@ uint32_t Interpret(const std::vector<bool> &asmap, const std::vector<bool> &ip) jump = DecodeJump(pos, endpos); if (jump == INVALID) break; // Jump offset straddles EOF if (bits == 0) break; // No input bits left - if (jump >= endpos - pos) break; // Jumping past EOF + if (pos + jump < pos) break; // overflow + if (pos + jump >= endpos) break; // Jumping past EOF if (ip[ip.size() - bits]) { pos += jump; } @@ -155,7 +156,8 @@ bool SanityCheckASMap(const std::vector<bool>& asmap, int bits) } else if (opcode == Instruction::JUMP) { uint32_t jump = DecodeJump(pos, endpos); if (jump == INVALID) return false; // Jump offset straddles EOF - if (jump > endpos - pos) return false; // Jump out of range + if (pos + jump < pos) return false; // overflow + if (pos + jump > endpos) return false; // Jump out of range if (bits == 0) return false; // Consuming bits past the end of the input --bits; uint32_t jump_offset = pos - begin + jump; diff --git a/src/util/check.h b/src/util/check.h index d18887ae95..5c0f32cf51 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -5,6 +5,10 @@ #ifndef BITCOIN_UTIL_CHECK_H #define BITCOIN_UTIL_CHECK_H +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + #include <tinyformat.h> #include <stdexcept> diff --git a/src/util/ref.h b/src/util/ref.h new file mode 100644 index 0000000000..9685ea9fec --- /dev/null +++ b/src/util/ref.h @@ -0,0 +1,38 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_REF_H +#define BITCOIN_UTIL_REF_H + +#include <util/check.h> + +#include <typeindex> + +namespace util { + +/** + * Type-safe dynamic reference. + * + * This implements a small subset of the functionality in C++17's std::any + * class, and can be dropped when the project updates to C++17 + * (https://github.com/bitcoin/bitcoin/issues/16684) + */ +class Ref +{ +public: + Ref() = default; + template<typename T> Ref(T& value) { Set(value); } + template<typename T> T& Get() const { CHECK_NONFATAL(Has<T>()); return *static_cast<T*>(m_value); } + template<typename T> void Set(T& value) { m_value = &value; m_type = std::type_index(typeid(T)); } + template<typename T> bool Has() const { return m_value && m_type == std::type_index(typeid(T)); } + void Clear() { m_value = nullptr; m_type = std::type_index(typeid(void)); } + +private: + void* m_value = nullptr; + std::type_index m_type = std::type_index(typeid(void)); +}; + +} // namespace util + +#endif // BITCOIN_UTIL_REF_H diff --git a/src/util/string.h b/src/util/string.h index b8e2a06235..cdb41630c6 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -30,10 +30,11 @@ NODISCARD inline std::string TrimString(const std::string& str, const std::strin * @param separator The separator * @param unary_op Apply this operator to each item in the list */ -template <typename T, typename UnaryOp> -std::string Join(const std::vector<T>& list, const std::string& separator, UnaryOp unary_op) +template <typename T, typename BaseType, typename UnaryOp> +auto Join(const std::vector<T>& list, const BaseType& separator, UnaryOp unary_op) + -> decltype(unary_op(list.at(0))) { - std::string ret; + decltype(unary_op(list.at(0))) ret; for (size_t i = 0; i < list.size(); ++i) { if (i > 0) ret += separator; ret += unary_op(list.at(i)); @@ -41,9 +42,16 @@ std::string Join(const std::vector<T>& list, const std::string& separator, Unary return ret; } +template <typename T> +T Join(const std::vector<T>& list, const T& separator) +{ + return Join(list, separator, [](const T& i) { return i; }); +} + +// Explicit overload needed for c_str arguments, which would otherwise cause a substitution failure in the template above. inline std::string Join(const std::vector<std::string>& list, const std::string& separator) { - return Join(list, separator, [](const std::string& i) { return i; }); + return Join<std::string>(list, separator); } /** diff --git a/src/util/system.cpp b/src/util/system.cpp index 2013b416db..7e7ba840cd 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <sync.h> #include <util/system.h> #include <chainparamsbase.h> @@ -75,18 +76,18 @@ const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; ArgsManager gArgs; +/** Mutex to protect dir_locks. */ +static Mutex cs_dir_locks; /** A map that contains all the currently held directory locks. After * successful locking, these will be held here until the global destructor * cleans them up and thus automatically unlocks them, or ReleaseDirectoryLocks * is called. */ -static std::map<std::string, std::unique_ptr<fsbridge::FileLock>> dir_locks; -/** Mutex to protect dir_locks. */ -static std::mutex cs_dir_locks; +static std::map<std::string, std::unique_ptr<fsbridge::FileLock>> dir_locks GUARDED_BY(cs_dir_locks); bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only) { - std::lock_guard<std::mutex> ulock(cs_dir_locks); + LOCK(cs_dir_locks); fs::path pathLockFile = directory / lockfile_name; // If a lock for this directory already exists in the map, don't try to re-lock it @@ -110,13 +111,13 @@ bool LockDirectory(const fs::path& directory, const std::string lockfile_name, b void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name) { - std::lock_guard<std::mutex> lock(cs_dir_locks); + LOCK(cs_dir_locks); dir_locks.erase((directory / lockfile_name).string()); } void ReleaseDirectoryLocks() { - std::lock_guard<std::mutex> ulock(cs_dir_locks); + LOCK(cs_dir_locks); dir_locks.clear(); } @@ -1162,8 +1163,9 @@ void ScheduleBatchPriority() { #ifdef SCHED_BATCH const static sched_param param{}; - if (pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m) != 0) { - LogPrintf("Failed to pthread_setschedparam: %s\n", strerror(errno)); + const int rc = pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m); + if (rc != 0) { + LogPrintf("Failed to pthread_setschedparam: %s\n", strerror(rc)); } #endif } diff --git a/src/util/translation.h b/src/util/translation.h index b2d964c977..695d6dac96 100644 --- a/src/util/translation.h +++ b/src/util/translation.h @@ -16,21 +16,28 @@ struct bilingual_str { std::string original; std::string translated; + + bilingual_str& operator+=(const bilingual_str& rhs) + { + original += rhs.original; + translated += rhs.translated; + return *this; + } + + bool empty() const + { + return original.empty(); + } }; -inline bilingual_str operator+(const bilingual_str& lhs, const bilingual_str& rhs) +inline bilingual_str operator+(bilingual_str lhs, const bilingual_str& rhs) { - return bilingual_str{ - lhs.original + rhs.original, - lhs.translated + rhs.translated}; + lhs += rhs; + return lhs; } /** Mark a bilingual_str as untranslated */ -inline static bilingual_str Untranslated(std::string original) { return {original, original}; } -/** Unary operator to return the original */ -inline static std::string OpOriginal(const bilingual_str& b) { return b.original; } -/** Unary operator to return the translation */ -inline static std::string OpTranslated(const bilingual_str& b) { return b.translated; } +inline bilingual_str Untranslated(std::string original) { return {original, original}; } namespace tinyformat { template <typename... Args> diff --git a/src/validation.cpp b/src/validation.cpp index a924af90af..c7a1423536 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -50,7 +50,6 @@ #include <string> #include <boost/algorithm/string/replace.hpp> -#include <boost/thread.hpp> #if defined(NDEBUG) # error "Bitcoin cannot be compiled without assertions." @@ -71,12 +70,20 @@ static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB /** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB -/** Time to wait (in seconds) between writing blocks/block index to disk. */ -static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; -/** Time to wait (in seconds) between flushing chainstate to disk. */ -static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; -/** Maximum age of our tip in seconds for us to be considered current for fee estimation */ -static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; +/** Time to wait between writing blocks/block index to disk. */ +static constexpr std::chrono::hours DATABASE_WRITE_INTERVAL{1}; +/** Time to wait between flushing chainstate to disk. */ +static constexpr std::chrono::hours DATABASE_FLUSH_INTERVAL{24}; +/** Maximum age of our tip for us to be considered current for fee estimation */ +static constexpr std::chrono::hours MAX_FEE_ESTIMATION_TIP_AGE{3}; +const std::vector<std::string> CHECKLEVEL_DOC { + "level 0 reads the blocks from disk", + "level 1 verifies block validity", + "level 2 verifies undo data", + "level 3 checks disconnection of tip blocks", + "level 4 tries to reconnect the blocks", + "each level includes the checks of the previous levels", +}; bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { // First sort by most total work, ... @@ -196,8 +203,8 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc std::unique_ptr<CBlockTreeDB> pblocktree; // See definition for documentation -static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); -static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight); +static void FindFilesToPruneManual(ChainstateManager& chainman, std::set<int>& setFilesToPrune, int nManualPruneHeight); +static void FindFilesToPrune(ChainstateManager& chainman, std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight); bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); static FlatFileSeq BlockFileSeq(); @@ -347,7 +354,7 @@ static bool IsCurrentForFeeEstimation() EXCLUSIVE_LOCKS_REQUIRED(cs_main) AssertLockHeld(cs_main); if (::ChainstateActive().IsInitialBlockDownload()) return false; - if (::ChainActive().Tip()->GetBlockTime() < (GetTime() - MAX_FEE_ESTIMATION_TIP_AGE)) + if (::ChainActive().Tip()->GetBlockTime() < count_seconds(GetTime<std::chrono::seconds>() - MAX_FEE_ESTIMATION_TIP_AGE)) return false; if (::ChainActive().Height() < pindexBestHeader->nHeight - 1) return false; @@ -1481,14 +1488,21 @@ int GetSpendHeight(const CCoinsViewCache& inputs) } -static CuckooCache::cache<uint256, SignatureCacheHasher> scriptExecutionCache; -static uint256 scriptExecutionCacheNonce(GetRandHash()); +static CuckooCache::cache<uint256, SignatureCacheHasher> g_scriptExecutionCache; +static CSHA256 g_scriptExecutionCacheHasher; void InitScriptExecutionCache() { + // Setup the salted hasher + uint256 nonce = GetRandHash(); + // We want the nonce to be 64 bytes long to force the hasher to process + // this chunk, which makes later hash computations more efficient. We + // just write our 32-byte entropy twice to fill the 64 bytes. + g_scriptExecutionCacheHasher.Write(nonce.begin(), 32); + g_scriptExecutionCacheHasher.Write(nonce.begin(), 32); // nMaxCacheSize is unsigned. If -maxsigcachesize is set to zero, // setup_bytes creates the minimum possible cache (2 elements). size_t nMaxCacheSize = std::min(std::max((int64_t)0, gArgs.GetArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) / 2), MAX_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20); - size_t nElems = scriptExecutionCache.setup_bytes(nMaxCacheSize); + size_t nElems = g_scriptExecutionCache.setup_bytes(nMaxCacheSize); LogPrintf("Using %zu MiB out of %zu/2 requested for script execution cache, able to store %zu elements\n", (nElems*sizeof(uint256)) >>20, (nMaxCacheSize*2)>>20, nElems); } @@ -1526,12 +1540,10 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const C // properly commits to the scriptPubKey in the inputs view of that // transaction). uint256 hashCacheEntry; - // We only use the first 19 bytes of nonce to avoid a second SHA - // round - giving us 19 + 32 + 4 = 55 bytes (+ 8 + 1 = 64) - static_assert(55 - sizeof(flags) - 32 >= 128/8, "Want at least 128 bits of nonce for script execution cache"); - CSHA256().Write(scriptExecutionCacheNonce.begin(), 55 - sizeof(flags) - 32).Write(tx.GetWitnessHash().begin(), 32).Write((unsigned char*)&flags, sizeof(flags)).Finalize(hashCacheEntry.begin()); + CSHA256 hasher = g_scriptExecutionCacheHasher; + hasher.Write(tx.GetWitnessHash().begin(), 32).Write((unsigned char*)&flags, sizeof(flags)).Finalize(hashCacheEntry.begin()); AssertLockHeld(cs_main); //TODO: Remove this requirement by making CuckooCache not require external locks - if (scriptExecutionCache.contains(hashCacheEntry, !cacheFullScriptStore)) { + if (g_scriptExecutionCache.contains(hashCacheEntry, !cacheFullScriptStore)) { return true; } @@ -1586,7 +1598,7 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const C if (cacheFullScriptStore && !pvChecks) { // We executed all of the provided scripts, and were told to // cache the result. Do so now. - scriptExecutionCache.insert(hashCacheEntry); + g_scriptExecutionCache.insert(hashCacheEntry); } return true; @@ -1651,14 +1663,15 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) } /** Abort with a message */ +// TODO: AbortNode() should take bilingual_str userMessage parameter. static bool AbortNode(const std::string& strMessage, const std::string& userMessage = "", unsigned int prefix = 0) { SetMiscWarning(strMessage); LogPrintf("*** %s\n", strMessage); if (!userMessage.empty()) { - uiInterface.ThreadSafeMessageBox(userMessage, "", CClientUIInterface::MSG_ERROR | prefix); + uiInterface.ThreadSafeMessageBox(Untranslated(userMessage), "", CClientUIInterface::MSG_ERROR | prefix); } else { - uiInterface.ThreadSafeMessageBox(_("Error: A fatal internal error occurred, see debug.log for details").translated, "", CClientUIInterface::MSG_ERROR | CClientUIInterface::MSG_NOPREFIX); + uiInterface.ThreadSafeMessageBox(_("Error: A fatal internal error occurred, see debug.log for details"), "", CClientUIInterface::MSG_ERROR | CClientUIInterface::MSG_NOPREFIX); } StartShutdown(); return false; @@ -1764,19 +1777,24 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } -void static FlushBlockFile(bool fFinalize = false) +static void FlushUndoFile(int block_file, bool finalize = false) { - LOCK(cs_LastBlockFile); + FlatFilePos undo_pos_old(block_file, vinfoBlockFile[block_file].nUndoSize); + if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { + AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error."); + } +} +static void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false) +{ + LOCK(cs_LastBlockFile); FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); - FlatFilePos undo_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nUndoSize); - - bool status = true; - status &= BlockFileSeq().Flush(block_pos_old, fFinalize); - status &= UndoFileSeq().Flush(undo_pos_old, fFinalize); - if (!status) { + if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); } + // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, + // e.g. during IBD or a sync after a node going offline + if (!fFinalize || finalize_undo) FlushUndoFile(nLastBlockFile, finalize_undo); } static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); @@ -1790,6 +1808,14 @@ static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationSt return error("ConnectBlock(): FindUndoPos failed"); if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) return AbortNode(state, "Failed to write undo data"); + // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) + // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height + // in the block file info as below; note that this does not catch the case where the undo writes are keeping up + // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in + // the FindBlockPos function + if (_pos.nFile < nLastBlockFile && static_cast<uint32_t>(pindex->nHeight) == vinfoBlockFile[_pos.nFile].nHeightLast) { + FlushUndoFile(_pos.nFile, true); + } // update nUndoPos in block index pindex->nUndoPos = _pos.nPos; @@ -2263,8 +2289,8 @@ bool CChainState::FlushStateToDisk( { LOCK(cs_main); assert(this->CanFlushToDisk()); - static int64_t nLastWrite = 0; - static int64_t nLastFlush = 0; + static std::chrono::microseconds nLastWrite{0}; + static std::chrono::microseconds nLastFlush{0}; std::set<int> setFilesToPrune; bool full_flush_completed = false; @@ -2281,11 +2307,11 @@ bool CChainState::FlushStateToDisk( if (nManualPruneHeight > 0) { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune (manual)", BCLog::BENCH); - FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight); + FindFilesToPruneManual(g_chainman, setFilesToPrune, nManualPruneHeight); } else { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCH); - FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight()); + FindFilesToPrune(g_chainman, setFilesToPrune, chainparams.PruneAfterHeight()); fCheckForPruning = false; } if (!setFilesToPrune.empty()) { @@ -2296,12 +2322,12 @@ bool CChainState::FlushStateToDisk( } } } - int64_t nNow = GetTimeMicros(); + const auto nNow = GetTime<std::chrono::microseconds>(); // Avoid writing/flushing immediately after startup. - if (nLastWrite == 0) { + if (nLastWrite.count() == 0) { nLastWrite = nNow; } - if (nLastFlush == 0) { + if (nLastFlush.count() == 0) { nLastFlush = nNow; } // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). @@ -2309,9 +2335,9 @@ bool CChainState::FlushStateToDisk( // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FlushStateMode::IF_NEEDED && cache_state >= CoinsCacheSizeState::CRITICAL; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. - bool fPeriodicWrite = mode == FlushStateMode::PERIODIC && nNow > nLastWrite + (int64_t)DATABASE_WRITE_INTERVAL * 1000000; + bool fPeriodicWrite = mode == FlushStateMode::PERIODIC && nNow > nLastWrite + DATABASE_WRITE_INTERVAL; // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. - bool fPeriodicFlush = mode == FlushStateMode::PERIODIC && nNow > nLastFlush + (int64_t)DATABASE_FLUSH_INTERVAL * 1000000; + bool fPeriodicFlush = mode == FlushStateMode::PERIODIC && nNow > nLastFlush + DATABASE_FLUSH_INTERVAL; // Combine all conditions that result in a full cache flush. fDoFullFlush = (mode == FlushStateMode::ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicFlush || fFlushForPrune; // Write blocks and block index to disk. @@ -2799,6 +2825,13 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai return true; } +static SynchronizationState GetSynchronizationState(bool init) +{ + if (!init) return SynchronizationState::POST_INIT; + if (::fReindex) return SynchronizationState::INIT_REINDEX; + return SynchronizationState::INIT_DOWNLOAD; +} + static bool NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { bool fNotify = false; bool fInitialBlockDownload = false; @@ -2816,7 +2849,7 @@ static bool NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { } // Send block tip changed notifications without cs_main if (fNotify) { - uiInterface.NotifyHeaderTip(fInitialBlockDownload, pindexHeader); + uiInterface.NotifyHeaderTip(GetSynchronizationState(fInitialBlockDownload), pindexHeader); } return fNotify; } @@ -2846,8 +2879,6 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar CBlockIndex *pindexNewTip = nullptr; int nStopAtHeight = gArgs.GetArg("-stopatheight", DEFAULT_STOPATHEIGHT); do { - boost::this_thread::interruption_point(); - // Block until the validation queue drains. This should largely // never happen in normal operation, however may happen during // reindex, causing memory blowup if we run too far ahead. @@ -2905,7 +2936,7 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); // Always notify the UI if a new block tip was connected - uiInterface.NotifyBlockTip(fInitialDownload, pindexNewTip); + uiInterface.NotifyBlockTip(GetSynchronizationState(fInitialDownload), pindexNewTip); } } // When we reach this point, we switched to a new tip (stored in pindexNewTip). @@ -2916,8 +2947,7 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar // never shutdown before connecting the genesis block during LoadChainTip(). Previously this // caused an assert() failure during shutdown in such cases as the UTXO DB flushing checks // that the best block hash is non-null. - if (ShutdownRequested()) - break; + if (ShutdownRequested()) break; } while (pindexNewTip != pindexMostWork); CheckBlockIndex(chainparams.GetConsensus()); @@ -3096,7 +3126,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, const CChainParam // Only notify about a new block tip if the active chain was modified. if (pindex_was_in_chain) { - uiInterface.NotifyBlockTip(IsInitialBlockDownload(), to_mark_failed->pprev); + uiInterface.NotifyBlockTip(GetSynchronizationState(IsInitialBlockDownload()), to_mark_failed->pprev); } return true; } @@ -3235,8 +3265,13 @@ static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int n vinfoBlockFile.resize(nFile + 1); } + bool finalize_undo = false; if (!fKnown) { while (vinfoBlockFile[nFile].nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { + // when the undo file is keeping up with the block file, we want to flush it explicitly + // when it is lagging behind (more blocks arrive than are being connected), we let the + // undo block write case handle it + finalize_undo = (vinfoBlockFile[nFile].nHeightLast == (unsigned int)ChainActive().Tip()->nHeight); nFile++; if (vinfoBlockFile.size() <= nFile) { vinfoBlockFile.resize(nFile + 1); @@ -3250,7 +3285,7 @@ static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int n if (!fKnown) { LogPrintf("Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); } - FlushBlockFile(!fKnown); + FlushBlockFile(!fKnown, finalize_undo); nLastBlockFile = nFile; } @@ -3683,13 +3718,14 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS } // Exposed wrapper for AcceptBlockHeader -bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) +bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) { + AssertLockNotHeld(cs_main); { LOCK(cs_main); for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast - bool accepted = g_chainman.m_blockman.AcceptBlockHeader( + bool accepted = m_blockman.AcceptBlockHeader( header, state, chainparams, &pindex); ::ChainstateActive().CheckBlockIndex(chainparams.GetConsensus()); @@ -3811,7 +3847,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, Block return true; } -bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool *fNewBlock) +bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock) { AssertLockNotHeld(cs_main); @@ -3887,12 +3923,12 @@ uint64_t CalculateCurrentUsage() return retval; } -/* Prune a block file (modify associated database entries)*/ -void PruneOneBlockFile(const int fileNumber) +void ChainstateManager::PruneOneBlockFile(const int fileNumber) { + AssertLockHeld(cs_main); LOCK(cs_LastBlockFile); - for (const auto& entry : g_chainman.BlockIndex()) { + for (const auto& entry : m_blockman.m_block_index) { CBlockIndex* pindex = entry.second; if (pindex->nFile == fileNumber) { pindex->nStatus &= ~BLOCK_HAVE_DATA; @@ -3906,12 +3942,12 @@ void PruneOneBlockFile(const int fileNumber) // to be downloaded again in order to consider its chain, at which // point it would be considered as a candidate for // m_blocks_unlinked or setBlockIndexCandidates. - auto range = g_chainman.m_blockman.m_blocks_unlinked.equal_range(pindex->pprev); + auto range = m_blockman.m_blocks_unlinked.equal_range(pindex->pprev); while (range.first != range.second) { std::multimap<CBlockIndex *, CBlockIndex *>::iterator _it = range.first; range.first++; if (_it->second == pindex) { - g_chainman.m_blockman.m_blocks_unlinked.erase(_it); + m_blockman.m_blocks_unlinked.erase(_it); } } } @@ -3933,7 +3969,7 @@ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) } /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ -static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight) +static void FindFilesToPruneManual(ChainstateManager& chainman, std::set<int>& setFilesToPrune, int nManualPruneHeight) { assert(fPruneMode && nManualPruneHeight > 0); @@ -3947,7 +3983,7 @@ static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPr for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) continue; - PruneOneBlockFile(fileNumber); + chainman.PruneOneBlockFile(fileNumber); setFilesToPrune.insert(fileNumber); count++; } @@ -3980,7 +4016,7 @@ void PruneBlockFilesManual(int nManualPruneHeight) * * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned */ -static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight) +static void FindFilesToPrune(ChainstateManager& chainman, std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight) { LOCK2(cs_main, cs_LastBlockFile); if (::ChainActive().Tip() == nullptr || nPruneTarget == 0) { @@ -4022,7 +4058,7 @@ static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfte if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) continue; - PruneOneBlockFile(fileNumber); + chainman.PruneOneBlockFile(fileNumber); // Queue up the files for removal setFilesToPrune.insert(fileNumber); nCurrentUsage -= nBytesToPrune; @@ -4146,9 +4182,9 @@ void BlockManager::Unload() { m_block_index.clear(); } -bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool static LoadBlockIndexDB(ChainstateManager& chainman, const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - if (!g_chainman.m_blockman.LoadBlockIndex( + if (!chainman.m_blockman.LoadBlockIndex( chainparams.GetConsensus(), *pblocktree, ::ChainstateActive().setBlockIndexCandidates)) { return false; @@ -4174,8 +4210,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE // Check presence of blk files LogPrintf("Checking all blk files are present...\n"); std::set<int> setBlkDataFiles; - for (const std::pair<const uint256, CBlockIndex*>& item : g_chainman.BlockIndex()) - { + for (const std::pair<const uint256, CBlockIndex*>& item : chainman.BlockIndex()) { CBlockIndex* pindex = item.second; if (pindex->nStatus & BLOCK_HAVE_DATA) { setBlkDataFiles.insert(pindex->nFile); @@ -4259,7 +4294,6 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, int reportDone = 0; LogPrintf("[0%%]..."); /* Continued */ for (pindex = ::ChainActive().Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { - boost::this_thread::interruption_point(); const int percentageDone = std::max(1, std::min(99, (int)(((double)(::ChainActive().Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); if (reportDone < percentageDone/10) { // report every 10% step @@ -4305,8 +4339,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, nGoodTransactions += block.vtx.size(); } } - if (ShutdownRequested()) - return true; + if (ShutdownRequested()) return true; } if (pindexFailure) return error("VerifyDB(): *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", ::ChainActive().Height() - pindexFailure->nHeight + 1, nGoodTransactions); @@ -4317,7 +4350,6 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, // check level 4: try reconnecting blocks if (nCheckLevel >= 4) { while (pindex != ::ChainActive().Tip()) { - boost::this_thread::interruption_point(); const int percentageDone = std::max(1, std::min(99, 100 - (int)(((double)(::ChainActive().Height() - pindex->nHeight)) / (double)nCheckDepth * 50))); if (reportDone < percentageDone/10) { // report every 10% step @@ -4331,6 +4363,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); if (!::ChainstateActive().ConnectBlock(block, state, pindex, coins, chainparams)) return error("VerifyDB(): *** found unconnectable block at %d, hash=%s (%s)", pindex->nHeight, pindex->GetBlockHash().ToString(), state.ToString()); + if (ShutdownRequested()) return true; } } @@ -4592,14 +4625,15 @@ void UnloadBlockIndex() fHavePruned = false; } -bool LoadBlockIndex(const CChainParams& chainparams) +bool ChainstateManager::LoadBlockIndex(const CChainParams& chainparams) { + AssertLockHeld(cs_main); // Load block index from databases bool needs_init = fReindex; if (!fReindex) { - bool ret = LoadBlockIndexDB(chainparams); + bool ret = LoadBlockIndexDB(*this, chainparams); if (!ret) return false; - needs_init = g_chainman.m_blockman.m_block_index.empty(); + needs_init = m_blockman.m_block_index.empty(); } if (needs_init) { @@ -5058,12 +5092,18 @@ bool LoadMempool(CTxMemPool& pool) pool.PrioritiseTransaction(i.first, i.second); } - std::set<uint256> unbroadcast_txids; - file >> unbroadcast_txids; - unbroadcast = unbroadcast_txids.size(); + // TODO: remove this try except in v0.22 + try { + std::set<uint256> unbroadcast_txids; + file >> unbroadcast_txids; + unbroadcast = unbroadcast_txids.size(); - for (const auto& txid : unbroadcast_txids) { + for (const auto& txid : unbroadcast_txids) { pool.AddUnbroadcastTx(txid); + } + } catch (const std::exception&) { + // mempool.dat files created prior to v0.21 will not have an + // unbroadcast set. No need to log a failure if parsing fails here. } } catch (const std::exception& e) { diff --git a/src/validation.h b/src/validation.h index c4a5cc4593..e403bcb51a 100644 --- a/src/validation.h +++ b/src/validation.h @@ -29,6 +29,7 @@ #include <memory> #include <set> #include <stdint.h> +#include <string> #include <utility> #include <vector> @@ -43,6 +44,7 @@ class CConnman; class CScriptCheck; class CBlockPolicyEstimator; class CTxMemPool; +class ChainstateManager; class TxValidationState; struct ChainTxData; @@ -103,6 +105,13 @@ struct BlockHasher size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } }; +/** Current sync state passed to tip changed callbacks. */ +enum class SynchronizationState { + INIT_REINDEX, + INIT_DOWNLOAD, + POST_INIT +}; + extern RecursiveMutex cs_main; extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; @@ -141,41 +150,8 @@ extern bool fHavePruned; extern bool fPruneMode; /** Number of MiB of block files that we're trying to stay below. */ extern uint64_t nPruneTarget; - -/** - * Process an incoming block. This only returns after the best known valid - * block is made active. Note that it does not, however, guarantee that the - * specific block passed to it has been checked for validity! - * - * If you want to *possibly* get feedback on whether pblock is valid, you must - * install a CValidationInterface (see validationinterface.h) - this will have - * its BlockChecked method called whenever *any* block completes validation. - * - * Note that we guarantee that either the proof-of-work is valid on pblock, or - * (and possibly also) BlockChecked will have been called. - * - * May not be called in a - * validationinterface callback. - * - * @param[in] pblock The block we want to process. - * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and whitelisted peers. - * @param[out] fNewBlock A boolean which is set to indicate if the block was first received via this call - * @returns If the block was processed, independently of block validity - */ -bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock) LOCKS_EXCLUDED(cs_main); - -/** - * Process incoming block headers. - * - * May not be called in a - * validationinterface callback. - * - * @param[in] block The block headers themselves - * @param[out] state This may be set to an Error state if any error occurred processing them - * @param[in] chainparams The params for the chain we want to connect to - * @param[out] ppindex If set, the pointer will be set to point to the last new block index object for the given headers - */ -bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr) LOCKS_EXCLUDED(cs_main); +/** Documentation for argument 'checklevel'. */ +extern const std::vector<std::string> CHECKLEVEL_DOC; /** Open a block file (blk?????.dat) */ FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); @@ -185,9 +161,6 @@ fs::path GetBlockPosFilename(const FlatFilePos &pos); void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp = nullptr); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); -/** Load the block tree and coins database from disk, - * initializing state if we're running with -reindex. */ -bool LoadBlockIndex(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Unload database information */ void UnloadBlockIndex(); /** Run an instance of the script checking thread */ @@ -210,11 +183,6 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pin uint64_t CalculateCurrentUsage(); /** - * Mark one block file as pruned. - */ -void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - -/** * Actually unlink the specified files */ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); @@ -486,9 +454,6 @@ enum class CoinsCacheSizeState OK = 0 }; -// Defined below, but needed for `friend` usage in CChainState. -class ChainstateManager; - /** * CChainState stores and provides an API to update our local knowledge of the * current best chain. @@ -863,6 +828,47 @@ public: CChain& ValidatedChain() const { return ValidatedChainstate().m_chain; } CBlockIndex* ValidatedTip() const { return ValidatedChain().Tip(); } + /** + * Process an incoming block. This only returns after the best known valid + * block is made active. Note that it does not, however, guarantee that the + * specific block passed to it has been checked for validity! + * + * If you want to *possibly* get feedback on whether pblock is valid, you must + * install a CValidationInterface (see validationinterface.h) - this will have + * its BlockChecked method called whenever *any* block completes validation. + * + * Note that we guarantee that either the proof-of-work is valid on pblock, or + * (and possibly also) BlockChecked will have been called. + * + * May not be called in a + * validationinterface callback. + * + * @param[in] pblock The block we want to process. + * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and whitelisted peers. + * @param[out] fNewBlock A boolean which is set to indicate if the block was first received via this call + * @returns If the block was processed, independently of block validity + */ + bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock) LOCKS_EXCLUDED(cs_main); + + /** + * Process incoming block headers. + * + * May not be called in a + * validationinterface callback. + * + * @param[in] block The block headers themselves + * @param[out] state This may be set to an Error state if any error occurred processing them + * @param[in] chainparams The params for the chain we want to connect to + * @param[out] ppindex If set, the pointer will be set to point to the last new block index object for the given headers + */ + bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr) LOCKS_EXCLUDED(cs_main); + + //! Mark one block file as pruned (modify associated database entries) + void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! Load the block tree and coins database from disk, initializing state if we're running with -reindex + bool LoadBlockIndex(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + //! Unload block index and chain data before shutdown. void Unload() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); @@ -870,6 +876,7 @@ public: void Reset(); }; +/** DEPRECATED! Please use node.chainman instead. May only be used in validation.cpp internally */ extern ChainstateManager g_chainman GUARDED_BY(::cs_main); /** @returns the most-work valid chainstate. */ diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 11000774c0..3dfbcc581c 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -89,22 +89,26 @@ public: static CMainSignals g_signals; -void CMainSignals::RegisterBackgroundSignalScheduler(CScheduler& scheduler) { +void CMainSignals::RegisterBackgroundSignalScheduler(CScheduler& scheduler) +{ assert(!m_internals); m_internals.reset(new MainSignalsInstance(&scheduler)); } -void CMainSignals::UnregisterBackgroundSignalScheduler() { +void CMainSignals::UnregisterBackgroundSignalScheduler() +{ m_internals.reset(nullptr); } -void CMainSignals::FlushBackgroundCallbacks() { +void CMainSignals::FlushBackgroundCallbacks() +{ if (m_internals) { m_internals->m_schedulerClient.EmptyQueue(); } } -size_t CMainSignals::CallbacksPending() { +size_t CMainSignals::CallbacksPending() +{ if (!m_internals) return 0; return m_internals->m_schedulerClient.CallbacksPending(); } @@ -114,10 +118,11 @@ CMainSignals& GetMainSignals() return g_signals; } -void RegisterSharedValidationInterface(std::shared_ptr<CValidationInterface> pwalletIn) { - // Each connection captures pwalletIn to ensure that each callback is - // executed before pwalletIn is destroyed. For more details see #18338. - g_signals.m_internals->Register(std::move(pwalletIn)); +void RegisterSharedValidationInterface(std::shared_ptr<CValidationInterface> callbacks) +{ + // Each connection captures the shared_ptr to ensure that each callback is + // executed before the subscriber is destroyed. For more details see #18338. + g_signals.m_internals->Register(std::move(callbacks)); } void RegisterValidationInterface(CValidationInterface* callbacks) @@ -132,24 +137,28 @@ void UnregisterSharedValidationInterface(std::shared_ptr<CValidationInterface> c UnregisterValidationInterface(callbacks.get()); } -void UnregisterValidationInterface(CValidationInterface* pwalletIn) { +void UnregisterValidationInterface(CValidationInterface* callbacks) +{ if (g_signals.m_internals) { - g_signals.m_internals->Unregister(pwalletIn); + g_signals.m_internals->Unregister(callbacks); } } -void UnregisterAllValidationInterfaces() { +void UnregisterAllValidationInterfaces() +{ if (!g_signals.m_internals) { return; } g_signals.m_internals->Clear(); } -void CallFunctionInValidationInterfaceQueue(std::function<void ()> func) { +void CallFunctionInValidationInterfaceQueue(std::function<void()> func) +{ g_signals.m_internals->m_schedulerClient.AddToProcessQueue(std::move(func)); } -void SyncWithValidationInterfaceQueue() { +void SyncWithValidationInterfaceQueue() +{ AssertLockNotHeld(cs_main); // Block until the validation queue drains std::promise<void> promise; @@ -190,22 +199,22 @@ void CMainSignals::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockInd fInitialDownload); } -void CMainSignals::TransactionAddedToMempool(const CTransactionRef &ptx) { - auto event = [ptx, this] { - m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.TransactionAddedToMempool(ptx); }); +void CMainSignals::TransactionAddedToMempool(const CTransactionRef& tx) { + auto event = [tx, this] { + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.TransactionAddedToMempool(tx); }); }; ENQUEUE_AND_LOG_EVENT(event, "%s: txid=%s wtxid=%s", __func__, - ptx->GetHash().ToString(), - ptx->GetWitnessHash().ToString()); + tx->GetHash().ToString(), + tx->GetWitnessHash().ToString()); } -void CMainSignals::TransactionRemovedFromMempool(const CTransactionRef &ptx) { - auto event = [ptx, this] { - m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.TransactionRemovedFromMempool(ptx); }); +void CMainSignals::TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) { + auto event = [tx, reason, this] { + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.TransactionRemovedFromMempool(tx, reason); }); }; ENQUEUE_AND_LOG_EVENT(event, "%s: txid=%s wtxid=%s", __func__, - ptx->GetHash().ToString(), - ptx->GetWitnessHash().ToString()); + tx->GetHash().ToString(), + tx->GetWitnessHash().ToString()); } void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex) { diff --git a/src/validationinterface.h b/src/validationinterface.h index cb0204a555..e96f2883fc 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -21,21 +21,22 @@ class CConnman; class CValidationInterface; class uint256; class CScheduler; +enum class MemPoolRemovalReason; -// These functions dispatch to one or all registered wallets - -/** Register a wallet to receive updates from core */ -void RegisterValidationInterface(CValidationInterface* pwalletIn); -/** Unregister a wallet from core */ -void UnregisterValidationInterface(CValidationInterface* pwalletIn); -/** Unregister all wallets from core */ +/** Register subscriber */ +void RegisterValidationInterface(CValidationInterface* callbacks); +/** Unregister subscriber. DEPRECATED. This is not safe to use when the RPC server or main message handler thread is running. */ +void UnregisterValidationInterface(CValidationInterface* callbacks); +/** Unregister all subscribers */ void UnregisterAllValidationInterfaces(); // Alternate registration functions that release a shared_ptr after the last // notification is sent. These are useful for race-free cleanup, since // unregistration is nonblocking and can return before the last notification is // processed. +/** Register subscriber */ void RegisterSharedValidationInterface(std::shared_ptr<CValidationInterface> callbacks); +/** Unregister subscriber */ void UnregisterSharedValidationInterface(std::shared_ptr<CValidationInterface> callbacks); /** @@ -96,7 +97,7 @@ protected: * * Called on a background thread. */ - virtual void TransactionAddedToMempool(const CTransactionRef &ptxn) {} + virtual void TransactionAddedToMempool(const CTransactionRef& tx) {} /** * Notifies listeners of a transaction leaving mempool. * @@ -129,7 +130,7 @@ protected: * * Called on a background thread. */ - virtual void TransactionRemovedFromMempool(const CTransactionRef &ptx) {} + virtual void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) {} /** * Notifies listeners of a block being connected. * Provides a vector of transactions evicted from the mempool as a result. @@ -196,8 +197,8 @@ public: void UpdatedBlockTip(const CBlockIndex *, const CBlockIndex *, bool fInitialDownload); - void TransactionAddedToMempool(const CTransactionRef &); - void TransactionRemovedFromMempool(const CTransactionRef &); + void TransactionAddedToMempool(const CTransactionRef&); + void TransactionRemovedFromMempool(const CTransactionRef&, MemPoolRemovalReason); void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex); void BlockDisconnected(const std::shared_ptr<const CBlock> &, const CBlockIndex* pindex); void ChainStateFlushed(const CBlockLocator &); diff --git a/src/wallet/context.cpp b/src/wallet/context.cpp new file mode 100644 index 0000000000..09b2f30467 --- /dev/null +++ b/src/wallet/context.cpp @@ -0,0 +1,8 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <wallet/context.h> + +WalletContext::WalletContext() {} +WalletContext::~WalletContext() {} diff --git a/src/wallet/context.h b/src/wallet/context.h new file mode 100644 index 0000000000..3c8fdd1c59 --- /dev/null +++ b/src/wallet/context.h @@ -0,0 +1,32 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_CONTEXT_H +#define BITCOIN_WALLET_CONTEXT_H + +namespace interfaces { +class Chain; +} // namespace interfaces + +//! WalletContext struct containing references to state shared between CWallet +//! instances, like the reference to the chain interface, and the list of opened +//! wallets. +//! +//! Future shared state can be added here as an alternative to adding global +//! variables. +//! +//! The struct isn't intended to have any member functions. It should just be a +//! collection of state pointers that doesn't pull in dependencies or implement +//! behavior. +struct WalletContext { + interfaces::Chain* chain{nullptr}; + + //! Declare default constructor and destructor that are not inline, so code + //! instantiating the WalletContext struct doesn't need to #include class + //! definitions for smart pointer and container members. + WalletContext(); + ~WalletContext(); +}; + +#endif // BITCOIN_WALLET_CONTEXT_H diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index f59c63260e..f2df786e2e 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -43,15 +43,9 @@ public: //! such as the various parameters to scrypt std::vector<unsigned char> vchOtherDerivationParameters; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(vchCryptedKey); - READWRITE(vchSalt); - READWRITE(nDerivationMethod); - READWRITE(nDeriveIterations); - READWRITE(vchOtherDerivationParameters); + SERIALIZE_METHODS(CMasterKey, obj) + { + READWRITE(obj.vchCryptedKey, obj.vchSalt, obj.nDerivationMethod, obj.nDeriveIterations, obj.vchOtherDerivationParameters); } CMasterKey() diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 3f7c2d09cc..d90e8e6433 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -14,8 +14,6 @@ #include <sys/stat.h> #endif -#include <boost/thread.hpp> - namespace { //! Make sure database has a unique fileid within the environment. If it @@ -268,21 +266,14 @@ BerkeleyEnvironment::BerkeleyEnvironment() fMockDb = true; } -BerkeleyEnvironment::VerifyResult BerkeleyEnvironment::Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename) +bool BerkeleyEnvironment::Verify(const std::string& strFile) { LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); Db db(dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); - if (result == 0) - return VerifyResult::VERIFY_OK; - else if (recoverFunc == nullptr) - return VerifyResult::RECOVER_FAIL; - - // Try to recover: - bool fRecovered = (*recoverFunc)(fs::path(strPath) / strFile, out_backup_filename); - return (fRecovered ? VerifyResult::RECOVER_OK : VerifyResult::RECOVER_FAIL); + return result == 0; } BerkeleyBatch::SafeDbt::SafeDbt() @@ -324,82 +315,13 @@ BerkeleyBatch::SafeDbt::operator Dbt*() return &m_dbt; } -bool BerkeleyBatch::Recover(const fs::path& file_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& newFilename) -{ - std::string filename; - std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename); - - // Recovery procedure: - // move wallet file to walletfilename.timestamp.bak - // Call Salvage with fAggressive=true to - // get as much data as possible. - // Rewrite salvaged data to fresh wallet file - // Set -rescan so any missing transactions will be - // found. - int64_t now = GetTime(); - newFilename = strprintf("%s.%d.bak", filename, now); - - int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr, - newFilename.c_str(), DB_AUTO_COMMIT); - if (result == 0) - LogPrintf("Renamed %s to %s\n", filename, newFilename); - else - { - LogPrintf("Failed to rename %s to %s\n", filename, newFilename); - return false; - } - - std::vector<BerkeleyEnvironment::KeyValPair> salvagedData; - bool fSuccess = env->Salvage(newFilename, true, salvagedData); - if (salvagedData.empty()) - { - LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); - return false; - } - LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); - - std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0); - int ret = pdbCopy->open(nullptr, // Txn pointer - filename.c_str(), // Filename - "main", // Logical db name - DB_BTREE, // Database type - DB_CREATE, // Flags - 0); - if (ret > 0) { - LogPrintf("Cannot create database file %s\n", filename); - pdbCopy->close(0); - return false; - } - - DbTxn* ptxn = env->TxnBegin(); - for (BerkeleyEnvironment::KeyValPair& row : salvagedData) - { - if (recoverKVcallback) - { - CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); - CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); - if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue)) - continue; - } - Dbt datKey(&row.first[0], row.first.size()); - Dbt datValue(&row.second[0], row.second.size()); - int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); - if (ret2 > 0) - fSuccess = false; - } - ptxn->commit(0); - pdbCopy->close(0); - - return fSuccess; -} - bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, bilingual_str& errorStr) { std::string walletFile; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile); fs::path walletDir = env->Directory(); - LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(nullptr, nullptr, nullptr)); + LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion()); LogPrintf("Using wallet %s\n", file_path.string()); if (!env->Open(true /* retry */)) { @@ -410,7 +332,7 @@ bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, bilingual_str& return true; } -bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::vector<bilingual_str>& warnings, bilingual_str& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc) +bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, bilingual_str& errorStr) { std::string walletFile; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile); @@ -418,19 +340,8 @@ bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::vector<bi if (fs::exists(walletDir / walletFile)) { - std::string backup_filename; - BerkeleyEnvironment::VerifyResult r = env->Verify(walletFile, recoverFunc, backup_filename); - if (r == BerkeleyEnvironment::VerifyResult::RECOVER_OK) - { - warnings.push_back(strprintf(_("Warning: Wallet file corrupt, data salvaged!" - " Original %s saved as %s in %s; if" - " your balance or transactions are incorrect you should" - " restore from a backup."), - walletFile, backup_filename, walletDir)); - } - if (r == BerkeleyEnvironment::VerifyResult::RECOVER_FAIL) - { - errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile); + if (!env->Verify(walletFile)) { + errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), walletFile); return false; } } @@ -438,72 +349,6 @@ bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::vector<bi return true; } -/* End of headers, beginning of key/value data */ -static const char *HEADER_END = "HEADER=END"; -/* End of key/value data */ -static const char *DATA_END = "DATA=END"; - -bool BerkeleyEnvironment::Salvage(const std::string& strFile, bool fAggressive, std::vector<BerkeleyEnvironment::KeyValPair>& vResult) -{ - LOCK(cs_db); - assert(mapFileUseCount.count(strFile) == 0); - - u_int32_t flags = DB_SALVAGE; - if (fAggressive) - flags |= DB_AGGRESSIVE; - - std::stringstream strDump; - - Db db(dbenv.get(), 0); - int result = db.verify(strFile.c_str(), nullptr, &strDump, flags); - if (result == DB_VERIFY_BAD) { - LogPrintf("BerkeleyEnvironment::Salvage: Database salvage found errors, all data may not be recoverable.\n"); - if (!fAggressive) { - LogPrintf("BerkeleyEnvironment::Salvage: Rerun with aggressive mode to ignore errors and continue.\n"); - return false; - } - } - if (result != 0 && result != DB_VERIFY_BAD) { - LogPrintf("BerkeleyEnvironment::Salvage: Database salvage failed with result %d.\n", result); - return false; - } - - // Format of bdb dump is ascii lines: - // header lines... - // HEADER=END - // hexadecimal key - // hexadecimal value - // ... repeated - // DATA=END - - std::string strLine; - while (!strDump.eof() && strLine != HEADER_END) - getline(strDump, strLine); // Skip past header - - std::string keyHex, valueHex; - while (!strDump.eof() && keyHex != DATA_END) { - getline(strDump, keyHex); - if (keyHex != DATA_END) { - if (strDump.eof()) - break; - getline(strDump, valueHex); - if (valueHex == DATA_END) { - LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Number of keys in data does not match number of values.\n"); - break; - } - vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex))); - } - } - - if (keyHex != DATA_END) { - LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Unexpected end of file while reading salvage output.\n"); - return false; - } - - return (result == 0); -} - - void BerkeleyEnvironment::CheckpointLSN(const std::string& strFile) { dbenv->txn_checkpoint(0, 0, 0); @@ -824,7 +669,6 @@ bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase& database) if (nRefCount == 0) { - boost::this_thread::interruption_point(); std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile); if (mi != env->mapFileUseCount.end()) { @@ -916,3 +760,8 @@ void BerkeleyDatabase::ReloadDbEnv() env->ReloadDbEnv(); } } + +std::string BerkeleyDatabaseVersion() +{ + return DbEnv::version(nullptr, nullptr, nullptr); +} diff --git a/src/wallet/db.h b/src/wallet/db.h index 1bf3375475..54ce144ffc 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -19,7 +19,14 @@ #include <unordered_map> #include <vector> +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsuggest-override" +#endif #include <db_cxx.h> +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif struct bilingual_str; @@ -59,26 +66,7 @@ public: bool IsDatabaseLoaded(const std::string& db_filename) const { return m_databases.find(db_filename) != m_databases.end(); } fs::path Directory() const { return strPath; } - /** - * Verify that database file strFile is OK. If it is not, - * call the callback to try to recover. - * This must be called BEFORE strFile is opened. - * Returns true if strFile is OK. - */ - enum class VerifyResult { VERIFY_OK, - RECOVER_OK, - RECOVER_FAIL }; - typedef bool (*recoverFunc_type)(const fs::path& file_path, std::string& out_backup_filename); - VerifyResult Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename); - /** - * Salvage data from a file that Verify says is bad. - * fAggressive sets the DB_AGGRESSIVE flag (see berkeley DB->verify() method documentation). - * Appends binary key/value pairs to vResult, returns true if successful. - * NOTE: reads the entire database into memory, so cannot be used - * for huge databases. - */ - typedef std::pair<std::vector<unsigned char>, std::vector<unsigned char> > KeyValPair; - bool Salvage(const std::string& strFile, bool fAggressive, std::vector<KeyValPair>& vResult); + bool Verify(const std::string& strFile); bool Open(bool retry); void Close(); @@ -238,7 +226,6 @@ public: void Flush(); void Close(); - static bool Recover(const fs::path& file_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& out_backup_filename); /* flush the wallet passively (TRY_LOCK) ideal to be called periodically */ @@ -246,7 +233,7 @@ public: /* verifies the database environment */ static bool VerifyEnvironment(const fs::path& file_path, bilingual_str& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const fs::path& file_path, std::vector<bilingual_str>& warnings, bilingual_str& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc); + static bool VerifyDatabaseFile(const fs::path& file_path, bilingual_str& errorStr); template <typename K, typename T> bool Read(const K& key, T& value) @@ -402,4 +389,6 @@ public: bool static Rewrite(BerkeleyDatabase& database, const char* pszSkip = nullptr); }; +std::string BerkeleyDatabaseVersion(); + #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 8688f3df5e..3885eb6185 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -54,7 +54,6 @@ void WalletInit::AddWalletOptions() const gArgs.AddArg("-paytxfee=<amt>", strprintf("Fee (in %s/kB) to add to transactions you send (default: %s)", CURRENCY_UNIT, FormatMoney(CFeeRate{DEFAULT_PAY_TX_FEE}.GetFeePerK())), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-rescan", "Rescan the block chain for missing wallet transactions on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - gArgs.AddArg("-salvagewallet", "Attempt to recover private keys from a corrupt wallet on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-spendzeroconfchange", strprintf("Spend unconfirmed change when sending transactions (default: %u)", DEFAULT_SPEND_ZEROCONF_CHANGE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-txconfirmtarget=<n>", strprintf("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)", DEFAULT_TX_CONFIRM_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-wallet=<path>", "Specify wallet database path. Can be specified multiple times to load multiple wallets. Path is interpreted relative to <walletdir> if it is not absolute, and will be created if it does not exist (as a directory containing a wallet.dat file and log files). For backwards compatibility this will also accept names of existing data files in <walletdir>.)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); @@ -89,16 +88,6 @@ bool WalletInit::ParameterInteraction() const LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -walletbroadcast=0\n", __func__); } - if (gArgs.GetBoolArg("-salvagewallet", false)) { - if (is_multiwallet) { - return InitError(strprintf("%s is only allowed with a single wallet file", "-salvagewallet")); - } - // Rewrite just private keys: rescan to find transactions - if (gArgs.SoftSetBoolArg("-rescan", true)) { - LogPrintf("%s: parameter interaction: -salvagewallet=1 -> setting -rescan=1\n", __func__); - } - } - bool zapwallettxes = gArgs.GetBoolArg("-zapwallettxes", false); // -zapwallettxes implies dropping the mempool on startup if (zapwallettxes && gArgs.SoftSetBoolArg("-persistmempool", false)) { @@ -108,7 +97,7 @@ bool WalletInit::ParameterInteraction() const // -zapwallettxes implies a rescan if (zapwallettxes) { if (is_multiwallet) { - return InitError(strprintf("%s is only allowed with a single wallet file", "-zapwallettxes")); + return InitError(strprintf(Untranslated("%s is only allowed with a single wallet file"), "-zapwallettxes")); } if (gArgs.SoftSetBoolArg("-rescan", true)) { LogPrintf("%s: parameter interaction: -zapwallettxes enabled -> setting -rescan=1\n", __func__); @@ -116,7 +105,7 @@ bool WalletInit::ParameterInteraction() const } if (gArgs.GetBoolArg("-sysperms", false)) - return InitError("-sysperms is not allowed in combination with enabled wallet functionality"); + return InitError(Untranslated("-sysperms is not allowed in combination with enabled wallet functionality")); return true; } diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 217a950457..8df3e78215 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -20,14 +20,14 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal // The canonical path cleans the path, preventing >1 Berkeley environment instances for the same directory fs::path canonical_wallet_dir = fs::canonical(wallet_dir, error); if (error || !fs::exists(wallet_dir)) { - chain.initError(strprintf(_("Specified -walletdir \"%s\" does not exist").translated, wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" does not exist"), wallet_dir.string())); return false; } else if (!fs::is_directory(wallet_dir)) { - chain.initError(strprintf(_("Specified -walletdir \"%s\" is not a directory").translated, wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), wallet_dir.string())); return false; // The canonical path transforms relative paths into absolute ones, so we check the non-canonical version } else if (!wallet_dir.is_absolute()) { - chain.initError(strprintf(_("Specified -walletdir \"%s\" is a relative path").translated, wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), wallet_dir.string())); return false; } gArgs.ForceSetArg("-walletdir", canonical_wallet_dir.string()); @@ -37,11 +37,6 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal chain.initMessage(_("Verifying wallet(s)...").translated); - // Parameter interaction code should have thrown an error if -salvagewallet - // was enabled with more than wallet file, so the wallet_files size check - // here should have no effect. - bool salvage_wallet = gArgs.GetBoolArg("-salvagewallet", false) && wallet_files.size() <= 1; - // Keep track of each wallet absolute path to detect duplicates. std::set<fs::path> wallet_paths; @@ -49,16 +44,16 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal WalletLocation location(wallet_file); if (!wallet_paths.insert(location.GetPath()).second) { - chain.initError(strprintf(_("Error loading wallet %s. Duplicate -wallet filename specified.").translated, wallet_file)); + chain.initError(strprintf(_("Error loading wallet %s. Duplicate -wallet filename specified."), wallet_file)); return false; } bilingual_str error_string; std::vector<bilingual_str> warnings; - bool verify_success = CWallet::Verify(chain, location, salvage_wallet, error_string, warnings); - if (!warnings.empty()) chain.initWarning(Join(warnings, "\n", OpTranslated)); + bool verify_success = CWallet::Verify(chain, location, error_string, warnings); + if (!warnings.empty()) chain.initWarning(Join(warnings, Untranslated("\n"))); if (!verify_success) { - chain.initError(error_string.translated); + chain.initError(error_string); return false; } } @@ -73,16 +68,16 @@ bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& walle bilingual_str error; std::vector<bilingual_str> warnings; std::shared_ptr<CWallet> pwallet = CWallet::CreateWalletFromFile(chain, WalletLocation(walletFile), error, warnings); - if (!warnings.empty()) chain.initWarning(Join(warnings, "\n", OpTranslated)); + if (!warnings.empty()) chain.initWarning(Join(warnings, Untranslated("\n"))); if (!pwallet) { - chain.initError(error.translated); + chain.initError(error); return false; } AddWallet(pwallet); } return true; } catch (const std::runtime_error& e) { - chain.initError(e.what()); + chain.initError(Untranslated(e.what())); return false; } } diff --git a/src/wallet/load.h b/src/wallet/load.h index 5a62e29303..e24b1f2e69 100644 --- a/src/wallet/load.h +++ b/src/wallet/load.h @@ -16,8 +16,6 @@ class Chain; } // namespace interfaces //! Responsible for reading and validating the -wallet arguments and verifying the wallet database. -//! This function will perform salvage on the wallet if requested, as long as only one wallet is -//! being loaded (WalletInit::ParameterInteraction() forbids -salvagewallet, -zapwallettxes or -upgradewallet with multiwallet). bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files); //! Load wallet databases. diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 7bf3d169c3..d5f6d63a46 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -746,7 +746,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now wallet.BlockUntilSyncedToCurrentChain(); - LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); + LOCK2(wallet.cs_wallet, spk_man.cs_KeyStore); EnsureWalletIsUnlocked(&wallet); @@ -769,7 +769,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) std::map<CKeyID, int64_t> mapKeyBirth; const std::map<CKeyID, int64_t>& mapKeyPool = spk_man.GetAllReserveKeys(); - pwallet->GetKeyBirthTimes(mapKeyBirth); + wallet.GetKeyBirthTimes(mapKeyBirth); std::set<CScriptID> scripts = spk_man.GetCScripts(); diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index dda00f1fe7..ae2b19e923 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -21,12 +21,14 @@ #include <util/fees.h> #include <util/message.h> // For MessageSign() #include <util/moneystr.h> +#include <util/ref.h> #include <util/string.h> #include <util/system.h> #include <util/translation.h> #include <util/url.h> #include <util/vector.h> #include <wallet/coincontrol.h> +#include <wallet/context.h> #include <wallet/feebumper.h> #include <wallet/rpcwallet.h> #include <wallet/wallet.h> @@ -121,6 +123,14 @@ void EnsureWalletIsUnlocked(const CWallet* pwallet) } } +WalletContext& EnsureWalletContext(const util::Ref& context) +{ + if (!context.Has<WalletContext>()) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Wallet context not found"); + } + return context.Get<WalletContext>(); +} + // also_create should only be set to true only when the RPC is expected to add things to a blank wallet and make it no longer blank LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create) { @@ -1482,10 +1492,9 @@ UniValue listtransactions(const JSONRPCRequest& request) static UniValue listsinceblock(const JSONRPCRequest& request) { - std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); - const CWallet* const pwallet = wallet.get(); + std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + if (!EnsureWalletIsAvailable(pwallet.get(), request.fHelp)) { return NullUniValue; } @@ -1542,11 +1551,12 @@ static UniValue listsinceblock(const JSONRPCRequest& request) }, }.Check(request); + const CWallet& wallet = *pwallet; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now - pwallet->BlockUntilSyncedToCurrentChain(); + wallet.BlockUntilSyncedToCurrentChain(); - LOCK(pwallet->cs_wallet); + LOCK(wallet.cs_wallet); // The way the 'height' is initialized is just a workaround for the gcc bug #47679 since version 4.6.0. Optional<int> height = MakeOptional(false, int()); // Height of the specified block or the common ancestor, if the block provided was in a deactivated chain. @@ -1557,9 +1567,9 @@ static UniValue listsinceblock(const JSONRPCRequest& request) uint256 blockId; if (!request.params[0].isNull() && !request.params[0].get_str().empty()) { blockId = ParseHashV(request.params[0], "blockhash"); - height.emplace(); - altheight.emplace(); - if (!pwallet->chain().findCommonAncestor(blockId, pwallet->GetLastBlockHash(), /* ancestor out */ FoundBlock().height(*height), /* blockId out */ FoundBlock().height(*altheight))) { + height = int{}; + altheight = int{}; + if (!wallet.chain().findCommonAncestor(blockId, wallet.GetLastBlockHash(), /* ancestor out */ FoundBlock().height(*height), /* blockId out */ FoundBlock().height(*altheight))) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } } @@ -1572,21 +1582,21 @@ static UniValue listsinceblock(const JSONRPCRequest& request) } } - if (ParseIncludeWatchonly(request.params[2], *pwallet)) { + if (ParseIncludeWatchonly(request.params[2], wallet)) { filter |= ISMINE_WATCH_ONLY; } bool include_removed = (request.params[3].isNull() || request.params[3].get_bool()); - int depth = height ? pwallet->GetLastBlockHeight() + 1 - *height : -1; + int depth = height ? wallet.GetLastBlockHeight() + 1 - *height : -1; UniValue transactions(UniValue::VARR); - for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + for (const std::pair<const uint256, CWalletTx>& pairWtx : wallet.mapWallet) { const CWalletTx& tx = pairWtx.second; if (depth == -1 || abs(tx.GetDepthInMainChain()) < depth) { - ListTransactions(pwallet, tx, 0, true, transactions, filter, nullptr /* filter_label */); + ListTransactions(&wallet, tx, 0, true, transactions, filter, nullptr /* filter_label */); } } @@ -1595,15 +1605,15 @@ static UniValue listsinceblock(const JSONRPCRequest& request) UniValue removed(UniValue::VARR); while (include_removed && altheight && *altheight > *height) { CBlock block; - if (!pwallet->chain().findBlock(blockId, FoundBlock().data(block)) || block.IsNull()) { + if (!wallet.chain().findBlock(blockId, FoundBlock().data(block)) || block.IsNull()) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); } for (const CTransactionRef& tx : block.vtx) { - auto it = pwallet->mapWallet.find(tx->GetHash()); - if (it != pwallet->mapWallet.end()) { + auto it = wallet.mapWallet.find(tx->GetHash()); + if (it != wallet.mapWallet.end()) { // We want all transactions regardless of confirmation count to appear here, // even negative confirmation ones, hence the big negative. - ListTransactions(pwallet, it->second, -100000000, true, removed, filter, nullptr /* filter_label */); + ListTransactions(&wallet, it->second, -100000000, true, removed, filter, nullptr /* filter_label */); } } blockId = block.hashPrevBlock; @@ -1611,7 +1621,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) } uint256 lastblock; - CHECK_NONFATAL(pwallet->chain().findAncestorByHeight(pwallet->GetLastBlockHash(), pwallet->GetLastBlockHeight() + 1 - target_confirms, FoundBlock().hash(lastblock))); + CHECK_NONFATAL(wallet.chain().findAncestorByHeight(wallet.GetLastBlockHash(), wallet.GetLastBlockHeight() + 1 - target_confirms, FoundBlock().hash(lastblock))); UniValue ret(UniValue::VOBJ); ret.pushKV("transactions", transactions); @@ -1892,6 +1902,9 @@ static UniValue walletpassphrase(const JSONRPCRequest& request) }.Check(request); int64_t nSleepTime; + int64_t relock_time; + // Prevent concurrent calls to walletpassphrase with the same wallet. + LOCK(pwallet->m_unlock_mutex); { LOCK(pwallet->cs_wallet); @@ -1929,6 +1942,7 @@ static UniValue walletpassphrase(const JSONRPCRequest& request) pwallet->TopUpKeyPool(); pwallet->nRelockTime = GetTime() + nSleepTime; + relock_time = pwallet->nRelockTime; } // rpcRunLater must be called without cs_wallet held otherwise a deadlock @@ -1940,9 +1954,11 @@ static UniValue walletpassphrase(const JSONRPCRequest& request) // wallet before the following callback is called. If a valid shared pointer // is acquired in the callback then the wallet is still loaded. std::weak_ptr<CWallet> weak_wallet = wallet; - pwallet->chain().rpcRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), [weak_wallet] { + pwallet->chain().rpcRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), [weak_wallet, relock_time] { if (auto shared_wallet = weak_wallet.lock()) { LOCK(shared_wallet->cs_wallet); + // Skip if this is not the most recent rpcRunLater callback. + if (shared_wallet->nRelockTime != relock_time) return; shared_wallet->Lock(); shared_wallet->nRelockTime = 0; } @@ -2578,6 +2594,7 @@ static UniValue loadwallet(const JSONRPCRequest& request) }, }.Check(request); + WalletContext& context = EnsureWalletContext(request.context); WalletLocation location(request.params[0].get_str()); if (!location.Exists()) { @@ -2592,12 +2609,12 @@ static UniValue loadwallet(const JSONRPCRequest& request) bilingual_str error; std::vector<bilingual_str> warnings; - std::shared_ptr<CWallet> const wallet = LoadWallet(*g_rpc_chain, location, error, warnings); + std::shared_ptr<CWallet> const wallet = LoadWallet(*context.chain, location, error, warnings); if (!wallet) throw JSONRPCError(RPC_WALLET_ERROR, error.original); UniValue obj(UniValue::VOBJ); obj.pushKV("name", wallet->GetName()); - obj.pushKV("warning", Join(warnings, "\n", OpOriginal)); + obj.pushKV("warning", Join(warnings, Untranslated("\n")).original); return obj; } @@ -2696,6 +2713,7 @@ static UniValue createwallet(const JSONRPCRequest& request) }, }.Check(request); + WalletContext& context = EnsureWalletContext(request.context); uint64_t flags = 0; if (!request.params[1].isNull() && request.params[1].get_bool()) { flags |= WALLET_FLAG_DISABLE_PRIVATE_KEYS; @@ -2720,11 +2738,12 @@ static UniValue createwallet(const JSONRPCRequest& request) } if (!request.params[5].isNull() && request.params[5].get_bool()) { flags |= WALLET_FLAG_DESCRIPTORS; + warnings.emplace_back(Untranslated("Wallet is an experimental descriptor wallet")); } bilingual_str error; std::shared_ptr<CWallet> wallet; - WalletCreationStatus status = CreateWallet(*g_rpc_chain, passphrase, flags, request.params[0].get_str(), error, warnings, wallet); + WalletCreationStatus status = CreateWallet(*context.chain, passphrase, flags, request.params[0].get_str(), error, warnings, wallet); switch (status) { case WalletCreationStatus::CREATION_FAILED: throw JSONRPCError(RPC_WALLET_ERROR, error.original); @@ -2737,7 +2756,7 @@ static UniValue createwallet(const JSONRPCRequest& request) UniValue obj(UniValue::VOBJ); obj.pushKV("name", wallet->GetName()); - obj.pushKV("warning", Join(warnings, "\n", OpOriginal)); + obj.pushKV("warning", Join(warnings, Untranslated("\n")).original); return obj; } @@ -3973,10 +3992,6 @@ UniValue sethdseed(const JSONRPCRequest& request) LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet, true); - if (pwallet->chain().isInitialBlockDownload()) { - throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Cannot set a new HD seed while still in Initial Block Download"); - } - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed to a wallet with private keys disabled"); } @@ -4260,7 +4275,7 @@ UniValue removeprunedfunds(const JSONRPCRequest& request); UniValue importmulti(const JSONRPCRequest& request); UniValue importdescriptors(const JSONRPCRequest& request); -void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers) +Span<const CRPCCommand> GetWalletRPCCommands() { // clang-format off static const CRPCCommand commands[] = @@ -4326,9 +4341,5 @@ static const CRPCCommand commands[] = { "wallet", "walletprocesspsbt", &walletprocesspsbt, {"psbt","sign","sighashtype","bip32derivs"} }, }; // clang-format on - - for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) - handlers.emplace_back(chain.handleRpc(commands[vcidx])); + return MakeSpan(commands); } - -interfaces::Chain* g_rpc_chain = nullptr; diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index 8c149d455b..d00221a04c 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -5,30 +5,22 @@ #ifndef BITCOIN_WALLET_RPCWALLET_H #define BITCOIN_WALLET_RPCWALLET_H +#include <span.h> + #include <memory> #include <string> #include <vector> -class CRPCTable; +class CRPCCommand; class CWallet; class JSONRPCRequest; class LegacyScriptPubKeyMan; class UniValue; -struct PartiallySignedTransaction; class CTransaction; +struct PartiallySignedTransaction; +struct WalletContext; -namespace interfaces { -class Chain; -class Handler; -} - -//! Pointer to chain interface that needs to be declared as a global to be -//! accessible loadwallet and createwallet methods. Due to limitations of the -//! RPC framework, there's currently no direct way to pass in state to RPC -//! methods without globals. -extern interfaces::Chain* g_rpc_chain; - -void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers); +Span<const CRPCCommand> GetWalletRPCCommands(); /** * Figures out what wallet, if any, to use for a JSONRPCRequest. @@ -40,6 +32,7 @@ std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& reques void EnsureWalletIsUnlocked(const CWallet*); bool EnsureWalletIsAvailable(const CWallet*, bool avoidException); +WalletContext& EnsureWalletContext(const util::Ref& context); LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create = false); UniValue getaddressinfo(const JSONRPCRequest& request); diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp new file mode 100644 index 0000000000..70067ebef0 --- /dev/null +++ b/src/wallet/salvage.cpp @@ -0,0 +1,150 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <fs.h> +#include <streams.h> +#include <wallet/salvage.h> +#include <wallet/wallet.h> +#include <wallet/walletdb.h> + +/* End of headers, beginning of key/value data */ +static const char *HEADER_END = "HEADER=END"; +/* End of key/value data */ +static const char *DATA_END = "DATA=END"; +typedef std::pair<std::vector<unsigned char>, std::vector<unsigned char> > KeyValPair; + +bool RecoverDatabaseFile(const fs::path& file_path) +{ + std::string filename; + std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename); + + // Recovery procedure: + // move wallet file to walletfilename.timestamp.bak + // Call Salvage with fAggressive=true to + // get as much data as possible. + // Rewrite salvaged data to fresh wallet file + // Set -rescan so any missing transactions will be + // found. + int64_t now = GetTime(); + std::string newFilename = strprintf("%s.%d.bak", filename, now); + + int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr, + newFilename.c_str(), DB_AUTO_COMMIT); + if (result == 0) + LogPrintf("Renamed %s to %s\n", filename, newFilename); + else + { + LogPrintf("Failed to rename %s to %s\n", filename, newFilename); + return false; + } + + /** + * Salvage data from a file. The DB_AGGRESSIVE flag is being used (see berkeley DB->verify() method documentation). + * key/value pairs are appended to salvagedData which are then written out to a new wallet file. + * NOTE: reads the entire database into memory, so cannot be used + * for huge databases. + */ + std::vector<KeyValPair> salvagedData; + + std::stringstream strDump; + + Db db(env->dbenv.get(), 0); + result = db.verify(newFilename.c_str(), nullptr, &strDump, DB_SALVAGE | DB_AGGRESSIVE); + if (result == DB_VERIFY_BAD) { + LogPrintf("Salvage: Database salvage found errors, all data may not be recoverable.\n"); + } + if (result != 0 && result != DB_VERIFY_BAD) { + LogPrintf("Salvage: Database salvage failed with result %d.\n", result); + return false; + } + + // Format of bdb dump is ascii lines: + // header lines... + // HEADER=END + // hexadecimal key + // hexadecimal value + // ... repeated + // DATA=END + + std::string strLine; + while (!strDump.eof() && strLine != HEADER_END) + getline(strDump, strLine); // Skip past header + + std::string keyHex, valueHex; + while (!strDump.eof() && keyHex != DATA_END) { + getline(strDump, keyHex); + if (keyHex != DATA_END) { + if (strDump.eof()) + break; + getline(strDump, valueHex); + if (valueHex == DATA_END) { + LogPrintf("Salvage: WARNING: Number of keys in data does not match number of values.\n"); + break; + } + salvagedData.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex))); + } + } + + bool fSuccess; + if (keyHex != DATA_END) { + LogPrintf("Salvage: WARNING: Unexpected end of file while reading salvage output.\n"); + fSuccess = false; + } else { + fSuccess = (result == 0); + } + + if (salvagedData.empty()) + { + LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); + return false; + } + LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); + + std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0); + int ret = pdbCopy->open(nullptr, // Txn pointer + filename.c_str(), // Filename + "main", // Logical db name + DB_BTREE, // Database type + DB_CREATE, // Flags + 0); + if (ret > 0) { + LogPrintf("Cannot create database file %s\n", filename); + pdbCopy->close(0); + return false; + } + + DbTxn* ptxn = env->TxnBegin(); + CWallet dummyWallet(nullptr, WalletLocation(), WalletDatabase::CreateDummy()); + for (KeyValPair& row : salvagedData) + { + /* Filter for only private key type KV pairs to be added to the salvaged wallet */ + CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); + CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); + std::string strType, strErr; + bool fReadOK; + { + // Required in LoadKeyMetadata(): + LOCK(dummyWallet.cs_wallet); + fReadOK = ReadKeyValue(&dummyWallet, ssKey, ssValue, strType, strErr); + } + if (!WalletBatch::IsKeyType(strType) && strType != DBKeys::HDCHAIN) { + continue; + } + if (!fReadOK) + { + LogPrintf("WARNING: WalletBatch::Recover skipping %s: %s\n", strType, strErr); + continue; + } + Dbt datKey(&row.first[0], row.first.size()); + Dbt datValue(&row.second[0], row.second.size()); + int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); + if (ret2 > 0) + fSuccess = false; + } + ptxn->commit(0); + pdbCopy->close(0); + + return fSuccess; +} diff --git a/src/wallet/salvage.h b/src/wallet/salvage.h new file mode 100644 index 0000000000..e361930f5e --- /dev/null +++ b/src/wallet/salvage.h @@ -0,0 +1,14 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_SALVAGE_H +#define BITCOIN_WALLET_SALVAGE_H + +#include <fs.h> +#include <streams.h> + +bool RecoverDatabaseFile(const fs::path& file_path); + +#endif // BITCOIN_WALLET_SALVAGE_H diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index e4be5045e1..8a2a798644 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -12,6 +12,9 @@ #include <util/translation.h> #include <wallet/scriptpubkeyman.h> +//! Value for the first BIP 32 hardened derivation. Can be used as a bit mask and as a value. See BIP 32 for more details. +const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; + bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) { LOCK(cs_KeyStore); @@ -220,6 +223,7 @@ bool LegacyScriptPubKeyMan::CheckDecryptionKey(const CKeyingMaterial& master_key bool keyPass = mapCryptedKeys.empty(); // Always pass when there are no encrypted keys bool keyFail = false; CryptedKeyMap::const_iterator mi = mapCryptedKeys.begin(); + WalletBatch batch(m_storage.GetDatabase()); for (; mi != mapCryptedKeys.end(); ++mi) { const CPubKey &vchPubKey = (*mi).second.first; @@ -233,6 +237,10 @@ bool LegacyScriptPubKeyMan::CheckDecryptionKey(const CKeyingMaterial& master_key keyPass = true; if (fDecryptionThoroughlyChecked) break; + else { + // Rewrite these encrypted keys with checksums + batch.WriteCryptedKey(vchPubKey, vchCryptedSecret, mapKeyMetadata[vchPubKey.GetID()]); + } } if (keyPass && keyFail) { @@ -290,6 +298,43 @@ bool LegacyScriptPubKeyMan::GetReservedDestination(const OutputType type, bool i return true; } +bool LegacyScriptPubKeyMan::TopUpInactiveHDChain(const CKeyID seed_id, int64_t index, bool internal) +{ + LOCK(cs_KeyStore); + + if (m_storage.IsLocked()) return false; + + auto it = m_inactive_hd_chains.find(seed_id); + if (it == m_inactive_hd_chains.end()) { + return false; + } + + CHDChain& chain = it->second; + + // Top up key pool + int64_t target_size = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 1); + + // "size" of the keypools. Not really the size, actually the difference between index and the chain counter + // Since chain counter is 1 based and index is 0 based, one of them needs to be offset by 1. + int64_t kp_size = (internal ? chain.nInternalChainCounter : chain.nExternalChainCounter) - (index + 1); + + // make sure the keypool fits the user-selected target (-keypool) + int64_t missing = std::max(target_size - kp_size, (int64_t) 0); + + if (missing > 0) { + WalletBatch batch(m_storage.GetDatabase()); + for (int64_t i = missing; i > 0; --i) { + GenerateNewKey(batch, chain, internal); + } + if (internal) { + WalletLogPrintf("inactive seed with id %s added %d internal keys\n", HexStr(seed_id), missing); + } else { + WalletLogPrintf("inactive seed with id %s added %d keys\n", HexStr(seed_id), missing); + } + } + return true; +} + void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) { LOCK(cs_KeyStore); @@ -297,13 +342,28 @@ void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) for (const auto& keyid : GetAffectedKeys(script, *this)) { std::map<CKeyID, int64_t>::const_iterator mi = m_pool_key_to_index.find(keyid); if (mi != m_pool_key_to_index.end()) { - WalletLogPrintf("%s: Detected a used keypool key, mark all keypool key up to this key as used\n", __func__); + WalletLogPrintf("%s: Detected a used keypool key, mark all keypool keys up to this key as used\n", __func__); MarkReserveKeysAsUsed(mi->second); if (!TopUp()) { WalletLogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); } } + + // Find the key's metadata and check if it's seed id (if it has one) is inactive, i.e. it is not the current m_hd_chain seed id. + // If so, TopUp the inactive hd chain + auto it = mapKeyMetadata.find(keyid); + if (it != mapKeyMetadata.end()){ + CKeyMetadata meta = it->second; + if (!meta.hd_seed_id.IsNull() && meta.hd_seed_id != m_hd_chain.seed_id) { + bool internal = (meta.key_origin.path[1] & ~BIP32_HARDENED_KEY_LIMIT) != 0; + int64_t index = meta.key_origin.path[2] & ~BIP32_HARDENED_KEY_LIMIT; + + if (!TopUpInactiveHDChain(meta.hd_seed_id, index, internal)) { + WalletLogPrintf("%s: Adding inactive seed keys failed\n", __func__); + } + } + } } } @@ -357,7 +417,7 @@ bool LegacyScriptPubKeyMan::SetupGeneration(bool force) bool LegacyScriptPubKeyMan::IsHDEnabled() const { - return !hdChain.seed_id.IsNull(); + return !m_hd_chain.seed_id.IsNull(); } bool LegacyScriptPubKeyMan::CanGetAddresses(bool internal) const @@ -713,8 +773,13 @@ bool LegacyScriptPubKeyMan::AddKeyPubKeyInner(const CKey& key, const CPubKey &pu return true; } -bool LegacyScriptPubKeyMan::LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) +bool LegacyScriptPubKeyMan::LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret, bool checksum_valid) { + // Set fDecryptionThoroughlyChecked to false when the checksum is invalid + if (!checksum_valid) { + fDecryptionThoroughlyChecked = false; + } + return AddCryptedKeyInner(vchPubKey, vchCryptedSecret); } @@ -838,10 +903,27 @@ bool LegacyScriptPubKeyMan::AddWatchOnly(const CScript& dest, int64_t nCreateTim void LegacyScriptPubKeyMan::SetHDChain(const CHDChain& chain, bool memonly) { LOCK(cs_KeyStore); - if (!memonly && !WalletBatch(m_storage.GetDatabase()).WriteHDChain(chain)) - throw std::runtime_error(std::string(__func__) + ": writing chain failed"); + // memonly == true means we are loading the wallet file + // memonly == false means that the chain is actually being changed + if (!memonly) { + // Store the new chain + if (!WalletBatch(m_storage.GetDatabase()).WriteHDChain(chain)) { + throw std::runtime_error(std::string(__func__) + ": writing chain failed"); + } + // When there's an old chain, add it as an inactive chain as we are now rotating hd chains + if (!m_hd_chain.seed_id.IsNull()) { + AddInactiveHDChain(m_hd_chain); + } + } + + m_hd_chain = chain; +} - hdChain = chain; +void LegacyScriptPubKeyMan::AddInactiveHDChain(const CHDChain& chain) +{ + LOCK(cs_KeyStore); + assert(!chain.seed_id.IsNull()); + m_inactive_hd_chains[chain.seed_id] = chain; } bool LegacyScriptPubKeyMan::HaveKey(const CKeyID &address) const @@ -920,7 +1002,7 @@ bool LegacyScriptPubKeyMan::GetPubKey(const CKeyID &address, CPubKey& vchPubKeyO return GetWatchPubKey(address, vchPubKeyOut); } -CPubKey LegacyScriptPubKeyMan::GenerateNewKey(WalletBatch &batch, bool internal) +CPubKey LegacyScriptPubKeyMan::GenerateNewKey(WalletBatch &batch, CHDChain& hd_chain, bool internal) { assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); @@ -935,7 +1017,7 @@ CPubKey LegacyScriptPubKeyMan::GenerateNewKey(WalletBatch &batch, bool internal) // use HD key derivation if HD was enabled during wallet creation and a seed is present if (IsHDEnabled()) { - DeriveNewChildKey(batch, metadata, secret, (m_storage.CanSupportFeature(FEATURE_HD_SPLIT) ? internal : false)); + DeriveNewChildKey(batch, metadata, secret, hd_chain, (m_storage.CanSupportFeature(FEATURE_HD_SPLIT) ? internal : false)); } else { secret.MakeNewKey(fCompressed); } @@ -957,9 +1039,7 @@ CPubKey LegacyScriptPubKeyMan::GenerateNewKey(WalletBatch &batch, bool internal) return pubkey; } -const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; - -void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey& secret, bool internal) +void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey& secret, CHDChain& hd_chain, bool internal) { // for now we use a fixed keypath scheme of m/0'/0'/k CKey seed; //seed (256bit) @@ -969,7 +1049,7 @@ void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& CExtKey childKey; //key at m/0'/0'/<n>' // try to get the seed - if (!GetKey(hdChain.seed_id, seed)) + if (!GetKey(hd_chain.seed_id, seed)) throw std::runtime_error(std::string(__func__) + ": seed not found"); masterKey.SetSeed(seed.begin(), seed.size()); @@ -988,30 +1068,30 @@ void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& // childIndex | BIP32_HARDENED_KEY_LIMIT = derive childIndex in hardened child-index-range // example: 1 | BIP32_HARDENED_KEY_LIMIT == 0x80000001 == 2147483649 if (internal) { - chainChildKey.Derive(childKey, hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - metadata.hdKeypath = "m/0'/1'/" + ToString(hdChain.nInternalChainCounter) + "'"; + chainChildKey.Derive(childKey, hd_chain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + metadata.hdKeypath = "m/0'/1'/" + ToString(hd_chain.nInternalChainCounter) + "'"; metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); metadata.key_origin.path.push_back(1 | BIP32_HARDENED_KEY_LIMIT); - metadata.key_origin.path.push_back(hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - hdChain.nInternalChainCounter++; + metadata.key_origin.path.push_back(hd_chain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + hd_chain.nInternalChainCounter++; } else { - chainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - metadata.hdKeypath = "m/0'/0'/" + ToString(hdChain.nExternalChainCounter) + "'"; + chainChildKey.Derive(childKey, hd_chain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + metadata.hdKeypath = "m/0'/0'/" + ToString(hd_chain.nExternalChainCounter) + "'"; metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); - metadata.key_origin.path.push_back(hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - hdChain.nExternalChainCounter++; + metadata.key_origin.path.push_back(hd_chain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + hd_chain.nExternalChainCounter++; } } while (HaveKey(childKey.key.GetPubKey().GetID())); secret = childKey.key; - metadata.hd_seed_id = hdChain.seed_id; + metadata.hd_seed_id = hd_chain.seed_id; CKeyID master_id = masterKey.key.GetPubKey().GetID(); std::copy(master_id.begin(), master_id.begin() + 4, metadata.key_origin.fingerprint); metadata.has_key_origin = true; // update the chain model in the database - if (!batch.WriteHDChain(hdChain)) - throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); + if (hd_chain.seed_id == m_hd_chain.seed_id && !batch.WriteHDChain(hd_chain)) + throw std::runtime_error(std::string(__func__) + ": writing HD chain model failed"); } void LegacyScriptPubKeyMan::LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) @@ -1166,7 +1246,7 @@ bool LegacyScriptPubKeyMan::TopUp(unsigned int kpSize) internal = true; } - CPubKey pubkey(GenerateNewKey(batch, internal)); + CPubKey pubkey(GenerateNewKey(batch, m_hd_chain, internal)); AddKeypoolPubkeyWithDB(pubkey, internal, batch); } if (missingInternal + missingExternal > 0) { @@ -1239,7 +1319,7 @@ bool LegacyScriptPubKeyMan::GetKeyFromPool(CPubKey& result, const OutputType typ if (!ReserveKeyFromKeyPool(nIndex, keypool, internal) && !m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { if (m_storage.IsLocked()) return false; WalletBatch batch(m_storage.GetDatabase()); - result = GenerateNewKey(batch, internal); + result = GenerateNewKey(batch, m_hd_chain, internal); return true; } KeepDestination(nIndex, type); @@ -1497,7 +1577,7 @@ std::set<CKeyID> LegacyScriptPubKeyMan::GetKeys() const return set_address; } -void LegacyScriptPubKeyMan::SetType(OutputType type, bool internal) {} +void LegacyScriptPubKeyMan::SetInternal(bool internal) {} bool DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) { @@ -1509,7 +1589,9 @@ bool DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDest { LOCK(cs_desc_man); assert(m_wallet_descriptor.descriptor->IsSingleType()); // This is a combo descriptor which should not be an active descriptor - if (type != m_address_type) { + Optional<OutputType> desc_addr_type = m_wallet_descriptor.descriptor->GetOutputType(); + assert(desc_addr_type); + if (type != *desc_addr_type) { throw std::runtime_error(std::string(__func__) + ": Types are inconsistent"); } @@ -1777,7 +1859,7 @@ bool DescriptorScriptPubKeyMan::AddDescriptorKeyWithDB(WalletBatch& batch, const } } -bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_key) +bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_key, OutputType addr_type) { LOCK(cs_desc_man); assert(m_storage.IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); @@ -1794,7 +1876,7 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_ // Build descriptor string std::string desc_prefix; std::string desc_suffix = "/*)"; - switch (m_address_type) { + switch (addr_type) { case OutputType::LEGACY: { desc_prefix = "pkh(" + xpub + "/44'"; break; @@ -2076,9 +2158,8 @@ uint256 DescriptorScriptPubKeyMan::GetID() const return id; } -void DescriptorScriptPubKeyMan::SetType(OutputType type, bool internal) +void DescriptorScriptPubKeyMan::SetInternal(bool internal) { - this->m_address_type = type; this->m_internal = internal; } diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 4c002edf2d..d62d30f339 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -18,6 +18,8 @@ #include <boost/signals2/signal.hpp> +#include <unordered_map> + enum class OutputType; struct bilingual_str; @@ -110,40 +112,52 @@ public: CKeyPool(); CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn); - ADD_SERIALIZE_METHODS; + template<typename Stream> + void Serialize(Stream& s) const + { + int nVersion = s.GetVersion(); + if (!(s.GetType() & SER_GETHASH)) { + s << nVersion; + } + s << nTime << vchPubKey << fInternal << m_pre_split; + } - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { + template<typename Stream> + void Unserialize(Stream& s) + { int nVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) - READWRITE(nVersion); - READWRITE(nTime); - READWRITE(vchPubKey); - if (ser_action.ForRead()) { - try { - READWRITE(fInternal); - } - catch (std::ios_base::failure&) { - /* flag as external address if we can't read the internal boolean - (this will be the case for any wallet before the HD chain split version) */ - fInternal = false; - } - try { - READWRITE(m_pre_split); - } - catch (std::ios_base::failure&) { - /* flag as postsplit address if we can't read the m_pre_split boolean - (this will be the case for any wallet that upgrades to HD chain split)*/ - m_pre_split = false; - } + if (!(s.GetType() & SER_GETHASH)) { + s >> nVersion; + } + s >> nTime >> vchPubKey; + try { + s >> fInternal; + } catch (std::ios_base::failure&) { + /* flag as external address if we can't read the internal boolean + (this will be the case for any wallet before the HD chain split version) */ + fInternal = false; } - else { - READWRITE(fInternal); - READWRITE(m_pre_split); + try { + s >> m_pre_split; + } catch (std::ios_base::failure&) { + /* flag as postsplit address if we can't read the m_pre_split boolean + (this will be the case for any wallet that upgrades to HD chain split) */ + m_pre_split = false; } } }; +class KeyIDHasher +{ +public: + KeyIDHasher() {} + + size_t operator()(const CKeyID& id) const + { + return id.GetUint64(0); + } +}; + /* * A class implementing ScriptPubKeyMan manages some (or all) scriptPubKeys used in a wallet. * It contains the scripts and keys related to the scriptPubKeys it manages. @@ -224,7 +238,7 @@ public: virtual uint256 GetID() const { return uint256(); } - virtual void SetType(OutputType type, bool internal) {} + virtual void SetInternal(bool internal) {} /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ template<typename... Params> @@ -243,7 +257,7 @@ class LegacyScriptPubKeyMan : public ScriptPubKeyMan, public FillableSigningProv { private: //! keeps track of whether Unlock has run a thorough check before - bool fDecryptionThoroughlyChecked = false; + bool fDecryptionThoroughlyChecked = true; using WatchOnlySet = std::set<CScript>; using WatchKeyMap = std::map<CKeyID, CPubKey>; @@ -288,10 +302,11 @@ private: bool AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info); /* the HD chain data model (external chain counters) */ - CHDChain hdChain; + CHDChain m_hd_chain; + std::unordered_map<CKeyID, CHDChain, KeyIDHasher> m_inactive_hd_chains; /* HD derive new child key (on internal or external chain) */ - void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, CHDChain& hd_chain, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_KeyStore); std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_KeyStore); @@ -320,6 +335,18 @@ private: */ bool ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal); + /** + * Like TopUp() but adds keys for inactive HD chains. + * Ensures that there are at least -keypool number of keys derived after the given index. + * + * @param seed_id the CKeyID for the HD seed. + * @param index the index to start generating keys from + * @param internal whether the internal chain should be used. true for internal chain, false for external chain. + * + * @return true if seed was found and keys were derived. false if unable to derive seeds + */ + bool TopUpInactiveHDChain(const CKeyID seed_id, int64_t index, bool internal); + public: using ScriptPubKeyMan::ScriptPubKeyMan; @@ -370,7 +397,7 @@ public: uint256 GetID() const override; - void SetType(OutputType type, bool internal) override; + void SetInternal(bool internal) override; // Map from Key ID to key metadata. std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_KeyStore); @@ -385,7 +412,7 @@ public: //! Adds an encrypted key to the store, and saves it to disk. bool AddCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); //! Adds an encrypted key to the store, without saving it to disk (used by LoadWallet) - bool LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); + bool LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret, bool checksum_valid); void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); //! Adds a CScript to the store bool LoadCScript(const CScript& redeemScript); @@ -393,11 +420,12 @@ public: void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata); void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata); //! Generate a new key - CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + CPubKey GenerateNewKey(WalletBatch& batch, CHDChain& hd_chain, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); /* Set the HD chain model (chain child index counters) */ void SetHDChain(const CHDChain& chain, bool memonly); - const CHDChain& GetHDChain() const { return hdChain; } + const CHDChain& GetHDChain() const { return m_hd_chain; } + void AddInactiveHDChain(const CHDChain& chain); //! Adds a watch-only address to the store, without saving it to disk (used by LoadWallet) bool LoadWatchOnly(const CScript &dest); @@ -497,14 +525,11 @@ private: PubKeyMap m_map_pubkeys GUARDED_BY(cs_desc_man); int32_t m_max_cached_index = -1; - OutputType m_address_type; bool m_internal = false; KeyMap m_map_keys GUARDED_BY(cs_desc_man); CryptedKeyMap m_map_crypted_keys GUARDED_BY(cs_desc_man); - bool SetCrypted(); - //! keeps track of whether Unlock has run a thorough check before bool m_decryption_thoroughly_checked = false; @@ -524,9 +549,9 @@ public: : ScriptPubKeyMan(storage), m_wallet_descriptor(descriptor) {} - DescriptorScriptPubKeyMan(WalletStorage& storage, OutputType address_type, bool internal) + DescriptorScriptPubKeyMan(WalletStorage& storage, bool internal) : ScriptPubKeyMan(storage), - m_address_type(address_type), m_internal(internal) + m_internal(internal) {} mutable RecursiveMutex cs_desc_man; @@ -551,7 +576,7 @@ public: bool IsHDEnabled() const override; //! Setup descriptors based on the given CExtkey - bool SetupDescriptorGeneration(const CExtKey& master_key); + bool SetupDescriptorGeneration(const CExtKey& master_key, OutputType addr_type); bool HavePrivateKeys() const override; @@ -575,7 +600,7 @@ public: uint256 GetID() const override; - void SetType(OutputType type, bool internal) override; + void SetInternal(bool internal) override; void SetCache(const DescriptorCache& cache); diff --git a/src/wallet/test/init_test_fixture.h b/src/wallet/test/init_test_fixture.h index 6ba7d66b7c..c95b4f1f6e 100644 --- a/src/wallet/test/init_test_fixture.h +++ b/src/wallet/test/init_test_fixture.h @@ -18,7 +18,6 @@ struct InitWalletDirTestingSetup: public BasicTestingSetup { fs::path m_datadir; fs::path m_cwd; std::map<std::string, fs::path> m_walletdir_path_cases; - NodeContext m_node; std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); std::unique_ptr<interfaces::ChainClient> m_chain_client; }; diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index a294935b64..6c32868b1e 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -19,7 +19,6 @@ struct WalletTestingSetup: public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); - NodeContext m_node; std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); std::unique_ptr<interfaces::ChainClient> m_chain_client = interfaces::MakeWalletClient(*m_chain, {}); CWallet m_wallet; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 3dba106c41..6f8c973f25 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -15,6 +15,7 @@ #include <rpc/server.h> #include <test/util/logging.h> #include <test/util/setup_common.h> +#include <util/ref.h> #include <util/translation.h> #include <validation.h> #include <wallet/coincontrol.h> @@ -117,7 +118,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Prune the older block file. { LOCK(cs_main); - PruneOneBlockFile(oldTip->GetBlockPos().nFile); + EnsureChainman(m_node).PruneOneBlockFile(oldTip->GetBlockPos().nFile); } UnlinkPrunedFiles({oldTip->GetBlockPos().nFile}); @@ -143,7 +144,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Prune the remaining block file. { LOCK(cs_main); - PruneOneBlockFile(newTip->GetBlockPos().nFile); + EnsureChainman(m_node).PruneOneBlockFile(newTip->GetBlockPos().nFile); } UnlinkPrunedFiles({newTip->GetBlockPos().nFile}); @@ -180,7 +181,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // Prune the older block file. { LOCK(cs_main); - PruneOneBlockFile(oldTip->GetBlockPos().nFile); + EnsureChainman(m_node).PruneOneBlockFile(oldTip->GetBlockPos().nFile); } UnlinkPrunedFiles({oldTip->GetBlockPos().nFile}); @@ -208,7 +209,8 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) key.pushKV("timestamp", newTip->GetBlockTimeMax() + TIMESTAMP_WINDOW + 1); key.pushKV("internal", UniValue(true)); keys.push_back(key); - JSONRPCRequest request; + util::Ref context; + JSONRPCRequest request(context); request.params.setArray(); request.params.push_back(keys); @@ -262,7 +264,8 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) AddWallet(wallet); wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } - JSONRPCRequest request; + util::Ref context; + JSONRPCRequest request(context); request.params.setArray(); request.params.push_back(backup_file); @@ -277,7 +280,8 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); - JSONRPCRequest request; + util::Ref context; + JSONRPCRequest request(context); request.params.setArray(); request.params.push_back(backup_file); AddWallet(wallet); @@ -531,7 +535,6 @@ public: return it->second; } - NodeContext m_node; std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); std::unique_ptr<CWallet> wallet; }; @@ -742,8 +745,8 @@ BOOST_FIXTURE_TEST_CASE(CreateWalletFromFile, TestChain100Setup) BOOST_CHECK_EQUAL(addtx_count, 2); { LOCK(wallet->cs_wallet); - BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1); - BOOST_CHECK_EQUAL(wallet->mapWallet.count(mempool_tx.GetHash()), 1); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(mempool_tx.GetHash()), 1U); } @@ -779,8 +782,8 @@ BOOST_FIXTURE_TEST_CASE(CreateWalletFromFile, TestChain100Setup) BOOST_CHECK_EQUAL(addtx_count, 4); { LOCK(wallet->cs_wallet); - BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1); - BOOST_CHECK_EQUAL(wallet->mapWallet.count(mempool_tx.GetHash()), 1); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(mempool_tx.GetHash()), 1U); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 2b45c6a536..89737ca7b5 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -21,6 +21,7 @@ #include <script/descriptor.h> #include <script/script.h> #include <script/signingprovider.h> +#include <txmempool.h> #include <util/bip32.h> #include <util/check.h> #include <util/error.h> @@ -153,7 +154,7 @@ void UnloadWallet(std::shared_ptr<CWallet>&& wallet) std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) { try { - if (!CWallet::Verify(chain, location, false, error, warnings)) { + if (!CWallet::Verify(chain, location, error, warnings)) { error = Untranslated("Wallet file verification failed.") + Untranslated(" ") + error; return nullptr; } @@ -195,7 +196,7 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& } // Wallet::Verify will check if we're trying to create a wallet with a duplicate name. - if (!CWallet::Verify(chain, location, false, error, warnings)) { + if (!CWallet::Verify(chain, location, error, warnings)) { error = Untranslated("Wallet file verification failed.") + Untranslated(" ") + error; return WalletCreationStatus::CREATION_FAILED; } @@ -1100,23 +1101,52 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmatio MarkInputsDirty(ptx); } -void CWallet::transactionAddedToMempool(const CTransactionRef& ptx) { +void CWallet::transactionAddedToMempool(const CTransactionRef& tx) { LOCK(cs_wallet); - CWalletTx::Confirmation confirm(CWalletTx::Status::UNCONFIRMED, /* block_height */ 0, {}, /* nIndex */ 0); - SyncTransaction(ptx, confirm); + SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /* block height */ 0, /* block hash */ {}, /* index */ 0}); - auto it = mapWallet.find(ptx->GetHash()); + auto it = mapWallet.find(tx->GetHash()); if (it != mapWallet.end()) { it->second.fInMempool = true; } } -void CWallet::transactionRemovedFromMempool(const CTransactionRef &ptx) { +void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) { LOCK(cs_wallet); - auto it = mapWallet.find(ptx->GetHash()); + auto it = mapWallet.find(tx->GetHash()); if (it != mapWallet.end()) { it->second.fInMempool = false; } + // Handle transactions that were removed from the mempool because they + // conflict with transactions in a newly connected block. + if (reason == MemPoolRemovalReason::CONFLICT) { + // Call SyncNotifications, so external -walletnotify notifications will + // be triggered for these transactions. Set Status::UNCONFIRMED instead + // of Status::CONFLICTED for a few reasons: + // + // 1. The transactionRemovedFromMempool callback does not currently + // provide the conflicting block's hash and height, and for backwards + // compatibility reasons it may not be not safe to store conflicted + // wallet transactions with a null block hash. See + // https://github.com/bitcoin/bitcoin/pull/18600#discussion_r420195993. + // 2. For most of these transactions, the wallet's internal conflict + // detection in the blockConnected handler will subsequently call + // MarkConflicted and update them with CONFLICTED status anyway. This + // applies to any wallet transaction that has inputs spent in the + // block, or that has ancestors in the wallet with inputs spent by + // the block. + // 3. Longstanding behavior since the sync implementation in + // https://github.com/bitcoin/bitcoin/pull/9371 and the prior sync + // implementation before that was to mark these transactions + // unconfirmed rather than conflicted. + // + // Nothing described above should be seen as an unchangeable requirement + // when improving this code in the future. The wallet's heuristics for + // distinguishing between conflicted and unconfirmed transactions are + // imperfect, and could be improved in general, see + // https://github.com/bitcoin-core/bitcoin-devwiki/wiki/Wallet-Transaction-Conflict-Tracking + SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /* block height */ 0, /* block hash */ {}, /* index */ 0}); + } } void CWallet::blockConnected(const CBlock& block, int height) @@ -1127,9 +1157,8 @@ void CWallet::blockConnected(const CBlock& block, int height) m_last_block_processed_height = height; m_last_block_processed = block_hash; for (size_t index = 0; index < block.vtx.size(); index++) { - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, height, block_hash, index); - SyncTransaction(block.vtx[index], confirm); - transactionRemovedFromMempool(block.vtx[index]); + SyncTransaction(block.vtx[index], {CWalletTx::Status::CONFIRMED, height, block_hash, (int)index}); + transactionRemovedFromMempool(block.vtx[index], MemPoolRemovalReason::BLOCK); } } @@ -1144,8 +1173,7 @@ void CWallet::blockDisconnected(const CBlock& block, int height) m_last_block_processed_height = height - 1; m_last_block_processed = block.hashPrevBlock; for (const CTransactionRef& ptx : block.vtx) { - CWalletTx::Confirmation confirm(CWalletTx::Status::UNCONFIRMED, /* block_height */ 0, {}, /* nIndex */ 0); - SyncTransaction(ptx, confirm); + SyncTransaction(ptx, {CWalletTx::Status::UNCONFIRMED, /* block height */ 0, /* block hash */ {}, /* index */ 0}); } } @@ -1685,8 +1713,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc break; } for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, block_height, block_hash, posInBlock); - SyncTransaction(block.vtx[posInBlock], confirm, fUpdate); + SyncTransaction(block.vtx[posInBlock], {CWalletTx::Status::CONFIRMED, block_height, block_hash, (int)posInBlock}, fUpdate); } // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; @@ -1982,10 +2009,6 @@ void CWallet::ResendWalletTransactions() nNextResend = GetTime() + (12 * 60 * 60) + GetRand(24 * 60 * 60); if (fFirst) return; - // Only do it if there's been a new block since last time - if (m_best_block_time < nLastResend) return; - nLastResend = GetTime(); - int submitted_tx_count = 0; { // cs_wallet scope @@ -3654,7 +3677,7 @@ std::vector<std::string> CWallet::GetDestValues(const std::string& prefix) const return values; } -bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, bool salvage_wallet, bilingual_str& error_string, std::vector<bilingual_str>& warnings) +bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error_string, std::vector<bilingual_str>& warnings) { // Do some checking on wallet path. It should be either a: // @@ -3694,16 +3717,7 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b return false; } - if (salvage_wallet) { - // Recover readable keypairs: - CWallet dummyWallet(&chain, WalletLocation(), WalletDatabase::CreateDummy()); - std::string backup_filename; - if (!WalletBatch::Recover(wallet_path, (void *)&dummyWallet, WalletBatch::RecoverKeysOnlyFilter, backup_filename)) { - return false; - } - } - - return WalletBatch::VerifyDatabaseFile(wallet_path, warnings, error_string); + return WalletBatch::VerifyDatabaseFile(wallet_path, error_string); } std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings, uint64_t wallet_creation_flags) @@ -4366,7 +4380,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() for (bool internal : {false, true}) { for (OutputType t : OUTPUT_TYPES) { - auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, t, internal)); + auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, internal)); if (IsCrypted()) { if (IsLocked()) { throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); @@ -4375,7 +4389,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); } } - spk_manager->SetupDescriptorGeneration(master_key); + spk_manager->SetupDescriptorGeneration(master_key, t); uint256 id = spk_manager->GetID(); m_spk_managers[id] = std::move(spk_manager); SetActiveScriptPubKeyMan(id, t, internal); @@ -4388,7 +4402,7 @@ void CWallet::SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool interna WalletLogPrintf("Setting spkMan to active: id = %s, type = %d, internal = %d\n", id.ToString(), static_cast<int>(type), static_cast<int>(internal)); auto& spk_mans = internal ? m_internal_spk_managers : m_external_spk_managers; auto spk_man = m_spk_managers.at(id).get(); - spk_man->SetType(type, internal); + spk_man->SetInternal(internal); spk_mans[type] = spk_man; if (!memonly) { diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 8f624c25d7..67331dc3be 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -631,7 +631,6 @@ private: std::atomic<bool> fScanningWallet{false}; // controlled by WalletRescanReserver std::atomic<int64_t> m_scanning_start{0}; std::atomic<double> m_scanning_progress{0}; - std::mutex mutexScanning; friend class WalletRescanReserver; //! the current wallet version: clients below this version are not able to load the wallet @@ -641,7 +640,6 @@ private: int nWalletMaxVersion GUARDED_BY(cs_wallet) = FEATURE_BASE; int64_t nNextResend = 0; - int64_t nLastResend = 0; bool fBroadcastTransactions = false; // Local time that the tip block was received. Used to schedule wallet rebroadcasts. std::atomic<int64_t> m_best_block_time {0}; @@ -871,8 +869,10 @@ public: std::vector<std::string> GetDestValues(const std::string& prefix) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Holds a timestamp at which point the wallet is scheduled (externally) to be relocked. Caller must arrange for actual relocking to occur via Lock(). - int64_t nRelockTime = 0; + int64_t nRelockTime GUARDED_BY(cs_wallet){0}; + // Used to prevent concurrent calls to walletpassphrase RPC. + Mutex m_unlock_mutex; bool Unlock(const SecureString& strWalletPassphrase, bool accept_no_keys = false); bool ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, const SecureString& strNewWalletPassphrase); bool EncryptWallet(const SecureString& strWalletPassphrase); @@ -921,7 +921,7 @@ public: uint256 last_failed_block; }; ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, Optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate); - void transactionRemovedFromMempool(const CTransactionRef &ptx) override; + void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) override; void ReacceptWalletTransactions() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ResendWalletTransactions(); struct Balance { @@ -1135,7 +1135,7 @@ public: bool MarkReplaced(const uint256& originalHash, const uint256& newHash); //! Verify wallet naming and perform salvage on the wallet if required - static bool Verify(interfaces::Chain& chain, const WalletLocation& location, bool salvage_wallet, bilingual_str& error_string, std::vector<bilingual_str>& warnings); + static bool Verify(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error_string, std::vector<bilingual_str>& warnings); /* Initializes the wallet, returns a new CWallet instance or a null pointer in case of an error */ static std::shared_ptr<CWallet> CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings, uint64_t wallet_creation_flags = 0); @@ -1286,13 +1286,11 @@ public: bool reserve() { assert(!m_could_reserve); - std::lock_guard<std::mutex> lock(m_wallet.mutexScanning); - if (m_wallet.fScanningWallet) { + if (m_wallet.fScanningWallet.exchange(true)) { return false; } m_wallet.m_scanning_start = GetTimeMillis(); m_wallet.m_scanning_progress = 0; - m_wallet.fScanningWallet = true; m_could_reserve = true; return true; } @@ -1304,7 +1302,6 @@ public: ~WalletRescanReserver() { - std::lock_guard<std::mutex> lock(m_wallet.mutexScanning); if (m_could_reserve) { m_wallet.fScanningWallet = false; } diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 98597bdb0f..cb516f70f0 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -10,6 +10,7 @@ #include <protocol.h> #include <serialize.h> #include <sync.h> +#include <util/bip32.h> #include <util/system.h> #include <util/time.h> #include <wallet/wallet.h> @@ -17,8 +18,6 @@ #include <atomic> #include <string> -#include <boost/thread.hpp> - namespace DBKeys { const std::string ACENTRY{"acentry"}; const std::string ACTIVEEXTERNALSPK{"activeexternalspk"}; @@ -115,8 +114,19 @@ bool WalletBatch::WriteCryptedKey(const CPubKey& vchPubKey, return false; } - if (!WriteIC(std::make_pair(DBKeys::CRYPTED_KEY, vchPubKey), vchCryptedSecret, false)) { - return false; + // Compute a checksum of the encrypted key + uint256 checksum = Hash(vchCryptedSecret.begin(), vchCryptedSecret.end()); + + const auto key = std::make_pair(DBKeys::CRYPTED_KEY, vchPubKey); + if (!WriteIC(key, std::make_pair(vchCryptedSecret, checksum), false)) { + // It may already exist, so try writing just the checksum + std::vector<unsigned char> val; + if (!m_batch.Read(key, val)) { + return false; + } + if (!WriteIC(key, std::make_pair(val, checksum), true)) { + return false; + } } EraseIC(std::make_pair(DBKeys::KEY, vchPubKey)); return true; @@ -245,6 +255,7 @@ public: std::map<uint256, DescriptorCache> m_descriptor_caches; std::map<std::pair<uint256, CKeyID>, CKey> m_descriptor_keys; std::map<std::pair<uint256, CKeyID>, std::pair<CPubKey, std::vector<unsigned char>>> m_descriptor_crypt_keys; + std::map<uint160, CHDChain> m_hd_chains; CWalletScanState() { } @@ -397,9 +408,21 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } std::vector<unsigned char> vchPrivKey; ssValue >> vchPrivKey; + + // Get the checksum and check it + bool checksum_valid = false; + if (!ssValue.eof()) { + uint256 checksum; + ssValue >> checksum; + if ((checksum_valid = Hash(vchPrivKey.begin(), vchPrivKey.end()) != checksum)) { + strErr = "Error reading wallet database: Crypted key corrupt"; + return false; + } + } + wss.nCKeys++; - if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCryptedKey(vchPubKey, vchPrivKey)) + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCryptedKey(vchPubKey, vchPrivKey, checksum_valid)) { strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCryptedKey failed"; return false; @@ -412,6 +435,65 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, ssValue >> keyMeta; wss.nKeyMeta++; pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); + + // Extract some CHDChain info from this metadata if it has any + if (keyMeta.nVersion >= CKeyMetadata::VERSION_WITH_HDDATA && !keyMeta.hd_seed_id.IsNull() && keyMeta.hdKeypath.size() > 0) { + // Get the path from the key origin or from the path string + // Not applicable when path is "s" as that indicates a seed + bool internal = false; + uint32_t index = 0; + if (keyMeta.hdKeypath != "s") { + std::vector<uint32_t> path; + if (keyMeta.has_key_origin) { + // We have a key origin, so pull it from its path vector + path = keyMeta.key_origin.path; + } else { + // No key origin, have to parse the string + if (!ParseHDKeypath(keyMeta.hdKeypath, path)) { + strErr = "Error reading wallet database: keymeta with invalid HD keypath"; + return false; + } + } + + // Extract the index and internal from the path + // Path string is m/0'/k'/i' + // Path vector is [0', k', i'] (but as ints OR'd with the hardened bit + // k == 0 for external, 1 for internal. i is the index + if (path.size() != 3) { + strErr = "Error reading wallet database: keymeta found with unexpected path"; + return false; + } + if (path[0] != 0x80000000) { + strErr = strprintf("Unexpected path index of 0x%08x (expected 0x80000000) for the element at index 0", path[0]); + return false; + } + if (path[1] != 0x80000000 && path[1] != (1 | 0x80000000)) { + strErr = strprintf("Unexpected path index of 0x%08x (expected 0x80000000 or 0x80000001) for the element at index 1", path[1]); + return false; + } + if ((path[2] & 0x80000000) == 0) { + strErr = strprintf("Unexpected path index of 0x%08x (expected to be greater than or equal to 0x80000000)", path[2]); + return false; + } + internal = path[1] == (1 | 0x80000000); + index = path[2] & ~0x80000000; + } + + // Insert a new CHDChain, or get the one that already exists + auto ins = wss.m_hd_chains.emplace(keyMeta.hd_seed_id, CHDChain()); + CHDChain& chain = ins.first->second; + if (ins.second) { + // For new chains, we want to default to VERSION_HD_BASE until we see an internal + chain.nVersion = CHDChain::VERSION_HD_BASE; + chain.seed_id = keyMeta.hd_seed_id; + } + if (internal) { + chain.nVersion = CHDChain::VERSION_HD_CHAIN_SPLIT; + chain.nInternalChainCounter = std::max(chain.nInternalChainCounter, index); + } else { + chain.nExternalChainCounter = std::max(chain.nExternalChainCounter, index); + } + } } else if (strType == DBKeys::WATCHMETA) { CScript script; ssKey >> script; @@ -588,6 +670,13 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, return true; } +bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr) +{ + CWalletScanState dummy_wss; + LOCK(pwallet->cs_wallet); + return ReadKeyValue(pwallet, ssKey, ssValue, dummy_wss, strType, strErr); +} + bool WalletBatch::IsKeyType(const std::string& strType) { return (strType == DBKeys::KEY || @@ -654,11 +743,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) pwallet->WalletLogPrintf("%s\n", strErr); } pcursor->close(); - } - catch (const boost::thread_interrupted&) { - throw; - } - catch (...) { + } catch (...) { result = DBErrors::CORRUPT; } @@ -735,6 +820,20 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) result = DBErrors::CORRUPT; } + // Set the inactive chain + if (wss.m_hd_chains.size() > 0) { + LegacyScriptPubKeyMan* legacy_spkm = pwallet->GetLegacyScriptPubKeyMan(); + if (!legacy_spkm) { + pwallet->WalletLogPrintf("Inactive HD Chains found but no Legacy ScriptPubKeyMan\n"); + return DBErrors::CORRUPT; + } + for (const auto& chain_pair : wss.m_hd_chains) { + if (chain_pair.first != pwallet->GetLegacyScriptPubKeyMan()->GetHDChain().seed_id) { + pwallet->GetLegacyScriptPubKeyMan()->AddInactiveHDChain(chain_pair.second); + } + } + } + return result; } @@ -782,11 +881,7 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWal } } pcursor->close(); - } - catch (const boost::thread_interrupted&) { - throw; - } - catch (...) { + } catch (...) { result = DBErrors::CORRUPT; } @@ -878,53 +973,14 @@ void MaybeCompactWalletDB() fOneThread = false; } -// -// Try to (very carefully!) recover wallet file if there is a problem. -// -bool WalletBatch::Recover(const fs::path& wallet_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& out_backup_filename) -{ - return BerkeleyBatch::Recover(wallet_path, callbackDataIn, recoverKVcallback, out_backup_filename); -} - -bool WalletBatch::Recover(const fs::path& wallet_path, std::string& out_backup_filename) -{ - // recover without a key filter callback - // results in recovering all record types - return WalletBatch::Recover(wallet_path, nullptr, nullptr, out_backup_filename); -} - -bool WalletBatch::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDataStream ssValue) -{ - CWallet *dummyWallet = reinterpret_cast<CWallet*>(callbackData); - CWalletScanState dummyWss; - std::string strType, strErr; - bool fReadOK; - { - // Required in LoadKeyMetadata(): - LOCK(dummyWallet->cs_wallet); - fReadOK = ReadKeyValue(dummyWallet, ssKey, ssValue, - dummyWss, strType, strErr); - } - if (!IsKeyType(strType) && strType != DBKeys::HDCHAIN) { - return false; - } - if (!fReadOK) - { - LogPrintf("WARNING: WalletBatch::Recover skipping %s: %s\n", strType, strErr); - return false; - } - - return true; -} - bool WalletBatch::VerifyEnvironment(const fs::path& wallet_path, bilingual_str& errorStr) { return BerkeleyBatch::VerifyEnvironment(wallet_path, errorStr); } -bool WalletBatch::VerifyDatabaseFile(const fs::path& wallet_path, std::vector<bilingual_str>& warnings, bilingual_str& errorStr) +bool WalletBatch::VerifyDatabaseFile(const fs::path& wallet_path, bilingual_str& errorStr) { - return BerkeleyBatch::VerifyDatabaseFile(wallet_path, warnings, errorStr, WalletBatch::Recover); + return BerkeleyBatch::VerifyDatabaseFile(wallet_path, errorStr); } bool WalletBatch::WriteDestData(const std::string &address, const std::string &key, const std::string &value) diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index e2bf229c68..b95ed24d12 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -98,15 +98,13 @@ public: int nVersion; CHDChain() { SetNull(); } - ADD_SERIALIZE_METHODS; - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) + + SERIALIZE_METHODS(CHDChain, obj) { - READWRITE(this->nVersion); - READWRITE(nExternalChainCounter); - READWRITE(seed_id); - if (this->nVersion >= VERSION_HD_CHAIN_SPLIT) - READWRITE(nInternalChainCounter); + READWRITE(obj.nVersion, obj.nExternalChainCounter, obj.seed_id); + if (obj.nVersion >= VERSION_HD_CHAIN_SPLIT) { + READWRITE(obj.nInternalChainCounter); + } } void SetNull() @@ -116,6 +114,11 @@ public: nInternalChainCounter = 0; seed_id.SetNull(); } + + bool operator==(const CHDChain& chain) const + { + return seed_id == chain.seed_id; + } }; class CKeyMetadata @@ -142,21 +145,16 @@ public: nCreateTime = nCreateTime_; } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(this->nVersion); - READWRITE(nCreateTime); - if (this->nVersion >= VERSION_WITH_HDDATA) - { - READWRITE(hdKeypath); - READWRITE(hd_seed_id); + SERIALIZE_METHODS(CKeyMetadata, obj) + { + READWRITE(obj.nVersion, obj.nCreateTime); + if (obj.nVersion >= VERSION_WITH_HDDATA) { + READWRITE(obj.hdKeypath, obj.hd_seed_id); } - if (this->nVersion >= VERSION_WITH_KEY_ORIGIN) + if (obj.nVersion >= VERSION_WITH_KEY_ORIGIN) { - READWRITE(key_origin); - READWRITE(has_key_origin); + READWRITE(obj.key_origin); + READWRITE(obj.has_key_origin); } } @@ -263,18 +261,12 @@ public: DBErrors FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWalletTx>& vWtx); DBErrors ZapWalletTx(std::list<CWalletTx>& vWtx); DBErrors ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut); - /* Try to (very carefully!) recover wallet database (with a possible key type filter) */ - static bool Recover(const fs::path& wallet_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& out_backup_filename); - /* Recover convenience-function to bypass the key filter callback, called when verify fails, recovers everything */ - static bool Recover(const fs::path& wallet_path, std::string& out_backup_filename); - /* Recover filter (used as callback), will only let keys (cryptographical keys) as KV/key-type pass through */ - static bool RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDataStream ssValue); /* Function to determine if a certain KV/key-type is a key (cryptographical key) type */ static bool IsKeyType(const std::string& strType); /* verifies the database environment */ static bool VerifyEnvironment(const fs::path& wallet_path, bilingual_str& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const fs::path& wallet_path, std::vector<bilingual_str>& warnings, bilingual_str& errorStr); + static bool VerifyDatabaseFile(const fs::path& wallet_path, bilingual_str& errorStr); //! write the hdchain model (external chain child index counter) bool WriteHDChain(const CHDChain& chain); @@ -294,4 +286,7 @@ private: //! Compacts BDB state so that wallet.dat is self-contained (if there are changes) void MaybeCompactWalletDB(); +//! Unserialize a given Key-Value pair and load it into the wallet +bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr); + #endif // BITCOIN_WALLET_WALLETDB_H diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 522efaa884..be07c28503 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -5,6 +5,7 @@ #include <fs.h> #include <util/system.h> #include <util/translation.h> +#include <wallet/salvage.h> #include <wallet/wallet.h> #include <wallet/walletutil.h> @@ -103,6 +104,27 @@ static void WalletShowInfo(CWallet* wallet_instance) tfm::format(std::cout, "Address Book: %zu\n", wallet_instance->m_address_book.size()); } +static bool SalvageWallet(const fs::path& path) +{ + // Create a Database handle to allow for the db to be initialized before recovery + std::unique_ptr<WalletDatabase> database = WalletDatabase::Create(path); + + // Initialize the environment before recovery + bilingual_str error_string; + try { + WalletBatch::VerifyEnvironment(path, error_string); + } catch (const fs::filesystem_error& e) { + error_string = Untranslated(strprintf("Error loading wallet. %s", fsbridge::get_filesystem_error_message(e))); + } + if (!error_string.original.empty()) { + tfm::format(std::cerr, "Failed to open wallet for salvage :%s\n", error_string.original); + return false; + } + + // Perform the recovery + return RecoverDatabaseFile(path); +} + bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) { fs::path path = fs::absolute(name, GetWalletDir()); @@ -113,7 +135,7 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) WalletShowInfo(wallet_instance.get()); wallet_instance->Flush(true); } - } else if (command == "info") { + } else if (command == "info" || command == "salvage") { if (!fs::exists(path)) { tfm::format(std::cerr, "Error: no wallet file at %s\n", name); return false; @@ -123,10 +145,15 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) tfm::format(std::cerr, "%s\nError loading %s. Is wallet being used by other process?\n", error.original, name); return false; } - std::shared_ptr<CWallet> wallet_instance = LoadWallet(name, path); - if (!wallet_instance) return false; - WalletShowInfo(wallet_instance.get()); - wallet_instance->Flush(true); + + if (command == "info") { + std::shared_ptr<CWallet> wallet_instance = LoadWallet(name, path); + if (!wallet_instance) return false; + WalletShowInfo(wallet_instance.get()); + wallet_instance->Flush(true); + } else if (command == "salvage") { + return SalvageWallet(path); + } } else { tfm::format(std::cerr, "Invalid command: %s\n", command); return false; diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index 599b1a9f5a..a4e4fda8a1 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -98,26 +98,22 @@ public: int32_t next_index = 0; // Position of the next item to generate DescriptorCache cache; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - if (ser_action.ForRead()) { - std::string desc; - std::string error; - READWRITE(desc); - FlatSigningProvider keys; - descriptor = Parse(desc, keys, error, true); - if (!descriptor) { - throw std::ios_base::failure("Invalid descriptor: " + error); - } - } else { - READWRITE(descriptor->ToString()); + void DeserializeDescriptor(const std::string& str) + { + std::string error; + FlatSigningProvider keys; + descriptor = Parse(str, keys, error, true); + if (!descriptor) { + throw std::ios_base::failure("Invalid descriptor: " + error); } - READWRITE(creation_time); - READWRITE(next_index); - READWRITE(range_start); - READWRITE(range_end); + } + + SERIALIZE_METHODS(WalletDescriptor, obj) + { + std::string descriptor_str; + SER_WRITE(obj, descriptor_str = obj.descriptor->ToString()); + READWRITE(descriptor_str, obj.creation_time, obj.next_index, obj.range_start, obj.range_end); + SER_READ(obj, obj.DeserializeDescriptor(descriptor_str)); } WalletDescriptor() {} diff --git a/test/README.md b/test/README.md index e1dab92a06..b036a66f67 100644 --- a/test/README.md +++ b/test/README.md @@ -225,6 +225,10 @@ gdb /home/example/bitcoind <pid> Note: gdb attach step may require ptrace_scope to be modified, or `sudo` preceding the `gdb`. See this link for considerations: https://www.kernel.org/doc/Documentation/security/Yama.txt +Often while debugging rpc calls from functional tests, the test might reach timeout before +process can return a response. Use `--timeout-factor 0` to disable all rpc timeouts for that partcular +functional test. Ex: `test/functional/wallet_hd.py --timeout-factor 0`. + ##### Profiling An easy way to profile node performance during functional tests is provided @@ -257,6 +261,7 @@ Use the `-v` option for verbose output. | Lint test | Dependency | Version [used by CI](../ci/lint/04_install.sh) | Installation |-----------|:----------:|:-------------------------------------------:|-------------- | [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) | [3.7.8](https://github.com/bitcoin/bitcoin/pull/15257) | `pip3 install flake8==3.7.8` +| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) | [0.700](https://github.com/bitcoin/bitcoin/pull/18210) | `pip3 install mypy==0.700` | [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) | [0.6.0](https://github.com/bitcoin/bitcoin/pull/15166) | [details...](https://github.com/koalaman/shellcheck#installing) | [`lint-shell.sh`](lint/lint-shell.sh) | [yq](https://github.com/kislyuk/yq) | default | `pip3 install yq` | [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) | [1.15.0](https://github.com/bitcoin/bitcoin/pull/16186) | `pip3 install codespell==1.15.0` diff --git a/test/config.ini.in b/test/config.ini.in index 9687206ee1..be1bfe8752 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -7,6 +7,7 @@ [environment] PACKAGE_NAME=@PACKAGE_NAME@ +PACKAGE_BUGREPORT=@PACKAGE_BUGREPORT@ SRCDIR=@abs_top_srcdir@ BUILDDIR=@abs_top_builddir@ EXEEXT=@EXEEXT@ diff --git a/test/functional/README.md b/test/functional/README.md index 004e0afb1d..aff5f714f2 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -26,10 +26,12 @@ don't have test cases for. The Travis linter also checks this, but [possibly not in all cases](https://github.com/bitcoin/bitcoin/pull/14884#discussion_r239585126). - See [the python lint script](/test/lint/lint-python.sh) that checks for violations that could lead to bugs and issues in the test code. +- Use [type hints](https://docs.python.org/3/library/typing.html) in your code to improve code readability + and to detect possible bugs earlier. - Avoid wildcard imports - Use a module-level docstring to describe what the test is testing, and how it is testing it. -- When subclassing the BitcoinTestFramwork, place overrides for the +- When subclassing the BitcoinTestFramework, place overrides for the `set_test_params()`, `add_options()` and `setup_xxxx()` methods at the top of the subclass, then locally-defined helper methods, then the `run_test()` method. - Use `'{}'.format(x)` for string formatting, not `'%s' % x`. @@ -45,7 +47,7 @@ don't have test cases for. - `rpc` for tests for individual RPC methods or features, eg `rpc_listtransactions.py` - `tool` for tests for tools, eg `tool_wallet.py` - `wallet` for tests for wallet features, eg `wallet_keypool.py` -- use an underscore to separate words +- Use an underscore to separate words - exception: for tests for specific RPCs or command line options which don't include underscores, name the test after the exact RPC or argument name, eg `rpc_decodescript.py`, not `rpc_decode_script.py` - Don't use the redundant word `test` in the name, eg `interface_zmq.py`, not `interface_zmq_test.py` diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index ae5721bec2..6e72db1d96 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -21,6 +21,7 @@ Invalid tx cases not covered here can be found by running: """ import abc +from typing import Optional from test_framework.messages import ( COutPoint, CTransaction, @@ -56,7 +57,7 @@ class BadTxTemplate: __metaclass__ = abc.ABCMeta # The expected error code given by bitcoind upon submission of the tx. - reject_reason = "" + reject_reason = "" # type: Optional[str] # Only specified if it differs from mempool acceptance error. block_reject_reason = "" diff --git a/test/functional/example_test.py b/test/functional/example_test.py index 70dfe81d4e..5d782026dc 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -15,7 +15,7 @@ from collections import defaultdict # Avoid wildcard * imports from test_framework.blocktools import (create_block, create_coinbase) -from test_framework.messages import CInv +from test_framework.messages import CInv, MSG_BLOCK from test_framework.mininode import ( P2PInterface, mininode_lock, @@ -198,7 +198,7 @@ class ExampleTest(BitcoinTestFramework): getdata_request = msg_getdata() for block in blocks: - getdata_request.inv.append(CInv(2, block)) + getdata_request.inv.append(CInv(MSG_BLOCK, block)) self.nodes[2].p2p.send_message(getdata_request) # wait_until() will loop until a predicate condition is met. Use it to test properties of the diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index 9cff79a42c..596ff206f2 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -6,7 +6,11 @@ Test various backwards compatibility scenarios. Download the previous node binaries: -contrib/devtools/previous_release.sh -b v0.19.0.1 v0.18.1 v0.17.1 +contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 + +v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py. +Due to a hardfork in regtest, it can't be used to sync nodes. + Due to RPC changes introduced in various versions the below tests won't work for older versions without some patches or workarounds. @@ -22,6 +26,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.descriptors import descsum_create from test_framework.util import ( + adjust_bitcoin_conf_for_pre_17, assert_equal, sync_blocks, sync_mempools, @@ -31,14 +36,15 @@ from test_framework.util import ( class BackwardsCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 5 + self.num_nodes = 6 # Add new version after each release: self.extra_args = [ ["-addresstype=bech32"], # Pre-release: use to mine blocks ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # Pre-release: use to receive coins, swap wallets, etc - ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.19.0.1 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.19.1 ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.18.1 - ["-nowallet", "-walletrbf=1", "-addresstype=bech32"] # v0.17.1 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.17.1 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.16.3 ] def skip_test_if_missing_module(self): @@ -49,10 +55,13 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ None, None, - 190001, + 190100, 180100, 170100, + 160300, ]) + # adapt bitcoin.conf, because older bitcoind's don't recognize config sections + adjust_bitcoin_conf_for_pre_17(self.nodes[5].bitcoinconf) self.start_nodes() @@ -65,10 +74,11 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): res = self.nodes[self.num_nodes - 1].getblockchaininfo() assert_equal(res['blocks'], 101) - node_master = self.nodes[self.num_nodes - 4] - node_v19 = self.nodes[self.num_nodes - 3] - node_v18 = self.nodes[self.num_nodes - 2] - node_v17 = self.nodes[self.num_nodes - 1] + node_master = self.nodes[self.num_nodes - 5] + node_v19 = self.nodes[self.num_nodes - 4] + node_v18 = self.nodes[self.num_nodes - 3] + node_v17 = self.nodes[self.num_nodes - 2] + node_v16 = self.nodes[self.num_nodes - 1] self.log.info("Test wallet backwards compatibility...") # Create a number of wallets and open them in older versions: @@ -167,6 +177,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): node_v19_wallets_dir = os.path.join(node_v19.datadir, "regtest/wallets") node_v18_wallets_dir = os.path.join(node_v18.datadir, "regtest/wallets") node_v17_wallets_dir = os.path.join(node_v17.datadir, "regtest/wallets") + node_v16_wallets_dir = os.path.join(node_v16.datadir, "regtest") node_master.unloadwallet("w1") node_master.unloadwallet("w2") node_v19.unloadwallet("w1_v19") @@ -174,6 +185,13 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): node_v18.unloadwallet("w1_v18") node_v18.unloadwallet("w2_v18") + # Copy wallets to v0.16 + for wallet in os.listdir(node_master_wallets_dir): + shutil.copytree( + os.path.join(node_master_wallets_dir, wallet), + os.path.join(node_v16_wallets_dir, wallet) + ) + # Copy wallets to v0.17 for wallet in os.listdir(node_master_wallets_dir): shutil.copytree( @@ -292,10 +310,17 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # assert_raises_rpc_error(-4, "Wallet loading failed.", node_v17.loadwallet, 'w3_v18') # Instead, we stop node and try to launch it with the wallet: - self.stop_node(self.num_nodes - 1) + self.stop_node(4) node_v17.assert_start_raises_init_error(["-wallet=w3_v18"], "Error: Error loading w3_v18: Wallet requires newer version of Bitcoin Core") node_v17.assert_start_raises_init_error(["-wallet=w3"], "Error: Error loading w3: Wallet requires newer version of Bitcoin Core") - self.start_node(self.num_nodes - 1) + self.start_node(4) + + # Open most recent wallet in v0.16 (no loadwallet RPC) + self.stop_node(5) + self.start_node(5, extra_args=["-wallet=w2"]) + wallet = node_v16.get_wallet_rpc("w2") + info = wallet.getwalletinfo() + assert info['keypoolsize'] == 1 self.log.info("Test wallet upgrade path...") # u1: regular wallet, created with v0.17 diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index 1a7c656274..a4dc455d57 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -71,7 +71,7 @@ class ConfArgsTest(BitcoinTestFramework): with open(inc_conf_file2_path, 'w', encoding='utf-8') as conf: conf.write('[testnet]\n') self.restart_node(0) - self.nodes[0].stop_node(expected_stderr='Warning: ' + inc_conf_file_path + ':1 Section [testnot] is not recognized.' + os.linesep + 'Warning: ' + inc_conf_file2_path + ':1 Section [testnet] is not recognized.') + self.nodes[0].stop_node(expected_stderr='Warning: ' + inc_conf_file_path + ':1 Section [testnot] is not recognized.' + os.linesep + inc_conf_file2_path + ':1 Section [testnet] is not recognized.') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('') # clear diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index 5bbdb8cda1..7b38e09bf9 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -256,7 +256,11 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): self.log.debug("Mining longer tip") block_hashes = [] while current_height + 1 > self.nodes[3].getblockcount(): - block_hashes.extend(self.nodes[3].generate(min(10, current_height + 1 - self.nodes[3].getblockcount()))) + block_hashes.extend(self.nodes[3].generatetoaddress( + nblocks=min(10, current_height + 1 - self.nodes[3].getblockcount()), + # new address to avoid mining a block that has just been invalidated + address=self.nodes[3].getnewaddress(), + )) self.log.debug("Syncing %d new blocks...", len(block_hashes)) self.sync_node3blocks(block_hashes) utxo_list = self.nodes[3].listunspent() @@ -281,5 +285,6 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): if self.restart_counts[i] == 0: self.log.warning("Node %d never crashed during utxo flush!", i) + if __name__ == "__main__": ChainstateWriteCrashTest().main() diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index d4a8f8a715..7eabf86cad 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -13,7 +13,7 @@ if uploadtarget has been reached. from collections import defaultdict import time -from test_framework.messages import CInv, msg_getdata +from test_framework.messages import CInv, MSG_BLOCK, msg_getdata from test_framework.mininode import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, mine_large_block @@ -84,7 +84,7 @@ class MaxUploadTest(BitcoinTestFramework): # the same big old block too many times (expect: disconnect) getdata_request = msg_getdata() - getdata_request.inv.append(CInv(2, big_old_block)) + getdata_request.inv.append(CInv(MSG_BLOCK, big_old_block)) max_bytes_per_day = 800*1024*1024 daily_buffer = 144 * 4000000 @@ -109,7 +109,7 @@ class MaxUploadTest(BitcoinTestFramework): # Requesting the current block on p2p_conns[1] should succeed indefinitely, # even when over the max upload target. # We'll try 800 times - getdata_request.inv = [CInv(2, big_new_block)] + getdata_request.inv = [CInv(MSG_BLOCK, big_new_block)] for i in range(800): p2p_conns[1].send_and_ping(getdata_request) assert_equal(p2p_conns[1].block_receive_map[big_new_block], i+1) @@ -117,7 +117,7 @@ class MaxUploadTest(BitcoinTestFramework): self.log.info("Peer 1 able to repeatedly download new block") # But if p2p_conns[1] tries for an old block, it gets disconnected too. - getdata_request.inv = [CInv(2, big_old_block)] + getdata_request.inv = [CInv(MSG_BLOCK, big_old_block)] p2p_conns[1].send_message(getdata_request) p2p_conns[1].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 1) @@ -138,23 +138,22 @@ class MaxUploadTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() self.log.info("Restarting node 0 with noban permission and 1MB maxuploadtarget") - self.stop_node(0) - self.start_node(0, ["-whitelist=noban@127.0.0.1", "-maxuploadtarget=1"]) + self.restart_node(0, ["-whitelist=noban@127.0.0.1", "-maxuploadtarget=1"]) # Reconnect to self.nodes[0] self.nodes[0].add_p2p_connection(TestP2PConn()) #retrieve 20 blocks which should be enough to break the 1MB limit - getdata_request.inv = [CInv(2, big_new_block)] + getdata_request.inv = [CInv(MSG_BLOCK, big_new_block)] for i in range(20): self.nodes[0].p2p.send_and_ping(getdata_request) assert_equal(self.nodes[0].p2p.block_receive_map[big_new_block], i+1) - getdata_request.inv = [CInv(2, big_old_block)] + getdata_request.inv = [CInv(MSG_BLOCK, big_old_block)] self.nodes[0].p2p.send_and_ping(getdata_request) - assert_equal(len(self.nodes[0].getpeerinfo()), 1) #node is still connected because of the whitelist + assert_equal(len(self.nodes[0].getpeerinfo()), 1) #node is still connected because of the noban permission - self.log.info("Peer still connected after trying to download old block (whitelisted)") + self.log.info("Peer still connected after trying to download old block (noban permission)") if __name__ == '__main__': MaxUploadTest().main() diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index b110a559c0..fb0c7ceed4 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -5,12 +5,14 @@ """Test the -alertnotify, -blocknotify and -walletnotify options.""" import os -from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE, keyhash_to_p2pkh from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, wait_until, connect_nodes, + disconnect_nodes, + hex_str_to_bytes, ) # Linux allow all characters other than \x00 @@ -81,8 +83,67 @@ class NotificationsTest(BitcoinTestFramework): # directory content should equal the generated transaction hashes txids_rpc = list(map(lambda t: notify_outputname(self.wallet, t['txid']), self.nodes[1].listtransactions("*", block_count))) assert_equal(sorted(txids_rpc), sorted(os.listdir(self.walletnotify_dir))) + for tx_file in os.listdir(self.walletnotify_dir): + os.remove(os.path.join(self.walletnotify_dir, tx_file)) + + # Conflicting transactions tests. Give node 0 same wallet seed as + # node 1, generate spends from node 0, and check notifications + # triggered by node 1 + self.log.info("test -walletnotify with conflicting transactions") + self.nodes[0].sethdseed(seed=self.nodes[1].dumpprivkey(keyhash_to_p2pkh(hex_str_to_bytes(self.nodes[1].getwalletinfo()['hdseedid'])[::-1]))) + self.nodes[0].rescanblockchain() + self.nodes[0].generatetoaddress(100, ADDRESS_BCRT1_UNSPENDABLE) + + # Generate transaction on node 0, sync mempools, and check for + # notification on node 1. + tx1 = self.nodes[0].sendtoaddress(address=ADDRESS_BCRT1_UNSPENDABLE, amount=1, replaceable=True) + assert_equal(tx1 in self.nodes[0].getrawmempool(), True) + self.sync_mempools() + self.expect_wallet_notify([tx1]) + + # Generate bump transaction, sync mempools, and check for bump1 + # notification. In the future, per + # https://github.com/bitcoin/bitcoin/pull/9371, it might be better + # to have notifications for both tx1 and bump1. + bump1 = self.nodes[0].bumpfee(tx1)["txid"] + assert_equal(bump1 in self.nodes[0].getrawmempool(), True) + self.sync_mempools() + self.expect_wallet_notify([bump1]) + + # Add bump1 transaction to new block, checking for a notification + # and the correct number of confirmations. + self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + self.sync_blocks() + self.expect_wallet_notify([bump1]) + assert_equal(self.nodes[1].gettransaction(bump1)["confirmations"], 1) + + # Generate a second transaction to be bumped. + tx2 = self.nodes[0].sendtoaddress(address=ADDRESS_BCRT1_UNSPENDABLE, amount=1, replaceable=True) + assert_equal(tx2 in self.nodes[0].getrawmempool(), True) + self.sync_mempools() + self.expect_wallet_notify([tx2]) + + # Bump tx2 as bump2 and generate a block on node 0 while + # disconnected, then reconnect and check for notifications on node 1 + # about newly confirmed bump2 and newly conflicted tx2. + disconnect_nodes(self.nodes[0], 1) + bump2 = self.nodes[0].bumpfee(tx2)["txid"] + self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + assert_equal(self.nodes[0].gettransaction(bump2)["confirmations"], 1) + assert_equal(tx2 in self.nodes[1].getrawmempool(), True) + connect_nodes(self.nodes[0], 1) + self.sync_blocks() + self.expect_wallet_notify([bump2, tx2]) + assert_equal(self.nodes[1].gettransaction(bump2)["confirmations"], 1) # TODO: add test for `-alertnotify` large fork notifications + def expect_wallet_notify(self, tx_ids): + wait_until(lambda: len(os.listdir(self.walletnotify_dir)) >= len(tx_ids), timeout=10) + assert_equal(sorted(notify_outputname(self.wallet, tx_id) for tx_id in tx_ids), sorted(os.listdir(self.walletnotify_dir))) + for tx_file in os.listdir(self.walletnotify_dir): + os.remove(os.path.join(self.walletnotify_dir, tx_file)) + + if __name__ == '__main__': NotificationsTest().main() diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index fdd86310c0..24c357091f 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -108,12 +108,7 @@ class SegWitTest(BitcoinTestFramework): assert tmpl['sigoplimit'] == 20000 assert tmpl['transactions'][0]['hash'] == txid assert tmpl['transactions'][0]['sigops'] == 2 - tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert tmpl['sizelimit'] == 1000000 - assert 'weightlimit' not in tmpl - assert tmpl['sigoplimit'] == 20000 - assert tmpl['transactions'][0]['hash'] == txid - assert tmpl['transactions'][0]['sigops'] == 2 + assert '!segwit' not in tmpl['rules'] self.nodes[0].generate(1) # block 162 balance_presetup = self.nodes[0].getbalance() @@ -213,6 +208,7 @@ class SegWitTest(BitcoinTestFramework): assert tmpl['sigoplimit'] == 80000 assert tmpl['transactions'][0]['txid'] == txid assert tmpl['transactions'][0]['sigops'] == 8 + assert '!segwit' in tmpl['rules'] self.nodes[0].generate(1) # Mine a block to clear the gbt cache diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 1c94305220..7530e7daf6 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -67,6 +67,7 @@ class TestBitcoinCli(BitcoinTestFramework): if self.is_wallet_compiled(): self.log.info("Test -getinfo and bitcoin-cli getwalletinfo return expected wallet info") assert_equal(cli_get_info['balance'], BALANCE) + assert 'balances' not in cli_get_info.keys() wallet_info = self.nodes[0].getwalletinfo() assert_equal(cli_get_info['keypoolsize'], wallet_info['keypoolsize']) assert_equal(cli_get_info['unlocked_until'], wallet_info['unlocked_until']) @@ -76,42 +77,60 @@ class TestBitcoinCli(BitcoinTestFramework): # Setup to test -getinfo and -rpcwallet= with multiple wallets. wallets = ['', 'Encrypted', 'secret'] - amounts = [Decimal('59.999928'), Decimal(9), Decimal(31)] + amounts = [BALANCE + Decimal('9.999928'), Decimal(9), Decimal(31)] self.nodes[0].createwallet(wallet_name=wallets[1]) self.nodes[0].createwallet(wallet_name=wallets[2]) w1 = self.nodes[0].get_wallet_rpc(wallets[0]) w2 = self.nodes[0].get_wallet_rpc(wallets[1]) w3 = self.nodes[0].get_wallet_rpc(wallets[2]) w1.walletpassphrase(password, self.rpc_timeout) + w2.encryptwallet(password) w1.sendtoaddress(w2.getnewaddress(), amounts[1]) w1.sendtoaddress(w3.getnewaddress(), amounts[2]) # Mine a block to confirm; adds a block reward (50 BTC) to the default wallet. self.nodes[0].generate(1) - self.log.info("Test -getinfo with multiple wallets loaded returns no balance") - assert_equal(set(self.nodes[0].listwallets()), set(wallets)) - assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli().keys() - self.log.info("Test -getinfo with multiple wallets and -rpcwallet returns specified wallet balance") for i in range(len(wallets)): - cli_get_info = self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[i])) + cli_get_info = self.nodes[0].cli('-getinfo', '-rpcwallet={}'.format(wallets[i])).send_cli() + assert 'balances' not in cli_get_info.keys() assert_equal(cli_get_info['balance'], amounts[i]) - self.log.info("Test -getinfo with multiple wallets and -rpcwallet=non-existing-wallet returns no balance") - assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli('-rpcwallet=does-not-exist').keys() + self.log.info("Test -getinfo with multiple wallets and -rpcwallet=non-existing-wallet returns no balances") + cli_get_info_keys = self.nodes[0].cli('-getinfo', '-rpcwallet=does-not-exist').send_cli().keys() + assert 'balance' not in cli_get_info_keys + assert 'balances' not in cli_get_info_keys - self.log.info("Test -getinfo after unloading all wallets except a non-default one returns its balance") + self.log.info("Test -getinfo with multiple wallets returns all loaded wallet names and balances") + assert_equal(set(self.nodes[0].listwallets()), set(wallets)) + cli_get_info = self.nodes[0].cli('-getinfo').send_cli() + assert 'balance' not in cli_get_info.keys() + assert_equal(cli_get_info['balances'], {k: v for k, v in zip(wallets, amounts)}) + + # Unload the default wallet and re-verify. self.nodes[0].unloadwallet(wallets[0]) + assert wallets[0] not in self.nodes[0].listwallets() + cli_get_info = self.nodes[0].cli('-getinfo').send_cli() + assert 'balance' not in cli_get_info.keys() + assert_equal(cli_get_info['balances'], {k: v for k, v in zip(wallets[1:], amounts[1:])}) + + self.log.info("Test -getinfo after unloading all wallets except a non-default one returns its balance") self.nodes[0].unloadwallet(wallets[2]) assert_equal(self.nodes[0].listwallets(), [wallets[1]]) - assert_equal(self.nodes[0].cli('-getinfo').send_cli()['balance'], amounts[1]) - - self.log.info("Test -getinfo -rpcwallet=remaining-non-default-wallet returns its balance") - assert_equal(self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[1]))['balance'], amounts[1]) - - self.log.info("Test -getinfo with -rpcwallet=unloaded wallet returns no balance") - assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[2])).keys() + cli_get_info = self.nodes[0].cli('-getinfo').send_cli() + assert 'balances' not in cli_get_info.keys() + assert_equal(cli_get_info['balance'], amounts[1]) + + self.log.info("Test -getinfo with -rpcwallet=remaining-non-default-wallet returns only its balance") + cli_get_info = self.nodes[0].cli('-getinfo', '-rpcwallet={}'.format(wallets[1])).send_cli() + assert 'balances' not in cli_get_info.keys() + assert_equal(cli_get_info['balance'], amounts[1]) + + self.log.info("Test -getinfo with -rpcwallet=unloaded wallet returns no balances") + cli_get_info = self.nodes[0].cli('-getinfo', '-rpcwallet={}'.format(wallets[2])).send_cli() + assert 'balance' not in cli_get_info_keys + assert 'balances' not in cli_get_info_keys else: self.log.info("*** Wallet not compiled; cli getwalletinfo and -getinfo wallet tests skipped") self.nodes[0].generate(1) # maintain block parity with the wallet_compiled conditional branch diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index a07dad18d6..5b7216b253 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -7,6 +7,7 @@ from decimal import Decimal from test_framework.messages import COIN +from test_framework.mininode import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -58,6 +59,7 @@ class MempoolPackagesTest(BitcoinTestFramework): def run_test(self): # Mine some blocks and have them mature. + self.nodes[0].add_p2p_connection(P2PTxInvStore()) # keep track of invs self.nodes[0].generate(101) utxo = self.nodes[0].listunspent(10) txid = utxo[0]['txid'] @@ -72,6 +74,10 @@ class MempoolPackagesTest(BitcoinTestFramework): value = sent_value chain.append(txid) + # Wait until mempool transactions have passed initial broadcast (sent inv and received getdata) + # Otherwise, getrawmempool may be inconsistent with getmempoolentry if unbroadcast changes in between + self.nodes[0].p2p.wait_for_broadcast(chain) + # Check mempool has MAX_ANCESTORS transactions in it, and descendant and ancestor # count and fees should look correct mempool = self.nodes[0].getrawmempool(True) @@ -212,6 +218,10 @@ class MempoolPackagesTest(BitcoinTestFramework): for tx in chain[:MAX_ANCESTORS_CUSTOM]: assert tx in mempool1 # TODO: more detailed check of node1's mempool (fees etc.) + # check transaction unbroadcast info (should be false if in both mempools) + mempool = self.nodes[0].getrawmempool(True) + for tx in mempool: + assert_equal(mempool[tx]['unbroadcast'], False) # TODO: test ancestor size limits diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 3969da2eb0..5d00648aed 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -84,7 +84,9 @@ class MempoolPersistTest(BitcoinTestFramework): assert_greater_than_or_equal(tx_creation_time_higher, tx_creation_time) # disconnect nodes & make a txn that remains in the unbroadcast set. - disconnect_nodes(self.nodes[0], 2) + disconnect_nodes(self.nodes[0], 1) + assert(len(self.nodes[0].getpeerinfo()) == 0) + assert(len(self.nodes[0].p2ps) == 0) self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("12")) connect_nodes(self.nodes[0], 2) @@ -157,8 +159,10 @@ class MempoolPersistTest(BitcoinTestFramework): # clear out mempool node0.generate(1) - # disconnect nodes to make a txn that remains in the unbroadcast set. - disconnect_nodes(node0, 1) + # ensure node0 doesn't have any connections + # make a transaction that will remain in the unbroadcast set + assert(len(node0.getpeerinfo()) == 0) + assert(len(node0.p2ps) == 0) node0.sendtoaddress(self.nodes[1].getnewaddress(), Decimal("12")) # shutdown, then startup with wallet disabled diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index a561f28b91..365d011157 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -16,6 +16,7 @@ from test_framework.util import ( disconnect_nodes, ) +MAX_INITIAL_BROADCAST_DELAY = 15 * 60 # 15 minutes in seconds class MempoolUnbroadcastTest(BitcoinTestFramework): def set_test_params(self): @@ -53,6 +54,13 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): txFS = node.signrawtransactionwithwallet(txF["hex"]) rpc_tx_hsh = node.sendrawtransaction(txFS["hex"]) + # check transactions are in unbroadcast using rpc + mempoolinfo = self.nodes[0].getmempoolinfo() + assert_equal(mempoolinfo['unbroadcastcount'], 2) + mempool = self.nodes[0].getrawmempool(True) + for tx in mempool: + assert_equal(mempool[tx]['unbroadcast'], True) + # check that second node doesn't have these two txns mempool = self.nodes[1].getrawmempool() assert rpc_tx_hsh not in mempool @@ -65,24 +73,30 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): connect_nodes(node, 1) # fast forward into the future & ensure that the second node has the txns - node.mockscheduler(15 * 60) # 15 min in seconds + node.mockscheduler(MAX_INITIAL_BROADCAST_DELAY) self.sync_mempools(timeout=30) mempool = self.nodes[1].getrawmempool() assert rpc_tx_hsh in mempool assert wallet_tx_hsh in mempool + # check that transactions are no longer in first node's unbroadcast set + mempool = self.nodes[0].getrawmempool(True) + for tx in mempool: + assert_equal(mempool[tx]['unbroadcast'], False) + self.log.info("Add another connection & ensure transactions aren't broadcast again") conn = node.add_p2p_connection(P2PTxInvStore()) - node.mockscheduler(15 * 60) - time.sleep(5) + node.mockscheduler(MAX_INITIAL_BROADCAST_DELAY) + time.sleep(2) # allow sufficient time for possibility of broadcast assert_equal(len(conn.get_invs()), 0) + disconnect_nodes(node, 1) + node.disconnect_p2ps() + def test_txn_removal(self): self.log.info("Test that transactions removed from mempool are removed from unbroadcast set") node = self.nodes[0] - disconnect_nodes(node, 1) - node.disconnect_p2ps # since the node doesn't have any connections, it will not receive # any GETDATAs & thus the transaction will remain in the unbroadcast set. diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 1bda167c87..86d7c78d63 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -29,8 +29,6 @@ from test_framework.util import ( assert_raises_rpc_error, connect_nodes, ) -from test_framework.script import CScriptNum - def assert_template(node, block, expect, rehash=True): if rehash: @@ -91,12 +89,6 @@ class MiningTest(BitcoinTestFramework): coinbase_tx.rehash() # round-trip the encoded bip34 block height commitment - assert_equal(CScriptNum.decode(coinbase_tx.vin[0].scriptSig), next_height) - # round-trip negative and multi-byte CScriptNums to catch python regression - assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(1500))), 1500) - assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1500))), -1500) - assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1))), -1) - block = CBlock() block.nVersion = tmpl["version"] block.hashPrevBlock = int(tmpl["previousblockhash"], 16) diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py new file mode 100755 index 0000000000..6d947ac660 --- /dev/null +++ b/test/functional/p2p_blockfilters.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Tests NODE_COMPACT_FILTERS (BIP 157/158). + +Tests that a node configured with -blockfilterindex and -peerblockfilters can serve +cfheaders and cfcheckpts. +""" + +from test_framework.messages import ( + FILTER_TYPE_BASIC, + hash256, + msg_getcfcheckpt, + msg_getcfheaders, + msg_getcfilters, + ser_uint256, + uint256_from_str, +) +from test_framework.mininode import P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + connect_nodes, + disconnect_nodes, + wait_until, +) + +class CFiltersClient(P2PInterface): + def __init__(self): + super().__init__() + # Store the cfilters received. + self.cfilters = [] + + def pop_cfilters(self): + cfilters = self.cfilters + self.cfilters = [] + return cfilters + + def on_cfilter(self, message): + """Store cfilters received in a list.""" + self.cfilters.append(message) + +class CompactFiltersTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.rpc_timeout = 480 + self.num_nodes = 2 + self.extra_args = [ + ["-blockfilterindex", "-peerblockfilters"], + ["-blockfilterindex"], + ] + + def run_test(self): + # Node 0 supports COMPACT_FILTERS, node 1 does not. + node0 = self.nodes[0].add_p2p_connection(CFiltersClient()) + node1 = self.nodes[1].add_p2p_connection(CFiltersClient()) + + # Nodes 0 & 1 share the same first 999 blocks in the chain. + self.nodes[0].generate(999) + self.sync_blocks(timeout=600) + + # Stale blocks by disconnecting nodes 0 & 1, mining, then reconnecting + disconnect_nodes(self.nodes[0], 1) + + self.nodes[0].generate(1) + wait_until(lambda: self.nodes[0].getblockcount() == 1000) + stale_block_hash = self.nodes[0].getblockhash(1000) + + self.nodes[1].generate(1001) + wait_until(lambda: self.nodes[1].getblockcount() == 2000) + + self.log.info("get cfcheckpt on chain to be re-orged out.") + request = msg_getcfcheckpt( + filter_type=FILTER_TYPE_BASIC, + stop_hash=int(stale_block_hash, 16) + ) + node0.send_and_ping(message=request) + response = node0.last_message['cfcheckpt'] + assert_equal(response.filter_type, request.filter_type) + assert_equal(response.stop_hash, request.stop_hash) + assert_equal(len(response.headers), 1) + + self.log.info("Reorg node 0 to a new chain.") + connect_nodes(self.nodes[0], 1) + self.sync_blocks(timeout=600) + + main_block_hash = self.nodes[0].getblockhash(1000) + assert main_block_hash != stale_block_hash, "node 0 chain did not reorganize" + + self.log.info("Check that peers can fetch cfcheckpt on active chain.") + tip_hash = self.nodes[0].getbestblockhash() + request = msg_getcfcheckpt( + filter_type=FILTER_TYPE_BASIC, + stop_hash=int(tip_hash, 16) + ) + node0.send_and_ping(request) + response = node0.last_message['cfcheckpt'] + assert_equal(response.filter_type, request.filter_type) + assert_equal(response.stop_hash, request.stop_hash) + + main_cfcheckpt = self.nodes[0].getblockfilter(main_block_hash, 'basic')['header'] + tip_cfcheckpt = self.nodes[0].getblockfilter(tip_hash, 'basic')['header'] + assert_equal( + response.headers, + [int(header, 16) for header in (main_cfcheckpt, tip_cfcheckpt)] + ) + + self.log.info("Check that peers can fetch cfcheckpt on stale chain.") + request = msg_getcfcheckpt( + filter_type=FILTER_TYPE_BASIC, + stop_hash=int(stale_block_hash, 16) + ) + node0.send_and_ping(request) + response = node0.last_message['cfcheckpt'] + + stale_cfcheckpt = self.nodes[0].getblockfilter(stale_block_hash, 'basic')['header'] + assert_equal( + response.headers, + [int(header, 16) for header in (stale_cfcheckpt,)] + ) + + self.log.info("Check that peers can fetch cfheaders on active chain.") + request = msg_getcfheaders( + filter_type=FILTER_TYPE_BASIC, + start_height=1, + stop_hash=int(main_block_hash, 16) + ) + node0.send_and_ping(request) + response = node0.last_message['cfheaders'] + main_cfhashes = response.hashes + assert_equal(len(main_cfhashes), 1000) + assert_equal( + compute_last_header(response.prev_header, response.hashes), + int(main_cfcheckpt, 16) + ) + + self.log.info("Check that peers can fetch cfheaders on stale chain.") + request = msg_getcfheaders( + filter_type=FILTER_TYPE_BASIC, + start_height=1, + stop_hash=int(stale_block_hash, 16) + ) + node0.send_and_ping(request) + response = node0.last_message['cfheaders'] + stale_cfhashes = response.hashes + assert_equal(len(stale_cfhashes), 1000) + assert_equal( + compute_last_header(response.prev_header, response.hashes), + int(stale_cfcheckpt, 16) + ) + + self.log.info("Check that peers can fetch cfilters.") + stop_hash = self.nodes[0].getblockhash(10) + request = msg_getcfilters( + filter_type=FILTER_TYPE_BASIC, + start_height=1, + stop_hash=int(stop_hash, 16) + ) + node0.send_message(request) + node0.sync_with_ping() + response = node0.pop_cfilters() + assert_equal(len(response), 10) + + self.log.info("Check that cfilter responses are correct.") + for cfilter, cfhash, height in zip(response, main_cfhashes, range(1, 11)): + block_hash = self.nodes[0].getblockhash(height) + assert_equal(cfilter.filter_type, FILTER_TYPE_BASIC) + assert_equal(cfilter.block_hash, int(block_hash, 16)) + computed_cfhash = uint256_from_str(hash256(cfilter.filter_data)) + assert_equal(computed_cfhash, cfhash) + + self.log.info("Check that peers can fetch cfilters for stale blocks.") + request = msg_getcfilters( + filter_type=FILTER_TYPE_BASIC, + start_height=1000, + stop_hash=int(stale_block_hash, 16) + ) + node0.send_message(request) + node0.sync_with_ping() + response = node0.pop_cfilters() + assert_equal(len(response), 1) + + cfilter = response[0] + assert_equal(cfilter.filter_type, FILTER_TYPE_BASIC) + assert_equal(cfilter.block_hash, int(stale_block_hash, 16)) + computed_cfhash = uint256_from_str(hash256(cfilter.filter_data)) + assert_equal(computed_cfhash, stale_cfhashes[999]) + + self.log.info("Requests to node 1 without NODE_COMPACT_FILTERS results in disconnection.") + requests = [ + msg_getcfcheckpt( + filter_type=FILTER_TYPE_BASIC, + stop_hash=int(main_block_hash, 16) + ), + msg_getcfheaders( + filter_type=FILTER_TYPE_BASIC, + start_height=1000, + stop_hash=int(main_block_hash, 16) + ), + msg_getcfilters( + filter_type=FILTER_TYPE_BASIC, + start_height=1000, + stop_hash=int(main_block_hash, 16) + ), + ] + for request in requests: + node1 = self.nodes[1].add_p2p_connection(P2PInterface()) + node1.send_message(request) + node1.wait_for_disconnect() + + self.log.info("Check that invalid requests result in disconnection.") + requests = [ + # Requesting too many filters results in disconnection. + msg_getcfilters( + filter_type=FILTER_TYPE_BASIC, + start_height=0, + stop_hash=int(main_block_hash, 16) + ), + # Requesting too many filter headers results in disconnection. + msg_getcfheaders( + filter_type=FILTER_TYPE_BASIC, + start_height=0, + stop_hash=int(tip_hash, 16) + ), + # Requesting unknown filter type results in disconnection. + msg_getcfcheckpt( + filter_type=255, + stop_hash=int(main_block_hash, 16) + ), + # Requesting unknown hash results in disconnection. + msg_getcfcheckpt( + filter_type=FILTER_TYPE_BASIC, + stop_hash=123456789, + ), + ] + for request in requests: + node0 = self.nodes[0].add_p2p_connection(P2PInterface()) + node0.send_message(request) + node0.wait_for_disconnect() + +def compute_last_header(prev_header, hashes): + """Compute the last filter header from a starting header and a sequence of filter hashes.""" + header = ser_uint256(prev_header) + for filter_hash in hashes: + header = hash256(ser_uint256(filter_hash) + header) + return uint256_from_str(header) + +if __name__ == '__main__': + CompactFiltersTest().main() diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 3258a38e3c..c155dda664 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -57,6 +57,29 @@ class P2PBlocksOnly(BitcoinTestFramework): self.nodes[0].p2p.wait_for_tx(txid) assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) + self.log.info('Check that txs from whitelisted peers are not rejected and relayed to others') + self.log.info("Restarting node 0 with whitelist permission and blocksonly") + self.restart_node(0, ["-persistmempool=0", "-whitelist=127.0.0.1", "-whitelistforcerelay", "-blocksonly"]) + assert_equal(self.nodes[0].getrawmempool(),[]) + first_peer = self.nodes[0].add_p2p_connection(P2PInterface()) + second_peer = self.nodes[0].add_p2p_connection(P2PInterface()) + peer_1_info = self.nodes[0].getpeerinfo()[0] + assert_equal(peer_1_info['whitelisted'], True) + assert_equal(peer_1_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool']) + peer_2_info = self.nodes[0].getpeerinfo()[1] + assert_equal(peer_2_info['whitelisted'], True) + assert_equal(peer_2_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool']) + assert_equal(self.nodes[0].testmempoolaccept([sigtx])[0]['allowed'], True) + txid = self.nodes[0].testmempoolaccept([sigtx])[0]['txid'] + + self.log.info('Check that the tx from whitelisted first_peer is relayed to others (ie.second_peer)') + with self.nodes[0].assert_debug_log(["received getdata"]): + first_peer.send_message(msg_tx(FromHex(CTransaction(), sigtx))) + self.log.info('Check that the whitelisted peer is still connected after sending the transaction') + assert_equal(first_peer.is_connected, True) + second_peer.wait_for_tx(txid) + assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) + self.log.info("Whitelisted peer's transaction is accepted and relayed") if __name__ == '__main__': P2PBlocksOnly().main() diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 66e6f8c424..d77a744758 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -10,7 +10,7 @@ Version 2 compact blocks are post-segwit (wtxids) import random from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment -from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_no_witness_block, msg_no_witness_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_block, msg_blocktxn, MSG_WITNESS_FLAG, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex +from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_no_witness_block, msg_no_witness_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_block, msg_blocktxn, MSG_BLOCK, MSG_CMPCT_BLOCK, MSG_WITNESS_FLAG, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex from test_framework.mininode import mininode_lock, P2PInterface from test_framework.script import CScript, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework @@ -44,7 +44,7 @@ class TestP2PConn(P2PInterface): def on_inv(self, message): for x in self.last_message["inv"].inv: - if x.type == 2: + if x.type == MSG_BLOCK: self.block_announced = True self.announced_blockhashes.add(x.hash) @@ -307,7 +307,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch the compact block using a normal non-announce getdata with mininode_lock: test_node.clear_block_announcement() - inv = CInv(4, block_hash) # 4 == "CompactBlock" + inv = CInv(MSG_CMPCT_BLOCK, block_hash) test_node.send_message(msg_getdata([inv])) wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) @@ -380,7 +380,7 @@ class CompactBlocksTest(BitcoinTestFramework): block = self.build_block_on_tip(node, segwit=segwit) if announce == "inv": - test_node.send_message(msg_inv([CInv(2, block.sha256)])) + test_node.send_message(msg_inv([CInv(MSG_BLOCK, block.sha256)])) wait_until(lambda: "getheaders" in test_node.last_message, timeout=30, lock=mininode_lock) test_node.send_header_for_blocks([block]) else: @@ -564,7 +564,8 @@ class CompactBlocksTest(BitcoinTestFramework): # We should receive a getdata request test_node.wait_for_getdata([block.sha256], timeout=10) - assert test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG + assert test_node.last_message["getdata"].inv[0].type == MSG_BLOCK or \ + test_node.last_message["getdata"].inv[0].type == MSG_BLOCK | MSG_WITNESS_FLAG # Deliver the block if version == 2: @@ -633,7 +634,7 @@ class CompactBlocksTest(BitcoinTestFramework): wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) test_node.clear_block_announcement() - test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock) test_node.clear_block_announcement() @@ -642,7 +643,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.clear_block_announcement() with mininode_lock: test_node.last_message.pop("block", None) - test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) wait_until(lambda: "block" in test_node.last_message, timeout=30, lock=mininode_lock) with mininode_lock: test_node.last_message["block"].block.calc_sha256() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index 4f242bd94a..805cb1e84f 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -7,7 +7,7 @@ from decimal import Decimal import time -from test_framework.messages import msg_feefilter +from test_framework.messages import MSG_TX, msg_feefilter from test_framework.mininode import mininode_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework @@ -31,7 +31,7 @@ class TestP2PConn(P2PInterface): def on_inv(self, message): for i in message.inv: - if (i.type == 1): + if (i.type == MSG_TX): self.txinvs.append(hashToHex(i.hash)) def clear_invs(self): diff --git a/test/functional/p2p_fingerprint.py b/test/functional/p2p_fingerprint.py index c9fbb830c8..d743abe681 100755 --- a/test/functional/p2p_fingerprint.py +++ b/test/functional/p2p_fingerprint.py @@ -11,7 +11,7 @@ the node should pretend that it does not have it to avoid fingerprinting. import time from test_framework.blocktools import (create_block, create_coinbase) -from test_framework.messages import CInv +from test_framework.messages import CInv, MSG_BLOCK from test_framework.mininode import ( P2PInterface, msg_headers, @@ -48,7 +48,7 @@ class P2PFingerprintTest(BitcoinTestFramework): # Send a getdata request for a given block hash def send_block_request(self, block_hash, node): msg = msg_getdata() - msg.inv.append(CInv(2, block_hash)) # 2 == "Block" + msg.inv.append(CInv(MSG_BLOCK, block_hash)) node.send_message(msg) # Send a getheaders request for a given single block hash diff --git a/test/functional/p2p_getdata.py b/test/functional/p2p_getdata.py new file mode 100755 index 0000000000..d1b11c2c61 --- /dev/null +++ b/test/functional/p2p_getdata.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test GETDATA processing behavior""" +from collections import defaultdict + +from test_framework.messages import ( + CInv, + msg_getdata, +) +from test_framework.mininode import P2PInterface +from test_framework.test_framework import BitcoinTestFramework + + +class P2PStoreBlock(P2PInterface): + def __init__(self): + super().__init__() + self.blocks = defaultdict(int) + + def on_block(self, message): + message.block.calc_sha256() + self.blocks[message.block.sha256] += 1 + + +class GetdataTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + p2p_block_store = self.nodes[0].add_p2p_connection(P2PStoreBlock()) + + self.log.info("test that an invalid GETDATA doesn't prevent processing of future messages") + + # Send invalid message and verify that node responds to later ping + invalid_getdata = msg_getdata() + invalid_getdata.inv.append(CInv(t=0, h=0)) # INV type 0 is invalid. + p2p_block_store.send_and_ping(invalid_getdata) + + # Check getdata still works by fetching tip block + best_block = int(self.nodes[0].getbestblockhash(), 16) + good_getdata = msg_getdata() + good_getdata.inv.append(CInv(t=2, h=best_block)) + p2p_block_store.send_and_ping(good_getdata) + p2p_block_store.wait_until(lambda: self.nodes[0].p2ps[0].blocks[best_block] == 1) + + +if __name__ == '__main__': + GetdataTest().main() diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 4bd832e8f7..81302374c9 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -7,7 +7,16 @@ import asyncio import struct import sys -from test_framework import messages +from test_framework.messages import ( + CBlockHeader, + CInv, + msg_getdata, + msg_headers, + msg_inv, + msg_ping, + MSG_TX, + ser_string, +) from test_framework.mininode import ( NetworkThread, P2PDataStore, @@ -25,7 +34,7 @@ class msg_unrecognized: self.str_data = str_data.encode() if not isinstance(str_data, bytes) else str_data def serialize(self): - return messages.ser_string(self.str_data) + return ser_string(self.str_data) def __repr__(self): return "{}(data={})".format(self.msgtype, self.str_data) @@ -135,7 +144,7 @@ class InvalidMessagesTest(BitcoinTestFramework): # For some reason unknown to me, we sometimes have to push additional data to the # peer in order for it to realize a disconnect. try: - node.p2p.send_message(messages.msg_ping(nonce=123123)) + node.p2p.send_message(msg_ping(nonce=123123)) except IOError: pass @@ -158,7 +167,7 @@ class InvalidMessagesTest(BitcoinTestFramework): asyncio.run_coroutine_threadsafe(swap_magic_bytes(), NetworkThread.network_event_loop).result() with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: INVALID MESSAGESTART ping']): - conn.send_message(messages.msg_ping(nonce=0xff)) + conn.send_message(msg_ping(nonce=0xff)) conn.wait_for_disconnect(timeout=1) self.nodes[0].disconnect_p2ps() @@ -206,13 +215,13 @@ class InvalidMessagesTest(BitcoinTestFramework): def test_large_inv(self): conn = self.nodes[0].add_p2p_connection(P2PInterface()) with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (0 -> 20): message inv size() = 50001']): - msg = messages.msg_inv([messages.CInv(1, 1)] * 50001) + msg = msg_inv([CInv(MSG_TX, 1)] * 50001) conn.send_and_ping(msg) with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (20 -> 40): message getdata size() = 50001']): - msg = messages.msg_getdata([messages.CInv(1, 1)] * 50001) + msg = msg_getdata([CInv(MSG_TX, 1)] * 50001) conn.send_and_ping(msg) with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (40 -> 60): headers message size = 2001']): - msg = messages.msg_headers([messages.CBlockHeader()] * 2001) + msg = msg_headers([CBlockHeader()] * 2001) conn.send_and_ping(msg) self.nodes[0].disconnect_p2ps() diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py index 6b3436fa5f..da30ad5977 100755 --- a/test/functional/p2p_leak_tx.py +++ b/test/functional/p2p_leak_tx.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that we don't leak txs to inbound peers that we haven't yet announced to""" -from test_framework.messages import msg_getdata, CInv +from test_framework.messages import msg_getdata, CInv, MSG_TX from test_framework.mininode import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -37,7 +37,7 @@ class P2PLeakTxTest(BitcoinTestFramework): txid = gen_node.sendtoaddress(gen_node.getnewaddress(), 0.01) want_tx = msg_getdata() - want_tx.inv.append(CInv(t=1, h=int(txid, 16))) + want_tx.inv.append(CInv(t=MSG_TX, h=int(txid, 16))) inbound_peer.last_message.pop('notfound', None) inbound_peer.send_and_ping(want_tx) diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index 4f4dd6c975..ed3429a037 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -8,7 +8,7 @@ Tests that a node configured with -prune=550 signals NODE_NETWORK_LIMITED correc and that it responds to getdata requests for blocks correctly: - send a block within 288 + 2 of the tip - disconnect peers who request blocks older than that.""" -from test_framework.messages import CInv, msg_getdata, msg_verack, NODE_NETWORK_LIMITED, NODE_WITNESS +from test_framework.messages import CInv, MSG_BLOCK, msg_getdata, msg_verack, NODE_NETWORK_LIMITED, NODE_WITNESS from test_framework.mininode import P2PInterface, mininode_lock from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -31,7 +31,7 @@ class P2PIgnoreInv(P2PInterface): wait_until(test_function, timeout=timeout, lock=mininode_lock) def send_getdata_for_block(self, blockhash): getdata_request = msg_getdata() - getdata_request.inv.append(CInv(2, int(blockhash, 16))) + getdata_request.inv.append(CInv(MSG_BLOCK, int(blockhash, 16))) self.send_message(getdata_request) class NodeNetworkLimitedTest(BitcoinTestFramework): diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 6fb0fec32b..8803086213 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -22,6 +22,8 @@ from test_framework.messages import ( CTxOut, CTxWitness, MAX_BLOCK_BASE_SIZE, + MSG_BLOCK, + MSG_TX, MSG_WITNESS_FLAG, NODE_NETWORK, NODE_WITNESS, @@ -157,7 +159,7 @@ class TestP2PConn(P2PInterface): def announce_tx_and_wait_for_getdata(self, tx, timeout=60, success=True): with mininode_lock: self.last_message.pop("getdata", None) - self.send_message(msg_inv(inv=[CInv(1, tx.sha256)])) + self.send_message(msg_inv(inv=[CInv(MSG_TX, tx.sha256)])) if success: self.wait_for_getdata([tx.sha256], timeout) else: @@ -173,7 +175,7 @@ class TestP2PConn(P2PInterface): if use_header: self.send_message(msg) else: - self.send_message(msg_inv(inv=[CInv(2, block.sha256)])) + self.send_message(msg_inv(inv=[CInv(MSG_BLOCK, block.sha256)])) self.wait_for_getheaders() self.send_message(msg) self.wait_for_getdata([block.sha256]) @@ -293,7 +295,7 @@ class SegWitTest(BitcoinTestFramework): return func_wrapper - @subtest + @subtest # type: ignore def test_non_witness_transaction(self): """See if sending a regular transaction works, and create a utxo to use in later tests.""" # Mine a block with an anyone-can-spend coinbase, @@ -322,7 +324,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.append(UTXO(tx.sha256, 0, 49 * 100000000)) self.nodes[0].generate(1) - @subtest + @subtest # type: ignore def test_unnecessary_witness_before_segwit_activation(self): """Verify that blocks with witnesses are rejected before activation.""" @@ -353,7 +355,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx.sha256, 0, tx.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_block_relay(self): """Test that block requests to NODE_WITNESS peer are with MSG_WITNESS_FLAG. @@ -449,7 +451,7 @@ class SegWitTest(BitcoinTestFramework): self.old_node.announce_tx_and_wait_for_getdata(block4.vtx[0]) assert block4.sha256 not in self.old_node.getdataset - @subtest + @subtest # type: ignore def test_v0_outputs_arent_spendable(self): """Test that v0 outputs aren't spendable before segwit activation. @@ -531,7 +533,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(txid, 2, value)) - @subtest + @subtest # type: ignore def test_getblocktemplate_before_lockin(self): txid = int(self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1), 16) @@ -557,7 +559,7 @@ class SegWitTest(BitcoinTestFramework): self.nodes[0].generate(1) self.sync_blocks() - @subtest + @subtest # type: ignore def test_witness_tx_relay_before_segwit_activation(self): # Generate a transaction that doesn't require a witness, but send it @@ -576,7 +578,7 @@ class SegWitTest(BitcoinTestFramework): # Verify that if a peer doesn't set nServices to include NODE_WITNESS, # the getdata is just for the non-witness portion. self.old_node.announce_tx_and_wait_for_getdata(tx) - assert self.old_node.last_message["getdata"].inv[0].type == 1 + assert self.old_node.last_message["getdata"].inv[0].type == MSG_TX # Since we haven't delivered the tx yet, inv'ing the same tx from # a witness transaction ought not result in a getdata. @@ -599,7 +601,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx_hash, 0, tx_value)) - @subtest + @subtest # type: ignore def test_standardness_v0(self): """Test V0 txout standardness. @@ -677,7 +679,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) assert_equal(len(self.nodes[1].getrawmempool()), 0) - @subtest + @subtest # type: ignore def advance_to_segwit_active(self): """Mine enough blocks to activate segwit.""" assert not softfork_active(self.nodes[0], 'segwit') @@ -688,7 +690,7 @@ class SegWitTest(BitcoinTestFramework): assert softfork_active(self.nodes[0], 'segwit') self.segwit_active = True - @subtest + @subtest # type: ignore def test_p2sh_witness(self): """Test P2SH wrapped witness programs.""" @@ -757,7 +759,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(spend_tx.sha256, 0, spend_tx.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_witness_commitments(self): """Test witness commitments. @@ -847,7 +849,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_block_malleability(self): # Make sure that a block that has too big a virtual size @@ -887,7 +889,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)] test_witness_block(self.nodes[0], self.test_node, block, accepted=True) - @subtest + @subtest # type: ignore def test_witness_block_size(self): # TODO: Test that non-witness carrying blocks can't exceed 1MB # Skipping this test for now; this is covered in p2p-fullblocktest.py @@ -965,7 +967,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue)) - @subtest + @subtest # type: ignore def test_submit_block(self): """Test that submitblock adds the nonce automatically when possible.""" block = self.build_next_block() @@ -1001,7 +1003,7 @@ class SegWitTest(BitcoinTestFramework): # Tip should not advance! assert self.nodes[0].getbestblockhash() != block_2.hash - @subtest + @subtest # type: ignore def test_extra_witness_data(self): """Test extra witness data in a transaction.""" @@ -1074,7 +1076,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_max_witness_push_length(self): """Test that witness stack can only allow up to 520 byte pushes.""" @@ -1111,7 +1113,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop() self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_max_witness_program_length(self): """Test that witness outputs greater than 10kB can't be spent.""" @@ -1159,7 +1161,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop() self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_witness_input_length(self): """Test that vin length must match vtxinwit length.""" @@ -1241,7 +1243,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop() self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_tx_relay_after_segwit_activation(self): """Test transaction relay after segwit activation. @@ -1310,9 +1312,9 @@ class SegWitTest(BitcoinTestFramework): tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_program] # Also check that old_node gets a tx announcement, even though this is # a witness transaction. - self.old_node.wait_for_inv([CInv(1, tx2.sha256)]) # wait until tx2 was inv'ed + self.old_node.wait_for_inv([CInv(MSG_TX, tx2.sha256)]) # wait until tx2 was inv'ed test_transaction_acceptance(self.nodes[0], self.test_node, tx3, with_witness=True, accepted=True) - self.old_node.wait_for_inv([CInv(1, tx3.sha256)]) + self.old_node.wait_for_inv([CInv(MSG_TX, tx3.sha256)]) # Test that getrawtransaction returns correct witness information # hash, size, vsize @@ -1334,7 +1336,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_segwit_versions(self): """Test validity of future segwit version transactions. @@ -1416,7 +1418,7 @@ class SegWitTest(BitcoinTestFramework): # Add utxo to our list self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_premature_coinbase_witness_spend(self): block = self.build_next_block() @@ -1451,7 +1453,7 @@ class SegWitTest(BitcoinTestFramework): test_witness_block(self.nodes[0], self.test_node, block2, accepted=True) self.sync_blocks() - @subtest + @subtest # type: ignore def test_uncompressed_pubkey(self): """Test uncompressed pubkey validity in segwit transactions. @@ -1556,7 +1558,7 @@ class SegWitTest(BitcoinTestFramework): test_witness_block(self.nodes[0], self.test_node, block, accepted=True) self.utxo.append(UTXO(tx5.sha256, 0, tx5.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_signature_version_1(self): key = ECKey() @@ -1738,7 +1740,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(len(tx.vout)): self.utxo.append(UTXO(tx.sha256, i, tx.vout[i].nValue)) - @subtest + @subtest # type: ignore def test_non_standard_witness_blinding(self): """Test behavior of unnecessary witnesses in transactions does not blind the node for the transaction""" @@ -1792,7 +1794,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_non_standard_witness(self): """Test detection of non-standard P2WSH witness""" pad = chr(1).encode('latin-1') @@ -1892,7 +1894,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) - @subtest + @subtest # type: ignore def test_upgrade_after_activation(self): """Test the behavior of starting up a segwit-aware node after the softfork has activated.""" @@ -1914,7 +1916,7 @@ class SegWitTest(BitcoinTestFramework): assert_equal(self.nodes[0].getblock(block_hash), self.nodes[2].getblock(block_hash)) height -= 1 - @subtest + @subtest # type: ignore def test_witness_sigops(self): """Test sigop counting is correct inside witnesses.""" diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index a8fba306a7..481b1c1841 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -92,6 +92,7 @@ from test_framework.mininode import ( NODE_WITNESS, P2PInterface, mininode_lock, + MSG_BLOCK, msg_block, msg_getblocks, msg_getdata, @@ -120,7 +121,7 @@ class BaseNode(P2PInterface): """Request data for a list of block hashes.""" msg = msg_getdata() for x in block_hashes: - msg.inv.append(CInv(2, x)) + msg.inv.append(CInv(MSG_BLOCK, x)) self.send_message(msg) def send_get_headers(self, locator, hashstop): @@ -131,7 +132,7 @@ class BaseNode(P2PInterface): def send_block_inv(self, blockhash): msg = msg_inv() - msg.inv = [CInv(2, blockhash)] + msg.inv = [CInv(MSG_BLOCK, blockhash)] self.send_message(msg) def send_header_for_blocks(self, new_blocks): diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index a999fba818..10f5eea0e5 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -63,7 +63,7 @@ class TxDownloadTest(BitcoinTestFramework): txid = 0xdeadbeef self.log.info("Announce the txid from each incoming peer to node 0") - msg = msg_inv([CInv(t=1, h=txid)]) + msg = msg_inv([CInv(t=MSG_TX, h=txid)]) for p in self.nodes[0].p2ps: p.send_and_ping(msg) @@ -104,7 +104,7 @@ class TxDownloadTest(BitcoinTestFramework): self.log.info( "Announce the transaction to all nodes from all {} incoming peers, but never send it".format(NUM_INBOUND)) - msg = msg_inv([CInv(t=1, h=txid)]) + msg = msg_inv([CInv(t=MSG_TX, h=txid)]) for p in self.peers: p.send_and_ping(msg) @@ -135,13 +135,13 @@ class TxDownloadTest(BitcoinTestFramework): with mininode_lock: p.tx_getdata_count = 0 - p.send_message(msg_inv([CInv(t=1, h=i) for i in txids])) + p.send_message(msg_inv([CInv(t=MSG_TX, h=i) for i in txids])) wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT, lock=mininode_lock) with mininode_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT) self.log.info("Now check that if we send a NOTFOUND for a transaction, we'll get one more request") - p.send_message(msg_notfound(vec=[CInv(t=1, h=txids[0])])) + p.send_message(msg_notfound(vec=[CInv(t=MSG_TX, h=txids[0])])) wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT + 1, timeout=10, lock=mininode_lock) with mininode_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT + 1) @@ -154,7 +154,7 @@ class TxDownloadTest(BitcoinTestFramework): def test_spurious_notfound(self): self.log.info('Check that spurious notfound is ignored') - self.nodes[0].p2ps[0].send_message(msg_notfound(vec=[CInv(1, 1)])) + self.nodes[0].p2ps[0].send_message(msg_notfound(vec=[CInv(MSG_TX, 1)])) def run_test(self): # Setup the p2p connections diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index 3aaf4b9977..c323168848 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -54,7 +54,7 @@ Node1 is unused in tests 3-7: import time from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script -from test_framework.messages import CBlockHeader, CInv, msg_block, msg_headers, msg_inv +from test_framework.messages import CBlockHeader, CInv, MSG_BLOCK, msg_block, msg_headers, msg_inv from test_framework.mininode import mininode_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -210,7 +210,7 @@ class AcceptBlockTest(BitcoinTestFramework): with mininode_lock: # Clear state so we can check the getdata request test_node.last_message.pop("getdata", None) - test_node.send_message(msg_inv([CInv(2, block_h3.sha256)])) + test_node.send_message(msg_inv([CInv(MSG_BLOCK, block_h3.sha256)])) test_node.sync_with_ping() with mininode_lock: diff --git a/test/functional/rpc_getaddressinfo_label_deprecation.py b/test/functional/rpc_getaddressinfo_label_deprecation.py index 4c6b2fe5cf..09545ebce7 100755 --- a/test/functional/rpc_getaddressinfo_label_deprecation.py +++ b/test/functional/rpc_getaddressinfo_label_deprecation.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ Test deprecation of the RPC getaddressinfo `label` field. It has been -superceded by the `labels` field. +superseded by the `labels` field. """ from test_framework.test_framework import BitcoinTestFramework diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 376bb35f07..58d8c4abe1 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -28,6 +28,7 @@ from test_framework.messages import ( NODE_WITNESS, ) + def assert_net_servicesnames(servicesflag, servicenames): """Utility that checks if all flags are correctly decoded in `getpeerinfo` and `getnetworkinfo`. @@ -40,6 +41,7 @@ def assert_net_servicesnames(servicesflag, servicenames): servicesflag_generated |= getattr(test_framework.messages, 'NODE_' + servicename) assert servicesflag_generated == servicesflag + class NetTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -57,6 +59,7 @@ class NetTest(BitcoinTestFramework): self._test_getnetworkinfo() self._test_getaddednodeinfo() self._test_getpeerinfo() + self.test_service_flags() self._test_getnodeaddresses() def _test_connection_count(self): @@ -139,6 +142,11 @@ class NetTest(BitcoinTestFramework): for info in peer_info: assert_net_servicesnames(int(info[0]["services"], 0x10), info[0]["servicesnames"]) + def test_service_flags(self): + self.nodes[0].add_p2p_connection(P2PInterface(), services=(1 << 4) | (1 << 63)) + assert_equal(['UNKNOWN[2^4]', 'UNKNOWN[2^63]'], self.nodes[0].getpeerinfo()[-1]['servicesnames']) + self.nodes[0].disconnect_p2ps() + def _test_getnodeaddresses(self): self.nodes[0].add_p2p_connection(P2PInterface()) @@ -174,5 +182,6 @@ class NetTest(BitcoinTestFramework): node_addresses = self.nodes[0].getnodeaddresses(LARGE_REQUEST_COUNT) assert_greater_than(LARGE_REQUEST_COUNT, len(node_addresses)) + if __name__ == '__main__': NetTest().main() diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index b75ce15f2e..daf02fc4f3 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -20,6 +20,7 @@ import string import configparser import sys + def call_with_auth(node, user, password): url = urllib.parse.urlparse(node.url) headers = {"Authorization": "Basic " + str_to_b64str('{}:{}'.format(user, password))} @@ -64,9 +65,9 @@ class HTTPBasicsTest(BitcoinTestFramework): self.password = lines[3] with open(os.path.join(get_datadir_path(self.options.tmpdir, 0), "bitcoin.conf"), 'a', encoding='utf8') as f: - f.write(rpcauth+"\n") - f.write(rpcauth2+"\n") - f.write(rpcauth3+"\n") + f.write(rpcauth + "\n") + f.write(rpcauth2 + "\n") + f.write(rpcauth3 + "\n") with open(os.path.join(get_datadir_path(self.options.tmpdir, 1), "bitcoin.conf"), 'a', encoding='utf8') as f: f.write("rpcuser={}\n".format(self.rpcuser)) f.write("rpcpassword={}\n".format(self.rpcpassword)) @@ -76,19 +77,16 @@ class HTTPBasicsTest(BitcoinTestFramework): assert_equal(200, call_with_auth(node, user, password).status) self.log.info('Wrong...') - assert_equal(401, call_with_auth(node, user, password+'wrong').status) + assert_equal(401, call_with_auth(node, user, password + 'wrong').status) self.log.info('Wrong...') - assert_equal(401, call_with_auth(node, user+'wrong', password).status) + assert_equal(401, call_with_auth(node, user + 'wrong', password).status) self.log.info('Wrong...') - assert_equal(401, call_with_auth(node, user+'wrong', password+'wrong').status) + assert_equal(401, call_with_auth(node, user + 'wrong', password + 'wrong').status) def run_test(self): - - ################################################## - # Check correctness of the rpcauth config option # - ################################################## + self.log.info('Check correctness of the rpcauth config option') url = urllib.parse.urlparse(self.nodes[0].url) self.test_auth(self.nodes[0], url.username, url.password) @@ -96,12 +94,18 @@ class HTTPBasicsTest(BitcoinTestFramework): self.test_auth(self.nodes[0], 'rt2', self.rt2password) self.test_auth(self.nodes[0], self.user, self.password) - ############################################################### - # Check correctness of the rpcuser/rpcpassword config options # - ############################################################### + self.log.info('Check correctness of the rpcuser/rpcpassword config options') url = urllib.parse.urlparse(self.nodes[1].url) self.test_auth(self.nodes[1], self.rpcuser, self.rpcpassword) + self.log.info('Check that failure to write cookie file will abort the node gracefully') + self.stop_node(0) + cookie_file = os.path.join(get_datadir_path(self.options.tmpdir, 0), self.chain, '.cookie.tmp') + os.mkdir(cookie_file) + init_error = 'Error: Unable to start HTTP server. See debug log for details.' + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error) + + if __name__ == '__main__': - HTTPBasicsTest ().main () + HTTPBasicsTest().main() diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 8f410f233e..9506b63f82 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -5,12 +5,15 @@ """Encode and decode BASE58, P2PKH and P2SH addresses.""" import enum +import unittest from .script import hash256, hash160, sha256, CScript, OP_0 from .util import hex_str_to_bytes from . import segwit_addr +from test_framework.util import assert_equal + ADDRESS_BCRT1_UNSPENDABLE = 'bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj' ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR = 'addr(bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj)#juyq9d97' # Coins sent to this address can be spent with a witness stack of just OP_TRUE @@ -41,7 +44,32 @@ def byte_to_base58(b, version): str = str[2:] return result -# TODO: def base58_decode + +def base58_to_byte(s, verify_checksum=True): + if not s: + return b'' + n = 0 + for c in s: + n *= 58 + assert c in chars + digit = chars.index(c) + n += digit + h = '%x' % n + if len(h) % 2: + h = '0' + h + res = n.to_bytes((n.bit_length() + 7) // 8, 'big') + pad = 0 + for c in s: + if c == chars[0]: + pad += 1 + else: + break + res = b'\x00' * pad + res + if verify_checksum: + assert_equal(hash256(res[:-4])[:4], res[-4:]) + + return res[1:-4], int(res[0]) + def keyhash_to_p2pkh(hash, main = False): assert len(hash) == 20 @@ -100,3 +128,22 @@ def check_script(script): if (type(script) is bytes or type(script) is CScript): return script assert False + + +class TestFrameworkScript(unittest.TestCase): + def test_base58encodedecode(self): + def check_base58(data, version): + self.assertEqual(base58_to_byte(byte_to_base58(data, version)), (data, version)) + + check_base58(b'\x1f\x8e\xa1p*{\xd4\x94\x1b\xca\tA\xb8R\xc4\xbb\xfe\xdb.\x05', 111) + check_base58(b':\x0b\x05\xf4\xd7\xf6l;\xa7\x00\x9fE50)l\x84\\\xc9\xcf', 111) + check_base58(b'A\xc1\xea\xf1\x11\x80%Y\xba\xd6\x1b`\xd6+\x1f\x89|c\x92\x8a', 111) + check_base58(b'\0A\xc1\xea\xf1\x11\x80%Y\xba\xd6\x1b`\xd6+\x1f\x89|c\x92\x8a', 111) + check_base58(b'\0\0A\xc1\xea\xf1\x11\x80%Y\xba\xd6\x1b`\xd6+\x1f\x89|c\x92\x8a', 111) + check_base58(b'\0\0\0A\xc1\xea\xf1\x11\x80%Y\xba\xd6\x1b`\xd6+\x1f\x89|c\x92\x8a', 111) + check_base58(b'\x1f\x8e\xa1p*{\xd4\x94\x1b\xca\tA\xb8R\xc4\xbb\xfe\xdb.\x05', 0) + check_base58(b':\x0b\x05\xf4\xd7\xf6l;\xa7\x00\x9fE50)l\x84\\\xc9\xcf', 0) + check_base58(b'A\xc1\xea\xf1\x11\x80%Y\xba\xd6\x1b`\xd6+\x1f\x89|c\x92\x8a', 0) + check_base58(b'\0A\xc1\xea\xf1\x11\x80%Y\xba\xd6\x1b`\xd6+\x1f\x89|c\x92\x8a', 0) + check_base58(b'\0\0A\xc1\xea\xf1\x11\x80%Y\xba\xd6\x1b`\xd6+\x1f\x89|c\x92\x8a', 0) + check_base58(b'\0\0\0A\xc1\xea\xf1\x11\x80%Y\xba\xd6\x1b`\xd6+\x1f\x89|c\x92\x8a', 0) diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index d741b00ba0..afc1995009 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -4,6 +4,8 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for manipulating blocks and transactions.""" +import unittest + from .address import ( key_to_p2sh_p2wpkh, key_to_p2wpkh, @@ -217,3 +219,9 @@ def send_to_witness(use_p2wsh, node, utxo, pubkey, encode_p2sh, amount, sign=Tru tx_to_witness = ToHex(tx) return node.sendrawtransaction(tx_to_witness) + +class TestFrameworkBlockTools(unittest.TestCase): + def test_create_coinbase(self): + height = 20 + coinbase_tx = create_coinbase(height=height) + assert_equal(CScriptNum.decode(coinbase_tx.vin[0].scriptSig), height) diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 4855f62a8f..4d1dd4422e 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -54,9 +54,12 @@ NODE_NETWORK_LIMITED = (1 << 10) MSG_TX = 1 MSG_BLOCK = 2 MSG_FILTERED_BLOCK = 3 +MSG_CMPCT_BLOCK = 4 MSG_WITNESS_FLAG = 1 << 30 MSG_TYPE_MASK = 0xffffffff >> 2 +FILTER_TYPE_BASIC = 0 + # Serialization/deserialization tools def sha256(s): return hashlib.new('sha256', s).digest() @@ -1512,3 +1515,154 @@ class msg_no_witness_blocktxn(msg_blocktxn): def serialize(self): return self.block_transactions.serialize(with_witness=False) + + +class msg_getcfilters: + __slots__ = ("filter_type", "start_height", "stop_hash") + msgtype = b"getcfilters" + + def __init__(self, filter_type, start_height, stop_hash): + self.filter_type = filter_type + self.start_height = start_height + self.stop_hash = stop_hash + + def deserialize(self, f): + self.filter_type = struct.unpack("<B", f.read(1))[0] + self.start_height = struct.unpack("<I", f.read(4))[0] + self.stop_hash = deser_uint256(f) + + def serialize(self): + r = b"" + r += struct.pack("<B", self.filter_type) + r += struct.pack("<I", self.start_height) + r += ser_uint256(self.stop_hash) + return r + + def __repr__(self): + return "msg_getcfilters(filter_type={:#x}, start_height={}, stop_hash={:x})".format( + self.filter_type, self.start_height, self.stop_hash) + +class msg_cfilter: + __slots__ = ("filter_type", "block_hash", "filter_data") + msgtype = b"cfilter" + + def __init__(self, filter_type=None, block_hash=None, filter_data=None): + self.filter_type = filter_type + self.block_hash = block_hash + self.filter_data = filter_data + + def deserialize(self, f): + self.filter_type = struct.unpack("<B", f.read(1))[0] + self.block_hash = deser_uint256(f) + self.filter_data = deser_string(f) + + def serialize(self): + r = b"" + r += struct.pack("<B", self.filter_type) + r += ser_uint256(self.block_hash) + r += ser_string(self.filter_data) + return r + + def __repr__(self): + return "msg_cfilter(filter_type={:#x}, block_hash={:x})".format( + self.filter_type, self.block_hash) + +class msg_getcfheaders: + __slots__ = ("filter_type", "start_height", "stop_hash") + msgtype = b"getcfheaders" + + def __init__(self, filter_type, start_height, stop_hash): + self.filter_type = filter_type + self.start_height = start_height + self.stop_hash = stop_hash + + def deserialize(self, f): + self.filter_type = struct.unpack("<B", f.read(1))[0] + self.start_height = struct.unpack("<I", f.read(4))[0] + self.stop_hash = deser_uint256(f) + + def serialize(self): + r = b"" + r += struct.pack("<B", self.filter_type) + r += struct.pack("<I", self.start_height) + r += ser_uint256(self.stop_hash) + return r + + def __repr__(self): + return "msg_getcfheaders(filter_type={:#x}, start_height={}, stop_hash={:x})".format( + self.filter_type, self.start_height, self.stop_hash) + +class msg_cfheaders: + __slots__ = ("filter_type", "stop_hash", "prev_header", "hashes") + msgtype = b"cfheaders" + + def __init__(self, filter_type=None, stop_hash=None, prev_header=None, hashes=None): + self.filter_type = filter_type + self.stop_hash = stop_hash + self.prev_header = prev_header + self.hashes = hashes + + def deserialize(self, f): + self.filter_type = struct.unpack("<B", f.read(1))[0] + self.stop_hash = deser_uint256(f) + self.prev_header = deser_uint256(f) + self.hashes = deser_uint256_vector(f) + + def serialize(self): + r = b"" + r += struct.pack("<B", self.filter_type) + r += ser_uint256(self.stop_hash) + r += ser_uint256(self.prev_header) + r += ser_uint256_vector(self.hashes) + return r + + def __repr__(self): + return "msg_cfheaders(filter_type={:#x}, stop_hash={:x})".format( + self.filter_type, self.stop_hash) + +class msg_getcfcheckpt: + __slots__ = ("filter_type", "stop_hash") + msgtype = b"getcfcheckpt" + + def __init__(self, filter_type, stop_hash): + self.filter_type = filter_type + self.stop_hash = stop_hash + + def deserialize(self, f): + self.filter_type = struct.unpack("<B", f.read(1))[0] + self.stop_hash = deser_uint256(f) + + def serialize(self): + r = b"" + r += struct.pack("<B", self.filter_type) + r += ser_uint256(self.stop_hash) + return r + + def __repr__(self): + return "msg_getcfcheckpt(filter_type={:#x}, stop_hash={:x})".format( + self.filter_type, self.stop_hash) + +class msg_cfcheckpt: + __slots__ = ("filter_type", "stop_hash", "headers") + msgtype = b"cfcheckpt" + + def __init__(self, filter_type=None, stop_hash=None, headers=None): + self.filter_type = filter_type + self.stop_hash = stop_hash + self.headers = headers + + def deserialize(self, f): + self.filter_type = struct.unpack("<B", f.read(1))[0] + self.stop_hash = deser_uint256(f) + self.headers = deser_uint256_vector(f) + + def serialize(self): + r = b"" + r += struct.pack("<B", self.filter_type) + r += ser_uint256(self.stop_hash) + r += ser_uint256_vector(self.headers) + return r + + def __repr__(self): + return "msg_cfcheckpt(filter_type={:#x}, stop_hash={:x})".format( + self.filter_type, self.stop_hash) diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 31cec66ee7..45063aaff2 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -31,6 +31,9 @@ from test_framework.messages import ( msg_block, MSG_BLOCK, msg_blocktxn, + msg_cfcheckpt, + msg_cfheaders, + msg_cfilter, msg_cmpctblock, msg_feefilter, msg_filteradd, @@ -67,6 +70,9 @@ MESSAGEMAP = { b"addr": msg_addr, b"block": msg_block, b"blocktxn": msg_blocktxn, + b"cfcheckpt": msg_cfcheckpt, + b"cfheaders": msg_cfheaders, + b"cfilter": msg_cfilter, b"cmpctblock": msg_cmpctblock, b"feefilter": msg_feefilter, b"filteradd": msg_filteradd, @@ -120,9 +126,9 @@ class P2PConnection(asyncio.Protocol): def is_connected(self): return self._transport is not None - def peer_connect(self, dstaddr, dstport, *, net, factor): + def peer_connect(self, dstaddr, dstport, *, net, timeout_factor): assert not self.is_connected - self.factor = factor + self.timeout_factor = timeout_factor self.dstaddr = dstaddr self.dstport = dstport # The initial message to send after the connection was made: @@ -328,6 +334,9 @@ class P2PInterface(P2PConnection): def on_addr(self, message): pass def on_block(self, message): pass def on_blocktxn(self, message): pass + def on_cfcheckpt(self, message): pass + def on_cfheaders(self, message): pass + def on_cfilter(self, message): pass def on_cmpctblock(self, message): pass def on_feefilter(self, message): pass def on_filteradd(self, message): pass @@ -368,8 +377,8 @@ class P2PInterface(P2PConnection): # Connection helper methods - def wait_until(self, test_function, timeout): - wait_until(test_function, timeout=timeout, lock=mininode_lock, factor=self.factor) + def wait_until(self, test_function, timeout=60): + wait_until(test_function, timeout=timeout, lock=mininode_lock, timeout_factor=self.timeout_factor) def wait_for_disconnect(self, timeout=60): test_function = lambda: not self.is_connected @@ -642,12 +651,24 @@ class P2PTxInvStore(P2PInterface): self.tx_invs_received = defaultdict(int) def on_inv(self, message): + super().on_inv(message) # Send getdata in response. # Store how many times invs have been received for each tx. for i in message.inv: if i.type == MSG_TX: # save txid self.tx_invs_received[i.hash] += 1 + super().on_inv(message) + def get_invs(self): with mininode_lock: return list(self.tx_invs_received.keys()) + + def wait_for_broadcast(self, txns, timeout=60): + """Waits for the txns (list of txids) to complete initial broadcast. + The mempool should mark unbroadcast=False for these transactions. + """ + # Wait until invs have been received (and getdatas sent) for each txid. + self.wait_until(lambda: set(self.get_invs()) == set([int(tx, 16) for tx in txns]), timeout) + # Flush messages and wait for the getdatas to be processed + self.sync_with_ping() diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 9102266456..cc5f8307d3 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -9,6 +9,7 @@ This file is modified from python-bitcoinlib. import hashlib import struct import unittest +from typing import List, Dict from .messages import ( CTransaction, @@ -21,7 +22,7 @@ from .messages import ( ) MAX_SCRIPT_ELEMENT_SIZE = 520 -OPCODE_NAMES = {} +OPCODE_NAMES = {} # type: Dict[CScriptOp, str] def hash160(s): return hashlib.new('ripemd160', sha256(s)).digest() @@ -37,7 +38,7 @@ def bn2vch(v): # Serialize to bytes return encoded_v.to_bytes(n_bytes, 'little') -_opcode_instances = [] +_opcode_instances = [] # type: List[CScriptOp] class CScriptOp(int): """A single script opcode""" __slots__ = () @@ -731,3 +732,9 @@ class TestFrameworkScript(unittest.TestCase): self.assertEqual(bn2vch(0xFFFFFFFF), bytes([0xFF, 0xFF, 0xFF, 0xFF, 0x00])) self.assertEqual(bn2vch(123456789), bytes([0x15, 0xCD, 0x5B, 0x07])) self.assertEqual(bn2vch(-54321), bytes([0x31, 0xD4, 0x80])) + + def test_cscriptnum_encoding(self): + # round-trip negative and multi-byte CScriptNums + values = [0, 1, -1, -2, 127, 128, -255, 256, (1 << 15) - 1, -(1 << 16), (1 << 24) - 1, (1 << 31), 1 - (1 << 32), 1 << 40, 1500, -1500] + for value in values: + self.assertEqual(CScriptNum.decode(CScriptNum.encode(CScriptNum(value))), value) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 17acd4dcd3..9f5e9e5f0d 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -91,6 +91,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): This class also contains various public and private helper methods.""" + chain = None # type: str + setup_clean_chain = None # type: bool + def __init__(self): """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method""" self.chain = 'regtest' @@ -102,7 +105,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.bind_to_localhost_only = True self.set_test_params() self.parse_args() - self.rpc_timeout = int(self.rpc_timeout * self.options.factor) # optionally, increase timeout by a factor + if self.options.timeout_factor == 0 : + self.options.timeout_factor = 99999 + self.rpc_timeout = int(self.rpc_timeout * self.options.timeout_factor) # optionally, increase timeout by a factor def main(self): """Main function. This should not be overridden by the subclass test scripts.""" @@ -138,6 +143,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): sys.exit(exit_code) def parse_args(self): + previous_releases_path = os.getenv("PREVIOUS_RELEASES_DIR") or os.getcwd() + "/releases" parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") @@ -152,6 +158,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="Print out all RPC calls as they are made") parser.add_argument("--portseed", dest="port_seed", default=os.getpid(), type=int, help="The seed to use for assigning port numbers (default: current process id)") + parser.add_argument("--previous-releases", dest="prev_releases", action="store_true", + default=os.path.isdir(previous_releases_path) and bool(os.listdir(previous_releases_path)), + help="Force test of previous releases (default: %(default)s)") parser.add_argument("--coveragedir", dest="coveragedir", help="Write tested RPC commands into this directory") parser.add_argument("--configfile", dest="configfile", @@ -169,9 +178,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="set a random seed for deterministically reproducing a previous test run") parser.add_argument("--descriptors", default=False, action="store_true", help="Run test using a descriptor wallet") - parser.add_argument('--factor', type=float, default=1.0, help='adjust test timeouts by a factor') + parser.add_argument('--timeout-factor', dest="timeout_factor", type=float, default=1.0, help='adjust test timeouts by a factor. Setting it to 0 disables all timeouts') self.add_options(parser) self.options = parser.parse_args() + self.options.previous_releases_path = previous_releases_path def setup(self): """Call this method to start up the test framework object with options set.""" @@ -185,10 +195,18 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): config = configparser.ConfigParser() config.read_file(open(self.options.configfile)) self.config = config - self.options.bitcoind = os.getenv("BITCOIND", default=config["environment"]["BUILDDIR"] + '/src/bitcoind' + config["environment"]["EXEEXT"]) - self.options.bitcoincli = os.getenv("BITCOINCLI", default=config["environment"]["BUILDDIR"] + '/src/bitcoin-cli' + config["environment"]["EXEEXT"]) - - self.options.previous_releases_path = os.getenv("PREVIOUS_RELEASES_DIR") or os.getcwd() + "/releases" + fname_bitcoind = os.path.join( + config["environment"]["BUILDDIR"], + "src", + "bitcoind" + config["environment"]["EXEEXT"], + ) + fname_bitcoincli = os.path.join( + config["environment"]["BUILDDIR"], + "src", + "bitcoin-cli" + config["environment"]["EXEEXT"], + ) + self.options.bitcoind = os.getenv("BITCOIND", default=fname_bitcoind) + self.options.bitcoincli = os.getenv("BITCOINCLI", default=fname_bitcoincli) os.environ['PATH'] = os.pathsep.join([ os.path.join(config['environment']['BUILDDIR'], 'src'), @@ -275,7 +293,12 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): exit_code = TEST_EXIT_SKIPPED else: self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) + self.log.error("") self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir)) + self.log.error("") + self.log.error("If this failure happened unexpectedly or intermittently, please file a bug and provide a link or upload of the combined log.") + self.log.error(self.config['environment']['PACKAGE_BUGREPORT']) + self.log.error("") exit_code = TEST_EXIT_FAILED # Logging.shutdown will not remove stream- and filehandlers, so we must # do it explicitly. Handlers are removed so the next test run can apply @@ -387,7 +410,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # Public helper methods. These can be accessed by the subclass test scripts. - def add_nodes(self, num_nodes, extra_args=None, *, rpchost=None, binary=None, binary_cli=None, versions=None): + def add_nodes(self, num_nodes: int, extra_args=None, *, rpchost=None, binary=None, binary_cli=None, versions=None): """Instantiate TestNode objects. Should only be called once after the nodes have been specified in @@ -435,7 +458,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): chain=self.chain, rpchost=rpchost, timewait=self.rpc_timeout, - factor=self.options.factor, + timeout_factor=self.options.timeout_factor, bitcoind=binary[i], bitcoin_cli=binary_cli[i], version=versions[i], @@ -581,7 +604,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): extra_args=['-disablewallet'], rpchost=None, timewait=self.rpc_timeout, - factor=self.options.factor, + timeout_factor=self.options.timeout_factor, bitcoind=self.options.bitcoind, bitcoin_cli=self.options.bitcoincli, coverage_dir=None, @@ -671,17 +694,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def has_previous_releases(self): """Checks whether previous releases are present and enabled.""" - if os.getenv("TEST_PREVIOUS_RELEASES") == "false": - # disabled - return False - if not os.path.isdir(self.options.previous_releases_path): - if os.getenv("TEST_PREVIOUS_RELEASES") == "true": - raise AssertionError("TEST_PREVIOUS_RELEASES=true but releases missing: {}".format( + if self.options.prev_releases: + raise AssertionError("Force test of previous releases but releases missing: {}".format( self.options.previous_releases_path)) - # missing - return False - return True + return self.options.prev_releases def is_cli_compiled(self): """Checks whether bitcoin-cli was compiled.""" diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 404c1b207b..ebc0501e11 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -62,7 +62,7 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, datadir, *, chain, rpchost, timewait, factor, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None, descriptors=False): + def __init__(self, i, datadir, *, chain, rpchost, timewait, timeout_factor, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None, descriptors=False): """ Kwargs: start_perf (bool): If True, begin profiling the node with `perf` as soon as @@ -128,7 +128,7 @@ class TestNode(): self.perf_subprocesses = {} self.p2ps = [] - self.factor = factor + self.timeout_factor = timeout_factor AddressKeyPair = collections.namedtuple('AddressKeyPair', ['address', 'key']) PRIV_KEYS = [ @@ -219,7 +219,12 @@ class TestNode(): raise FailedToStartError(self._node_msg( 'bitcoind exited with status {} during initialization'.format(self.process.returncode))) try: - rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.chain, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir) + rpc = get_rpc_proxy( + rpc_url(self.datadir, self.index, self.chain, self.rpchost), + self.index, + timeout=self.rpc_timeout // 2, # Shorter timeout to allow for one retry in case of ETIMEDOUT + coveragedir=self.coverage_dir, + ) rpc.getblockcount() # If the call to getblockcount() succeeds then the RPC connection is up if self.version_is_at_least(190000): @@ -241,7 +246,7 @@ class TestNode(): # The wait is done here to make tests as robust as possible # and prevent racy tests and intermittent failures as much # as possible. Some tests might not need this, but the - # overhead is trivial, and the added gurantees are worth + # overhead is trivial, and the added guarantees are worth # the minimal performance cost. self.log.debug("RPC successfully started") if self.use_cli: @@ -260,7 +265,11 @@ class TestNode(): # succeeds. Try again to properly raise the FailedToStartError pass except OSError as e: - if e.errno != errno.ECONNREFUSED: # Port not yet open? + if e.errno == errno.ETIMEDOUT: + pass # Treat identical to ConnectionResetError + elif e.errno == errno.ECONNREFUSED: + pass # Port not yet open? + else: raise # unknown OS error except ValueError as e: # cookie file not found and no rpcuser or rpcpassword; bitcoind is still starting if "No RPC credentials" not in str(e): @@ -349,13 +358,13 @@ class TestNode(): return True def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT): - wait_until(self.is_node_stopped, timeout=timeout, factor=self.factor) + wait_until(self.is_node_stopped, timeout=timeout, timeout_factor=self.timeout_factor) @contextlib.contextmanager def assert_debug_log(self, expected_msgs, unexpected_msgs=None, timeout=2): if unexpected_msgs is None: unexpected_msgs = [] - time_end = time.time() + timeout * self.factor + time_end = time.time() + timeout * self.timeout_factor debug_log = os.path.join(self.datadir, self.chain, 'debug.log') with open(debug_log, encoding='utf-8') as dl: dl.seek(0, 2) @@ -512,7 +521,7 @@ class TestNode(): if 'dstaddr' not in kwargs: kwargs['dstaddr'] = '127.0.0.1' - p2p_conn.peer_connect(**kwargs, net=self.chain, factor=self.factor)() + p2p_conn.peer_connect(**kwargs, net=self.chain, timeout_factor=self.timeout_factor)() self.p2ps.append(p2p_conn) if wait_for_verack: # Wait for the node to send us the version and verack @@ -526,7 +535,7 @@ class TestNode(): # transaction that will be added to the mempool as soon as we return here. # # So syncing here is redundant when we only want to send a message, but the cost is low (a few milliseconds) - # in comparision to the upside of making tests less fragile and unexpected intermittent errors less likely. + # in comparison to the upside of making tests less fragile and unexpected intermittent errors less likely. p2p_conn.sync_with_ping() return p2p_conn @@ -562,6 +571,8 @@ class TestNodeCLIAttr: def arg_to_cli(arg): if isinstance(arg, bool): return str(arg).lower() + elif arg is None: + return 'null' elif isinstance(arg, dict) or isinstance(arg, list): return json.dumps(arg, default=EncodeDecimal) else: @@ -632,27 +643,13 @@ class RPCOverloadWrapper(): def __getattr__(self, name): return getattr(self.rpc, name) - def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase=None, avoid_reuse=None, descriptors=None): - if self.is_cli: - if disable_private_keys is None: - disable_private_keys = 'null' - if blank is None: - blank = 'null' - if passphrase is None: - passphrase = '' - if avoid_reuse is None: - avoid_reuse = 'null' + def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase='', avoid_reuse=None, descriptors=None): if descriptors is None: descriptors = self.descriptors return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors) def importprivkey(self, privkey, label=None, rescan=None): wallet_info = self.getwalletinfo() - if self.is_cli: - if label is None: - label = 'null' - if rescan is None: - rescan = 'null' if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): return self.__getattr__('importprivkey')(privkey, label, rescan) desc = descsum_create('combo(' + privkey + ')') @@ -667,11 +664,6 @@ class RPCOverloadWrapper(): def addmultisigaddress(self, nrequired, keys, label=None, address_type=None): wallet_info = self.getwalletinfo() - if self.is_cli: - if label is None: - label = 'null' - if address_type is None: - address_type = 'null' if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): return self.__getattr__('addmultisigaddress')(nrequired, keys, label, address_type) cms = self.createmultisig(nrequired, keys, address_type) @@ -687,11 +679,6 @@ class RPCOverloadWrapper(): def importpubkey(self, pubkey, label=None, rescan=None): wallet_info = self.getwalletinfo() - if self.is_cli: - if label is None: - label = 'null' - if rescan is None: - rescan = 'null' if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): return self.__getattr__('importpubkey')(pubkey, label, rescan) desc = descsum_create('combo(' + pubkey + ')') @@ -706,13 +693,6 @@ class RPCOverloadWrapper(): def importaddress(self, address, label=None, rescan=None, p2sh=None): wallet_info = self.getwalletinfo() - if self.is_cli: - if label is None: - label = 'null' - if rescan is None: - rescan = 'null' - if p2sh is None: - p2sh = 'null' if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): return self.__getattr__('importaddress')(address, label, rescan, p2sh) is_hex = False diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 6cfb22befe..52306c8c3d 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -208,10 +208,10 @@ def str_to_b64str(string): def satoshi_round(amount): return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) -def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None, factor=1.0): +def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None, timeout_factor=1.0): if attempts == float('inf') and timeout == float('inf'): timeout = 60 - timeout = timeout * factor + timeout = timeout * timeout_factor attempt = 0 time_end = time.time() + timeout @@ -413,7 +413,11 @@ def connect_nodes(from_connection, node_num): from_connection.addnode(ip_port, "onetry") # poll until version handshake complete to avoid race conditions # with transaction relaying - wait_until(lambda: all(peer['version'] != 0 for peer in from_connection.getpeerinfo())) + # See comments in net_processing: + # * Must have a version message before anything else + # * Must have a verack message before anything else + wait_until(lambda: all(peer['version'] != 0 for peer in from_connection.getpeerinfo())) + wait_until(lambda: all(peer['bytesrecv_per_msg'].pop('verack', 0) == 24 for peer in from_connection.getpeerinfo())) def sync_blocks(rpc_connections, *, wait=1, timeout=60): diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 45f591ba4a..0ea65c68b8 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -42,7 +42,7 @@ except UnicodeDecodeError: if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): if os.name == 'nt': import ctypes - kernel32 = ctypes.windll.kernel32 + kernel32 = ctypes.windll.kernel32 # type: ignore ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 STD_OUTPUT_HANDLE = -11 STD_ERROR_HANDLE = -12 @@ -67,6 +67,8 @@ TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 TEST_FRAMEWORK_MODULES = [ + "address", + "blocktools", "script", ] @@ -157,6 +159,7 @@ BASE_SCRIPTS = [ 'rpc_deprecated.py', 'wallet_disable.py', 'p2p_addr_relay.py', + 'p2p_getdata.py', 'rpc_net.py', 'wallet_keypool.py', 'wallet_keypool.py --descriptors', @@ -225,6 +228,7 @@ BASE_SCRIPTS = [ 'feature_loadblock.py', 'p2p_dos_header_tree.py', 'p2p_unrequested_blocks.py', + 'p2p_blockfilters.py', 'feature_includeconf.py', 'feature_asmap.py', 'mempool_unbroadcast.py', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 039ce7daee..524e1593ba 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -203,6 +203,14 @@ class ToolWalletTest(BitcoinTestFramework): assert_equal(shasum_after, shasum_before) self.log.debug('Wallet file shasum unchanged\n') + def test_salvage(self): + # TODO: Check salvage actually salvages and doesn't break things. https://github.com/bitcoin/bitcoin/issues/7463 + self.log.info('Check salvage') + self.start_node(0, ['-wallet=salvage']) + self.stop_node(0) + + self.assert_tool_output('', '-wallet=salvage', 'salvage') + def run_test(self): self.wallet_path = os.path.join(self.nodes[0].datadir, self.chain, 'wallets', 'wallet.dat') self.test_invalid_tool_commands_and_args() @@ -211,7 +219,7 @@ class ToolWalletTest(BitcoinTestFramework): self.test_tool_wallet_info_after_transaction() self.test_tool_wallet_create_on_existing_wallet() self.test_getwalletinfo_on_different_wallet() - + self.test_salvage() if __name__ == '__main__': ToolWalletTest().main() diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 9e295af330..797c903dd3 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -404,8 +404,6 @@ class WalletTest(BitcoinTestFramework): '-reindex', '-zapwallettxes=1', '-zapwallettxes=2', - # disabled until issue is fixed: https://github.com/bitcoin/bitcoin/issues/7463 - # '-salvagewallet', ] chainlimit = 6 for m in maintenance: diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 09f89eb59d..c441b75652 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -11,7 +11,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes, - assert_raises_rpc_error + assert_raises_rpc_error, ) @@ -32,11 +32,11 @@ class WalletHDTest(BitcoinTestFramework): # create an internal key change_addr = self.nodes[1].getrawchangeaddress() - change_addrV= self.nodes[1].getaddressinfo(change_addr) + change_addrV = self.nodes[1].getaddressinfo(change_addr) if self.options.descriptors: assert_equal(change_addrV["hdkeypath"], "m/84'/1'/0'/1/0") else: - assert_equal(change_addrV["hdkeypath"], "m/0'/1'/0'") #first internal child key + assert_equal(change_addrV["hdkeypath"], "m/0'/1'/0'") #first internal child key # Import a non-HD private key in the HD wallet non_hd_add = 'bcrt1qmevj8zfx0wdvp05cqwkmr6mxkfx60yezwjksmt' @@ -58,7 +58,7 @@ class WalletHDTest(BitcoinTestFramework): if self.options.descriptors: assert_equal(hd_info["hdkeypath"], "m/84'/1'/0'/0/" + str(i)) else: - assert_equal(hd_info["hdkeypath"], "m/0'/0'/"+str(i)+"'") + assert_equal(hd_info["hdkeypath"], "m/0'/0'/" + str(i) + "'") assert_equal(hd_info["hdmasterfingerprint"], hd_fingerprint) self.nodes[0].sendtoaddress(hd_add, 1) self.nodes[0].generate(1) @@ -67,11 +67,11 @@ class WalletHDTest(BitcoinTestFramework): # create an internal key (again) change_addr = self.nodes[1].getrawchangeaddress() - change_addrV= self.nodes[1].getaddressinfo(change_addr) + change_addrV = self.nodes[1].getaddressinfo(change_addr) if self.options.descriptors: assert_equal(change_addrV["hdkeypath"], "m/84'/1'/0'/1/1") else: - assert_equal(change_addrV["hdkeypath"], "m/0'/1'/1'") #second internal child key + assert_equal(change_addrV["hdkeypath"], "m/0'/1'/1'") #second internal child key self.sync_all() assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + 1) @@ -82,7 +82,10 @@ class WalletHDTest(BitcoinTestFramework): # otherwise node1 would auto-recover all funds in flag the keypool keys as used shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "blocks")) shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "chainstate")) - shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, self.chain, 'wallets', "wallet.dat")) + shutil.copyfile( + os.path.join(self.nodes[1].datadir, "hd.bak"), + os.path.join(self.nodes[1].datadir, self.chain, 'wallets', "wallet.dat"), + ) self.start_node(1) # Assert that derivation is deterministic @@ -93,7 +96,7 @@ class WalletHDTest(BitcoinTestFramework): if self.options.descriptors: assert_equal(hd_info_2["hdkeypath"], "m/84'/1'/0'/0/" + str(i)) else: - assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/"+str(i)+"'") + assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/" + str(i) + "'") assert_equal(hd_info_2["hdmasterfingerprint"], hd_fingerprint) assert_equal(hd_add, hd_add_2) connect_nodes(self.nodes[0], 1) @@ -108,7 +111,10 @@ class WalletHDTest(BitcoinTestFramework): self.stop_node(1) shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "blocks")) shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "chainstate")) - shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, self.chain, "wallets", "wallet.dat")) + shutil.copyfile( + os.path.join(self.nodes[1].datadir, "hd.bak"), + os.path.join(self.nodes[1].datadir, self.chain, "wallets", "wallet.dat"), + ) self.start_node(1, extra_args=self.extra_args[1]) connect_nodes(self.nodes[0], 1) self.sync_all() @@ -142,8 +148,9 @@ class WalletHDTest(BitcoinTestFramework): new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] assert orig_masterkeyid != new_masterkeyid addr = self.nodes[1].getnewaddress() - assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/0\'') # Make sure the new address is the first from the keypool - self.nodes[1].keypoolrefill(1) # Fill keypool with 1 key + # Make sure the new address is the first from the keypool + assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/0\'') + self.nodes[1].keypoolrefill(1) # Fill keypool with 1 key # Set a new HD seed on node 1 without flushing the keypool new_seed = self.nodes[0].dumpprivkey(self.nodes[0].getnewaddress()) @@ -153,13 +160,15 @@ class WalletHDTest(BitcoinTestFramework): assert orig_masterkeyid != new_masterkeyid addr = self.nodes[1].getnewaddress() assert_equal(orig_masterkeyid, self.nodes[1].getaddressinfo(addr)['hdseedid']) - assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/1\'') # Make sure the new address continues previous keypool + # Make sure the new address continues previous keypool + assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/1\'') # Check that the next address is from the new seed self.nodes[1].keypoolrefill(1) next_addr = self.nodes[1].getnewaddress() assert_equal(new_masterkeyid, self.nodes[1].getaddressinfo(next_addr)['hdseedid']) - assert_equal(self.nodes[1].getaddressinfo(next_addr)['hdkeypath'], 'm/0\'/0\'/0\'') # Make sure the new address is not from previous keypool + # Make sure the new address is not from previous keypool + assert_equal(self.nodes[1].getaddressinfo(next_addr)['hdkeypath'], 'm/0\'/0\'/0\'') assert next_addr != addr # Sethdseed parameter validity @@ -170,5 +179,104 @@ class WalletHDTest(BitcoinTestFramework): assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, new_seed) assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, self.nodes[1].dumpprivkey(self.nodes[1].getnewaddress())) + self.log.info('Test sethdseed restoring with keys outside of the initial keypool') + self.nodes[0].generate(10) + # Restart node 1 with keypool of 3 and a different wallet + self.nodes[1].createwallet(wallet_name='origin', blank=True) + self.stop_node(1) + self.start_node(1, extra_args=['-keypool=3', '-wallet=origin']) + connect_nodes(self.nodes[0], 1) + + # sethdseed restoring and seeing txs to addresses out of the keypool + origin_rpc = self.nodes[1].get_wallet_rpc('origin') + seed = self.nodes[0].dumpprivkey(self.nodes[0].getnewaddress()) + origin_rpc.sethdseed(True, seed) + + self.nodes[1].createwallet(wallet_name='restore', blank=True) + restore_rpc = self.nodes[1].get_wallet_rpc('restore') + restore_rpc.sethdseed(True, seed) # Set to be the same seed as origin_rpc + restore_rpc.sethdseed(True) # Rotate to a new seed, making original `seed` inactive + + self.nodes[1].createwallet(wallet_name='restore2', blank=True) + restore2_rpc = self.nodes[1].get_wallet_rpc('restore2') + restore2_rpc.sethdseed(True, seed) # Set to be the same seed as origin_rpc + restore2_rpc.sethdseed(True) # Rotate to a new seed, making original `seed` inactive + + # Check persistence of inactive seed by reloading restore. restore2 is still loaded to test the case where the wallet is not reloaded + restore_rpc.unloadwallet() + self.nodes[1].loadwallet('restore') + restore_rpc = self.nodes[1].get_wallet_rpc('restore') + + # Empty origin keypool and get an address that is beyond the initial keypool + origin_rpc.getnewaddress() + origin_rpc.getnewaddress() + last_addr = origin_rpc.getnewaddress() # Last address of initial keypool + addr = origin_rpc.getnewaddress() # First address beyond initial keypool + + # Check that the restored seed has last_addr but does not have addr + info = restore_rpc.getaddressinfo(last_addr) + assert_equal(info['ismine'], True) + info = restore_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], False) + info = restore2_rpc.getaddressinfo(last_addr) + assert_equal(info['ismine'], True) + info = restore2_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], False) + # Check that the origin seed has addr + info = origin_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], True) + + # Send a transaction to addr, which is out of the initial keypool. + # The wallet that has set a new seed (restore_rpc) should not detect this transaction. + txid = self.nodes[0].sendtoaddress(addr, 1) + origin_rpc.sendrawtransaction(self.nodes[0].gettransaction(txid)['hex']) + self.nodes[0].generate(1) + self.sync_blocks() + origin_rpc.gettransaction(txid) + assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore_rpc.gettransaction, txid) + out_of_kp_txid = txid + + # Send a transaction to last_addr, which is in the initial keypool. + # The wallet that has set a new seed (restore_rpc) should detect this transaction and generate 3 new keys from the initial seed. + # The previous transaction (out_of_kp_txid) should still not be detected as a rescan is required. + txid = self.nodes[0].sendtoaddress(last_addr, 1) + origin_rpc.sendrawtransaction(self.nodes[0].gettransaction(txid)['hex']) + self.nodes[0].generate(1) + self.sync_blocks() + origin_rpc.gettransaction(txid) + restore_rpc.gettransaction(txid) + assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore_rpc.gettransaction, out_of_kp_txid) + restore2_rpc.gettransaction(txid) + assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore2_rpc.gettransaction, out_of_kp_txid) + + # After rescanning, restore_rpc should now see out_of_kp_txid and generate an additional key. + # addr should now be part of restore_rpc and be ismine + restore_rpc.rescanblockchain() + restore_rpc.gettransaction(out_of_kp_txid) + info = restore_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], True) + restore2_rpc.rescanblockchain() + restore2_rpc.gettransaction(out_of_kp_txid) + info = restore2_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], True) + + # Check again that 3 keys were derived. + # Empty keypool and get an address that is beyond the initial keypool + origin_rpc.getnewaddress() + origin_rpc.getnewaddress() + last_addr = origin_rpc.getnewaddress() + addr = origin_rpc.getnewaddress() + + # Check that the restored seed has last_addr but does not have addr + info = restore_rpc.getaddressinfo(last_addr) + assert_equal(info['ismine'], True) + info = restore_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], False) + info = restore2_rpc.getaddressinfo(last_addr) + assert_equal(info['ismine'], True) + info = restore2_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], False) + + if __name__ == '__main__': - WalletHDTest().main () + WalletHDTest().main() diff --git a/test/functional/wallet_labels.py b/test/functional/wallet_labels.py index f8d1720469..fb4a1f9792 100755 --- a/test/functional/wallet_labels.py +++ b/test/functional/wallet_labels.py @@ -134,6 +134,33 @@ class WalletLabelsTest(BitcoinTestFramework): # in the label. This is a no-op. change_label(node, labels[2].addresses[0], labels[2], labels[2]) + self.log.info('Check watchonly labels') + node.createwallet(wallet_name='watch_only', disable_private_keys=True, descriptors=False) + wallet_watch_only = node.get_wallet_rpc('watch_only') + BECH32_VALID = { + '✔️_VER15_PROG40': 'bcrt10qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqn2cjv3', + '✔️_VER16_PROG03': 'bcrt1sqqqqqjq8pdp', + '✔️_VER16_PROB02': 'bcrt1sqqqqqjq8pv', + } + BECH32_INVALID = { + '❌_VER15_PROG41': 'bcrt10qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzc7xyq', + '❌_VER16_PROB01': 'bcrt1sqqpl9r5c', + } + for l in BECH32_VALID: + ad = BECH32_VALID[l] + wallet_watch_only.importaddress(label=l, rescan=False, address=ad) + node.generatetoaddress(1, ad) + assert_equal(wallet_watch_only.getaddressesbylabel(label=l), {ad: {'purpose': 'receive'}}) + assert_equal(wallet_watch_only.getreceivedbylabel(label=l), 0) + for l in BECH32_INVALID: + ad = BECH32_INVALID[l] + assert_raises_rpc_error( + -5, + "Invalid Bitcoin address or script", + lambda: wallet_watch_only.importaddress(label=l, rescan=False, address=ad), + ) + + class Label: def __init__(self, name): # Label name diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 580a61f9f3..ff9ff34185 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -122,10 +122,6 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].assert_start_raises_init_error(['-zapwallettxes=1', '-wallet=w1', '-wallet=w2'], "Error: -zapwallettxes is only allowed with a single wallet file") self.nodes[0].assert_start_raises_init_error(['-zapwallettxes=2', '-wallet=w1', '-wallet=w2'], "Error: -zapwallettxes is only allowed with a single wallet file") - self.log.info("Do not allow -salvagewallet with multiwallet") - self.nodes[0].assert_start_raises_init_error(['-salvagewallet', '-wallet=w1', '-wallet=w2'], "Error: -salvagewallet is only allowed with a single wallet file") - self.nodes[0].assert_start_raises_init_error(['-salvagewallet=1', '-wallet=w1', '-wallet=w2'], "Error: -salvagewallet is only allowed with a single wallet file") - # if wallets/ doesn't exist, datadir should be the default wallet dir wallet_dir2 = data_dir('walletdir') os.rename(wallet_dir(), wallet_dir2) diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index b384998d56..3417616d77 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -49,16 +49,21 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): block.solve() node.submitblock(ToHex(block)) - # Transaction should not be rebroadcast node.syncwithvalidationinterfacequeue() - node.p2ps[1].sync_with_ping() - assert_equal(node.p2ps[1].tx_invs_received[txid], 0) + now = int(time.time()) + + # Transaction should not be rebroadcast within first 12 hours + # Leave 2 mins for buffer + twelve_hrs = 12 * 60 * 60 + two_min = 2 * 60 + node.setmocktime(now + twelve_hrs - two_min) + time.sleep(2) # ensure enough time has passed for rebroadcast attempt to occur + assert_equal(txid in node.p2ps[1].get_invs(), False) self.log.info("Bump time & check that transaction is rebroadcast") # Transaction should be rebroadcast approximately 24 hours in the future, # but can range from 12-36. So bump 36 hours to be sure. - rebroadcast_time = int(time.time()) + 36 * 60 * 60 - node.setmocktime(rebroadcast_time) + node.setmocktime(now + 36 * 60 * 60) wait_until(lambda: node.p2ps[1].tx_invs_received[txid] >= 1, lock=mininode_lock) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index e7e71bf3f6..bb81746715 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -4,9 +4,11 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """upgradewallet RPC functional test -Test upgradewallet RPC. Download v0.15.2 v0.16.3 node binaries: +Test upgradewallet RPC. Download node binaries: -contrib/devtools/previous_release.sh -b v0.15.2 v0.16.3 +contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 + +Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py """ import os diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index e2454c4237..56b18752ec 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -38,6 +38,7 @@ def main(): ) parser.add_argument( '--par', + '-j', type=int, default=4, help='How many targets to merge or execute in parallel.', diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index bd9c8337ac..5404565b94 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -67,9 +67,9 @@ EXPECTED_BOOST_INCLUDES=( boost/signals2/last_value.hpp boost/signals2/signal.hpp boost/test/unit_test.hpp - boost/thread.hpp boost/thread/condition_variable.hpp boost/thread/mutex.hpp + boost/thread/shared_mutex.hpp boost/thread/thread.hpp boost/variant.hpp boost/variant/apply_visitor.hpp diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index 86ac5a930f..decea38c4f 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -7,6 +7,7 @@ # Check for specified flake8 warnings in python files. export LC_ALL=C +export MYPY_CACHE_DIR="${BASE_ROOT_DIR}/test/.mypy_cache" enabled=( E101 # indentation contains mixed spaces and tabs @@ -89,10 +90,20 @@ elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then exit 0 fi -PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}") $( +EXIT_CODE=0 + +if ! PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}") $( if [[ $# == 0 ]]; then git ls-files "*.py" else echo "$@" fi -) +); then + EXIT_CODE=1 +fi + +if ! mypy --ignore-missing-imports $(git ls-files "test/functional/*.py"); then + EXIT_CODE=1 +fi + +exit $EXIT_CODE diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 70eea34363..fd2e1a329c 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -1,6 +1,34 @@ # ThreadSanitizer suppressions # ============================ +# double locks (TODO fix) +mutex:g_genesis_wait_mutex +mutex:Interrupt +mutex:CThreadInterrupt +mutex:CConnman::Interrupt +mutex:CConnman::WakeMessageHandler +mutex:CConnman::ThreadOpenConnections +mutex:CConnman::ThreadOpenAddedConnections +mutex:CConnman::SocketHandler +mutex:UpdateTip +mutex:PeerLogicValidation::UpdatedBlockTip +mutex:g_best_block_mutex +# race (TODO fix) +race:CConnman::WakeMessageHandler +race:CConnman::ThreadMessageHandler +race:fHaveGenesis +race:ProcessNewBlock +race:ThreadImport +race:WalletBatch::WriteHDChain +race:zmq::* +race:bitcoin-qt +# deadlock (TODO fix) +deadlock:CConnman::ForNode +deadlock:CConnman::GetNodeStats +deadlock:CChainState::ConnectTip +deadlock:UpdateTip +deadlock:wallet_tests::CreateWalletFromFile + # WalletBatch (unidentified deadlock) deadlock:WalletBatch |