diff options
190 files changed, 2422 insertions, 1640 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index eeb2d6590b..2f908dc718 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -5,8 +5,6 @@ configuration: Release platform: x64 clone_depth: 5 environment: - APPVEYOR_SAVE_CACHE_ON_ERROR: true - CLCACHE_SERVER: 1 PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' PYTHONUTF8: 1 QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip' @@ -14,59 +12,41 @@ environment: QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' -cache: -- C:\tools\vcpkg\installed -> build_msvc\vcpkg-packages.txt -- C:\Qt5.9.8_x64_static_vs2019 install: # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. # - cmd: pip install zmq # Powershell block below is to install the c++ dependencies via vcpkg. The pseudo code is: -# 1. Check whether the vcpkg install directory exists (note that updating the vcpkg-packages.txt file -# will cause the appveyor cache rules to invalidate the directory) -# 2. If the directory is missing: # a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary, # b. Install the missing packages. - ps: | $env:PACKAGES = Get-Content -Path build_msvc\vcpkg-packages.txt - Write-Host "vcpkg list: $env:PACKAGES" - if(!(Test-Path -Path ($env:VCPKG_INSTALL_PATH))) { - cd c:\tools\vcpkg - $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. - git pull origin master - git checkout $env:VCPKG_COMMIT_ID - .\bootstrap-vcpkg.bat - Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" - .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null - cd "$env:APPVEYOR_BUILD_FOLDER" - } - else { - Write-Host "required vcpkg packages already installed." - } - c:\tools\vcpkg\vcpkg integrate install + Write-Host "vcpkg installing packages: $env:PACKAGES" + cd c:\tools\vcpkg + $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. + git pull origin master > $null + git -c advice.detachedHead=false checkout $env:VCPKG_COMMIT_ID + .\bootstrap-vcpkg.bat > $null + Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" + .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null + Write-Host "vcpkg packages installed successfully." + .\vcpkg integrate install + cd "$env:APPVEYOR_BUILD_FOLDER" before_build: # Powershell block below is to download and extract the Qt static libraries. The pseudo code is: -# 1. If the Qt destination directory exists assume it is correct and do nothing. To -# force a fresh install of the packages delete the job's appveyor cache. -# 2. Otherwise: # a. Download the zip file with the prebuilt Qt static libraries. # b. Check that the downloaded file matches the expected hash. # c. Extract the zip file to the specific destination path expected by the msbuild projects. - ps: | - if(!(Test-Path -Path ($env:QT_LOCAL_PATH))) { - Write-Host "Downloading Qt binaries."; - Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip; - Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH..."; - if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) { - Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH; - Write-Host "Qt binary download matched the expected hash."; - } - else { - Write-Host "ERROR: Qt binary download did not match the expected hash."; - Exit-AppveyorBuild; - } + Write-Host "Downloading Qt binaries."; + Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip; + Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH..."; + if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) { + Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH; + Write-Host "Qt binary download matched the expected hash."; } else { - Write-Host "Qt binaries already present."; + Write-Host "ERROR: Qt binary download did not match the expected hash."; + Exit-AppveyorBuild; } - cmd: python build_msvc\msvc-autogen.py build_script: diff --git a/.cirrus.yml b/.cirrus.yml index 9c2598d606..446d3e35a9 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,4 +1,5 @@ -# Global defaults +### Global defaults + timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out container: # https://cirrus-ci.org/faq/#are-there-any-limits @@ -12,7 +13,9 @@ env: TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache CCACHE_SIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" -# Global task template + +### Global task template + # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks global_task_template: &GLOBAL_TASK_TEMPLATE ccache_cache: @@ -31,6 +34,7 @@ global_task_template: &GLOBAL_TASK_TEMPLATE - git merge FETCH_HEAD # Merge base to detect silent merge conflicts ci_script: - ./ci/test_run_all.sh + #task: # name: "Windows" # windows_container: @@ -48,6 +52,14 @@ global_task_template: &GLOBAL_TASK_TEMPLATE # - choco install python --version=3.7.7 -y task: + name: 'x86_64 Linux [GOAL: install] [focal] [depends, sanitizers: thread (TSan), no gui]' + << : *GLOBAL_TASK_TEMPLATE + container: + image: ubuntu:focal + env: + FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" + +task: name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]' << : *GLOBAL_TASK_TEMPLATE container: diff --git a/.travis.yml b/.travis.yml index edec60afba..2d69ad7d23 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,14 +18,11 @@ os: linux language: minimal arch: amd64 cache: - ccache: true directories: - $TRAVIS_BUILD_DIR/depends/built - $TRAVIS_BUILD_DIR/depends/sdk-sources - $TRAVIS_BUILD_DIR/ci/scratch/.ccache - $TRAVIS_BUILD_DIR/releases/$HOST -before_cache: - - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then brew cleanup; fi stages: - lint - test @@ -47,7 +44,7 @@ script: - if [ $SECONDS -gt 1200 ]; then export CONTINUE=0; fi # Likely the depends build took very long - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # continue on repos with extended build time (90 minutes) - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi - - if [ $SECONDS -gt 2000 ]; then export CONTINUE=0; fi # Likely the build took very long; The tests take about 1000s, so we should abort if we have less than 50*60-1000=2000s left + - if [[ $SECONDS -gt 50*60-$EXPECTED_TESTS_DURATION_IN_SECONDS ]]; then export CONTINUE=0; fi - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # continue on repos with extended build time (90 minutes) - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi after_script: @@ -100,11 +97,9 @@ jobs: FILE_ENV="./ci/test/00_setup_env_native_qt5.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [focal] [depends, sanitizers: thread (TSan), no gui]' - # Not enough memory on travis machines, so feature_block is excluded for now + name: 'x86_64 Linux [GOAL: install] [focal] [depends, sanitizers: memory (MSan)]' env: >- - TEST_RUNNER_EXTRA="--exclude feature_block" - FILE_ENV="./ci/test/00_setup_env_native_tsan.sh" + FILE_ENV="./ci/test/00_setup_env_native_msan.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, sanitizers: fuzzer,address,undefined]' @@ -138,22 +133,12 @@ jobs: # Xcode 11.3.1, macOS 10.14, SDK 10.15 # https://docs.travis-ci.com/user/reference/osx/#macos-version osx_image: xcode11.3 - cache: - directories: - - $TRAVIS_BUILD_DIR/ci/scratch/.ccache - - $TRAVIS_BUILD_DIR/releases/$HOST - - $HOME/Library/Caches/Homebrew - - /usr/local/Homebrew addons: homebrew: packages: - - libtool - berkeley-db4 - - boost - miniupnpc - - qt - qrencode - - python3 - ccache - zeromq env: >- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 65bc45a00b..11a0f4bac7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,27 +6,28 @@ welcome to contribute towards development in the form of peer review, testing and patches. This document explains the practical process and guidelines for contributing. -Firstly in terms of structure, there is no particular concept of "Core +First, in terms of structure, there is no particular concept of "Bitcoin Core developers" in the sense of privileged people. Open source often naturally -revolves around meritocracy where longer term contributors gain more trust from -the developer community. However, some hierarchy is necessary for practical -purposes. As such there are repository "maintainers" who are responsible for -merging pull requests as well as a "lead maintainer" who is responsible for the -release cycle, overall merging, moderation and appointment of maintainers. +revolves around a meritocracy where contributors earn trust from the developer +community over time. Nevertheless, some hierarchy is necessary for practical +purposes. As such, there are repository "maintainers" who are responsible for +merging pull requests, as well as a "lead maintainer" who is responsible for the +release cycle as well as overall merging, moderation and appointment of +maintainers. Getting Started --------------- New contributors are very welcome and needed. -Reviewing and testing is the most effective way you can contribute as a new -contributor, and it also will teach you much more about the code and process -than opening PRs. Please refer to the section [peer review](#peer-review) later -in this document. +Reviewing and testing is highly valued and the most effective way you can contribute +as a new contributor. It also will teach you much more about the code and +process than opening pull requests. Please refer to the [peer review](#peer-review) +section below. Before you start contributing, familiarize yourself with the Bitcoin Core build system and tests. Refer to the documentation in the repository on how to build -Bitcoin Core and how to run the unit and functional tests. +Bitcoin Core and how to run the unit tests, functional tests, and fuzz tests. There are many open issues of varying difficulty waiting to be fixed. If you're looking for somewhere to start contributing, check out the @@ -62,7 +63,7 @@ history logs can be found on [http://www.erisian.com.au/bitcoin-core-dev/](http://www.erisian.com.au/bitcoin-core-dev/) and [http://gnusha.org/bitcoin-core-dev/](http://gnusha.org/bitcoin-core-dev/). -Discussion about code base improvements happens in GitHub issues and on pull +Discussion about codebase improvements happens in GitHub issues and pull requests. The developer @@ -75,7 +76,7 @@ Contributor Workflow -------------------- The codebase is maintained using the "contributor workflow" where everyone -without exception contributes patch proposals using "pull requests". This +without exception contributes patch proposals using "pull requests" (PRs). This facilitates social contribution, easy testing and peer review. To contribute a patch, the workflow is as follows: @@ -113,6 +114,9 @@ In general, [commits should be atomic](https://en.wikipedia.org/wiki/Atomic_comm and diffs should be easy to read. For this reason, do not mix any formatting fixes or code moves with actual code changes. +Make sure each individual commit is hygienic: that it builds successfully on its +own without warnings, errors, regressions, or test failures. + Commit messages should be verbose by default consisting of a short subject line (50 chars max), a blank line and detailed explanatory text as separate paragraph(s), unless the title alone is self-explanatory (like "Corrected typo @@ -124,7 +128,7 @@ If a particular commit references another issue, please add the reference. For example: `refs #1234` or `fixes #4321`. Using the `fixes` or `closes` keywords will cause the corresponding issue to be closed when the pull request is merged. -Commit messages should never contain any `@` mentions. +Commit messages should never contain any `@` mentions (usernames prefixed with "@"). Please refer to the [Git manual](https://git-scm.com/doc) for more information about Git. @@ -158,10 +162,16 @@ Examples: qt: Add feed bump button log: Fix typo in log message -The body of the pull request should contain enough description about what the -patch does together with any justification/reasoning. You should include -references to any discussions (for example other tickets or mailing list -discussions). +The body of the pull request should contain sufficient description of *what* the +patch does, and even more importantly, *why*, with justification and reasoning. +You should include references to any discussions (for example, other issues or +mailing list discussions). + +The description for a new pull request should not contain any `@` mentions. The +PR description will be included in the commit message when the PR is merged and +any users mentioned in the description will be annoyingly notified each time a +fork of Bitcoin Core copies the merge. Instead, make any username mentions in a +subsequent comment to the PR. ### Translation changes @@ -197,13 +207,13 @@ before it will be merged. The basic squashing workflow is shown below. # Save and quit. git push -f # (force push to GitHub) -Please update the resulting commit message if needed. It should read as a -coherent message. In most cases, this means that you should not just list the -interim commits. +Please update the resulting commit message, if needed. It should read as a +coherent message. In most cases, this means not just listing the interim +commits. -If you have problems with squashing (or other workflows with `git`), you can -alternatively enable "Allow edits from maintainers" in the right GitHub -sidebar and ask for help in the pull request. +If you have problems with squashing or other git workflows, you can enable +"Allow edits from maintainers" in the right-hand sidebar of the GitHub web +interface and ask for help in the pull request. Please refrain from creating several pull requests for the same change. Use the pull request that is already open (or was created earlier) to amend @@ -287,8 +297,8 @@ In general, all pull requests must: - Have a clear use case, fix a demonstrable bug or serve the greater good of the project (for example refactoring for modularisation); - - Be well peer reviewed; - - Have unit tests and functional tests where appropriate; + - Be well peer-reviewed; + - Have unit tests, functional tests, and fuzz tests, where appropriate; - Follow code style guidelines ([C++](doc/developer-notes.md), [functional tests](test/functional/README.md)); - Not break the existing test suite; - Where bugs are fixed, where possible, there should be unit tests @@ -315,7 +325,7 @@ spread out over GitHub, mailing list and IRC discussions). #### Conceptual Review A review can be a conceptual review, where the reviewer leaves a comment - * `Concept (N)ACK`, meaning "I do (not) agree in the general goal of this pull + * `Concept (N)ACK`, meaning "I do (not) agree with the general goal of this pull request", * `Approach (N)ACK`, meaning `Concept ACK`, but "I do (not) agree with the approach of this change". @@ -325,30 +335,28 @@ NACKs without accompanying reasoning may be disregarded. #### Code Review -After conceptual agreement on the change, code review can be provided. It is -starting with `ACK BRANCH_COMMIT`, where `BRANCH_COMMIT` is the top of the -topic branch. The review is followed by a description of how the reviewer did -the review. The following -language is used within pull-request comments: +After conceptual agreement on the change, code review can be provided. A review +begins with `ACK BRANCH_COMMIT`, where `BRANCH_COMMIT` is the top of the PR +branch, followed by a description of how the reviewer did the review. The +following language is used within pull request comments: - - "I have tested the code", involving - change-specific manual testing in addition to running the unit and functional - tests, and in case it is not obvious how the manual testing was done, it should - be described; + - "I have tested the code", involving change-specific manual testing in + addition to running the unit, functional, or fuzz tests, and in case it is + not obvious how the manual testing was done, it should be described; - "I have not tested the code, but I have reviewed it and it looks OK, I agree it can be merged"; - - Nit refers to trivial, often non-blocking issues. + - A "nit" refers to a trivial, often non-blocking issue. Project maintainers reserve the right to weigh the opinions of peer reviewers -using common sense judgement and also may weight based on meritocracy: Those -that have demonstrated a deeper commitment and understanding towards the project -(over time) or have clear domain expertise may naturally have more weight, as -one would expect in all walks of life. +using common sense judgement and may also weigh based on merit. Reviewers that +have demonstrated a deeper commitment and understanding of the project over time +or who have clear domain expertise may naturally have more weight, as one would +expect in all walks of life. -Where a patch set affects consensus critical code, the bar will be set much +Where a patch set affects consensus-critical code, the bar will be much higher in terms of discussion and peer review requirements, keeping in mind that mistakes could be very costly to the wider community. This includes refactoring -of consensus critical code. +of consensus-critical code. Where a patch set proposes to change the Bitcoin consensus, it must have been discussed extensively on the mailing list and IRC, be accompanied by a widely @@ -365,7 +373,7 @@ about: - It may be because of a feature freeze due to an upcoming release. During this time, only bug fixes are taken into consideration. If your pull request is a new feature, - it will not be prioritized until the release is over. Wait for release. + it will not be prioritized until after the release. Wait for the release. - It may be because the changes you are suggesting do not appeal to people. Rather than nits and critique, which require effort and means they care enough to spend time on your contribution, thundering silence is a good sign of widespread (mild) dislike of a given change @@ -375,16 +383,18 @@ about: [developer notes](doc/developer-notes.md), is dangerous or insecure, is messily written, etc. Identify and address any of the issues you find. Then ask e.g. on IRC if someone could give their opinion on the concept itself. - - It may be because your code is too complex for all but a few people. And those people + - It may be because your code is too complex for all but a few people, and those people may not have realized your pull request even exists. A great way to find people who are qualified and care about the code you are touching is the [Git Blame feature](https://help.github.com/articles/tracing-changes-in-a-file/). Simply - find the person touching the code you are touching before you and see if you can find - them and give them a nudge. Don't be incessant about the nudging though. + look up who last modified the code you are changing and see if you can find + them and give them a nudge. Don't be incessant about the nudging, though. - Finally, if all else fails, ask on IRC or elsewhere for someone to give your pull request - a look. If you think you've been waiting an unreasonably long amount of time (month+) for - no particular reason (few lines changed, etc), this is totally fine. Try to return the favor - when someone else is asking for feedback on their code, and universe balances out. + a look. If you think you've been waiting for an unreasonably long time (say, + more than a month) for no particular reason (a few lines changed, etc.), + this is totally fine. Try to return the favor when someone else is asking + for feedback on their code, and the universe balances out. + - Remember that the best thing you can do while waiting is give review to others! Backporting @@ -393,11 +403,11 @@ Backporting Security and bug fixes can be backported from `master` to release branches. If the backport is non-trivial, it may be appropriate to open an -additional PR, to backport the change, only after the original PR +additional PR to backport the change, but only after the original PR has been merged. Otherwise, backports will be done in batches and the maintainers will use the proper `Needs backport (...)` labels -when needed (the original author does not need to worry). +when needed (the original author does not need to worry about it). A backport should contain the following metadata in the commit body: diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index 35ba8425b3..fbbe1a2156 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -137,18 +137,6 @@ don't. */ #define HAVE_DECL_STRNLEN 1 -/* Define to 1 if you have the declaration of `__builtin_clz', and to 0 if you - don't. */ -//#define HAVE_DECL___BUILTIN_CLZ 1 - -/* Define to 1 if you have the declaration of `__builtin_clzl', and to 0 if - you don't. */ -//#define HAVE_DECL___BUILTIN_CLZL 1 - -/* Define to 1 if you have the declaration of `__builtin_clzll', and to 0 if - you don't. */ -//#define HAVE_DECL___BUILTIN_CLZLL 1 - /* Define to 1 if you have the <dlfcn.h> header file. */ /* #undef HAVE_DLFCN_H */ diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index c09997d39d..4fd516fff5 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -107,10 +107,10 @@ <ClCompile> <WarningLevel>Level3</WarningLevel> <PrecompiledHeader>NotUsing</PrecompiledHeader> - <AdditionalOptions>/utf-8 %(AdditionalOptions)</AdditionalOptions> + <AdditionalOptions>/utf-8 /std:c++17 %(AdditionalOptions)</AdditionalOptions> <DisableSpecificWarnings>4018;4221;4244;4267;4334;4715;4805;4834</DisableSpecificWarnings> <TreatWarningAsError>true</TreatWarningAsError> - <PreprocessorDefinitions>ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;%(PreprocessorDefinitions)</PreprocessorDefinitions> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 80d0df4a78..fae424051d 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -6,6 +6,10 @@ export LC_ALL=C +travis_retry sudo apt update && sudo apt install -y clang-format-9 +sudo update-alternatives --install /usr/bin/clang-format clang-format $(which clang-format-9 ) 100 +sudo update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(which clang-format-diff-9) 100 + travis_retry pip3 install codespell==1.17.1 travis_retry pip3 install flake8==3.8.3 travis_retry pip3 install yq diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 711f436630..2413cfca9f 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -31,11 +31,14 @@ export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch} export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} # Whether to prefer BusyBox over GNU utilities export USE_BUSY_BOX=${USE_BUSY_BOX:-false} + export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} +export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000} + export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04} # Randomize test order. diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index 45a29928cb..7ec66eeb4f 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -14,4 +14,4 @@ export XCODE_BUILD_ID=11C505 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" +export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror" diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index f50efcc33a..b8a9ccaae5 100644 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -10,9 +10,13 @@ export HOST=x86_64-apple-darwin16 export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well) export PIP_PACKAGES="zmq" export GOAL="install" -export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" -export TEST_RUNNER_EXTRA="wallet_disable" # Only run wallet_disable as a smoke test, see https://github.com/bitcoin/bitcoin/pull/17240#issuecomment-546022121 why the other tests are disabled -export RUN_SECURITY_TESTS="true" -# Run without depends +export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror" export NO_DEPENDS=1 export OSX_SDK="" +export CCACHE_SIZE=300M + +export RUN_SECURITY_TESTS="true" +if [ "$TRAVIS_REPO_SLUG" != "bitcoin/bitcoin" ]; then + export RUN_FUNCTIONAL_TESTS="false" + export EXPECTED_TESTS_DURATION_IN_SECONDS=200 +fi diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh new file mode 100644 index 0000000000..cc583edf17 --- /dev/null +++ b/ci/test/00_setup_env_native_msan.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export DOCKER_NAME_TAG="ubuntu:20.04" +LIBCXX_DIR="${BASE_ROOT_DIR}/ci/scratch/msan/build/" +export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" +LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" +export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" +export BDB_PREFIX="${BASE_ROOT_DIR}/db4" + +export CONTAINER_NAME="ci_native_msan" +export PACKAGES="clang-9 llvm-9 cmake" +export DEP_OPTS="NO_WALLET=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++11 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++11 ${MSAN_AND_LIBCXX_FLAGS}'" +export GOAL="install" +export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${BASE_ROOT_DIR}/depends/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'" +export USE_MEMORY_SANITIZER="true" +export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index 553dab1491..1a0b14b62b 100644 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet export DOCKER_NAME_TAG=ubuntu:16.04 # Use xenial to have one config run the tests in python3.5, see doc/dependencies.md -export PACKAGES="python3-zmq" +export PACKAGES="python3-zmq clang-3.8 llvm-3.8" # Use clang-3.8 to test C++11 compatibility, see doc/dependencies.md export DEP_OPTS="NO_WALLET=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" +export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CC=clang-3.8 CXX=clang++-3.8" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index ada7a65b8f..c4fe0eff2f 100644 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -10,6 +10,6 @@ export CONTAINER_NAME=ci_native_tsan export DOCKER_NAME_TAG=ubuntu:20.04 export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq" export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'" -export TEST_RUNNER_EXTRA="--timeout-factor=4" # Increase timeout because sanitizers slow down +export TEST_RUNNER_EXTRA="--exclude feature_block --timeout-factor=4" # Increase timeout because sanitizers slow down. Low memory on Travis machines, exclude feature_block. export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-gui=no CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++'" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 165983d906..d3566914ac 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -14,7 +14,6 @@ if [[ $QEMU_USER_CMD == qemu-s390* ]]; then fi if [ "$TRAVIS_OS_NAME" == "osx" ]; then - export PATH="/usr/local/opt/ccache/libexec:$PATH" ${CI_RETRY_EXE} pip3 install $PIP_PACKAGES fi @@ -90,6 +89,15 @@ export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" +if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then + DOCKER_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-9) 100" + DOCKER_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-9) 100" + DOCKER_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" + DOCKER_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-10.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" + DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" + DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" +fi + if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Create $BASE_ROOT_DIR" DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index de33881419..057395eba6 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -21,6 +21,14 @@ OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}" if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" fi + +if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then + # Use BDB compiled using install_db4.sh script to work around linking issue when using BDB + # from depends. See https://github.com/bitcoin/bitcoin/pull/18288#discussion_r433189350 for + # details. + DOCKER_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +fi + if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index b68cd9d3f8..17d765b862 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -37,6 +37,14 @@ END_FOLD set -o errtrace trap 'DOCKER_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR +if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then + # MemorySanitizer (MSAN) does not support tracking memory initialization done by + # using the Linux getrandom syscall. Avoid using getrandom by undefining + # HAVE_SYS_GETRANDOM. See https://github.com/google/sanitizers/issues/852 for + # details. + DOCKER_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' +fi + BEGIN_FOLD build DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) END_FOLD diff --git a/configure.ac b/configure.ac index 8f5a183dde..fb63b9fc68 100644 --- a/configure.ac +++ b/configure.ac @@ -14,6 +14,12 @@ AC_CONFIG_HEADERS([src/config/bitcoin-config.h]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) +m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR([PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh])]) +PKG_PROG_PKG_CONFIG +if test "x$PKG_CONFIG" = x; then + AC_MSG_ERROR([pkg-config not found]) +fi + BITCOIN_DAEMON_NAME=bitcoind BITCOIN_GUI_NAME=bitcoin-qt BITCOIN_CLI_NAME=bitcoin-cli @@ -229,16 +235,6 @@ AC_ARG_ENABLE([zmq], [use_zmq=$enableval], [use_zmq=yes]) -AC_ARG_ENABLE([bip70], - [AS_HELP_STRING([--enable-bip70], - [BIP70 (payment protocol) support in the GUI (no longer supported)])], - [enable_bip70=$enableval], - [enable_bip70=no]) - -if test x$enable_bip70 != xno; then - AC_MSG_ERROR([BIP70 is no longer supported!]) -fi - AC_ARG_WITH([libmultiprocess], [AS_HELP_STRING([--with-libmultiprocess=yes|no|auto], [Build with libmultiprocess library. (default: auto, i.e. detect with pkg-config)])], @@ -396,11 +392,11 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wextra],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wgnu],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wgnu"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wformat],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat"],,[[$CXXFLAG_WERROR]]) + dnl some compilers will ignore -Wformat-security without -Wformat, so just combine the two here. + AX_CHECK_COMPILE_FLAG([-Wformat -Wformat-security],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat -Wformat-security"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wvla],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wshadow-field],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wswitch],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wswitch"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wformat-security],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat-security"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wthread-safety],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) @@ -570,13 +566,8 @@ AC_ARG_WITH([daemon], [build_bitcoind=$withval], [build_bitcoind=yes]) -use_pkgconfig=yes case $host in *mingw*) - - dnl pkgconfig does more harm than good with MinGW - use_pkgconfig=no - TARGET_OS=windows AC_CHECK_LIB([kernel32], [GetModuleFileNameA],, AC_MSG_ERROR(libkernel32 missing)) AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(libuser32 missing)) @@ -680,16 +671,6 @@ case $host in ;; esac -if test x$use_pkgconfig = xyes; then - m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR(PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh.)]) - m4_ifdef([PKG_PROG_PKG_CONFIG], [ - PKG_PROG_PKG_CONFIG - if test x"$PKG_CONFIG" = "x"; then - AC_MSG_ERROR(pkg-config not found.) - fi - ]) -fi - if test x$use_extended_functional_tests != xno; then AC_SUBST(EXTENDED_FUNCTIONAL_TESTS, --extended) fi @@ -850,7 +831,21 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,, #include <byteswap.h> #endif]) -AC_CHECK_DECLS([__builtin_clz, __builtin_clzl, __builtin_clzll]) +AC_MSG_CHECKING(for __builtin_clzl) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ + (void) __builtin_clzl(0); + ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])], + [ AC_MSG_RESULT(no)] +) + +AC_MSG_CHECKING(for __builtin_clzll) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ + (void) __builtin_clzll(0); + ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])], + [ AC_MSG_RESULT(no)] +) dnl Check for malloc_info (for memory statistics information in getmemoryinfo) AC_MSG_CHECKING(for getmemoryinfo) @@ -1299,115 +1294,66 @@ CPPFLAGS="$TEMP_CPPFLAGS" fi -if test x$use_pkgconfig = xyes; then - : dnl - m4_ifdef( - [PKG_CHECK_MODULES], - [ - if test x$use_qr != xno; then - BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) - fi - if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR(libevent version 2.0.21 or greater not found.)]) - if test x$TARGET_OS != xwindows; then - PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR(libevent_pthreads version 2.0.21 or greater not found.)]) - fi - fi - - if test "x$use_zmq" = "xyes"; then - PKG_CHECK_MODULES([ZMQ],[libzmq >= 4], - [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], - [AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) - AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) - use_zmq=no]) - else - AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) - fi - ] - ) -else +dnl libevent check - if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - AC_CHECK_HEADER([event2/event.h], [use_libevent=yes], AC_MSG_ERROR(libevent headers missing),) - AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing)) - if test x$TARGET_OS != xwindows; then - AC_CHECK_LIB([event_pthreads],[main],EVENT_PTHREADS_LIBS=-levent_pthreads,AC_MSG_ERROR(libevent_pthreads missing)) - fi +if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then + PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.0.21 or greater not found.])]) + if test x$TARGET_OS != xwindows; then + PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR([libevent_pthreads version 2.0.21 or greater not found.])]) fi +fi - if test "x$use_zmq" = "xyes"; then - AC_CHECK_HEADER([zmq.h], - [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], - [AC_MSG_WARN([zmq.h not found, disabling zmq support]) - use_zmq=no - AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) - AC_CHECK_LIB([zmq],[zmq_ctx_shutdown],ZMQ_LIBS=-lzmq, - [AC_MSG_WARN([libzmq >= 4.0 not found, disabling zmq support]) - use_zmq=no - AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) - else - AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) - fi +dnl QR Code encoding library check - if test "x$use_zmq" = "xyes"; then - dnl Assume libzmq was built for static linking - case $host in - *mingw*) - ZMQ_CFLAGS="$ZMQ_CFLAGS -DZMQ_STATIC" - ;; - esac - fi +if test "x$use_qr" != xno; then + BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) +fi - if test x$use_qr != xno; then - BITCOIN_QT_CHECK([AC_CHECK_LIB([qrencode], [main],[QR_LIBS=-lqrencode], [have_qrencode=no])]) - BITCOIN_QT_CHECK([AC_CHECK_HEADER([qrencode.h],, have_qrencode=no)]) - fi +dnl ZMQ check + +if test "x$use_zmq" = xyes; then + PKG_CHECK_MODULES([ZMQ], [libzmq >= 4], + AC_DEFINE([ENABLE_ZMQ], [1], [Define to 1 to enable ZMQ functions]), + [AC_DEFINE([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) + AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) + use_zmq=no]) +else + AC_DEFINE_UNQUOTED([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) +fi + +if test "x$use_zmq" = xyes; then + dnl Assume libzmq was built for static linking + case $host in + *mingw*) + ZMQ_CFLAGS="$ZMQ_CFLAGS -DZMQ_STATIC" + ;; + esac fi dnl univalue check need_bundled_univalue=yes - if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononononono; then need_bundled_univalue=no else - -if test x$system_univalue != xno ; then - found_univalue=no - if test x$use_pkgconfig = xyes; then - : #NOP - m4_ifdef( - [PKG_CHECK_MODULES], - [ - PKG_CHECK_MODULES([UNIVALUE],[libunivalue >= 1.0.4],[found_univalue=yes],[true]) - ] - ) - else - AC_CHECK_HEADER([univalue.h],[ - AC_CHECK_LIB([univalue], [main],[ - UNIVALUE_LIBS=-lunivalue - found_univalue=yes - ],[true]) - ],[true]) + if test x$system_univalue != xno; then + PKG_CHECK_MODULES([UNIVALUE], [libunivalue >= 1.0.4], [found_univalue=yes], [found_univalue=no]) + if test x$found_univalue = xyes; then + system_univalue=yes + need_bundled_univalue=no + elif test x$system_univalue = xyes; then + AC_MSG_ERROR([univalue not found]) + else + system_univalue=no + fi fi - if test x$found_univalue = xyes ; then - system_univalue=yes - need_bundled_univalue=no - elif test x$system_univalue = xyes ; then - AC_MSG_ERROR([univalue not found]) - else - system_univalue=no + if test x$need_bundled_univalue = xyes; then + UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' + UNIVALUE_LIBS='univalue/libunivalue.la' fi fi -if test x$need_bundled_univalue = xyes ; then - UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' - UNIVALUE_LIBS='univalue/libunivalue.la' -fi - -fi - AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes]) AC_SUBST(UNIVALUE_CFLAGS) AC_SUBST(UNIVALUE_LIBS) @@ -1416,12 +1362,10 @@ dnl libmultiprocess library check libmultiprocess_found=no if test "x$with_libmultiprocess" = xyes || test "x$with_libmultiprocess" = xauto; then - if test "x$use_pkgconfig" = xyes; then - m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ - libmultiprocess_found=yes; - libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`; - ], [true])]) - fi + m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ + libmultiprocess_found=yes; + libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`; + ], [true])]) elif test "x$with_libmultiprocess" != xno; then AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no]) fi diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 5f0bffa5cb..0d4fab937d 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -5,6 +5,9 @@ XCODE_BUILD_ID=11C505 LD64_VERSION=530 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers + +# When cross-compiling for Darwin using Clang, -mlinker-version must be passed to +# ensure that modern linker features are enabled. darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION) diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index eb45e14f6f..1cd5a1749a 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -3,17 +3,23 @@ $(package)_version=2.1.11-stable $(package)_download_path=https://github.com/libevent/libevent/archive/ $(package)_file_name=release-$($(package)_version).tar.gz $(package)_sha256_hash=229393ab2bf0dc94694f21836846b424f3532585bac3468738b7bf752c03901e +$(package)_patches=0001-fix-windows-getaddrinfo.patch define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/0001-fix-windows-getaddrinfo.patch && \ ./autogen.sh endef +# When building for Windows, we set _WIN32_WINNT to target the same Windows +# version as we do in configure. Due to quirks in libevents build system, this +# is also required to enable support for ipv6. See #19375. define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress --disable-samples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_release=--disable-debug-mode $(package)_config_opts_linux=--with-pic $(package)_config_opts_android=--with-pic + $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601 endef define $(package)_config_cmds diff --git a/depends/patches/libevent/0001-fix-windows-getaddrinfo.patch b/depends/patches/libevent/0001-fix-windows-getaddrinfo.patch new file mode 100644 index 0000000000..a98cd90bd5 --- /dev/null +++ b/depends/patches/libevent/0001-fix-windows-getaddrinfo.patch @@ -0,0 +1,15 @@ +diff -ur libevent-2.1.8-stable.orig/configure.ac libevent-2.1.8-stable/configure.ac +--- libevent-2.1.8-stable.orig/configure.ac 2017-01-29 17:51:00.000000000 +0000 ++++ libevent-2.1.8-stable/configure.ac 2020-03-07 01:11:16.311335005 +0000 +@@ -389,6 +389,10 @@ + #ifdef HAVE_NETDB_H + #include <netdb.h> + #endif ++#ifdef _WIN32 ++#include <winsock2.h> ++#include <ws2tcpip.h> ++#endif + ]], + [[ + getaddrinfo; +Only in libevent-2.1.8-stable: configure.ac~ diff --git a/doc/build-windows.md b/doc/build-windows.md index d3dc467f19..28b6aceb3c 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -91,15 +91,22 @@ Note that for WSL the Bitcoin Core source path MUST be somewhere in the default example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. This means you cannot use a directory that is located directly on the host Windows file system to perform the build. +Additional WSL Note: WSL support for [launching Win32 applications](https://docs.microsoft.com/en-us/archive/blogs/wsl/windows-and-ubuntu-interoperability#launching-win32-applications-from-within-wsl) +results in `Autoconf` configure scripts being able to execute Windows Portable Executable files. This can cause +unexpected behaviour during the build, such as Win32 error dialogs for missing libraries. The recommended approach +is to temporarily disable WSL support for Win32 applications. + Build using: PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var + sudo bash -c "echo 0 > /proc/sys/fs/binfmt_misc/status" # Disable WSL support for Win32 applications. cd depends make HOST=x86_64-w64-mingw32 cd .. ./autogen.sh # not required when building from tarball CONFIG_SITE=$PWD/depends/x86_64-w64-mingw32/share/config.site ./configure --prefix=/ make + sudo bash -c "echo 1 > /proc/sys/fs/binfmt_misc/status" # Enable WSL support for Win32 applications. ## Depends system diff --git a/doc/developer-notes.md b/doc/developer-notes.md index bd3daa3202..6ae7e770e8 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -887,7 +887,7 @@ Others are external projects without a tight relationship with our project. Chan be sent upstream, but bugfixes may also be prudent to PR against Bitcoin Core so that they can be integrated quickly. Cosmetic changes should be purely taken upstream. -There is a tool in `test/lint/git-subtree-check.sh` to check a subtree directory for consistency with +There is a tool in `test/lint/git-subtree-check.sh` ([instructions](../test/lint#git-subtree-checksh)) to check a subtree directory for consistency with its upstream repository. Current subtrees include: diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 419b1db44e..c97b8d4d50 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -121,6 +121,8 @@ $ git clone https://github.com/google/afl $ make -C afl/ $ make -C afl/llvm_mode/ $ ./autogen.sh +# It is possible to compile with afl-gcc and afl-g++ instead of afl-clang. However, running afl-fuzz +# may require more memory via the -m flag. $ CC=$(pwd)/afl/afl-clang-fast CXX=$(pwd)/afl/afl-clang-fast++ ./configure --enable-fuzz $ make # For macOS you may need to ignore x86 compilation checks when running "make". If so, diff --git a/doc/reduce-traffic.md b/doc/reduce-traffic.md index ce77a00dd5..86943b1f72 100644 --- a/doc/reduce-traffic.md +++ b/doc/reduce-traffic.md @@ -23,7 +23,7 @@ longer serving historic blocks (blocks older than one week). Keep in mind that new nodes require other nodes that are willing to serve historic blocks. -Peers with the `noban` permission will never be disconnected, although their traffic counts for +Peers with the `download` permission will never be disconnected, although their traffic counts for calculating the target. ## 2. Disable "listening" (`-listen=0`) @@ -50,7 +50,7 @@ Be reminded of the effects of this setting. Doing so disables the automatic broadcasting of transactions from wallet. Not relaying other's transactions could hurt your privacy if used while a wallet is loaded or if you use the node to broadcast transactions. -- If a peer is whitelisted and "-whitelistforcerelay" is set to "1" (which will - also set "whitelistrelay" to "1"), we will still receive and relay their transactions. +- If a peer has the forcerelay permission, we will still receive and relay + their transactions. - It makes block propagation slower because compact block relay can only be used when transaction relay is enabled. diff --git a/doc/release-notes-11413.md b/doc/release-notes-11413.md deleted file mode 100644 index 32735e37f6..0000000000 --- a/doc/release-notes-11413.md +++ /dev/null @@ -1,11 +0,0 @@ -Updated or changed RPC ----------------------- - -The `bumpfee`, `fundrawtransaction`, `sendmany`, `sendtoaddress`, and `walletcreatefundedpsbt` -RPC commands have been updated to include two new fee estimation methods "BTC/kB" and "sat/B". -The target is the fee expressed explicitly in the given form. Note that use of this feature -will trigger BIP 125 (replace-by-fee) opt-in. - -In addition, the `estimate_mode` parameter is now case insensitive for all of the above RPC commands. - -The `bumpfee` command now uses `conf_target` rather than `confTarget` in the options. diff --git a/doc/release-notes-16377.md b/doc/release-notes-16377.md deleted file mode 100644 index 3442fa451b..0000000000 --- a/doc/release-notes-16377.md +++ /dev/null @@ -1,9 +0,0 @@ -RPC changes ------------ -- The `walletcreatefundedpsbt` RPC call will now fail with - `Insufficient funds` when inputs are manually selected but are not enough to cover - the outputs and fee. Additional inputs can automatically be added through the - new `add_inputs` option. - -- The `fundrawtransaction` RPC now supports `add_inputs` option that when `false` - prevents adding more inputs if necessary and consequently the RPC fails. diff --git a/doc/release-notes-16528.md b/doc/release-notes-16528.md deleted file mode 100644 index e69de29bb2..0000000000 --- a/doc/release-notes-16528.md +++ /dev/null diff --git a/doc/release-notes-18594.md b/doc/release-notes-18594.md deleted file mode 100644 index 6a2ef0a67c..0000000000 --- a/doc/release-notes-18594.md +++ /dev/null @@ -1,5 +0,0 @@ -## CLI - -The `bitcoin-cli -getinfo` command now displays the wallet name and balance for -each of the loaded wallets when more than one is loaded (e.g. in multiwallet -mode) and a wallet is not specified with `-rpcwallet`. (#18594) diff --git a/doc/release-notes-18918.md b/doc/release-notes-18918.md deleted file mode 100644 index e69de29bb2..0000000000 --- a/doc/release-notes-18918.md +++ /dev/null diff --git a/doc/release-notes-19133.md b/doc/release-notes-19133.md deleted file mode 100644 index 5150fbe1c7..0000000000 --- a/doc/release-notes-19133.md +++ /dev/null @@ -1,7 +0,0 @@ -## CLI - -A new `bitcoin-cli -generate` command, equivalent to RPC `generatenewaddress` -followed by `generatetoaddress`, can generate blocks for command line testing -purposes. This is a client-side version of the -[former](https://github.com/bitcoin/bitcoin/issues/14299) `generate` RPC. See -the help for details. (#19133) diff --git a/doc/release-notes-19200.md b/doc/release-notes-19200.md deleted file mode 100644 index 4670cb2e75..0000000000 --- a/doc/release-notes-19200.md +++ /dev/null @@ -1,7 +0,0 @@ -## Wallet - -- Backwards compatibility has been dropped for two `getaddressinfo` RPC - deprecations, as notified in the 0.20 release notes. The deprecated `label` - field has been removed as well as the deprecated `labels` behavior of - returning a JSON object containing `name` and `purpose` key-value pairs. Since - 0.20, the `labels` field returns a JSON array of label names. (#19200) diff --git a/doc/release-notes.md b/doc/release-notes.md index e73bedfb10..23983dcd7b 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -87,6 +87,28 @@ Updated RPCs whether initial broadcast of the transaction has been acknowledged by a peer. `getmempoolancestors` and `getmempooldescendants` are also updated. +- The `bumpfee`, `fundrawtransaction`, `sendmany`, `sendtoaddress`, and `walletcreatefundedpsbt` +RPC commands have been updated to include two new fee estimation methods "BTC/kB" and "sat/B". +The target is the fee expressed explicitly in the given form. Note that use of this feature +will trigger BIP 125 (replace-by-fee) opt-in. (#11413) + +- In addition, the `estimate_mode` parameter is now case insensitive for all of + the above RPC commands. (#11413) + +- The `bumpfee` command now uses `conf_target` rather than `confTarget` in the + options. (#11413) + +- The `getpeerinfo` RPC no longer returns the `banscore` field unless the configuration + option `-deprecatedrpc=banscore` is used. The `banscore` field will be fully + removed in the next major release. (#19469) + +- The `walletcreatefundedpsbt` RPC call will now fail with + `Insufficient funds` when inputs are manually selected but are not enough to cover + the outputs and fee. Additional inputs can automatically be added through the + new `add_inputs` option. (#16377) + +- The `fundrawtransaction` RPC now supports `add_inputs` option that when `false` + prevents adding more inputs if necessary and consequently the RPC fails. Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section below. @@ -99,17 +121,46 @@ Build System Updated settings ---------------- +- The `-banscore` configuration option, which modified the default threshold for + disconnecting and discouraging misbehaving peers, has been removed as part of + changes in 0.20.1 and in this release to the handling of misbehaving peers. + Refer to "Changes regarding misbehaving peers" in the 0.20.1 release notes for + details. (#19464) + - The `-debug=db` logging category, which was deprecated in 0.20 and replaced by `-debug=walletdb` to distinguish it from `coindb`, has been removed. (#19202) +- A `download` permission has been extracted from the `noban` permission. For + compatibility, `noban` implies the `download` permission, but this may change + in future releases. Refer to the help of the affected settings `-whitebind` + and `-whitelist` for more details. (#19191) + Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. +Tools and Utilities +------------------- + +- A new `bitcoin-cli -generate` command, equivalent to RPC `generatenewaddress` + followed by `generatetoaddress`, can generate blocks for command line testing + purposes. This is a client-side version of the + former `generate` RPC. See the help for details. (#19133) + +- The `bitcoin-cli -getinfo` command now displays the wallet name and balance for + each of the loaded wallets when more than one is loaded (e.g. in multiwallet + mode) and a wallet is not specified with `-rpcwallet`. (#18594) + New settings ------------ Wallet ------ +- Backwards compatibility has been dropped for two `getaddressinfo` RPC + deprecations, as notified in the 0.20 release notes. The deprecated `label` + field has been removed as well as the deprecated `labels` behavior of + returning a JSON object containing `name` and `purpose` key-value pairs. Since + 0.20, the `labels` field returns a JSON array of label names. (#19200) + - To improve wallet privacy, the frequency of wallet rebroadcast attempts is reduced from approximately once every 15 minutes to once every 12-36 hours. To maintain a similar level of guarantee for initial broadcast of wallet @@ -253,9 +304,23 @@ issue. GUI changes ----------- +- The GUI Peers window no longer displays a "Ban Score" field. This is part of + changes in 0.20.1 and in this release to the handling of misbehaving + peers. Refer to "Changes regarding misbehaving peers" in the 0.20.1 release + notes for details. (#19512) + Low-level changes ================= +RPC +--- + +- To make RPC `sendtoaddress` more consistent with `sendmany` the following error + `sendtoaddress` codes were changed from `-4` to `-6`: + - Insufficient funds + - Fee estimation failed + - Transaction has too long of a mempool chain + Tests ----- diff --git a/share/examples/bitcoin.conf b/share/examples/bitcoin.conf index 96fb6658a0..90a592cc63 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/bitcoin.conf @@ -20,8 +20,8 @@ # Bind to given address and always listen on it. Use [host]:port notation for IPv6 #bind=<addr> -# Bind to given address and whitelist peers connecting to it. Use [host]:port notation for IPv6 -#whitebind=<addr> +# Bind to given address and add permission flags to peers connecting to it. Use [host]:port notation for IPv6 +#whitebind=perm@<addr> ############################################################## ## Quick Primer on addnode vs connect ## diff --git a/src/.clang-format b/src/.clang-format index a8f8565f80..ef7a0ef5c7 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -3,7 +3,6 @@ AccessModifierOffset: -4 AlignAfterOpenBracket: true AlignEscapedNewlinesLeft: true AlignTrailingComments: true -AllowAllArgumentsOnNextLine : true AllowAllParametersOfDeclarationOnNextLine: true AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: true diff --git a/src/Makefile.am b/src/Makefile.am index 632ed3e31f..cd3cc95707 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -110,9 +110,9 @@ BITCOIN_CORE_H = \ banman.h \ base58.h \ bech32.h \ - bloom.h \ blockencodings.h \ blockfilter.h \ + bloom.h \ chain.h \ chainparams.h \ chainparamsbase.h \ @@ -133,6 +133,7 @@ BITCOIN_CORE_H = \ core_io.h \ core_memusage.h \ cuckoocache.h \ + dbwrapper.h \ flatfile.h \ fs.h \ httprpc.h \ @@ -148,7 +149,6 @@ BITCOIN_CORE_H = \ interfaces/wallet.h \ key.h \ key_io.h \ - dbwrapper.h \ limitedmap.h \ logging.h \ logging/timer.h \ @@ -167,6 +167,7 @@ BITCOIN_CORE_H = \ node/context.h \ node/psbt.h \ node/transaction.h \ + node/ui_interface.h \ node/utxo_snapshot.h \ noui.h \ optional.h \ @@ -206,13 +207,12 @@ BITCOIN_CORE_H = \ support/events.h \ support/lockedpool.h \ sync.h \ - threadsafety.h \ threadinterrupt.h \ + threadsafety.h \ timedata.h \ torcontrol.h \ txdb.h \ txmempool.h \ - ui_interface.h \ undo.h \ util/asmap.h \ util/bip32.h \ @@ -221,8 +221,6 @@ BITCOIN_CORE_H = \ util/error.h \ util/fees.h \ util/golombrice.h \ - util/spanparsing.h \ - util/system.h \ util/macros.h \ util/memory.h \ util/message.h \ @@ -230,19 +228,22 @@ BITCOIN_CORE_H = \ util/rbf.h \ util/ref.h \ util/settings.h \ + util/spanparsing.h \ util/string.h \ + util/system.h \ util/threadnames.h \ util/time.h \ util/translation.h \ + util/ui_change_type.h \ util/url.h \ util/vector.h \ validation.h \ validationinterface.h \ versionbits.h \ versionbitsinfo.h \ - walletinitinterface.h \ wallet/bdb.h \ wallet/coincontrol.h \ + wallet/coinselection.h \ wallet/context.h \ wallet/crypter.h \ wallet/db.h \ @@ -257,7 +258,7 @@ BITCOIN_CORE_H = \ wallet/walletdb.h \ wallet/wallettool.h \ wallet/walletutil.h \ - wallet/coinselection.h \ + walletinitinterface.h \ warnings.h \ zmq/zmqabstractnotifier.h \ zmq/zmqconfig.h\ @@ -286,16 +287,16 @@ libbitcoin_server_a_SOURCES = \ blockfilter.cpp \ chain.cpp \ consensus/tx_verify.cpp \ + dbwrapper.cpp \ flatfile.cpp \ httprpc.cpp \ httpserver.cpp \ index/base.cpp \ index/blockfilterindex.cpp \ index/txindex.cpp \ + init.cpp \ interfaces/chain.cpp \ interfaces/node.cpp \ - init.cpp \ - dbwrapper.cpp \ miner.cpp \ net.cpp \ net_processing.cpp \ @@ -304,6 +305,7 @@ libbitcoin_server_a_SOURCES = \ node/context.cpp \ node/psbt.cpp \ node/transaction.cpp \ + node/ui_interface.cpp \ noui.cpp \ policy/fees.cpp \ policy/rbf.cpp \ @@ -322,7 +324,6 @@ libbitcoin_server_a_SOURCES = \ torcontrol.cpp \ txdb.cpp \ txmempool.cpp \ - ui_interface.cpp \ validation.cpp \ validationinterface.cpp \ versionbits.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 9dc3078487..3b51503948 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -11,6 +11,7 @@ FUZZ_TARGETS = \ test/fuzz/asmap \ test/fuzz/asmap_direct \ test/fuzz/banentry_deserialize \ + test/fuzz/banman \ test/fuzz/base_encode_decode \ test/fuzz/bech32 \ test/fuzz/block \ @@ -33,7 +34,13 @@ FUZZ_TARGETS = \ test/fuzz/coins_deserialize \ test/fuzz/coins_view \ test/fuzz/crypto \ + test/fuzz/crypto_aes256 \ + test/fuzz/crypto_aes256cbc \ + test/fuzz/crypto_chacha20 \ + test/fuzz/crypto_chacha20_poly1305_aead \ test/fuzz/crypto_common \ + test/fuzz/crypto_hkdf_hmac_sha256_l32 \ + test/fuzz/crypto_poly1305 \ test/fuzz/cuckoocache \ test/fuzz/decode_tx \ test/fuzz/descriptor_parse \ @@ -355,6 +362,12 @@ test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_banentry_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_banman_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_banman_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_banman_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_banman_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_banman_SOURCES = test/fuzz/banman.cpp + test_fuzz_base_encode_decode_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_base_encode_decode_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_base_encode_decode_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -487,12 +500,48 @@ test_fuzz_crypto_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_crypto_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_crypto_SOURCES = test/fuzz/crypto.cpp +test_fuzz_crypto_aes256_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_aes256_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_aes256_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_aes256_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_aes256_SOURCES = test/fuzz/crypto_aes256.cpp + +test_fuzz_crypto_aes256cbc_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_aes256cbc_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_aes256cbc_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_aes256cbc_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_aes256cbc_SOURCES = test/fuzz/crypto_aes256cbc.cpp + +test_fuzz_crypto_chacha20_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_chacha20_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_chacha20_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_chacha20_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_chacha20_SOURCES = test/fuzz/crypto_chacha20.cpp + +test_fuzz_crypto_chacha20_poly1305_aead_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_chacha20_poly1305_aead_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_chacha20_poly1305_aead_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_chacha20_poly1305_aead_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_chacha20_poly1305_aead_SOURCES = test/fuzz/crypto_chacha20_poly1305_aead.cpp + test_fuzz_crypto_common_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_crypto_common_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_crypto_common_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_crypto_common_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_crypto_common_SOURCES = test/fuzz/crypto_common.cpp +test_fuzz_crypto_hkdf_hmac_sha256_l32_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_hkdf_hmac_sha256_l32_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_hkdf_hmac_sha256_l32_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_hkdf_hmac_sha256_l32_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_hkdf_hmac_sha256_l32_SOURCES = test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp + +test_fuzz_crypto_poly1305_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_poly1305_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_poly1305_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_poly1305_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_poly1305_SOURCES = test/fuzz/crypto_poly1305.cpp + test_fuzz_cuckoocache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_cuckoocache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_cuckoocache_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 835c5d6c65..f3e8a19de2 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -8,6 +8,7 @@ #include <addrman.h> #include <chainparams.h> #include <clientversion.h> +#include <cstdint> #include <hash.h> #include <random.h> #include <streams.h> @@ -36,7 +37,7 @@ template <typename Data> bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data) { // Generate random temporary filename - unsigned short randv = 0; + uint16_t randv = 0; GetRandBytes((unsigned char*)&randv, sizeof(randv)); std::string tmpfn = strprintf("%s.%04x", prefix, randv); diff --git a/src/addrdb.h b/src/addrdb.h index c6d4307d69..8410c3776c 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -17,13 +17,6 @@ class CSubNet; class CAddrMan; class CDataStream; -typedef enum BanReason -{ - BanReasonUnknown = 0, - BanReasonNodeMisbehaving = 1, - BanReasonManuallyAdded = 2 -} BanReason; - class CBanEntry { public: @@ -31,7 +24,6 @@ public: int nVersion; int64_t nCreateTime; int64_t nBanUntil; - uint8_t banReason; CBanEntry() { @@ -44,31 +36,17 @@ public: nCreateTime = nCreateTimeIn; } - explicit CBanEntry(int64_t n_create_time_in, BanReason ban_reason_in) : CBanEntry(n_create_time_in) + SERIALIZE_METHODS(CBanEntry, obj) { - banReason = ban_reason_in; + uint8_t ban_reason = 2; //! For backward compatibility + READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, ban_reason); } - SERIALIZE_METHODS(CBanEntry, obj) { READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, obj.banReason); } - void SetNull() { nVersion = CBanEntry::CURRENT_VERSION; nCreateTime = 0; nBanUntil = 0; - banReason = BanReasonUnknown; - } - - std::string banReasonToString() const - { - switch (banReason) { - case BanReasonNodeMisbehaving: - return "node misbehaving"; - case BanReasonManuallyAdded: - return "manually added"; - default: - return "unknown"; - } } }; diff --git a/src/banman.cpp b/src/banman.cpp index 9cc584f0e4..8752185a60 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -6,7 +6,7 @@ #include <banman.h> #include <netaddress.h> -#include <ui_interface.h> +#include <node/ui_interface.h> #include <util/system.h> #include <util/time.h> #include <util/translation.h> @@ -26,7 +26,7 @@ BanMan::BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t SweepBanned(); // sweep out unused entries LogPrint(BCLog::NET, "Loaded %d banned node ips/subnets from banlist.dat %dms\n", - banmap.size(), GetTimeMillis() - n_start); + m_banned.size(), GetTimeMillis() - n_start); } else { LogPrintf("Invalid or missing banlist.dat; recreating\n"); SetBannedSetDirty(true); // force write @@ -68,28 +68,13 @@ void BanMan::ClearBanned() if (m_client_interface) m_client_interface->BannedListChanged(); } -int BanMan::IsBannedLevel(CNetAddr net_addr) +bool BanMan::IsDiscouraged(const CNetAddr& net_addr) { - // Returns the most severe level of banning that applies to this address. - // 0 - Not banned - // 1 - Automatic misbehavior ban - // 2 - Any other ban - int level = 0; - auto current_time = GetTime(); LOCK(m_cs_banned); - for (const auto& it : m_banned) { - CSubNet sub_net = it.first; - CBanEntry ban_entry = it.second; - - if (current_time < ban_entry.nBanUntil && sub_net.Match(net_addr)) { - if (ban_entry.banReason != BanReasonNodeMisbehaving) return 2; - level = 1; - } - } - return level; + return m_discouraged.contains(net_addr.GetAddrBytes()); } -bool BanMan::IsBanned(CNetAddr net_addr) +bool BanMan::IsBanned(const CNetAddr& net_addr) { auto current_time = GetTime(); LOCK(m_cs_banned); @@ -104,7 +89,7 @@ bool BanMan::IsBanned(CNetAddr net_addr) return false; } -bool BanMan::IsBanned(CSubNet sub_net) +bool BanMan::IsBanned(const CSubNet& sub_net) { auto current_time = GetTime(); LOCK(m_cs_banned); @@ -118,15 +103,21 @@ bool BanMan::IsBanned(CSubNet sub_net) return false; } -void BanMan::Ban(const CNetAddr& net_addr, const BanReason& ban_reason, int64_t ban_time_offset, bool since_unix_epoch) +void BanMan::Ban(const CNetAddr& net_addr, int64_t ban_time_offset, bool since_unix_epoch) { CSubNet sub_net(net_addr); - Ban(sub_net, ban_reason, ban_time_offset, since_unix_epoch); + Ban(sub_net, ban_time_offset, since_unix_epoch); +} + +void BanMan::Discourage(const CNetAddr& net_addr) +{ + LOCK(m_cs_banned); + m_discouraged.insert(net_addr.GetAddrBytes()); } -void BanMan::Ban(const CSubNet& sub_net, const BanReason& ban_reason, int64_t ban_time_offset, bool since_unix_epoch) +void BanMan::Ban(const CSubNet& sub_net, int64_t ban_time_offset, bool since_unix_epoch) { - CBanEntry ban_entry(GetTime(), ban_reason); + CBanEntry ban_entry(GetTime()); int64_t normalized_ban_time_offset = ban_time_offset; bool normalized_since_unix_epoch = since_unix_epoch; @@ -146,8 +137,8 @@ void BanMan::Ban(const CSubNet& sub_net, const BanReason& ban_reason, int64_t ba } if (m_client_interface) m_client_interface->BannedListChanged(); - //store banlist to disk immediately if user requested ban - if (ban_reason == BanReasonManuallyAdded) DumpBanlist(); + //store banlist to disk immediately + DumpBanlist(); } bool BanMan::Unban(const CNetAddr& net_addr) diff --git a/src/banman.h b/src/banman.h index 6bea2e75e9..f6bfbd1e49 100644 --- a/src/banman.h +++ b/src/banman.h @@ -6,6 +6,7 @@ #define BITCOIN_BANMAN_H #include <addrdb.h> +#include <bloom.h> #include <fs.h> #include <net_types.h> // For banmap_t #include <sync.h> @@ -23,32 +24,55 @@ class CClientUIInterface; class CNetAddr; class CSubNet; -// Denial-of-service detection/prevention -// The idea is to detect peers that are behaving -// badly and disconnect/ban them, but do it in a -// one-coding-mistake-won't-shatter-the-entire-network -// way. -// IMPORTANT: There should be nothing I can give a -// node that it will forward on that will make that -// node's peers drop it. If there is, an attacker -// can isolate a node and/or try to split the network. -// Dropping a node for sending stuff that is invalid -// now but might be valid in a later version is also -// dangerous, because it can cause a network split -// between nodes running old code and nodes running -// new code. +// Banman manages two related but distinct concepts: +// +// 1. Banning. This is configured manually by the user, through the setban RPC. +// If an address or subnet is banned, we never accept incoming connections from +// it and never create outgoing connections to it. We won't gossip its address +// to other peers in addr messages. Banned addresses and subnets are stored to +// banlist.dat on shutdown and reloaded on startup. Banning can be used to +// prevent connections with spy nodes or other griefers. +// +// 2. Discouragement. If a peer misbehaves enough (see Misbehaving() in +// net_processing.cpp), we'll mark that address as discouraged. We still allow +// incoming connections from them, but they're preferred for eviction when +// we receive new incoming connections. We never make outgoing connections to +// them, and do not gossip their address to other peers. This is implemented as +// a bloom filter. We can (probabilistically) test for membership, but can't +// list all discouraged addresses or unmark them as discouraged. Discouragement +// can prevent our limited connection slots being used up by incompatible +// or broken peers. +// +// Neither banning nor discouragement are protections against denial-of-service +// attacks, since if an attacker has a way to waste our resources and we +// disconnect from them and ban that address, it's trivial for them to +// reconnect from another IP address. +// +// Attempting to automatically disconnect or ban any class of peer carries the +// risk of splitting the network. For example, if we banned/disconnected for a +// transaction that fails a policy check and a future version changes the +// policy check so the transaction is accepted, then that transaction could +// cause the network to split between old nodes and new nodes. class BanMan { public: ~BanMan(); BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t default_ban_time); - void Ban(const CNetAddr& net_addr, const BanReason& ban_reason, int64_t ban_time_offset = 0, bool since_unix_epoch = false); - void Ban(const CSubNet& sub_net, const BanReason& ban_reason, int64_t ban_time_offset = 0, bool since_unix_epoch = false); + void Ban(const CNetAddr& net_addr, int64_t ban_time_offset = 0, bool since_unix_epoch = false); + void Ban(const CSubNet& sub_net, int64_t ban_time_offset = 0, bool since_unix_epoch = false); + void Discourage(const CNetAddr& net_addr); void ClearBanned(); - int IsBannedLevel(CNetAddr net_addr); - bool IsBanned(CNetAddr net_addr); - bool IsBanned(CSubNet sub_net); + + //! Return whether net_addr is banned + bool IsBanned(const CNetAddr& net_addr); + + //! Return whether sub_net is exactly banned + bool IsBanned(const CSubNet& sub_net); + + //! Return whether net_addr is discouraged. + bool IsDiscouraged(const CNetAddr& net_addr); + bool Unban(const CNetAddr& net_addr); bool Unban(const CSubNet& sub_net); void GetBanned(banmap_t& banmap); @@ -68,6 +92,7 @@ private: CClientUIInterface* m_client_interface = nullptr; CBanDB m_ban_db; const int64_t m_default_ban_time; + CRollingBloomFilter m_discouraged GUARDED_BY(m_cs_banned) {50000, 0.000001}; }; #endif diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 1b75854210..1f872ce700 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> +#include <crypto/sha256.h> #include <util/strencodings.h> #include <util/system.h> @@ -35,6 +36,7 @@ int main(int argc, char** argv) { ArgsManager argsman; SetupBenchArgs(argsman); + SHA256AutoDetect(); std::string error; if (!argsman.ParseParameters(argc, argv, error)) { tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index f5125f22db..9afcda4578 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -516,8 +516,8 @@ static void ParseError(const UniValue& error, std::string& strPrint, int& nRet) */ static void GetWalletBalances(UniValue& result) { - std::unique_ptr<BaseRequestHandler> rh{MakeUnique<DefaultRequestHandler>()}; - const UniValue listwallets = ConnectAndCallRPC(rh.get(), "listwallets", /* args=*/{}); + DefaultRequestHandler rh; + const UniValue listwallets = ConnectAndCallRPC(&rh, "listwallets", /* args=*/{}); if (!find_value(listwallets, "error").isNull()) return; const UniValue& wallets = find_value(listwallets, "result"); if (wallets.size() <= 1) return; @@ -525,7 +525,7 @@ static void GetWalletBalances(UniValue& result) UniValue balances(UniValue::VOBJ); for (const UniValue& wallet : wallets.getValues()) { const std::string wallet_name = wallet.get_str(); - const UniValue getbalances = ConnectAndCallRPC(rh.get(), "getbalances", /* args=*/{}, wallet_name); + const UniValue getbalances = ConnectAndCallRPC(&rh, "getbalances", /* args=*/{}, wallet_name); const UniValue& balance = find_value(getbalances, "result")["mine"]["trusted"]; balances.pushKV(wallet_name, balance); } @@ -540,8 +540,8 @@ static UniValue GetNewAddress() { Optional<std::string> wallet_name{}; if (gArgs.IsArgSet("-rpcwallet")) wallet_name = gArgs.GetArg("-rpcwallet", ""); - std::unique_ptr<BaseRequestHandler> rh{MakeUnique<DefaultRequestHandler>()}; - return ConnectAndCallRPC(rh.get(), "getnewaddress", /* args=*/{}, wallet_name); + DefaultRequestHandler rh; + return ConnectAndCallRPC(&rh, "getnewaddress", /* args=*/{}, wallet_name); } /** diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index b8e8717896..3dcce92ab5 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -13,9 +13,9 @@ #include <init.h> #include <interfaces/chain.h> #include <node/context.h> +#include <node/ui_interface.h> #include <noui.h> #include <shutdown.h> -#include <ui_interface.h> #include <util/ref.h> #include <util/strencodings.h> #include <util/system.h> diff --git a/src/crypto/common.h b/src/crypto/common.h index e7bb020a19..5b4932c992 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -82,12 +82,12 @@ void static inline WriteBE64(unsigned char* ptr, uint64_t x) /** Return the smallest number n such that (x >> n) == 0 (or 64 if the highest bit in x is set. */ uint64_t static inline CountBits(uint64_t x) { -#if HAVE_DECL___BUILTIN_CLZL +#if HAVE_BUILTIN_CLZL if (sizeof(unsigned long) >= sizeof(uint64_t)) { return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0; } #endif -#if HAVE_DECL___BUILTIN_CLZLL +#if HAVE_BUILTIN_CLZLL if (sizeof(unsigned long long) >= sizeof(uint64_t)) { return x ? 8 * sizeof(unsigned long long) - __builtin_clzll(x) : 0; } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 116d7d8679..215b033708 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -292,18 +292,6 @@ public: // Get an estimate of LevelDB memory usage (in bytes). size_t DynamicMemoryUsage() const; - // not available for LevelDB; provide for compatibility with BDB - bool Flush() - { - return true; - } - - bool Sync() - { - CDBBatch batch(*this); - return WriteBatch(batch, true); - } - CDBIterator *NewIterator() { return new CDBIterator(*this, pdb->NewIterator(iteroptions)); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 5e78fd1d71..1e5ea2de83 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -7,10 +7,10 @@ #include <chainparamsbase.h> #include <compat.h> #include <netbase.h> +#include <node/ui_interface.h> #include <rpc/protocol.h> // For HTTP status codes #include <shutdown.h> #include <sync.h> -#include <ui_interface.h> #include <util/strencodings.h> #include <util/system.h> #include <util/threadnames.h> diff --git a/src/index/base.cpp b/src/index/base.cpp index a93b67395d..f587205a28 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -4,9 +4,9 @@ #include <chainparams.h> #include <index/base.h> +#include <node/ui_interface.h> #include <shutdown.h> #include <tinyformat.h> -#include <ui_interface.h> #include <util/system.h> #include <util/translation.h> #include <validation.h> diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 59d1888fff..64472714cc 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -3,8 +3,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <index/txindex.h> +#include <node/ui_interface.h> #include <shutdown.h> -#include <ui_interface.h> #include <util/system.h> #include <util/translation.h> #include <validation.h> diff --git a/src/init.cpp b/src/init.cpp index 8d9566edc3..9864bad291 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -31,6 +31,7 @@ #include <net_processing.h> #include <netbase.h> #include <node/context.h> +#include <node/ui_interface.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/policy.h> @@ -48,8 +49,8 @@ #include <torcontrol.h> #include <txdb.h> #include <txmempool.h> -#include <ui_interface.h> #include <util/asmap.h> +#include <util/check.h> #include <util/moneystr.h> #include <util/string.h> #include <util/system.h> @@ -395,7 +396,7 @@ void SetupServerArgs(NodeContext& node) gArgs.AddArg("-blocknotify=<cmd>", "Execute command when the best block changes (%s in cmd is replaced by block hash)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #endif gArgs.AddArg("-blockreconstructionextratxn=<n>", strprintf("Extra transactions to keep in memory for compact block reconstructions (default: %u)", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Automatic broadcast and rebroadcast of any transactions from inbound peers is disabled, unless '-whitelistforcerelay' is '1', in which case whitelisted peers' transactions will be relayed. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Automatic broadcast and rebroadcast of any transactions from inbound peers is disabled, unless the peer has the 'forcerelay' permission. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); @@ -430,8 +431,7 @@ void SetupServerArgs(NodeContext& node) gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-asmap=<file>", strprintf("Specify asn mapping used for bucketing of the peers (default: %s). Relative paths will be prefixed by the net-specific datadir location.", DEFAULT_ASMAP_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-bantime=<n>", strprintf("Number of seconds to keep misbehaving peers from reconnecting (default: %u)", DEFAULT_MISBEHAVING_BANTIME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-bantime=<n>", strprintf("Default duration (in seconds) of manually configured bans (default: %u)", DEFAULT_MISBEHAVING_BANTIME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-bind=<addr>", "Bind to given address and always listen on it. Use [host]:port notation for IPv6", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-connect=<ip>", "Connect only to the specified node; -noconnect disables automatic connections (the rules for this peer are the same as for -addnode). This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-discover", "Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -445,7 +445,7 @@ void SetupServerArgs(NodeContext& node) gArgs.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by peers forward or backward by this amount. (default: %u seconds)", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h). Limit does not apply to peers with 'noban' permission. 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h). Limit does not apply to peers with 'download' permission. 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor hidden services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-onlynet=<net>", "Make outgoing connections only through network <net> (ipv4, ipv6 or onion). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -468,12 +468,12 @@ void SetupServerArgs(NodeContext& node) #else hidden_args.emplace_back("-upnp"); #endif - gArgs.AddArg("-whitebind=<[permissions@]addr>", "Bind to given address and whitelist peers connecting to it. " + gArgs.AddArg("-whitebind=<[permissions@]addr>", "Bind to the given address and add permission flags to the peers connecting to it. " "Use [host]:port notation for IPv6. Allowed permissions: " + Join(NET_PERMISSIONS_DOC, ", ") + ". " - "Specify multiple permissions separated by commas (default: noban,mempool,relay). Can be specified multiple times.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + "Specify multiple permissions separated by commas (default: download,noban,mempool,relay). Can be specified multiple times.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-whitelist=<[permissions@]IP address or network>", "Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) or " - "CIDR notated network(e.g. 1.2.3.0/24). Uses same permissions as " + gArgs.AddArg("-whitelist=<[permissions@]IP address or network>", "Add permission flags to the peers connecting from the given IP address (e.g. 1.2.3.4) or " + "CIDR-notated network (e.g. 1.2.3.0/24). Uses the same permissions as " "-whitebind. Can be specified multiple times." , ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); g_wallet_init_interface.AddWalletOptions(); @@ -774,13 +774,14 @@ static bool InitSanityCheck() return true; } -static bool AppInitServers(const util::Ref& context) +static bool AppInitServers(const util::Ref& context, NodeContext& node) { RPCServer::OnStarted(&OnRPCStarted); RPCServer::OnStopped(&OnRPCStopped); if (!InitHTTPServer()) return false; StartRPC(); + node.rpc_interruption_point = RpcInterruptionPoint; if (!StartHTTPRPC(context)) return false; if (gArgs.GetBoolArg("-rest", DEFAULT_REST_ENABLE)) StartREST(context); @@ -1317,8 +1318,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) node.scheduler = MakeUnique<CScheduler>(); // Start the lightweight task scheduler thread - CScheduler::Function serviceLoop = [&node]{ node.scheduler->serviceQueue(); }; - threadGroup.create_thread(std::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); + threadGroup.create_thread([&] { TraceThread("scheduler", [&] { node.scheduler->serviceQueue(); }); }); // Gather some entropy once per minute. node.scheduler->scheduleEvery([]{ @@ -1352,7 +1352,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) if (gArgs.GetBoolArg("-server", false)) { uiInterface.InitMessage_connect(SetRPCWarmupStatus); - if (!AppInitServers(context)) + if (!AppInitServers(context, node)) return InitError(_("Unable to start HTTP server. See debug log for details.")); } @@ -1379,9 +1379,9 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) node.mempool = &::mempool; assert(!node.chainman); node.chainman = &g_chainman; - ChainstateManager& chainman = EnsureChainman(node); + ChainstateManager& chainman = *Assert(node.chainman); - node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), *node.scheduler, *node.chainman, *node.mempool)); + node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), *node.scheduler, chainman, *node.mempool)); RegisterValidationInterface(node.peer_logic.get()); // sanitize comments per BIP-0014, format user agent and check total size @@ -1589,7 +1589,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) // If the loaded chain has a wrong genesis, bail out immediately // (we're likely using a testnet datadir, or the other way around). - if (!::BlockIndex().empty() && + if (!chainman.BlockIndex().empty() && !LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) { return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); } @@ -1869,8 +1869,8 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) //// debug print { LOCK(cs_main); - LogPrintf("block tree size = %u\n", ::BlockIndex().size()); - chain_active_height = ::ChainActive().Height(); + LogPrintf("block tree size = %u\n", chainman.BlockIndex().size()); + chain_active_height = chainman.ActiveChain().Height(); } LogPrintf("nBestHeight = %d\n", chain_active_height); diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index d1e04b114d..d49e4454af 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -13,6 +13,7 @@ #include <node/coin.h> #include <node/context.h> #include <node/transaction.h> +#include <node/ui_interface.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -25,7 +26,6 @@ #include <sync.h> #include <timedata.h> #include <txmempool.h> -#include <ui_interface.h> #include <uint256.h> #include <univalue.h> #include <util/system.h> diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 65695707f7..bbeb0fa801 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -15,6 +15,7 @@ #include <string> #include <vector> +class ArgsManager; class CBlock; class CFeeRate; class CRPCCommand; @@ -322,7 +323,7 @@ std::unique_ptr<Chain> MakeChain(NodeContext& node); //! analysis, or fee estimation. These clients need to expose their own //! MakeXXXClient functions returning their implementations of the ChainClient //! interface. -std::unique_ptr<ChainClient> MakeWalletClient(Chain& chain, std::vector<std::string> wallet_filenames); +std::unique_ptr<ChainClient> MakeWalletClient(Chain& chain, ArgsManager& args, std::vector<std::string> wallet_filenames); } // namespace interfaces diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index d420788dbe..834a16ecf5 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -17,6 +17,7 @@ #include <netaddress.h> #include <netbase.h> #include <node/context.h> +#include <node/ui_interface.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/settings.h> @@ -26,7 +27,6 @@ #include <support/allocators/secure.h> #include <sync.h> #include <txmempool.h> -#include <ui_interface.h> #include <util/ref.h> #include <util/system.h> #include <util/translation.h> @@ -146,10 +146,10 @@ public: } return false; } - bool ban(const CNetAddr& net_addr, BanReason reason, int64_t ban_time_offset) override + bool ban(const CNetAddr& net_addr, int64_t ban_time_offset) override { if (m_context.banman) { - m_context.banman->Ban(net_addr, reason, ban_time_offset); + m_context.banman->Ban(net_addr, ban_time_offset); return true; } return false; diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 877a40568f..b88b5bc14e 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -122,7 +122,7 @@ public: virtual bool getBanned(banmap_t& banmap) = 0; //! Ban node. - virtual bool ban(const CNetAddr& net_addr, BanReason reason, int64_t ban_time_offset) = 0; + virtual bool ban(const CNetAddr& net_addr, int64_t ban_time_offset) = 0; //! Unban node. virtual bool unban(const CSubNet& ip) = 0; diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index b65eb72b1c..7fd24425cf 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -13,11 +13,11 @@ #include <script/standard.h> #include <support/allocators/secure.h> #include <sync.h> -#include <ui_interface.h> #include <uint256.h> #include <util/check.h> #include <util/ref.h> #include <util/system.h> +#include <util/ui_change_type.h> #include <wallet/context.h> #include <wallet/feebumper.h> #include <wallet/fees.h> @@ -438,7 +438,6 @@ public: bool canGetAddresses() override { return m_wallet->CanGetAddresses(); } bool privateKeysDisabled() override { return m_wallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); } OutputType getDefaultAddressType() override { return m_wallet->m_default_address_type; } - OutputType getDefaultChangeType() override { return m_wallet->m_default_change_type; } CAmount getDefaultMaxTxFee() override { return m_wallet->m_default_max_tx_fee; } void remove() override { @@ -484,10 +483,11 @@ public: class WalletClientImpl : public ChainClient { public: - WalletClientImpl(Chain& chain, std::vector<std::string> wallet_filenames) + WalletClientImpl(Chain& chain, ArgsManager& args, std::vector<std::string> wallet_filenames) : m_wallet_filenames(std::move(wallet_filenames)) { m_context.chain = &chain; + m_context.args = &args; } void registerRpcs() override { @@ -500,7 +500,7 @@ public: } bool verify() override { return VerifyWallets(*m_context.chain, m_wallet_filenames); } bool load() override { return LoadWallets(*m_context.chain, m_wallet_filenames); } - void start(CScheduler& scheduler) override { return StartWallets(scheduler); } + void start(CScheduler& scheduler) override { return StartWallets(scheduler, *Assert(m_context.args)); } void flush() override { return FlushWallets(); } void stop() override { return StopWallets(); } void setMockTime(int64_t time) override { return SetMockTime(time); } @@ -515,7 +515,7 @@ public: ~WalletClientImpl() override { UnloadWallets(); } WalletContext m_context; - std::vector<std::string> m_wallet_filenames; + const std::vector<std::string> m_wallet_filenames; std::vector<std::unique_ptr<Handler>> m_rpc_handlers; std::list<CRPCCommand> m_rpc_commands; }; @@ -524,9 +524,9 @@ public: std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet) { return wallet ? MakeUnique<WalletImpl>(wallet) : nullptr; } -std::unique_ptr<ChainClient> MakeWalletClient(Chain& chain, std::vector<std::string> wallet_filenames) +std::unique_ptr<ChainClient> MakeWalletClient(Chain& chain, ArgsManager& args, std::vector<std::string> wallet_filenames) { - return MakeUnique<WalletClientImpl>(chain, std::move(wallet_filenames)); + return MakeUnique<WalletClientImpl>(chain, args, std::move(wallet_filenames)); } } // namespace interfaces diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index e2161521f6..3cdadbc72e 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -9,8 +9,8 @@ #include <pubkey.h> // For CKeyID and CScriptID (definitions needed in CTxDestination instantiation) #include <script/standard.h> // For CTxDestination #include <support/allocators/secure.h> // For SecureString -#include <ui_interface.h> // For ChangeType #include <util/message.h> +#include <util/ui_change_type.h> #include <functional> #include <map> @@ -256,9 +256,6 @@ public: // Get default address type. virtual OutputType getDefaultAddressType() = 0; - // Get default change type. - virtual OutputType getDefaultChangeType() = 0; - //! Get max tx fee. virtual CAmount getDefaultMaxTxFee() = 0; diff --git a/src/miner.cpp b/src/miner.cpp index d9dcbe8a70..41a835f70a 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -109,7 +109,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc if(!pblocktemplate.get()) return nullptr; - pblock = &pblocktemplate->block; // pointer for convenience + CBlock* const pblock = &pblocktemplate->block; // pointer for convenience // Add dummy coinbase tx as first transaction pblock->vtx.emplace_back(); @@ -226,7 +226,7 @@ bool BlockAssembler::TestPackageTransactions(const CTxMemPool::setEntries& packa void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) { - pblock->vtx.emplace_back(iter->GetSharedTx()); + pblocktemplate->block.vtx.emplace_back(iter->GetSharedTx()); pblocktemplate->vTxFees.push_back(iter->GetFee()); pblocktemplate->vTxSigOpsCost.push_back(iter->GetSigOpCost()); nBlockWeight += iter->GetTxWeight(); diff --git a/src/miner.h b/src/miner.h index 69296f9078..096585dfe4 100644 --- a/src/miner.h +++ b/src/miner.h @@ -128,8 +128,6 @@ class BlockAssembler private: // The constructed block template std::unique_ptr<CBlockTemplate> pblocktemplate; - // A convenience pointer that always refers to the CBlock in pblocktemplate - CBlock* pblock; // Configuration parameters for the block size bool fIncludeWitness; diff --git a/src/net.cpp b/src/net.cpp index 371fbeed59..cf5757d6c0 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -14,12 +14,12 @@ #include <clientversion.h> #include <consensus/consensus.h> #include <crypto/sha256.h> -#include <netbase.h> #include <net_permissions.h> +#include <netbase.h> +#include <node/ui_interface.h> #include <protocol.h> #include <random.h> #include <scheduler.h> -#include <ui_interface.h> #include <util/strencodings.h> #include <util/translation.h> @@ -42,6 +42,7 @@ static_assert(MINIUPNPC_API_VERSION >= 10, "miniUPnPc API version >= 10 assumed"); #endif +#include <cstdint> #include <unordered_map> #include <math.h> @@ -110,9 +111,9 @@ void CConnman::AddOneShot(const std::string& strDest) vOneShots.push_back(strDest); } -unsigned short GetListenPort() +uint16_t GetListenPort() { - return (unsigned short)(gArgs.GetArg("-port", Params().GetDefaultPort())); + return (uint16_t)(gArgs.GetArg("-port", Params().GetDefaultPort())); } // find 'best' local address for a particular peer @@ -563,15 +564,15 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) // since pingtime does not update until the ping is complete, which might take a while. // So, if a ping is taking an unusually long time in flight, // the caller can immediately detect that this is happening. - int64_t nPingUsecWait = 0; - if ((0 != nPingNonceSent) && (0 != nPingUsecStart)) { - nPingUsecWait = GetTimeMicros() - nPingUsecStart; + std::chrono::microseconds ping_wait{0}; + if ((0 != nPingNonceSent) && (0 != m_ping_start.load().count())) { + ping_wait = GetTime<std::chrono::microseconds>() - m_ping_start.load(); } // Raw ping time is in microseconds, but show it to user as whole seconds (Bitcoin users should be well used to small numbers with many decimal places by now :) stats.m_ping_usec = nPingUsecTime; stats.m_min_ping_usec = nMinPingUsecTime; - stats.m_ping_wait_usec = nPingUsecWait; + stats.m_ping_wait_usec = count_microseconds(ping_wait); // Leave string empty if addrLocal invalid (not filled in yet) CService addrLocalUnlocked = GetAddrLocal(); @@ -582,9 +583,9 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete) { complete = false; - int64_t nTimeMicros = GetTimeMicros(); + const auto time = GetTime<std::chrono::microseconds>(); LOCK(cs_vRecv); - nLastRecv = nTimeMicros / 1000000; + nLastRecv = std::chrono::duration_cast<std::chrono::seconds>(time).count(); nRecvBytes += nBytes; while (nBytes > 0) { // absorb network data @@ -596,7 +597,7 @@ bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete if (m_deserializer->Complete()) { // decompose a transport agnostic CNetMessage from the deserializer - CNetMessage msg = m_deserializer->GetMessage(Params().MessageStart(), nTimeMicros); + CNetMessage msg = m_deserializer->GetMessage(Params().MessageStart(), time); //store received bytes per message command //to prevent a memory DOS, only allow valid commands @@ -699,7 +700,8 @@ const uint256& V1TransportDeserializer::GetMessageHash() const return data_hash; } -CNetMessage V1TransportDeserializer::GetMessage(const CMessageHeader::MessageStartChars& message_start, int64_t time) { +CNetMessage V1TransportDeserializer::GetMessage(const CMessageHeader::MessageStartChars& message_start, const std::chrono::microseconds time) +{ // decompose a single CNetMessage from the TransportDeserializer CNetMessage msg(std::move(vRecv)); @@ -1010,17 +1012,24 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { // on all platforms. Set it again here just to be sure. SetSocketNoDelay(hSocket); - int bannedlevel = m_banman ? m_banman->IsBannedLevel(addr) : 0; - - // Don't accept connections from banned peers, but if our inbound slots aren't almost full, accept - // if the only banning reason was an automatic misbehavior ban. - if (!NetPermissions::HasFlag(permissionFlags, NetPermissionFlags::PF_NOBAN) && bannedlevel > ((nInbound + 1 < nMaxInbound) ? 1 : 0)) + // Don't accept connections from banned peers. + bool banned = m_banman && m_banman->IsBanned(addr); + if (!NetPermissions::HasFlag(permissionFlags, NetPermissionFlags::PF_NOBAN) && banned) { LogPrint(BCLog::NET, "connection from %s dropped (banned)\n", addr.ToString()); CloseSocket(hSocket); return; } + // Only accept connections from discouraged peers if our inbound slots aren't (almost) full. + bool discouraged = m_banman && m_banman->IsDiscouraged(addr); + if (!NetPermissions::HasFlag(permissionFlags, NetPermissionFlags::PF_NOBAN) && nInbound + 1 >= nMaxInbound && discouraged) + { + LogPrint(BCLog::NET, "connection from %s dropped (discouraged)\n", addr.ToString()); + CloseSocket(hSocket); + return; + } + if (nInbound >= nMaxInbound) { if (!AttemptToEvictConnection()) { @@ -1044,7 +1053,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { pnode->m_permissionFlags = permissionFlags; // If this flag is present, the user probably expect that RPC and QT report it as whitelisted (backward compatibility) pnode->m_legacyWhitelisted = legacyWhitelisted; - pnode->m_prefer_evict = bannedlevel > 0; + pnode->m_prefer_evict = discouraged; m_msgproc->InitializeNode(pnode); LogPrint(BCLog::NET, "connection from %s accepted\n", addr.ToString()); @@ -1103,12 +1112,9 @@ void CConnman::DisconnectNodes() if (pnode->GetRefCount() <= 0) { bool fDelete = false; { - TRY_LOCK(pnode->cs_inventory, lockInv); - if (lockInv) { - TRY_LOCK(pnode->cs_vSend, lockSend); - if (lockSend) { - fDelete = true; - } + TRY_LOCK(pnode->cs_vSend, lockSend); + if (lockSend) { + fDelete = true; } } if (fDelete) { @@ -1154,9 +1160,9 @@ void CConnman::InactivityCheck(CNode *pnode) LogPrintf("socket receive timeout: %is\n", nTime - pnode->nLastRecv); pnode->fDisconnect = true; } - else if (pnode->nPingNonceSent && pnode->nPingUsecStart + TIMEOUT_INTERVAL * 1000000 < GetTimeMicros()) + else if (pnode->nPingNonceSent && pnode->m_ping_start.load() + std::chrono::seconds{TIMEOUT_INTERVAL} < GetTime<std::chrono::microseconds>()) { - LogPrintf("ping timeout: %fs\n", 0.000001 * (GetTimeMicros() - pnode->nPingUsecStart)); + LogPrintf("ping timeout: %fs\n", 0.000001 * count_microseconds(GetTime<std::chrono::microseconds>() - pnode->m_ping_start.load())); pnode->fDisconnect = true; } else if (!pnode->fSuccessfullyConnected) @@ -2045,10 +2051,10 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai return; } if (!pszDest) { - if (IsLocal(addrConnect) || - FindNode(static_cast<CNetAddr>(addrConnect)) || (m_banman && m_banman->IsBanned(addrConnect)) || - FindNode(addrConnect.ToStringIPPort())) + bool banned_or_discouraged = m_banman && (m_banman->IsDiscouraged(addrConnect) || m_banman->IsBanned(addrConnect)); + if (IsLocal(addrConnect) || FindNode(static_cast<CNetAddr>(addrConnect)) || banned_or_discouraged || FindNode(addrConnect.ToStringIPPort())) { return; + } } else if (FindNode(std::string(pszDest))) return; @@ -2505,11 +2511,6 @@ CConnman::~CConnman() Stop(); } -size_t CConnman::GetAddressCount() const -{ - return addrman.size(); -} - void CConnman::SetServices(const CService &addr, ServiceFlags nServices) { addrman.SetServices(addr, nServices); @@ -2639,7 +2640,7 @@ void CConnman::RecordBytesSent(uint64_t bytes) nMaxOutboundTotalBytesSentInCycle = 0; } - // TODO, exclude peers with noban permission + // TODO, exclude peers with download permission nMaxOutboundTotalBytesSentInCycle += bytes; } @@ -25,8 +25,8 @@ #include <uint256.h> #include <atomic> +#include <cstdint> #include <deque> -#include <stdint.h> #include <thread> #include <memory> #include <condition_variable> @@ -247,7 +247,6 @@ public: }; // Addrman functions - size_t GetAddressCount() const; void SetServices(const CService &addr, ServiceFlags nServices); void MarkAddressGood(const CAddress& addr); void AddNewAddresses(const std::vector<CAddress>& vAddr, const CAddress& addrFrom, int64_t nTimePenalty = 0); @@ -448,6 +447,7 @@ private: std::atomic<int> nBestHeight; CClientUIInterface* clientInterface; NetEventsInterface* m_msgproc; + /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ BanMan* m_banman; /** SipHasher seeds for deterministic randomness */ @@ -482,7 +482,7 @@ void Discover(); void StartMapPort(); void InterruptMapPort(); void StopMapPort(); -unsigned short GetListenPort(); +uint16_t GetListenPort(); struct CombinerAll { @@ -612,13 +612,13 @@ public: */ class CNetMessage { public: - CDataStream m_recv; // received message data - int64_t m_time = 0; // time (in microseconds) of message receipt. + CDataStream m_recv; //!< received message data + std::chrono::microseconds m_time{0}; //!< time of message receipt bool m_valid_netmagic = false; bool m_valid_header = false; bool m_valid_checksum = false; - uint32_t m_message_size = 0; // size of the payload - uint32_t m_raw_message_size = 0; // used wire size of the message (including header/checksum) + uint32_t m_message_size{0}; //!< size of the payload + uint32_t m_raw_message_size{0}; //!< used wire size of the message (including header/checksum) std::string m_command; CNetMessage(CDataStream&& recv_in) : m_recv(std::move(recv_in)) {} @@ -642,7 +642,7 @@ public: // read and deserialize data virtual int Read(const char *data, unsigned int bytes) = 0; // decomposes a message from the context - virtual CNetMessage GetMessage(const CMessageHeader::MessageStartChars& message_start, int64_t time) = 0; + virtual CNetMessage GetMessage(const CMessageHeader::MessageStartChars& message_start, std::chrono::microseconds time) = 0; virtual ~TransportDeserializer() {} }; @@ -695,7 +695,7 @@ public: if (ret < 0) Reset(); return ret; } - CNetMessage GetMessage(const CMessageHeader::MessageStartChars& message_start, int64_t time) override; + CNetMessage GetMessage(const CMessageHeader::MessageStartChars& message_start, std::chrono::microseconds time) override; }; /** The TransportSerializer prepares messages for the network transport @@ -803,7 +803,7 @@ public: // There is no final sorting before sending, as they are always sent immediately // and in the order requested. std::vector<uint256> vInventoryBlockToSend GUARDED_BY(cs_inventory); - RecursiveMutex cs_inventory; + Mutex cs_inventory; struct TxRelay { mutable RecursiveMutex cs_filter; @@ -845,8 +845,8 @@ public: // Ping time measurement: // The pong reply we're expecting, or 0 if no pong expected. std::atomic<uint64_t> nPingNonceSent{0}; - // Time (in usec) the last ping was sent, or 0 if no ping was ever sent. - std::atomic<int64_t> nPingUsecStart{0}; + /** When the last ping was sent, or 0 if no ping was ever sent */ + std::atomic<std::chrono::microseconds> m_ping_start{std::chrono::microseconds{0}}; // Last measured round-trip time. std::atomic<int64_t> nPingUsecTime{0}; // Best measured round-trip time. @@ -982,18 +982,6 @@ public: } } - void PushBlockInventory(const uint256& hash) - { - LOCK(cs_inventory); - vInventoryBlockToSend.push_back(hash); - } - - void PushBlockHash(const uint256 &hash) - { - LOCK(cs_inventory); - vBlockHashesToAnnounce.push_back(hash); - } - void CloseSocketDisconnect(); void copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap); diff --git a/src/net_permissions.cpp b/src/net_permissions.cpp index da09149856..a75838307c 100644 --- a/src/net_permissions.cpp +++ b/src/net_permissions.cpp @@ -10,10 +10,11 @@ const std::vector<std::string> NET_PERMISSIONS_DOC{ "bloomfilter (allow requesting BIP37 filtered blocks and transactions)", - "noban (do not ban for misbehavior)", + "noban (do not ban for misbehavior; implies download)", "forcerelay (relay transactions that are already in the mempool; implies relay)", "relay (relay even in -blocksonly mode)", "mempool (allow requesting BIP35 mempool contents)", + "download (allow getheaders during IBD, no disconnect after maxuploadtarget limit)", }; namespace { @@ -46,6 +47,7 @@ bool TryParsePermissionFlags(const std::string str, NetPermissionFlags& output, else if (permission == "noban") NetPermissions::AddFlag(flags, PF_NOBAN); else if (permission == "forcerelay") NetPermissions::AddFlag(flags, PF_FORCERELAY); else if (permission == "mempool") NetPermissions::AddFlag(flags, PF_MEMPOOL); + else if (permission == "download") NetPermissions::AddFlag(flags, PF_DOWNLOAD); else if (permission == "all") NetPermissions::AddFlag(flags, PF_ALL); else if (permission == "relay") NetPermissions::AddFlag(flags, PF_RELAY); else if (permission.length() == 0); // Allow empty entries @@ -72,6 +74,7 @@ std::vector<std::string> NetPermissions::ToStrings(NetPermissionFlags flags) if (NetPermissions::HasFlag(flags, PF_FORCERELAY)) strings.push_back("forcerelay"); if (NetPermissions::HasFlag(flags, PF_RELAY)) strings.push_back("relay"); if (NetPermissions::HasFlag(flags, PF_MEMPOOL)) strings.push_back("mempool"); + if (NetPermissions::HasFlag(flags, PF_DOWNLOAD)) strings.push_back("download"); return strings; } diff --git a/src/net_permissions.h b/src/net_permissions.h index e004067e75..a9633ee2ae 100644 --- a/src/net_permissions.h +++ b/src/net_permissions.h @@ -14,8 +14,7 @@ struct bilingual_str; extern const std::vector<std::string> NET_PERMISSIONS_DOC; -enum NetPermissionFlags -{ +enum NetPermissionFlags { PF_NONE = 0, // Can query bloomfilter even if -peerbloomfilters is false PF_BLOOMFILTER = (1U << 1), @@ -24,14 +23,16 @@ enum NetPermissionFlags // Always relay transactions from this peer, even if already in mempool // Keep parameter interaction: forcerelay implies relay PF_FORCERELAY = (1U << 2) | PF_RELAY, - // Can't be banned for misbehavior - PF_NOBAN = (1U << 4), + // Allow getheaders during IBD and block-download after maxuploadtarget limit + PF_DOWNLOAD = (1U << 6), + // Can't be banned/disconnected/discouraged for misbehavior + PF_NOBAN = (1U << 4) | PF_DOWNLOAD, // Can query the mempool PF_MEMPOOL = (1U << 5), // True if the user did not specifically set fine grained permissions PF_ISIMPLICIT = (1U << 31), - PF_ALL = PF_BLOOMFILTER | PF_FORCERELAY | PF_RELAY | PF_NOBAN | PF_MEMPOOL, + PF_ALL = PF_BLOOMFILTER | PF_FORCERELAY | PF_RELAY | PF_NOBAN | PF_MEMPOOL | PF_DOWNLOAD, }; class NetPermissions diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 80e58a6dba..7a58de35d7 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -13,10 +13,9 @@ #include <consensus/validation.h> #include <hash.h> #include <index/blockfilterindex.h> -#include <validation.h> #include <merkleblock.h> -#include <netmessagemaker.h> #include <netbase.h> +#include <netmessagemaker.h> #include <policy/fees.h> #include <policy/policy.h> #include <primitives/block.h> @@ -26,22 +25,22 @@ #include <scheduler.h> #include <tinyformat.h> #include <txmempool.h> -#include <util/system.h> +#include <util/check.h> // For NDEBUG compile time check #include <util/strencodings.h> +#include <util/system.h> +#include <validation.h> #include <memory> #include <typeinfo> -#if defined(NDEBUG) -# error "Bitcoin cannot be compiled without assertions." -#endif - /** Expiration time for orphan transactions in seconds */ static constexpr int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60; /** Minimum time between orphan transactions expire time checks in seconds */ static constexpr int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; /** How long to cache transactions in mapRelay for normal relay */ -static constexpr std::chrono::seconds RELAY_TX_CACHE_TIME{15 * 60}; +static constexpr std::chrono::seconds RELAY_TX_CACHE_TIME = std::chrono::minutes{15}; +/** How long a transaction has to be in the mempool before it can unconditionally be relayed (even when not in mapRelay). */ +static constexpr std::chrono::seconds UNCONDITIONAL_RELAY_DELAY = std::chrono::minutes{2}; /** Headers download timeout expressed in microseconds * Timeout = base + per_header * (expected number of headers) */ static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_BASE = 15 * 60 * 1000000; // 15 minutes @@ -66,8 +65,8 @@ static constexpr int STALE_RELAY_AGE_LIMIT = 30 * 24 * 60 * 60; /// Age after which a block is considered historical for purposes of rate /// limiting block relay. Set to one week, denominated in seconds. static constexpr int HISTORICAL_BLOCK_AGE = 7 * 24 * 60 * 60; -/** Time between pings automatically sent out for latency probing and keepalive (in seconds). */ -static const int PING_INTERVAL = 2 * 60; +/** Time between pings automatically sent out for latency probing and keepalive */ +static constexpr std::chrono::minutes PING_INTERVAL{2}; /** The maximum number of entries in a locator */ static const unsigned int MAX_LOCATOR_SZ = 101; /** The maximum number of entries in an 'inv' protocol message */ @@ -120,11 +119,20 @@ static constexpr std::chrono::hours AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL{24}; /** Average delay between peer address broadcasts */ static constexpr std::chrono::seconds AVG_ADDRESS_BROADCAST_INTERVAL{30}; /** Average delay between trickled inventory transmissions in seconds. - * Blocks and whitelisted receivers bypass this, outbound peers get half this delay. */ + * Blocks and peers with noban permission bypass this, outbound peers get half this delay. */ static const unsigned int INVENTORY_BROADCAST_INTERVAL = 5; -/** Maximum number of inventory items to send per transmission. +/** Maximum rate of inventory items to send per second. * Limits the impact of low-fee transaction floods. */ -static constexpr unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_INTERVAL; +static constexpr unsigned int INVENTORY_BROADCAST_PER_SECOND = 7; +/** Maximum number of inventory items to send per transmission. */ +static constexpr unsigned int INVENTORY_BROADCAST_MAX = INVENTORY_BROADCAST_PER_SECOND * INVENTORY_BROADCAST_INTERVAL; +/** The number of most recently announced transactions a peer can request. */ +static constexpr unsigned int INVENTORY_MAX_RECENT_RELAY = 3500; +/** Verify that INVENTORY_MAX_RECENT_RELAY is enough to cache everything typically + * relayed before unconditional relay from the mempool kicks in. This is only a + * lower bound, and it should be larger to account for higher inv rate to outbound + * peers, and random variations in the broadcast mechanism. */ +static_assert(INVENTORY_MAX_RECENT_RELAY >= INVENTORY_BROADCAST_PER_SECOND * UNCONDITIONAL_RELAY_DELAY / std::chrono::seconds{1}, "INVENTORY_RELAY_MAX too low"); /** Average delay between feefilter broadcasts in seconds. */ static constexpr unsigned int AVG_FEEFILTER_BROADCAST_INTERVAL = 10 * 60; /** Maximum feefilter broadcast delay after significant change. */ @@ -252,8 +260,8 @@ struct CNodeState { bool fCurrentlyConnected; //! Accumulated misbehaviour score for this peer. int nMisbehavior; - //! Whether this peer should be disconnected and banned (unless whitelisted). - bool fShouldBan; + //! Whether this peer should be disconnected and marked as discouraged (unless it has the noban permission). + bool m_should_discourage; //! String name of this peer (debugging/logging purposes). const std::string name; //! The best known block we know this peer has announced. @@ -398,13 +406,16 @@ struct CNodeState { //! Whether this peer is a manual connection bool m_is_manual_connection; + //! A rolling bloom filter of all announced tx CInvs to this peer. + CRollingBloomFilter m_recently_announced_invs = CRollingBloomFilter{INVENTORY_MAX_RECENT_RELAY, 0.000001}; + CNodeState(CAddress addrIn, std::string addrNameIn, bool is_inbound, bool is_manual) : address(addrIn), name(std::move(addrNameIn)), m_is_inbound(is_inbound), m_is_manual_connection (is_manual) { fCurrentlyConnected = false; nMisbehavior = 0; - fShouldBan = false; + m_should_discourage = false; pindexBestKnownBlock = nullptr; hashLastUnknownBlock.SetNull(); pindexLastCommonBlock = nullptr; @@ -425,6 +436,7 @@ struct CNodeState { fSupportsDesiredCmpctVersion = false; m_chain_sync = { 0, nullptr, false, false }; m_last_block_announcement = 0; + m_recently_announced_invs.reset(); } }; @@ -451,7 +463,7 @@ static void UpdatePreferredDownload(const CNode& node, CNodeState* state) EXCLUS nPreferredDownload += state->fPreferredDownload; } -static void PushNodeVersion(CNode& pnode, CConnman* connman, int64_t nTime) +static void PushNodeVersion(CNode& pnode, CConnman& connman, int64_t nTime) { // Note that pnode->GetLocalServices() is a reflection of the local // services we were offering when the CNode object was created for this @@ -465,7 +477,7 @@ static void PushNodeVersion(CNode& pnode, CConnman* connman, int64_t nTime) CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService(), addr.nServices)); CAddress addrMe = CAddress(CService(), nLocalNodeServices); - connman->PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, + connman.PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, nonce, strSubVersion, nNodeStartingHeight, ::g_relay_txes && pnode.m_tx_relay != nullptr)); if (fLogIPs) { @@ -576,7 +588,7 @@ static void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) EXCLUSIV * lNodesAnnouncingHeaderAndIDs, and keeping that list under a certain size by * removing the first element if necessary. */ -static void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman* connman) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman& connman) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(cs_main); CNodeState* nodestate = State(nodeid); @@ -592,20 +604,20 @@ static void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman* connma return; } } - connman->ForNode(nodeid, [connman](CNode* pfrom){ + connman.ForNode(nodeid, [&connman](CNode* pfrom){ AssertLockHeld(cs_main); uint64_t nCMPCTBLOCKVersion = (pfrom->GetLocalServices() & NODE_WITNESS) ? 2 : 1; if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { // As per BIP152, we only get 3 of our peers to announce // blocks using compact encodings. - connman->ForNode(lNodesAnnouncingHeaderAndIDs.front(), [connman, nCMPCTBLOCKVersion](CNode* pnodeStop){ + connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, nCMPCTBLOCKVersion](CNode* pnodeStop){ AssertLockHeld(cs_main); - connman->PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/false, nCMPCTBLOCKVersion)); + connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/false, nCMPCTBLOCKVersion)); return true; }); lNodesAnnouncingHeaderAndIDs.pop_front(); } - connman->PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/true, nCMPCTBLOCKVersion)); + connman.PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/true, nCMPCTBLOCKVersion)); lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); return true; }); @@ -813,7 +825,7 @@ void PeerLogicValidation::InitializeNode(CNode *pnode) { mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName), pnode->fInbound, pnode->m_manual_connection)); } if(!pnode->fInbound) - PushNodeVersion(*pnode, connman, GetTime()); + PushNodeVersion(*pnode, *connman, GetTime()); } void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const @@ -1019,7 +1031,8 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) } /** - * Mark a misbehaving peer to be banned depending upon the value of `-banscore`. + * Increment peer's misbehavior score. If the new value >= DISCOURAGEMENT_THRESHOLD, mark the node + * to be discouraged, meaning the peer might be disconnected and added to the discouragement filter. */ void Misbehaving(NodeId pnode, int howmuch, const std::string& message) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { @@ -1031,18 +1044,17 @@ void Misbehaving(NodeId pnode, int howmuch, const std::string& message) EXCLUSIV return; state->nMisbehavior += howmuch; - int banscore = gArgs.GetArg("-banscore", DEFAULT_BANSCORE_THRESHOLD); std::string message_prefixed = message.empty() ? "" : (": " + message); - if (state->nMisbehavior >= banscore && state->nMisbehavior - howmuch < banscore) + if (state->nMisbehavior >= DISCOURAGEMENT_THRESHOLD && state->nMisbehavior - howmuch < DISCOURAGEMENT_THRESHOLD) { - LogPrint(BCLog::NET, "%s: %s peer=%d (%d -> %d) BAN THRESHOLD EXCEEDED%s\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior, message_prefixed); - state->fShouldBan = true; + LogPrint(BCLog::NET, "%s: %s peer=%d (%d -> %d) DISCOURAGE THRESHOLD EXCEEDED%s\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior, message_prefixed); + state->m_should_discourage = true; } else LogPrint(BCLog::NET, "%s: %s peer=%d (%d -> %d)%s\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior, message_prefixed); } /** - * Potentially ban a node based on the contents of a BlockValidationState object + * Potentially mark a node discouraged based on the contents of a BlockValidationState object * * @param[in] via_compact_block this bool is passed in because net_processing should * punish peers differently depending on whether the data was provided in a compact @@ -1072,7 +1084,7 @@ static bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& s break; } - // Ban outbound (but not inbound) peers if on an invalid chain. + // Discourage outbound (but not inbound) peers if on an invalid chain. // Exempt HB compact block peers and manual connections. if (!via_compact_block && !node_state->m_is_inbound && !node_state->m_is_manual_connection) { Misbehaving(nodeid, 100, message); @@ -1107,7 +1119,7 @@ static bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& s } /** - * Potentially ban a node based on the contents of a TxValidationState object + * Potentially disconnect and discourage a node based on the contents of a TxValidationState object * * @return Returns true if the peer was punished (probably disconnected) */ @@ -1328,9 +1340,10 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB } // Relay inventory, but don't relay old inventory during initial block download. connman->ForEachNode([nNewHeight, &vHashes](CNode* pnode) { + LOCK(pnode->cs_inventory); if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : 0)) { for (const uint256& hash : reverse_iterate(vHashes)) { - pnode->PushBlockHash(hash); + pnode->vBlockHashesToAnnounce.push_back(hash); } } }); @@ -1339,7 +1352,7 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB } /** - * Handle invalid block rejection and consequent peer banning, maintain which + * Handle invalid block rejection and consequent peer discouragement, maintain which * peers announce compact blocks. */ void PeerLogicValidation::BlockChecked(const CBlock& block, const BlockValidationState& state) { @@ -1365,7 +1378,7 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const BlockValidatio !::ChainstateActive().IsInitialBlockDownload() && mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) { if (it != mapBlockSource.end()) { - MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, connman); + MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, *connman); } } if (it != mapBlockSource.end()) @@ -1440,7 +1453,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& assert(nRelayNodes <= best.size()); auto sortfunc = [&best, &hasher, nRelayNodes](CNode* pnode) { - if (pnode->nVersion >= CADDR_TIME_VERSION && pnode->IsAddrRelayPeer()) { + if (pnode->IsAddrRelayPeer()) { uint64_t hashKey = CSipHasher(hasher).Write(pnode->GetId()).Finalize(); for (unsigned int i = 0; i < nRelayNodes; i++) { if (hashKey > best[i].first) { @@ -1461,7 +1474,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& connman.ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); } -void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, const CInv& inv, CConnman* connman) +void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, const CInv& inv, CConnman& connman) { bool send = false; std::shared_ptr<const CBlock> a_recent_block; @@ -1509,9 +1522,9 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); // disconnect node in case we have reached the outbound limit for serving historical blocks if (send && - connman->OutboundTargetReached(true) && + connman.OutboundTargetReached(true) && (((pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - pindex->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.type == MSG_FILTERED_BLOCK) && - !pfrom.HasPermission(PF_NOBAN) // never disconnect nodes with the noban permission + !pfrom.HasPermission(PF_DOWNLOAD) // nodes with the download permission may exceed target ) { LogPrint(BCLog::NET, "historical block serving limit reached, disconnect peer=%d\n", pfrom.GetId()); @@ -1543,7 +1556,7 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c if (!ReadRawBlockFromDisk(block_data, pindex, chainparams.MessageStart())) { assert(!"cannot load block from disk"); } - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, MakeSpan(block_data))); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, MakeSpan(block_data))); // Don't set pblock as we've sent the block } else { // Send block from disk @@ -1554,9 +1567,9 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c } if (pblock) { if (inv.type == MSG_BLOCK) - connman->PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); + connman.PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_WITNESS_BLOCK) - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_FILTERED_BLOCK) { bool sendMerkleBlock = false; @@ -1569,7 +1582,7 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c } } if (sendMerkleBlock) { - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see // This avoids hurting performance by pointlessly requiring a round-trip // Note that there is currently no way for a node to request any single transactions we didn't send here - @@ -1578,7 +1591,7 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; for (PairType& pair : merkleBlock.vMatchedTxn) - connman->PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); + connman.PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); } // else // no response @@ -1593,13 +1606,13 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c int nSendFlags = fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; if (CanDirectFetch(consensusParams) && pindex->nHeight >= ::ChainActive().Height() - MAX_CMPCTBLOCK_DEPTH) { if ((fPeerWantsWitness || !fWitnessesPresentInARecentCompactBlock) && a_recent_compact_block && a_recent_compact_block->header.GetHash() == pindex->GetBlockHash()) { - connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); + connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); } else { CBlockHeaderAndShortTxIDs cmpctblock(*pblock, fPeerWantsWitness); - connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); } } else { - connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, *pblock)); + connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, *pblock)); } } } @@ -1607,49 +1620,47 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c // Trigger the peer node to send a getblocks request for the next batch of inventory if (inv.hash == pfrom.hashContinue) { - // Bypass PushBlockInventory, this must send even if redundant, + // Send immediately. This must send even if redundant, // and we want it right after the last block so they don't // wait for other stuff first. std::vector<CInv> vInv; vInv.push_back(CInv(MSG_BLOCK, ::ChainActive().Tip()->GetBlockHash())); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::INV, vInv)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::INV, vInv)); pfrom.hashContinue.SetNull(); } } } //! Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). -CTransactionRef static FindTxForGetData(CNode& peer, const uint256& txid, const std::chrono::seconds mempool_req, const std::chrono::seconds longlived_mempool_time) LOCKS_EXCLUDED(cs_main) +CTransactionRef static FindTxForGetData(const CNode& peer, const uint256& txid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) LOCKS_EXCLUDED(cs_main) { - // Check if the requested transaction is so recent that we're just - // about to announce it to the peer; if so, they certainly shouldn't - // know we already have it. - { - LOCK(peer.m_tx_relay->cs_tx_inventory); - if (peer.m_tx_relay->setInventoryTxToSend.count(txid)) return {}; + auto txinfo = mempool.info(txid); + if (txinfo.tx) { + // If a TX could have been INVed in reply to a MEMPOOL request, + // or is older than UNCONDITIONAL_RELAY_DELAY, permit the request + // unconditionally. + if ((mempool_req.count() && txinfo.m_time <= mempool_req) || txinfo.m_time <= now - UNCONDITIONAL_RELAY_DELAY) { + return std::move(txinfo.tx); + } } { LOCK(cs_main); - // Look up transaction in relay pool - auto mi = mapRelay.find(txid); - if (mi != mapRelay.end()) return mi->second; - } - auto txinfo = mempool.info(txid); - if (txinfo.tx) { - // To protect privacy, do not answer getdata using the mempool when - // that TX couldn't have been INVed in reply to a MEMPOOL request, - // or when it's too recent to have expired from mapRelay. - if ((mempool_req.count() && txinfo.m_time <= mempool_req) || txinfo.m_time <= longlived_mempool_time) { - return txinfo.tx; + // Otherwise, the transaction must have been announced recently. + if (State(peer.GetId())->m_recently_announced_invs.contains(txid)) { + // If it was, it can be relayed from either the mempool... + if (txinfo.tx) return std::move(txinfo.tx); + // ... or the relay pool. + auto mi = mapRelay.find(txid); + if (mi != mapRelay.end()) return mi->second; } } return {}; } -void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnman* connman, CTxMemPool& mempool, const std::atomic<bool>& interruptMsgProc) LOCKS_EXCLUDED(cs_main) +void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnman& connman, CTxMemPool& mempool, const std::atomic<bool>& interruptMsgProc) LOCKS_EXCLUDED(cs_main) { AssertLockNotHeld(cs_main); @@ -1657,8 +1668,7 @@ void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnm std::vector<CInv> vNotFound; const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); - // mempool entries added before this time have likely expired from mapRelay - const std::chrono::seconds longlived_mempool_time = GetTime<std::chrono::seconds>() - RELAY_TX_CACHE_TIME; + const std::chrono::seconds now = GetTime<std::chrono::seconds>(); // Get last mempool request time const std::chrono::seconds mempool_req = pfrom.m_tx_relay != nullptr ? pfrom.m_tx_relay->m_last_mempool_req.load() : std::chrono::seconds::min(); @@ -1679,11 +1689,22 @@ void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnm continue; } - CTransactionRef tx = FindTxForGetData(pfrom, inv.hash, mempool_req, longlived_mempool_time); + CTransactionRef tx = FindTxForGetData(pfrom, inv.hash, mempool_req, now); if (tx) { int nSendFlags = (inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); - connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); + connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); mempool.RemoveUnbroadcastTx(inv.hash); + // As we're going to send tx, make sure its unconfirmed parents are made requestable. + for (const auto& txin : tx->vin) { + auto txinfo = mempool.info(txin.prevout.hash); + if (txinfo.tx && txinfo.m_time > now - UNCONDITIONAL_RELAY_DELAY) { + // Relaying a transaction with a recent but unconfirmed parent. + if (WITH_LOCK(pfrom.m_tx_relay->cs_tx_inventory, return !pfrom.m_tx_relay->filterInventoryKnown.contains(txin.prevout.hash))) { + LOCK(cs_main); + State(pfrom.GetId())->m_recently_announced_invs.insert(txin.prevout.hash); + } + } + } } else { vNotFound.push_back(inv); } @@ -1717,7 +1738,7 @@ void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnm // In normal operation, we often send NOTFOUND messages for parents of // transactions that we relay; if a peer is missing a parent, they may // assume we have them and request the parents from us. - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); } } @@ -1729,7 +1750,7 @@ static uint32_t GetFetchFlags(const CNode& pfrom) EXCLUSIVE_LOCKS_REQUIRED(cs_ma return nFetchFlags; } -inline void static SendBlockTransactions(const CBlock& block, const BlockTransactionsRequest& req, CNode& pfrom, CConnman* connman) { +inline void static SendBlockTransactions(const CBlock& block, const BlockTransactionsRequest& req, CNode& pfrom, CConnman& connman) { BlockTransactions resp(req); for (size_t i = 0; i < req.indexes.size(); i++) { if (req.indexes[i] >= block.vtx.size()) { @@ -1742,10 +1763,10 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac LOCK(cs_main); const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); int nSendFlags = State(pfrom.GetId())->fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; - connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); + connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); } -static void ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateManager& chainman, CTxMemPool& mempool, const std::vector<CBlockHeader>& headers, const CChainParams& chainparams, bool via_compact_block) +static void ProcessHeadersMessage(CNode& pfrom, CConnman& connman, ChainstateManager& chainman, CTxMemPool& mempool, const std::vector<CBlockHeader>& headers, const CChainParams& chainparams, bool via_compact_block) { const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); size_t nCount = headers.size(); @@ -1771,7 +1792,7 @@ static void ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateMan // nUnconnectingHeaders gets reset back to 0. if (!LookupBlockIndex(headers[0].hashPrevBlock) && nCount < MAX_BLOCKS_TO_ANNOUNCE) { nodestate->nUnconnectingHeaders++; - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), @@ -1836,7 +1857,7 @@ static void ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateMan // TODO: optimize: if pindexLast is an ancestor of ::ChainActive().Tip or pindexBestHeader, continue // from there instead. LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom.GetId(), pfrom.nStartingHeight); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexLast), uint256())); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexLast), uint256())); } bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); @@ -1886,7 +1907,7 @@ static void ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateMan // In any case, we want to download using a compact block, not a regular one vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); } - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); } } } @@ -1897,8 +1918,8 @@ static void ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateMan // headers to fetch from this peer. if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { // This peer has too little work on their headers chain to help - // us sync -- disconnect if using an outbound slot (unless - // whitelisted or addnode). + // us sync -- disconnect if it is an outbound disconnection + // candidate. // Note: We compare their tip to nMinimumChainWork (rather than // ::ChainActive().Tip()) because we won't start block download // until we have a headers chain that has at least @@ -1927,7 +1948,7 @@ static void ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateMan return; } -void static ProcessOrphanTx(CConnman* connman, CTxMemPool& mempool, std::set<uint256>& orphan_work_set, std::list<CTransactionRef>& removed_txn) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans) +void static ProcessOrphanTx(CConnman& connman, CTxMemPool& mempool, std::set<uint256>& orphan_work_set, std::list<CTransactionRef>& removed_txn) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans) { AssertLockHeld(cs_main); AssertLockHeld(g_cs_orphans); @@ -1951,7 +1972,7 @@ void static ProcessOrphanTx(CConnman* connman, CTxMemPool& mempool, std::set<uin if (setMisbehaving.count(fromPeer)) continue; if (AcceptToMemoryPool(mempool, orphan_state, porphanTx, &removed_txn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); - RelayTransaction(orphanHash, *connman); + RelayTransaction(orphanHash, connman); for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(orphanHash, i)); if (it_by_prev != mapOrphanTransactionsByPrev.end()) { @@ -1968,7 +1989,10 @@ void static ProcessOrphanTx(CConnman* connman, CTxMemPool& mempool, std::set<uin if (MaybePunishNodeForTx(fromPeer, orphan_state)) { setMisbehaving.insert(fromPeer); } - LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s\n", orphanHash.ToString()); + LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s from peer=%d. %s\n", + orphanHash.ToString(), + fromPeer, + orphan_state.ToString()); } // Has inputs but not accepted to mempool // Probably non-standard or insufficient fee @@ -2208,11 +2232,11 @@ void ProcessMessage( CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, - int64_t nTimeReceived, + const std::chrono::microseconds time_received, const CChainParams& chainparams, ChainstateManager& chainman, CTxMemPool& mempool, - CConnman* connman, + CConnman& connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) { @@ -2250,7 +2274,7 @@ void ProcessMessage( nServices = ServiceFlags(nServiceInt); if (!pfrom.fInbound) { - connman->SetServices(pfrom.addr, nServices); + connman.SetServices(pfrom.addr, nServices); } if (!pfrom.fInbound && !pfrom.fFeeler && !pfrom.m_manual_connection && !HasAllDesirableServiceFlags(nServices)) { @@ -2279,7 +2303,7 @@ void ProcessMessage( if (!vRecv.empty()) vRecv >> fRelay; // Disconnect if we connected to ourself - if (pfrom.fInbound && !connman->CheckIncomingNonce(nNonce)) + if (pfrom.fInbound && !connman.CheckIncomingNonce(nNonce)) { LogPrintf("connected to self at %s, disconnecting\n", pfrom.addr.ToString()); pfrom.fDisconnect = true; @@ -2295,7 +2319,7 @@ void ProcessMessage( if (pfrom.fInbound) PushNodeVersion(pfrom, connman, GetAdjustedTime()); - connman->PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); + connman.PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); pfrom.nServices = nServices; pfrom.SetAddrLocal(addrMe); @@ -2351,12 +2375,9 @@ void ProcessMessage( } // Get recent addresses - if (pfrom.fOneShot || pfrom.nVersion >= CADDR_TIME_VERSION || connman->GetAddressCount() < 1000) - { - connman->PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); - pfrom.fGetAddr = true; - } - connman->MarkAddressGood(pfrom.addr); + connman.PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); + pfrom.fGetAddr = true; + connman.MarkAddressGood(pfrom.addr); } std::string remoteAddr; @@ -2375,7 +2396,7 @@ void ProcessMessage( // If the peer is old enough to have the old alert system, send it the final alert. if (pfrom.nVersion <= 70012) { CDataStream finalAlert(ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50"), SER_NETWORK, PROTOCOL_VERSION); - connman->PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make("alert", finalAlert)); + connman.PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make("alert", finalAlert)); } // Feeler connections exist only to verify if address is online. @@ -2415,7 +2436,7 @@ void ProcessMessage( // We send this to non-NODE NETWORK peers as well, because even // non-NODE NETWORK peers can announce blocks (such as pruning // nodes) - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); } if (pfrom.nVersion >= SHORT_IDS_BLOCKS_VERSION) { // Tell our peer we are willing to provide version 1 or 2 cmpctblocks @@ -2426,9 +2447,9 @@ void ProcessMessage( bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 2; if (pfrom.GetLocalServices() & NODE_WITNESS) - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); nCMPCTBLOCKVersion = 1; - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); } pfrom.fSuccessfullyConnected = true; return; @@ -2445,9 +2466,6 @@ void ProcessMessage( std::vector<CAddress> vAddr; vRecv >> vAddr; - // Don't want addr from older versions unless seeding - if (pfrom.nVersion < CADDR_TIME_VERSION && connman->GetAddressCount() > 1000) - return; if (!pfrom.IsAddrRelayPeer()) { return; } @@ -2476,18 +2494,21 @@ void ProcessMessage( if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; pfrom.AddAddressKnown(addr); - if (banman->IsBanned(addr)) continue; // Do not process banned addresses beyond remembering we received them + if (banman && (banman->IsDiscouraged(addr) || banman->IsBanned(addr))) { + // Do not process banned/discouraged addresses beyond remembering we received them + continue; + } bool fReachable = IsReachable(addr); if (addr.nTime > nSince && !pfrom.fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { // Relay to a limited number of other nodes - RelayAddress(addr, fReachable, *connman); + RelayAddress(addr, fReachable, connman); } // Do not store addresses outside our network if (fReachable) vAddrOk.push_back(addr); } - connman->AddNewAddresses(vAddrOk, pfrom.addr, 2 * 60 * 60); + connman.AddNewAddresses(vAddrOk, pfrom.addr, 2 * 60 * 60); if (vAddr.size() < 1000) pfrom.fGetAddr = false; if (pfrom.fOneShot) @@ -2538,9 +2559,10 @@ void ProcessMessage( // block-relay-only peer bool fBlocksOnly = !g_relay_txes || (pfrom.m_tx_relay == nullptr); - // Allow whitelisted peers to send data other than blocks in blocks only mode if whitelistrelay is true - if (pfrom.HasPermission(PF_RELAY)) + // Allow peers with relay permission to send data other than blocks in blocks only mode + if (pfrom.HasPermission(PF_RELAY)) { fBlocksOnly = false; + } LOCK(cs_main); @@ -2583,7 +2605,7 @@ void ProcessMessage( } if (best_block != nullptr) { - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), *best_block)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), *best_block)); LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, best_block->ToString(), pfrom.GetId()); } @@ -2666,7 +2688,7 @@ void ProcessMessage( LogPrint(BCLog::NET, " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); break; } - pfrom.PushBlockInventory(pindex->GetBlockHash()); + WITH_LOCK(pfrom.cs_inventory, pfrom.vInventoryBlockToSend.push_back(pindex->GetBlockHash())); if (--nLimit <= 0) { // When this block is requested, we'll send an inv that'll @@ -2740,7 +2762,7 @@ void ProcessMessage( } LOCK(cs_main); - if (::ChainstateActive().IsInitialBlockDownload() && !pfrom.HasPermission(PF_NOBAN)) { + if (::ChainstateActive().IsInitialBlockDownload() && !pfrom.HasPermission(PF_DOWNLOAD)) { LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom.GetId()); return; } @@ -2791,7 +2813,7 @@ void ProcessMessage( // will re-announce the new block via headers (or compact blocks again) // in the SendMessages logic. nodestate->pindexBestHeaderSent = pindex ? pindex : ::ChainActive().Tip(); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); return; } @@ -2827,7 +2849,7 @@ void ProcessMessage( if (!AlreadyHave(inv, mempool) && AcceptToMemoryPool(mempool, state, ptx, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { mempool.check(&::ChainstateActive().CoinsTip()); - RelayTransaction(tx.GetHash(), *connman); + RelayTransaction(tx.GetHash(), connman); for (unsigned int i = 0; i < tx.vout.size(); i++) { auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(inv.hash, i)); if (it_by_prev != mapOrphanTransactionsByPrev.end()) { @@ -2894,15 +2916,15 @@ void ProcessMessage( } if (pfrom.HasPermission(PF_FORCERELAY)) { - // Always relay transactions received from whitelisted peers, even + // Always relay transactions received from peers with forcerelay permission, even // if they were already in the mempool, // allowing the node to function as a gateway for // nodes hidden behind it. if (!mempool.exists(tx.GetHash())) { - LogPrintf("Not relaying non-mempool transaction %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); + LogPrintf("Not relaying non-mempool transaction %s from forcerelay peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); } else { - LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); - RelayTransaction(tx.GetHash(), *connman); + LogPrintf("Force relaying tx %s from peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); + RelayTransaction(tx.GetHash(), connman); } } } @@ -2927,8 +2949,7 @@ void ProcessMessage( // peer simply for relaying a tx that our recentRejects has caught, // regardless of false positives. - if (state.IsInvalid()) - { + if (state.IsInvalid()) { LogPrint(BCLog::MEMPOOLREJ, "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), pfrom.GetId(), state.ToString()); @@ -2956,7 +2977,7 @@ void ProcessMessage( if (!LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers if (!::ChainstateActive().IsInitialBlockDownload()) - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); return; } @@ -3017,7 +3038,7 @@ void ProcessMessage( // so we just grab the block via normal getdata std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } return; } @@ -3051,14 +3072,14 @@ void ProcessMessage( PartiallyDownloadedBlock& partialBlock = *(*queuedBlockIt)->partialBlock; ReadStatus status = partialBlock.InitData(cmpctblock, vExtraTxnForCompact); if (status == READ_STATUS_INVALID) { - MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist + MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case Misbehaving does not result in a disconnect Misbehaving(pfrom.GetId(), 100, strprintf("Peer %d sent us invalid compact block\n", pfrom.GetId())); return; } else if (status == READ_STATUS_FAILED) { // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return; } @@ -3075,7 +3096,7 @@ void ProcessMessage( fProcessBLOCKTXN = true; } else { req.blockhash = pindex->GetBlockHash(); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); } } else { // This block is either already in flight from a different @@ -3101,7 +3122,7 @@ void ProcessMessage( // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return; } else { // If this was an announce-cmpctblock, we want the same treatment as a header message @@ -3111,14 +3132,14 @@ void ProcessMessage( } // cs_main if (fProcessBLOCKTXN) - return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, chainman, mempool, connman, banman, interruptMsgProc); + return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, time_received, chainparams, chainman, mempool, connman, banman, interruptMsgProc); if (fRevertToHeaderProcessing) { // Headers received from HB compact block peers are permitted to be // relayed before full validation (see BIP 152), so we don't want to disconnect // the peer if the header turns out to be for an invalid block. // Note that if a peer tries to build on an invalid chain, that - // will be detected and the peer will be banned. + // will be detected and the peer will be disconnected/discouraged. return ProcessHeadersMessage(pfrom, connman, chainman, mempool, {cmpctblock.header}, chainparams, /*via_compact_block=*/true); } @@ -3184,14 +3205,14 @@ void ProcessMessage( PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; ReadStatus status = partialBlock.FillBlock(*pblock, resp.txn); if (status == READ_STATUS_INVALID) { - MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist + MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case Misbehaving does not result in a disconnect Misbehaving(pfrom.GetId(), 100, strprintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom.GetId())); return; } else if (status == READ_STATUS_FAILED) { // Might have collided, fall back to getdata now :( std::vector<CInv> invs; invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom), resp.blockhash)); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { // Block is either okay, or possibly we received // READ_STATUS_CHECKBLOCK_FAILED. @@ -3204,7 +3225,7 @@ void ProcessMessage( // 3. the block is otherwise invalid (eg invalid coinbase, // block is too big, too many legacy sigops, etc). // So if CheckBlock failed, #3 is the only possibility. - // Under BIP 152, we don't DoS-ban unless proof of work is + // Under BIP 152, we don't discourage the peer unless proof of work is // invalid (we don't require all the stateless checks to have // been run). This is handled below, so just treat this as // though the block was successfully read, and rely on the @@ -3326,10 +3347,11 @@ void ProcessMessage( pfrom.fSentAddr = true; pfrom.vAddrToSend.clear(); - std::vector<CAddress> vAddr = connman->GetAddresses(); + std::vector<CAddress> vAddr = connman.GetAddresses(); FastRandomContext insecure_rand; for (const CAddress &addr : vAddr) { - if (!banman->IsBanned(addr)) { + bool banned_or_discouraged = banman && (banman->IsDiscouraged(addr) || banman->IsBanned(addr)); + if (!banned_or_discouraged) { pfrom.PushAddress(addr, insecure_rand); } } @@ -3347,7 +3369,7 @@ void ProcessMessage( return; } - if (connman->OutboundTargetReached(false) && !pfrom.HasPermission(PF_MEMPOOL)) + if (connman.OutboundTargetReached(false) && !pfrom.HasPermission(PF_MEMPOOL)) { if (!pfrom.HasPermission(PF_NOBAN)) { @@ -3380,13 +3402,13 @@ void ProcessMessage( // it, if the remote node sends a ping once per second and this node takes 5 // seconds to respond to each, the 5th ping the remote sends would appear to // return very quickly. - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); } return; } if (msg_type == NetMsgType::PONG) { - int64_t pingUsecEnd = nTimeReceived; + const auto ping_end = time_received; uint64_t nonce = 0; size_t nAvail = vRecv.in_avail(); bool bPingFinished = false; @@ -3400,11 +3422,11 @@ void ProcessMessage( if (nonce == pfrom.nPingNonceSent) { // Matching pong received, this ping is no longer outstanding bPingFinished = true; - int64_t pingUsecTime = pingUsecEnd - pfrom.nPingUsecStart; - if (pingUsecTime > 0) { + const auto ping_time = ping_end - pfrom.m_ping_start.load(); + if (ping_time.count() > 0) { // Successful ping time measurement, replace previous - pfrom.nPingUsecTime = pingUsecTime; - pfrom.nMinPingUsecTime = std::min(pfrom.nMinPingUsecTime.load(), pingUsecTime); + pfrom.nPingUsecTime = count_microseconds(ping_time); + pfrom.nMinPingUsecTime = std::min(pfrom.nMinPingUsecTime.load(), count_microseconds(ping_time)); } else { // This should never happen sProblem = "Timing mishap"; @@ -3520,17 +3542,17 @@ void ProcessMessage( } if (msg_type == NetMsgType::GETCFILTERS) { - ProcessGetCFilters(pfrom, vRecv, chainparams, *connman); + ProcessGetCFilters(pfrom, vRecv, chainparams, connman); return; } if (msg_type == NetMsgType::GETCFHEADERS) { - ProcessGetCFHeaders(pfrom, vRecv, chainparams, *connman); + ProcessGetCFHeaders(pfrom, vRecv, chainparams, connman); return; } if (msg_type == NetMsgType::GETCFCHECKPT) { - ProcessGetCFCheckPt(pfrom, vRecv, chainparams, *connman); + ProcessGetCFCheckPt(pfrom, vRecv, chainparams, connman); return; } @@ -3564,25 +3586,26 @@ void ProcessMessage( return; } -bool PeerLogicValidation::CheckIfBanned(CNode& pnode) +bool PeerLogicValidation::MaybeDiscourageAndDisconnect(CNode& pnode) { AssertLockHeld(cs_main); CNodeState &state = *State(pnode.GetId()); - if (state.fShouldBan) { - state.fShouldBan = false; - if (pnode.HasPermission(PF_NOBAN)) + if (state.m_should_discourage) { + state.m_should_discourage = false; + if (pnode.HasPermission(PF_NOBAN)) { LogPrintf("Warning: not punishing whitelisted peer %s!\n", pnode.addr.ToString()); - else if (pnode.m_manual_connection) + } else if (pnode.m_manual_connection) { LogPrintf("Warning: not punishing manually-connected peer %s!\n", pnode.addr.ToString()); - else if (pnode.addr.IsLocal()) { - // Disconnect but don't ban _this_ local node - LogPrintf("Warning: disconnecting but not banning local peer %s!\n", pnode.addr.ToString()); + } else if (pnode.addr.IsLocal()) { + // Disconnect but don't discourage this local node + LogPrintf("Warning: disconnecting but not discouraging local peer %s!\n", pnode.addr.ToString()); pnode.fDisconnect = true; } else { - // Disconnect and ban all nodes sharing the address + // Disconnect and discourage all nodes sharing the address + LogPrintf("Disconnecting and discouraging peer %s!\n", pnode.addr.ToString()); if (m_banman) { - m_banman->Ban(pnode.addr, BanReasonNodeMisbehaving); + m_banman->Discourage(pnode.addr); } connman->DisconnectNode(pnode.addr); } @@ -3605,12 +3628,12 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter bool fMoreWork = false; if (!pfrom->vRecvGetData.empty()) - ProcessGetData(*pfrom, chainparams, connman, m_mempool, interruptMsgProc); + ProcessGetData(*pfrom, chainparams, *connman, m_mempool, interruptMsgProc); if (!pfrom->orphan_work_set.empty()) { std::list<CTransactionRef> removed_txn; LOCK2(cs_main, g_cs_orphans); - ProcessOrphanTx(connman, m_mempool, pfrom->orphan_work_set, removed_txn); + ProcessOrphanTx(*connman, m_mempool, pfrom->orphan_work_set, removed_txn); for (const CTransactionRef& removedTx : removed_txn) { AddToCompactExtraTransactions(removedTx); } @@ -3670,7 +3693,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter } try { - ProcessMessage(*pfrom, msg_type, vRecv, msg.m_time, chainparams, m_chainman, m_mempool, connman, m_banman, interruptMsgProc); + ProcessMessage(*pfrom, msg_type, vRecv, msg.m_time, chainparams, m_chainman, m_mempool, *connman, m_banman, interruptMsgProc); if (interruptMsgProc) return false; if (!pfrom->vRecvGetData.empty()) @@ -3682,7 +3705,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter } LOCK(cs_main); - CheckIfBanned(*pfrom); + MaybeDiscourageAndDisconnect(*pfrom); return fMoreWork; } @@ -3860,7 +3883,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // RPC ping request by user pingSend = true; } - if (pto->nPingNonceSent == 0 && pto->nPingUsecStart + PING_INTERVAL * 1000000 < GetTimeMicros()) { + if (pto->nPingNonceSent == 0 && pto->m_ping_start.load() + PING_INTERVAL < GetTime<std::chrono::microseconds>()) { // Ping automatically sent as a latency probe & keepalive. pingSend = true; } @@ -3870,7 +3893,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); } pto->fPingQueued = false; - pto->nPingUsecStart = GetTimeMicros(); + pto->m_ping_start = GetTime<std::chrono::microseconds>(); if (pto->nVersion > BIP0031_VERSION) { pto->nPingNonceSent = nonce; connman->PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); @@ -3885,7 +3908,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) if (!lockMain) return true; - if (CheckIfBanned(*pto)) return true; + if (MaybeDiscourageAndDisconnect(*pto)) return true; CNodeState &state = *State(pto->GetId()); @@ -4083,7 +4106,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // If the peer's chain has this block, don't inv it back. if (!PeerHasHeader(&state, pindex)) { - pto->PushBlockInventory(hashToAnnounce); + pto->vInventoryBlockToSend.push_back(hashToAnnounce); LogPrint(BCLog::NET, "%s: sending inv peer=%d hash=%s\n", __func__, pto->GetId(), hashToAnnounce.ToString()); } @@ -4154,6 +4177,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) if (!pto->m_tx_relay->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; } pto->m_tx_relay->filterInventoryKnown.insert(hash); + // Responses to MEMPOOL requests bypass the m_recently_announced_invs filter. vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); @@ -4207,6 +4231,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) } if (pto->m_tx_relay->pfilter && !pto->m_tx_relay->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; // Send + State(pto->GetId())->m_recently_announced_invs.insert(hash); vInv.push_back(CInv(MSG_TX, hash)); nRelayedTransactions++; { @@ -4264,9 +4289,9 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // Check for headers sync timeouts if (state.fSyncStarted && state.nHeadersSyncTimeout < std::numeric_limits<int64_t>::max()) { // Detect whether this is a stalling initial-headers-sync peer - if (pindexBestHeader->GetBlockTime() <= GetAdjustedTime() - 24*60*60) { + if (pindexBestHeader->GetBlockTime() <= GetAdjustedTime() - 24 * 60 * 60) { if (nNow > state.nHeadersSyncTimeout && nSyncStarted == 1 && (nPreferredDownload - state.fPreferredDownload >= 1)) { - // Disconnect a (non-whitelisted) peer if it is our only sync peer, + // Disconnect a peer (without the noban permission) if it is our only sync peer, // and we have others we could be using instead. // Note: If all our peers are inbound, then we won't // disconnect our sync peer for stalling; we have bigger @@ -4276,7 +4301,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) pto->fDisconnect = true; return true; } else { - LogPrintf("Timeout downloading headers from whitelisted peer=%d, not disconnecting\n", pto->GetId()); + LogPrintf("Timeout downloading headers from noban peer=%d, not disconnecting\n", pto->GetId()); // Reset the headers sync state so that we have a // chance to try downloading from a different peer. // Note: this will also result in at least one more diff --git a/src/net_processing.h b/src/net_processing.h index 19beca0cc4..fa1555fbe6 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -23,15 +23,18 @@ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; static const bool DEFAULT_PEERBLOOMFILTERS = false; static const bool DEFAULT_PEERBLOCKFILTERS = false; +/** Threshold for marking a node to be discouraged, e.g. disconnected and added to the discouragement filter. */ +static const int DISCOURAGEMENT_THRESHOLD{100}; class PeerLogicValidation final : public CValidationInterface, public NetEventsInterface { private: CConnman* const connman; + /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ BanMan* const m_banman; ChainstateManager& m_chainman; CTxMemPool& m_mempool; - bool CheckIfBanned(CNode& pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool MaybeDiscourageAndDisconnect(CNode& pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main); public: PeerLogicValidation(CConnman* connman, BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool); diff --git a/src/netaddress.cpp b/src/netaddress.cpp index f79425a52e..0aaba440b8 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <cstdint> #include <netaddress.h> #include <hash.h> #include <util/strencodings.h> @@ -627,15 +628,15 @@ CService::CService() : port(0) { } -CService::CService(const CNetAddr& cip, unsigned short portIn) : CNetAddr(cip), port(portIn) +CService::CService(const CNetAddr& cip, uint16_t portIn) : CNetAddr(cip), port(portIn) { } -CService::CService(const struct in_addr& ipv4Addr, unsigned short portIn) : CNetAddr(ipv4Addr), port(portIn) +CService::CService(const struct in_addr& ipv4Addr, uint16_t portIn) : CNetAddr(ipv4Addr), port(portIn) { } -CService::CService(const struct in6_addr& ipv6Addr, unsigned short portIn) : CNetAddr(ipv6Addr), port(portIn) +CService::CService(const struct in6_addr& ipv6Addr, uint16_t portIn) : CNetAddr(ipv6Addr), port(portIn) { } @@ -663,7 +664,7 @@ bool CService::SetSockAddr(const struct sockaddr *paddr) } } -unsigned short CService::GetPort() const +uint16_t CService::GetPort() const { return port; } @@ -725,12 +726,10 @@ bool CService::GetSockAddr(struct sockaddr* paddr, socklen_t *addrlen) const */ std::vector<unsigned char> CService::GetKey() const { - std::vector<unsigned char> vKey; - vKey.resize(18); - memcpy(vKey.data(), ip, 16); - vKey[16] = port / 0x100; // most significant byte of our port - vKey[17] = port & 0x0FF; // least significant byte of our port - return vKey; + auto key = GetAddrBytes(); + key.push_back(port / 0x100); // most significant byte of our port + key.push_back(port & 0x0FF); // least significant byte of our port + return key; } std::string CService::ToStringPort() const diff --git a/src/netaddress.h b/src/netaddress.h index e640c07d32..f2daad7fb6 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -12,7 +12,7 @@ #include <compat.h> #include <serialize.h> -#include <stdint.h> +#include <cstdint> #include <string> #include <vector> @@ -90,6 +90,7 @@ class CNetAddr uint32_t GetMappedAS(const std::vector<bool> &asmap) const; std::vector<unsigned char> GetGroup(const std::vector<bool> &asmap) const; + std::vector<unsigned char> GetAddrBytes() const { return {std::begin(ip), std::end(ip)}; } int GetReachabilityFrom(const CNetAddr *paddrPartner = nullptr) const; explicit CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); @@ -142,10 +143,10 @@ class CService : public CNetAddr public: CService(); - CService(const CNetAddr& ip, unsigned short port); - CService(const struct in_addr& ipv4Addr, unsigned short port); + CService(const CNetAddr& ip, uint16_t port); + CService(const struct in_addr& ipv4Addr, uint16_t port); explicit CService(const struct sockaddr_in& addr); - unsigned short GetPort() const; + uint16_t GetPort() const; bool GetSockAddr(struct sockaddr* paddr, socklen_t *addrlen) const; bool SetSockAddr(const struct sockaddr* paddr); friend bool operator==(const CService& a, const CService& b); @@ -156,10 +157,14 @@ class CService : public CNetAddr std::string ToStringPort() const; std::string ToStringIPPort() const; - CService(const struct in6_addr& ipv6Addr, unsigned short port); + CService(const struct in6_addr& ipv6Addr, uint16_t port); explicit CService(const struct sockaddr_in6& addr); - SERIALIZE_METHODS(CService, obj) { READWRITE(obj.ip, Using<BigEndianFormatter<2>>(obj.port)); } + SERIALIZE_METHODS(CService, obj) + { + READWRITEAS(CNetAddr, obj); + READWRITE(Using<BigEndianFormatter<2>>(obj.port)); + } }; bool SanityCheckASMap(const std::vector<bool>& asmap); diff --git a/src/netbase.cpp b/src/netbase.cpp index 9fe03c6a24..3a3b5f3e66 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -12,6 +12,7 @@ #include <util/system.h> #include <atomic> +#include <cstdint> #ifndef WIN32 #include <fcntl.h> @@ -798,11 +799,11 @@ bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int ProxyCredentials random_auth; static std::atomic_int counter(0); random_auth.username = random_auth.password = strprintf("%i", counter++); - if (!Socks5(strDest, (unsigned short)port, &random_auth, hSocket)) { + if (!Socks5(strDest, (uint16_t)port, &random_auth, hSocket)) { return false; } } else { - if (!Socks5(strDest, (unsigned short)port, 0, hSocket)) { + if (!Socks5(strDest, (uint16_t)port, 0, hSocket)) { return false; } } diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index e3c4c828b6..fb46ea1731 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -8,13 +8,23 @@ #include <coins.h> #include <hash.h> #include <serialize.h> -#include <validation.h> #include <uint256.h> #include <util/system.h> +#include <validation.h> #include <map> -static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +static uint64_t GetBogoSize(const CScript& scriptPubKey) +{ + return 32 /* txid */ + + 4 /* vout index */ + + 4 /* height + coinbase */ + + 8 /* amount */ + + 2 /* scriptPubKey len */ + + scriptPubKey.size() /* scriptPubKey */; +} + +static void ApplyStats(CCoinsStats& stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) { assert(!outputs.empty()); ss << hash; @@ -26,26 +36,38 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, ss << VARINT_MODE(output.second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); stats.nTransactionOutputs++; stats.nTotalAmount += output.second.out.nValue; - stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + - 2 /* scriptPubKey len */ + output.second.out.scriptPubKey.size() /* scriptPubKey */; + stats.nBogoSize += GetBogoSize(output.second.out.scriptPubKey); } ss << VARINT(0u); } +static void ApplyStats(CCoinsStats& stats, std::nullptr_t, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +{ + assert(!outputs.empty()); + stats.nTransactions++; + for (const auto& output : outputs) { + stats.nTransactionOutputs++; + stats.nTotalAmount += output.second.out.nValue; + stats.nBogoSize += GetBogoSize(output.second.out.scriptPubKey); + } +} + //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void()>& interruption_point) +template <typename T> +static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj, const std::function<void()>& interruption_point) { stats = CCoinsStats(); std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); assert(pcursor); - CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); stats.hashBlock = pcursor->GetBestBlock(); { LOCK(cs_main); stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight; } - ss << stats.hashBlock; + + PrepareHash(hash_obj, stats); + uint256 prevkey; std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { @@ -54,7 +76,7 @@ bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { if (!outputs.empty() && key.hash != prevkey) { - ApplyStats(stats, ss, prevkey, outputs); + ApplyStats(stats, hash_obj, prevkey, outputs); outputs.clear(); } prevkey = key.hash; @@ -66,9 +88,38 @@ bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void pcursor->Next(); } if (!outputs.empty()) { - ApplyStats(stats, ss, prevkey, outputs); + ApplyStats(stats, hash_obj, prevkey, outputs); } - stats.hashSerialized = ss.GetHash(); + + FinalizeHash(hash_obj, stats); + stats.nDiskSize = view->EstimateSize(); return true; } + +bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, CoinStatsHashType hash_type, const std::function<void()>& interruption_point) +{ + switch (hash_type) { + case(CoinStatsHashType::HASH_SERIALIZED): { + CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); + return GetUTXOStats(view, stats, ss, interruption_point); + } + case(CoinStatsHashType::NONE): { + return GetUTXOStats(view, stats, nullptr, interruption_point); + } + } // no default case, so the compiler can warn about missing cases + assert(false); +} + +// The legacy hash serializes the hashBlock +static void PrepareHash(CHashWriter& ss, CCoinsStats& stats) +{ + ss << stats.hashBlock; +} +static void PrepareHash(std::nullptr_t, CCoinsStats& stats) {} + +static void FinalizeHash(CHashWriter& ss, CCoinsStats& stats) +{ + stats.hashSerialized = ss.GetHash(); +} +static void FinalizeHash(std::nullptr_t, CCoinsStats& stats) {} diff --git a/src/node/coinstats.h b/src/node/coinstats.h index d9cdaa3036..2a7441c10e 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -14,6 +14,11 @@ class CCoinsView; +enum class CoinStatsHashType { + HASH_SERIALIZED, + NONE, +}; + struct CCoinsStats { int nHeight{0}; @@ -30,6 +35,6 @@ struct CCoinsStats }; //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void()>& interruption_point = {}); +bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const CoinStatsHashType hash_type, const std::function<void()>& interruption_point = {}); #endif // BITCOIN_NODE_COINSTATS_H diff --git a/src/node/context.h b/src/node/context.h index c45d9e6689..be568cba36 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -6,6 +6,7 @@ #define BITCOIN_NODE_CONTEXT_H #include <cassert> +#include <functional> #include <memory> #include <vector> @@ -41,6 +42,7 @@ struct NodeContext { std::unique_ptr<interfaces::Chain> chain; std::vector<std::unique_ptr<interfaces::ChainClient>> chain_clients; std::unique_ptr<CScheduler> scheduler; + std::function<void()> rpc_interruption_point = [] {}; //! Declare default constructor and destructor that are not inline, so code //! instantiating the NodeContext struct doesn't need to #include class @@ -49,10 +51,4 @@ struct NodeContext { ~NodeContext(); }; -inline ChainstateManager& EnsureChainman(const NodeContext& node) -{ - assert(node.chainman); - return *node.chainman; -} - #endif // BITCOIN_NODE_CONTEXT_H diff --git a/src/ui_interface.cpp b/src/node/ui_interface.cpp index 15795bd67f..8d3665975d 100644 --- a/src/ui_interface.cpp +++ b/src/node/ui_interface.cpp @@ -2,18 +2,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <ui_interface.h> +#include <node/ui_interface.h> #include <util/translation.h> -#include <boost/signals2/last_value.hpp> +#include <boost/signals2/optional_last_value.hpp> #include <boost/signals2/signal.hpp> CClientUIInterface uiInterface; struct UISignals { - boost::signals2::signal<CClientUIInterface::ThreadSafeMessageBoxSig, boost::signals2::last_value<bool>> ThreadSafeMessageBox; - boost::signals2::signal<CClientUIInterface::ThreadSafeQuestionSig, boost::signals2::last_value<bool>> ThreadSafeQuestion; + boost::signals2::signal<CClientUIInterface::ThreadSafeMessageBoxSig, boost::signals2::optional_last_value<bool>> ThreadSafeMessageBox; + boost::signals2::signal<CClientUIInterface::ThreadSafeQuestionSig, boost::signals2::optional_last_value<bool>> ThreadSafeQuestion; boost::signals2::signal<CClientUIInterface::InitMessageSig> InitMessage; boost::signals2::signal<CClientUIInterface::NotifyNumConnectionsChangedSig> NotifyNumConnectionsChanged; boost::signals2::signal<CClientUIInterface::NotifyNetworkActiveChangedSig> NotifyNetworkActiveChanged; @@ -42,8 +42,8 @@ ADD_SIGNALS_IMPL_WRAPPER(NotifyBlockTip); ADD_SIGNALS_IMPL_WRAPPER(NotifyHeaderTip); ADD_SIGNALS_IMPL_WRAPPER(BannedListChanged); -bool CClientUIInterface::ThreadSafeMessageBox(const bilingual_str& message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeMessageBox(message, caption, style); } -bool CClientUIInterface::ThreadSafeQuestion(const bilingual_str& message, const std::string& non_interactive_message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeQuestion(message, non_interactive_message, caption, style); } +bool CClientUIInterface::ThreadSafeMessageBox(const bilingual_str& message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeMessageBox(message, caption, style).value_or(false);} +bool CClientUIInterface::ThreadSafeQuestion(const bilingual_str& message, const std::string& non_interactive_message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeQuestion(message, non_interactive_message, caption, style).value_or(false);} void CClientUIInterface::InitMessage(const std::string& message) { return g_ui_signals.InitMessage(message); } void CClientUIInterface::NotifyNumConnectionsChanged(int newNumConnections) { return g_ui_signals.NotifyNumConnectionsChanged(newNumConnections); } void CClientUIInterface::NotifyNetworkActiveChanged(bool networkActive) { return g_ui_signals.NotifyNetworkActiveChanged(networkActive); } diff --git a/src/ui_interface.h b/src/node/ui_interface.h index b7895e373f..d574ab879f 100644 --- a/src/ui_interface.h +++ b/src/node/ui_interface.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_UI_INTERFACE_H -#define BITCOIN_UI_INTERFACE_H +#ifndef BITCOIN_NODE_UI_INTERFACE_H +#define BITCOIN_NODE_UI_INTERFACE_H #include <functional> #include <memory> @@ -20,14 +20,6 @@ class connection; } } // namespace boost -/** General change type (added, updated, removed). */ -enum ChangeType -{ - CT_NEW, - CT_UPDATED, - CT_DELETED -}; - /** Signals for UI communication. */ class CClientUIInterface { @@ -122,8 +114,8 @@ void InitWarning(const bilingual_str& str); /** Show error message **/ bool InitError(const bilingual_str& str); -inline bool AbortError(const bilingual_str& str) { return InitError(str); } +constexpr auto AbortError = InitError; extern CClientUIInterface uiInterface; -#endif // BITCOIN_UI_INTERFACE_H +#endif // BITCOIN_NODE_UI_INTERFACE_H diff --git a/src/noui.cpp b/src/noui.cpp index 821d10e3bc..3c82512fac 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -6,7 +6,7 @@ #include <noui.h> #include <logging.h> -#include <ui_interface.h> +#include <node/ui_interface.h> #include <util/translation.h> #include <string> diff --git a/src/outputtype.cpp b/src/outputtype.cpp index 871474d56e..e978852826 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -42,8 +42,8 @@ const std::string& FormatOutputType(OutputType type) case OutputType::LEGACY: return OUTPUT_TYPE_STRING_LEGACY; case OutputType::P2SH_SEGWIT: return OUTPUT_TYPE_STRING_P2SH_SEGWIT; case OutputType::BECH32: return OUTPUT_TYPE_STRING_BECH32; - default: assert(false); - } + } // no default case, so the compiler can warn about missing cases + assert(false); } CTxDestination GetDestinationForKey(const CPubKey& key, OutputType type) @@ -61,8 +61,8 @@ CTxDestination GetDestinationForKey(const CPubKey& key, OutputType type) return witdest; } } - default: assert(false); - } + } // no default case, so the compiler can warn about missing cases + assert(false); } std::vector<CTxDestination> GetAllDestinationsForKey(const CPubKey& key) @@ -100,6 +100,6 @@ CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, return ScriptHash(witprog); } } - default: assert(false); - } + } // no default case, so the compiler can warn about missing cases + assert(false); } diff --git a/src/outputtype.h b/src/outputtype.h index 1438f65844..77a16b1d05 100644 --- a/src/outputtype.h +++ b/src/outputtype.h @@ -18,14 +18,6 @@ enum class OutputType { LEGACY, P2SH_SEGWIT, BECH32, - - /** - * Special output type for change outputs only. Automatically choose type - * based on address type setting and the types other of non-change outputs - * (see -changetype option documentation and implementation in - * CWallet::TransactionChangeType for details). - */ - CHANGE_AUTO, }; extern const std::array<OutputType, 3> OUTPUT_TYPES; diff --git a/src/policy/fees.h b/src/policy/fees.h index e445c1590d..e79dbc9868 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -273,7 +273,7 @@ public: /** Create new FeeFilterRounder */ explicit FeeFilterRounder(const CFeeRate& minIncrementalFee); - /** Quantize a minimum fee for privacy purpose before broadcast **/ + /** Quantize a minimum fee for privacy purpose before broadcast. Not thread-safe due to use of FastRandomContext */ CAmount round(CAmount currentMinFee); private: diff --git a/src/protocol.h b/src/protocol.h index 985f44640b..9ab63a30fb 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -371,7 +371,13 @@ public: READWRITE(nVersion); } if ((s.GetType() & SER_DISK) || - (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) { + (nVersion != INIT_PROTO_VERSION && !(s.GetType() & SER_GETHASH))) { + // The only time we serialize a CAddress object without nTime is in + // the initial VERSION messages which contain two CAddress records. + // At that point, the serialization version is INIT_PROTO_VERSION. + // After the version handshake, serialization version is >= + // MIN_PEER_PROTO_VERSION and all ADDR messages are serialized with + // nTime. READWRITE(obj.nTime); } READWRITE(Using<CustomUintFormatter<8>>(obj.nServices)); diff --git a/src/psbt.cpp b/src/psbt.cpp index 10260740f0..3fb743e5db 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -35,14 +35,6 @@ bool PartiallySignedTransaction::Merge(const PartiallySignedTransaction& psbt) return true; } -bool PartiallySignedTransaction::IsSane() const -{ - for (PSBTInput input : inputs) { - if (!input.IsSane()) return false; - } - return true; -} - bool PartiallySignedTransaction::AddInput(const CTxIn& txin, PSBTInput& psbtin) { if (std::find(tx->vin.begin(), tx->vin.end(), txin) != tx->vin.end()) { @@ -144,8 +136,8 @@ void PSBTInput::Merge(const PSBTInput& input) { if (!non_witness_utxo && input.non_witness_utxo) non_witness_utxo = input.non_witness_utxo; if (witness_utxo.IsNull() && !input.witness_utxo.IsNull()) { + // TODO: For segwit v1, we will want to clear out the non-witness utxo when setting a witness one. For v0 and non-segwit, this is not safe witness_utxo = input.witness_utxo; - non_witness_utxo = nullptr; // Clear out any non-witness utxo when we set a witness one. } partial_sigs.insert(input.partial_sigs.begin(), input.partial_sigs.end()); @@ -158,18 +150,6 @@ void PSBTInput::Merge(const PSBTInput& input) if (final_script_witness.IsNull() && !input.final_script_witness.IsNull()) final_script_witness = input.final_script_witness; } -bool PSBTInput::IsSane() const -{ - // Cannot have both witness and non-witness utxos - if (!witness_utxo.IsNull() && non_witness_utxo) return false; - - // If we have a witness_script or a scriptWitness, we must also have a witness utxo - if (!witness_script.empty() && witness_utxo.IsNull()) return false; - if (!final_script_witness.IsNull() && witness_utxo.IsNull()) return false; - - return true; -} - void PSBTOutput::FillSignatureData(SignatureData& sigdata) const { if (!redeem_script.empty()) { @@ -261,11 +241,6 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& bool require_witness_sig = false; CTxOut utxo; - // Verify input sanity, which checks that at most one of witness or non-witness utxos is provided. - if (!input.IsSane()) { - return false; - } - if (input.non_witness_utxo) { // If we're taking our information from a non-witness UTXO, verify that it matches the prevout. COutPoint prevout = tx.vin[index].prevout; @@ -299,10 +274,11 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& if (require_witness_sig && !sigdata.witness) return false; input.FromSignatureData(sigdata); - // If we have a witness signature, use the smaller witness UTXO. + // If we have a witness signature, put a witness UTXO. + // TODO: For segwit v1, we should remove the non_witness_utxo if (sigdata.witness) { input.witness_utxo = utxo; - input.non_witness_utxo = nullptr; + // input.non_witness_utxo = nullptr; } // Fill in the missing info @@ -356,10 +332,6 @@ TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector return TransactionError::PSBT_MISMATCH; } } - if (!out.IsSane()) { - return TransactionError::INVALID_PSBT; - } - return TransactionError::OK; } diff --git a/src/psbt.h b/src/psbt.h index 0a8ea2ea0b..0951b76f83 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -62,18 +62,17 @@ struct PSBTInput void FillSignatureData(SignatureData& sigdata) const; void FromSignatureData(const SignatureData& sigdata); void Merge(const PSBTInput& input); - bool IsSane() const; PSBTInput() {} template <typename Stream> inline void Serialize(Stream& s) const { // Write the utxo - // If there is a non-witness utxo, then don't add the witness one. if (non_witness_utxo) { SerializeToVector(s, PSBT_IN_NON_WITNESS_UTXO); OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS); SerializeToVector(os, non_witness_utxo); - } else if (!witness_utxo.IsNull()) { + } + if (!witness_utxo.IsNull()) { SerializeToVector(s, PSBT_IN_WITNESS_UTXO); SerializeToVector(s, witness_utxo); } @@ -284,7 +283,6 @@ struct PSBTOutput void FillSignatureData(SignatureData& sigdata) const; void FromSignatureData(const SignatureData& sigdata); void Merge(const PSBTOutput& output); - bool IsSane() const; PSBTOutput() {} template <typename Stream> @@ -401,7 +399,6 @@ struct PartiallySignedTransaction /** Merge psbt into this. The two psbts must have the same underlying CTransaction (i.e. the * same actual Bitcoin transaction.) Returns true if the merge succeeded, false otherwise. */ NODISCARD bool Merge(const PartiallySignedTransaction& psbt); - bool IsSane() const; bool AddInput(const CTxIn& txin, PSBTInput& psbtin); bool AddOutput(const CTxOut& txout, const PSBTOutput& psbtout); PartiallySignedTransaction() {} @@ -551,10 +548,6 @@ struct PartiallySignedTransaction if (outputs.size() != tx->vout.size()) { throw std::ios_base::failure("Outputs provided does not match the number of outputs in transaction."); } - // Sanity check - if (!IsSane()) { - throw std::ios_base::failure("PSBT is not sane."); - } } template <typename Stream> diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index e0b9345a32..a304c23a2c 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -30,11 +30,10 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <noui.h> -#include <ui_interface.h> #include <uint256.h> #include <util/system.h> -#include <util/translation.h> #include <util/threadnames.h> +#include <util/translation.h> #include <validation.h> #include <memory> @@ -557,6 +556,8 @@ int GuiMain(int argc, char* argv[]) /// 9. Main GUI initialization // Install global event filter that makes sure that long tooltips can be word-wrapped app.installEventFilter(new GUIUtil::ToolTipToRichTextFilter(TOOLTIP_WRAP_THRESHOLD, &app)); + // Install global event filter that makes sure that out-of-focus labels do not contain text cursor. + app.installEventFilter(new GUIUtil::LabelOutOfFocusEventFilter(&app)); #if defined(Q_OS_WIN) // Install global event filter for processing Windows session related Windows messages (WM_QUERYENDSESSION and WM_ENDSESSION) qApp->installNativeEventFilter(new WinShutdownMonitor()); diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index e807eea61e..ebcc04a5eb 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -34,7 +34,7 @@ #include <chainparams.h> #include <interfaces/handler.h> #include <interfaces/node.h> -#include <ui_interface.h> +#include <node/ui_interface.h> #include <util/system.h> #include <util/translation.h> #include <validation.h> @@ -112,6 +112,8 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty Q_EMIT consoleShown(rpcConsole); } + modalOverlay = new ModalOverlay(enableWallet, this->centralWidget()); + // Accept D&D of URIs setAcceptDrops(true); @@ -201,7 +203,6 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty openOptionsDialogWithTab(OptionsDialog::TAB_NETWORK); }); - modalOverlay = new ModalOverlay(enableWallet, this->centralWidget()); connect(labelBlocksIcon, &GUIUtil::ClickableLabel::clicked, this, &BitcoinGUI::showModalOverlay); connect(progressBar, &GUIUtil::ClickableProgressBar::clicked, this, &BitcoinGUI::showModalOverlay); #ifdef ENABLE_WALLET @@ -238,6 +239,7 @@ BitcoinGUI::~BitcoinGUI() void BitcoinGUI::createActions() { QActionGroup *tabGroup = new QActionGroup(this); + connect(modalOverlay, &ModalOverlay::triggered, tabGroup, &QActionGroup::setEnabled); overviewAction = new QAction(platformStyle->SingleColorIcon(":/icons/overview"), tr("&Overview"), this); overviewAction->setStatusTip(tr("Show general overview of wallet")); @@ -683,6 +685,7 @@ void BitcoinGUI::removeWallet(WalletModel* walletModel) m_wallet_selector->removeItem(index); if (m_wallet_selector->count() == 0) { setWalletActionsEnabled(false); + overviewAction->setChecked(true); } else if (m_wallet_selector->count() == 1) { m_wallet_selector_label_action->setVisible(false); m_wallet_selector_action->setVisible(false); diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index 1217ca3e2e..93840b4169 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -1082,12 +1082,12 @@ <item row="0" column="0"> <widget class="QLabel" name="label_30"> <property name="text"> - <string>Whitelisted</string> + <string>Permissions</string> </property> </widget> </item> <item row="0" column="1"> - <widget class="QLabel" name="peerWhitelisted"> + <widget class="QLabel" name="peerPermissions"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> </property> @@ -1264,36 +1264,13 @@ </widget> </item> <item row="8" column="0"> - <widget class="QLabel" name="label_24"> - <property name="text"> - <string>Ban Score</string> - </property> - </widget> - </item> - <item row="8" column="1"> - <widget class="QLabel" name="peerBanScore"> - <property name="cursor"> - <cursorShape>IBeamCursor</cursorShape> - </property> - <property name="text"> - <string>N/A</string> - </property> - <property name="textFormat"> - <enum>Qt::PlainText</enum> - </property> - <property name="textInteractionFlags"> - <set>Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse</set> - </property> - </widget> - </item> - <item row="9" column="0"> <widget class="QLabel" name="label_22"> <property name="text"> <string>Connection Time</string> </property> </widget> </item> - <item row="9" column="1"> + <item row="8" column="1"> <widget class="QLabel" name="peerConnTime"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1309,14 +1286,14 @@ </property> </widget> </item> - <item row="10" column="0"> + <item row="9" column="0"> <widget class="QLabel" name="label_15"> <property name="text"> <string>Last Send</string> </property> </widget> </item> - <item row="10" column="1"> + <item row="9" column="1"> <widget class="QLabel" name="peerLastSend"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1332,14 +1309,14 @@ </property> </widget> </item> - <item row="11" column="0"> + <item row="10" column="0"> <widget class="QLabel" name="label_19"> <property name="text"> <string>Last Receive</string> </property> </widget> </item> - <item row="11" column="1"> + <item row="10" column="1"> <widget class="QLabel" name="peerLastRecv"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1355,14 +1332,14 @@ </property> </widget> </item> - <item row="12" column="0"> + <item row="11" column="0"> <widget class="QLabel" name="label_18"> <property name="text"> <string>Sent</string> </property> </widget> </item> - <item row="12" column="1"> + <item row="11" column="1"> <widget class="QLabel" name="peerBytesSent"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1378,14 +1355,14 @@ </property> </widget> </item> - <item row="13" column="0"> + <item row="12" column="0"> <widget class="QLabel" name="label_20"> <property name="text"> <string>Received</string> </property> </widget> </item> - <item row="13" column="1"> + <item row="12" column="1"> <widget class="QLabel" name="peerBytesRecv"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1401,14 +1378,14 @@ </property> </widget> </item> - <item row="14" column="0"> + <item row="13" column="0"> <widget class="QLabel" name="label_26"> <property name="text"> <string>Ping Time</string> </property> </widget> </item> - <item row="14" column="1"> + <item row="13" column="1"> <widget class="QLabel" name="peerPingTime"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1424,7 +1401,7 @@ </property> </widget> </item> - <item row="15" column="0"> + <item row="14" column="0"> <widget class="QLabel" name="peerPingWaitLabel"> <property name="toolTip"> <string>The duration of a currently outstanding ping.</string> @@ -1434,7 +1411,7 @@ </property> </widget> </item> - <item row="15" column="1"> + <item row="14" column="1"> <widget class="QLabel" name="peerPingWait"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1450,14 +1427,14 @@ </property> </widget> </item> - <item row="16" column="0"> + <item row="15" column="0"> <widget class="QLabel" name="peerMinPingLabel"> <property name="text"> <string>Min Ping</string> </property> </widget> </item> - <item row="16" column="1"> + <item row="15" column="1"> <widget class="QLabel" name="peerMinPing"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1473,14 +1450,14 @@ </property> </widget> </item> - <item row="17" column="0"> + <item row="16" column="0"> <widget class="QLabel" name="label_timeoffset"> <property name="text"> <string>Time Offset</string> </property> </widget> </item> - <item row="17" column="1"> + <item row="16" column="1"> <widget class="QLabel" name="timeoffset"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1496,7 +1473,7 @@ </property> </widget> </item> - <item row="18" column="0"> + <item row="17" column="0"> <widget class="QLabel" name="peerMappedASLabel"> <property name="toolTip"> <string>The mapped Autonomous System used for diversifying peer selection.</string> @@ -1506,7 +1483,7 @@ </property> </widget> </item> - <item row="18" column="1"> + <item row="17" column="1"> <widget class="QLabel" name="peerMappedAS"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1522,7 +1499,7 @@ </property> </widget> </item> - <item row="19" column="0"> + <item row="18" column="0"> <spacer name="verticalSpacer_3"> <property name="orientation"> <enum>Qt::Vertical</enum> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 3cadac2f2f..7f439fa45e 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -450,6 +450,28 @@ bool ToolTipToRichTextFilter::eventFilter(QObject *obj, QEvent *evt) return QObject::eventFilter(obj, evt); } +LabelOutOfFocusEventFilter::LabelOutOfFocusEventFilter(QObject* parent) + : QObject(parent) +{ +} + +bool LabelOutOfFocusEventFilter::eventFilter(QObject* watched, QEvent* event) +{ + if (event->type() == QEvent::FocusOut) { + auto focus_out = static_cast<QFocusEvent*>(event); + if (focus_out->reason() != Qt::PopupFocusReason) { + auto label = qobject_cast<QLabel*>(watched); + if (label) { + auto flags = label->textInteractionFlags(); + label->setTextInteractionFlags(Qt::NoTextInteraction); + label->setTextInteractionFlags(flags); + } + } + } + + return QObject::eventFilter(watched, event); +} + void TableViewLastColumnResizingFixer::connectViewHeadersSignals() { connect(tableView->horizontalHeader(), &QHeaderView::sectionResized, this, &TableViewLastColumnResizingFixer::on_sectionResized); diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 8741d90102..2bd94b5eb3 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -162,6 +162,21 @@ namespace GUIUtil }; /** + * Qt event filter that intercepts QEvent::FocusOut events for QLabel objects, and + * resets their `textInteractionFlags' property to get rid of the visible cursor. + * + * This is a temporary fix of QTBUG-59514. + */ + class LabelOutOfFocusEventFilter : public QObject + { + Q_OBJECT + + public: + explicit LabelOutOfFocusEventFilter(QObject* parent); + bool eventFilter(QObject* watched, QEvent* event) override; + }; + + /** * Makes a QTableView last column feel as if it was being resized from its left border. * Also makes sure the column widths are never larger than the table's viewport. * In Qt, all columns are resizable from the right, but it's not intuitive resizing the last column from the right. diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index 0ba1beaf3e..8070aa627c 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -171,6 +171,8 @@ void ModalOverlay::showHide(bool hide, bool userRequested) if ( (layerIsVisible && !hide) || (!layerIsVisible && hide) || (!hide && userClosed && !userRequested)) return; + Q_EMIT triggered(hide); + if (!isVisible() && !hide) setVisible(true); diff --git a/src/qt/modaloverlay.h b/src/qt/modaloverlay.h index 1d84046d3d..7b07777641 100644 --- a/src/qt/modaloverlay.h +++ b/src/qt/modaloverlay.h @@ -25,16 +25,20 @@ public: explicit ModalOverlay(bool enable_wallet, QWidget *parent); ~ModalOverlay(); -public Q_SLOTS: void tipUpdate(int count, const QDateTime& blockDate, double nVerificationProgress); void setKnownBestHeight(int count, const QDateTime& blockDate); - void toggleVisibility(); // will show or hide the modal layer void showHide(bool hide = false, bool userRequested = false); - void closeClicked(); bool isLayerVisible() const { return layerIsVisible; } +public Q_SLOTS: + void toggleVisibility(); + void closeClicked(); + +Q_SIGNALS: + void triggered(bool hidden); + protected: bool eventFilter(QObject * obj, QEvent * ev) override; bool event(QEvent* ev) override; diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 6ca5ac9d75..14fdf9046e 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -6,6 +6,7 @@ #define BITCOIN_QT_OPTIONSMODEL_H #include <amount.h> +#include <cstdint> #include <qt/guiconstants.h> #include <QAbstractListModel> @@ -15,7 +16,7 @@ class Node; } extern const char *DEFAULT_GUI_PROXY_HOST; -static constexpr unsigned short DEFAULT_GUI_PROXY_PORT = 9050; +static constexpr uint16_t DEFAULT_GUI_PROXY_PORT = 9050; /** * Convert configured prune target MiB to displayed GB. Round up to avoid underestimating max disk usage. diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index beca78a021..a1da85bda7 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -14,9 +14,9 @@ #include <chainparams.h> #include <interfaces/node.h> -#include <policy/policy.h> #include <key_io.h> -#include <ui_interface.h> +#include <node/ui_interface.h> +#include <policy/policy.h> #include <util/system.h> #include <wallet/wallet.h> diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index dafd517ca8..29fd720244 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -467,6 +467,7 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty // Install event filter for up and down arrow ui->lineEdit->installEventFilter(this); + ui->lineEdit->setMaxLength(16 * 1024 * 1024); ui->messagesWidget->installEventFilter(this); connect(ui->clearButton, &QPushButton::clicked, this, &RPCConsole::clear); @@ -1119,15 +1120,20 @@ void RPCConsole::updateNodeDetail(const CNodeCombinedStats *stats) ui->peerSubversion->setText(QString::fromStdString(stats->nodeStats.cleanSubVer)); ui->peerDirection->setText(stats->nodeStats.fInbound ? tr("Inbound") : tr("Outbound")); ui->peerHeight->setText(QString::number(stats->nodeStats.nStartingHeight)); - ui->peerWhitelisted->setText(stats->nodeStats.m_legacyWhitelisted ? tr("Yes") : tr("No")); + if (stats->nodeStats.m_permissionFlags == PF_NONE) { + ui->peerPermissions->setText(tr("N/A")); + } else { + QStringList permissions; + for (const auto& permission : NetPermissions::ToStrings(stats->nodeStats.m_permissionFlags)) { + permissions.append(QString::fromStdString(permission)); + } + ui->peerPermissions->setText(permissions.join(" & ")); + } ui->peerMappedAS->setText(stats->nodeStats.m_mapped_as != 0 ? QString::number(stats->nodeStats.m_mapped_as) : tr("N/A")); // This check fails for example if the lock was busy and // nodeStateStats couldn't be fetched. if (stats->fNodeStateStatsAvailable) { - // Ban score is init to 0 - ui->peerBanScore->setText(QString("%1").arg(stats->nodeStateStats.nMisbehavior)); - // Sync height is init to -1 if (stats->nodeStateStats.nSyncHeight > -1) ui->peerSyncHeight->setText(QString("%1").arg(stats->nodeStateStats.nSyncHeight)); @@ -1218,7 +1224,7 @@ void RPCConsole::banSelectedNode(int bantime) // Find possible nodes, ban it and clear the selected node const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(detailNodeRow); if (stats) { - m_node.ban(stats->nodeStats.addr, BanReasonManuallyAdded, bantime); + m_node.ban(stats->nodeStats.addr, bantime); m_node.disconnectByAddress(stats->nodeStats.addr); } } diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 0ac61f3adc..97fb88d71c 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -21,9 +21,9 @@ #include <chainparams.h> #include <interfaces/node.h> #include <key_io.h> +#include <node/ui_interface.h> #include <policy/fees.h> #include <txmempool.h> -#include <ui_interface.h> #include <wallet/coincontrol.h> #include <wallet/fees.h> #include <wallet/wallet.h> diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index ced6a299d5..6e6b2b8466 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -14,7 +14,6 @@ #include <interfaces/wallet.h> #include <qt/guiutil.h> #include <qt/networkstyle.h> -#include <ui_interface.h> #include <util/system.h> #include <util/translation.h> diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index f88d57c716..443e2d047d 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -62,9 +62,10 @@ void AppTests::appTests() } #endif - BasicTestingSetup test{CBaseChainParams::REGTEST}; // Create a temp data directory to backup the gui settings to - ECC_Stop(); // Already started by the common test setup, so stop it to avoid interference - LogInstance().DisconnectTestLogger(); + fs::create_directories([] { + BasicTestingSetup test{CBaseChainParams::REGTEST}; // Create a temp data directory to backup the gui settings to + return GetDataDir() / "blocks"; + }()); m_app.parameterSetup(); m_app.createOptionsModel(true /* reset settings */); @@ -80,6 +81,7 @@ void AppTests::appTests() m_app.exec(); // Reset global state to avoid interfering with later tests. + LogInstance().DisconnectTestLogger(); AbortShutdown(); UnloadBlockIndex(); WITH_LOCK(::cs_main, g_chainman.Reset()); diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index aefdcd2716..12efca2503 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -40,7 +40,7 @@ Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin); const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; // This is all you need to run all the tests -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { // Initialize persistent globals with the testing setup state for sanity. // E.g. -datadir in gArgs is set to a temp directory dummy value (instead @@ -70,6 +70,8 @@ int main(int argc, char *argv[]) BitcoinApplication app(*node); app.setApplicationName("Bitcoin-Qt-test"); + node->setupServerArgs(); // Make gArgs available in the NodeContext + node->context()->args->ClearArgs(); // Clear added args again AppTests app_tests(app); if (QTest::qExec(&app_tests) != 0) { fInvalid = true; diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 22ba5187bb..c560dc58e7 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -178,17 +178,12 @@ public: TransactionRecord* index(interfaces::Wallet& wallet, const uint256& cur_block_hash, const int idx) { - if(idx >= 0 && idx < cachedWallet.size()) - { + if (idx >= 0 && idx < cachedWallet.size()) { TransactionRecord *rec = &cachedWallet[idx]; - // Get required locks upfront. This avoids the GUI from getting - // stuck if the core is holding the locks for a longer time - for - // example, during a wallet rescan. - // // If a status update is needed (blocks came in since last check), - // update the status of this transaction from the wallet. Otherwise, - // simply re-use the cached status. + // try to update the status of this transaction from the wallet. + // Otherwise, simply re-use the cached status. interfaces::WalletTxStatus wtx; int numBlocks; int64_t block_time; diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 3df81807f0..54ecfc38ec 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -17,7 +17,7 @@ #include <qt/transactiontablemodel.h> #include <qt/walletmodel.h> -#include <ui_interface.h> +#include <node/ui_interface.h> #include <QApplication> #include <QComboBox> diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 1143969352..e374dd191c 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -21,8 +21,8 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <key_io.h> +#include <node/ui_interface.h> #include <psbt.h> -#include <ui_interface.h> #include <util/system.h> // for GetBoolArg #include <util/translation.h> #include <wallet/coincontrol.h> diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index cec9b0eeb8..2fc883a5f5 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -20,8 +20,8 @@ #include <qt/walletmodel.h> #include <interfaces/node.h> +#include <node/ui_interface.h> #include <psbt.h> -#include <ui_interface.h> #include <util/strencodings.h> #include <QAction> diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 64f8a5bb3b..2afc9a3d4a 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -75,7 +75,10 @@ CTxMemPool& EnsureMemPool(const util::Ref& context) ChainstateManager& EnsureChainman(const util::Ref& context) { NodeContext& node = EnsureNodeContext(context); - return EnsureChainman(node); + if (!node.chainman) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Node chainman not found"); + } + return *node.chainman; } /* Calculate the difficulty for a given block index. @@ -795,10 +798,8 @@ static CBlock GetBlockChecked(const CBlockIndex* pblockindex) if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { // Block not found on disk. This could be because we have the block - // header in our index but don't have the block (for example if a - // non-whitelisted node sends us an unrequested long chain of valid - // blocks, we add the headers to our index, but don't accept the - // block). + // header in our index but not yet have the block or did not accept the + // block. throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); } @@ -972,7 +973,9 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) RPCHelpMan{"gettxoutsetinfo", "\nReturns statistics about the unspent transaction output set.\n" "Note this call may take some time.\n", - {}, + { + {"hash_type", RPCArg::Type::STR, /* default */ "hash_serialized_2", "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'none'."}, + }, RPCResult{ RPCResult::Type::OBJ, "", "", { @@ -981,7 +984,7 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs"}, {RPCResult::Type::NUM, "txouts", "The number of unspent transaction outputs"}, {RPCResult::Type::NUM, "bogosize", "A meaningless metric for UTXO set size"}, - {RPCResult::Type::STR_HEX, "hash_serialized_2", "The serialized hash"}, + {RPCResult::Type::STR_HEX, "hash_serialized_2", "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"}, {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk"}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount"}, }}, @@ -996,14 +999,19 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) CCoinsStats stats; ::ChainstateActive().ForceFlushStateToDisk(); + const CoinStatsHashType hash_type = ParseHashType(request.params[0], CoinStatsHashType::HASH_SERIALIZED); + CCoinsView* coins_view = WITH_LOCK(cs_main, return &ChainstateActive().CoinsDB()); - if (GetUTXOStats(coins_view, stats, RpcInterruptionPoint)) { + NodeContext& node = EnsureNodeContext(request.context); + if (GetUTXOStats(coins_view, stats, hash_type, node.rpc_interruption_point)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); ret.pushKV("transactions", (int64_t)stats.nTransactions); ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs); ret.pushKV("bogosize", (int64_t)stats.nBogoSize); - ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); + if (hash_type == CoinStatsHashType::HASH_SERIALIZED) { + ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); + } ret.pushKV("disk_size", stats.nDiskSize); ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { @@ -1325,50 +1333,48 @@ static UniValue getchaintips(const JSONRPCRequest& request) }, }.Check(request); + ChainstateManager& chainman = EnsureChainman(request.context); LOCK(cs_main); /* - * Idea: the set of chain tips is ::ChainActive().tip, plus orphan blocks which do not have another orphan building off of them. + * Idea: The set of chain tips is the active chain tip, plus orphan blocks which do not have another orphan building off of them. * Algorithm: * - Make one pass through BlockIndex(), picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. * - Iterate through the orphan blocks. If the block isn't pointed to by another orphan, it is a chain tip. - * - add ::ChainActive().Tip() + * - Add the active chain tip */ std::set<const CBlockIndex*, CompareBlocksByHeight> setTips; std::set<const CBlockIndex*> setOrphans; std::set<const CBlockIndex*> setPrevs; - for (const std::pair<const uint256, CBlockIndex*>& item : ::BlockIndex()) - { - if (!::ChainActive().Contains(item.second)) { + for (const std::pair<const uint256, CBlockIndex*>& item : chainman.BlockIndex()) { + if (!chainman.ActiveChain().Contains(item.second)) { setOrphans.insert(item.second); setPrevs.insert(item.second->pprev); } } - for (std::set<const CBlockIndex*>::iterator it = setOrphans.begin(); it != setOrphans.end(); ++it) - { + for (std::set<const CBlockIndex*>::iterator it = setOrphans.begin(); it != setOrphans.end(); ++it) { if (setPrevs.erase(*it) == 0) { setTips.insert(*it); } } // Always report the currently active tip. - setTips.insert(::ChainActive().Tip()); + setTips.insert(chainman.ActiveChain().Tip()); /* Construct the output array. */ UniValue res(UniValue::VARR); - for (const CBlockIndex* block : setTips) - { + for (const CBlockIndex* block : setTips) { UniValue obj(UniValue::VOBJ); obj.pushKV("height", block->nHeight); obj.pushKV("hash", block->phashBlock->GetHex()); - const int branchLen = block->nHeight - ::ChainActive().FindFork(block)->nHeight; + const int branchLen = block->nHeight - chainman.ActiveChain().FindFork(block)->nHeight; obj.pushKV("branchlen", branchLen); std::string status; - if (::ChainActive().Contains(block)) { + if (chainman.ActiveChain().Contains(block)) { // This block is part of the currently active chain. status = "active"; } else if (block->nStatus & BLOCK_FAILED_MASK) { @@ -1967,8 +1973,10 @@ static UniValue savemempool(const JSONRPCRequest& request) return NullUniValue; } +namespace { //! Search for a given set of pubkey scripts -bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& should_abort, int64_t& count, CCoinsViewCursor* cursor, const std::set<CScript>& needles, std::map<COutPoint, Coin>& out_results) { +bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& should_abort, int64_t& count, CCoinsViewCursor* cursor, const std::set<CScript>& needles, std::map<COutPoint, Coin>& out_results, std::function<void()>& interruption_point) +{ scan_progress = 0; count = 0; while (cursor->Valid()) { @@ -1976,7 +1984,7 @@ bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& Coin coin; if (!cursor->GetKey(key) || !cursor->GetValue(coin)) return false; if (++count % 8192 == 0) { - RpcInterruptionPoint(); + interruption_point(); if (should_abort) { // allow to abort the scan via the abort reference return false; @@ -1995,6 +2003,7 @@ bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& scan_progress = 100; return true; } +} // namespace /** RAII object to prevent concurrency issue when scanning the txout set */ static std::atomic<int> g_scan_progress; @@ -2143,7 +2152,8 @@ UniValue scantxoutset(const JSONRPCRequest& request) tip = ::ChainActive().Tip(); CHECK_NONFATAL(tip); } - bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins); + NodeContext& node = EnsureNodeContext(request.context); + bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins, node.rpc_interruption_point); result.pushKV("success", res); result.pushKV("txouts", count); result.pushKV("height", tip->nHeight); @@ -2298,6 +2308,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) std::unique_ptr<CCoinsViewCursor> pcursor; CCoinsStats stats; CBlockIndex* tip; + NodeContext& node = EnsureNodeContext(request.context); { // We need to lock cs_main to ensure that the coinsdb isn't written to @@ -2316,7 +2327,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) ::ChainstateActive().ForceFlushStateToDisk(); - if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats, RpcInterruptionPoint)) { + if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats, CoinStatsHashType::NONE, node.rpc_interruption_point)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } @@ -2334,7 +2345,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) unsigned int iter{0}; while (pcursor->Valid()) { - if (iter % 5000 == 0) RpcInterruptionPoint(); + if (iter % 5000 == 0) node.rpc_interruption_point(); ++iter; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { afile << key; @@ -2377,7 +2388,7 @@ static const CRPCCommand commands[] = { "blockchain", "getmempoolinfo", &getmempoolinfo, {} }, { "blockchain", "getrawmempool", &getrawmempool, {"verbose"} }, { "blockchain", "gettxout", &gettxout, {"txid","n","include_mempool"} }, - { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, {} }, + { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, {"hash_type"} }, { "blockchain", "pruneblockchain", &pruneblockchain, {"height"} }, { "blockchain", "savemempool", &savemempool, {} }, { "blockchain", "verifychain", &verifychain, {"checklevel","nblocks"} }, diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index df1e0fe623..9981ea35df 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -112,7 +112,7 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) {RPCResult::Type::BOOL, "inbound", "Inbound (true) or Outbound (false)"}, {RPCResult::Type::BOOL, "addnode", "Whether connection was due to addnode/-connect or if it was an automatic/inbound connection"}, {RPCResult::Type::NUM, "startingheight", "The starting height (block) of the peer"}, - {RPCResult::Type::NUM, "banscore", "The ban score"}, + {RPCResult::Type::NUM, "banscore", "The ban score (DEPRECATED, returned only if config option -deprecatedrpc=banscore is passed)"}, {RPCResult::Type::NUM, "synced_headers", "The last header we have in common with this peer"}, {RPCResult::Type::NUM, "synced_blocks", "The last block we have in common with this peer"}, {RPCResult::Type::ARR, "inflight", "", @@ -191,7 +191,10 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) obj.pushKV("addnode", stats.m_manual_connection); obj.pushKV("startingheight", stats.nStartingHeight); if (fStateStats) { - obj.pushKV("banscore", statestats.nMisbehavior); + if (IsDeprecatedRPCEnabled("banscore")) { + // banscore is deprecated in v0.21 for removal in v0.22 + obj.pushKV("banscore", statestats.nMisbehavior); + } obj.pushKV("synced_headers", statestats.nSyncHeight); obj.pushKV("synced_blocks", statestats.nCommonHeight); UniValue heights(UniValue::VARR); @@ -614,12 +617,12 @@ static UniValue setban(const JSONRPCRequest& request) absolute = true; if (isSubnet) { - node.banman->Ban(subNet, BanReasonManuallyAdded, banTime, absolute); + node.banman->Ban(subNet, banTime, absolute); if (node.connman) { node.connman->DisconnectNode(subNet); } } else { - node.banman->Ban(netAddr, BanReasonManuallyAdded, banTime, absolute); + node.banman->Ban(netAddr, banTime, absolute); if (node.connman) { node.connman->DisconnectNode(netAddr); } @@ -628,7 +631,7 @@ static UniValue setban(const JSONRPCRequest& request) else if(strCommand == "remove") { if (!( isSubnet ? node.banman->Unban(subNet) : node.banman->Unban(netAddr) )) { - throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Unban failed. Requested address/subnet was not previously banned."); + throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Unban failed. Requested address/subnet was not previously manually banned."); } } return NullUniValue; @@ -637,7 +640,7 @@ static UniValue setban(const JSONRPCRequest& request) static UniValue listbanned(const JSONRPCRequest& request) { RPCHelpMan{"listbanned", - "\nList all banned IPs/Subnets.\n", + "\nList all manually banned IPs/Subnets.\n", {}, RPCResult{RPCResult::Type::ARR, "", "", { @@ -646,7 +649,6 @@ static UniValue listbanned(const JSONRPCRequest& request) {RPCResult::Type::STR, "address", ""}, {RPCResult::Type::NUM_TIME, "banned_until", ""}, {RPCResult::Type::NUM_TIME, "ban_created", ""}, - {RPCResult::Type::STR, "ban_reason", ""}, }}, }}, RPCExamples{ @@ -671,7 +673,6 @@ static UniValue listbanned(const JSONRPCRequest& request) rec.pushKV("address", entry.first.ToString()); rec.pushKV("banned_until", banEntry.nBanUntil); rec.pushKV("ban_created", banEntry.nCreateTime); - rec.pushKV("ban_reason", banEntry.banReasonToString()); bannedAddresses.push_back(rec); } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index faec359d1c..7b1da6fdcd 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1104,30 +1104,34 @@ UniValue decodepsbt(const JSONRPCRequest& request) const PSBTInput& input = psbtx.inputs[i]; UniValue in(UniValue::VOBJ); // UTXOs + bool have_a_utxo = false; + CTxOut txout; if (!input.witness_utxo.IsNull()) { - const CTxOut& txout = input.witness_utxo; - - UniValue out(UniValue::VOBJ); - - out.pushKV("amount", ValueFromAmount(txout.nValue)); - if (MoneyRange(txout.nValue) && MoneyRange(total_in + txout.nValue)) { - total_in += txout.nValue; - } else { - // Hack to just not show fee later - have_all_utxos = false; - } + txout = input.witness_utxo; UniValue o(UniValue::VOBJ); ScriptToUniv(txout.scriptPubKey, o, true); + + UniValue out(UniValue::VOBJ); + out.pushKV("amount", ValueFromAmount(txout.nValue)); out.pushKV("scriptPubKey", o); + in.pushKV("witness_utxo", out); - } else if (input.non_witness_utxo) { + + have_a_utxo = true; + } + if (input.non_witness_utxo) { + txout = input.non_witness_utxo->vout[psbtx.tx->vin[i].prevout.n]; + UniValue non_wit(UniValue::VOBJ); TxToUniv(*input.non_witness_utxo, uint256(), non_wit, false); in.pushKV("non_witness_utxo", non_wit); - CAmount utxo_val = input.non_witness_utxo->vout[psbtx.tx->vin[i].prevout.n].nValue; - if (MoneyRange(utxo_val) && MoneyRange(total_in + utxo_val)) { - total_in += utxo_val; + + have_a_utxo = true; + } + if (have_a_utxo) { + if (MoneyRange(txout.nValue) && MoneyRange(total_in + txout.nValue)) { + total_in += txout.nValue; } else { // Hack to just not show fee later have_all_utxos = false; diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 844f62cbc6..e5f6b1b9f1 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -20,10 +20,10 @@ #include <mutex> #include <unordered_map> -static RecursiveMutex cs_rpcWarmup; +static Mutex g_rpc_warmup_mutex; static std::atomic<bool> g_rpc_running{false}; -static bool fRPCInWarmup GUARDED_BY(cs_rpcWarmup) = true; -static std::string rpcWarmupStatus GUARDED_BY(cs_rpcWarmup) = "RPC server started"; +static bool fRPCInWarmup GUARDED_BY(g_rpc_warmup_mutex) = true; +static std::string rpcWarmupStatus GUARDED_BY(g_rpc_warmup_mutex) = "RPC server started"; /* Timer-creating functions */ static RPCTimerInterface* timerInterface = nullptr; /* Map of name to timer. */ @@ -130,11 +130,9 @@ std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest& return strRet; } -UniValue help(const JSONRPCRequest& jsonRequest) +static RPCHelpMan help() { - if (jsonRequest.fHelp || jsonRequest.params.size() > 1) - throw std::runtime_error( - RPCHelpMan{"help", + return RPCHelpMan{"help", "\nList all commands, or get help for a specified command.\n", { {"command", RPCArg::Type::STR, /* default */ "all commands", "The command to get help on"}, @@ -143,32 +141,32 @@ UniValue help(const JSONRPCRequest& jsonRequest) RPCResult::Type::STR, "", "The help text" }, RPCExamples{""}, - }.ToString() - ); - + [&](const RPCHelpMan& self, const JSONRPCRequest& jsonRequest) -> UniValue +{ std::string strCommand; if (jsonRequest.params.size() > 0) strCommand = jsonRequest.params[0].get_str(); return tableRPC.help(strCommand, jsonRequest); +}, + }; } - -UniValue stop(const JSONRPCRequest& jsonRequest) +static RPCHelpMan stop() { static const std::string RESULT{PACKAGE_NAME " stopping"}; - // Accept the deprecated and ignored 'detach' boolean argument + return RPCHelpMan{"stop", // Also accept the hidden 'wait' integer argument (milliseconds) // For instance, 'stop 1000' makes the call wait 1 second before returning // to the client (intended for testing) - if (jsonRequest.fHelp || jsonRequest.params.size() > 1) - throw std::runtime_error( - RPCHelpMan{"stop", "\nRequest a graceful shutdown of " PACKAGE_NAME ".", - {}, + { + {"wait", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "how long to wait in ms", "", {}, /* hidden */ true}, + }, RPCResult{RPCResult::Type::STR, "", "A string with the content '" + RESULT + "'"}, RPCExamples{""}, - }.ToString()); + [&](const RPCHelpMan& self, const JSONRPCRequest& jsonRequest) -> UniValue +{ // Event loop will exit after current HTTP requests have been handled, so // this reply will get back to the client. StartShutdown(); @@ -176,11 +174,13 @@ UniValue stop(const JSONRPCRequest& jsonRequest) UninterruptibleSleep(std::chrono::milliseconds{jsonRequest.params[0].get_int()}); } return RESULT; +}, + }; } -static UniValue uptime(const JSONRPCRequest& jsonRequest) +static RPCHelpMan uptime() { - RPCHelpMan{"uptime", + return RPCHelpMan{"uptime", "\nReturns the total uptime of the server.\n", {}, RPCResult{ @@ -190,14 +190,16 @@ static UniValue uptime(const JSONRPCRequest& jsonRequest) HelpExampleCli("uptime", "") + HelpExampleRpc("uptime", "") }, - }.Check(jsonRequest); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ return GetTime() - GetStartupTime(); } + }; +} -static UniValue getrpcinfo(const JSONRPCRequest& request) +static RPCHelpMan getrpcinfo() { - RPCHelpMan{"getrpcinfo", + return RPCHelpMan{"getrpcinfo", "\nReturns details of the RPC server.\n", {}, RPCResult{ @@ -217,8 +219,8 @@ static UniValue getrpcinfo(const JSONRPCRequest& request) RPCExamples{ HelpExampleCli("getrpcinfo", "") + HelpExampleRpc("getrpcinfo", "")}, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ LOCK(g_rpc_server_info.mutex); UniValue active_commands(UniValue::VARR); for (const RPCCommandExecutionInfo& info : g_rpc_server_info.active_commands) { @@ -237,6 +239,8 @@ static UniValue getrpcinfo(const JSONRPCRequest& request) return result; } + }; +} // clang-format off static const CRPCCommand vRPCCommands[] = @@ -327,20 +331,20 @@ void RpcInterruptionPoint() void SetRPCWarmupStatus(const std::string& newStatus) { - LOCK(cs_rpcWarmup); + LOCK(g_rpc_warmup_mutex); rpcWarmupStatus = newStatus; } void SetRPCWarmupFinished() { - LOCK(cs_rpcWarmup); + LOCK(g_rpc_warmup_mutex); assert(fRPCInWarmup); fRPCInWarmup = false; } bool RPCIsInWarmup(std::string *outStatus) { - LOCK(cs_rpcWarmup); + LOCK(g_rpc_warmup_mutex); if (outStatus) *outStatus = rpcWarmupStatus; return fRPCInWarmup; @@ -439,7 +443,7 @@ UniValue CRPCTable::execute(const JSONRPCRequest &request) const { // Return immediately if in warmup { - LOCK(cs_rpcWarmup); + LOCK(g_rpc_warmup_mutex); if (fRPCInWarmup) throw JSONRPCError(RPC_IN_WARMUP, rpcWarmupStatus); } diff --git a/src/rpc/server.h b/src/rpc/server.h index d7a04ff6e8..6da3e94ea2 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -8,6 +8,7 @@ #include <amount.h> #include <rpc/request.h> +#include <rpc/util.h> #include <functional> #include <map> @@ -85,6 +86,7 @@ void RPCUnsetTimerInterface(RPCTimerInterface *iface); void RPCRunLater(const std::string& name, std::function<void()> func, int64_t nSeconds); typedef UniValue(*rpcfn_type)(const JSONRPCRequest& jsonRequest); +typedef RPCHelpMan (*RpcMethodFnType)(); class CRPCCommand { @@ -101,6 +103,19 @@ public: { } + //! Simplified constructor taking plain RpcMethodFnType function pointer. + CRPCCommand(std::string category, std::string name_in, RpcMethodFnType fn, std::vector<std::string> args_in) + : CRPCCommand( + category, + fn().m_name, + [fn](const JSONRPCRequest& request, UniValue& result, bool) { result = fn().HandleRequest(request); return true; }, + fn().GetArgNames(), + intptr_t(fn)) + { + CHECK_NONFATAL(fn().m_name == name_in); + CHECK_NONFATAL(fn().GetArgNames() == args_in); + } + //! Simplified constructor taking plain rpcfn_type function pointer. CRPCCommand(const char* category, const char* name, rpcfn_type fn, std::initializer_list<const char*> args) : CRPCCommand(category, name, @@ -117,7 +132,7 @@ public: }; /** - * Bitcoin RPC command dispatcher. + * RPC command dispatcher. */ class CRPCTable { diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 54ea352a72..9f4c7bee9c 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -113,6 +113,23 @@ std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey) return ParseHexV(find_value(o, strKey), strKey); } +CoinStatsHashType ParseHashType(const UniValue& param, const CoinStatsHashType default_type) +{ + if (param.isNull()) { + return default_type; + } else { + std::string hash_type_input = param.get_str(); + + if (hash_type_input == "hash_serialized_2") { + return CoinStatsHashType::HASH_SERIALIZED; + } else if (hash_type_input == "none") { + return CoinStatsHashType::NONE; + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%d is not a valid hash_type", hash_type_input)); + } + } +} + std::string HelpExampleCli(const std::string& methodname, const std::string& args) { return "> bitcoin-cli " + methodname + " " + args + "\n"; @@ -368,9 +385,7 @@ struct Sections { PushSection({indent + "]" + (outer_type != OuterType::NONE ? "," : ""), ""}); break; } - - // no default case, so the compiler can warn about missing cases - } + } // no default case, so the compiler can warn about missing cases } /** @@ -381,6 +396,9 @@ struct Sections { std::string ret; const size_t pad = m_max_pad + 4; for (const auto& s : m_sections) { + // The left part of a section is assumed to be a single line, usually it is the name of the JSON struct or a + // brace like {, }, [, or ] + CHECK_NONFATAL(s.m_left.find('\n') == std::string::npos); if (s.m_right.empty()) { ret += s.m_left; ret += "\n"; @@ -415,7 +433,11 @@ struct Sections { }; RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples) + : RPCHelpMan{std::move(name), std::move(description), std::move(args), std::move(results), std::move(examples), nullptr} {} + +RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples, RPCMethodImpl fun) : m_name{std::move(name)}, + m_fun{std::move(fun)}, m_description{std::move(description)}, m_args{std::move(args)}, m_results{std::move(results)}, @@ -464,6 +486,16 @@ bool RPCHelpMan::IsValidNumArgs(size_t num_args) const } return num_required_args <= num_args && num_args <= m_args.size(); } + +std::vector<std::string> RPCHelpMan::GetArgNames() const +{ + std::vector<std::string> ret; + for (const auto& arg : m_args) { + ret.emplace_back(arg.m_names); + } + return ret; +} + std::string RPCHelpMan::ToString() const { std::string ret; @@ -472,6 +504,7 @@ std::string RPCHelpMan::ToString() const ret += m_name; bool was_optional{false}; for (const auto& arg : m_args) { + if (arg.m_hidden) continue; const bool optional = arg.IsOptional(); ret += " "; if (optional) { @@ -493,6 +526,7 @@ std::string RPCHelpMan::ToString() const Sections sections; for (size_t i{0}; i < m_args.size(); ++i) { const auto& arg = m_args.at(i); + if (arg.m_hidden) continue; if (i == 0) ret += "\nArguments:\n"; @@ -572,9 +606,7 @@ std::string RPCArg::ToDescriptionString() const ret += "json array"; break; } - - // no default case, so the compiler can warn about missing cases - } + } // no default case, so the compiler can warn about missing cases } if (m_fallback.which() == 1) { ret += ", optional, default=" + boost::get<std::string>(m_fallback); @@ -592,9 +624,7 @@ std::string RPCArg::ToDescriptionString() const ret += ", required"; break; } - - // no default case, so the compiler can warn about missing cases - } + } // no default case, so the compiler can warn about missing cases } ret += ")"; ret += m_description.empty() ? "" : " " + m_description; @@ -689,10 +719,7 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const sections.PushSection({indent + "}" + maybe_separator, ""}); return; } - - // no default case, so the compiler can warn about missing cases - } - + } // no default case, so the compiler can warn about missing cases CHECK_NONFATAL(false); } @@ -729,9 +756,7 @@ std::string RPCArg::ToStringObj(const bool oneline) const case Type::OBJ_USER_KEYS: // Currently unused, so avoid writing dead code CHECK_NONFATAL(false); - - // no default case, so the compiler can warn about missing cases - } + } // no default case, so the compiler can warn about missing cases CHECK_NONFATAL(false); } @@ -766,9 +791,7 @@ std::string RPCArg::ToString(const bool oneline) const } return "[" + res + "...]"; } - - // no default case, so the compiler can warn about missing cases - } + } // no default case, so the compiler can warn about missing cases CHECK_NONFATAL(false); } diff --git a/src/rpc/util.h b/src/rpc/util.h index 53dce2c397..45b0bb0c7e 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_RPC_UTIL_H #define BITCOIN_RPC_UTIL_H +#include <node/coinstats.h> #include <node/transaction.h> #include <outputtype.h> #include <protocol.h> @@ -77,6 +78,8 @@ extern uint256 ParseHashO(const UniValue& o, std::string strKey); extern std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName); extern std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey); +CoinStatsHashType ParseHashType(const UniValue& param, const CoinStatsHashType default_type); + extern CAmount AmountFromValue(const UniValue& value); extern std::string HelpExampleCli(const std::string& methodname, const std::string& args); extern std::string HelpExampleRpc(const std::string& methodname, const std::string& args); @@ -144,6 +147,7 @@ struct RPCArg { using Fallback = boost::variant<Optional, /* default value for optional args */ std::string>; const std::string m_names; //!< The name of the arg (can be empty for inner args, can contain multiple aliases separated by | for named request arguments) const Type m_type; + const bool m_hidden; const std::vector<RPCArg> m_inner; //!< Only used for arrays or dicts const Fallback m_fallback; const std::string m_description; @@ -156,9 +160,11 @@ struct RPCArg { const Fallback fallback, const std::string description, const std::string oneline_description = "", - const std::vector<std::string> type_str = {}) + const std::vector<std::string> type_str = {}, + const bool hidden = false) : m_names{std::move(name)}, m_type{std::move(type)}, + m_hidden{hidden}, m_fallback{std::move(fallback)}, m_description{std::move(description)}, m_oneline_description{std::move(oneline_description)}, @@ -177,6 +183,7 @@ struct RPCArg { const std::vector<std::string> type_str = {}) : m_names{std::move(name)}, m_type{std::move(type)}, + m_hidden{false}, m_inner{std::move(inner)}, m_fallback{std::move(fallback)}, m_description{std::move(description)}, @@ -326,8 +333,15 @@ class RPCHelpMan { public: RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples); + using RPCMethodImpl = std::function<UniValue(const RPCHelpMan&, const JSONRPCRequest&)>; + RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples, RPCMethodImpl fun); std::string ToString() const; + UniValue HandleRequest(const JSONRPCRequest& request) + { + Check(request); + return m_fun(*this, request); + } /** If the supplied number of args is neither too small nor too high */ bool IsValidNumArgs(size_t num_args) const; /** @@ -340,8 +354,12 @@ public: } } -private: + std::vector<std::string> GetArgNames() const; + const std::string m_name; + +private: + const RPCMethodImpl m_fun; const std::string m_description; const std::vector<RPCArg> m_args; const RPCResults m_results; diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 39dd4ff39f..1c4990791c 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -290,14 +290,11 @@ public: return CScript() << CScript::EncodeOP_N(id.version) << std::vector<unsigned char>(id.program, id.program + id.length); } }; - -const CScriptVisitor g_script_visitor; - } // namespace CScript GetScriptForDestination(const CTxDestination& dest) { - return boost::apply_visitor(::g_script_visitor, dest); + return boost::apply_visitor(CScriptVisitor(), dest); } CScript GetScriptForRawPubKey(const CPubKey& pubKey) diff --git a/src/serialize.h b/src/serialize.h index 71c2cfa164..7a94e704b2 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -9,13 +9,13 @@ #include <compat/endian.h> #include <algorithm> +#include <cstdint> #include <cstring> #include <ios> #include <limits> #include <map> #include <memory> #include <set> -#include <stdint.h> #include <string> #include <string.h> #include <utility> @@ -272,7 +272,7 @@ template<typename Stream> inline void Unserialize(Stream& s, bool& a) { char f=s inline unsigned int GetSizeOfCompactSize(uint64_t nSize) { if (nSize < 253) return sizeof(unsigned char); - else if (nSize <= std::numeric_limits<unsigned short>::max()) return sizeof(unsigned char) + sizeof(unsigned short); + else if (nSize <= std::numeric_limits<uint16_t>::max()) return sizeof(unsigned char) + sizeof(uint16_t); else if (nSize <= std::numeric_limits<unsigned int>::max()) return sizeof(unsigned char) + sizeof(unsigned int); else return sizeof(unsigned char) + sizeof(uint64_t); } @@ -286,7 +286,7 @@ void WriteCompactSize(Stream& os, uint64_t nSize) { ser_writedata8(os, nSize); } - else if (nSize <= std::numeric_limits<unsigned short>::max()) + else if (nSize <= std::numeric_limits<uint16_t>::max()) { ser_writedata8(os, 253); ser_writedata16(os, nSize); diff --git a/src/sync.cpp b/src/sync.cpp index 9abdedbed4..10f0483189 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -60,7 +60,7 @@ struct CLockLocation { std::string ToString() const { return strprintf( - "%s %s:%s%s (in thread %s)", + "'%s' in %s:%s%s (in thread '%s')", mutexName, sourceFile, sourceLine, (fTry ? " (TRY)" : ""), m_thread_name); } @@ -105,7 +105,7 @@ static void potential_deadlock_detected(const LockPair& mismatch, const LockStac { LogPrintf("POTENTIAL DEADLOCK DETECTED\n"); LogPrintf("Previous lock order was:\n"); - for (const LockStackItem& i : s2) { + for (const LockStackItem& i : s1) { if (i.first == mismatch.first) { LogPrintf(" (1)"); /* Continued */ } @@ -114,21 +114,25 @@ static void potential_deadlock_detected(const LockPair& mismatch, const LockStac } LogPrintf(" %s\n", i.second.ToString()); } + + std::string mutex_a, mutex_b; LogPrintf("Current lock order is:\n"); - for (const LockStackItem& i : s1) { + for (const LockStackItem& i : s2) { if (i.first == mismatch.first) { LogPrintf(" (1)"); /* Continued */ + mutex_a = i.second.Name(); } if (i.first == mismatch.second) { LogPrintf(" (2)"); /* Continued */ + mutex_b = i.second.Name(); } LogPrintf(" %s\n", i.second.ToString()); } if (g_debug_lockorder_abort) { - tfm::format(std::cerr, "Assertion failed: detected inconsistent lock order at %s:%i, details in debug log.\n", __FILE__, __LINE__); + tfm::format(std::cerr, "Assertion failed: detected inconsistent lock order for %s, details in debug log.\n", s2.back().second.ToString()); abort(); } - throw std::logic_error("potential deadlock detected"); + throw std::logic_error(strprintf("potential deadlock detected: %s -> %s -> %s", mutex_b, mutex_a, mutex_b)); } static void push_lock(void* c, const CLockLocation& locklocation) diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 7dff2e6e86..00c4bdc14e 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -94,7 +94,7 @@ bool BuildChainTestingSetup::BuildChain(const CBlockIndex* pindex, CBlockHeader header = block->GetBlockHeader(); BlockValidationState state; - if (!EnsureChainman(m_node).ProcessNewBlockHeaders({header}, state, Params(), &pindex)) { + if (!Assert(m_node.chainman)->ProcessNewBlockHeaders({header}, state, Params(), &pindex)) { return false; } } @@ -171,7 +171,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) uint256 chainA_last_header = last_header; for (size_t i = 0; i < 2; i++) { const auto& block = chainA[i]; - BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, nullptr)); } for (size_t i = 0; i < 2; i++) { const auto& block = chainA[i]; @@ -189,7 +189,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) uint256 chainB_last_header = last_header; for (size_t i = 0; i < 3; i++) { const auto& block = chainB[i]; - BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, nullptr)); } for (size_t i = 0; i < 3; i++) { const auto& block = chainB[i]; @@ -220,7 +220,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) // Reorg back to chain A. for (size_t i = 2; i < 4; i++) { const auto& block = chainA[i]; - BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, nullptr)); } // Check that chain A and B blocks can be retrieved. diff --git a/src/test/data/script_tests.json b/src/test/data/script_tests.json index c01ef307b7..724789bbf9 100644 --- a/src/test/data/script_tests.json +++ b/src/test/data/script_tests.json @@ -678,7 +678,7 @@ ["0 0x02 0x0000 0", "CHECKMULTISIGVERIFY 1", "", "OK"], ["While not really correctly DER encoded, the empty signature is allowed by"], -["STRICTENC to provide a compact way to provide a delibrately invalid signature."], +["STRICTENC to provide a compact way to provide a deliberately invalid signature."], ["0", "0x21 0x02865c40293a680cb9c020e7b1e106d8c1916d3cef99aa431a56d253e69256dac0 CHECKSIG NOT", "STRICTENC", "OK"], ["0 0", "1 0x21 0x02865c40293a680cb9c020e7b1e106d8c1916d3cef99aa431a56d253e69256dac0 1 CHECKMULTISIG NOT", "STRICTENC", "OK"], diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 348b170536..d49b51926c 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -217,7 +217,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) connman->ClearNodes(); } -BOOST_AUTO_TEST_CASE(DoS_banning) +BOOST_AUTO_TEST_CASE(peer_discouragement) { auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); @@ -232,14 +232,14 @@ BOOST_AUTO_TEST_CASE(DoS_banning) dummyNode1.fSuccessfullyConnected = true; { LOCK(cs_main); - Misbehaving(dummyNode1.GetId(), 100); // Should get banned + Misbehaving(dummyNode1.GetId(), DISCOURAGEMENT_THRESHOLD); // Should be discouraged } { LOCK2(cs_main, dummyNode1.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); } - BOOST_CHECK(banman->IsBanned(addr1)); - BOOST_CHECK(!banman->IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned + BOOST_CHECK(banman->IsDiscouraged(addr1)); + BOOST_CHECK(!banman->IsDiscouraged(ip(0xa0b0c001|0x0000ff00))); // Different IP, not discouraged CAddress addr2(ip(0xa0b0c002), NODE_NONE); CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, CAddress(), "", true); @@ -249,76 +249,30 @@ BOOST_AUTO_TEST_CASE(DoS_banning) dummyNode2.fSuccessfullyConnected = true; { LOCK(cs_main); - Misbehaving(dummyNode2.GetId(), 50); + Misbehaving(dummyNode2.GetId(), DISCOURAGEMENT_THRESHOLD - 1); } { LOCK2(cs_main, dummyNode2.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode2)); } - BOOST_CHECK(!banman->IsBanned(addr2)); // 2 not banned yet... - BOOST_CHECK(banman->IsBanned(addr1)); // ... but 1 still should be + BOOST_CHECK(!banman->IsDiscouraged(addr2)); // 2 not discouraged yet... + BOOST_CHECK(banman->IsDiscouraged(addr1)); // ... but 1 still should be { LOCK(cs_main); - Misbehaving(dummyNode2.GetId(), 50); + Misbehaving(dummyNode2.GetId(), 1); // 2 reaches discouragement threshold } { LOCK2(cs_main, dummyNode2.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode2)); } - BOOST_CHECK(banman->IsBanned(addr2)); + BOOST_CHECK(banman->IsDiscouraged(addr1)); // Expect both 1 and 2 + BOOST_CHECK(banman->IsDiscouraged(addr2)); // to be discouraged now bool dummy; peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); peerLogic->FinalizeNode(dummyNode2.GetId(), dummy); } -BOOST_AUTO_TEST_CASE(DoS_banscore) -{ - auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); - auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); - - banman->ClearBanned(); - gArgs.ForceSetArg("-banscore", "111"); // because 11 is my favorite number - CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, 1, CAddress(), "", true); - dummyNode1.SetSendVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(&dummyNode1); - dummyNode1.nVersion = 1; - dummyNode1.fSuccessfullyConnected = true; - { - LOCK(cs_main); - Misbehaving(dummyNode1.GetId(), 100); - } - { - LOCK2(cs_main, dummyNode1.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); - } - BOOST_CHECK(!banman->IsBanned(addr1)); - { - LOCK(cs_main); - Misbehaving(dummyNode1.GetId(), 10); - } - { - LOCK2(cs_main, dummyNode1.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); - } - BOOST_CHECK(!banman->IsBanned(addr1)); - { - LOCK(cs_main); - Misbehaving(dummyNode1.GetId(), 1); - } - { - LOCK2(cs_main, dummyNode1.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); - } - BOOST_CHECK(banman->IsBanned(addr1)); - gArgs.ForceSetArg("-banscore", ToString(DEFAULT_BANSCORE_THRESHOLD)); - - bool dummy; - peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); -} - BOOST_AUTO_TEST_CASE(DoS_bantime) { auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); @@ -338,19 +292,13 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) { LOCK(cs_main); - Misbehaving(dummyNode.GetId(), 100); + Misbehaving(dummyNode.GetId(), DISCOURAGEMENT_THRESHOLD); } { LOCK2(cs_main, dummyNode.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode)); } - BOOST_CHECK(banman->IsBanned(addr)); - - SetMockTime(nStartTime+60*60); - BOOST_CHECK(banman->IsBanned(addr)); - - SetMockTime(nStartTime+60*60*24+1); - BOOST_CHECK(!banman->IsBanned(addr)); + BOOST_CHECK(banman->IsDiscouraged(addr)); bool dummy; peerLogic->FinalizeNode(dummyNode.GetId(), dummy); diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 5d7065dafb..20132d5782 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -135,7 +135,7 @@ void DoCheck(const std::string& prv, const std::string& pub, int flags, const st // When the descriptor is hardened, evaluate with access to the private keys inside. const FlatSigningProvider& key_provider = (flags & HARDENED) ? keys_priv : keys_pub; - // Evaluate the descriptor selected by `t` in poisition `i`. + // Evaluate the descriptor selected by `t` in position `i`. FlatSigningProvider script_provider, script_provider_cached; std::vector<CScript> spks, spks_cached; DescriptorCache desc_cache; diff --git a/src/test/fuzz/addrdb.cpp b/src/test/fuzz/addrdb.cpp index 524cea83fe..ad6461650f 100644 --- a/src/test/fuzz/addrdb.cpp +++ b/src/test/fuzz/addrdb.cpp @@ -18,18 +18,11 @@ void test_one_input(const std::vector<uint8_t>& buffer) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const CBanEntry ban_entry = [&] { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 3)) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 2)) { case 0: return CBanEntry{fuzzed_data_provider.ConsumeIntegral<int64_t>()}; break; - case 1: - return CBanEntry{fuzzed_data_provider.ConsumeIntegral<int64_t>(), fuzzed_data_provider.PickValueInArray<BanReason>({ - BanReason::BanReasonUnknown, - BanReason::BanReasonNodeMisbehaving, - BanReason::BanReasonManuallyAdded, - })}; - break; - case 2: { + case 1: { const std::optional<CBanEntry> ban_entry = ConsumeDeserializable<CBanEntry>(fuzzed_data_provider); if (ban_entry) { return *ban_entry; @@ -39,5 +32,4 @@ void test_one_input(const std::vector<uint8_t>& buffer) } return CBanEntry{}; }(); - assert(!ban_entry.banReasonToString().empty()); } diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp new file mode 100644 index 0000000000..fc4a1d9261 --- /dev/null +++ b/src/test/fuzz/banman.cpp @@ -0,0 +1,88 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <banman.h> +#include <fs.h> +#include <netaddress.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <util/system.h> + +#include <cstdint> +#include <limits> +#include <string> +#include <vector> + +namespace { +int64_t ConsumeBanTimeOffset(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + // Avoid signed integer overflow by capping to int32_t max: + // banman.cpp:137:73: runtime error: signed integer overflow: 1591700817 + 9223372036854775807 cannot be represented in type 'long' + return fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(std::numeric_limits<int64_t>::min(), std::numeric_limits<int32_t>::max()); +} +} // namespace + +void initialize() +{ + InitializeFuzzingContext(); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + const fs::path banlist_file = GetDataDir() / "fuzzed_banlist.dat"; + fs::remove(banlist_file); + { + BanMan ban_man{banlist_file, nullptr, ConsumeBanTimeOffset(fuzzed_data_provider)}; + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 11)) { + case 0: { + ban_man.Ban(ConsumeNetAddr(fuzzed_data_provider), + ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); + break; + } + case 1: { + ban_man.Ban(ConsumeSubNet(fuzzed_data_provider), + ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); + break; + } + case 2: { + ban_man.ClearBanned(); + break; + } + case 4: { + ban_man.IsBanned(ConsumeNetAddr(fuzzed_data_provider)); + break; + } + case 5: { + ban_man.IsBanned(ConsumeSubNet(fuzzed_data_provider)); + break; + } + case 6: { + ban_man.Unban(ConsumeNetAddr(fuzzed_data_provider)); + break; + } + case 7: { + ban_man.Unban(ConsumeSubNet(fuzzed_data_provider)); + break; + } + case 8: { + banmap_t banmap; + ban_man.GetBanned(banmap); + break; + } + case 9: { + ban_man.DumpBanlist(); + break; + } + case 11: { + ban_man.Discourage(ConsumeNetAddr(fuzzed_data_provider)); + break; + } + } + } + } + fs::remove(banlist_file); +} diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 52dd62a145..c186bef7ae 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -278,7 +278,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) CCoinsStats stats; bool expected_code_path = false; try { - (void)GetUTXOStats(&coins_view_cache, stats); + (void)GetUTXOStats(&coins_view_cache, stats, CoinStatsHashType::HASH_SERIALIZED); } catch (const std::logic_error&) { expected_code_path = true; } diff --git a/src/test/fuzz/crypto_aes256.cpp b/src/test/fuzz/crypto_aes256.cpp new file mode 100644 index 0000000000..ae14073c96 --- /dev/null +++ b/src/test/fuzz/crypto_aes256.cpp @@ -0,0 +1,30 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/aes.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cassert> +#include <cstdint> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + const std::vector<uint8_t> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, AES256_KEYSIZE); + + AES256Encrypt encrypt{key.data()}; + AES256Decrypt decrypt{key.data()}; + + while (fuzzed_data_provider.ConsumeBool()) { + const std::vector<uint8_t> plaintext = ConsumeFixedLengthByteVector(fuzzed_data_provider, AES_BLOCKSIZE); + std::vector<uint8_t> ciphertext(AES_BLOCKSIZE); + encrypt.Encrypt(ciphertext.data(), plaintext.data()); + std::vector<uint8_t> decrypted_plaintext(AES_BLOCKSIZE); + decrypt.Decrypt(decrypted_plaintext.data(), ciphertext.data()); + assert(decrypted_plaintext == plaintext); + } +} diff --git a/src/test/fuzz/crypto_aes256cbc.cpp b/src/test/fuzz/crypto_aes256cbc.cpp new file mode 100644 index 0000000000..52983c7e79 --- /dev/null +++ b/src/test/fuzz/crypto_aes256cbc.cpp @@ -0,0 +1,34 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/aes.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cassert> +#include <cstdint> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + const std::vector<uint8_t> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, AES256_KEYSIZE); + const std::vector<uint8_t> iv = ConsumeFixedLengthByteVector(fuzzed_data_provider, AES_BLOCKSIZE); + const bool pad = fuzzed_data_provider.ConsumeBool(); + + AES256CBCEncrypt encrypt{key.data(), iv.data(), pad}; + AES256CBCDecrypt decrypt{key.data(), iv.data(), pad}; + + while (fuzzed_data_provider.ConsumeBool()) { + const std::vector<uint8_t> plaintext = ConsumeRandomLengthByteVector(fuzzed_data_provider); + std::vector<uint8_t> ciphertext(plaintext.size() + AES_BLOCKSIZE); + const int encrypt_ret = encrypt.Encrypt(plaintext.data(), plaintext.size(), ciphertext.data()); + ciphertext.resize(encrypt_ret); + std::vector<uint8_t> decrypted_plaintext(ciphertext.size()); + const int decrypt_ret = decrypt.Decrypt(ciphertext.data(), ciphertext.size(), decrypted_plaintext.data()); + decrypted_plaintext.resize(decrypt_ret); + assert(decrypted_plaintext == plaintext || (!pad && plaintext.size() % AES_BLOCKSIZE != 0 && encrypt_ret == 0 && decrypt_ret == 0)); + } +} diff --git a/src/test/fuzz/crypto_chacha20.cpp b/src/test/fuzz/crypto_chacha20.cpp new file mode 100644 index 0000000000..b7438d312d --- /dev/null +++ b/src/test/fuzz/crypto_chacha20.cpp @@ -0,0 +1,50 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/chacha20.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + ChaCha20 chacha20; + if (fuzzed_data_provider.ConsumeBool()) { + const std::vector<unsigned char> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, fuzzed_data_provider.ConsumeIntegralInRange<size_t>(16, 32)); + chacha20 = ChaCha20{key.data(), key.size()}; + } + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 4)) { + case 0: { + const std::vector<unsigned char> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, fuzzed_data_provider.ConsumeIntegralInRange<size_t>(16, 32)); + chacha20.SetKey(key.data(), key.size()); + break; + } + case 1: { + chacha20.SetIV(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + break; + } + case 2: { + chacha20.Seek(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + break; + } + case 3: { + std::vector<uint8_t> output(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + chacha20.Keystream(output.data(), output.size()); + break; + } + case 4: { + std::vector<uint8_t> output(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + const std::vector<uint8_t> input = ConsumeFixedLengthByteVector(fuzzed_data_provider, output.size()); + chacha20.Crypt(input.data(), output.data(), input.size()); + break; + } + } + } +} diff --git a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp new file mode 100644 index 0000000000..48e4263f27 --- /dev/null +++ b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp @@ -0,0 +1,72 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/chacha_poly_aead.h> +#include <crypto/poly1305.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cassert> +#include <cstdint> +#include <limits> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + const std::vector<uint8_t> k1 = ConsumeFixedLengthByteVector(fuzzed_data_provider, CHACHA20_POLY1305_AEAD_KEY_LEN); + const std::vector<uint8_t> k2 = ConsumeFixedLengthByteVector(fuzzed_data_provider, CHACHA20_POLY1305_AEAD_KEY_LEN); + + ChaCha20Poly1305AEAD aead(k1.data(), k1.size(), k2.data(), k2.size()); + uint64_t seqnr_payload = 0; + uint64_t seqnr_aad = 0; + int aad_pos = 0; + size_t buffer_size = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096); + std::vector<uint8_t> in(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); + std::vector<uint8_t> out(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); + bool is_encrypt = fuzzed_data_provider.ConsumeBool(); + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 6)) { + case 0: { + buffer_size = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(64, 4096); + in = std::vector<uint8_t>(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); + out = std::vector<uint8_t>(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); + break; + } + case 1: { + (void)aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffer_size, is_encrypt); + break; + } + case 2: { + uint32_t len = 0; + const bool ok = aead.GetLength(&len, seqnr_aad, aad_pos, in.data()); + assert(ok); + break; + } + case 3: { + seqnr_payload += 1; + aad_pos += CHACHA20_POLY1305_AEAD_AAD_LEN; + if (aad_pos + CHACHA20_POLY1305_AEAD_AAD_LEN > CHACHA20_ROUND_OUTPUT) { + aad_pos = 0; + seqnr_aad += 1; + } + break; + } + case 4: { + seqnr_payload = fuzzed_data_provider.ConsumeIntegral<int>(); + break; + } + case 5: { + seqnr_aad = fuzzed_data_provider.ConsumeIntegral<int>(); + break; + } + case 6: { + is_encrypt = fuzzed_data_provider.ConsumeBool(); + break; + } + } + } +} diff --git a/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp b/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp new file mode 100644 index 0000000000..e0a4e90c10 --- /dev/null +++ b/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp @@ -0,0 +1,25 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/hkdf_sha256_32.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + const std::vector<uint8_t> initial_key_material = ConsumeRandomLengthByteVector(fuzzed_data_provider); + + CHKDF_HMAC_SHA256_L32 hkdf_hmac_sha256_l32(initial_key_material.data(), initial_key_material.size(), fuzzed_data_provider.ConsumeRandomLengthString(1024)); + while (fuzzed_data_provider.ConsumeBool()) { + std::vector<uint8_t> out(32); + hkdf_hmac_sha256_l32.Expand32(fuzzed_data_provider.ConsumeRandomLengthString(128), out.data()); + } +} diff --git a/src/test/fuzz/crypto_poly1305.cpp b/src/test/fuzz/crypto_poly1305.cpp new file mode 100644 index 0000000000..5681e6a693 --- /dev/null +++ b/src/test/fuzz/crypto_poly1305.cpp @@ -0,0 +1,22 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/poly1305.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + const std::vector<uint8_t> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, POLY1305_KEYLEN); + const std::vector<uint8_t> in = ConsumeRandomLengthByteVector(fuzzed_data_provider); + + std::vector<uint8_t> tag_out(POLY1305_TAGLEN); + poly1305_auth(tag_out.data(), in.data(), in.size(), key.data()); +} diff --git a/src/test/fuzz/http_request.cpp b/src/test/fuzz/http_request.cpp index ebf89749e9..36d44e361f 100644 --- a/src/test/fuzz/http_request.cpp +++ b/src/test/fuzz/http_request.cpp @@ -7,6 +7,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/strencodings.h> #include <event2/buffer.h> #include <event2/event.h> @@ -48,7 +49,14 @@ void test_one_input(const std::vector<uint8_t>& buffer) assert(evbuf != nullptr); const std::vector<uint8_t> http_buffer = ConsumeRandomLengthByteVector(fuzzed_data_provider, 4096); evbuffer_add(evbuf, http_buffer.data(), http_buffer.size()); - if (evhttp_parse_firstline_(evreq, evbuf) != 1 || evhttp_parse_headers_(evreq, evbuf) != 1) { + // Avoid constructing requests that will be interpreted by libevent as PROXY requests to avoid triggering + // a nullptr dereference. The dereference (req->evcon->http_server) takes place in evhttp_parse_request_line + // and is a consequence of our hacky but necessary use of the internal function evhttp_parse_firstline_ in + // this fuzzing harness. The workaround is not aesthetically pleasing, but it successfully avoids the troublesome + // code path. " http:// HTTP/1.1\n" was a crashing input prior to this workaround. + const std::string http_buffer_str = ToLower({http_buffer.begin(), http_buffer.end()}); + if (http_buffer_str.find(" http://") != std::string::npos || http_buffer_str.find(" https://") != std::string::npos || + evhttp_parse_firstline_(evreq, evbuf) != 1 || evhttp_parse_headers_(evreq, evbuf) != 1) { evbuffer_free(evbuf); evhttp_request_free(evreq); return; diff --git a/src/test/fuzz/netaddress.cpp b/src/test/fuzz/netaddress.cpp index d8d53566c7..2901c704f6 100644 --- a/src/test/fuzz/netaddress.cpp +++ b/src/test/fuzz/netaddress.cpp @@ -5,41 +5,13 @@ #include <netaddress.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> #include <cassert> #include <cstdint> #include <netinet/in.h> #include <vector> -namespace { -CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); - if (network == Network::NET_IPV4) { - const in_addr v4_addr = { - .s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; - return CNetAddr{v4_addr}; - } else if (network == Network::NET_IPV6) { - if (fuzzed_data_provider.remaining_bytes() < 16) { - return CNetAddr{}; - } - in6_addr v6_addr = {}; - memcpy(v6_addr.s6_addr, fuzzed_data_provider.ConsumeBytes<uint8_t>(16).data(), 16); - return CNetAddr{v6_addr, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; - } else if (network == Network::NET_INTERNAL) { - CNetAddr net_addr; - net_addr.SetInternal(fuzzed_data_provider.ConsumeBytesAsString(32)); - return net_addr; - } else if (network == Network::NET_ONION) { - CNetAddr net_addr; - net_addr.SetSpecial(fuzzed_data_provider.ConsumeBytesAsString(32)); - return net_addr; - } else { - assert(false); - } -} -}; // namespace - void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/p2p_transport_deserializer.cpp b/src/test/fuzz/p2p_transport_deserializer.cpp index 57393fed45..6fba2bfaba 100644 --- a/src/test/fuzz/p2p_transport_deserializer.cpp +++ b/src/test/fuzz/p2p_transport_deserializer.cpp @@ -30,7 +30,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) pch += handled; n_bytes -= handled; if (deserializer.Complete()) { - const int64_t m_time = std::numeric_limits<int64_t>::max(); + const std::chrono::microseconds m_time{std::numeric_limits<int64_t>::max()}; const CNetMessage msg = deserializer.GetMessage(Params().MessageStart(), m_time); assert(msg.m_command.size() <= CMessageHeader::COMMAND_SIZE); assert(msg.m_raw_message_size <= buffer.size()); diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 2fa751b987..9e40d5cd55 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -34,11 +34,11 @@ void ProcessMessage( CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, - int64_t nTimeReceived, + const std::chrono::microseconds time_received, const CChainParams& chainparams, ChainstateManager& chainman, CTxMemPool& mempool, - CConnman* connman, + CConnman& connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc); @@ -87,9 +87,9 @@ void test_one_input(const std::vector<uint8_t>& buffer) connman.AddTestNode(p2p_node); g_setup->m_node.peer_logic->InitializeNode(&p2p_node); try { - ProcessMessage(p2p_node, random_message_type, random_bytes_data_stream, GetTimeMillis(), + ProcessMessage(p2p_node, random_message_type, random_bytes_data_stream, GetTime<std::chrono::microseconds>(), Params(), *g_setup->m_node.chainman, *g_setup->m_node.mempool, - g_setup->m_node.connman.get(), g_setup->m_node.banman.get(), + *g_setup->m_node.connman, g_setup->m_node.banman.get(), std::atomic<bool>{false}); } catch (const std::ios_base::failure&) { } diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp index 64328fb66e..908e2b16f2 100644 --- a/src/test/fuzz/psbt.cpp +++ b/src/test/fuzz/psbt.cpp @@ -39,7 +39,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) } (void)psbt.IsNull(); - (void)psbt.IsSane(); Optional<CMutableTransaction> tx = psbt.tx; if (tx) { @@ -50,7 +49,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) for (const PSBTInput& input : psbt.inputs) { (void)PSBTInputSigned(input); (void)input.IsNull(); - (void)input.IsSane(); } for (const PSBTOutput& output : psbt.outputs) { diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 1c1b2cd254..8cf91ef940 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -8,8 +8,11 @@ #include <amount.h> #include <arith_uint256.h> #include <attributes.h> +#include <chainparamsbase.h> #include <coins.h> #include <consensus/consensus.h> +#include <netaddress.h> +#include <netbase.h> #include <primitives/transaction.h> #include <script/script.h> #include <script/standard.h> @@ -17,6 +20,7 @@ #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/util/setup_common.h> #include <txmempool.h> #include <uint256.h> #include <version.h> @@ -228,4 +232,36 @@ NODISCARD inline std::vector<uint8_t> ConsumeFixedLengthByteVector(FuzzedDataPro return result; } +CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); + CNetAddr net_addr; + if (network == Network::NET_IPV4) { + const in_addr v4_addr = { + .s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + net_addr = CNetAddr{v4_addr}; + } else if (network == Network::NET_IPV6) { + if (fuzzed_data_provider.remaining_bytes() >= 16) { + in6_addr v6_addr = {}; + memcpy(v6_addr.s6_addr, fuzzed_data_provider.ConsumeBytes<uint8_t>(16).data(), 16); + net_addr = CNetAddr{v6_addr, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + } + } else if (network == Network::NET_INTERNAL) { + net_addr.SetInternal(fuzzed_data_provider.ConsumeBytesAsString(32)); + } else if (network == Network::NET_ONION) { + net_addr.SetSpecial(fuzzed_data_provider.ConsumeBytesAsString(32)); + } + return net_addr; +} + +CSubNet ConsumeSubNet(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<int32_t>()}; +} + +void InitializeFuzzingContext(const std::string& chain_name = CBaseChainParams::REGTEST) +{ + static const BasicTestingSetup basic_testing_setup{chain_name, {"-nodebuglogfile"}}; +} + #endif // BITCOIN_TEST_FUZZ_UTIL_H diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 11ff7b833b..62a0dc4241 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -253,7 +253,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) pblock->nNonce = blockinfo[i].nonce; } std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); - BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); + BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); pblock->hashPrevBlock = pblock->GetHash(); } diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 84bf593497..ab42be21bd 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -6,6 +6,7 @@ #include <addrman.h> #include <chainparams.h> #include <clientversion.h> +#include <cstdint> #include <net.h> #include <netbase.h> #include <serialize.h> @@ -83,10 +84,10 @@ BOOST_FIXTURE_TEST_SUITE(net_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(cnode_listen_port) { // test default - unsigned short port = GetListenPort(); + uint16_t port = GetListenPort(); BOOST_CHECK(port == Params().GetDefaultPort()); // test set port - unsigned short altPort = 12345; + uint16_t altPort = 12345; BOOST_CHECK(gArgs.SoftSetArg("-port", ToString(altPort))); port = GetListenPort(); BOOST_CHECK(port == altPort); diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 0fbf257f0e..ea3e633cc2 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -383,7 +383,7 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) BOOST_CHECK(!NetWhitebindPermissions::TryParse("bloom,forcerelay,oopsie@1.2.3.4:32", whitebindPermissions, error)); BOOST_CHECK(error.original.find("Invalid P2P permission") != std::string::npos); - // Check whitelist error + // Check netmask error BOOST_CHECK(!NetWhitelistPermissions::TryParse("bloom,forcerelay,noban@1.2.3.4:32", whitelistPermissions, error)); BOOST_CHECK(error.original.find("Invalid netmask specified in -whitelist") != std::string::npos); @@ -397,12 +397,13 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) BOOST_CHECK(NetWhitelistPermissions::TryParse("bloom,forcerelay,noban,relay,mempool@1.2.3.4/32", whitelistPermissions, error)); const auto strings = NetPermissions::ToStrings(PF_ALL); - BOOST_CHECK_EQUAL(strings.size(), 5U); + BOOST_CHECK_EQUAL(strings.size(), 6U); BOOST_CHECK(std::find(strings.begin(), strings.end(), "bloomfilter") != strings.end()); BOOST_CHECK(std::find(strings.begin(), strings.end(), "forcerelay") != strings.end()); BOOST_CHECK(std::find(strings.begin(), strings.end(), "relay") != strings.end()); BOOST_CHECK(std::find(strings.begin(), strings.end(), "noban") != strings.end()); BOOST_CHECK(std::find(strings.begin(), strings.end(), "mempool") != strings.end()); + BOOST_CHECK(std::find(strings.begin(), strings.end(), "download") != strings.end()); } BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) diff --git a/src/test/sync_tests.cpp b/src/test/sync_tests.cpp index 5c6c2ee38e..3ea8714f3a 100644 --- a/src/test/sync_tests.cpp +++ b/src/test/sync_tests.cpp @@ -18,7 +18,7 @@ void TestPotentialDeadLockDetected(MutexType& mutex1, MutexType& mutex2) try { LOCK2(mutex2, mutex1); } catch (const std::logic_error& e) { - BOOST_CHECK_EQUAL(e.what(), "potential deadlock detected"); + BOOST_CHECK_EQUAL(e.what(), "potential deadlock detected: mutex1 -> mutex2 -> mutex1"); error_thrown = true; } #ifdef DEBUG_LOCKORDER diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index dac7f1a07b..74536ae74c 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -11,6 +11,7 @@ #include <node/context.h> #include <pow.h> #include <script/standard.h> +#include <util/check.h> #include <validation.h> CTxIn generatetoaddress(const NodeContext& node, const std::string& address) @@ -31,7 +32,7 @@ CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) assert(block->nNonce); } - bool processed{EnsureChainman(node).ProcessNewBlock(Params(), block, true, nullptr)}; + bool processed{Assert(node.chainman)->ProcessNewBlock(Params(), block, true, nullptr)}; assert(processed); return CTxIn{block->vtx[0]->GetHash(), 0}; @@ -39,9 +40,8 @@ CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) { - assert(node.mempool); auto block = std::make_shared<CBlock>( - BlockAssembler{*node.mempool, Params()} + BlockAssembler{*Assert(node.mempool), Params()} .CreateNewBlock(coinbase_scriptPubKey) ->block); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 709d357b8a..24c0d6382b 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -75,11 +75,13 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve "dummy", "-printtoconsole=0", "-logtimemicros", + "-logthreadnames", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", }, extra_args); + util::ThreadRename("test"); fs::create_directories(m_path_root); gArgs.ForceSetArg("-datadir", m_path_root.string()); ClearDatadirCache(); @@ -130,7 +132,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const // We have to run a scheduler thread to prevent ActivateBestChain // from blocking due to queue overrun. - threadGroup.create_thread([&]{ m_node.scheduler->serviceQueue(); }); + threadGroup.create_thread([&] { TraceThread("scheduler", [&] { m_node.scheduler->serviceQueue(); }); }); GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); @@ -229,7 +231,7 @@ CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransa while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); - EnsureChainman(m_node).ProcessNewBlock(chainparams, shared_pblock, true, nullptr); + Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr); CBlock result = block; return result; diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index e480782c12..78b279e42a 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -12,6 +12,7 @@ #include <pubkey.h> #include <random.h> #include <txmempool.h> +#include <util/check.h> #include <util/string.h> #include <type_traits> diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 257328974b..e247c09a97 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -41,6 +41,16 @@ namespace BCLog { BOOST_FIXTURE_TEST_SUITE(util_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(util_check) +{ + // Check that Assert can forward + const std::unique_ptr<int> p_two = Assert(MakeUnique<int>(2)); + // Check that Assert works on lvalues and rvalues + const int two = *Assert(p_two); + Assert(two == 2); + Assert(true); +} + BOOST_AUTO_TEST_CASE(util_criticalsection) { RecursiveMutex cs; diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index 4dcc080b2d..f3f9fb2bff 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -53,8 +53,6 @@ std::set<std::string> RenameEnMasse(int num_threads) */ BOOST_AUTO_TEST_CASE(util_threadnames_test_rename_threaded) { - BOOST_CHECK_EQUAL(util::ThreadGetInternalName(), ""); - #if !defined(HAVE_THREAD_LOCAL) // This test doesn't apply to platforms where we don't have thread_local. return; diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 45e0c5484e..8e85b7df3e 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -163,10 +163,10 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) std::transform(blocks.begin(), blocks.end(), std::back_inserter(headers), [](std::shared_ptr<const CBlock> b) { return b->GetBlockHeader(); }); // Process all the headers so we understand the toplogy of the chain - BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlockHeaders(headers, state, Params())); + BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlockHeaders(headers, state, Params())); // Connect the genesis block and drain any outstanding events - BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored)); + BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored)); SyncWithValidationInterfaceQueue(); // subscribe to events (this subscriber will validate event ordering) @@ -188,13 +188,13 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) FastRandomContext insecure; for (int i = 0; i < 1000; i++) { auto block = blocks[insecure.randrange(blocks.size() - 1)]; - EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, &ignored); + Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, &ignored); } // to make sure that eventually we process the full chain - do it here for (auto block : blocks) { if (block->vtx.size() == 1) { - bool processed = EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, &ignored); + bool processed = Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, &ignored); assert(processed); } } @@ -233,7 +233,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) { bool ignored; auto ProcessBlock = [&](std::shared_ptr<const CBlock> block) -> bool { - return EnsureChainman(m_node).ProcessNewBlock(Params(), block, /* fForceProcessing */ true, /* fNewBlock */ &ignored); + return Assert(m_node.chainman)->ProcessNewBlock(Params(), block, /* fForceProcessing */ true, /* fNewBlock */ &ignored); }; // Process all mined blocks diff --git a/src/timedata.cpp b/src/timedata.cpp index 16dac24a48..6b3a79017b 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -9,8 +9,8 @@ #include <timedata.h> #include <netaddress.h> +#include <node/ui_interface.h> #include <sync.h> -#include <ui_interface.h> #include <util/system.h> #include <util/translation.h> #include <warnings.h> diff --git a/src/txdb.cpp b/src/txdb.cpp index 6f652c1375..047560f45d 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -5,10 +5,10 @@ #include <txdb.h> +#include <node/ui_interface.h> #include <pow.h> #include <random.h> #include <shutdown.h> -#include <ui_interface.h> #include <uint256.h> #include <util/system.h> #include <util/translation.h> diff --git a/src/util/check.h b/src/util/check.h index 5c0f32cf51..9edf394492 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -25,7 +25,7 @@ class NonFatalCheckError : public std::runtime_error * - where the condition is assumed to be true, not for error handling or validating user input * - where a failure to fulfill the condition is recoverable and does not abort the program * - * For example in RPC code, where it is undersirable to crash the whole program, this can be generally used to replace + * For example in RPC code, where it is undesirable to crash the whole program, this can be generally used to replace * asserts or recoverable logic errors. A NonFatalCheckError in RPC code is caught and passed as a string to the RPC * caller, which can then report the issue to the developers. */ @@ -42,4 +42,18 @@ class NonFatalCheckError : public std::runtime_error } \ } while (false) +#if defined(NDEBUG) +#error "Cannot compile without assertions!" +#endif + +/** Helper for Assert(). TODO remove in C++14 and replace `decltype(get_pure_r_value(val))` with `T` (templated lambda) */ +template <typename T> +T get_pure_r_value(T&& val) +{ + return std::forward<T>(val); +} + +/** Identity function. Abort if the value compares equal to zero */ +#define Assert(val) [&]() -> decltype(get_pure_r_value(val)) { auto&& check = (val); assert(#val && check); return std::forward<decltype(get_pure_r_value(val))>(check); }() + #endif // BITCOIN_UTIL_CHECK_H diff --git a/src/util/time.h b/src/util/time.h index b00c25f67c..af934e423b 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -15,10 +15,15 @@ void UninterruptibleSleep(const std::chrono::microseconds& n); /** * Helper to count the seconds of a duration. * - * All durations should be using std::chrono and calling this should generally be avoided in code. Though, it is still - * preferred to an inline t.count() to protect against a reliance on the exact type of t. + * All durations should be using std::chrono and calling this should generally + * be avoided in code. Though, it is still preferred to an inline t.count() to + * protect against a reliance on the exact type of t. + * + * This helper is used to convert durations before passing them over an + * interface that doesn't support std::chrono (e.g. RPC, debug log, or the GUI) */ inline int64_t count_seconds(std::chrono::seconds t) { return t.count(); } +inline int64_t count_microseconds(std::chrono::microseconds t) { return t.count(); } /** * DEPRECATED diff --git a/src/util/ui_change_type.h b/src/util/ui_change_type.h new file mode 100644 index 0000000000..1db761a18d --- /dev/null +++ b/src/util/ui_change_type.h @@ -0,0 +1,15 @@ +// Copyright (c) 2012-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_UI_CHANGE_TYPE_H +#define BITCOIN_UTIL_UI_CHANGE_TYPE_H + +/** General change type (added, updated, removed). */ +enum ChangeType { + CT_NEW, + CT_UPDATED, + CT_DELETED +}; + +#endif // BITCOIN_UTIL_UI_CHANGE_TYPE_H diff --git a/src/validation.cpp b/src/validation.cpp index fcaa62c9b2..b90ff440be 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -20,6 +20,7 @@ #include <index/txindex.h> #include <logging.h> #include <logging/timer.h> +#include <node/ui_interface.h> #include <optional.h> #include <policy/fees.h> #include <policy/policy.h> @@ -36,9 +37,9 @@ #include <tinyformat.h> #include <txdb.h> #include <txmempool.h> -#include <ui_interface.h> #include <uint256.h> #include <undo.h> +#include <util/check.h> // For NDEBUG compile time check #include <util/moneystr.h> #include <util/rbf.h> #include <util/strencodings.h> @@ -51,10 +52,6 @@ #include <boost/algorithm/string/replace.hpp> -#if defined(NDEBUG) -# error "Bitcoin cannot be compiled without assertions." -#endif - #define MICRO 0.000001 #define MILLI 0.001 @@ -576,8 +573,9 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) CAmount& nConflictingFees = ws.m_conflicting_fees; size_t& nConflictingSize = ws.m_conflicting_size; - if (!CheckTransaction(tx, state)) + if (!CheckTransaction(tx, state)) { return false; // state filled in by CheckTransaction + } // Coinbase is only valid in a block, not as a loose transaction if (tx.IsCoinBase()) @@ -687,7 +685,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) CAmount nFees = 0; if (!Consensus::CheckTxInputs(tx, state, m_view, GetSpendHeight(m_view), nFees)) { - return error("%s: Consensus::CheckTxInputs: %s, %s", __func__, tx.GetHash().ToString(), state.ToString()); + return false; // state filled in by CheckTxInputs } // Check for non-standard pay-to-script-hash in inputs @@ -1321,12 +1319,6 @@ bool CChainState::IsInitialBlockDownload() const static CBlockIndex *pindexBestForkTip = nullptr, *pindexBestForkBase = nullptr; -BlockMap& BlockIndex() -{ - LOCK(::cs_main); - return g_chainman.m_blockman.m_block_index; -} - static void AlertNotify(const std::string& strMessage) { uiInterface.NotifyAlertChanged(); @@ -1430,12 +1422,12 @@ void static InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(c pindexBestHeader = ::ChainActive().Tip(); } - LogPrintf("%s: invalid block=%s height=%d log2_work=%.8g date=%s\n", __func__, + LogPrintf("%s: invalid block=%s height=%d log2_work=%f date=%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, log(pindexNew->nChainWork.getdouble())/log(2.0), FormatISO8601DateTime(pindexNew->GetBlockTime())); CBlockIndex *tip = ::ChainActive().Tip(); assert (tip); - LogPrintf("%s: current best=%s height=%d log2_work=%.8g date=%s\n", __func__, + LogPrintf("%s: current best=%s height=%d log2_work=%f date=%s\n", __func__, tip->GetBlockHash().ToString(), ::ChainActive().Height(), log(tip->nChainWork.getdouble())/log(2.0), FormatISO8601DateTime(tip->GetBlockTime())); CheckForkWarningConditions(); @@ -2485,7 +2477,7 @@ void static UpdateTip(const CBlockIndex* pindexNew, const CChainParams& chainPar if (nUpgraded > 0) AppendWarning(warning_messages, strprintf(_("%d of last 100 blocks have unexpected version"), nUpgraded)); } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n", __func__, + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, log(pindexNew->nChainWork.getdouble())/log(2.0), (unsigned long)pindexNew->nChainTx, FormatISO8601DateTime(pindexNew->GetBlockTime()), diff --git a/src/validation.h b/src/validation.h index 58383ad923..acadf151c5 100644 --- a/src/validation.h +++ b/src/validation.h @@ -74,7 +74,6 @@ static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; -static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; /** Default for -persistmempool */ static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for using fee filter */ @@ -845,7 +844,7 @@ public: * validationinterface callback. * * @param[in] pblock The block we want to process. - * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and whitelisted peers. + * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources. * @param[out] fNewBlock A boolean which is set to indicate if the block was first received via this call * @returns If the block was processed, independently of block validity */ @@ -886,9 +885,6 @@ CChainState& ChainstateActive(); /** Please prefer the identical ChainstateManager::ActiveChain */ CChain& ChainActive(); -/** Please prefer the identical ChainstateManager::BlockIndex */ -BlockMap& BlockIndex(); - /** Global variable that points to the active block tree (protected by cs_main) */ extern std::unique_ptr<CBlockTreeDB> pblocktree; diff --git a/src/version.h b/src/version.h index d932b512d4..e5d1f5a7f9 100644 --- a/src/version.h +++ b/src/version.h @@ -14,15 +14,8 @@ static const int PROTOCOL_VERSION = 70015; //! initial proto version, to be increased after version/verack negotiation static const int INIT_PROTO_VERSION = 209; -//! In this version, 'getheaders' was introduced. -static const int GETHEADERS_VERSION = 31800; - //! disconnect from peers older than this proto version -static const int MIN_PEER_PROTO_VERSION = GETHEADERS_VERSION; - -//! nTime field added to CAddress, starting with this version; -//! if possible, avoid requesting addresses nodes older than this -static const int CADDR_TIME_VERSION = 31402; +static const int MIN_PEER_PROTO_VERSION = 31800; //! BIP 0031, pong message, is enabled for all versions AFTER this one static const int BIP0031_VERSION = 60000; diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 125bf004e4..44d1bafaf6 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -292,11 +292,10 @@ BerkeleyBatch::SafeDbt::operator Dbt*() return &m_dbt; } -bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, bilingual_str& errorStr) +bool BerkeleyDatabase::Verify(bilingual_str& errorStr) { - std::string walletFile; - std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile); fs::path walletDir = env->Directory(); + fs::path file_path = walletDir / strFile; LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion()); LogPrintf("Using wallet %s\n", file_path.string()); @@ -306,19 +305,10 @@ bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, bilingual_str& return false; } - return true; -} - -bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, bilingual_str& errorStr) -{ - std::string walletFile; - std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile); - fs::path walletDir = env->Directory(); - - if (fs::exists(walletDir / walletFile)) + if (fs::exists(file_path)) { - if (!env->Verify(walletFile)) { - errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), walletFile); + if (!env->Verify(strFile)) { + errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), file_path); return false; } } @@ -335,7 +325,7 @@ void BerkeleyEnvironment::CheckpointLSN(const std::string& strFile) } -BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr) +BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr), m_cursor(nullptr) { fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); fFlushOnClose = fFlushOnCloseIn; @@ -442,6 +432,7 @@ void BerkeleyBatch::Close() activeTxn->abort(); activeTxn = nullptr; pdb = nullptr; + CloseCursor(); if (fFlushOnClose) Flush(); @@ -494,13 +485,11 @@ void BerkeleyEnvironment::ReloadDbEnv() Open(true); } -bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip) +bool BerkeleyDatabase::Rewrite(const char* pszSkip) { - if (database.IsDummy()) { + if (IsDummy()) { return true; } - BerkeleyEnvironment *env = database.env.get(); - const std::string& strFile = database.strFile; while (true) { { LOCK(cs_db); @@ -514,7 +503,7 @@ bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip) LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile); std::string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} - BerkeleyBatch db(database, "r"); + BerkeleyBatch db(*this, "r"); std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0); int ret = pdbCopy->open(nullptr, // Txn pointer @@ -528,17 +517,15 @@ bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip) fSuccess = false; } - Dbc* pcursor = db.GetCursor(); - if (pcursor) + if (db.StartCursor()) { while (fSuccess) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue); - if (ret1 == DB_NOTFOUND) { - pcursor->close(); + bool complete; + bool ret1 = db.ReadAtCursor(ssKey, ssValue, complete); + if (complete) { break; - } else if (ret1 != 0) { - pcursor->close(); + } else if (!ret1) { fSuccess = false; break; } @@ -556,6 +543,8 @@ bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip) if (ret2 > 0) fSuccess = false; } + db.CloseCursor(); + } if (fSuccess) { db.Close(); env->CloseDb(strFile); @@ -624,51 +613,35 @@ void BerkeleyEnvironment::Flush(bool fShutdown) } } -bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase& database) +bool BerkeleyDatabase::PeriodicFlush() { - if (database.IsDummy()) { - return true; - } - bool ret = false; - BerkeleyEnvironment *env = database.env.get(); - const std::string& strFile = database.strFile; + // There's nothing to do for dummy databases. Return true. + if (IsDummy()) return true; + + // Don't flush if we can't acquire the lock. TRY_LOCK(cs_db, lockDb); - if (lockDb) - { - // Don't do this if any databases are in use - int nRefCount = 0; - std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin(); - while (mit != env->mapFileUseCount.end()) - { - nRefCount += (*mit).second; - mit++; - } + if (!lockDb) return false; - if (nRefCount == 0) - { - std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile); - if (mi != env->mapFileUseCount.end()) - { - LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); - int64_t nStart = GetTimeMillis(); + // Don't flush if any databases are in use + for (const auto& use_count : env->mapFileUseCount) { + if (use_count.second > 0) return false; + } - // Flush wallet file so it's self contained - env->CloseDb(strFile); - env->CheckpointLSN(strFile); + // Don't flush if there haven't been any batch writes for this database. + auto it = env->mapFileUseCount.find(strFile); + if (it == env->mapFileUseCount.end()) return false; - env->mapFileUseCount.erase(mi++); - LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); - ret = true; - } - } - } + LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); + int64_t nStart = GetTimeMillis(); - return ret; -} + // Flush wallet file so it's self contained + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + env->mapFileUseCount.erase(it); -bool BerkeleyDatabase::Rewrite(const char* pszSkip) -{ - return BerkeleyBatch::Rewrite(*this, pszSkip); + LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); + + return true; } bool BerkeleyDatabase::Backup(const std::string& strDest) const @@ -738,27 +711,30 @@ void BerkeleyDatabase::ReloadDbEnv() } } -Dbc* BerkeleyBatch::GetCursor() +bool BerkeleyBatch::StartCursor() { + assert(!m_cursor); if (!pdb) - return nullptr; - Dbc* pcursor = nullptr; - int ret = pdb->cursor(nullptr, &pcursor, 0); - if (ret != 0) - return nullptr; - return pcursor; + return false; + int ret = pdb->cursor(nullptr, &m_cursor, 0); + return ret == 0; } -int BerkeleyBatch::ReadAtCursor(Dbc* pcursor, CDataStream& ssKey, CDataStream& ssValue) +bool BerkeleyBatch::ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) { + complete = false; + if (m_cursor == nullptr) return false; // Read at cursor SafeDbt datKey; SafeDbt datValue; - int ret = pcursor->get(datKey, datValue, DB_NEXT); + int ret = m_cursor->get(datKey, datValue, DB_NEXT); + if (ret == DB_NOTFOUND) { + complete = true; + } if (ret != 0) - return ret; + return false; else if (datKey.get_data() == nullptr || datValue.get_data() == nullptr) - return 99999; + return false; // Convert to streams ssKey.SetType(SER_DISK); @@ -767,7 +743,14 @@ int BerkeleyBatch::ReadAtCursor(Dbc* pcursor, CDataStream& ssKey, CDataStream& s ssValue.SetType(SER_DISK); ssValue.clear(); ssValue.write((char*)datValue.get_data(), datValue.get_size()); - return 0; + return true; +} + +void BerkeleyBatch::CloseCursor() +{ + if (!m_cursor) return; + m_cursor->close(); + m_cursor = nullptr; } bool BerkeleyBatch::TxnBegin() @@ -804,15 +787,13 @@ std::string BerkeleyDatabaseVersion() return DbEnv::version(nullptr, nullptr, nullptr); } -bool BerkeleyBatch::ReadKey(CDataStream& key, CDataStream& value) +bool BerkeleyBatch::ReadKey(CDataStream&& key, CDataStream& value) { if (!pdb) return false; - // Key SafeDbt datKey(key.data(), key.size()); - // Read SafeDbt datValue; int ret = pdb->get(activeTxn, datKey, datValue, 0); if (ret == 0 && datValue.get_data() != nullptr) { @@ -822,48 +803,46 @@ bool BerkeleyBatch::ReadKey(CDataStream& key, CDataStream& value) return false; } -bool BerkeleyBatch::WriteKey(CDataStream& key, CDataStream& value, bool overwrite) +bool BerkeleyBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite) { if (!pdb) return true; if (fReadOnly) assert(!"Write called on database in read-only mode"); - // Key SafeDbt datKey(key.data(), key.size()); - // Value SafeDbt datValue(value.data(), value.size()); - // Write int ret = pdb->put(activeTxn, datKey, datValue, (overwrite ? 0 : DB_NOOVERWRITE)); return (ret == 0); } -bool BerkeleyBatch::EraseKey(CDataStream& key) +bool BerkeleyBatch::EraseKey(CDataStream&& key) { if (!pdb) return false; if (fReadOnly) assert(!"Erase called on database in read-only mode"); - // Key SafeDbt datKey(key.data(), key.size()); - // Erase int ret = pdb->del(activeTxn, datKey, 0); return (ret == 0 || ret == DB_NOTFOUND); } -bool BerkeleyBatch::HasKey(CDataStream& key) +bool BerkeleyBatch::HasKey(CDataStream&& key) { if (!pdb) return false; - // Key SafeDbt datKey(key.data(), key.size()); - // Exists int ret = pdb->exists(activeTxn, datKey, 0); return ret == 0; } + +std::unique_ptr<BerkeleyBatch> BerkeleyDatabase::MakeBatch(const char* mode, bool flush_on_close) +{ + return MakeUnique<BerkeleyBatch>(*this, mode, flush_on_close); +} diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index c121bb4228..e54776fc0d 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -90,9 +90,11 @@ public: /** Get BerkeleyEnvironment and database filename given a wallet path. */ std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename); -/** Return wheter a BDB wallet database is currently loaded. */ +/** Return whether a BDB wallet database is currently loaded. */ bool IsBDBWalletLoaded(const fs::path& wallet_path); +class BerkeleyBatch; + /** An instance of this class represents one database. * For BerkeleyDB this is just a (env, strFile) tuple. **/ @@ -131,6 +133,9 @@ public: /** Make sure all changes are flushed to disk. */ void Flush(bool shutdown); + /* flush the wallet passively (TRY_LOCK) + ideal to be called periodically */ + bool PeriodicFlush(); void IncrementUpdateCounter(); @@ -141,6 +146,9 @@ public: unsigned int nLastFlushed; int64_t nLastWalletUpdate; + /** Verifies the environment and database file */ + bool Verify(bilingual_str& error); + /** * Pointer to shared database environment. * @@ -155,6 +163,9 @@ public: /** Database pointer. This is initialized lazily and reset during flushes, so it can be null. */ std::unique_ptr<Db> m_db; + /** Make a BerkeleyBatch connected to this database */ + std::unique_ptr<BerkeleyBatch> MakeBatch(const char* mode, bool flush_on_close); + private: std::string strFile; @@ -166,7 +177,7 @@ private: }; /** RAII class that provides access to a Berkeley database */ -class BerkeleyBatch +class BerkeleyBatch : public DatabaseBatch { /** RAII class that automatically cleanses its data on destruction */ class SafeDbt final @@ -189,108 +200,36 @@ class BerkeleyBatch }; private: - bool ReadKey(CDataStream& key, CDataStream& value); - bool WriteKey(CDataStream& key, CDataStream& value, bool overwrite=true); - bool EraseKey(CDataStream& key); - bool HasKey(CDataStream& key); + bool ReadKey(CDataStream&& key, CDataStream& value) override; + bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite = true) override; + bool EraseKey(CDataStream&& key) override; + bool HasKey(CDataStream&& key) override; protected: Db* pdb; std::string strFile; DbTxn* activeTxn; + Dbc* m_cursor; bool fReadOnly; bool fFlushOnClose; BerkeleyEnvironment *env; public: explicit BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode = "r+", bool fFlushOnCloseIn=true); - ~BerkeleyBatch() { Close(); } + ~BerkeleyBatch() override { Close(); } BerkeleyBatch(const BerkeleyBatch&) = delete; BerkeleyBatch& operator=(const BerkeleyBatch&) = delete; - void Flush(); - void Close(); - - /* flush the wallet passively (TRY_LOCK) - ideal to be called periodically */ - static bool PeriodicFlush(BerkeleyDatabase& database); - /* verifies the database environment */ - static bool VerifyEnvironment(const fs::path& file_path, bilingual_str& errorStr); - /* verifies the database file */ - static bool VerifyDatabaseFile(const fs::path& file_path, bilingual_str& errorStr); - - template <typename K, typename T> - bool Read(const K& key, T& value) - { - // Key - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - bool success = false; - bool ret = ReadKey(ssKey, ssValue); - if (ret) { - // Unserialize value - try { - ssValue >> value; - success = true; - } catch (const std::exception&) { - // In this case success remains 'false' - } - } - return ret && success; - } - - template <typename K, typename T> - bool Write(const K& key, const T& value, bool fOverwrite = true) - { - // Key - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - - // Value - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - ssValue.reserve(10000); - ssValue << value; - - // Write - return WriteKey(ssKey, ssValue, fOverwrite); - } - - template <typename K> - bool Erase(const K& key) - { - // Key - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - - // Erase - return EraseKey(ssKey); - } - - template <typename K> - bool Exists(const K& key) - { - // Key - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - - // Exists - return HasKey(ssKey); - } - - Dbc* GetCursor(); - int ReadAtCursor(Dbc* pcursor, CDataStream& ssKey, CDataStream& ssValue); - bool TxnBegin(); - bool TxnCommit(); - bool TxnAbort(); + void Flush() override; + void Close() override; - bool static Rewrite(BerkeleyDatabase& database, const char* pszSkip = nullptr); + bool StartCursor() override; + bool ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) override; + void CloseCursor() override; + bool TxnBegin() override; + bool TxnCommit() override; + bool TxnAbort() override; }; std::string BerkeleyDatabaseVersion(); diff --git a/src/wallet/context.h b/src/wallet/context.h index 3c8fdd1c59..a83591154f 100644 --- a/src/wallet/context.h +++ b/src/wallet/context.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_WALLET_CONTEXT_H #define BITCOIN_WALLET_CONTEXT_H +class ArgsManager; namespace interfaces { class Chain; } // namespace interfaces @@ -21,6 +22,7 @@ class Chain; //! behavior. struct WalletContext { interfaces::Chain* chain{nullptr}; + ArgsManager* args{nullptr}; //! Declare default constructor and destructor that are not inline, so code //! instantiating the WalletContext struct doesn't need to #include class diff --git a/src/wallet/db.h b/src/wallet/db.h index 1322bf54fa..76668f8dc2 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -6,7 +6,9 @@ #ifndef BITCOIN_WALLET_DB_H #define BITCOIN_WALLET_DB_H +#include <clientversion.h> #include <fs.h> +#include <streams.h> #include <string> @@ -14,4 +16,82 @@ fs::path WalletDataFilePath(const fs::path& wallet_path); void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename); +/** RAII class that provides access to a WalletDatabase */ +class DatabaseBatch +{ +private: + virtual bool ReadKey(CDataStream&& key, CDataStream& value) = 0; + virtual bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite=true) = 0; + virtual bool EraseKey(CDataStream&& key) = 0; + virtual bool HasKey(CDataStream&& key) = 0; + +public: + explicit DatabaseBatch() {} + virtual ~DatabaseBatch() {} + + DatabaseBatch(const DatabaseBatch&) = delete; + DatabaseBatch& operator=(const DatabaseBatch&) = delete; + + virtual void Flush() = 0; + virtual void Close() = 0; + + template <typename K, typename T> + bool Read(const K& key, T& value) + { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(1000); + ssKey << key; + + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + if (!ReadKey(std::move(ssKey), ssValue)) return false; + try { + ssValue >> value; + return true; + } catch (const std::exception&) { + return false; + } + } + + template <typename K, typename T> + bool Write(const K& key, const T& value, bool fOverwrite = true) + { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(1000); + ssKey << key; + + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + ssValue.reserve(10000); + ssValue << value; + + return WriteKey(std::move(ssKey), std::move(ssValue), fOverwrite); + } + + template <typename K> + bool Erase(const K& key) + { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(1000); + ssKey << key; + + return EraseKey(std::move(ssKey)); + } + + template <typename K> + bool Exists(const K& key) + { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(1000); + ssKey << key; + + return HasKey(std::move(ssKey)); + } + + virtual bool StartCursor() = 0; + virtual bool ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) = 0; + virtual void CloseCursor() = 0; + virtual bool TxnBegin() = 0; + virtual bool TxnCommit() = 0; + virtual bool TxnAbort() = 0; +}; + #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 3885eb6185..781920755c 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -7,8 +7,9 @@ #include <interfaces/chain.h> #include <net.h> #include <node/context.h> +#include <node/ui_interface.h> #include <outputtype.h> -#include <ui_interface.h> +#include <util/check.h> #include <util/moneystr.h> #include <util/system.h> #include <util/translation.h> @@ -16,9 +17,9 @@ #include <wallet/wallet.h> #include <walletinitinterface.h> -class WalletInit : public WalletInitInterface { +class WalletInit : public WalletInitInterface +{ public: - //! Was the wallet component compiled in. bool HasWalletSupport() const override {return true;} @@ -112,10 +113,11 @@ bool WalletInit::ParameterInteraction() const void WalletInit::Construct(NodeContext& node) const { - if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { + ArgsManager& args = *Assert(node.args); + if (args.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { LogPrintf("Wallet disabled!\n"); return; } - gArgs.SoftSetArg("-wallet", ""); - node.chain_clients.emplace_back(interfaces::MakeWalletClient(*node.chain, gArgs.GetArgs("-wallet"))); + args.SoftSetArg("-wallet", ""); + node.chain_clients.emplace_back(interfaces::MakeWalletClient(*node.chain, args, args.GetArgs("-wallet"))); } diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 8df3e78215..c2818a41e7 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -11,6 +11,7 @@ #include <util/system.h> #include <util/translation.h> #include <wallet/wallet.h> +#include <wallet/walletdb.h> bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files) { @@ -82,14 +83,16 @@ bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& walle } } -void StartWallets(CScheduler& scheduler) +void StartWallets(CScheduler& scheduler, const ArgsManager& args) { for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { pwallet->postInitProcess(); } // Schedule periodic wallet flushes and tx rebroadcasts - scheduler.scheduleEvery(MaybeCompactWalletDB, std::chrono::milliseconds{500}); + if (args.GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) { + scheduler.scheduleEvery(MaybeCompactWalletDB, std::chrono::milliseconds{500}); + } scheduler.scheduleEvery(MaybeResendWalletTxs, std::chrono::milliseconds{1000}); } diff --git a/src/wallet/load.h b/src/wallet/load.h index e24b1f2e69..ff4f5b4b23 100644 --- a/src/wallet/load.h +++ b/src/wallet/load.h @@ -9,6 +9,7 @@ #include <string> #include <vector> +class ArgsManager; class CScheduler; namespace interfaces { @@ -22,7 +23,7 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files); //! Complete startup of wallets. -void StartWallets(CScheduler& scheduler); +void StartWallets(CScheduler& scheduler, const ArgsManager& args); //! Flush all wallets in preparation for shutdown. void FlushWallets(); diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index c9ea6c2ad9..3b752ca936 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -1547,7 +1547,7 @@ static UniValue ProcessDescriptorImport(CWallet * const pwallet, const UniValue& if (!w_desc.descriptor->GetOutputType()) { warnings.push_back("Unknown output type, cannot set descriptor to active."); } else { - pwallet->SetActiveScriptPubKeyMan(spk_manager->GetID(), *w_desc.descriptor->GetOutputType(), internal); + pwallet->AddActiveScriptPubKeyMan(spk_manager->GetID(), *w_desc.descriptor->GetOutputType(), internal); } } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 00927a2e55..9d334063c4 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -223,7 +223,7 @@ static void SetFeeEstimateMode(const CWallet* pwallet, CCoinControl& cc, const U cc.m_feerate = CFeeRate(fee_rate); // default RBF to true for explicit fee rate modes - if (cc.m_signal_bip125_rbf == boost::none) cc.m_signal_bip125_rbf = true; + if (cc.m_signal_bip125_rbf == nullopt) cc.m_signal_bip125_rbf = true; } else if (!estimate_param.isNull()) { cc.m_confirm_target = ParseConfirmTarget(estimate_param, pwallet->chain().estimateMaxBlocks()); } @@ -306,7 +306,7 @@ static UniValue getrawchangeaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Error: This wallet has no available keys"); } - OutputType output_type = pwallet->m_default_change_type != OutputType::CHANGE_AUTO ? pwallet->m_default_change_type : pwallet->m_default_address_type; + OutputType output_type = pwallet->m_default_change_type.get_value_or(pwallet->m_default_address_type); if (!request.params[0].isNull()) { if (!ParseOutputType(request.params[0].get_str(), output_type)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[0].get_str())); @@ -359,36 +359,54 @@ static UniValue setlabel(const JSONRPCRequest& request) return NullUniValue; } +void ParseRecipients(const UniValue& address_amounts, const UniValue& subtract_fee_outputs, std::vector<CRecipient> &recipients) { + std::set<CTxDestination> destinations; + int i = 0; + for (const std::string& address: address_amounts.getKeys()) { + CTxDestination dest = DecodeDestination(address); + if (!IsValidDestination(dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, std::string("Invalid Bitcoin address: ") + address); + } -static CTransactionRef SendMoney(CWallet* const pwallet, const CTxDestination& address, CAmount nValue, bool fSubtractFeeFromAmount, const CCoinControl& coin_control, mapValue_t mapValue) -{ - CAmount curBalance = pwallet->GetBalance(0, coin_control.m_avoid_address_reuse).m_mine_trusted; + if (destinations.count(dest)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated address: ") + address); + } + destinations.insert(dest); - // Check amount - if (nValue <= 0) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid amount"); + CScript script_pub_key = GetScriptForDestination(dest); + CAmount amount = AmountFromValue(address_amounts[i++]); - if (nValue > curBalance) - throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Insufficient funds"); + bool subtract_fee = false; + for (unsigned int idx = 0; idx < subtract_fee_outputs.size(); idx++) { + const UniValue& addr = subtract_fee_outputs[idx]; + if (addr.get_str() == address) { + subtract_fee = true; + } + } - // Parse Bitcoin address - CScript scriptPubKey = GetScriptForDestination(address); + CRecipient recipient = {script_pub_key, amount, subtract_fee}; + recipients.push_back(recipient); + } +} + +UniValue SendMoney(CWallet* const pwallet, const CCoinControl &coin_control, std::vector<CRecipient> &recipients, mapValue_t map_value) +{ + EnsureWalletIsUnlocked(pwallet); - // Create and send the transaction + // Shuffle recipient list + std::shuffle(recipients.begin(), recipients.end(), FastRandomContext()); + + // Send CAmount nFeeRequired = 0; - bilingual_str error; - std::vector<CRecipient> vecSend; int nChangePosRet = -1; - CRecipient recipient = {scriptPubKey, nValue, fSubtractFeeFromAmount}; - vecSend.push_back(recipient); + bilingual_str error; CTransactionRef tx; - if (!pwallet->CreateTransaction(vecSend, tx, nFeeRequired, nChangePosRet, error, coin_control)) { - if (!fSubtractFeeFromAmount && nValue + nFeeRequired > curBalance) - error = strprintf(Untranslated("Error: This transaction requires a transaction fee of at least %s"), FormatMoney(nFeeRequired)); - throw JSONRPCError(RPC_WALLET_ERROR, error.original); + bool fCreated = pwallet->CreateTransaction(recipients, tx, nFeeRequired, nChangePosRet, error, coin_control, !pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + if (!fCreated) { + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, error.original); } - pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */); - return tx; + pwallet->CommitTransaction(tx, std::move(map_value), {} /* orderForm */); + return tx->GetHash().GetHex(); } static UniValue sendtoaddress(const JSONRPCRequest& request) @@ -436,16 +454,6 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) LOCK(pwallet->cs_wallet); - CTxDestination dest = DecodeDestination(request.params[0].get_str()); - if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); - } - - // Amount - CAmount nAmount = AmountFromValue(request.params[1]); - if (nAmount <= 0) - throw JSONRPCError(RPC_TYPE_ERROR, "Invalid amount for send"); - // Wallet comments mapValue_t mapValue; if (!request.params[2].isNull() && !request.params[2].get_str().empty()) @@ -471,8 +479,18 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) EnsureWalletIsUnlocked(pwallet); - CTransactionRef tx = SendMoney(pwallet, dest, nAmount, fSubtractFeeFromAmount, coin_control, std::move(mapValue)); - return tx->GetHash().GetHex(); + UniValue address_amounts(UniValue::VOBJ); + const std::string address = request.params[0].get_str(); + address_amounts.pushKV(address, request.params[1]); + UniValue subtractFeeFromAmount(UniValue::VARR); + if (fSubtractFeeFromAmount) { + subtractFeeFromAmount.push_back(address); + } + + std::vector<CRecipient> recipients; + ParseRecipients(address_amounts, subtractFeeFromAmount, recipients); + + return SendMoney(pwallet, coin_control, recipients, mapValue); } static UniValue listaddressgroupings(const JSONRPCRequest& request) @@ -860,52 +878,10 @@ static UniValue sendmany(const JSONRPCRequest& request) SetFeeEstimateMode(pwallet, coin_control, request.params[7], request.params[6]); - std::set<CTxDestination> destinations; - std::vector<CRecipient> vecSend; + std::vector<CRecipient> recipients; + ParseRecipients(sendTo, subtractFeeFromAmount, recipients); - std::vector<std::string> keys = sendTo.getKeys(); - for (const std::string& name_ : keys) { - CTxDestination dest = DecodeDestination(name_); - if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, std::string("Invalid Bitcoin address: ") + name_); - } - - if (destinations.count(dest)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated address: ") + name_); - } - destinations.insert(dest); - - CScript scriptPubKey = GetScriptForDestination(dest); - CAmount nAmount = AmountFromValue(sendTo[name_]); - if (nAmount <= 0) - throw JSONRPCError(RPC_TYPE_ERROR, "Invalid amount for send"); - - bool fSubtractFeeFromAmount = false; - for (unsigned int idx = 0; idx < subtractFeeFromAmount.size(); idx++) { - const UniValue& addr = subtractFeeFromAmount[idx]; - if (addr.get_str() == name_) - fSubtractFeeFromAmount = true; - } - - CRecipient recipient = {scriptPubKey, nAmount, fSubtractFeeFromAmount}; - vecSend.push_back(recipient); - } - - EnsureWalletIsUnlocked(pwallet); - - // Shuffle recipient list - std::shuffle(vecSend.begin(), vecSend.end(), FastRandomContext()); - - // Send - CAmount nFeeRequired = 0; - int nChangePosRet = -1; - bilingual_str error; - CTransactionRef tx; - bool fCreated = pwallet->CreateTransaction(vecSend, tx, nFeeRequired, nChangePosRet, error, coin_control); - if (!fCreated) - throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, error.original); - pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */); - return tx->GetHash().GetHex(); + return SendMoney(pwallet, coin_control, recipients, std::move(mapValue)); } static UniValue addmultisigaddress(const JSONRPCRequest& request) @@ -2993,10 +2969,11 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f if (options.exists("changeAddress")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both changeAddress and address_type options"); } - coinControl.m_change_type = pwallet->m_default_change_type; - if (!ParseOutputType(options["change_type"].get_str(), *coinControl.m_change_type)) { + OutputType out_type; + if (!ParseOutputType(options["change_type"].get_str(), out_type)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown change type '%s'", options["change_type"].get_str())); } + coinControl.m_change_type.emplace(out_type); } coinControl.fAllowWatchOnly = ParseIncludeWatchonly(options["includeWatching"], *pwallet); @@ -3140,7 +3117,7 @@ static UniValue fundrawtransaction(const JSONRPCRequest& request) CAmount fee; int change_position; CCoinControl coin_control; - // Automatically select (additional) coins. Can be overriden by options.add_inputs. + // Automatically select (additional) coins. Can be overridden by options.add_inputs. coin_control.m_add_inputs = true; FundTransaction(pwallet, tx, fee, change_position, request.params[1], coin_control); @@ -4074,7 +4051,7 @@ UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) CMutableTransaction rawTx = ConstructTransaction(request.params[0], request.params[1], request.params[2], rbf); CCoinControl coin_control; // Automatically select coins, unless at least one is manually selected. Can - // be overriden by options.add_inputs. + // be overridden by options.add_inputs. coin_control.m_add_inputs = rawTx.vin.size() == 0; FundTransaction(pwallet, rawTx, fee, change_position, request.params[3], coin_control); diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index d42950ee42..e6e62332c0 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -20,6 +20,11 @@ bool RecoverDatabaseFile(const fs::path& file_path) std::string filename; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename); + if (!env->Open(true /* retry */)) { + tfm::format(std::cerr, "Error initializing wallet database environment %s!", env->Directory()); + return false; + } + // Recovery procedure: // move wallet file to walletfilename.timestamp.bak // Call Salvage with fAggressive=true to diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index d7c50a9d2a..51715462c5 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -597,11 +597,6 @@ TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psb continue; } - // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. - if (!input.IsSane()) { - return TransactionError::INVALID_PSBT; - } - // Get the Sighash type if (sign && input.sighash_type > 0 && input.sighash_type != sighash_type) { return TransactionError::SIGHASH_MISMATCH; @@ -910,20 +905,22 @@ bool LegacyScriptPubKeyMan::AddWatchOnly(const CScript& dest, int64_t nCreateTim return AddWatchOnly(dest); } -void LegacyScriptPubKeyMan::SetHDChain(const CHDChain& chain, bool memonly) +void LegacyScriptPubKeyMan::LoadHDChain(const CHDChain& chain) { LOCK(cs_KeyStore); - // memonly == true means we are loading the wallet file - // memonly == false means that the chain is actually being changed - if (!memonly) { - // Store the new chain - if (!WalletBatch(m_storage.GetDatabase()).WriteHDChain(chain)) { - throw std::runtime_error(std::string(__func__) + ": writing chain failed"); - } - // When there's an old chain, add it as an inactive chain as we are now rotating hd chains - if (!m_hd_chain.seed_id.IsNull()) { - AddInactiveHDChain(m_hd_chain); - } + m_hd_chain = chain; +} + +void LegacyScriptPubKeyMan::AddHDChain(const CHDChain& chain) +{ + LOCK(cs_KeyStore); + // Store the new chain + if (!WalletBatch(m_storage.GetDatabase()).WriteHDChain(chain)) { + throw std::runtime_error(std::string(__func__) + ": writing chain failed"); + } + // When there's an old chain, add it as an inactive chain as we are now rotating hd chains + if (!m_hd_chain.seed_id.IsNull()) { + AddInactiveHDChain(m_hd_chain); } m_hd_chain = chain; @@ -1177,7 +1174,7 @@ void LegacyScriptPubKeyMan::SetHDSeed(const CPubKey& seed) CHDChain newHdChain; newHdChain.nVersion = m_storage.CanSupportFeature(FEATURE_HD_SPLIT) ? CHDChain::VERSION_HD_CHAIN_SPLIT : CHDChain::VERSION_HD_BASE; newHdChain.seed_id = seed.GetID(); - SetHDChain(newHdChain, false); + AddHDChain(newHdChain); NotifyCanGetAddressesChanged(); WalletBatch batch(m_storage.GetDatabase()); m_storage.UnsetBlankWalletFlag(batch); @@ -1900,8 +1897,8 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_ desc_prefix = "wpkh(" + xpub + "/84'"; break; } - default: assert(false); - } + } // no default case, so the compiler can warn about missing cases + assert(!desc_prefix.empty()); // Mainnet derives at 0', testnet and regtest derive at 1' if (Params().IsTestChain()) { @@ -2086,11 +2083,6 @@ TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& continue; } - // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. - if (!input.IsSane()) { - return TransactionError::INVALID_PSBT; - } - // Get the Sighash type if (sign && input.sighash_type > 0 && input.sighash_type != sighash_type) { return TransactionError::SIGHASH_MISMATCH; diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 9fa2a68284..a96d971734 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -422,8 +422,10 @@ public: //! Generate a new key CPubKey GenerateNewKey(WalletBatch& batch, CHDChain& hd_chain, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); - /* Set the HD chain model (chain child index counters) */ - void SetHDChain(const CHDChain& chain, bool memonly); + /* Set the HD chain model (chain child index counters) and writes it to the database */ + void AddHDChain(const CHDChain& chain); + //! Load a HD chain model (used by LoadWallet) + void LoadHDChain(const CHDChain& chain); const CHDChain& GetHDChain() const { return m_hd_chain; } void AddInactiveHDChain(const CHDChain& chain); diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index 797a0d634f..35bd965673 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -3,13 +3,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <fs.h> +#include <util/check.h> #include <util/system.h> #include <wallet/test/init_test_fixture.h> -InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainName): BasicTestingSetup(chainName) +InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) { - m_chain_client = MakeWalletClient(*m_chain, {}); + m_chain_client = MakeWalletClient(*m_chain, *Assert(m_node.args), {}); std::string sep; sep += fs::path::preferred_separator; diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 3f85a48ff3..ce7e661b67 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -64,7 +64,7 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; std::string final_hex = HexStr(ssTx); - BOOST_CHECK_EQUAL(final_hex, "70736274ff01009a020000000258e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd750000000000ffffffff838d0427d0ec650a68aa46bb0b098aea4422c071b2ca78352a077959d07cea1d0100000000ffffffff0270aaf00800000000160014d85c2b71d0060b09c9886aeb815e50991dda124d00e1f5050000000016001400aea9a2e5f0f876a588df5546e8742d1d87008f00000000000100bb0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f6187650000000104475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae2206029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f10d90c6a4f000000800000008000000080220602dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d710d90c6a4f0000008000000080010000800001012000c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88701042200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903010547522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae2206023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7310d90c6a4f000000800000008003000080220603089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc10d90c6a4f00000080000000800200008000220203a9a4c37f5996d3aa25dbac6b570af0650394492942460b354753ed9eeca5877110d90c6a4f000000800000008004000080002202027f6399757d2eff55a136ad02c684b1838b6556e5f1b6b34282a94b6b5005109610d90c6a4f00000080000000800500008000"); + BOOST_CHECK_EQUAL(final_hex, "70736274ff01009a020000000258e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd750000000000ffffffff838d0427d0ec650a68aa46bb0b098aea4422c071b2ca78352a077959d07cea1d0100000000ffffffff0270aaf00800000000160014d85c2b71d0060b09c9886aeb815e50991dda124d00e1f5050000000016001400aea9a2e5f0f876a588df5546e8742d1d87008f00000000000100bb0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f6187650000000104475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae2206029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f10d90c6a4f000000800000008000000080220602dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d710d90c6a4f0000008000000080010000800001008a020000000158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e8876500000001012000c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88701042200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903010547522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae2206023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7310d90c6a4f000000800000008003000080220603089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc10d90c6a4f00000080000000800200008000220203a9a4c37f5996d3aa25dbac6b570af0650394492942460b354753ed9eeca5877110d90c6a4f000000800000008004000080002202027f6399757d2eff55a136ad02c684b1838b6556e5f1b6b34282a94b6b5005109610d90c6a4f00000080000000800500008000"); // Mutate the transaction so that one of the inputs is invalid psbtx.tx->vin[0].prevout.n = 2; diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index 6c32868b1e..99d7cfe921 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -10,17 +10,18 @@ #include <interfaces/chain.h> #include <interfaces/wallet.h> #include <node/context.h> +#include <util/check.h> #include <wallet/wallet.h> #include <memory> /** Testing setup and teardown for wallet. */ -struct WalletTestingSetup: public TestingSetup { +struct WalletTestingSetup : public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); - std::unique_ptr<interfaces::ChainClient> m_chain_client = interfaces::MakeWalletClient(*m_chain, {}); + std::unique_ptr<interfaces::ChainClient> m_chain_client = interfaces::MakeWalletClient(*m_chain, *Assert(m_node.args), {}); CWallet m_wallet; std::unique_ptr<interfaces::Handler> m_chain_notifications_handler; }; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 497ccd14bb..9cc847b2d0 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -118,7 +118,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Prune the older block file. { LOCK(cs_main); - EnsureChainman(m_node).PruneOneBlockFile(oldTip->GetBlockPos().nFile); + Assert(m_node.chainman)->PruneOneBlockFile(oldTip->GetBlockPos().nFile); } UnlinkPrunedFiles({oldTip->GetBlockPos().nFile}); @@ -144,7 +144,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Prune the remaining block file. { LOCK(cs_main); - EnsureChainman(m_node).PruneOneBlockFile(newTip->GetBlockPos().nFile); + Assert(m_node.chainman)->PruneOneBlockFile(newTip->GetBlockPos().nFile); } UnlinkPrunedFiles({newTip->GetBlockPos().nFile}); @@ -181,7 +181,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // Prune the older block file. { LOCK(cs_main); - EnsureChainman(m_node).PruneOneBlockFile(oldTip->GetBlockPos().nFile); + Assert(m_node.chainman)->PruneOneBlockFile(oldTip->GetBlockPos().nFile); } UnlinkPrunedFiles({oldTip->GetBlockPos().nFile}); @@ -333,7 +333,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 50*COIN); } -static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) +static int64_t AddTx(ChainstateManager& chainman, CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) { CMutableTransaction tx; CWalletTx::Confirmation confirm; @@ -341,7 +341,8 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 SetMockTime(mockTime); CBlockIndex* block = nullptr; if (blockTime > 0) { - auto inserted = ::BlockIndex().emplace(GetRandHash(), new CBlockIndex); + LOCK(cs_main); + auto inserted = chainman.BlockIndex().emplace(GetRandHash(), new CBlockIndex); assert(inserted.second); const uint256& hash = inserted.first->first; block = inserted.first->second; @@ -363,24 +364,24 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 BOOST_AUTO_TEST_CASE(ComputeTimeSmart) { // New transaction should use clock time if lower than block time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 1, 100, 120), 100); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 1, 100, 120), 100); // Test that updating existing transaction does not change smart time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 1, 200, 220), 100); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 1, 200, 220), 100); // New transaction should use clock time if there's no block time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 2, 300, 0), 300); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 2, 300, 0), 300); // New transaction should use block time if lower than clock time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 3, 420, 400), 400); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 3, 420, 400), 400); // New transaction should use latest entry time if higher than // min(block time, clock time). - BOOST_CHECK_EQUAL(AddTx(m_wallet, 4, 500, 390), 400); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 4, 500, 390), 400); // If there are future entries, new transaction should use time of the // newest entry that is no more than 300 seconds ahead of the clock time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 5, 50, 600), 300); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 5, 50, 600), 300); // Reset mock time for other tests. SetMockTime(0); @@ -790,4 +791,37 @@ BOOST_FIXTURE_TEST_CASE(CreateWalletFromFile, TestChain100Setup) TestUnloadWallet(std::move(wallet)); } +BOOST_FIXTURE_TEST_CASE(ZapSelectTx, TestChain100Setup) +{ + auto chain = interfaces::MakeChain(m_node); + auto wallet = TestLoadWallet(*chain); + CKey key; + key.MakeNewKey(true); + AddKey(*wallet, key); + + std::string error; + m_coinbase_txns.push_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + auto block_tx = TestSimpleSpend(*m_coinbase_txns[0], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); + CreateAndProcessBlock({block_tx}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + + SyncWithValidationInterfaceQueue(); + + { + auto block_hash = block_tx.GetHash(); + auto prev_hash = m_coinbase_txns[0]->GetHash(); + + LOCK(wallet->cs_wallet); + BOOST_CHECK(wallet->HasWalletSpend(prev_hash)); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_hash), 1u); + + std::vector<uint256> vHashIn{ block_hash }, vHashOut; + BOOST_CHECK_EQUAL(wallet->ZapSelectTx(vHashIn, vHashOut), DBErrors::LOAD_OK); + + BOOST_CHECK(!wallet->HasWalletSpend(prev_hash)); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_hash), 0u); + } + + TestUnloadWallet(std::move(wallet)); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 57eec9baf9..8eec00993f 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -99,9 +99,11 @@ std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet) return interfaces::MakeHandler([it] { LOCK(cs_wallets); g_load_wallet_fns.erase(it); }); } +static Mutex g_loading_wallet_mutex; static Mutex g_wallet_release_mutex; static std::condition_variable g_wallet_release_cv; -static std::set<std::string> g_unloading_wallet_set; +static std::set<std::string> g_loading_wallet_set GUARDED_BY(g_loading_wallet_mutex); +static std::set<std::string> g_unloading_wallet_set GUARDED_BY(g_wallet_release_mutex); // Custom deleter for shared_ptr<CWallet>. static void ReleaseWallet(CWallet* wallet) @@ -145,7 +147,8 @@ void UnloadWallet(std::shared_ptr<CWallet>&& wallet) } } -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) +namespace { +std::shared_ptr<CWallet> LoadWalletInternal(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) { try { if (!CWallet::Verify(chain, location, error, warnings)) { @@ -166,6 +169,19 @@ std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocati return nullptr; } } +} // namespace + +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) +{ + auto result = WITH_LOCK(g_loading_wallet_mutex, return g_loading_wallet_set.insert(location.GetName())); + if (!result.second) { + error = Untranslated("Wallet already being loading."); + return nullptr; + } + auto wallet = LoadWalletInternal(chain, location, error, warnings); + WITH_LOCK(g_loading_wallet_mutex, g_loading_wallet_set.erase(result.first)); + return wallet; +} std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) { @@ -1406,19 +1422,28 @@ bool CWallet::IsWalletFlagSet(uint64_t flag) const return (m_wallet_flags & flag); } -bool CWallet::SetWalletFlags(uint64_t overwriteFlags, bool memonly) +bool CWallet::LoadWalletFlags(uint64_t flags) { LOCK(cs_wallet); - m_wallet_flags = overwriteFlags; - if (((overwriteFlags & KNOWN_WALLET_FLAGS) >> 32) ^ (overwriteFlags >> 32)) { + if (((flags & KNOWN_WALLET_FLAGS) >> 32) ^ (flags >> 32)) { // contains unknown non-tolerable wallet flags return false; } - if (!memonly && !WalletBatch(*database).WriteWalletFlags(m_wallet_flags)) { + m_wallet_flags = flags; + + return true; +} + +bool CWallet::AddWalletFlags(uint64_t flags) +{ + LOCK(cs_wallet); + // We should never be writing unknown non-tolerable wallet flags + assert(((flags & KNOWN_WALLET_FLAGS) >> 32) == (flags >> 32)); + if (!WalletBatch(*database).WriteWalletFlags(flags)) { throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed"); } - return true; + return LoadWalletFlags(flags); } int64_t CWalletTx::GetTxTime() const @@ -2491,13 +2516,8 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp continue; } - // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. - if (!input.IsSane()) { - return TransactionError::INVALID_PSBT; - } - // If we have no utxo, grab it from the wallet. - if (!input.non_witness_utxo && input.witness_utxo.IsNull()) { + if (!input.non_witness_utxo) { const uint256& txhash = txin.prevout.hash; const auto it = mapWallet.find(txhash); if (it != mapWallet.end()) { @@ -2653,11 +2673,11 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, const uin return locktime; } -OutputType CWallet::TransactionChangeType(OutputType change_type, const std::vector<CRecipient>& vecSend) +OutputType CWallet::TransactionChangeType(const Optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend) { // If -changetype is specified, always use that change type. - if (change_type != OutputType::CHANGE_AUTO) { - return change_type; + if (change_type) { + return *change_type; } // if m_default_address_type is legacy, use legacy address as change (even @@ -3109,9 +3129,11 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 { AssertLockHeld(cs_wallet); DBErrors nZapSelectTxRet = WalletBatch(*database, "cr+").ZapSelectTx(vHashIn, vHashOut); - for (uint256 hash : vHashOut) { + for (const uint256& hash : vHashOut) { const auto& it = mapWallet.find(hash); wtxOrdered.erase(it->second.m_it_wtxOrdered); + for (const auto& txin : it->second.tx->vin) + mapTxSpends.erase(txin.prevout); mapWallet.erase(it); NotifyTransactionChanged(this, hash, CT_DELETED); } @@ -3721,15 +3743,11 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b std::unique_ptr<WalletDatabase> database = CreateWalletDatabase(wallet_path); try { - if (!WalletBatch::VerifyEnvironment(wallet_path, error_string)) { - return false; - } + return database->Verify(error_string); } catch (const fs::filesystem_error& e) { error_string = Untranslated(strprintf("Error loading wallet %s. %s", location.GetName(), fsbridge::get_filesystem_error_message(e))); return false; } - - return WalletBatch::VerifyDatabaseFile(wallet_path, error_string); } std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings, uint64_t wallet_creation_flags) @@ -3789,7 +3807,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // ensure this wallet.dat can only be opened by clients supporting HD with chain split and expects no default key walletInstance->SetMinVersion(FEATURE_LATEST); - walletInstance->SetWalletFlags(wallet_creation_flags, false); + walletInstance->AddWalletFlags(wallet_creation_flags); // Only create LegacyScriptPubKeyMan when not descriptor wallet if (!walletInstance->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { @@ -3826,14 +3844,20 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - if (!gArgs.GetArg("-addresstype", "").empty() && !ParseOutputType(gArgs.GetArg("-addresstype", ""), walletInstance->m_default_address_type)) { - error = strprintf(_("Unknown address type '%s'"), gArgs.GetArg("-addresstype", "")); - return nullptr; + if (!gArgs.GetArg("-addresstype", "").empty()) { + if (!ParseOutputType(gArgs.GetArg("-addresstype", ""), walletInstance->m_default_address_type)) { + error = strprintf(_("Unknown address type '%s'"), gArgs.GetArg("-addresstype", "")); + return nullptr; + } } - if (!gArgs.GetArg("-changetype", "").empty() && !ParseOutputType(gArgs.GetArg("-changetype", ""), walletInstance->m_default_change_type)) { - error = strprintf(_("Unknown change type '%s'"), gArgs.GetArg("-changetype", "")); - return nullptr; + if (!gArgs.GetArg("-changetype", "").empty()) { + OutputType out_type; + if (!ParseOutputType(gArgs.GetArg("-changetype", ""), out_type)) { + error = strprintf(_("Unknown change type '%s'"), gArgs.GetArg("-changetype", "")); + return nullptr; + } + walletInstance->m_default_change_type = out_type; } if (gArgs.IsArgSet("-mintxfee")) { @@ -4404,12 +4428,21 @@ void CWallet::SetupDescriptorScriptPubKeyMans() spk_manager->SetupDescriptorGeneration(master_key, t); uint256 id = spk_manager->GetID(); m_spk_managers[id] = std::move(spk_manager); - SetActiveScriptPubKeyMan(id, t, internal); + AddActiveScriptPubKeyMan(id, t, internal); } } } -void CWallet::SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal, bool memonly) +void CWallet::AddActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal) +{ + WalletBatch batch(*database); + if (!batch.WriteActiveScriptPubKeyMan(static_cast<uint8_t>(type), id, internal)) { + throw std::runtime_error(std::string(__func__) + ": writing active ScriptPubKeyMan id failed"); + } + LoadActiveScriptPubKeyMan(id, type, internal); +} + +void CWallet::LoadActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal) { WalletLogPrintf("Setting spkMan to active: id = %s, type = %d, internal = %d\n", id.ToString(), static_cast<int>(type), static_cast<int>(internal)); auto& spk_mans = internal ? m_internal_spk_managers : m_external_spk_managers; @@ -4417,12 +4450,6 @@ void CWallet::SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool interna spk_man->SetInternal(internal); spk_mans[type] = spk_man; - if (!memonly) { - WalletBatch batch(*database); - if (!batch.WriteActiveScriptPubKeyMan(static_cast<uint8_t>(type), id, internal)) { - throw std::runtime_error(std::string(__func__) + ": writing active ScriptPubKeyMan id failed"); - } - } NotifyCanGetAddressesChanged(); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 9931671fb4..8cb2a64484 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -13,11 +13,11 @@ #include <policy/feerate.h> #include <psbt.h> #include <tinyformat.h> -#include <ui_interface.h> #include <util/message.h> #include <util/strencodings.h> #include <util/string.h> #include <util/system.h> +#include <util/ui_change_type.h> #include <validationinterface.h> #include <wallet/coinselection.h> #include <wallet/crypter.h> @@ -105,9 +105,6 @@ class ReserveDestination; //! Default for -addresstype constexpr OutputType DEFAULT_ADDRESS_TYPE{OutputType::BECH32}; -//! Default for -changetype -constexpr OutputType DEFAULT_CHANGE_TYPE{OutputType::CHANGE_AUTO}; - static constexpr uint64_t KNOWN_WALLET_FLAGS = WALLET_FLAG_AVOID_REUSE | WALLET_FLAG_BLANK_WALLET @@ -934,7 +931,7 @@ public: Balance GetBalance(int min_depth = 0, bool avoid_reuse = true) const; CAmount GetAvailableBalance(const CCoinControl* coinControl = nullptr) const; - OutputType TransactionChangeType(OutputType change_type, const std::vector<CRecipient>& vecSend); + OutputType TransactionChangeType(const Optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend); /** * Insert additional inputs into the transaction by @@ -1012,7 +1009,13 @@ public: CFeeRate m_fallback_fee{DEFAULT_FALLBACK_FEE}; CFeeRate m_discard_rate{DEFAULT_DISCARD_FEE}; OutputType m_default_address_type{DEFAULT_ADDRESS_TYPE}; - OutputType m_default_change_type{DEFAULT_CHANGE_TYPE}; + /** + * Default output type for change outputs. When unset, automatically choose type + * based on address type setting and the types other of non-change outputs + * (see -changetype option documentation and implementation in + * CWallet::TransactionChangeType for details). + */ + Optional<OutputType> m_default_change_type{}; /** Absolute maximum transaction fee (in satoshis) used by default for the wallet */ CAmount m_default_max_tx_fee{DEFAULT_TRANSACTION_MAXFEE}; @@ -1173,7 +1176,9 @@ public: /** overwrite all flags by the given uint64_t returns false if unknown, non-tolerable flags are present */ - bool SetWalletFlags(uint64_t overwriteFlags, bool memOnly); + bool AddWalletFlags(uint64_t flags); + /** Loads the flags into the wallet. (used by LoadWallet) */ + bool LoadWalletFlags(uint64_t flags); /** Determine if we are a legacy wallet */ bool IsLegacy() const; @@ -1251,12 +1256,17 @@ public: //! Instantiate a descriptor ScriptPubKeyMan from the WalletDescriptor and load it void LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc); - //! Sets the active ScriptPubKeyMan for the specified type and internal + //! Adds the active ScriptPubKeyMan for the specified type and internal. Writes it to the wallet file + //! @param[in] id The unique id for the ScriptPubKeyMan + //! @param[in] type The OutputType this ScriptPubKeyMan provides addresses for + //! @param[in] internal Whether this ScriptPubKeyMan provides change addresses + void AddActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal); + + //! Loads an active ScriptPubKeyMan for the specified type and internal. (used by LoadWallet) //! @param[in] id The unique id for the ScriptPubKeyMan //! @param[in] type The OutputType this ScriptPubKeyMan provides addresses for //! @param[in] internal Whether this ScriptPubKeyMan provides change addresses - //! @param[in] memonly Whether to record this update to the database. Set to true for wallet loading, normally false when actually updating the wallet. - void SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal, bool memonly = false); + void LoadActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal); //! Create new DescriptorScriptPubKeyMans and add them to the wallet void SetupDescriptorScriptPubKeyMans(); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 603887ee58..1478687bf9 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -121,7 +121,7 @@ bool WalletBatch::WriteCryptedKey(const CPubKey& vchPubKey, if (!WriteIC(key, std::make_pair(vchCryptedSecret, checksum), false)) { // It may already exist, so try writing just the checksum std::vector<unsigned char> val; - if (!m_batch.Read(key, val)) { + if (!m_batch->Read(key, val)) { return false; } if (!WriteIC(key, std::make_pair(val, checksum), true)) { @@ -166,8 +166,8 @@ bool WalletBatch::WriteBestBlock(const CBlockLocator& locator) bool WalletBatch::ReadBestBlock(CBlockLocator& locator) { - if (m_batch.Read(DBKeys::BESTBLOCK, locator) && !locator.vHave.empty()) return true; - return m_batch.Read(DBKeys::BESTBLOCK_NOMERKLE, locator); + if (m_batch->Read(DBKeys::BESTBLOCK, locator) && !locator.vHave.empty()) return true; + return m_batch->Read(DBKeys::BESTBLOCK_NOMERKLE, locator); } bool WalletBatch::WriteOrderPosNext(int64_t nOrderPosNext) @@ -177,7 +177,7 @@ bool WalletBatch::WriteOrderPosNext(int64_t nOrderPosNext) bool WalletBatch::ReadPool(int64_t nPool, CKeyPool& keypool) { - return m_batch.Read(std::make_pair(DBKeys::POOL, nPool), keypool); + return m_batch->Read(std::make_pair(DBKeys::POOL, nPool), keypool); } bool WalletBatch::WritePool(int64_t nPool, const CKeyPool& keypool) @@ -539,11 +539,11 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } else if (strType == DBKeys::HDCHAIN) { CHDChain chain; ssValue >> chain; - pwallet->GetOrCreateLegacyScriptPubKeyMan()->SetHDChain(chain, true); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadHDChain(chain); } else if (strType == DBKeys::FLAGS) { uint64_t flags; ssValue >> flags; - if (!pwallet->SetWalletFlags(flags, true)) { + if (!pwallet->LoadWalletFlags(flags)) { strErr = "Error reading wallet database: Unknown non-tolerable wallet flags found"; return false; } @@ -592,9 +592,6 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, ssValue >> ser_xpub; CExtPubKey xpub; xpub.Decode(ser_xpub.data()); - if (wss.m_descriptor_caches.count(desc_id)) { - wss.m_descriptor_caches[desc_id] = DescriptorCache(); - } if (parent) { wss.m_descriptor_caches[desc_id].CacheParentExtPubKey(key_exp_index, xpub); } else { @@ -693,15 +690,14 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) LOCK(pwallet->cs_wallet); try { int nMinVersion = 0; - if (m_batch.Read(DBKeys::MINVERSION, nMinVersion)) { + if (m_batch->Read(DBKeys::MINVERSION, nMinVersion)) { if (nMinVersion > FEATURE_LATEST) return DBErrors::TOO_NEW; pwallet->LoadMinVersion(nMinVersion); } // Get cursor - Dbc* pcursor = m_batch.GetCursor(); - if (!pcursor) + if (!m_batch->StartCursor()) { pwallet->WalletLogPrintf("Error getting wallet database cursor\n"); return DBErrors::CORRUPT; @@ -712,11 +708,14 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = m_batch.ReadAtCursor(pcursor, ssKey, ssValue); - if (ret == DB_NOTFOUND) + bool complete; + bool ret = m_batch->ReadAtCursor(ssKey, ssValue, complete); + if (complete) { break; - else if (ret != 0) + } + else if (!ret) { + m_batch->CloseCursor(); pwallet->WalletLogPrintf("Error reading next record from wallet database\n"); return DBErrors::CORRUPT; } @@ -743,17 +742,17 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (!strErr.empty()) pwallet->WalletLogPrintf("%s\n", strErr); } - pcursor->close(); } catch (...) { result = DBErrors::CORRUPT; } + m_batch->CloseCursor(); // Set the active ScriptPubKeyMans for (auto spk_man_pair : wss.m_active_external_spks) { - pwallet->SetActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ false, /* memonly */ true); + pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ false); } for (auto spk_man_pair : wss.m_active_internal_spks) { - pwallet->SetActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ true, /* memonly */ true); + pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ true); } // Set the descriptor caches @@ -783,7 +782,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) // Last client version to open this wallet, was previously the file version number int last_client = CLIENT_VERSION; - m_batch.Read(DBKeys::VERSION, last_client); + m_batch->Read(DBKeys::VERSION, last_client); int wallet_version = pwallet->GetVersion(); pwallet->WalletLogPrintf("Wallet File Version = %d\n", wallet_version > 0 ? wallet_version : last_client); @@ -808,7 +807,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) return DBErrors::NEED_REWRITE; if (last_client < CLIENT_VERSION) // Update - m_batch.Write(DBKeys::VERSION, CLIENT_VERSION); + m_batch->Write(DBKeys::VERSION, CLIENT_VERSION); if (wss.fAnyUnordered) result = pwallet->ReorderTransactions(); @@ -844,14 +843,13 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWal try { int nMinVersion = 0; - if (m_batch.Read(DBKeys::MINVERSION, nMinVersion)) { + if (m_batch->Read(DBKeys::MINVERSION, nMinVersion)) { if (nMinVersion > FEATURE_LATEST) return DBErrors::TOO_NEW; } // Get cursor - Dbc* pcursor = m_batch.GetCursor(); - if (!pcursor) + if (!m_batch->StartCursor()) { LogPrintf("Error getting wallet database cursor\n"); return DBErrors::CORRUPT; @@ -862,11 +860,12 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWal // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = m_batch.ReadAtCursor(pcursor, ssKey, ssValue); - if (ret == DB_NOTFOUND) + bool complete; + bool ret = m_batch->ReadAtCursor(ssKey, ssValue, complete); + if (complete) { break; - else if (ret != 0) - { + } else if (!ret) { + m_batch->CloseCursor(); LogPrintf("Error reading next record from wallet database\n"); return DBErrors::CORRUPT; } @@ -881,10 +880,10 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWal ssValue >> vWtx.back(); } } - pcursor->close(); } catch (...) { result = DBErrors::CORRUPT; } + m_batch->CloseCursor(); return result; } @@ -950,9 +949,6 @@ void MaybeCompactWalletDB() if (fOneThread.exchange(true)) { return; } - if (!gArgs.GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) { - return; - } for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { WalletDatabase& dbh = pwallet->GetDBHandle(); @@ -965,7 +961,7 @@ void MaybeCompactWalletDB() } if (dbh.nLastFlushed != nUpdateCounter && GetTime() - dbh.nLastWalletUpdate >= 2) { - if (BerkeleyBatch::PeriodicFlush(dbh)) { + if (dbh.PeriodicFlush()) { dbh.nLastFlushed = nUpdateCounter; } } @@ -974,16 +970,6 @@ void MaybeCompactWalletDB() fOneThread = false; } -bool WalletBatch::VerifyEnvironment(const fs::path& wallet_path, bilingual_str& errorStr) -{ - return BerkeleyBatch::VerifyEnvironment(wallet_path, errorStr); -} - -bool WalletBatch::VerifyDatabaseFile(const fs::path& wallet_path, bilingual_str& errorStr) -{ - return BerkeleyBatch::VerifyDatabaseFile(wallet_path, errorStr); -} - bool WalletBatch::WriteDestData(const std::string &address, const std::string &key, const std::string &value) { return WriteIC(std::make_pair(DBKeys::DESTDATA, std::make_pair(address, key)), value); @@ -1007,17 +993,17 @@ bool WalletBatch::WriteWalletFlags(const uint64_t flags) bool WalletBatch::TxnBegin() { - return m_batch.TxnBegin(); + return m_batch->TxnBegin(); } bool WalletBatch::TxnCommit() { - return m_batch.TxnCommit(); + return m_batch->TxnCommit(); } bool WalletBatch::TxnAbort() { - return m_batch.TxnAbort(); + return m_batch->TxnAbort(); } bool IsWalletLoaded(const fs::path& wallet_path) diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 61e0f19e56..6b55361c07 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -183,12 +183,12 @@ private: template <typename K, typename T> bool WriteIC(const K& key, const T& value, bool fOverwrite = true) { - if (!m_batch.Write(key, value, fOverwrite)) { + if (!m_batch->Write(key, value, fOverwrite)) { return false; } m_database.IncrementUpdateCounter(); if (m_database.nUpdateCounter % 1000 == 0) { - m_batch.Flush(); + m_batch->Flush(); } return true; } @@ -196,19 +196,19 @@ private: template <typename K> bool EraseIC(const K& key) { - if (!m_batch.Erase(key)) { + if (!m_batch->Erase(key)) { return false; } m_database.IncrementUpdateCounter(); if (m_database.nUpdateCounter % 1000 == 0) { - m_batch.Flush(); + m_batch->Flush(); } return true; } public: explicit WalletBatch(WalletDatabase& database, const char* pszMode = "r+", bool _fFlushOnClose = true) : - m_batch(database, pszMode, _fFlushOnClose), + m_batch(database.MakeBatch(pszMode, _fFlushOnClose)), m_database(database) { } @@ -280,7 +280,7 @@ public: //! Abort current transaction bool TxnAbort(); private: - BerkeleyBatch m_batch; + std::unique_ptr<BerkeleyBatch> m_batch; WalletDatabase& m_database; }; diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 77ed6beb5d..8a45d81456 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -112,7 +112,7 @@ static bool SalvageWallet(const fs::path& path) // Initialize the environment before recovery bilingual_str error_string; try { - WalletBatch::VerifyEnvironment(path, error_string); + database->Verify(error_string); } catch (const fs::filesystem_error& e) { error_string = Untranslated(strprintf("Error loading wallet. %s", fsbridge::get_filesystem_error_message(e))); } @@ -140,11 +140,6 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) tfm::format(std::cerr, "Error: no wallet file at %s\n", name); return false; } - bilingual_str error; - if (!WalletBatch::VerifyEnvironment(path, error)) { - tfm::format(std::cerr, "%s\nError loading %s. Is wallet being used by other process?\n", error.original, name); - return false; - } if (command == "info") { std::shared_ptr<CWallet> wallet_instance = LoadWallet(name, path); diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 549e8b2029..19cdc10935 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -30,7 +30,10 @@ class BIP68Test(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.extra_args = [ - ["-acceptnonstdtxn=1"], + [ + "-acceptnonstdtxn=1", + "-peertimeout=9999", # bump because mocktime might cause a disconnect otherwise + ], ["-acceptnonstdtxn=0"], ] diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 7eabf86cad..5538d6d3b4 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -35,7 +35,11 @@ class MaxUploadTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 - self.extra_args = [["-maxuploadtarget=800", "-acceptnonstdtxn=1"]] + self.extra_args = [[ + "-maxuploadtarget=800", + "-acceptnonstdtxn=1", + "-peertimeout=9999", # bump because mocktime might cause a disconnect otherwise + ]] self.supports_cli = False # Cache for utxos, as the listunspent may take a long time later in the test @@ -137,8 +141,8 @@ class MaxUploadTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() - self.log.info("Restarting node 0 with noban permission and 1MB maxuploadtarget") - self.restart_node(0, ["-whitelist=noban@127.0.0.1", "-maxuploadtarget=1"]) + self.log.info("Restarting node 0 with download permission and 1MB maxuploadtarget") + self.restart_node(0, ["-whitelist=download@127.0.0.1", "-maxuploadtarget=1"]) # Reconnect to self.nodes[0] self.nodes[0].add_p2p_connection(TestP2PConn()) @@ -151,9 +155,12 @@ class MaxUploadTest(BitcoinTestFramework): getdata_request.inv = [CInv(MSG_BLOCK, big_old_block)] self.nodes[0].p2p.send_and_ping(getdata_request) - assert_equal(len(self.nodes[0].getpeerinfo()), 1) #node is still connected because of the noban permission - self.log.info("Peer still connected after trying to download old block (noban permission)") + self.log.info("Peer still connected after trying to download old block (download permission)") + peer_info = self.nodes[0].getpeerinfo() + assert_equal(len(peer_info), 1) # node is still connected + assert_equal(peer_info[0]['permissions'], ['download']) + if __name__ == '__main__': MaxUploadTest().main() diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index c155dda664..f42a343042 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -57,29 +57,30 @@ class P2PBlocksOnly(BitcoinTestFramework): self.nodes[0].p2p.wait_for_tx(txid) assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) - self.log.info('Check that txs from whitelisted peers are not rejected and relayed to others') - self.log.info("Restarting node 0 with whitelist permission and blocksonly") + self.log.info('Check that txs from forcerelay peers are not rejected and relayed to others') + self.log.info("Restarting node 0 with forcerelay permission and blocksonly") self.restart_node(0, ["-persistmempool=0", "-whitelist=127.0.0.1", "-whitelistforcerelay", "-blocksonly"]) - assert_equal(self.nodes[0].getrawmempool(),[]) + assert_equal(self.nodes[0].getrawmempool(), []) first_peer = self.nodes[0].add_p2p_connection(P2PInterface()) second_peer = self.nodes[0].add_p2p_connection(P2PInterface()) peer_1_info = self.nodes[0].getpeerinfo()[0] assert_equal(peer_1_info['whitelisted'], True) - assert_equal(peer_1_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool']) + assert_equal(peer_1_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool', 'download']) peer_2_info = self.nodes[0].getpeerinfo()[1] assert_equal(peer_2_info['whitelisted'], True) - assert_equal(peer_2_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool']) + assert_equal(peer_2_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool', 'download']) assert_equal(self.nodes[0].testmempoolaccept([sigtx])[0]['allowed'], True) txid = self.nodes[0].testmempoolaccept([sigtx])[0]['txid'] - self.log.info('Check that the tx from whitelisted first_peer is relayed to others (ie.second_peer)') + self.log.info('Check that the tx from forcerelay first_peer is relayed to others (ie.second_peer)') with self.nodes[0].assert_debug_log(["received getdata"]): first_peer.send_message(msg_tx(FromHex(CTransaction(), sigtx))) - self.log.info('Check that the whitelisted peer is still connected after sending the transaction') + self.log.info('Check that the forcerelay peer is still connected after sending the transaction') assert_equal(first_peer.is_connected, True) second_peer.wait_for_tx(txid) assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) - self.log.info("Whitelisted peer's transaction is accepted and relayed") + self.log.info("Forcerelay peer's transaction is accepted and relayed") + if __name__ == '__main__': P2PBlocksOnly().main() diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 3b3dbd08f2..fe6e236fc4 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -26,7 +26,7 @@ from test_framework.util import ( wait_until, ) -banscore = 10 +DISCOURAGEMENT_THRESHOLD = 100 class CLazyNode(P2PInterface): @@ -65,12 +65,13 @@ class CLazyNode(P2PInterface): # Node that never sends a version. We'll use this to send a bunch of messages # anyway, and eventually get disconnected. -class CNodeNoVersionBan(CLazyNode): - # send a bunch of veracks without sending a message. This should get us disconnected. - # NOTE: implementation-specific check here. Remove if bitcoind ban behavior changes +class CNodeNoVersionMisbehavior(CLazyNode): + # Send enough veracks without a message to reach the peer discouragement + # threshold. This should get us disconnected. NOTE: implementation-specific + # test; update if our discouragement policy for peer misbehavior changes. def on_open(self): super().on_open() - for i in range(banscore): + for _ in range(DISCOURAGEMENT_THRESHOLD): self.send_message(msg_verack()) # Node that never sends a version. This one just sits idle and hopes to receive @@ -106,10 +107,10 @@ class P2PVersionStore(P2PInterface): class P2PLeakTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [['-banscore=' + str(banscore)]] def run_test(self): - no_version_bannode = self.nodes[0].add_p2p_connection(CNodeNoVersionBan(), send_version=False, wait_for_verack=False) + no_version_disconnect_node = self.nodes[0].add_p2p_connection( + CNodeNoVersionMisbehavior(), send_version=False, wait_for_verack=False) no_version_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVersionIdle(), send_version=False, wait_for_verack=False) no_verack_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVerackIdle(), wait_for_verack=False) @@ -117,7 +118,7 @@ class P2PLeakTest(BitcoinTestFramework): # verack, since we never sent one no_verack_idlenode.wait_for_verack() - wait_until(lambda: no_version_bannode.ever_connected, timeout=10, lock=mininode_lock) + wait_until(lambda: no_version_disconnect_node.ever_connected, timeout=10, lock=mininode_lock) wait_until(lambda: no_version_idlenode.ever_connected, timeout=10, lock=mininode_lock) wait_until(lambda: no_verack_idlenode.version_received, timeout=10, lock=mininode_lock) @@ -127,13 +128,13 @@ class P2PLeakTest(BitcoinTestFramework): #Give the node enough time to possibly leak out a message time.sleep(5) - #This node should have been banned - assert not no_version_bannode.is_connected + # Expect this node to be disconnected for misbehavior + assert not no_version_disconnect_node.is_connected self.nodes[0].disconnect_p2ps() # Make sure no unexpected messages came in - assert no_version_bannode.unexpected_msg == False + assert no_version_disconnect_node.unexpected_msg == False assert no_version_idlenode.unexpected_msg == False assert no_verack_idlenode.unexpected_msg == False diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index bea202855d..32a795e345 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -39,7 +39,8 @@ class P2PPermissionsTests(BitcoinTestFramework): self.checkpermission( # default permissions (no specific permissions) ["-whitelist=127.0.0.1"], - ["relay", "noban", "mempool"], + # Make sure the default values in the command line documentation match the ones here + ["relay", "noban", "mempool", "download"], True) self.checkpermission( @@ -51,7 +52,7 @@ class P2PPermissionsTests(BitcoinTestFramework): self.checkpermission( # relay permission removed (no specific permissions) ["-whitelist=127.0.0.1", "-whitelistrelay=0"], - ["noban", "mempool"], + ["noban", "mempool", "download"], True) self.checkpermission( @@ -59,7 +60,7 @@ class P2PPermissionsTests(BitcoinTestFramework): # Legacy parameter interaction which set whitelistrelay to true # if whitelistforcerelay is true ["-whitelist=127.0.0.1", "-whitelistforcerelay"], - ["forcerelay", "relay", "noban", "mempool"], + ["forcerelay", "relay", "noban", "mempool", "download"], True) # Let's make sure permissions are merged correctly @@ -70,32 +71,32 @@ class P2PPermissionsTests(BitcoinTestFramework): self.checkpermission( ["-whitelist=noban@127.0.0.1"], # Check parameter interaction forcerelay should activate relay - ["noban", "bloomfilter", "forcerelay", "relay"], + ["noban", "bloomfilter", "forcerelay", "relay", "download"], False) self.replaceinconfig(1, "whitebind=bloomfilter,forcerelay@" + ip_port, "bind=127.0.0.1") self.checkpermission( # legacy whitelistrelay should be ignored ["-whitelist=noban,mempool@127.0.0.1", "-whitelistrelay"], - ["noban", "mempool"], + ["noban", "mempool", "download"], False) self.checkpermission( # legacy whitelistforcerelay should be ignored ["-whitelist=noban,mempool@127.0.0.1", "-whitelistforcerelay"], - ["noban", "mempool"], + ["noban", "mempool", "download"], False) self.checkpermission( # missing mempool permission to be considered legacy whitelisted ["-whitelist=noban@127.0.0.1"], - ["noban"], + ["noban", "download"], False) self.checkpermission( # all permission added ["-whitelist=all@127.0.0.1"], - ["forcerelay", "noban", "mempool", "bloomfilter", "relay"], + ["forcerelay", "noban", "mempool", "bloomfilter", "relay", "download"], False) self.stop_node(1) @@ -107,9 +108,9 @@ class P2PPermissionsTests(BitcoinTestFramework): block_op_true = self.nodes[0].getblock(self.nodes[0].generatetoaddress(100, ADDRESS_BCRT1_P2WSH_OP_TRUE)[0]) self.sync_all() - self.log.debug("Create a connection from a whitelisted wallet that rebroadcasts raw txs") + self.log.debug("Create a connection from a forcerelay peer that rebroadcasts raw txs") # A python mininode is needed to send the raw transaction directly. If a full node was used, it could only - # rebroadcast via the inv-getdata mechanism. However, even for whitelisted connections, a full node would + # rebroadcast via the inv-getdata mechanism. However, even for forcerelay connections, a full node would # currently not request a txid that is already in the mempool. self.restart_node(1, extra_args=["-whitelist=forcerelay@127.0.0.1"]) p2p_rebroadcast_wallet = self.nodes[1].add_p2p_connection(P2PDataStore()) @@ -134,7 +135,7 @@ class P2PPermissionsTests(BitcoinTestFramework): self.log.debug("Check that node[1] will send the tx to node[0] even though it is already in the mempool") connect_nodes(self.nodes[1], 0) - with self.nodes[1].assert_debug_log(["Force relaying tx {} from whitelisted peer=0".format(txid)]): + with self.nodes[1].assert_debug_log(["Force relaying tx {} from peer=0".format(txid)]): p2p_rebroadcast_wallet.send_txs_and_test([tx], self.nodes[1]) wait_until(lambda: txid in self.nodes[0].getrawmempool()) @@ -145,7 +146,7 @@ class P2PPermissionsTests(BitcoinTestFramework): [tx], self.nodes[1], success=False, - reject_reason='Not relaying non-mempool transaction {} from whitelisted peer=0'.format(txid), + reject_reason='Not relaying non-mempool transaction {} from forcerelay peer=0'.format(txid), ) def checkpermission(self, args, expectedPermissions, whitelisted): diff --git a/test/functional/p2p_ping.py b/test/functional/p2p_ping.py new file mode 100755 index 0000000000..e00af88cc4 --- /dev/null +++ b/test/functional/p2p_ping.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test ping message +""" + +import time + +from test_framework.messages import ( + msg_pong, +) +from test_framework.mininode import ( + P2PInterface, + wait_until, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +PING_INTERVAL = 2 * 60 + + +class msg_pong_corrupt(msg_pong): + def serialize(self): + return b"" + + +class NodePongAdd1(P2PInterface): + def on_ping(self, message): + self.send_message(msg_pong(message.nonce + 1)) + + +class NodeNoPong(P2PInterface): + def on_ping(self, message): + pass + + +class PingPongTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [['-peertimeout=3']] + + def check_peer_info(self, *, pingtime, minping, pingwait): + stats = self.nodes[0].getpeerinfo()[0] + assert_equal(stats.pop('pingtime', None), pingtime) + assert_equal(stats.pop('minping', None), minping) + assert_equal(stats.pop('pingwait', None), pingwait) + + def mock_forward(self, delta): + self.mock_time += delta + self.nodes[0].setmocktime(self.mock_time) + + def run_test(self): + self.mock_time = int(time.time()) + self.mock_forward(0) + + self.log.info('Check that ping is sent after connection is established') + no_pong_node = self.nodes[0].add_p2p_connection(NodeNoPong()) + self.mock_forward(3) + assert no_pong_node.last_message.pop('ping').nonce != 0 + self.check_peer_info(pingtime=None, minping=None, pingwait=3) + + self.log.info('Reply without nonce cancels ping') + with self.nodes[0].assert_debug_log(['pong peer=0: Short payload']): + no_pong_node.send_and_ping(msg_pong_corrupt()) + self.check_peer_info(pingtime=None, minping=None, pingwait=None) + + self.log.info('Reply without ping') + with self.nodes[0].assert_debug_log([ + 'pong peer=0: Unsolicited pong without ping, 0 expected, 0 received, 8 bytes', + ]): + no_pong_node.send_and_ping(msg_pong()) + self.check_peer_info(pingtime=None, minping=None, pingwait=None) + + self.log.info('Reply with wrong nonce does not cancel ping') + assert 'ping' not in no_pong_node.last_message + with self.nodes[0].assert_debug_log(['pong peer=0: Nonce mismatch']): + # mock time PING_INTERVAL ahead to trigger node into sending a ping + self.mock_forward(PING_INTERVAL + 1) + wait_until(lambda: 'ping' in no_pong_node.last_message) + self.mock_forward(9) + # Send the wrong pong + no_pong_node.send_and_ping(msg_pong(no_pong_node.last_message.pop('ping').nonce - 1)) + self.check_peer_info(pingtime=None, minping=None, pingwait=9) + + self.log.info('Reply with zero nonce does cancel ping') + with self.nodes[0].assert_debug_log(['pong peer=0: Nonce zero']): + no_pong_node.send_and_ping(msg_pong(0)) + self.check_peer_info(pingtime=None, minping=None, pingwait=None) + + self.log.info('Check that ping is properly reported on RPC') + assert 'ping' not in no_pong_node.last_message + # mock time PING_INTERVAL ahead to trigger node into sending a ping + self.mock_forward(PING_INTERVAL + 1) + wait_until(lambda: 'ping' in no_pong_node.last_message) + ping_delay = 29 + self.mock_forward(ping_delay) + wait_until(lambda: 'ping' in no_pong_node.last_message) + no_pong_node.send_and_ping(msg_pong(no_pong_node.last_message.pop('ping').nonce)) + self.check_peer_info(pingtime=ping_delay, minping=ping_delay, pingwait=None) + + self.log.info('Check that minping is decreased after a fast roundtrip') + # mock time PING_INTERVAL ahead to trigger node into sending a ping + self.mock_forward(PING_INTERVAL + 1) + wait_until(lambda: 'ping' in no_pong_node.last_message) + ping_delay = 9 + self.mock_forward(ping_delay) + wait_until(lambda: 'ping' in no_pong_node.last_message) + no_pong_node.send_and_ping(msg_pong(no_pong_node.last_message.pop('ping').nonce)) + self.check_peer_info(pingtime=ping_delay, minping=ping_delay, pingwait=None) + + self.log.info('Check that peer is disconnected after ping timeout') + assert 'ping' not in no_pong_node.last_message + self.nodes[0].ping() + wait_until(lambda: 'ping' in no_pong_node.last_message) + with self.nodes[0].assert_debug_log(['ping timeout: 1201.000000s']): + self.mock_forward(20 * 60 + 1) + time.sleep(4) # peertimeout + 1 + + +if __name__ == '__main__': + PingPongTest().main() diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index c323168848..71b0b0f63a 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of unrequested blocks. -Setup: two nodes, node0+node1, not connected to each other. Node1 will have +Setup: two nodes, node0 + node1, not connected to each other. Node1 will have nMinimumChainWork set to 0x10, so it won't process low-work unrequested blocks. We have one P2PInterface connection to node0 called test_node, and one to node1 @@ -71,18 +71,10 @@ class AcceptBlockTest(BitcoinTestFramework): self.extra_args = [[], ["-minimumchainwork=0x10"]] def setup_network(self): - # Node0 will be used to test behavior of processing unrequested blocks - # from peers which are not whitelisted, while Node1 will be used for - # the whitelisted case. - # Node2 will be used for non-whitelisted peers to test the interaction - # with nMinimumChainWork. self.setup_nodes() def run_test(self): - # Setup the p2p connections - # test_node connects to node0 (not whitelisted) test_node = self.nodes[0].add_p2p_connection(P2PInterface()) - # min_work_node connects to node1 (whitelisted) min_work_node = self.nodes[1].add_p2p_connection(P2PInterface()) # 1. Have nodes mine a block (leave IBD) @@ -226,7 +218,7 @@ class AcceptBlockTest(BitcoinTestFramework): self.nodes[0].getblock(all_blocks[286].hash) assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[287].hash) - self.log.info("Successfully reorged to longer chain from non-whitelisted peer") + self.log.info("Successfully reorged to longer chain") # 8. Create a chain which is invalid at a height longer than the # current chain, but which has more blocks on top of that diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 6273c229ae..7c70f30ca3 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -241,6 +241,17 @@ class BlockchainTest(BitcoinTestFramework): del res['disk_size'], res3['disk_size'] assert_equal(res, res3) + self.log.info("Test hash_type option for gettxoutsetinfo()") + # Adding hash_type 'hash_serialized_2', which is the default, should + # not change the result. + res4 = node.gettxoutsetinfo(hash_type='hash_serialized_2') + del res4['disk_size'] + assert_equal(res, res4) + + # hash_type none should not return a UTXO set hash. + res5 = node.gettxoutsetinfo(hash_type='none') + assert 'hash_serialized_2' not in res5 + def _test_getblockheader(self): node = self.nodes[0] diff --git a/test/functional/rpc_getpeerinfo_banscore_deprecation.py b/test/functional/rpc_getpeerinfo_banscore_deprecation.py new file mode 100755 index 0000000000..b830248e1e --- /dev/null +++ b/test/functional/rpc_getpeerinfo_banscore_deprecation.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test deprecation of getpeerinfo RPC banscore field.""" + +from test_framework.test_framework import BitcoinTestFramework + + +class GetpeerinfoBanscoreDeprecationTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.extra_args = [[], ["-deprecatedrpc=banscore"]] + + def run_test(self): + self.log.info("Test getpeerinfo by default no longer returns a banscore field") + assert "banscore" not in self.nodes[0].getpeerinfo()[0].keys() + + self.log.info("Test getpeerinfo returns banscore with -deprecatedrpc=banscore") + assert "banscore" in self.nodes[1].getpeerinfo()[0].keys() + + +if __name__ == "__main__": + GetpeerinfoBanscoreDeprecationTest().main() diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 660953be9b..e5e62fd646 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -38,6 +38,7 @@ class PSBTTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + # TODO: Re-enable this test with segwit v1 def test_utxo_conversion(self): mining_node = self.nodes[2] offline_node = self.nodes[0] @@ -156,6 +157,10 @@ class PSBTTest(BitcoinTestFramework): # spend single key from node 1 rawtx = self.nodes[1].walletcreatefundedpsbt([{"txid":txid,"vout":p2wpkh_pos},{"txid":txid,"vout":p2sh_p2wpkh_pos},{"txid":txid,"vout":p2pkh_pos}], {self.nodes[1].getnewaddress():29.99})['psbt'] walletprocesspsbt_out = self.nodes[1].walletprocesspsbt(rawtx) + # Make sure it has both types of UTXOs + decoded = self.nodes[1].decodepsbt(walletprocesspsbt_out['psbt']) + assert 'non_witness_utxo' in decoded['inputs'][0] + assert 'witness_utxo' in decoded['inputs'][0] assert_equal(walletprocesspsbt_out['complete'], True) self.nodes[1].sendrawtransaction(self.nodes[1].finalizepsbt(walletprocesspsbt_out['psbt'])['hex']) @@ -352,7 +357,8 @@ class PSBTTest(BitcoinTestFramework): for i, signer in enumerate(signers): self.nodes[2].unloadwallet("wallet{}".format(i)) - self.test_utxo_conversion() + # TODO: Re-enable this for segwit v1 + # self.test_utxo_conversion() # Test that psbts with p2pkh outputs are created properly p2pkh = self.nodes[0].getnewaddress(address_type='legacy') diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 05308931e3..81eb881234 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -115,6 +115,8 @@ class AuthServiceProxy(): except OSError as e: retry = ( '[WinError 10053] An established connection was aborted by the software in your host machine' in str(e)) + # Workaround for a bug on macOS. See https://bugs.python.org/issue33450 + retry = retry or ('[Errno 41] Protocol wrong type for socket' in str(e)) if retry: self.__conn.close() self.__conn.request(method, path, postdata, headers) diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index eb1244035f..12302cdbc3 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -207,17 +207,19 @@ class CAddress: self.ip = "0.0.0.0" self.port = 0 - def deserialize(self, f, with_time=True): + def deserialize(self, f, *, with_time=True): if with_time: + # VERSION messages serialize CAddress objects without time self.time = struct.unpack("<i", f.read(4))[0] self.nServices = struct.unpack("<Q", f.read(8))[0] self.pchReserved = f.read(12) self.ip = socket.inet_ntoa(f.read(4)) self.port = struct.unpack(">H", f.read(2))[0] - def serialize(self, with_time=True): + def serialize(self, *, with_time=True): r = b"" if with_time: + # VERSION messages serialize CAddress objects without time r += struct.pack("<i", self.time) r += struct.pack("<Q", self.nServices) r += self.pchReserved @@ -973,10 +975,10 @@ class msg_version: self.nServices = struct.unpack("<Q", f.read(8))[0] self.nTime = struct.unpack("<q", f.read(8))[0] self.addrTo = CAddress() - self.addrTo.deserialize(f, False) + self.addrTo.deserialize(f, with_time=False) self.addrFrom = CAddress() - self.addrFrom.deserialize(f, False) + self.addrFrom.deserialize(f, with_time=False) self.nNonce = struct.unpack("<Q", f.read(8))[0] self.strSubVer = deser_string(f) @@ -996,8 +998,8 @@ class msg_version: r += struct.pack("<i", self.nVersion) r += struct.pack("<Q", self.nServices) r += struct.pack("<q", self.nTime) - r += self.addrTo.serialize(False) - r += self.addrFrom.serialize(False) + r += self.addrTo.serialize(with_time=False) + r += self.addrFrom.serialize(with_time=False) r += struct.pack("<Q", self.nNonce) r += ser_string(self.strSubVer) r += struct.pack("<i", self.nStartingHeight) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 41f9bde183..867a9909a8 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -236,6 +236,7 @@ BASE_SCRIPTS = [ 'mempool_compatibility.py', 'rpc_deriveaddresses.py', 'rpc_deriveaddresses.py --usecli', + 'p2p_ping.py', 'rpc_scantxoutset.py', 'feature_logging.py', 'p2p_node_network_limited.py', @@ -243,6 +244,7 @@ BASE_SCRIPTS = [ 'feature_blocksdir.py', 'feature_config_args.py', 'rpc_getdescriptorinfo.py', + 'rpc_getpeerinfo_banscore_deprecation.py', 'rpc_help.py', 'feature_help.py', 'feature_shutdown.py', @@ -396,11 +398,12 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= args = args or [] # Warn if bitcoind is already running - # pidof might fail or return an empty string if bitcoind is not running try: - if subprocess.check_output(["pidof", "bitcoind"]) not in [b'']: + # pgrep exits with code zero when one or more matching processes found + if subprocess.run(["pgrep", "-x", "bitcoind"], stdout=subprocess.DEVNULL).returncode == 0: print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0])) - except (OSError, subprocess.SubprocessError): + except OSError: + # pgrep not supported pass # Warn if there is a cache directory diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 524e1593ba..18f0beb598 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -71,8 +71,7 @@ class ToolWalletTest(BitcoinTestFramework): self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create') self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') self.assert_raises_tool_error( - 'Error initializing wallet database environment "{}"!\nError loading wallet.dat. Is wallet being used by other process?' - .format(os.path.join(self.nodes[0].datadir, self.chain, 'wallets')), + 'Error loading wallet.dat. Is wallet being used by another process?', '-wallet=wallet.dat', 'info', ) diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 8962362276..81382d94ad 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -119,7 +119,7 @@ class WalletTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "Invalid parameter, expected locked output", self.nodes[2].lockunspent, True, [unspent_0]) self.nodes[2].lockunspent(False, [unspent_0]) assert_raises_rpc_error(-8, "Invalid parameter, output already locked", self.nodes[2].lockunspent, False, [unspent_0]) - assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 20) + assert_raises_rpc_error(-6, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 20) assert_equal([unspent_0], self.nodes[2].listlockunspent()) self.nodes[2].lockunspent(True, [unspent_0]) assert_equal(len(self.nodes[2].listlockunspent()), 0) @@ -363,6 +363,9 @@ class WalletTest(BitcoinTestFramework): assert_equal(tx_obj['amount'], Decimal('-0.0001')) # General checks for errors from incorrect inputs + # This will raise an exception because the amount is negative + assert_raises_rpc_error(-3, "Amount out of range", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "-1") + # This will raise an exception because the amount type is wrong assert_raises_rpc_error(-3, "Invalid amount", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "1f-4") @@ -590,7 +593,7 @@ class WalletTest(BitcoinTestFramework): node0_balance = self.nodes[0].getbalance() # With walletrejectlongchains we will not create the tx and store it in our wallet. - assert_raises_rpc_error(-4, "Transaction has too long of a mempool chain", self.nodes[0].sendtoaddress, sending_addr, node0_balance - Decimal('0.01')) + assert_raises_rpc_error(-6, "Transaction has too long of a mempool chain", self.nodes[0].sendtoaddress, sending_addr, node0_balance - Decimal('0.01')) # Verify nothing new in wallet assert_equal(total_txs, len(self.nodes[0].listtransactions("*", 99999))) diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index ba1e494d9a..6bfb468823 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -202,5 +202,10 @@ class WalletDumpTest(BitcoinTestFramework): result = self.nodes[0].getaddressinfo(multisig_addr) assert result['ismine'] + self.log.info('Check that wallet is flushed') + with self.nodes[0].assert_debug_log(['Flushing wallet.dat'], timeout=20): + self.nodes[0].getnewaddress() + + if __name__ == '__main__': WalletDumpTest().main() diff --git a/test/functional/wallet_encryption.py b/test/functional/wallet_encryption.py index 6cd82ad250..4509c1e0b2 100755 --- a/test/functional/wallet_encryption.py +++ b/test/functional/wallet_encryption.py @@ -13,6 +13,7 @@ from test_framework.util import ( assert_greater_than_or_equal, ) + class WalletEncryptionTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -72,20 +73,25 @@ class WalletEncryptionTest(BitcoinTestFramework): # Test timeout bounds assert_raises_rpc_error(-8, "Timeout cannot be negative.", self.nodes[0].walletpassphrase, passphrase2, -10) - # Check the timeout - # Check a time less than the limit + + self.log.info('Check a timeout less than the limit') MAX_VALUE = 100000000 expected_time = int(time.time()) + MAX_VALUE - 600 self.nodes[0].walletpassphrase(passphrase2, MAX_VALUE - 600) + # give buffer for walletpassphrase, since it iterates over all crypted keys + expected_time_with_buffer = time.time() + MAX_VALUE - 600 actual_time = self.nodes[0].getwalletinfo()['unlocked_until'] assert_greater_than_or_equal(actual_time, expected_time) - assert_greater_than(expected_time + 5, actual_time) # 5 second buffer - # Check a time greater than the limit + assert_greater_than(expected_time_with_buffer, actual_time) + + self.log.info('Check a timeout greater than the limit') expected_time = int(time.time()) + MAX_VALUE - 1 self.nodes[0].walletpassphrase(passphrase2, MAX_VALUE + 1000) + expected_time_with_buffer = time.time() + MAX_VALUE actual_time = self.nodes[0].getwalletinfo()['unlocked_until'] assert_greater_than_or_equal(actual_time, expected_time) - assert_greater_than(expected_time + 5, actual_time) # 5 second buffer + assert_greater_than(expected_time_with_buffer, actual_time) + if __name__ == '__main__': WalletEncryptionTest().main() diff --git a/test/functional/wallet_fallbackfee.py b/test/functional/wallet_fallbackfee.py index 0c67982bbe..dbf853b35c 100755 --- a/test/functional/wallet_fallbackfee.py +++ b/test/functional/wallet_fallbackfee.py @@ -22,7 +22,7 @@ class WalletRBFTest(BitcoinTestFramework): # test sending a tx with disabled fallback fee (must fail) self.restart_node(0, extra_args=["-fallbackfee=0"]) - assert_raises_rpc_error(-4, "Fee estimation failed", lambda: self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1)) + assert_raises_rpc_error(-6, "Fee estimation failed", lambda: self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1)) assert_raises_rpc_error(-4, "Fee estimation failed", lambda: self.nodes[0].fundrawtransaction(self.nodes[0].createrawtransaction([], {self.nodes[0].getnewaddress(): 1}))) assert_raises_rpc_error(-6, "Fee estimation failed", lambda: self.nodes[0].sendmany("", {self.nodes[0].getnewaddress(): 1})) diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index fc5d653a91..2d982edef8 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -146,6 +146,14 @@ class ImportDescriptorsTest(BitcoinTestFramework): ismine=True, solvable=True) + # Check persistence of data and that loading works correctly + w1.unloadwallet() + self.nodes[1].loadwallet('w1') + test_address(w1, + key.p2sh_p2wpkh_addr, + ismine=True, + solvable=True) + # # Test importing of a multisig descriptor key1 = get_generate_key() key2 = get_generate_key() @@ -370,6 +378,10 @@ class ImportDescriptorsTest(BitcoinTestFramework): self.sync_all() assert_equal(wmulti_pub.getbalance(), wmulti_priv.getbalance()) + # Make sure that descriptor wallets containing multiple xpubs in a single descriptor load correctly + wmulti_pub.unloadwallet() + self.nodes[1].loadwallet('wmulti_pub') + self.log.info("Multisig with distributed keys") self.nodes[1].createwallet(wallet_name="wmulti_priv1", descriptors=True) wmulti_priv1 = self.nodes[1].get_wallet_rpc("wmulti_priv1") diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index ff9ff34185..88beef1034 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -7,19 +7,36 @@ Verify that a bitcoind node can load multiple wallet files """ from decimal import Decimal +from threading import Thread import os import shutil import time +from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework from test_framework.test_node import ErrorMatch from test_framework.util import ( assert_equal, assert_raises_rpc_error, + get_rpc_proxy, ) FEATURE_LATEST = 169900 +got_loading_error = False +def test_load_unload(node, name): + global got_loading_error + for i in range(10): + if got_loading_error: + return + try: + node.loadwallet(name) + node.unloadwallet(name) + except JSONRPCException as e: + if e.error['code'] == -4 and 'Wallet already being loading' in e.error['message']: + got_loading_error = True + return + class MultiWalletTest(BitcoinTestFramework): def set_test_params(self): @@ -212,6 +229,18 @@ class MultiWalletTest(BitcoinTestFramework): w2 = node.get_wallet_rpc(wallet_names[1]) w2.getwalletinfo() + self.log.info("Concurrent wallet loading") + threads = [] + for _ in range(3): + n = node.cli if self.options.usecli else get_rpc_proxy(node.url, 1, timeout=600, coveragedir=node.coverage_dir) + t = Thread(target=test_load_unload, args=(n, wallet_names[2], )) + t.start() + threads.append(t) + for t in threads: + t.join() + global got_loading_error + assert_equal(got_loading_error, True) + self.log.info("Load remaining wallets") for wallet_name in wallet_names[2:]: loadwallet_name = self.nodes[0].loadwallet(wallet_name) diff --git a/test/lint/README.md b/test/lint/README.md index 6b95cc3540..d15c061288 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -23,6 +23,12 @@ maintained: * for `src/crypto/ctaes`: https://github.com/bitcoin-core/ctaes.git (branch master) * for `src/crc32c`: https://github.com/google/crc32c.git (branch master) +To do so, add the upstream repository as remote: + +``` +git remote add --fetch secp256k1 https://github.com/bitcoin-core/secp256k1.git +``` + Usage: `git-subtree-check.sh DIR (COMMIT)` `COMMIT` may be omitted, in which case `HEAD` is used. diff --git a/test/lint/git-subtree-check.sh b/test/lint/git-subtree-check.sh index caa7affc63..5a0500df25 100755 --- a/test/lint/git-subtree-check.sh +++ b/test/lint/git-subtree-check.sh @@ -81,7 +81,7 @@ fi # get the tree in the subtree commit referred to if [ "d$(git cat-file -t $rev 2>/dev/null)" != dcommit ]; then - echo "subtree commit $rev unavailable: cannot compare" >&2 + echo "subtree commit $rev unavailable: cannot compare. Did you add and fetch the remote?" >&2 exit fi tree_subtree=$(git show -s --format="%T" $rev) diff --git a/test/lint/lint-assertions.sh b/test/lint/lint-assertions.sh index 1aacc09bcc..d30a8ca231 100755 --- a/test/lint/lint-assertions.sh +++ b/test/lint/lint-assertions.sh @@ -23,7 +23,7 @@ fi # Macro CHECK_NONFATAL(condition) should be used instead of assert for RPC code, where it # is undesirable to crash the whole program. See: src/util/check.h # src/rpc/server.cpp is excluded from this check since it's mostly meta-code. -OUTPUT=$(git grep -nE 'assert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp") +OUTPUT=$(git grep -nE '\<(A|a)ssert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp") if [[ ${OUTPUT} != "" ]]; then echo "CHECK_NONFATAL(condition) should be used instead of assert for RPC code." echo diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index 5404565b94..611bd4a8c4 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -64,7 +64,7 @@ EXPECTED_BOOST_INCLUDES=( boost/preprocessor/cat.hpp boost/preprocessor/stringize.hpp boost/signals2/connection.hpp - boost/signals2/last_value.hpp + boost/signals2/optional_last_value.hpp boost/signals2/signal.hpp boost/test/unit_test.hpp boost/thread/condition_variable.hpp diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index e2bb403c4d..2e5b801849 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -97,6 +97,7 @@ LOCALE_DEPENDENT_FUNCTIONS=( snprintf sprintf sscanf + std::locale::global std::to_string stod stof diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index b8fe75c5c5..cb33dd6232 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -19,7 +19,9 @@ race:CConnman::ThreadMessageHandler race:fHaveGenesis race:ProcessNewBlock race:ThreadImport +race:LoadWallet race:WalletBatch::WriteHDChain +race:BerkeleyBatch race:BerkeleyDatabase race:zmq::* race:bitcoin-qt |