diff options
474 files changed, 11709 insertions, 7416 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 287c8a7f56..7dcf9388b9 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -5,8 +5,6 @@ configuration: Release platform: x64 clone_depth: 5 environment: - APPVEYOR_SAVE_CACHE_ON_ERROR: true - CLCACHE_SERVER: 1 PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' PYTHONUTF8: 1 QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip' @@ -14,59 +12,41 @@ environment: QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' -cache: -- C:\tools\vcpkg\installed -> build_msvc\vcpkg-packages.txt -- C:\Qt5.9.8_x64_static_vs2019 install: # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. # - cmd: pip install zmq # Powershell block below is to install the c++ dependencies via vcpkg. The pseudo code is: -# 1. Check whether the vcpkg install directory exists (note that updating the vcpkg-packages.txt file -# will cause the appveyor cache rules to invalidate the directory) -# 2. If the directory is missing: # a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary, # b. Install the missing packages. - ps: | $env:PACKAGES = Get-Content -Path build_msvc\vcpkg-packages.txt - Write-Host "vcpkg list: $env:PACKAGES" - if(!(Test-Path -Path ($env:VCPKG_INSTALL_PATH))) { - cd c:\tools\vcpkg - $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. - git pull origin master - git checkout $env:VCPKG_COMMIT_ID - .\bootstrap-vcpkg.bat - Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" - .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null - cd "$env:APPVEYOR_BUILD_FOLDER" - } - else { - Write-Host "required vcpkg packages already installed." - } - c:\tools\vcpkg\vcpkg integrate install + Write-Host "vcpkg installing packages: $env:PACKAGES" + cd c:\tools\vcpkg + $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. + git pull origin master > $null + git -c advice.detachedHead=false checkout $env:VCPKG_COMMIT_ID + .\bootstrap-vcpkg.bat > $null + Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" + .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null + Write-Host "vcpkg packages installed successfully." + .\vcpkg integrate install + cd "$env:APPVEYOR_BUILD_FOLDER" before_build: # Powershell block below is to download and extract the Qt static libraries. The pseudo code is: -# 1. If the Qt destination directory exists assume it is correct and do nothing. To -# force a fresh install of the packages delete the job's appveyor cache. -# 2. Otherwise: # a. Download the zip file with the prebuilt Qt static libraries. # b. Check that the downloaded file matches the expected hash. # c. Extract the zip file to the specific destination path expected by the msbuild projects. - ps: | - if(!(Test-Path -Path ($env:QT_LOCAL_PATH))) { - Write-Host "Downloading Qt binaries."; - Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip; - Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH..."; - if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) { - Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH; - Write-Host "Qt binary download matched the expected hash."; - } - else { - Write-Host "ERROR: Qt binary download did not match the expected hash."; - Exit-AppveyorBuild; - } + Write-Host "Downloading Qt binaries."; + Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip; + Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH..."; + if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) { + Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH; + Write-Host "Qt binary download matched the expected hash."; } else { - Write-Host "Qt binaries already present."; + Write-Host "ERROR: Qt binary download did not match the expected hash."; + Exit-AppveyorBuild; } - cmd: python build_msvc\msvc-autogen.py build_script: diff --git a/.cirrus.yml b/.cirrus.yml index 44b49bb397..446d3e35a9 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,3 +1,40 @@ +### Global defaults + +timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out +container: + # https://cirrus-ci.org/faq/#are-there-any-limits + # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel + cpu: 2 + memory: 6G # https://cirrus-ci.org/guide/linux/#linux-containers +env: + PACKAGE_MANAGER_INSTALL : "apt-get update && apt-get install -y" + MAKEJOBS: "-j4" + DANGER_RUN_CI_ON_HOST: "1" # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system + TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache + CCACHE_SIZE: "200M" + CCACHE_DIR: "/tmp/ccache_dir" + +### Global task template + +# https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks +global_task_template: &GLOBAL_TASK_TEMPLATE + ccache_cache: + folder: "/tmp/ccache_dir" + depends_built_cache: + folder: "/tmp/cirrus-ci-build/depends/built" + depends_sdk_cache: + folder: "/tmp/cirrus-ci-build/depends/sdk-sources" + depends_releases_cache: + folder: "/tmp/cirrus-ci-build/releases" + merge_base_script: + - bash -c "$PACKAGE_MANAGER_INSTALL git" + - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH + - git config --global user.email "ci@ci.ci" + - git config --global user.name "ci" + - git merge FETCH_HEAD # Merge base to detect silent merge conflicts + ci_script: + - ./ci/test_run_all.sh + #task: # name: "Windows" # windows_container: @@ -13,22 +50,19 @@ # VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' # install_script: # - choco install python --version=3.7.7 -y + task: - name: "x86_64 Linux [GOAL: install] [bionic] [Using ./ci/ system]" + name: 'x86_64 Linux [GOAL: install] [focal] [depends, sanitizers: thread (TSan), no gui]' + << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:18.04 - cpu: 8 - memory: 8G - timeout_in: 60m + image: ubuntu:focal env: - MAKEJOBS: "-j9" - DANGER_RUN_CI_ON_HOST: "1" - TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache - CCACHE_SIZE: "200M" - CCACHE_DIR: "/tmp/ccache_dir" - ccache_cache: - folder: "/tmp/ccache_dir" - depends_built_cache: - folder: "/tmp/cirrus-ci-build/depends/built" - ci_script: - - ./ci/test_run_all.sh + FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" + +task: + name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]' + << : *GLOBAL_TASK_TEMPLATE + container: + image: ubuntu:focal + env: + FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 35b42424ad..eedeeb4e54 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -4,7 +4,11 @@ General bitcoin questions and/or support requests are best directed to the Bitco For reporting security issues, please read instructions at https://bitcoincore.org/en/contact/. -If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running memtest and observe CPU temperature with a load-test tool such as linpack before creating an issue! --> +If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running memtest and observe CPU temperature with a load-test tool such as linpack before creating an issue! + +Any report, issue or feature request related to the GUI should be reported at +https://github.com/bitcoin-core/gui/issues/ +--> <!-- Describe the issue --> <!--- What behavior did you expect? --> diff --git a/.github/ISSUE_TEMPLATE/gui_issue.md b/.github/ISSUE_TEMPLATE/gui_issue.md new file mode 100644 index 0000000000..37acc81e21 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/gui_issue.md @@ -0,0 +1,11 @@ +--- +name: An issue or feature request related to the GUI +about: Any report, issue or feature request related to the GUI should be reported at https://github.com/bitcoin-core/gui/issues/ +title: Any report, issue or feature request related to the GUI should be reported at https://github.com/bitcoin-core/gui/issues/ +labels: GUI +assignees: '' + +--- + +Any report, issue or feature request related to the GUI should be reported at +https://github.com/bitcoin-core/gui/issues/ diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d2c3b23375..ae92fc78f2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,6 +3,10 @@ Pull requests without a rationale and clear improvement may be closed immediately. + +GUI-related pull requests should be opened against +https://github.com/bitcoin-core/gui +first. See CONTRIBUTING.md --> <!-- diff --git a/.gitignore b/.gitignore index 23b6090265..3e5d284aa3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ src/bitcoin-gui src/bitcoin-node src/bitcoin-tx src/bitcoin-wallet -src/test/fuzz +src/test/fuzz/* !src/test/fuzz/*.* src/test/test_bitcoin src/qt/test/test_bitcoin-qt diff --git a/.travis.yml b/.travis.yml index 03ab9d2613..f9932cfaca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,4 @@ -# The test build matrix (stage: test) is constructed to test a wide range of -# configurations, rather than a single pass/fail. This helps to catch build -# failures and logic errors that present on platforms other than the ones the -# author has tested. -# -# Some builders use the dependency-generator in `./depends`, rather than using -# apt-get to install build dependencies. This guarantees that the tester is -# using the same versions as Gitian, so the build results are nearly identical -# to what would be found in a final release. -# -# In order to avoid rebuilding all dependencies for each build, the binaries -# are cached and re-used when possible. Changes in the dependency-generator -# will trigger cache-invalidation and rebuilds as necessary. -# -# These caches can be manually removed if necessary. This is one of the very +# Travis caches can be manually removed if necessary. This is one of the very # few manual operations that is possible with Travis, and it can be done by a # Bitcoin Core GitHub member via the Travis web interface [0]. # @@ -27,19 +13,16 @@ version: ~> 1.0 -dist: xenial +dist: bionic os: linux language: minimal arch: amd64 cache: - ccache: true directories: - $TRAVIS_BUILD_DIR/depends/built - $TRAVIS_BUILD_DIR/depends/sdk-sources - $TRAVIS_BUILD_DIR/ci/scratch/.ccache - $TRAVIS_BUILD_DIR/releases/$HOST -before_cache: - - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then brew cleanup; fi stages: - lint - test @@ -53,14 +36,16 @@ before_install: install: - set -o errexit; source ./ci/test/04_install.sh before_script: - - set -o errexit; source ./ci/test/05_before_script.sh + # Temporary workaround for https://github.com/bitcoin/bitcoin/issues/16368 + - for i in {1..4}; do echo "$(sleep 500)" ; done & + - set -o errexit; source ./ci/test/05_before_script.sh &> "/dev/null" script: - export CONTINUE=1 - if [ $SECONDS -gt 1200 ]; then export CONTINUE=0; fi # Likely the depends build took very long - - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # Whitelisted repo (90 minutes build time) + - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # continue on repos with extended build time (90 minutes) - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi - - if [ $SECONDS -gt 2000 ]; then export CONTINUE=0; fi # Likely the build took very long; The tests take about 1000s, so we should abort if we have less than 50*60-1000=2000s left - - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # Whitelisted repo (90 minutes build time) + - if [[ $SECONDS -gt 50*60-$EXPECTED_TESTS_DURATION_IN_SECONDS ]]; then export CONTINUE=0; fi + - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # continue on repos with extended build time (90 minutes) - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi after_script: - echo $TRAVIS_COMMIT_RANGE @@ -87,14 +72,12 @@ jobs: FILE_ENV="./ci/test/00_setup_env_arm.sh" QEMU_USER_CMD="" -# s390 build was disabled temporarily because of disk space issues on the Travis VM -# -# - stage: test -# name: 'S390x [GOAL: install] [buster] [unit tests, functional tests]' -# arch: s390x # Can disable QEMU_USER_CMD and run the tests natively without qemu -# env: >- -# FILE_ENV="./ci/test/00_setup_env_s390x.sh" -# QEMU_USER_CMD="" + - stage: test + name: 'S390x [GOAL: install] [buster] [unit tests, functional tests]' + arch: s390x # Can disable QEMU_USER_CMD and run the tests natively without qemu + env: >- + FILE_ENV="./ci/test/00_setup_env_s390x.sh" + QEMU_USER_CMD="" - stage: test name: 'Win64 [GOAL: deploy] [unit tests, no gui, no functional tests]' @@ -112,16 +95,9 @@ jobs: FILE_ENV="./ci/test/00_setup_env_native_qt5.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [focal] [depends, sanitizers: thread (TSan), no gui]' - # Not enough memory on travis machines, so feature_block is excluded for now + name: 'x86_64 Linux [GOAL: install] [focal] [depends, sanitizers: memory (MSan)]' env: >- - TEST_RUNNER_EXTRA="--exclude feature_block" - FILE_ENV="./ci/test/00_setup_env_native_tsan.sh" - - - stage: test - name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]' - env: >- - FILE_ENV="./ci/test/00_setup_env_native_asan.sh" + FILE_ENV="./ci/test/00_setup_env_native_msan.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, sanitizers: fuzzer,address,undefined]' @@ -129,13 +105,17 @@ jobs: FILE_ENV="./ci/test/00_setup_env_native_fuzz.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [bionic] [multiprocess]' - if: type != pull_request OR commit_message =~ /depends:|multiprocess:/ # Skip on non-depends, non-multiprocess PRs + name: 'x86_64 Linux [GOAL: install] [focal] [multiprocess]' env: >- FILE_ENV="./ci/test/00_setup_env_native_multiprocess.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [bionic] [no wallet]' + name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, fuzzers under valgrind]' + env: >- + FILE_ENV="./ci/test/00_setup_env_native_fuzz_with_valgrind.sh" + + - stage: test + name: 'x86_64 Linux [GOAL: install] [xenial] [no wallet]' env: >- FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh" @@ -151,22 +131,12 @@ jobs: # Xcode 11.3.1, macOS 10.14, SDK 10.15 # https://docs.travis-ci.com/user/reference/osx/#macos-version osx_image: xcode11.3 - cache: - directories: - - $TRAVIS_BUILD_DIR/ci/scratch/.ccache - - $TRAVIS_BUILD_DIR/releases/$HOST - - $HOME/Library/Caches/Homebrew - - /usr/local/Homebrew addons: homebrew: packages: - - libtool - berkeley-db4 - - boost - miniupnpc - - qt - qrencode - - python3 - ccache - zeromq env: >- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f34b1e1e15..11a0f4bac7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,27 +6,28 @@ welcome to contribute towards development in the form of peer review, testing and patches. This document explains the practical process and guidelines for contributing. -Firstly in terms of structure, there is no particular concept of "Core +First, in terms of structure, there is no particular concept of "Bitcoin Core developers" in the sense of privileged people. Open source often naturally -revolves around meritocracy where longer term contributors gain more trust from -the developer community. However, some hierarchy is necessary for practical -purposes. As such there are repository "maintainers" who are responsible for -merging pull requests as well as a "lead maintainer" who is responsible for the -release cycle, overall merging, moderation and appointment of maintainers. +revolves around a meritocracy where contributors earn trust from the developer +community over time. Nevertheless, some hierarchy is necessary for practical +purposes. As such, there are repository "maintainers" who are responsible for +merging pull requests, as well as a "lead maintainer" who is responsible for the +release cycle as well as overall merging, moderation and appointment of +maintainers. Getting Started --------------- New contributors are very welcome and needed. -Reviewing and testing is the most effective way you can contribute as a new -contributor, and it also will teach you much more about the code and process -than opening PRs. Please refer to the section [peer review](#peer-review) later -in this document. +Reviewing and testing is highly valued and the most effective way you can contribute +as a new contributor. It also will teach you much more about the code and +process than opening pull requests. Please refer to the [peer review](#peer-review) +section below. Before you start contributing, familiarize yourself with the Bitcoin Core build system and tests. Refer to the documentation in the repository on how to build -Bitcoin Core and how to run the unit and functional tests. +Bitcoin Core and how to run the unit tests, functional tests, and fuzz tests. There are many open issues of varying difficulty waiting to be fixed. If you're looking for somewhere to start contributing, check out the @@ -62,7 +63,7 @@ history logs can be found on [http://www.erisian.com.au/bitcoin-core-dev/](http://www.erisian.com.au/bitcoin-core-dev/) and [http://gnusha.org/bitcoin-core-dev/](http://gnusha.org/bitcoin-core-dev/). -Discussion about code base improvements happens in GitHub issues and on pull +Discussion about codebase improvements happens in GitHub issues and pull requests. The developer @@ -75,7 +76,7 @@ Contributor Workflow -------------------- The codebase is maintained using the "contributor workflow" where everyone -without exception contributes patch proposals using "pull requests". This +without exception contributes patch proposals using "pull requests" (PRs). This facilitates social contribution, easy testing and peer review. To contribute a patch, the workflow is as follows: @@ -84,6 +85,26 @@ To contribute a patch, the workflow is as follows: 1. Create topic branch 1. Commit patches +For GUI-related issues or pull requests, the https://github.com/bitcoin-core/gui repository should be used. +For all other issues and pull requests, the https://github.com/bitcoin/bitcoin node repository should be used. + +The master branch for all monotree repositories is identical. + +As a rule of thumb, everything that only modifies `src/qt` is a GUI-only pull +request. However: + +* For global refactoring or other transversal changes the node repository + should be used. +* For GUI-related build system changes, the node repository should be used + because the change needs review by the build systems reviewers. +* Changes in `src/interfaces` need to go to the node repository because they + might affect other components like the wallet. + +For large GUI changes that include build system and interface changes, it is +recommended to first open a pull request against the GUI repository. When there +is agreement to proceed with the changes, a pull request with the build system +and interfaces changes can be submitted to the node repository. + The project coding conventions in the [developer notes](doc/developer-notes.md) must be followed. @@ -93,6 +114,9 @@ In general, [commits should be atomic](https://en.wikipedia.org/wiki/Atomic_comm and diffs should be easy to read. For this reason, do not mix any formatting fixes or code moves with actual code changes. +Make sure each individual commit is hygienic: that it builds successfully on its +own without warnings, errors, regressions, or test failures. + Commit messages should be verbose by default consisting of a short subject line (50 chars max), a blank line and detailed explanatory text as separate paragraph(s), unless the title alone is self-explanatory (like "Corrected typo @@ -104,7 +128,7 @@ If a particular commit references another issue, please add the reference. For example: `refs #1234` or `fixes #4321`. Using the `fixes` or `closes` keywords will cause the corresponding issue to be closed when the pull request is merged. -Commit messages should never contain any `@` mentions. +Commit messages should never contain any `@` mentions (usernames prefixed with "@"). Please refer to the [Git manual](https://git-scm.com/doc) for more information about Git. @@ -138,10 +162,16 @@ Examples: qt: Add feed bump button log: Fix typo in log message -The body of the pull request should contain enough description about what the -patch does together with any justification/reasoning. You should include -references to any discussions (for example other tickets or mailing list -discussions). +The body of the pull request should contain sufficient description of *what* the +patch does, and even more importantly, *why*, with justification and reasoning. +You should include references to any discussions (for example, other issues or +mailing list discussions). + +The description for a new pull request should not contain any `@` mentions. The +PR description will be included in the commit message when the PR is merged and +any users mentioned in the description will be annoyingly notified each time a +fork of Bitcoin Core copies the merge. Instead, make any username mentions in a +subsequent comment to the PR. ### Translation changes @@ -177,13 +207,13 @@ before it will be merged. The basic squashing workflow is shown below. # Save and quit. git push -f # (force push to GitHub) -Please update the resulting commit message if needed. It should read as a -coherent message. In most cases, this means that you should not just list the -interim commits. +Please update the resulting commit message, if needed. It should read as a +coherent message. In most cases, this means not just listing the interim +commits. -If you have problems with squashing (or other workflows with `git`), you can -alternatively enable "Allow edits from maintainers" in the right GitHub -sidebar and ask for help in the pull request. +If you have problems with squashing or other git workflows, you can enable +"Allow edits from maintainers" in the right-hand sidebar of the GitHub web +interface and ask for help in the pull request. Please refrain from creating several pull requests for the same change. Use the pull request that is already open (or was created earlier) to amend @@ -267,8 +297,8 @@ In general, all pull requests must: - Have a clear use case, fix a demonstrable bug or serve the greater good of the project (for example refactoring for modularisation); - - Be well peer reviewed; - - Have unit tests and functional tests where appropriate; + - Be well peer-reviewed; + - Have unit tests, functional tests, and fuzz tests, where appropriate; - Follow code style guidelines ([C++](doc/developer-notes.md), [functional tests](test/functional/README.md)); - Not break the existing test suite; - Where bugs are fixed, where possible, there should be unit tests @@ -295,7 +325,7 @@ spread out over GitHub, mailing list and IRC discussions). #### Conceptual Review A review can be a conceptual review, where the reviewer leaves a comment - * `Concept (N)ACK`, meaning "I do (not) agree in the general goal of this pull + * `Concept (N)ACK`, meaning "I do (not) agree with the general goal of this pull request", * `Approach (N)ACK`, meaning `Concept ACK`, but "I do (not) agree with the approach of this change". @@ -305,30 +335,28 @@ NACKs without accompanying reasoning may be disregarded. #### Code Review -After conceptual agreement on the change, code review can be provided. It is -starting with `ACK BRANCH_COMMIT`, where `BRANCH_COMMIT` is the top of the -topic branch. The review is followed by a description of how the reviewer did -the review. The following -language is used within pull-request comments: +After conceptual agreement on the change, code review can be provided. A review +begins with `ACK BRANCH_COMMIT`, where `BRANCH_COMMIT` is the top of the PR +branch, followed by a description of how the reviewer did the review. The +following language is used within pull request comments: - - "I have tested the code", involving - change-specific manual testing in addition to running the unit and functional - tests, and in case it is not obvious how the manual testing was done, it should - be described; + - "I have tested the code", involving change-specific manual testing in + addition to running the unit, functional, or fuzz tests, and in case it is + not obvious how the manual testing was done, it should be described; - "I have not tested the code, but I have reviewed it and it looks OK, I agree it can be merged"; - - Nit refers to trivial, often non-blocking issues. + - A "nit" refers to a trivial, often non-blocking issue. Project maintainers reserve the right to weigh the opinions of peer reviewers -using common sense judgement and also may weight based on meritocracy: Those -that have demonstrated a deeper commitment and understanding towards the project -(over time) or have clear domain expertise may naturally have more weight, as -one would expect in all walks of life. +using common sense judgement and may also weigh based on merit. Reviewers that +have demonstrated a deeper commitment and understanding of the project over time +or who have clear domain expertise may naturally have more weight, as one would +expect in all walks of life. -Where a patch set affects consensus critical code, the bar will be set much +Where a patch set affects consensus-critical code, the bar will be much higher in terms of discussion and peer review requirements, keeping in mind that mistakes could be very costly to the wider community. This includes refactoring -of consensus critical code. +of consensus-critical code. Where a patch set proposes to change the Bitcoin consensus, it must have been discussed extensively on the mailing list and IRC, be accompanied by a widely @@ -345,7 +373,7 @@ about: - It may be because of a feature freeze due to an upcoming release. During this time, only bug fixes are taken into consideration. If your pull request is a new feature, - it will not be prioritized until the release is over. Wait for release. + it will not be prioritized until after the release. Wait for the release. - It may be because the changes you are suggesting do not appeal to people. Rather than nits and critique, which require effort and means they care enough to spend time on your contribution, thundering silence is a good sign of widespread (mild) dislike of a given change @@ -355,16 +383,18 @@ about: [developer notes](doc/developer-notes.md), is dangerous or insecure, is messily written, etc. Identify and address any of the issues you find. Then ask e.g. on IRC if someone could give their opinion on the concept itself. - - It may be because your code is too complex for all but a few people. And those people + - It may be because your code is too complex for all but a few people, and those people may not have realized your pull request even exists. A great way to find people who are qualified and care about the code you are touching is the [Git Blame feature](https://help.github.com/articles/tracing-changes-in-a-file/). Simply - find the person touching the code you are touching before you and see if you can find - them and give them a nudge. Don't be incessant about the nudging though. + look up who last modified the code you are changing and see if you can find + them and give them a nudge. Don't be incessant about the nudging, though. - Finally, if all else fails, ask on IRC or elsewhere for someone to give your pull request - a look. If you think you've been waiting an unreasonably long amount of time (month+) for - no particular reason (few lines changed, etc), this is totally fine. Try to return the favor - when someone else is asking for feedback on their code, and universe balances out. + a look. If you think you've been waiting for an unreasonably long time (say, + more than a month) for no particular reason (a few lines changed, etc.), + this is totally fine. Try to return the favor when someone else is asking + for feedback on their code, and the universe balances out. + - Remember that the best thing you can do while waiting is give review to others! Backporting @@ -373,11 +403,11 @@ Backporting Security and bug fixes can be backported from `master` to release branches. If the backport is non-trivial, it may be appropriate to open an -additional PR, to backport the change, only after the original PR +additional PR to backport the change, but only after the original PR has been merged. Otherwise, backports will be done in batches and the maintainers will use the proper `Needs backport (...)` labels -when needed (the original author does not need to worry). +when needed (the original author does not need to worry about it). A backport should contain the following metadata in the commit body: diff --git a/Makefile.am b/Makefile.am index 45dab3930d..75a164f49e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -342,3 +342,14 @@ clean-local: clean-docs rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ fuzz.coverage/ test/tmp/ cache/ $(OSX_APP) rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__ rm -rf osx_volname dist/ dpi36.background.tiff dpi72.background.tiff + +test-security-check: +if TARGET_DARWIN + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO +endif +if TARGET_WINDOWS + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE +endif +if TARGET_LINUX + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF +endif @@ -25,9 +25,14 @@ information or see https://opensource.org/licenses/MIT. Development Process ------------------- -The `master` branch is regularly built (see doc/build-*.md for instructions) and tested, but is not guaranteed to be +The `master` branch is regularly built (see `doc/build-*.md` for instructions) and tested, but it is not guaranteed to be completely stable. [Tags](https://github.com/bitcoin/bitcoin/tags) are created -regularly to indicate new official, stable release versions of Bitcoin Core. +regularly from release branches to indicate new official, stable release versions of Bitcoin Core. + +The https://github.com/bitcoin-core/gui repository is used exclusively for the +development of the GUI. Its master branch is identical in all monotree +repositories. Release branches and tags do not exist, so please do not fork +that repository unless it is for development reasons. The contribution workflow is described in [CONTRIBUTING.md](CONTRIBUTING.md) and useful hints for developers can be found in [doc/developer-notes.md](doc/developer-notes.md). diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 83d054af5f..e171238cbc 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -79,25 +79,12 @@ AC_DEFUN([BITCOIN_QT_INIT],[ AC_SUBST(QT_TRANSLATION_DIR,$qt_translation_path) ]) -dnl Find the appropriate version of Qt libraries and includes. -dnl Inputs: $1: Whether or not pkg-config should be used. yes|no. Default: yes. -dnl Inputs: $2: If $1 is "yes" and --with-gui=auto, which qt version should be -dnl tried first. -dnl Outputs: See _BITCOIN_QT_FIND_LIBS_* +dnl Find Qt libraries and includes. +dnl Outputs: See _BITCOIN_QT_FIND_LIBS dnl Outputs: Sets variables for all qt-related tools. dnl Outputs: bitcoin_enable_qt, bitcoin_enable_qt_dbus, bitcoin_enable_qt_test AC_DEFUN([BITCOIN_QT_CONFIGURE],[ - use_pkgconfig=$1 - - if test "x$use_pkgconfig" = x; then - use_pkgconfig=yes - fi - - if test "x$use_pkgconfig" = xyes; then - BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS_WITH_PKGCONFIG]) - else - BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS_WITHOUT_PKGCONFIG]) - fi + BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS]) dnl This is ugly and complicated. Yuck. Works as follows: dnl For Qt5, we can check a header to find out whether Qt is build @@ -117,8 +104,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ _BITCOIN_QT_FIND_STATIC_PLUGINS AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol if qt plugins are static]) if test "x$TARGET_OS" != xandroid; then - _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal]) - AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) + _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal]) + AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) fi if test "x$TARGET_OS" = xwindows; then _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QWindowsIntegrationPlugin)],[-lqwindows]) @@ -139,7 +126,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ CXXFLAGS=$TEMP_CXXFLAGS ]) - if test "x$use_pkgconfig$qt_bin_path" = xyes; then + if test "x$qt_bin_path" = x; then qt_bin_path="`$PKG_CONFIG --variable=host_bins Qt5Core 2>/dev/null`" fi @@ -213,7 +200,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ dnl enable qt support - AC_MSG_CHECKING(whether to build ]AC_PACKAGE_NAME[ GUI) + AC_MSG_CHECKING([whether to build ]AC_PACKAGE_NAME[ GUI]) BITCOIN_QT_CHECK([ bitcoin_enable_qt=yes bitcoin_enable_qt_test=yes @@ -255,57 +242,15 @@ dnl All macros below are internal and should _not_ be used from the main dnl configure.ac. dnl ---- -dnl Internal. Check included version of Qt against minimum specified in doc/dependencies.md -dnl Requires: INCLUDES must be populated as necessary. -dnl Output: bitcoin_cv_qt5=yes|no -AC_DEFUN([_BITCOIN_QT_CHECK_QT5],[ - AC_CACHE_CHECK(for Qt 5, bitcoin_cv_qt5,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include <QtCore/qconfig.h> - #ifndef QT_VERSION - # include <QtCore/qglobal.h> - #endif - ]], - [[ - #if QT_VERSION < 0x050501 - choke - #endif - ]])], - [bitcoin_cv_qt5=yes], - [bitcoin_cv_qt5=no]) -])]) - -dnl Internal. Check if the included version of Qt is greater than Qt58. -dnl Requires: INCLUDES must be populated as necessary. -dnl Output: bitcoin_cv_qt58=yes|no -AC_DEFUN([_BITCOIN_QT_CHECK_QT58],[ - AC_CACHE_CHECK(for > Qt 5.7, bitcoin_cv_qt58,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include <QtCore/qconfig.h> - #ifndef QT_VERSION - # include <QtCore/qglobal.h> - #endif - ]], - [[ - #if QT_VERSION_MINOR < 8 - choke - #endif - ]])], - [bitcoin_cv_qt58=yes], - [bitcoin_cv_qt58=no]) -])]) - - dnl Internal. Check if the linked version of Qt was built as static libs. dnl Requires: Qt5. dnl Requires: INCLUDES and LIBS must be populated as necessary. dnl Output: bitcoin_cv_static_qt=yes|no -dnl Output: Defines QT_STATICPLUGIN if plugins are static. AC_DEFUN([_BITCOIN_QT_IS_STATIC],[ AC_CACHE_CHECK(for static Qt, bitcoin_cv_static_qt,[ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <QtCore/qconfig.h> - #ifndef QT_VERSION OR QT_VERSION_STR + #ifndef QT_VERSION # include <QtCore/qglobal.h> #endif ]], @@ -317,9 +262,6 @@ AC_DEFUN([_BITCOIN_QT_IS_STATIC],[ [bitcoin_cv_static_qt=yes], [bitcoin_cv_static_qt=no]) ]) - if test "x$bitcoin_cv_static_qt" = xyes; then - AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol for static Qt plugins]) - fi ]) dnl Internal. Check if the link-requirements for static plugins are met. @@ -353,70 +295,32 @@ AC_DEFUN([_BITCOIN_QT_FIND_STATIC_PLUGINS],[ if test -d "$qt_plugin_path/platforms/android"; then QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms/android -lqtfreetype -lEGL" fi - if test "x$use_pkgconfig" = xyes; then - : dnl - m4_ifdef([PKG_CHECK_MODULES],[ - if test x$bitcoin_cv_qt58 = xno; then - PKG_CHECK_MODULES([QTPLATFORM], [Qt5PlatformSupport], [QT_LIBS="$QTPLATFORM_LIBS $QT_LIBS"]) - else - PKG_CHECK_MODULES([QTFONTDATABASE], [Qt5FontDatabaseSupport], [QT_LIBS="-lQt5FontDatabaseSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTEVENTDISPATCHER], [Qt5EventDispatcherSupport], [QT_LIBS="-lQt5EventDispatcherSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTTHEME], [Qt5ThemeSupport], [QT_LIBS="-lQt5ThemeSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTDEVICEDISCOVERY], [Qt5DeviceDiscoverySupport], [QT_LIBS="-lQt5DeviceDiscoverySupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTACCESSIBILITY], [Qt5AccessibilitySupport], [QT_LIBS="-lQt5AccessibilitySupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTFB], [Qt5FbSupport], [QT_LIBS="-lQt5FbSupport $QT_LIBS"]) - fi - if test "x$TARGET_OS" = xlinux; then - PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) - elif test "x$TARGET_OS" = xdarwin; then - PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport], [QT_LIBS="-lQt5GraphicsSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTCGL], [Qt5CglSupport], [QT_LIBS="-lQt5CglSupport $QT_LIBS"]) - fi - ]) - else - if test "x$TARGET_OS" = xwindows; then - AC_CACHE_CHECK(for Qt >= 5.6, bitcoin_cv_need_platformsupport,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include <QtCore/qconfig.h> - #ifndef QT_VERSION - # include <QtCore/qglobal.h> - #endif - ]], - [[ - #if QT_VERSION < 0x050600 || QT_VERSION_MINOR < 6 - choke - #endif - ]])], - [bitcoin_cv_need_platformsupport=yes], - [bitcoin_cv_need_platformsupport=no]) - ]) - if test "x$bitcoin_cv_need_platformsupport" = xyes; then - if test x$bitcoin_cv_qt58 = xno; then - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}PlatformSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXPlatformSupport not found))) - else - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}FontDatabaseSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXFontDatabaseSupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}EventDispatcherSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXEventDispatcherSupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}ThemeSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXThemeSupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}FbSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXFbSupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}DeviceDiscoverySupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXDeviceDiscoverySupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}AccessibilitySupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXAccessibilitySupport not found))) - QT_LIBS="$QT_LIBS -lversion -ldwmapi -luxtheme" - fi - fi - fi - fi - fi + m4_ifdef([PKG_CHECK_MODULES],[ + if test x$bitcoin_cv_qt58 = xno; then + PKG_CHECK_MODULES([QTPLATFORM], [Qt5PlatformSupport], [QT_LIBS="$QTPLATFORM_LIBS $QT_LIBS"]) + else + PKG_CHECK_MODULES([QTFONTDATABASE], [Qt5FontDatabaseSupport], [QT_LIBS="-lQt5FontDatabaseSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTEVENTDISPATCHER], [Qt5EventDispatcherSupport], [QT_LIBS="-lQt5EventDispatcherSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTTHEME], [Qt5ThemeSupport], [QT_LIBS="-lQt5ThemeSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTDEVICEDISCOVERY], [Qt5DeviceDiscoverySupport], [QT_LIBS="-lQt5DeviceDiscoverySupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTACCESSIBILITY], [Qt5AccessibilitySupport], [QT_LIBS="-lQt5AccessibilitySupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTFB], [Qt5FbSupport], [QT_LIBS="-lQt5FbSupport $QT_LIBS"]) + fi + if test "x$TARGET_OS" = xlinux; then + PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) + elif test "x$TARGET_OS" = xdarwin; then + PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport], [QT_LIBS="-lQt5GraphicsSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTCGL], [Qt5CglSupport], [QT_LIBS="-lQt5CglSupport $QT_LIBS"]) + fi + ]) + fi ]) dnl Internal. Find Qt libraries using pkg-config. -dnl Inputs: bitcoin_qt_want_version (from --with-gui=). The version to check -dnl first. -dnl Inputs: $1: If bitcoin_qt_want_version is "auto", check for this version -dnl first. dnl Outputs: All necessary QT_* variables are set. dnl Outputs: have_qt_test and have_qt_dbus are set (if applicable) to yes|no. -AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITH_PKGCONFIG],[ +AC_DEFUN([_BITCOIN_QT_FIND_LIBS],[ m4_ifdef([PKG_CHECK_MODULES],[ QT_LIB_PREFIX=Qt5 qt5_modules="Qt5Core Qt5Gui Qt5Network Qt5Widgets" @@ -437,83 +341,3 @@ AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITH_PKGCONFIG],[ ]) true; dnl ]) - -dnl Internal. Find Qt libraries without using pkg-config. Version is deduced -dnl from the discovered headers. -dnl Inputs: bitcoin_qt_want_version (from --with-gui=). The version to use. -dnl If "auto", the version will be discovered by _BITCOIN_QT_CHECK_QT5. -dnl Outputs: All necessary QT_* variables are set. -dnl Outputs: have_qt_test and have_qt_dbus are set (if applicable) to yes|no. -AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITHOUT_PKGCONFIG],[ - TEMP_CPPFLAGS="$CPPFLAGS" - TEMP_CXXFLAGS="$CXXFLAGS" - CXXFLAGS="$PIC_FLAGS $CXXFLAGS" - TEMP_LIBS="$LIBS" - BITCOIN_QT_CHECK([ - if test "x$qt_include_path" != x; then - QT_INCLUDES="-I$qt_include_path -I$qt_include_path/QtCore -I$qt_include_path/QtGui -I$qt_include_path/QtWidgets -I$qt_include_path/QtNetwork -I$qt_include_path/QtTest -I$qt_include_path/QtDBus" - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" - fi - ]) - - BITCOIN_QT_CHECK([AC_CHECK_HEADER([QtPlugin],,BITCOIN_QT_FAIL(QtCore headers missing))]) - BITCOIN_QT_CHECK([AC_CHECK_HEADER([QApplication],, BITCOIN_QT_FAIL(QtGui headers missing))]) - BITCOIN_QT_CHECK([AC_CHECK_HEADER([QLocalSocket],, BITCOIN_QT_FAIL(QtNetwork headers missing))]) - - BITCOIN_QT_CHECK([ - if test "x$bitcoin_qt_want_version" = xauto; then - _BITCOIN_QT_CHECK_QT5 - _BITCOIN_QT_CHECK_QT58 - fi - QT_LIB_PREFIX=Qt5 - ]) - - BITCOIN_QT_CHECK([ - LIBS= - if test "x$qt_lib_path" != x; then - LIBS="$LIBS -L$qt_lib_path" - fi - - if test "x$TARGET_OS" = xwindows; then - AC_CHECK_LIB([imm32], [main],, BITCOIN_QT_FAIL(libimm32 not found)) - fi - ]) - - BITCOIN_QT_CHECK(AC_CHECK_LIB([z] ,[main],,AC_MSG_WARN([zlib not found. Assuming qt has it built-in]))) - if test x$bitcoin_cv_qt58 = xno; then - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([png_error] ,[qtpng png],,AC_MSG_WARN([libpng not found. Assuming qt has it built-in]))) - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([pcre16_exec], [qtpcre pcre16],,AC_MSG_WARN([libpcre16 not found. Assuming qt has it built-in]))) - else - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([png_error] ,[qtlibpng png],,AC_MSG_WARN([libpng not found. Assuming qt has it built-in]))) - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([pcre2_match_16], [qtpcre2 libqtpcre2],,AC_MSG_WARN([libqtpcre2 not found. Assuming qt has it built-in]))) - fi - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([hb_ot_tags_from_script] ,[qtharfbuzzng qtharfbuzz harfbuzz],,AC_MSG_WARN([libharfbuzz not found. Assuming qt has it built-in or support is disabled]))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Core] ,[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Core not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Gui] ,[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Gui not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Network],[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Network not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Widgets],[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Widgets not found))) - QT_LIBS="$LIBS" - LIBS="$TEMP_LIBS" - - BITCOIN_QT_CHECK([ - LIBS= - if test "x$qt_lib_path" != x; then - LIBS="-L$qt_lib_path" - fi - AC_CHECK_LIB([${QT_LIB_PREFIX}Test], [main],, have_qt_test=no) - AC_CHECK_HEADER([QTest],, have_qt_test=no) - QT_TEST_LIBS="$LIBS" - if test "x$use_dbus" != xno; then - LIBS= - if test "x$qt_lib_path" != x; then - LIBS="-L$qt_lib_path" - fi - AC_CHECK_LIB([${QT_LIB_PREFIX}DBus], [main],, have_qt_dbus=no) - AC_CHECK_HEADER([QtDBus],, have_qt_dbus=no) - QT_DBUS_LIBS="$LIBS" - fi - ]) - CPPFLAGS="$TEMP_CPPFLAGS" - CXXFLAGS="$TEMP_CXXFLAGS" - LIBS="$TEMP_LIBS" -]) diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index 35ba8425b3..fbbe1a2156 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -137,18 +137,6 @@ don't. */ #define HAVE_DECL_STRNLEN 1 -/* Define to 1 if you have the declaration of `__builtin_clz', and to 0 if you - don't. */ -//#define HAVE_DECL___BUILTIN_CLZ 1 - -/* Define to 1 if you have the declaration of `__builtin_clzl', and to 0 if - you don't. */ -//#define HAVE_DECL___BUILTIN_CLZL 1 - -/* Define to 1 if you have the declaration of `__builtin_clzll', and to 0 if - you don't. */ -//#define HAVE_DECL___BUILTIN_CLZLL 1 - /* Define to 1 if you have the <dlfcn.h> header file. */ /* #undef HAVE_DLFCN_H */ diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index c09997d39d..4fd516fff5 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -107,10 +107,10 @@ <ClCompile> <WarningLevel>Level3</WarningLevel> <PrecompiledHeader>NotUsing</PrecompiledHeader> - <AdditionalOptions>/utf-8 %(AdditionalOptions)</AdditionalOptions> + <AdditionalOptions>/utf-8 /std:c++17 %(AdditionalOptions)</AdditionalOptions> <DisableSpecificWarnings>4018;4221;4244;4267;4334;4715;4805;4834</DisableSpecificWarnings> <TreatWarningAsError>true</TreatWarningAsError> - <PreprocessorDefinitions>ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;%(PreprocessorDefinitions)</PreprocessorDefinitions> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> diff --git a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj index 992f64ec2e..6a3c9f1dc1 100644 --- a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj +++ b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj @@ -35,6 +35,7 @@ <ClCompile Include="..\..\src\qt\paymentserver.cpp" /> <ClCompile Include="..\..\src\qt\peertablemodel.cpp" /> <ClCompile Include="..\..\src\qt\platformstyle.cpp" /> + <ClCompile Include="..\..\src\qt\psbtoperationsdialog.cpp" /> <ClCompile Include="..\..\src\qt\qrimagewidget.cpp" /> <ClCompile Include="..\..\src\qt\qvalidatedlineedit.cpp" /> <ClCompile Include="..\..\src\qt\qvaluecombobox.cpp" /> @@ -87,6 +88,7 @@ <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_paymentserver.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_peertablemodel.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_platformstyle.cpp" /> + <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_psbtoperationsdialog.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_qrimagewidget.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_qvalidatedlineedit.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_qvaluecombobox.cpp" /> diff --git a/build_msvc/libsecp256k1_config.h b/build_msvc/libsecp256k1_config.h index 5187c946a0..5978b9a0d9 100644 --- a/build_msvc/libsecp256k1_config.h +++ b/build_msvc/libsecp256k1_config.h @@ -26,4 +26,7 @@ #define USE_FIELD_10X26 1 #define USE_SCALAR_8X32 1 +#define ECMULT_GEN_PREC_BITS 4 +#define ECMULT_WINDOW_SIZE 15 + #endif /* BITCOIN_LIBSECP256K1_CONFIG_H */ diff --git a/ci/README.md b/ci/README.md index d2ea255b4b..3c5f04c39e 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,12 +1,8 @@ -## ci scripts +## CI Scripts This directory contains scripts for each build step in each build stage. -Currently three stages `lint`, `extended_lint` and `test` are defined. Each stage has its own lifecycle, similar to the -[Travis CI lifecycle](https://docs.travis-ci.com/user/job-lifecycle#the-job-lifecycle). Every script in here is named -and numbered according to which stage and lifecycle step it belongs to. - -### Running a stage locally +### Running a Stage Locally Be aware that the tests will be built and run in-place, so please run at your own risk. If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. @@ -36,3 +32,34 @@ To run the test stage with a specific configuration, ``` FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh ``` + +### Configurations + +The test files (`FILE_ENV`) are constructed to test a wide range of +configurations, rather than a single pass/fail. This helps to catch build +failures and logic errors that present on platforms other than the ones the +author has tested. + +Some builders use the dependency-generator in `./depends`, rather than using +the system package manager to install build dependencies. This guarantees that +the tester is using the same versions as the release builds, which also use +`./depends`. + +If no `FILE_ENV` has been specified or values are left out, `00_setup_env.sh` +is used as the default configuration with fallback values. + +It is also possible to force a specific configuration without modifying the +file. For example, + +``` +MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh +``` + +The files starting with `0n` (`n` greater than 0) are the scripts that are run +in order. + +### Cache + +In order to avoid rebuilding all dependencies for each build, the binaries are +cached and re-used when possible. Changes in the dependency-generator will +trigger cache-invalidation and rebuilds as necessary. diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 26b576c1ae..fae424051d 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -6,11 +6,15 @@ export LC_ALL=C -travis_retry pip3 install codespell==1.15.0 -travis_retry pip3 install flake8==3.7.8 +travis_retry sudo apt update && sudo apt install -y clang-format-9 +sudo update-alternatives --install /usr/bin/clang-format clang-format $(which clang-format-9 ) 100 +sudo update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(which clang-format-diff-9) 100 + +travis_retry pip3 install codespell==1.17.1 +travis_retry pip3 install flake8==3.8.3 travis_retry pip3 install yq -travis_retry pip3 install mypy==0.700 +travis_retry pip3 install mypy==0.781 -SHELLCHECK_VERSION=v0.6.0 -curl -s "https://storage.googleapis.com/shellcheck/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ +SHELLCHECK_VERSION=v0.7.1 +curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ export PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}" diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 56b8fc976e..2413cfca9f 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -31,10 +31,14 @@ export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch} export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} # Whether to prefer BusyBox over GNU utilities export USE_BUSY_BOX=${USE_BUSY_BOX:-false} + export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} +export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} +export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000} + export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04} # Randomize test order. diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index a4dc54d1c1..7ec66eeb4f 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -9,8 +9,9 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross export HOST=x86_64-apple-darwin16 export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" -export OSX_SDK=10.14 +export XCODE_VERSION=11.3.1 +export XCODE_BUILD_ID=11C505 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" +export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror" diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index 982e38daee..b8a9ccaae5 100644 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -7,10 +7,16 @@ export LC_ALL=C.UTF-8 export HOST=x86_64-apple-darwin16 +export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well) export PIP_PACKAGES="zmq" export GOAL="install" -export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" -export TEST_RUNNER_EXTRA="wallet_disable" # Only run wallet_disable as a smoke test, see https://github.com/bitcoin/bitcoin/pull/17240#issuecomment-546022121 why the other tests are disabled -# Run without depends +export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror" export NO_DEPENDS=1 export OSX_SDK="" +export CCACHE_SIZE=300M + +export RUN_SECURITY_TESTS="true" +if [ "$TRAVIS_REPO_SLUG" != "bitcoin/bitcoin" ]; then + export RUN_FUNCTIONAL_TESTS="false" + export EXPECTED_TESTS_DURATION_IN_SECONDS=200 +fi diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 28c63f1cf6..d57c673069 100644 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -7,8 +7,9 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_asan -export PACKAGES="clang-8 llvm-8 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" -# Use clang-8 instead of default clang (which is clang-6 on Bionic) to avoid spurious segfaults when running on ppc64le +export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export DOCKER_NAME_TAG=ubuntu:20.04 export NO_DEPENDS=1 +export TEST_RUNNER_EXTRA="--timeout-factor=4" # Increase timeout because sanitizers slow down export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang-8 CXX=clang++-8" +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 43ee219ef9..31f437f0e8 100644 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -15,3 +15,4 @@ export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++" +export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index c27d525003..e06a40eb23 100644 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -16,3 +16,4 @@ export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++" +export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh new file mode 100644 index 0000000000..6a4979990b --- /dev/null +++ b/ci/test/00_setup_env_native_msan.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export DOCKER_NAME_TAG="ubuntu:20.04" +LIBCXX_DIR="${BASE_ROOT_DIR}/ci/scratch/msan/build/" +export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" +LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" +export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" +export BDB_PREFIX="${BASE_ROOT_DIR}/db4" + +export CONTAINER_NAME="ci_native_msan" +export PACKAGES="clang-9 llvm-9 cmake" +export DEP_OPTS="NO_WALLET=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++11 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++11 ${MSAN_AND_LIBCXX_FLAGS}'" +export GOAL="install" +export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${BASE_ROOT_DIR}/depends/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'" +export USE_MEMORY_SANITIZER="true" +export RUN_FUNCTIONAL_TESTS="false" +export CCACHE_SIZE=250M diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh index 0fc989085c..786f0f927f 100644 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -7,6 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_multiprocess +export DOCKER_NAME_TAG=ubuntu:20.04 export PACKAGES="cmake python3" export DEP_OPTS="MULTIPROCESS=1" export GOAL="install" diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index 9c2be4cfac..1a0b14b62b 100644 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -7,7 +7,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet -export PACKAGES="python3-zmq" +export DOCKER_NAME_TAG=ubuntu:16.04 # Use xenial to have one config run the tests in python3.5, see doc/dependencies.md +export PACKAGES="python3-zmq clang-3.8 llvm-3.8" # Use clang-3.8 to test C++11 compatibility, see doc/dependencies.md export DEP_OPTS="NO_WALLET=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" +export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CC=clang-3.8 CXX=clang++-3.8" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index fa07990756..6e2ff729a2 100644 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -7,9 +7,11 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 +export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can compile our c++17 and run our functional tests in python3 export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash +export RUN_SECURITY_TESTS="true" export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 87b9d9da95..5695c43ec3 100644 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -10,5 +10,6 @@ export CONTAINER_NAME=ci_native_tsan export DOCKER_NAME_TAG=ubuntu:20.04 export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq" export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'" +export TEST_RUNNER_EXTRA="--exclude feature_block --timeout-factor=4" # Increase timeout because sanitizers slow down. Low memory on Travis machines, exclude feature_block. export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-gui=no CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++'" +export BITCOIN_CONFIG="--enable-zmq --with-gui=no CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++'" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index ff7c5fe913..0041122f1e 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -10,6 +10,6 @@ export CONTAINER_NAME=ci_native_valgrind export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude rpc_bind --factor=2" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export TEST_RUNNER_EXTRA="--exclude rpc_bind --timeout-factor=4" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 8f0c62a1a6..eb8b870dd6 100644 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -7,8 +7,10 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 +export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to win64 (bionic is used in the gitian build as well) export HOST=x86_64-w64-mingw32 export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64" export RUN_FUNCTIONAL_TESTS=false +export RUN_SECURITY_TESTS="true" export GOAL="deploy" export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 165983d906..d3566914ac 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -14,7 +14,6 @@ if [[ $QEMU_USER_CMD == qemu-s390* ]]; then fi if [ "$TRAVIS_OS_NAME" == "osx" ]; then - export PATH="/usr/local/opt/ccache/libexec:$PATH" ${CI_RETRY_EXE} pip3 install $PIP_PACKAGES fi @@ -90,6 +89,15 @@ export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" +if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then + DOCKER_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-9) 100" + DOCKER_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-9) 100" + DOCKER_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" + DOCKER_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-10.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" + DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" + DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" +fi + if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Create $BASE_ROOT_DIR" DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index efaaf154b1..131ea21677 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -15,11 +15,22 @@ fi DOCKER_EXEC mkdir -p ${DEPENDS_DIR}/SDKs ${DEPENDS_DIR}/sdk-sources -if [ -n "$OSX_SDK" ] && [ ! -f ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz +OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" +OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}" + +if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then + curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" +fi + +if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then + # Use BDB compiled using install_db4.sh script to work around linking issue when using BDB + # from depends. See https://github.com/bitcoin/bitcoin/pull/18288#discussion_r433189350 for + # details. + DOCKER_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" fi -if [ -n "$OSX_SDK" ] && [ -f ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - DOCKER_EXEC tar -C ${DEPENDS_DIR}/SDKs -xf ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz + +if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then + DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" fi if [[ $HOST = *-mingw32 ]]; then DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\) @@ -33,12 +44,10 @@ if [ -z "$NO_DEPENDS" ]; then else SHELL_OPTS="CONFIG_SHELL=" fi - # Temporary workaround for https://github.com/bitcoin/bitcoin/issues/16368 - python3 -c 'import time; [print(".") or time.sleep(500) for _ in range(4)]' & - ( DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS ) &> /dev/null + DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS fi if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then BEGIN_FOLD previous-versions - DOCKER_EXEC contrib/devtools/previous_release.sh -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" + DOCKER_EXEC contrib/devtools/previous_release.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" END_FOLD fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index b68cd9d3f8..17d765b862 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -37,6 +37,14 @@ END_FOLD set -o errtrace trap 'DOCKER_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR +if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then + # MemorySanitizer (MSAN) does not support tracking memory initialization done by + # using the Linux getrandom syscall. Avoid using getrandom by undefining + # HAVE_SYS_GETRANDOM. See https://github.com/google/sanitizers/issues/852 for + # details. + DOCKER_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' +fi + BEGIN_FOLD build DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) END_FOLD diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 51d84ee39d..96d44328b8 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -21,8 +21,6 @@ if [ -n "$USE_VALGRIND" ]; then END_FOLD fi -bash -c "${CI_WAIT}" & # Print dots in case the tests take a long time to run - if [ "$RUN_UNIT_TESTS" = "true" ]; then BEGIN_FOLD unit-tests DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib make $MAKEJOBS check VERBOSE=1 @@ -41,6 +39,12 @@ if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then END_FOLD fi +if [ "$RUN_SECURITY_TESTS" = "true" ]; then + BEGIN_FOLD security-tests + DOCKER_EXEC make test-security-check + END_FOLD +fi + if [ "$RUN_FUZZ_TESTS" = "true" ]; then BEGIN_FOLD fuzz-tests DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN} diff --git a/configure.ac b/configure.ac index 3bfbe2e7b0..f11f2b2059 100644 --- a/configure.ac +++ b/configure.ac @@ -14,6 +14,12 @@ AC_CONFIG_HEADERS([src/config/bitcoin-config.h]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) +m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR([PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh])]) +PKG_PROG_PKG_CONFIG +if test "x$PKG_CONFIG" = x; then + AC_MSG_ERROR([pkg-config not found]) +fi + BITCOIN_DAEMON_NAME=bitcoind BITCOIN_GUI_NAME=bitcoin-qt BITCOIN_CLI_NAME=bitcoin-cli @@ -111,9 +117,6 @@ AC_PATH_TOOL(READELF, readelf) AC_PATH_TOOL(CPPFILT, c++filt) AC_PATH_TOOL(OBJCOPY, objcopy) AC_PATH_PROG(DOXYGEN, doxygen) -if test -z "$DOXYGEN"; then - AC_MSG_WARN([Doxygen not found]) -fi AM_CONDITIONAL([HAVE_DOXYGEN], [test -n "$DOXYGEN"]) AC_ARG_VAR(PYTHONPATH, Augments the default search path for python module files) @@ -232,16 +235,6 @@ AC_ARG_ENABLE([zmq], [use_zmq=$enableval], [use_zmq=yes]) -AC_ARG_ENABLE([bip70], - [AS_HELP_STRING([--enable-bip70], - [BIP70 (payment protocol) support in the GUI (no longer supported)])], - [enable_bip70=$enableval], - [enable_bip70=no]) - -if test x$enable_bip70 != xno; then - AC_MSG_ERROR([BIP70 is no longer supported!]) -fi - AC_ARG_WITH([libmultiprocess], [AS_HELP_STRING([--with-libmultiprocess=yes|no|auto], [Build with libmultiprocess library. (default: auto, i.e. detect with pkg-config)])], @@ -399,11 +392,11 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wextra],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wgnu],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wgnu"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wformat],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat"],,[[$CXXFLAG_WERROR]]) + dnl some compilers will ignore -Wformat-security without -Wformat, so just combine the two here. + AX_CHECK_COMPILE_FLAG([-Wformat -Wformat-security],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat -Wformat-security"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wvla],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wshadow-field],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wswitch],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wswitch"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wformat-security],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat-security"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wthread-safety],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) @@ -573,13 +566,8 @@ AC_ARG_WITH([daemon], [build_bitcoind=$withval], [build_bitcoind=yes]) -use_pkgconfig=yes case $host in *mingw*) - - dnl pkgconfig does more harm than good with MinGW - use_pkgconfig=no - TARGET_OS=windows AC_CHECK_LIB([kernel32], [GetModuleFileNameA],, AC_MSG_ERROR(libkernel32 missing)) AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(libuser32 missing)) @@ -638,9 +626,10 @@ case $host in bdb_prefix=$($BREW --prefix berkeley-db4 2>/dev/null) qt5_prefix=$($BREW --prefix qt5 2>/dev/null) - if test x$bdb_prefix != x; then - CPPFLAGS="$CPPFLAGS -I$bdb_prefix/include" - LIBS="$LIBS -L$bdb_prefix/lib" + if test x$bdb_prefix != x && test "x$BDB_CFLAGS" = "x" && test "x$BDB_LIBS" = "x"; then + dnl This must precede the call to BITCOIN_FIND_BDB48 below. + BDB_CFLAGS="-I$bdb_prefix/include" + BDB_LIBS="-L$bdb_prefix/lib -ldb_cxx-4.8" fi if test x$qt5_prefix != x; then PKG_CONFIG_PATH="$qt5_prefix/lib/pkgconfig:$PKG_CONFIG_PATH" @@ -653,6 +642,7 @@ case $host in BUILD_OS=darwin ;; *) + AC_PATH_TOOL([DSYMUTIL], [dsymutil], dsymutil) AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool) AC_PATH_TOOL([OTOOL], [otool], otool) AC_PATH_PROGS([GENISOIMAGE], [genisoimage mkisofs],genisoimage) @@ -682,16 +672,6 @@ case $host in ;; esac -if test x$use_pkgconfig = xyes; then - m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR(PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh.)]) - m4_ifdef([PKG_PROG_PKG_CONFIG], [ - PKG_PROG_PKG_CONFIG - if test x"$PKG_CONFIG" = "x"; then - AC_MSG_ERROR(pkg-config not found.) - fi - ]) -fi - if test x$use_extended_functional_tests != xno; then AC_SUBST(EXTENDED_FUNCTIONAL_TESTS, --extended) fi @@ -805,6 +785,7 @@ if test x$use_hardening != xno; then AX_CHECK_LINK_FLAG([[-Wl,--high-entropy-va]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"],, [[$LDFLAG_WERROR]]) AX_CHECK_LINK_FLAG([[-Wl,-z,relro]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,relro"],, [[$LDFLAG_WERROR]]) AX_CHECK_LINK_FLAG([[-Wl,-z,now]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,now"],, [[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([[-Wl,-z,separate-code]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,separate-code"],, [[$LDFLAG_WERROR]]) AX_CHECK_LINK_FLAG([[-fPIE -pie]], [PIE_FLAGS="-fPIE"; HARDENED_LDFLAGS="$HARDENED_LDFLAGS -pie"],, [[$CXXFLAG_WERROR]]) case $host in @@ -852,7 +833,21 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,, #include <byteswap.h> #endif]) -AC_CHECK_DECLS([__builtin_clz, __builtin_clzl, __builtin_clzll]) +AC_MSG_CHECKING(for __builtin_clzl) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ + (void) __builtin_clzl(0); + ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])], + [ AC_MSG_RESULT(no)] +) + +AC_MSG_CHECKING(for __builtin_clzll) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ + (void) __builtin_clzll(0); + ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])], + [ AC_MSG_RESULT(no)] +) dnl Check for malloc_info (for memory statistics information in getmemoryinfo) AC_MSG_CHECKING(for getmemoryinfo) @@ -1139,7 +1134,7 @@ else BITCOIN_QT_INIT dnl sets $bitcoin_enable_qt, $bitcoin_enable_qt_test, $bitcoin_enable_qt_dbus - BITCOIN_QT_CONFIGURE([$use_pkgconfig]) + BITCOIN_QT_CONFIGURE fi if test x$enable_wallet != xno; then @@ -1301,115 +1296,66 @@ CPPFLAGS="$TEMP_CPPFLAGS" fi -if test x$use_pkgconfig = xyes; then - : dnl - m4_ifdef( - [PKG_CHECK_MODULES], - [ - if test x$use_qr != xno; then - BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) - fi - if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR(libevent version 2.0.21 or greater not found.)]) - if test x$TARGET_OS != xwindows; then - PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR(libevent_pthreads version 2.0.21 or greater not found.)]) - fi - fi - - if test "x$use_zmq" = "xyes"; then - PKG_CHECK_MODULES([ZMQ],[libzmq >= 4], - [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], - [AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) - AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) - use_zmq=no]) - else - AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) - fi - ] - ) -else +dnl libevent check - if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - AC_CHECK_HEADER([event2/event.h], [use_libevent=yes], AC_MSG_ERROR(libevent headers missing),) - AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing)) - if test x$TARGET_OS != xwindows; then - AC_CHECK_LIB([event_pthreads],[main],EVENT_PTHREADS_LIBS=-levent_pthreads,AC_MSG_ERROR(libevent_pthreads missing)) - fi +if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then + PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.0.21 or greater not found.])]) + if test x$TARGET_OS != xwindows; then + PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR([libevent_pthreads version 2.0.21 or greater not found.])]) fi +fi - if test "x$use_zmq" = "xyes"; then - AC_CHECK_HEADER([zmq.h], - [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], - [AC_MSG_WARN([zmq.h not found, disabling zmq support]) - use_zmq=no - AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) - AC_CHECK_LIB([zmq],[zmq_ctx_shutdown],ZMQ_LIBS=-lzmq, - [AC_MSG_WARN([libzmq >= 4.0 not found, disabling zmq support]) - use_zmq=no - AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) - else - AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) - fi +dnl QR Code encoding library check - if test "x$use_zmq" = "xyes"; then - dnl Assume libzmq was built for static linking - case $host in - *mingw*) - ZMQ_CFLAGS="$ZMQ_CFLAGS -DZMQ_STATIC" - ;; - esac - fi +if test "x$use_qr" != xno; then + BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) +fi - if test x$use_qr != xno; then - BITCOIN_QT_CHECK([AC_CHECK_LIB([qrencode], [main],[QR_LIBS=-lqrencode], [have_qrencode=no])]) - BITCOIN_QT_CHECK([AC_CHECK_HEADER([qrencode.h],, have_qrencode=no)]) - fi +dnl ZMQ check + +if test "x$use_zmq" = xyes; then + PKG_CHECK_MODULES([ZMQ], [libzmq >= 4], + AC_DEFINE([ENABLE_ZMQ], [1], [Define to 1 to enable ZMQ functions]), + [AC_DEFINE([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) + AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) + use_zmq=no]) +else + AC_DEFINE_UNQUOTED([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) +fi + +if test "x$use_zmq" = xyes; then + dnl Assume libzmq was built for static linking + case $host in + *mingw*) + ZMQ_CFLAGS="$ZMQ_CFLAGS -DZMQ_STATIC" + ;; + esac fi dnl univalue check need_bundled_univalue=yes - if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononononono; then need_bundled_univalue=no else - -if test x$system_univalue != xno ; then - found_univalue=no - if test x$use_pkgconfig = xyes; then - : #NOP - m4_ifdef( - [PKG_CHECK_MODULES], - [ - PKG_CHECK_MODULES([UNIVALUE],[libunivalue >= 1.0.4],[found_univalue=yes],[true]) - ] - ) - else - AC_CHECK_HEADER([univalue.h],[ - AC_CHECK_LIB([univalue], [main],[ - UNIVALUE_LIBS=-lunivalue - found_univalue=yes - ],[true]) - ],[true]) + if test x$system_univalue != xno; then + PKG_CHECK_MODULES([UNIVALUE], [libunivalue >= 1.0.4], [found_univalue=yes], [found_univalue=no]) + if test x$found_univalue = xyes; then + system_univalue=yes + need_bundled_univalue=no + elif test x$system_univalue = xyes; then + AC_MSG_ERROR([univalue not found]) + else + system_univalue=no + fi fi - if test x$found_univalue = xyes ; then - system_univalue=yes - need_bundled_univalue=no - elif test x$system_univalue = xyes ; then - AC_MSG_ERROR([univalue not found]) - else - system_univalue=no + if test x$need_bundled_univalue = xyes; then + UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' + UNIVALUE_LIBS='univalue/libunivalue.la' fi fi -if test x$need_bundled_univalue = xyes ; then - UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' - UNIVALUE_LIBS='univalue/libunivalue.la' -fi - -fi - AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes]) AC_SUBST(UNIVALUE_CFLAGS) AC_SUBST(UNIVALUE_LIBS) @@ -1418,12 +1364,10 @@ dnl libmultiprocess library check libmultiprocess_found=no if test "x$with_libmultiprocess" = xyes || test "x$with_libmultiprocess" = xauto; then - if test "x$use_pkgconfig" = xyes; then - m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ - libmultiprocess_found=yes; - libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`; - ], [true])]) - fi + m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ + libmultiprocess_found=yes; + libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`; + ], [true])]) elif test "x$with_libmultiprocess" != xno; then AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no]) fi @@ -1600,6 +1544,7 @@ fi AM_CONDITIONAL([TARGET_DARWIN], [test x$TARGET_OS = xdarwin]) AM_CONDITIONAL([BUILD_DARWIN], [test x$BUILD_OS = xdarwin]) +AM_CONDITIONAL([TARGET_LINUX], [test x$TARGET_OS = xlinux]) AM_CONDITIONAL([TARGET_WINDOWS], [test x$TARGET_OS = xwindows]) AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes]) AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes]) @@ -1688,6 +1633,8 @@ AC_SUBST(HAVE_WEAK_GETAUXVAL) AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) AM_COND_IF([HAVE_DOXYGEN], [AC_CONFIG_FILES([doc/Doxyfile])]) +AC_CONFIG_LINKS([contrib/devtools/security-check.py:contrib/devtools/security-check.py]) +AC_CONFIG_LINKS([contrib/devtools/test-security-check.py:contrib/devtools/test-security-check.py]) AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/fuzz/test_runner.py:test/fuzz/test_runner.py]) @@ -1721,7 +1668,7 @@ if test x$need_bundled_univalue = xyes; then AC_CONFIG_SUBDIRS([src/univalue]) fi -ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --with-bignum=no --enable-module-recovery --disable-jni" +ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --with-bignum=no --enable-module-recovery" AC_CONFIG_SUBDIRS([src/secp256k1]) AC_OUTPUT diff --git a/contrib/devtools/previous_release.py b/contrib/devtools/previous_release.py new file mode 100755 index 0000000000..5599051cf3 --- /dev/null +++ b/contrib/devtools/previous_release.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Download or build previous releases. +# Needs curl and tar to download a release, or the build dependencies when +# building a release. + +import argparse +import contextlib +from fnmatch import fnmatch +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import hashlib + + +@contextlib.contextmanager +def pushd(new_dir) -> None: + previous_dir = os.getcwd() + os.chdir(new_dir) + try: + yield + finally: + os.chdir(previous_dir) + + +def download_binary(tag, args) -> int: + if Path(tag).is_dir(): + if not args.remove_dir: + print('Using cached {}'.format(tag)) + return 0 + shutil.rmtree(tag) + Path(tag).mkdir() + bin_path = 'bin/bitcoin-core-{}'.format(tag[1:]) + match = re.compile('v(.*)(rc[0-9]+)$').search(tag) + if match: + bin_path = 'bin/bitcoin-core-{}/test.{}'.format( + match.group(1), match.group(2)) + tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format( + tag=tag[1:], platform=args.platform) + sha256Sums = "SHA256SUMS-{tag}.asc".format(tag=tag[1:]) + tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format( + bin_path=bin_path, tarball=tarball) + sha256SumsUrl = 'https://bitcoincore.org/{bin_path}/SHA256SUMS.asc'.format( + bin_path=bin_path) + + print('Fetching: {tarballUrl}'.format(tarballUrl=tarballUrl)) + print('Fetching: {sha256SumsUrl}'.format(sha256SumsUrl=sha256SumsUrl)) + + header, status = subprocess.Popen( + ['curl', '--head', tarballUrl], stdout=subprocess.PIPE).communicate() + if re.search("404 Not Found", header.decode("utf-8")): + print("Binary tag was not found") + return 1 + + curlCmds = [ + ['curl', '--remote-name', tarballUrl], + ['curl', '--output', sha256Sums, sha256SumsUrl], + ] + + for cmd in curlCmds: + ret = subprocess.run(cmd).returncode + if ret: + return ret + + hasher = hashlib.sha256() + with open(tarball, "rb") as afile: + hasher.update(afile.read()) + tarballHash = hasher.hexdigest() + tarballHash = '{} {}\n'.format(tarballHash, tarball) + with open(sha256Sums, 'r', encoding="utf-8") as afile: + shasums = afile.readlines() + + if tarballHash not in shasums: + print("Checksum did not match") + Path(tarball).unlink() + return 1 + print("Checksum matched") + + # Bitcoin Core Release Signing Keys v0.11.0+ + signingKey = "01EA5486DE18A882D4C2684590C8019E36C2E964" + + isKeyPresent = subprocess.run( + ["gpg", "--list-keys", signingKey]).returncode + if isKeyPresent: + return isKeyPresent + + isVerified = subprocess.run( + ["gpg", "--verify", sha256Sums]).returncode + if isVerified: + return isVerified + + # Extract tarball + ret = subprocess.run(['tar', '-zxf', tarball, '-C', tag, + '--strip-components=1', + 'bitcoin-{tag}'.format(tag=tag[1:])]).returncode + if ret: + return ret + + Path(tarball).unlink() + Path(sha256Sums).unlink() + return 0 + + +def build_release(tag, args) -> int: + githubUrl = "https://github.com/bitcoin/bitcoin" + if args.remove_dir: + if Path(tag).is_dir(): + shutil.rmtree(tag) + if not Path(tag).is_dir(): + # fetch new tags + subprocess.run( + ["git", "fetch", githubUrl, "--tags"]) + output = subprocess.check_output(['git', 'tag', '-l', tag]) + if not output: + print('Tag {} not found'.format(tag)) + return 1 + ret = subprocess.run([ + 'git', 'clone', githubUrl, tag + ]).returncode + if ret: + return ret + with pushd(tag): + ret = subprocess.run(['git', 'checkout', tag]).returncode + if ret: + return ret + host = args.host + if args.depends: + with pushd('depends'): + ret = subprocess.run(['make', 'NO_QT=1']).returncode + if ret: + return ret + host = os.environ.get( + 'HOST', subprocess.check_output(['./config.guess'])) + config_flags = '--prefix={pwd}/depends/{host} '.format( + pwd=os.getcwd(), + host=host) + args.config_flags + cmds = [ + './autogen.sh', + './configure {}'.format(config_flags), + 'make', + ] + for cmd in cmds: + ret = subprocess.run(cmd.split()).returncode + if ret: + return ret + # Move binaries, so they're in the same place as in the + # release download + Path('bin').mkdir(exist_ok=True) + files = ['bitcoind', 'bitcoin-cli', 'bitcoin-tx'] + for f in files: + Path('src/'+f).rename('bin/'+f) + return 0 + + +def check_host(args) -> int: + args.host = os.environ.get('HOST', subprocess.check_output( + './depends/config.guess').decode()) + if args.download_binary: + platforms = { + 'x86_64-*-linux*': 'x86_64-linux-gnu', + 'x86_64-apple-darwin*': 'osx64', + } + args.platform = '' + for pattern, target in platforms.items(): + if fnmatch(args.host, pattern): + args.platform = target + if not args.platform: + print('Not sure which binary to download for {}'.format(args.host)) + return 1 + return 0 + + +def main(args) -> int: + Path(args.target_dir).mkdir(exist_ok=True, parents=True) + print("Releases directory: {}".format(args.target_dir)) + ret = check_host(args) + if ret: + return ret + if args.download_binary: + with pushd(args.target_dir): + for tag in args.tags: + ret = download_binary(tag, args) + if ret: + return ret + return 0 + args.config_flags = os.environ.get('CONFIG_FLAGS', '') + args.config_flags += ' --without-gui --disable-tests --disable-bench' + with pushd(args.target_dir): + for tag in args.tags: + ret = build_release(tag, args) + if ret: + return ret + return 0 + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('-r', '--remove-dir', action='store_true', + help='remove existing directory.') + parser.add_argument('-d', '--depends', action='store_true', + help='use depends.') + parser.add_argument('-b', '--download-binary', action='store_true', + help='download release binary.') + parser.add_argument('-t', '--target-dir', action='store', + help='target directory.', default='releases') + parser.add_argument('tags', nargs='+', + help="release tags. e.g.: v0.18.1 v0.20.0rc2") + args = parser.parse_args() + sys.exit(main(args)) diff --git a/contrib/devtools/previous_release.sh b/contrib/devtools/previous_release.sh deleted file mode 100755 index d375291f47..0000000000 --- a/contrib/devtools/previous_release.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Build previous releases. - -export LC_ALL=C - -CONFIG_FLAGS="" -FUNCTIONAL_TESTS=0 -DELETE_EXISTING=0 -USE_DEPENDS=0 -DOWNLOAD_BINARY=0 -CONFIG_FLAGS="" -TARGET="releases" - -while getopts ":hfrdbt:" opt; do - case $opt in - h) - echo "Usage: .previous_release.sh [options] tag1 tag2" - echo " options:" - echo " -h Print this message" - echo " -f Configure for functional tests" - echo " -r Remove existing directory" - echo " -d Use depends" - echo " -b Download release binary" - echo " -t Target directory (default: releases)" - exit 0 - ;; - f) - FUNCTIONAL_TESTS=1 - CONFIG_FLAGS="$CONFIG_FLAGS --without-gui --disable-tests --disable-bench" - ;; - r) - DELETE_EXISTING=1 - ;; - d) - USE_DEPENDS=1 - ;; - b) - DOWNLOAD_BINARY=1 - ;; - t) - TARGET=$OPTARG - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1 - ;; - esac -done - -shift $((OPTIND-1)) - -if [ -z "$1" ]; then - echo "Specify release tag(s), e.g.: .previous_release v0.15.1" - exit 1 -fi - -if [ ! -d "$TARGET" ]; then - mkdir -p $TARGET -fi - -if [ "$DOWNLOAD_BINARY" -eq "1" ]; then - HOST="${HOST:-$(./depends/config.guess)}" - case "$HOST" in - x86_64-*-linux*) - PLATFORM=x86_64-linux-gnu - ;; - x86_64-apple-darwin*) - PLATFORM=osx64 - ;; - *) - echo "Not sure which binary to download for $HOST." - exit 1 - ;; - esac -fi - -echo "Releases directory: $TARGET" -pushd "$TARGET" || exit 1 -{ - for tag in "$@" - do - if [ "$DELETE_EXISTING" -eq "1" ]; then - if [ -d "$tag" ]; then - rm -r "$tag" - fi - fi - - if [ "$DOWNLOAD_BINARY" -eq "0" ]; then - - if [ ! -d "$tag" ]; then - if [ -z $(git tag -l "$tag") ]; then - echo "Tag $tag not found" - exit 1 - fi - - git clone https://github.com/bitcoin/bitcoin "$tag" - pushd "$tag" || exit 1 - { - git checkout "$tag" - if [ "$USE_DEPENDS" -eq "1" ]; then - pushd depends || exit 1 - { - if [ "$FUNCTIONAL_TESTS" -eq "1" ]; then - make NO_QT=1 - else - make - fi - HOST="${HOST:-$(./config.guess)}" - } - popd || exit 1 - CONFIG_FLAGS="--prefix=$PWD/depends/$HOST $CONFIG_FLAGS" - fi - ./autogen.sh - ./configure $CONFIG_FLAGS - make - # Move binaries, so they're in the same place as in the release download: - mkdir bin - mv src/bitcoind src/bitcoin-cli src/bitcoin-tx bin - if [ "$FUNCTIONAL_TESTS" -eq "0" ]; then - mv src/qt/bitcoin-qt bin - fi - } - popd || exit 1 - fi - else - if [ -d "$tag" ]; then - echo "Using cached $tag" - else - mkdir "$tag" - if [[ "$tag" =~ v(.*)(rc[0-9]+)$ ]]; then - BIN_PATH="bin/bitcoin-core-${BASH_REMATCH[1]}/test.${BASH_REMATCH[2]}" - else - BIN_PATH="bin/bitcoin-core-${tag:1}" - fi - URL="https://bitcoincore.org/$BIN_PATH/bitcoin-${tag:1}-$PLATFORM.tar.gz" - echo "Fetching: $URL" - if ! curl -O -f $URL; then - echo "Download failed." - exit 1 - fi - tar -zxf "bitcoin-${tag:1}-$PLATFORM.tar.gz" -C "$tag" --strip-components=1 "bitcoin-${tag:1}" - rm "bitcoin-${tag:1}-$PLATFORM.tar.gz" - fi - fi - done -} -popd || exit 1 diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index ca587ca9e5..dc74de9198 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -40,25 +40,48 @@ def get_ELF_program_headers(executable): stdout = run_command([READELF_CMD, '-l', '-W', executable]) in_headers = False - count = 0 headers = [] for line in stdout.splitlines(): if line.startswith('Program Headers:'): in_headers = True + count = 0 if line == '': in_headers = False if in_headers: if count == 1: # header line - ofs_typ = line.find('Type') - ofs_offset = line.find('Offset') - ofs_flags = line.find('Flg') - ofs_align = line.find('Align') - if ofs_typ == -1 or ofs_offset == -1 or ofs_flags == -1 or ofs_align == -1: + header = [x.strip() for x in line.split()] + ofs_typ = header.index('Type') + ofs_flags = header.index('Flg') + # assert readelf output is what we expect + if ofs_typ == -1 or ofs_flags == -1: raise ValueError('Cannot parse elfread -lW output') elif count > 1: - typ = line[ofs_typ:ofs_offset].rstrip() - flags = line[ofs_flags:ofs_align].rstrip() - headers.append((typ, flags)) + splitline = [x.strip() for x in line.split()] + typ = splitline[ofs_typ] + if not typ.startswith('[R'): # skip [Requesting ...] + splitline = [x.strip() for x in line.split()] + flags = splitline[ofs_flags] + # check for 'R', ' E' + if splitline[ofs_flags + 1] is 'E': + flags += ' E' + headers.append((typ, flags, [])) + count += 1 + + if line.startswith(' Section to Segment mapping:'): + in_mapping = True + count = 0 + if line == '': + in_mapping = False + if in_mapping: + if count == 1: # header line + ofs_segment = line.find('Segment') + ofs_sections = line.find('Sections...') + if ofs_segment == -1 or ofs_sections == -1: + raise ValueError('Cannot parse elfread -lW output') + elif count > 1: + segment = int(line[ofs_segment:ofs_sections].strip()) + sections = line[ofs_sections:].strip().split() + headers[segment][2].extend(sections) count += 1 return headers @@ -68,7 +91,7 @@ def check_ELF_NX(executable) -> bool: ''' have_wx = False have_gnu_stack = False - for (typ, flags) in get_ELF_program_headers(executable): + for (typ, flags, _) in get_ELF_program_headers(executable): if typ == 'GNU_STACK': have_gnu_stack = True if 'W' in flags and 'E' in flags: # section is both writable and executable @@ -82,7 +105,7 @@ def check_ELF_RELRO(executable) -> bool: Dynamic section must have BIND_NOW flag ''' have_gnu_relro = False - for (typ, flags) in get_ELF_program_headers(executable): + for (typ, flags, _) in get_ELF_program_headers(executable): # Note: not checking flags == 'R': here as linkers set the permission differently # This does not affect security: the permission flags of the GNU_RELRO program # header are ignored, the PT_LOAD header determines the effective permissions. @@ -113,6 +136,62 @@ def check_ELF_Canary(executable) -> bool: ok = True return ok +def check_ELF_separate_code(executable): + ''' + Check that sections are appropriately separated in virtual memory, + based on their permissions. This checks for missing -Wl,-z,separate-code + and potentially other problems. + ''' + EXPECTED_FLAGS = { + # Read + execute + '.init': 'R E', + '.plt': 'R E', + '.plt.got': 'R E', + '.plt.sec': 'R E', + '.text': 'R E', + '.fini': 'R E', + # Read-only data + '.interp': 'R', + '.note.gnu.property': 'R', + '.note.gnu.build-id': 'R', + '.note.ABI-tag': 'R', + '.gnu.hash': 'R', + '.dynsym': 'R', + '.dynstr': 'R', + '.gnu.version': 'R', + '.gnu.version_r': 'R', + '.rela.dyn': 'R', + '.rela.plt': 'R', + '.rodata': 'R', + '.eh_frame_hdr': 'R', + '.eh_frame': 'R', + '.qtmetadata': 'R', + '.gcc_except_table': 'R', + '.stapsdt.base': 'R', + # Writable data + '.init_array': 'RW', + '.fini_array': 'RW', + '.dynamic': 'RW', + '.got': 'RW', + '.data': 'RW', + '.bss': 'RW', + } + # For all LOAD program headers get mapping to the list of sections, + # and for each section, remember the flags of the associated program header. + flags_per_section = {} + for (typ, flags, sections) in get_ELF_program_headers(executable): + if typ == 'LOAD': + for section in sections: + assert(section not in flags_per_section) + flags_per_section[section] = flags + # Spot-check ELF LOAD program header flags per section + # If these sections exist, check them against the expected R/W/E flags + for (section, flags) in flags_per_section.items(): + if section in EXPECTED_FLAGS: + if EXPECTED_FLAGS[section] != flags: + return False + return True + def get_PE_dll_characteristics(executable) -> int: '''Get PE DllCharacteristics bits''' stdout = run_command([OBJDUMP_CMD, '-x', executable]) @@ -225,7 +304,8 @@ CHECKS = { ('PIE', check_ELF_PIE), ('NX', check_ELF_NX), ('RELRO', check_ELF_RELRO), - ('Canary', check_ELF_Canary) + ('Canary', check_ELF_Canary), + ('separate_code', check_ELF_separate_code), ], 'PE': [ ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index ea70b27941..ec2d886653 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -20,10 +20,9 @@ def write_testcode(filename): ''') def call_security_check(cc, source, executable, options): - subprocess.check_call([cc,source,'-o',executable] + options) - p = subprocess.Popen(['./security-check.py',executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - return (p.returncode, stdout.rstrip()) + subprocess.run([cc,source,'-o',executable] + options, check=True) + p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) + return (p.returncode, p.stdout.rstrip()) class TestSecurityChecks(unittest.TestCase): def test_ELF(self): @@ -32,15 +31,17 @@ class TestSecurityChecks(unittest.TestCase): cc = 'gcc' write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), (1, executable+': failed PIE NX RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), (1, executable+': failed PIE RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), (1, executable+': failed PIE RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), (1, executable+': failed RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), + (1, executable+': failed separate_code')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), (0, '')) def test_PE(self): diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh index 95b1553215..8501c72f04 100755 --- a/contrib/devtools/test_deterministic_coverage.sh +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -16,7 +16,6 @@ GCOV_EXECUTABLE="gcov" NON_DETERMINISTIC_TESTS=( "blockfilter_index_tests/blockfilter_index_initial_sync" # src/checkqueue.h: In CCheckQueue::Loop(): while (queue.empty()) { ... } "coinselector_tests/knapsack_solver_test" # coinselector_tests.cpp: if (equal_sets(setCoinsRet, setCoinsRet2)) - "denialofservice_tests/DoS_mapOrphans" # denialofservice_tests.cpp: it = mapOrphanTransactions.lower_bound(InsecureRand256()); "fs_tests/fsbridge_fstream" # deterministic test failure? "miner_tests/CreateNewBlock_validity" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) "scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue() diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index 4a3df93cea..d498c9e2c8 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -209,7 +209,7 @@ def main(): args.macos = 'm' in args.os # Disable for MacOS if no SDK found - if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.14.sdk.tar.gz'): + if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz'): print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') args.macos = False diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index bbae7201e5..e0aaafc15a 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -32,7 +32,7 @@ remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: -- "MacOSX10.14.sdk.tar.gz" +- "Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz" script: | set -e -o pipefail @@ -90,7 +90,7 @@ script: | BASEPREFIX="${PWD}/depends" mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.14.sdk.tar.gz + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz # Build dependencies for each host for i in $HOSTS; do diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 68ebb5def1..fe677e3a1f 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -14,55 +14,44 @@ When complete, it will have produced `Bitcoin-Qt.dmg`. ## SDK Extraction -Our current macOS SDK (`macOSX10.14.sdk`) can be extracted from -[Xcode_10.2.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). +### Step 1: Obtaining `Xcode.app` + +Our current macOS SDK +(`Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`) can be +extracted from +[Xcode_11.3.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip). An Apple ID is needed to download this. -`Xcode.app` is packaged in a `.xip` archive. -This makes the SDK less-trivial to extract on non-macOS machines. -One approach (tested on Debian Buster) is outlined below: +After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip` +archive. This makes the SDK less-trivial to extract on non-macOS machines. One +approach (tested on Debian Buster) is outlined below: ```bash +# Install/clone tools needed for extracting Xcode.app +apt install cpio +git clone https://github.com/bitcoin-core/apple-sdk-tools.git -apt install clang cpio git liblzma-dev libxml2-dev libssl-dev make - -git clone https://github.com/tpoechtrager/xar -pushd xar/xar -./configure -make -make install -popd - -git clone https://github.com/NiklasRosenstein/pbzx -pushd pbzx -clang -llzma -lxar pbzx.c -o pbzx -Wl,-rpath=/usr/local/lib -popd - -xar -xf Xcode_10.2.1.xip -C . - -./pbzx/pbzx -n Content | cpio -i - -find Xcode.app -type d -name MacOSX.sdk -exec sh -c 'tar --transform="s/MacOSX.sdk/MacOSX10.14.sdk/" -c -C$(dirname {}) MacOSX.sdk/ | gzip -9n > MacOSX10.14.sdk.tar.gz' \; +# Unpack Xcode_11.3.1.xip and place the resulting Xcode.app in your current +# working directory +python3 apple-sdk-tools/extract_xcode.py -f Xcode_11.3.1.xip | cpio -d -i ``` -on macOS the process is more straightforward: +On macOS the process is more straightforward: ```bash -xip -x Xcode_10.2.1.xip -tar -s "/MacOSX.sdk/MacOSX10.14.sdk/" -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk +xip -x Xcode_11.3.1.xip ``` -Our previously used macOS SDK (`MacOSX10.11.sdk`) can be extracted from -[Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg). -The script [`extract-osx-sdk.sh`](./extract-osx-sdk.sh) automates this. First -ensure the DMG file is in the current directory, and then run the script. You -may wish to delete the `intermediate 5.hfs` file and `MacOSX10.11.sdk` (the -directory) when you've confirmed the extraction succeeded. +### Step 2: Generating `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` + +To generate `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`, run +the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the +previous stage) as the first argument. ```bash -apt-get install p7zip-full sleuthkit -contrib/macdeploy/extract-osx-sdk.sh -rm -rf 5.hfs MacOSX10.11.sdk +# Generate a Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz from +# the supplied Xcode.app +./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' ``` ## Deterministic macOS DMG Notes @@ -91,13 +80,13 @@ and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done To complicate things further, all builds must target an Apple SDK. These SDKs are free to download, but not redistributable. To obtain it, register for an Apple Developer Account, -then download [Xcode 10.2.1](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). +then download [Xcode_11.3.1](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip). This file is many gigabytes in size, but most (but not all) of what we need is contained only in a single directory: ```bash -Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk +Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk ``` See the SDK Extraction notes above for how to obtain it. diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk new file mode 100755 index 0000000000..457d8f5e64 --- /dev/null +++ b/contrib/macdeploy/gen-sdk @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +import argparse +import plistlib +import pathlib +import sys +import tarfile +import gzip +import os +import contextlib + +@contextlib.contextmanager +def cd(path): + """Context manager that restores PWD even if an exception was raised.""" + old_pwd = os.getcwd() + os.chdir(str(path)) + try: + yield + finally: + os.chdir(old_pwd) + +def run(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument('xcode_app', metavar='XCODEAPP', nargs=1) + parser.add_argument("-o", metavar='OUTSDKTGZ', nargs=1, dest='out_sdktgz', required=False) + + args = parser.parse_args() + + xcode_app = pathlib.Path(args.xcode_app[0]).resolve() + assert xcode_app.is_dir(), "The supplied Xcode.app path '{}' either does not exist or is not a directory".format(xcode_app) + + xcode_app_plist = xcode_app.joinpath("Contents/version.plist") + with xcode_app_plist.open('rb') as fp: + pl = plistlib.load(fp) + xcode_version = pl['CFBundleShortVersionString'] + xcode_build_id = pl['ProductBuildVersion'] + print("Found Xcode (version: {xcode_version}, build id: {xcode_build_id})".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id)) + + sdk_dir = xcode_app.joinpath("Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk") + sdk_plist = sdk_dir.joinpath("System/Library/CoreServices/SystemVersion.plist") + with sdk_plist.open('rb') as fp: + pl = plistlib.load(fp) + sdk_version = pl['ProductVersion'] + sdk_build_id = pl['ProductBuildVersion'] + print("Found MacOSX SDK (version: {sdk_version}, build id: {sdk_build_id})".format(sdk_version=sdk_version, sdk_build_id=sdk_build_id)) + + out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id) + + xcode_libcxx_dir = xcode_app.joinpath("Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1") + assert xcode_libcxx_dir.is_dir() + + if args.out_sdktgz: + out_sdktgz_path = pathlib.Path(args.out_sdktgz_path) + else: + # Construct our own out_sdktgz if not specified on the command line + out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name)) + + def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir): + """Add all files in dir_to_add to tarfp, but prepent MEMBERPREFIX to the files' + names + + e.g. if the only file under /root/bazdir is /root/bazdir/qux, invoking: + + tarfp_add_with_base_change(tarfp, "foo/bar", "/root/bazdir") + + would result in the following members being added to tarfp: + + foo/bar/ -> corresponding to /root/bazdir + foo/bar/qux -> corresponding to /root/bazdir/qux + + """ + def change_tarinfo_base(tarinfo): + if tarinfo.name and tarinfo.name.startswith("./"): + tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name)) + if tarinfo.linkname and tarinfo.linkname.startswith("./"): + tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname)) + return tarinfo + with cd(dir_to_add): + tarfp.add(".", recursive=True, filter=change_tarinfo_base) + + print("Creating output .tar.gz file...") + with out_sdktgz_path.open("wb") as fp: + with gzip.GzipFile(fileobj=fp, compresslevel=9, mtime=0) as gzf: + with tarfile.open(mode="w", fileobj=gzf) as tarfp: + print("Adding MacOSX SDK {} files...".format(sdk_version)) + tarfp_add_with_base_change(tarfp, sdk_dir, out_name) + print("Adding libc++ headers...") + tarfp_add_with_base_change(tarfp, xcode_libcxx_dir, "{}/usr/include/c++/v1".format(out_name)) + print("Done! Find the resulting gzipped tarball at:") + print(out_sdktgz_path.resolve()) + +if __name__ == '__main__': + run() diff --git a/contrib/testgen/README.md b/contrib/testgen/README.md index 573a71a675..eaca473b40 100644 --- a/contrib/testgen/README.md +++ b/contrib/testgen/README.md @@ -4,5 +4,5 @@ Utilities to generate test vectors for the data-driven Bitcoin tests. Usage: - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 50 > ../../src/test/data/key_io_keys_valid.json - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 50 > ../../src/test/data/key_io_keys_invalid.json + PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 50 > ../../src/test/data/key_io_valid.json + PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 50 > ../../src/test/data/key_io_invalid.json diff --git a/contrib/testgen/base58.py b/contrib/testgen/base58.py index da67cb2d90..c7ebac50d4 100644 --- a/contrib/testgen/base58.py +++ b/contrib/testgen/base58.py @@ -107,7 +107,7 @@ def get_bcaddress_version(strAddress): if __name__ == '__main__': # Test case (from http://gitorious.org/bitcoin/python-base58.git) - assert get_bcaddress_version('15VjRaDX9zpbA8LVnbrCAFzrVzN7ixHNsC') is 0 + assert get_bcaddress_version('15VjRaDX9zpbA8LVnbrCAFzrVzN7ixHNsC') == 0 _ohai = 'o hai'.encode('ascii') _tmp = b58encode(_ohai) assert _tmp == 'DYB3oMS' diff --git a/depends/Makefile b/depends/Makefile index 3d0784cb6b..2bc5df974a 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -125,6 +125,11 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) +ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +build_id_string+=system_clang +$(host_arch)_$(host_os)_id_string+=system_clang +endif + qrencode_packages_$(NO_QR) = $(qrencode_packages) qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) $(qrencode_packages_) @@ -150,11 +155,17 @@ all_packages = $(packages) $(native_packages) meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk +$(host_arch)_$(host_os)_native_binutils?=$($(host_os)_native_binutils) $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) include funcs.mk +binutils_path=$($($(host_arch)_$(host_os)_native_binutils)_prefixbin) +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) +else +toolchain_path= +endif final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) @@ -170,10 +181,10 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ $(AT)sed -e 's|@HOST@|$(host)|' \ -e 's|@CC@|$(toolchain_path)$(host_CC)|' \ -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \ - -e 's|@AR@|$(toolchain_path)$(host_AR)|' \ - -e 's|@RANLIB@|$(toolchain_path)$(host_RANLIB)|' \ - -e 's|@NM@|$(toolchain_path)$(host_NM)|' \ - -e 's|@STRIP@|$(toolchain_path)$(host_STRIP)|' \ + -e 's|@AR@|$(binutils_path)$(host_AR)|' \ + -e 's|@RANLIB@|$(binutils_path)$(host_RANLIB)|' \ + -e 's|@NM@|$(binutils_path)$(host_NM)|' \ + -e 's|@STRIP@|$(binutils_path)$(host_STRIP)|' \ -e 's|@build_os@|$(build_os)|' \ -e 's|@host_os@|$(host_os)|' \ -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \ diff --git a/depends/README.md b/depends/README.md index c12ea8bcb3..11733024b1 100644 --- a/depends/README.md +++ b/depends/README.md @@ -44,7 +44,7 @@ The paths are automatically configured and no other options are needed unless ta #### For macOS cross compilation - sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python3-setuptools + sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python3-setuptools libtinfo5 #### For Win64 cross compilation @@ -80,21 +80,40 @@ For linux S390X cross compilation: sudo apt-get install g++-s390x-linux-gnu binutils-s390x-linux-gnu ### Dependency Options -The following can be set when running make: make FOO=bar - - SOURCES_PATH: downloaded sources will be placed here - BASE_CACHE: built packages will be placed here - SDK_PATH: Path where sdk's can be found (used by macOS) - FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up - NO_QT: Don't download/build/cache qt and its dependencies - NO_QR: Don't download/build/cache packages needed for enabling qrencode - NO_ZMQ: Don't download/build/cache packages needed for enabling zeromq - NO_WALLET: Don't download/build/cache libs needed to enable the wallet - NO_UPNP: Don't download/build/cache packages needed for enabling upnp - MULTIPROCESS: build libmultiprocess (experimental, requires cmake) - DEBUG: disable some optimizations and enable more runtime checking - HOST_ID_SALT: Optional salt to use when generating host package ids - BUILD_ID_SALT: Optional salt to use when generating build package ids +The following can be set when running make: `make FOO=bar` + +<dl> +<dt>SOURCES_PATH</dt> +<dd>downloaded sources will be placed here</dd> +<dt>BASE_CACHE</dt> +<dd>built packages will be placed here</dd> +<dt>SDK_PATH</dt> +<dd>Path where sdk's can be found (used by macOS)</dd> +<dt>FALLBACK_DOWNLOAD_PATH</dt> +<dd>If a source file can't be fetched, try here before giving up</dd> +<dt>NO_QT</dt> +<dd>Don't download/build/cache qt and its dependencies</dd> +<dt>NO_QR</dt> +<dd>Don't download/build/cache packages needed for enabling qrencode</dd> +<dt>NO_ZMQ</dt> +<dd>Don't download/build/cache packages needed for enabling zeromq</dd> +<dt>NO_WALLET</dt> +<dd>Don't download/build/cache libs needed to enable the wallet</dd> +<dt>NO_UPNP</dt> +<dd>Don't download/build/cache packages needed for enabling upnp</dd> +<dt>MULTIPROCESS</dt> +<dd>build libmultiprocess (experimental, requires cmake)</dd> +<dt>DEBUG</dt> +<dd>disable some optimizations and enable more runtime checking</dd> +<dt>HOST_ID_SALT</dt> +<dd>Optional salt to use when generating host package ids</dd> +<dt>BUILD_ID_SALT</dt> +<dd>Optional salt to use when generating build package ids</dd> +<dt>FORCE_USE_SYSTEM_CLANG</dt> +<dd>(EXPERTS ONLY) When cross-compiling for macOS, use clang found in the +system's <code>$PATH</code> rather than the default prebuilt release of clang +from llvm.org</dd> +</dl> If some packages are not built, for example `make NO_WALLET=1`, the appropriate options will be passed to bitcoin's configure. In this case, `--disable-wallet`. diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index 69c394ec1d..f4103fc1f2 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -19,4 +19,5 @@ darwin_LIBTOOL:=$(shell xcrun -f libtool) darwin_OTOOL:=$(shell xcrun -f otool) darwin_NM:=$(shell xcrun -f nm) darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) +darwin_native_binutils= darwin_native_toolchain= diff --git a/depends/config.site.in b/depends/config.site.in index 103aa0d2ff..87a3379303 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -49,15 +49,6 @@ if test x@host_os@ = xdarwin; then PORT=no fi -if test x@host_os@ = xmingw32; then - if test -z $with_qt_incdir; then - with_qt_incdir=$depends_prefix/include - fi - if test -z $with_qt_libdir; then - with_qt_libdir=$depends_prefix/lib - fi -fi - PATH=$depends_prefix/native/bin:$PATH PKG_CONFIG="`which pkg-config` --static" diff --git a/depends/funcs.mk b/depends/funcs.mk index 135ebba9f8..6fc20543bb 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -41,7 +41,7 @@ endef define int_get_build_id $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies)) -$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies))) +$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($($(1)_type)_native_binutils) $($(1)_dependencies))) $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash))) $(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id_string)) $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))) @@ -260,4 +260,4 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$ $(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package)))) #special exception: if a toolchain package exists, all non-native packages depend on it -$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) )) +$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) $($($(host_arch)_$(host_os)_native_binutils)_cached) )) diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 82e086a326..6099fd4c71 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,8 +1,35 @@ OSX_MIN_VERSION=10.12 -OSX_SDK_VERSION=10.14 -OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk -darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ +OSX_SDK_VERSION=10.15.1 +XCODE_VERSION=11.3.1 +XCODE_BUILD_ID=11C505 +LD64_VERSION=530 + +OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers + +# Flag explanations: +# +# -mlinker-version +# +# Ensures that modern linker features are enabled. See here for more +# details: https://github.com/bitcoin/bitcoin/pull/19407. +# +# -B$(build_prefix)/bin +# +# Explicitly point to our binaries (e.g. cctools) so that they are +# ensured to be found and preferred over other possibilities. +# +# -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1 +# +# Forces clang to use the libc++ headers from our SDK and completely +# forget about the libc++ headers from the standard directories +# +# TODO: Once we start requiring a clang version that has the +# -stdlib++-isystem<directory> flag first introduced here: +# https://reviews.llvm.org/D64089, we should use that instead. Read the +# differential summary there for more details. +# +darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin +darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1 darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS) @@ -13,5 +40,11 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) darwin_debug_CFLAGS=-O1 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) +darwin_native_binutils=native_cctools +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) darwin_native_toolchain=native_cctools +else +darwin_native_toolchain= +endif + darwin_cmake_system=Darwin diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index eb45e14f6f..1cd5a1749a 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -3,17 +3,23 @@ $(package)_version=2.1.11-stable $(package)_download_path=https://github.com/libevent/libevent/archive/ $(package)_file_name=release-$($(package)_version).tar.gz $(package)_sha256_hash=229393ab2bf0dc94694f21836846b424f3532585bac3468738b7bf752c03901e +$(package)_patches=0001-fix-windows-getaddrinfo.patch define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/0001-fix-windows-getaddrinfo.patch && \ ./autogen.sh endef +# When building for Windows, we set _WIN32_WINNT to target the same Windows +# version as we do in configure. Due to quirks in libevents build system, this +# is also required to enable support for ipv6. See #19375. define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress --disable-samples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_release=--disable-debug-mode $(package)_config_opts_linux=--with-pic $(package)_config_opts_android=--with-pic + $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601 endef define $(package)_config_cmds diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 4195230b40..5022ed980f 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -1,14 +1,16 @@ package=native_cctools -$(package)_version=3764b223c011574971ee3ae09ce968ba5dc2f00f +$(package)_version=55562e4073dea0fbfd0b20e0bf69ffe6390c7f97 $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=3e35907bf376269a844df08e03cbb43e345c88125374f2228e03724b5f9a2a04 +$(package)_sha256_hash=e51995a843533a3dac155dd0c71362dd471597a2d23f13dff194c6285362f875 $(package)_build_subdir=cctools -$(package)_clang_version=6.0.1 +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +$(package)_clang_version=8.0.0 $(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version) $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz -$(package)_clang_sha256_hash=fa5416553ca94a8c071a27134c094a5fb736fe1bd0ecc5ef2d9bc02754e1bef0 +$(package)_clang_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0 +endif $(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef $(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive @@ -16,15 +18,25 @@ $(package)_libtapi_download_file=$($(package)_libtapi_version).tar.gz $(package)_libtapi_file_name=$($(package)_libtapi_version).tar.gz $(package)_libtapi_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3 -$(package)_extra_sources=$($(package)_clang_file_name) -$(package)_extra_sources += $($(package)_libtapi_file_name) +$(package)_extra_sources=$($(package)_libtapi_file_name) +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +$(package)_extra_sources += $($(package)_clang_file_name) +endif +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) define $(package)_fetch_cmds $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ $(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \ $(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash)) endef +else +define $(package)_fetch_cmds +$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash)) +endef +endif +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) define $(package)_extract_cmds mkdir -p $($(package)_extract_dir) && \ echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ @@ -38,12 +50,29 @@ define $(package)_extract_cmds rm -f toolchain/lib/libc++abi.so* && \ tar --no-same-owner --strip-components=1 -xf $($(package)_source) endef +else +define $(package)_extract_cmds + mkdir -p $($(package)_extract_dir) && \ + echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + echo "$($(package)_libtapi_sha256_hash) $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + mkdir -p libtapi && \ + tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \ + tar --no-same-owner --strip-components=1 -xf $($(package)_source) +endef +endif define $(package)_set_vars - $(package)_config_opts=--target=$(host) --disable-lto-support --with-libtapi=$($(package)_extract_dir) + $(package)_config_opts=--target=$(host) --with-libtapi=$($(package)_extract_dir) $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib + ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) + $(package)_config_opts+=--enable-lto-support --with-llvm-config=$($(package)_extract_dir)/toolchain/bin/llvm-config $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++ + else + $(package)_cc=clang + $(package)_cxx=clang++ + endif endef define $(package)_preprocess_cmds @@ -60,6 +89,7 @@ define $(package)_build_cmds $(MAKE) endef +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install && \ mkdir -p $($(package)_staging_prefix_dir)/lib/ && \ @@ -72,7 +102,13 @@ define $(package)_stage_cmds cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\ cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \ - cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ - if `test -d include/c++/`; then cp -rf include/c++/ $($(package)_staging_prefix_dir)/include/; fi && \ - if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi + cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil +endef +else +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install && \ + mkdir -p $($(package)_staging_prefix_dir)/lib/ && \ + cd $($(package)_extract_dir) && \ + cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/ endef +endif diff --git a/depends/packages/native_cdrkit.mk b/depends/packages/native_cdrkit.mk index 8243458ec8..14c37a0fc7 100644 --- a/depends/packages/native_cdrkit.mk +++ b/depends/packages/native_cdrkit.mk @@ -9,8 +9,10 @@ define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/cdrkit-deterministic.patch endef +# Starting with 10.1, GCC defaults to -fno-common, resulting in linking errors. +# Pass -fcommon to retain the legacy behaviour. define $(package)_config_cmds - cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix) + cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix) -DCMAKE_C_FLAGS="-fcommon" endef define $(package)_build_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 631851855a..500881e442 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -80,7 +80,6 @@ $(package)_config_opts += -no-feature-colordialog $(package)_config_opts += -no-feature-commandlineparser $(package)_config_opts += -no-feature-concurrent $(package)_config_opts += -no-feature-dial -$(package)_config_opts += -no-feature-filesystemwatcher $(package)_config_opts += -no-feature-fontcombobox $(package)_config_opts += -no-feature-ftp $(package)_config_opts += -no-feature-http diff --git a/depends/patches/libevent/0001-fix-windows-getaddrinfo.patch b/depends/patches/libevent/0001-fix-windows-getaddrinfo.patch new file mode 100644 index 0000000000..a98cd90bd5 --- /dev/null +++ b/depends/patches/libevent/0001-fix-windows-getaddrinfo.patch @@ -0,0 +1,15 @@ +diff -ur libevent-2.1.8-stable.orig/configure.ac libevent-2.1.8-stable/configure.ac +--- libevent-2.1.8-stable.orig/configure.ac 2017-01-29 17:51:00.000000000 +0000 ++++ libevent-2.1.8-stable/configure.ac 2020-03-07 01:11:16.311335005 +0000 +@@ -389,6 +389,10 @@ + #ifdef HAVE_NETDB_H + #include <netdb.h> + #endif ++#ifdef _WIN32 ++#include <winsock2.h> ++#include <ws2tcpip.h> ++#endif + ]], + [[ + getaddrinfo; +Only in libevent-2.1.8-stable: configure.ac~ diff --git a/depends/patches/qt/fix_qt_pkgconfig.patch b/depends/patches/qt/fix_qt_pkgconfig.patch index 34302a9f2d..8c722ffb46 100644 --- a/depends/patches/qt/fix_qt_pkgconfig.patch +++ b/depends/patches/qt/fix_qt_pkgconfig.patch @@ -1,11 +1,23 @@ --- old/qtbase/mkspecs/features/qt_module.prf +++ new/qtbase/mkspecs/features/qt_module.prf -@@ -245,7 +245,7 @@ +@@ -264,7 +264,7 @@ load(qt_targets) # this builds on top of qt_common -!internal_module:!lib_bundle:if(unix|mingw) { -+unix|mingw { ++if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { CONFIG += create_pc QMAKE_PKGCONFIG_DESTDIR = pkgconfig host_build: \ +@@ -274,9 +274,9 @@ + QMAKE_PKGCONFIG_INCDIR = $$[QT_INSTALL_HEADERS/raw] + QMAKE_PKGCONFIG_CFLAGS = -I${includedir}/$$MODULE_INCNAME + QMAKE_PKGCONFIG_NAME = $$replace(TARGET, ^Qt, "Qt$$QT_MAJOR_VERSION ") +- QMAKE_PKGCONFIG_FILE = $$replace(TARGET, ^Qt, Qt$$QT_MAJOR_VERSION) ++ QMAKE_PKGCONFIG_FILE = $$replace(TARGET, ^Qt, Qt$$QT_MAJOR_VERSION)$$qtPlatformTargetSuffix() + for(i, MODULE_DEPENDS): \ +- QMAKE_PKGCONFIG_REQUIRES += $$replace(QT.$${i}.name, ^Qt, Qt$$section(QT.$${i}.VERSION, ., 0, 0)) ++ QMAKE_PKGCONFIG_REQUIRES += $$replace(QT.$${i}.name, ^Qt, Qt$$section(QT.$${i}.VERSION, ., 0, 0))$$qtPlatformTargetSuffix() + isEmpty(QMAKE_PKGCONFIG_DESCRIPTION): \ + QMAKE_PKGCONFIG_DESCRIPTION = $$replace(TARGET, ^Qt, "Qt ") module + pclib_replace.match = $$lib_replace.match diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 11e9c90f49..842a3964df 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -61,7 +61,6 @@ Only supports JSON as output format. * pruned : (boolean) if the blocks are subject to pruning * pruneheight : (numeric) highest block available * softforks : (array) status of softforks in progress -* bip9_softforks : (object) status of BIP9 softforks in progress #### Query UTXO set `GET /rest/getutxos/<checkmempool>/<txid>-<n>/<txid>-<n>/.../<txid>-<n>.<bin|hex|json>` @@ -79,7 +78,6 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 "bitmap": "1", "utxos" : [ { - "txvers" : 1 "height" : 2147483647, "value" : 8.8687, "scriptPubKey" : { diff --git a/doc/build-windows.md b/doc/build-windows.md index d3dc467f19..28b6aceb3c 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -91,15 +91,22 @@ Note that for WSL the Bitcoin Core source path MUST be somewhere in the default example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. This means you cannot use a directory that is located directly on the host Windows file system to perform the build. +Additional WSL Note: WSL support for [launching Win32 applications](https://docs.microsoft.com/en-us/archive/blogs/wsl/windows-and-ubuntu-interoperability#launching-win32-applications-from-within-wsl) +results in `Autoconf` configure scripts being able to execute Windows Portable Executable files. This can cause +unexpected behaviour during the build, such as Win32 error dialogs for missing libraries. The recommended approach +is to temporarily disable WSL support for Win32 applications. + Build using: PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var + sudo bash -c "echo 0 > /proc/sys/fs/binfmt_misc/status" # Disable WSL support for Win32 applications. cd depends make HOST=x86_64-w64-mingw32 cd .. ./autogen.sh # not required when building from tarball CONFIG_SITE=$PWD/depends/x86_64-w64-mingw32/share/config.site ./configure --prefix=/ make + sudo bash -c "echo 1 > /proc/sys/fs/binfmt_misc/status" # Enable WSL support for Win32 applications. ## Depends system diff --git a/doc/developer-notes.md b/doc/developer-notes.md index b33b3ad18a..6ae7e770e8 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -620,6 +620,19 @@ class A - *Rationale*: Easier to understand what is happening, thus easier to spot mistakes, even for those that are not language lawyers. +- Use `Span` as function argument when it can operate on any range-like container. + + - *Rationale*: Compared to `Foo(const vector<int>&)` this avoids the need for a (potentially expensive) + conversion to vector if the caller happens to have the input stored in another type of container. + However, be aware of the pitfalls documented in [span.h](../src/span.h). + +```cpp +void Foo(Span<const int> data); + +std::vector<int> vec{1,2,3}; +Foo(vec); +``` + - Prefer `enum class` (scoped enumerations) over `enum` (traditional enumerations) where possible. - *Rationale*: Scoped enumerations avoid two potential pitfalls/problems with traditional C++ enumerations: implicit conversions to `int`, and name clashes due to enumerators being exported to the surrounding scope. @@ -874,7 +887,7 @@ Others are external projects without a tight relationship with our project. Chan be sent upstream, but bugfixes may also be prudent to PR against Bitcoin Core so that they can be integrated quickly. Cosmetic changes should be purely taken upstream. -There is a tool in `test/lint/git-subtree-check.sh` to check a subtree directory for consistency with +There is a tool in `test/lint/git-subtree-check.sh` ([instructions](../test/lint#git-subtree-checksh)) to check a subtree directory for consistency with its upstream repository. Current subtrees include: diff --git a/doc/files.md b/doc/files.md index cd23d547bb..5475826329 100644 --- a/doc/files.md +++ b/doc/files.md @@ -50,7 +50,7 @@ Subdirectory | File(s) | Description `indexes/blockfilter/basic/` | `fltrNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Blockfilter index filters for the basic filtertype; *optional*, used if `-blockfilterindex=basic` `wallets/` | | [Contains wallets](#multi-wallet-environment); can be specified by `-walletdir` option; if `wallets/` subdirectory does not exist, a wallet resides in the data directory `./` | `banlist.dat` | Stores the IPs/subnets of banned nodes -`./` | `bitcoin.conf` | Contains [configuration settings](bitcoin-conf.md) for `bitcoind` or `bitcoin-qt`; can be specified by `-conf` option +`./` | `bitcoin.conf` | User-defined [configuration settings](bitcoin-conf.md) for `bitcoind` or `bitcoin-qt`. File is not written to by the software and must be created manually. Path can be specified by `-conf` option `./` | `bitcoind.pid` | Stores the process ID (PID) of `bitcoind` or `bitcoin-qt` while running; created at start and deleted on shutdown; can be specified by `-pid` option `./` | `debug.log` | Contains debug information and general logging generated by `bitcoind` or `bitcoin-qt`; can be specified by `-debuglogfile` option `./` | `fee_estimates.dat` | Stores statistics used to estimate minimum transaction fees and priorities required for confirmation @@ -58,6 +58,7 @@ Subdirectory | File(s) | Description `./` | `mempool.dat` | Dump of the mempool's transactions `./` | `onion_private_key` | Cached Tor hidden service private key for `-listenonion` option `./` | `peers.dat` | Peer IP address database (custom format) +`./` | `settings.json` | Read-write settings set through GUI or RPC interfaces, augmenting manual settings from [bitcoin.conf](bitcoin-conf.md). File is created automatically if read-write settings storage is not disabled with `-nosettings` option. Path can be specified with `-settings` option `./` | `.cookie` | Session RPC authentication cookie; if used, created at start and deleted on shutdown; can be specified by `-rpccookiefile` option `./` | `.lock` | Data directory lock file diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 419b1db44e..c97b8d4d50 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -121,6 +121,8 @@ $ git clone https://github.com/google/afl $ make -C afl/ $ make -C afl/llvm_mode/ $ ./autogen.sh +# It is possible to compile with afl-gcc and afl-g++ instead of afl-clang. However, running afl-fuzz +# may require more memory via the -m flag. $ CC=$(pwd)/afl/afl-clang-fast CXX=$(pwd)/afl/afl-clang-fast++ ./configure --enable-fuzz $ make # For macOS you may need to ignore x86 compilation checks when running "make". If so, diff --git a/doc/reduce-traffic.md b/doc/reduce-traffic.md index ce77a00dd5..86943b1f72 100644 --- a/doc/reduce-traffic.md +++ b/doc/reduce-traffic.md @@ -23,7 +23,7 @@ longer serving historic blocks (blocks older than one week). Keep in mind that new nodes require other nodes that are willing to serve historic blocks. -Peers with the `noban` permission will never be disconnected, although their traffic counts for +Peers with the `download` permission will never be disconnected, although their traffic counts for calculating the target. ## 2. Disable "listening" (`-listen=0`) @@ -50,7 +50,7 @@ Be reminded of the effects of this setting. Doing so disables the automatic broadcasting of transactions from wallet. Not relaying other's transactions could hurt your privacy if used while a wallet is loaded or if you use the node to broadcast transactions. -- If a peer is whitelisted and "-whitelistforcerelay" is set to "1" (which will - also set "whitelistrelay" to "1"), we will still receive and relay their transactions. +- If a peer has the forcerelay permission, we will still receive and relay + their transactions. - It makes block propagation slower because compact block relay can only be used when transaction relay is enabled. diff --git a/doc/release-notes-16525.md b/doc/release-notes-16525.md new file mode 100644 index 0000000000..220cb78de4 --- /dev/null +++ b/doc/release-notes-16525.md @@ -0,0 +1,9 @@ +RPC changes +----------- + +Exposed transaction version numbers are now treated as unsigned 32-bit integers +instead of signed 32-bit integers. This matches their treatment in consensus +logic. Versions greater than 2 continue to be non-standard (matching previous +behavior of smaller than 1 or greater than 2 being non-standard). Note that +this includes the joinpsbt command, which combines partially-signed +transactions by selecting the highest version number. diff --git a/doc/release-notes-16528.md b/doc/release-notes-16528.md deleted file mode 100644 index e69de29bb2..0000000000 --- a/doc/release-notes-16528.md +++ /dev/null diff --git a/doc/release-notes-18918.md b/doc/release-notes-18918.md deleted file mode 100644 index e69de29bb2..0000000000 --- a/doc/release-notes-18918.md +++ /dev/null diff --git a/doc/release-notes.md b/doc/release-notes.md index d9d0ecd631..23983dcd7b 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -45,6 +45,11 @@ wallet versions of Bitcoin Core are generally supported. Compatibility ============== +During this release cycle, work has been done to ensure that the codebase is fully +compatible with C++17. The intention is to begin using C++17 features starting +with the 0.22.0 release. This means that a compiler that supports C++17 will be +required to compile 0.22.0. + Bitcoin Core is supported and extensively tested on operating systems using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin Core should also work on most other Unix-like systems but is not as @@ -82,6 +87,28 @@ Updated RPCs whether initial broadcast of the transaction has been acknowledged by a peer. `getmempoolancestors` and `getmempooldescendants` are also updated. +- The `bumpfee`, `fundrawtransaction`, `sendmany`, `sendtoaddress`, and `walletcreatefundedpsbt` +RPC commands have been updated to include two new fee estimation methods "BTC/kB" and "sat/B". +The target is the fee expressed explicitly in the given form. Note that use of this feature +will trigger BIP 125 (replace-by-fee) opt-in. (#11413) + +- In addition, the `estimate_mode` parameter is now case insensitive for all of + the above RPC commands. (#11413) + +- The `bumpfee` command now uses `conf_target` rather than `confTarget` in the + options. (#11413) + +- The `getpeerinfo` RPC no longer returns the `banscore` field unless the configuration + option `-deprecatedrpc=banscore` is used. The `banscore` field will be fully + removed in the next major release. (#19469) + +- The `walletcreatefundedpsbt` RPC call will now fail with + `Insufficient funds` when inputs are manually selected but are not enough to cover + the outputs and fee. Additional inputs can automatically be added through the + new `add_inputs` option. (#16377) + +- The `fundrawtransaction` RPC now supports `add_inputs` option that when `false` + prevents adding more inputs if necessary and consequently the RPC fails. Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section below. @@ -94,17 +121,46 @@ Build System Updated settings ---------------- +- The `-banscore` configuration option, which modified the default threshold for + disconnecting and discouraging misbehaving peers, has been removed as part of + changes in 0.20.1 and in this release to the handling of misbehaving peers. + Refer to "Changes regarding misbehaving peers" in the 0.20.1 release notes for + details. (#19464) + - The `-debug=db` logging category, which was deprecated in 0.20 and replaced by `-debug=walletdb` to distinguish it from `coindb`, has been removed. (#19202) +- A `download` permission has been extracted from the `noban` permission. For + compatibility, `noban` implies the `download` permission, but this may change + in future releases. Refer to the help of the affected settings `-whitebind` + and `-whitelist` for more details. (#19191) + Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. +Tools and Utilities +------------------- + +- A new `bitcoin-cli -generate` command, equivalent to RPC `generatenewaddress` + followed by `generatetoaddress`, can generate blocks for command line testing + purposes. This is a client-side version of the + former `generate` RPC. See the help for details. (#19133) + +- The `bitcoin-cli -getinfo` command now displays the wallet name and balance for + each of the loaded wallets when more than one is loaded (e.g. in multiwallet + mode) and a wallet is not specified with `-rpcwallet`. (#18594) + New settings ------------ Wallet ------ +- Backwards compatibility has been dropped for two `getaddressinfo` RPC + deprecations, as notified in the 0.20 release notes. The deprecated `label` + field has been removed as well as the deprecated `labels` behavior of + returning a JSON object containing `name` and `purpose` key-value pairs. Since + 0.20, the `labels` field returns a JSON array of label names. (#19200) + - To improve wallet privacy, the frequency of wallet rebroadcast attempts is reduced from approximately once every 15 minutes to once every 12-36 hours. To maintain a similar level of guarantee for initial broadcast of wallet @@ -248,9 +304,23 @@ issue. GUI changes ----------- +- The GUI Peers window no longer displays a "Ban Score" field. This is part of + changes in 0.20.1 and in this release to the handling of misbehaving + peers. Refer to "Changes regarding misbehaving peers" in the 0.20.1 release + notes for details. (#19512) + Low-level changes ================= +RPC +--- + +- To make RPC `sendtoaddress` more consistent with `sendmany` the following error + `sendtoaddress` codes were changed from `-4` to `-6`: + - Insufficient funds + - Fee estimation failed + - Transaction has too long of a mempool chain + Tests ----- diff --git a/share/examples/bitcoin.conf b/share/examples/bitcoin.conf index 96fb6658a0..90a592cc63 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/bitcoin.conf @@ -20,8 +20,8 @@ # Bind to given address and always listen on it. Use [host]:port notation for IPv6 #bind=<addr> -# Bind to given address and whitelist peers connecting to it. Use [host]:port notation for IPv6 -#whitebind=<addr> +# Bind to given address and add permission flags to peers connecting to it. Use [host]:port notation for IPv6 +#whitebind=perm@<addr> ############################################################## ## Quick Primer on addnode vs connect ## diff --git a/src/.clang-format b/src/.clang-format index aae039dd77..ef7a0ef5c7 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -1,9 +1,9 @@ Language: Cpp AccessModifierOffset: -4 -AlignAfterOpenBracket: false +AlignAfterOpenBracket: true AlignEscapedNewlinesLeft: true AlignTrailingComments: true -AllowAllParametersOfDeclarationOnNextLine: false +AllowAllParametersOfDeclarationOnNextLine: true AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: true AllowShortFunctionsOnASingleLine: All diff --git a/src/Makefile.am b/src/Makefile.am index 7a280a67a7..cd3cc95707 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -110,9 +110,9 @@ BITCOIN_CORE_H = \ banman.h \ base58.h \ bech32.h \ - bloom.h \ blockencodings.h \ blockfilter.h \ + bloom.h \ chain.h \ chainparams.h \ chainparamsbase.h \ @@ -133,6 +133,7 @@ BITCOIN_CORE_H = \ core_io.h \ core_memusage.h \ cuckoocache.h \ + dbwrapper.h \ flatfile.h \ fs.h \ httprpc.h \ @@ -148,7 +149,6 @@ BITCOIN_CORE_H = \ interfaces/wallet.h \ key.h \ key_io.h \ - dbwrapper.h \ limitedmap.h \ logging.h \ logging/timer.h \ @@ -167,6 +167,7 @@ BITCOIN_CORE_H = \ node/context.h \ node/psbt.h \ node/transaction.h \ + node/ui_interface.h \ node/utxo_snapshot.h \ noui.h \ optional.h \ @@ -184,6 +185,7 @@ BITCOIN_CORE_H = \ reverse_iterator.h \ rpc/blockchain.h \ rpc/client.h \ + rpc/mining.h \ rpc/protocol.h \ rpc/rawtransaction_util.h \ rpc/register.h \ @@ -205,13 +207,12 @@ BITCOIN_CORE_H = \ support/events.h \ support/lockedpool.h \ sync.h \ - threadsafety.h \ threadinterrupt.h \ + threadsafety.h \ timedata.h \ torcontrol.h \ txdb.h \ txmempool.h \ - ui_interface.h \ undo.h \ util/asmap.h \ util/bip32.h \ @@ -220,8 +221,6 @@ BITCOIN_CORE_H = \ util/error.h \ util/fees.h \ util/golombrice.h \ - util/spanparsing.h \ - util/system.h \ util/macros.h \ util/memory.h \ util/message.h \ @@ -229,18 +228,22 @@ BITCOIN_CORE_H = \ util/rbf.h \ util/ref.h \ util/settings.h \ + util/spanparsing.h \ util/string.h \ + util/system.h \ util/threadnames.h \ util/time.h \ util/translation.h \ + util/ui_change_type.h \ util/url.h \ util/vector.h \ validation.h \ validationinterface.h \ versionbits.h \ versionbitsinfo.h \ - walletinitinterface.h \ + wallet/bdb.h \ wallet/coincontrol.h \ + wallet/coinselection.h \ wallet/context.h \ wallet/crypter.h \ wallet/db.h \ @@ -255,7 +258,7 @@ BITCOIN_CORE_H = \ wallet/walletdb.h \ wallet/wallettool.h \ wallet/walletutil.h \ - wallet/coinselection.h \ + walletinitinterface.h \ warnings.h \ zmq/zmqabstractnotifier.h \ zmq/zmqconfig.h\ @@ -284,16 +287,16 @@ libbitcoin_server_a_SOURCES = \ blockfilter.cpp \ chain.cpp \ consensus/tx_verify.cpp \ + dbwrapper.cpp \ flatfile.cpp \ httprpc.cpp \ httpserver.cpp \ index/base.cpp \ index/blockfilterindex.cpp \ index/txindex.cpp \ + init.cpp \ interfaces/chain.cpp \ interfaces/node.cpp \ - init.cpp \ - dbwrapper.cpp \ miner.cpp \ net.cpp \ net_processing.cpp \ @@ -302,6 +305,7 @@ libbitcoin_server_a_SOURCES = \ node/context.cpp \ node/psbt.cpp \ node/transaction.cpp \ + node/ui_interface.cpp \ noui.cpp \ policy/fees.cpp \ policy/rbf.cpp \ @@ -320,7 +324,6 @@ libbitcoin_server_a_SOURCES = \ torcontrol.cpp \ txdb.cpp \ txmempool.cpp \ - ui_interface.cpp \ validation.cpp \ validationinterface.cpp \ versionbits.cpp \ @@ -350,6 +353,7 @@ libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ interfaces/wallet.cpp \ + wallet/bdb.cpp \ wallet/coincontrol.cpp \ wallet/context.cpp \ wallet/crypter.cpp \ diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 13bfea7646..848053e841 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -25,6 +25,7 @@ QT_FORMS_UI = \ qt/forms/openuridialog.ui \ qt/forms/optionsdialog.ui \ qt/forms/overviewpage.ui \ + qt/forms/psbtoperationsdialog.ui \ qt/forms/receivecoinsdialog.ui \ qt/forms/receiverequestdialog.ui \ qt/forms/debugwindow.ui \ @@ -61,6 +62,7 @@ QT_MOC_CPP = \ qt/moc_overviewpage.cpp \ qt/moc_peertablemodel.cpp \ qt/moc_paymentserver.cpp \ + qt/moc_psbtoperationsdialog.cpp \ qt/moc_qrimagewidget.cpp \ qt/moc_qvalidatedlineedit.cpp \ qt/moc_qvaluecombobox.cpp \ @@ -132,6 +134,7 @@ BITCOIN_QT_H = \ qt/paymentserver.h \ qt/peertablemodel.h \ qt/platformstyle.h \ + qt/psbtoperationsdialog.h \ qt/qrimagewidget.h \ qt/qvalidatedlineedit.h \ qt/qvaluecombobox.h \ @@ -245,6 +248,7 @@ BITCOIN_QT_WALLET_CPP = \ qt/openuridialog.cpp \ qt/overviewpage.cpp \ qt/paymentserver.cpp \ + qt/psbtoperationsdialog.cpp \ qt/qrimagewidget.cpp \ qt/receivecoinsdialog.cpp \ qt/receiverequestdialog.cpp \ @@ -272,9 +276,7 @@ if ENABLE_WALLET BITCOIN_QT_CPP += $(BITCOIN_QT_WALLET_CPP) endif # ENABLE_WALLET -RES_IMAGES = - -RES_MOVIES = $(wildcard $(srcdir)/qt/res/movies/spinner-*.png) +RES_ANIMATION = $(wildcard $(srcdir)/qt/res/animation/spinner-*.png) BITCOIN_RC = qt/res/bitcoin-qt-res.rc @@ -286,7 +288,7 @@ qt_libbitcoinqt_a_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_OBJCXXFLAGS = $(AM_OBJCXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_SOURCES = $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(QT_FORMS_UI) \ - $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) + $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(RES_ICONS) $(RES_ANIMATION) if TARGET_DARWIN qt_libbitcoinqt_a_SOURCES += $(BITCOIN_MM) endif @@ -357,7 +359,7 @@ $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ @rm $(@D)/temp_$(<F) -$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) +$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_ICONS) $(RES_ANIMATION) @test -f $(RCC) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin $< | \ $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 75b1a14e36..b961f8d5b9 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -10,7 +10,9 @@ FUZZ_TARGETS = \ test/fuzz/addrman_deserialize \ test/fuzz/asmap \ test/fuzz/asmap_direct \ + test/fuzz/autofile \ test/fuzz/banentry_deserialize \ + test/fuzz/banman \ test/fuzz/base_encode_decode \ test/fuzz/bech32 \ test/fuzz/block \ @@ -28,11 +30,19 @@ FUZZ_TARGETS = \ test/fuzz/blockundo_deserialize \ test/fuzz/bloom_filter \ test/fuzz/bloomfilter_deserialize \ + test/fuzz/buffered_file \ test/fuzz/chain \ test/fuzz/checkqueue \ test/fuzz/coins_deserialize \ test/fuzz/coins_view \ + test/fuzz/crypto \ + test/fuzz/crypto_aes256 \ + test/fuzz/crypto_aes256cbc \ + test/fuzz/crypto_chacha20 \ + test/fuzz/crypto_chacha20_poly1305_aead \ test/fuzz/crypto_common \ + test/fuzz/crypto_hkdf_hmac_sha256_l32 \ + test/fuzz/crypto_poly1305 \ test/fuzz/cuckoocache \ test/fuzz/decode_tx \ test/fuzz/descriptor_parse \ @@ -53,6 +63,7 @@ FUZZ_TARGETS = \ test/fuzz/key_io \ test/fuzz/key_origin_info_deserialize \ test/fuzz/kitchen_sink \ + test/fuzz/load_external_block_file \ test/fuzz/locale \ test/fuzz/merkle_block_deserialize \ test/fuzz/merkleblock \ @@ -72,6 +83,7 @@ FUZZ_TARGETS = \ test/fuzz/partial_merkle_tree_deserialize \ test/fuzz/partially_signed_transaction_deserialize \ test/fuzz/policy_estimator \ + test/fuzz/policy_estimator_io \ test/fuzz/pow \ test/fuzz/prefilled_transaction_deserialize \ test/fuzz/prevector \ @@ -231,6 +243,7 @@ BITCOIN_TESTS =\ test/net_tests.cpp \ test/netbase_tests.cpp \ test/pmt_tests.cpp \ + test/policy_fee_tests.cpp \ test/policyestimator_tests.cpp \ test/pow_tests.cpp \ test/prevector_tests.cpp \ @@ -262,6 +275,7 @@ BITCOIN_TESTS =\ test/uint256_tests.cpp \ test/util_tests.cpp \ test/validation_block_tests.cpp \ + test/validation_chainstate_tests.cpp \ test/validation_chainstatemanager_tests.cpp \ test/validation_flush_tests.cpp \ test/validationinterface_tests.cpp \ @@ -347,12 +361,24 @@ test_fuzz_asmap_direct_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_asmap_direct_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_asmap_direct_SOURCES = test/fuzz/asmap_direct.cpp +test_fuzz_autofile_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_autofile_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_autofile_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_autofile_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_autofile_SOURCES = test/fuzz/autofile.cpp + test_fuzz_banentry_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBANENTRY_DESERIALIZE=1 test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_banentry_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_banman_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_banman_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_banman_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_banman_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_banman_SOURCES = test/fuzz/banman.cpp + test_fuzz_base_encode_decode_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_base_encode_decode_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_base_encode_decode_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -455,6 +481,12 @@ test_fuzz_bloomfilter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_bloomfilter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_bloomfilter_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_buffered_file_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_buffered_file_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_buffered_file_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_buffered_file_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_buffered_file_SOURCES = test/fuzz/buffered_file.cpp + test_fuzz_chain_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_chain_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_chain_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -479,12 +511,54 @@ test_fuzz_coins_view_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_coins_view_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_coins_view_SOURCES = test/fuzz/coins_view.cpp +test_fuzz_crypto_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_SOURCES = test/fuzz/crypto.cpp + +test_fuzz_crypto_aes256_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_aes256_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_aes256_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_aes256_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_aes256_SOURCES = test/fuzz/crypto_aes256.cpp + +test_fuzz_crypto_aes256cbc_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_aes256cbc_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_aes256cbc_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_aes256cbc_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_aes256cbc_SOURCES = test/fuzz/crypto_aes256cbc.cpp + +test_fuzz_crypto_chacha20_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_chacha20_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_chacha20_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_chacha20_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_chacha20_SOURCES = test/fuzz/crypto_chacha20.cpp + +test_fuzz_crypto_chacha20_poly1305_aead_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_chacha20_poly1305_aead_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_chacha20_poly1305_aead_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_chacha20_poly1305_aead_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_chacha20_poly1305_aead_SOURCES = test/fuzz/crypto_chacha20_poly1305_aead.cpp + test_fuzz_crypto_common_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_crypto_common_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_crypto_common_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_crypto_common_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_crypto_common_SOURCES = test/fuzz/crypto_common.cpp +test_fuzz_crypto_hkdf_hmac_sha256_l32_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_hkdf_hmac_sha256_l32_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_hkdf_hmac_sha256_l32_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_hkdf_hmac_sha256_l32_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_hkdf_hmac_sha256_l32_SOURCES = test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp + +test_fuzz_crypto_poly1305_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_poly1305_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_poly1305_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_poly1305_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_poly1305_SOURCES = test/fuzz/crypto_poly1305.cpp + test_fuzz_cuckoocache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_cuckoocache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_cuckoocache_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -605,6 +679,12 @@ test_fuzz_kitchen_sink_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_kitchen_sink_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_kitchen_sink_SOURCES = test/fuzz/kitchen_sink.cpp +test_fuzz_load_external_block_file_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_load_external_block_file_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_load_external_block_file_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_load_external_block_file_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_load_external_block_file_SOURCES = test/fuzz/load_external_block_file.cpp + test_fuzz_locale_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_locale_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_locale_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -725,6 +805,12 @@ test_fuzz_policy_estimator_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_policy_estimator_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_policy_estimator_SOURCES = test/fuzz/policy_estimator.cpp +test_fuzz_policy_estimator_io_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_policy_estimator_io_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_policy_estimator_io_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_policy_estimator_io_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_policy_estimator_io_SOURCES = test/fuzz/policy_estimator_io.cpp + test_fuzz_pow_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_pow_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_pow_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 835c5d6c65..f3e8a19de2 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -8,6 +8,7 @@ #include <addrman.h> #include <chainparams.h> #include <clientversion.h> +#include <cstdint> #include <hash.h> #include <random.h> #include <streams.h> @@ -36,7 +37,7 @@ template <typename Data> bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data) { // Generate random temporary filename - unsigned short randv = 0; + uint16_t randv = 0; GetRandBytes((unsigned char*)&randv, sizeof(randv)); std::string tmpfn = strprintf("%s.%04x", prefix, randv); diff --git a/src/addrdb.h b/src/addrdb.h index c6d4307d69..8410c3776c 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -17,13 +17,6 @@ class CSubNet; class CAddrMan; class CDataStream; -typedef enum BanReason -{ - BanReasonUnknown = 0, - BanReasonNodeMisbehaving = 1, - BanReasonManuallyAdded = 2 -} BanReason; - class CBanEntry { public: @@ -31,7 +24,6 @@ public: int nVersion; int64_t nCreateTime; int64_t nBanUntil; - uint8_t banReason; CBanEntry() { @@ -44,31 +36,17 @@ public: nCreateTime = nCreateTimeIn; } - explicit CBanEntry(int64_t n_create_time_in, BanReason ban_reason_in) : CBanEntry(n_create_time_in) + SERIALIZE_METHODS(CBanEntry, obj) { - banReason = ban_reason_in; + uint8_t ban_reason = 2; //! For backward compatibility + READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, ban_reason); } - SERIALIZE_METHODS(CBanEntry, obj) { READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, obj.banReason); } - void SetNull() { nVersion = CBanEntry::CURRENT_VERSION; nCreateTime = 0; nBanUntil = 0; - banReason = BanReasonUnknown; - } - - std::string banReasonToString() const - { - switch (banReason) { - case BanReasonNodeMisbehaving: - return "node misbehaving"; - case BanReasonManuallyAdded: - return "manually added"; - default: - return "unknown"; - } } }; diff --git a/src/banman.cpp b/src/banman.cpp index 9cc584f0e4..8752185a60 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -6,7 +6,7 @@ #include <banman.h> #include <netaddress.h> -#include <ui_interface.h> +#include <node/ui_interface.h> #include <util/system.h> #include <util/time.h> #include <util/translation.h> @@ -26,7 +26,7 @@ BanMan::BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t SweepBanned(); // sweep out unused entries LogPrint(BCLog::NET, "Loaded %d banned node ips/subnets from banlist.dat %dms\n", - banmap.size(), GetTimeMillis() - n_start); + m_banned.size(), GetTimeMillis() - n_start); } else { LogPrintf("Invalid or missing banlist.dat; recreating\n"); SetBannedSetDirty(true); // force write @@ -68,28 +68,13 @@ void BanMan::ClearBanned() if (m_client_interface) m_client_interface->BannedListChanged(); } -int BanMan::IsBannedLevel(CNetAddr net_addr) +bool BanMan::IsDiscouraged(const CNetAddr& net_addr) { - // Returns the most severe level of banning that applies to this address. - // 0 - Not banned - // 1 - Automatic misbehavior ban - // 2 - Any other ban - int level = 0; - auto current_time = GetTime(); LOCK(m_cs_banned); - for (const auto& it : m_banned) { - CSubNet sub_net = it.first; - CBanEntry ban_entry = it.second; - - if (current_time < ban_entry.nBanUntil && sub_net.Match(net_addr)) { - if (ban_entry.banReason != BanReasonNodeMisbehaving) return 2; - level = 1; - } - } - return level; + return m_discouraged.contains(net_addr.GetAddrBytes()); } -bool BanMan::IsBanned(CNetAddr net_addr) +bool BanMan::IsBanned(const CNetAddr& net_addr) { auto current_time = GetTime(); LOCK(m_cs_banned); @@ -104,7 +89,7 @@ bool BanMan::IsBanned(CNetAddr net_addr) return false; } -bool BanMan::IsBanned(CSubNet sub_net) +bool BanMan::IsBanned(const CSubNet& sub_net) { auto current_time = GetTime(); LOCK(m_cs_banned); @@ -118,15 +103,21 @@ bool BanMan::IsBanned(CSubNet sub_net) return false; } -void BanMan::Ban(const CNetAddr& net_addr, const BanReason& ban_reason, int64_t ban_time_offset, bool since_unix_epoch) +void BanMan::Ban(const CNetAddr& net_addr, int64_t ban_time_offset, bool since_unix_epoch) { CSubNet sub_net(net_addr); - Ban(sub_net, ban_reason, ban_time_offset, since_unix_epoch); + Ban(sub_net, ban_time_offset, since_unix_epoch); +} + +void BanMan::Discourage(const CNetAddr& net_addr) +{ + LOCK(m_cs_banned); + m_discouraged.insert(net_addr.GetAddrBytes()); } -void BanMan::Ban(const CSubNet& sub_net, const BanReason& ban_reason, int64_t ban_time_offset, bool since_unix_epoch) +void BanMan::Ban(const CSubNet& sub_net, int64_t ban_time_offset, bool since_unix_epoch) { - CBanEntry ban_entry(GetTime(), ban_reason); + CBanEntry ban_entry(GetTime()); int64_t normalized_ban_time_offset = ban_time_offset; bool normalized_since_unix_epoch = since_unix_epoch; @@ -146,8 +137,8 @@ void BanMan::Ban(const CSubNet& sub_net, const BanReason& ban_reason, int64_t ba } if (m_client_interface) m_client_interface->BannedListChanged(); - //store banlist to disk immediately if user requested ban - if (ban_reason == BanReasonManuallyAdded) DumpBanlist(); + //store banlist to disk immediately + DumpBanlist(); } bool BanMan::Unban(const CNetAddr& net_addr) diff --git a/src/banman.h b/src/banman.h index 6bea2e75e9..f6bfbd1e49 100644 --- a/src/banman.h +++ b/src/banman.h @@ -6,6 +6,7 @@ #define BITCOIN_BANMAN_H #include <addrdb.h> +#include <bloom.h> #include <fs.h> #include <net_types.h> // For banmap_t #include <sync.h> @@ -23,32 +24,55 @@ class CClientUIInterface; class CNetAddr; class CSubNet; -// Denial-of-service detection/prevention -// The idea is to detect peers that are behaving -// badly and disconnect/ban them, but do it in a -// one-coding-mistake-won't-shatter-the-entire-network -// way. -// IMPORTANT: There should be nothing I can give a -// node that it will forward on that will make that -// node's peers drop it. If there is, an attacker -// can isolate a node and/or try to split the network. -// Dropping a node for sending stuff that is invalid -// now but might be valid in a later version is also -// dangerous, because it can cause a network split -// between nodes running old code and nodes running -// new code. +// Banman manages two related but distinct concepts: +// +// 1. Banning. This is configured manually by the user, through the setban RPC. +// If an address or subnet is banned, we never accept incoming connections from +// it and never create outgoing connections to it. We won't gossip its address +// to other peers in addr messages. Banned addresses and subnets are stored to +// banlist.dat on shutdown and reloaded on startup. Banning can be used to +// prevent connections with spy nodes or other griefers. +// +// 2. Discouragement. If a peer misbehaves enough (see Misbehaving() in +// net_processing.cpp), we'll mark that address as discouraged. We still allow +// incoming connections from them, but they're preferred for eviction when +// we receive new incoming connections. We never make outgoing connections to +// them, and do not gossip their address to other peers. This is implemented as +// a bloom filter. We can (probabilistically) test for membership, but can't +// list all discouraged addresses or unmark them as discouraged. Discouragement +// can prevent our limited connection slots being used up by incompatible +// or broken peers. +// +// Neither banning nor discouragement are protections against denial-of-service +// attacks, since if an attacker has a way to waste our resources and we +// disconnect from them and ban that address, it's trivial for them to +// reconnect from another IP address. +// +// Attempting to automatically disconnect or ban any class of peer carries the +// risk of splitting the network. For example, if we banned/disconnected for a +// transaction that fails a policy check and a future version changes the +// policy check so the transaction is accepted, then that transaction could +// cause the network to split between old nodes and new nodes. class BanMan { public: ~BanMan(); BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t default_ban_time); - void Ban(const CNetAddr& net_addr, const BanReason& ban_reason, int64_t ban_time_offset = 0, bool since_unix_epoch = false); - void Ban(const CSubNet& sub_net, const BanReason& ban_reason, int64_t ban_time_offset = 0, bool since_unix_epoch = false); + void Ban(const CNetAddr& net_addr, int64_t ban_time_offset = 0, bool since_unix_epoch = false); + void Ban(const CSubNet& sub_net, int64_t ban_time_offset = 0, bool since_unix_epoch = false); + void Discourage(const CNetAddr& net_addr); void ClearBanned(); - int IsBannedLevel(CNetAddr net_addr); - bool IsBanned(CNetAddr net_addr); - bool IsBanned(CSubNet sub_net); + + //! Return whether net_addr is banned + bool IsBanned(const CNetAddr& net_addr); + + //! Return whether sub_net is exactly banned + bool IsBanned(const CSubNet& sub_net); + + //! Return whether net_addr is discouraged. + bool IsDiscouraged(const CNetAddr& net_addr); + bool Unban(const CNetAddr& net_addr); bool Unban(const CSubNet& sub_net); void GetBanned(banmap_t& banmap); @@ -68,6 +92,7 @@ private: CClientUIInterface* m_client_interface = nullptr; CBanDB m_ban_db; const int64_t m_default_ban_time; + CRollingBloomFilter m_discouraged GUARDED_BY(m_cs_banned) {50000, 0.000001}; }; #endif diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 981800c68e..135659f87f 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> +#include <crypto/sha256.h> #include <util/strencodings.h> #include <util/system.h> @@ -39,6 +40,7 @@ int main(int argc, char** argv) { ArgsManager argsman; SetupBenchArgs(argsman); + SHA256AutoDetect(); std::string error; if (!argsman.ParseParameters(argc, argv, error)) { tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 6106b7977c..3a71a6ca03 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -31,7 +31,7 @@ static void CoinSelection(benchmark::Bench& bench) { NodeContext node; auto chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); wallet.SetupLegacyScriptPubKeyMan(); std::vector<std::unique_ptr<CWalletTx>> wtxs; LOCK(wallet.cs_wallet); @@ -65,7 +65,7 @@ static void CoinSelection(benchmark::Bench& bench) typedef std::set<CInputCoin> CoinSet; static NodeContext testNode; static auto testChain = interfaces::MakeChain(testNode); -static CWallet testWallet(testChain.get(), WalletLocation(), WalletDatabase::CreateDummy()); +static CWallet testWallet(testChain.get(), WalletLocation(), CreateDummyWalletDatabase()); std::vector<std::unique_ptr<CWalletTx>> wtxn; // Copied from src/wallet/test/coinselector_tests.cpp diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index d55d5359f0..e16182b48e 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -26,7 +26,7 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b NodeContext node; std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(node); - CWallet wallet{chain.get(), WalletLocation(), WalletDatabase::CreateMock()}; + CWallet wallet{chain.get(), WalletLocation(), CreateMockWalletDatabase()}; { wallet.SetupLegacyScriptPubKeyMan(); bool first_run; diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 8d85789b4e..9afcda4578 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -11,6 +11,7 @@ #include <clientversion.h> #include <optional.h> #include <rpc/client.h> +#include <rpc/mining.h> #include <rpc/protocol.h> #include <rpc/request.h> #include <util/strencodings.h> @@ -39,6 +40,9 @@ static const int DEFAULT_HTTP_CLIENT_TIMEOUT=900; static const bool DEFAULT_NAMED=false; static const int CONTINUE_EXECUTION=-1; +/** Default number of blocks to generate for RPC generatetoaddress. */ +static const std::string DEFAULT_NBLOCKS = "1"; + static void SetupCliArgs() { SetupHelpOptions(gArgs); @@ -50,6 +54,7 @@ static void SetupCliArgs() gArgs.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-generate", strprintf("Generate blocks immediately, equivalent to RPC generatenewaddress followed by RPC generatetoaddress. Optional positional integer arguments are number of blocks to generate (default: %s) and maximum iterations to try (default: %s), equivalent to RPC generatetoaddress nblocks and maxtries arguments. Example: bitcoin-cli -generate 4 1000", DEFAULT_NBLOCKS, DEFAULT_MAX_TRIES), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-getinfo", "Get general information from the remote server. Note that unlike server-side RPC calls, the results of -getinfo is the result of multiple non-atomic requests. Some entries in the result may represent results from different states (e.g. wallet balance may be as of a different block from the chain state reported)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); SetupChainParamsBaseOptions(); gArgs.AddArg("-named", strprintf("Pass named instead of positional arguments (default: %s)", DEFAULT_NAMED), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -286,6 +291,28 @@ public: } }; +/** Process RPC generatetoaddress request. */ +class GenerateToAddressRequestHandler : public BaseRequestHandler +{ +public: + UniValue PrepareRequest(const std::string& method, const std::vector<std::string>& args) override + { + address_str = args.at(1); + UniValue params{RPCConvertValues("generatetoaddress", args)}; + return JSONRPCRequestObj("generatetoaddress", params, 1); + } + + UniValue ProcessReply(const UniValue &reply) override + { + UniValue result(UniValue::VOBJ); + result.pushKV("address", address_str); + result.pushKV("blocks", reply.get_obj()["result"]); + return JSONRPCReplyObj(result, NullUniValue, 1); + } +protected: + std::string address_str; +}; + /** Process default single requests */ class DefaultRequestHandler: public BaseRequestHandler { public: @@ -453,6 +480,34 @@ static UniValue ConnectAndCallRPC(BaseRequestHandler* rh, const std::string& str return response; } +/** Parse UniValue result to update the message to print to std::cout. */ +static void ParseResult(const UniValue& result, std::string& strPrint) +{ + if (result.isNull()) return; + strPrint = result.isStr() ? result.get_str() : result.write(2); +} + +/** Parse UniValue error to update the message to print to std::cerr and the code to return. */ +static void ParseError(const UniValue& error, std::string& strPrint, int& nRet) +{ + if (error.isObject()) { + const UniValue& err_code = find_value(error, "code"); + const UniValue& err_msg = find_value(error, "message"); + if (!err_code.isNull()) { + strPrint = "error code: " + err_code.getValStr() + "\n"; + } + if (err_msg.isStr()) { + strPrint += ("error message:\n" + err_msg.get_str()); + } + if (err_code.isNum() && err_code.get_int() == RPC_WALLET_NOT_SPECIFIED) { + strPrint += "\nTry adding \"-rpcwallet=<filename>\" option to bitcoin-cli command line."; + } + } else { + strPrint = "error: " + error.write(); + } + nRet = abs(error["code"].get_int()); +} + /** * GetWalletBalances calls listwallets; if more than one wallet is loaded, it then * fetches mine.trusted balances for each loaded wallet and pushes them to `result`. @@ -461,8 +516,8 @@ static UniValue ConnectAndCallRPC(BaseRequestHandler* rh, const std::string& str */ static void GetWalletBalances(UniValue& result) { - std::unique_ptr<BaseRequestHandler> rh{MakeUnique<DefaultRequestHandler>()}; - const UniValue listwallets = ConnectAndCallRPC(rh.get(), "listwallets", /* args=*/{}); + DefaultRequestHandler rh; + const UniValue listwallets = ConnectAndCallRPC(&rh, "listwallets", /* args=*/{}); if (!find_value(listwallets, "error").isNull()) return; const UniValue& wallets = find_value(listwallets, "result"); if (wallets.size() <= 1) return; @@ -470,13 +525,41 @@ static void GetWalletBalances(UniValue& result) UniValue balances(UniValue::VOBJ); for (const UniValue& wallet : wallets.getValues()) { const std::string wallet_name = wallet.get_str(); - const UniValue getbalances = ConnectAndCallRPC(rh.get(), "getbalances", /* args=*/{}, wallet_name); + const UniValue getbalances = ConnectAndCallRPC(&rh, "getbalances", /* args=*/{}, wallet_name); const UniValue& balance = find_value(getbalances, "result")["mine"]["trusted"]; balances.pushKV(wallet_name, balance); } result.pushKV("balances", balances); } +/** + * Call RPC getnewaddress. + * @returns getnewaddress response as a UniValue object. + */ +static UniValue GetNewAddress() +{ + Optional<std::string> wallet_name{}; + if (gArgs.IsArgSet("-rpcwallet")) wallet_name = gArgs.GetArg("-rpcwallet", ""); + DefaultRequestHandler rh; + return ConnectAndCallRPC(&rh, "getnewaddress", /* args=*/{}, wallet_name); +} + +/** + * Check bounds and set up args for RPC generatetoaddress params: nblocks, address, maxtries. + * @param[in] address Reference to const string address to insert into the args. + * @param args Reference to vector of string args to modify. + */ +static void SetGenerateToAddressArgs(const std::string& address, std::vector<std::string>& args) +{ + if (args.size() > 2) throw std::runtime_error("too many arguments (maximum 2 for nblocks and maxtries)"); + if (args.size() == 0) { + args.emplace_back(DEFAULT_NBLOCKS); + } else if (args.at(0) == "0") { + throw std::runtime_error("the first argument (number of blocks to generate, default: " + DEFAULT_NBLOCKS + ") must be an integer value greater than zero"); + } + args.emplace(args.begin() + 1, address); +} + static int CommandLineRPC(int argc, char *argv[]) { std::string strPrint; @@ -535,6 +618,15 @@ static int CommandLineRPC(int argc, char *argv[]) std::string method; if (gArgs.IsArgSet("-getinfo")) { rh.reset(new GetinfoRequestHandler()); + } else if (gArgs.GetBoolArg("-generate", false)) { + const UniValue getnewaddress{GetNewAddress()}; + const UniValue& error{find_value(getnewaddress, "error")}; + if (error.isNull()) { + SetGenerateToAddressArgs(find_value(getnewaddress, "result").get_str(), args); + rh.reset(new GenerateToAddressRequestHandler()); + } else { + ParseError(error, strPrint, nRet); + } } else { rh.reset(new DefaultRequestHandler()); if (args.size() < 1) { @@ -543,40 +635,22 @@ static int CommandLineRPC(int argc, char *argv[]) method = args[0]; args.erase(args.begin()); // Remove trailing method name from arguments vector } - Optional<std::string> wallet_name{}; - if (gArgs.IsArgSet("-rpcwallet")) wallet_name = gArgs.GetArg("-rpcwallet", ""); - const UniValue reply = ConnectAndCallRPC(rh.get(), method, args, wallet_name); - - // Parse reply - UniValue result = find_value(reply, "result"); - const UniValue& error = find_value(reply, "error"); - if (!error.isNull()) { - // Error - strPrint = "error: " + error.write(); - nRet = abs(error["code"].get_int()); - if (error.isObject()) { - const UniValue& errCode = find_value(error, "code"); - const UniValue& errMsg = find_value(error, "message"); - strPrint = errCode.isNull() ? "" : ("error code: " + errCode.getValStr() + "\n"); - - if (errMsg.isStr()) { - strPrint += ("error message:\n" + errMsg.get_str()); - } - if (errCode.isNum() && errCode.get_int() == RPC_WALLET_NOT_SPECIFIED) { - strPrint += "\nTry adding \"-rpcwallet=<filename>\" option to bitcoin-cli command line."; + if (nRet == 0) { + // Perform RPC call + Optional<std::string> wallet_name{}; + if (gArgs.IsArgSet("-rpcwallet")) wallet_name = gArgs.GetArg("-rpcwallet", ""); + const UniValue reply = ConnectAndCallRPC(rh.get(), method, args, wallet_name); + + // Parse reply + UniValue result = find_value(reply, "result"); + const UniValue& error = find_value(reply, "error"); + if (error.isNull()) { + if (gArgs.IsArgSet("-getinfo") && !gArgs.IsArgSet("-rpcwallet")) { + GetWalletBalances(result); // fetch multiwallet balances and append to result } - } - } else { - if (gArgs.IsArgSet("-getinfo") && !gArgs.IsArgSet("-rpcwallet")) { - GetWalletBalances(result); // fetch multiwallet balances and append to result - } - // Result - if (result.isNull()) { - strPrint = ""; - } else if (result.isStr()) { - strPrint = result.get_str(); + ParseResult(result, strPrint); } else { - strPrint = result.write(2); + ParseError(error, strPrint, nRet); } } } catch (const std::exception& e) { diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index b8e8717896..b04cc12059 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -13,9 +13,9 @@ #include <init.h> #include <interfaces/chain.h> #include <node/context.h> +#include <node/ui_interface.h> #include <noui.h> #include <shutdown.h> -#include <ui_interface.h> #include <util/ref.h> #include <util/strencodings.h> #include <util/system.h> @@ -101,6 +101,11 @@ static bool AppInit(int argc, char* argv[]) } } + if (!gArgs.InitSettings(error)) { + InitError(Untranslated(error)); + return false; + } + // -server defaults to true for bitcoind but not for the GUI so do this here gArgs.SoftSetBoolArg("-server", true); // Set this early so that parameter interactions go to console diff --git a/src/bloom.cpp b/src/bloom.cpp index 54fcf487e4..d182f0728e 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -135,8 +135,8 @@ bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx) else if ((nFlags & BLOOM_UPDATE_MASK) == BLOOM_UPDATE_P2PUBKEY_ONLY) { std::vector<std::vector<unsigned char> > vSolutions; - txnouttype type = Solver(txout.scriptPubKey, vSolutions); - if (type == TX_PUBKEY || type == TX_MULTISIG) { + TxoutType type = Solver(txout.scriptPubKey, vSolutions); + if (type == TxoutType::PUBKEY || type == TxoutType::MULTISIG) { insert(COutPoint(hash, i)); } } diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 799474fae2..092c45e4ce 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -121,6 +121,7 @@ public: vSeeds.emplace_back("seed.btc.petertodd.org"); // Peter Todd, only supports x1, x5, x9, and xd vSeeds.emplace_back("seed.bitcoin.sprovoost.nl"); // Sjors Provoost vSeeds.emplace_back("dnsseed.emzy.de"); // Stephan Oeste + vSeeds.emplace_back("seed.bitcoin.wiz.biz"); // Jason Maurice base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,0); base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,5); diff --git a/src/coins.cpp b/src/coins.cpp index 7b76c13f98..5de2ed7810 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -245,6 +245,14 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const return true; } +void CCoinsViewCache::ReallocateCache() +{ + // Cache should be empty when we're calling this. + assert(cacheCoins.size() == 0); + cacheCoins.~CCoinsMap(); + ::new (&cacheCoins) CCoinsMap(); +} + static const size_t MIN_TRANSACTION_OUTPUT_WEIGHT = WITNESS_SCALE_FACTOR * ::GetSerializeSize(CTxOut(), PROTOCOL_VERSION); static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_WEIGHT / MIN_TRANSACTION_OUTPUT_WEIGHT; diff --git a/src/coins.h b/src/coins.h index a3f34bb0ee..a3e241ac90 100644 --- a/src/coins.h +++ b/src/coins.h @@ -318,6 +318,13 @@ public: //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view bool HaveInputs(const CTransaction& tx) const; + //! Force a reallocation of the cache map. This is required when downsizing + //! the cache because the map's allocator may be hanging onto a lot of + //! memory despite having called .clear(). + //! + //! See: https://stackoverflow.com/questions/42114044/how-to-release-unordered-map-memory + void ReallocateCache(); + private: /** * @note this is marked const, but may actually append to `cacheCoins`, increasing diff --git a/src/consensus/validation.h b/src/consensus/validation.h index a79e7b9d12..8de7a8f2d8 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -30,12 +30,16 @@ enum class TxValidationResult { TX_MISSING_INPUTS, //!< transaction was missing some of its inputs TX_PREMATURE_SPEND, //!< transaction spends a coinbase too early, or violates locktime/sequence locks /** - * Transaction might be missing a witness, have a witness prior to SegWit + * Transaction might have a witness prior to SegWit * activation, or witness may have been malleated (which includes * non-standard witnesses). */ TX_WITNESS_MUTATED, /** + * Transaction is missing a witness. + */ + TX_WITNESS_STRIPPED, + /** * Tx already in mempool or conflicts with a tx in the chain * (if it conflicts with another tx in mempool, we use MEMPOOL_POLICY as it failed to reach the RBF threshold) * Currently this is only used if the transaction already exists in the mempool or on chain. @@ -75,37 +79,39 @@ enum class BlockValidationResult { * by TxValidationState and BlockValidationState for validation information on transactions * and blocks respectively. */ template <typename Result> -class ValidationState { +class ValidationState +{ private: - enum mode_state { - MODE_VALID, //!< everything ok - MODE_INVALID, //!< network rule violation (DoS value may be set) - MODE_ERROR, //!< run-time error - } m_mode{MODE_VALID}; + enum class ModeState { + M_VALID, //!< everything ok + M_INVALID, //!< network rule violation (DoS value may be set) + M_ERROR, //!< run-time error + } m_mode{ModeState::M_VALID}; Result m_result{}; std::string m_reject_reason; std::string m_debug_message; + public: bool Invalid(Result result, - const std::string &reject_reason="", - const std::string &debug_message="") + const std::string& reject_reason = "", + const std::string& debug_message = "") { m_result = result; m_reject_reason = reject_reason; m_debug_message = debug_message; - if (m_mode != MODE_ERROR) m_mode = MODE_INVALID; + if (m_mode != ModeState::M_ERROR) m_mode = ModeState::M_INVALID; return false; } bool Error(const std::string& reject_reason) { - if (m_mode == MODE_VALID) + if (m_mode == ModeState::M_VALID) m_reject_reason = reject_reason; - m_mode = MODE_ERROR; + m_mode = ModeState::M_ERROR; return false; } - bool IsValid() const { return m_mode == MODE_VALID; } - bool IsInvalid() const { return m_mode == MODE_INVALID; } - bool IsError() const { return m_mode == MODE_ERROR; } + bool IsValid() const { return m_mode == ModeState::M_VALID; } + bool IsInvalid() const { return m_mode == ModeState::M_INVALID; } + bool IsError() const { return m_mode == ModeState::M_ERROR; } Result GetResult() const { return m_result; } std::string GetRejectReason() const { return m_reject_reason; } std::string GetDebugMessage() const { return m_debug_message; } diff --git a/src/core_write.cpp b/src/core_write.cpp index eb0cc35f06..34cfeecc6f 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -131,20 +131,20 @@ std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags) { CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | serializeFlags); ssTx << tx; - return HexStr(ssTx.begin(), ssTx.end()); + return HexStr(ssTx); } void ScriptToUniv(const CScript& script, UniValue& out, bool include_address) { out.pushKV("asm", ScriptToAsmStr(script)); - out.pushKV("hex", HexStr(script.begin(), script.end())); + out.pushKV("hex", HexStr(script)); std::vector<std::vector<unsigned char>> solns; - txnouttype type = Solver(script, solns); + TxoutType type = Solver(script, solns); out.pushKV("type", GetTxnOutputType(type)); CTxDestination address; - if (include_address && ExtractDestination(script, address) && type != TX_PUBKEY) { + if (include_address && ExtractDestination(script, address) && type != TxoutType::PUBKEY) { out.pushKV("address", EncodeDestination(address)); } } @@ -152,15 +152,15 @@ void ScriptToUniv(const CScript& script, UniValue& out, bool include_address) void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex) { - txnouttype type; + TxoutType type; std::vector<CTxDestination> addresses; int nRequired; out.pushKV("asm", ScriptToAsmStr(scriptPubKey)); if (fIncludeHex) - out.pushKV("hex", HexStr(scriptPubKey.begin(), scriptPubKey.end())); + out.pushKV("hex", HexStr(scriptPubKey)); - if (!ExtractDestinations(scriptPubKey, type, addresses, nRequired) || type == TX_PUBKEY) { + if (!ExtractDestinations(scriptPubKey, type, addresses, nRequired) || type == TxoutType::PUBKEY) { out.pushKV("type", GetTxnOutputType(type)); return; } @@ -179,7 +179,9 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, { entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); - entry.pushKV("version", tx.nVersion); + // Transaction version is actually unsigned in consensus checks, just signed in memory, + // so cast to unsigned before giving it to the user. + entry.pushKV("version", static_cast<int64_t>(static_cast<uint32_t>(tx.nVersion))); entry.pushKV("size", (int)::GetSerializeSize(tx, PROTOCOL_VERSION)); entry.pushKV("vsize", (GetTransactionWeight(tx) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR); entry.pushKV("weight", GetTransactionWeight(tx)); @@ -190,19 +192,19 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, const CTxIn& txin = tx.vin[i]; UniValue in(UniValue::VOBJ); if (tx.IsCoinBase()) - in.pushKV("coinbase", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())); + in.pushKV("coinbase", HexStr(txin.scriptSig)); else { in.pushKV("txid", txin.prevout.hash.GetHex()); in.pushKV("vout", (int64_t)txin.prevout.n); UniValue o(UniValue::VOBJ); o.pushKV("asm", ScriptToAsmStr(txin.scriptSig, true)); - o.pushKV("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())); + o.pushKV("hex", HexStr(txin.scriptSig)); in.pushKV("scriptSig", o); } if (!tx.vin[i].scriptWitness.IsNull()) { UniValue txinwitness(UniValue::VARR); for (const auto& item : tx.vin[i].scriptWitness.stack) { - txinwitness.push_back(HexStr(item.begin(), item.end())); + txinwitness.push_back(HexStr(item)); } in.pushKV("txinwitness", txinwitness); } diff --git a/src/crypto/common.h b/src/crypto/common.h index e7bb020a19..5b4932c992 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -82,12 +82,12 @@ void static inline WriteBE64(unsigned char* ptr, uint64_t x) /** Return the smallest number n such that (x >> n) == 0 (or 64 if the highest bit in x is set. */ uint64_t static inline CountBits(uint64_t x) { -#if HAVE_DECL___BUILTIN_CLZL +#if HAVE_BUILTIN_CLZL if (sizeof(unsigned long) >= sizeof(uint64_t)) { return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0; } #endif -#if HAVE_DECL___BUILTIN_CLZLL +#if HAVE_BUILTIN_CLZLL if (sizeof(unsigned long long) >= sizeof(uint64_t)) { return x ? 8 * sizeof(unsigned long long) - __builtin_clzll(x) : 0; } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 116d7d8679..215b033708 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -292,18 +292,6 @@ public: // Get an estimate of LevelDB memory usage (in bytes). size_t DynamicMemoryUsage() const; - // not available for LevelDB; provide for compatibility with BDB - bool Flush() - { - return true; - } - - bool Sync() - { - CDBBatch batch(*this); - return WriteBatch(batch, true); - } - CDBIterator *NewIterator() { return new CDBIterator(*this, pdb->NewIterator(iteroptions)); diff --git a/src/fs.cpp b/src/fs.cpp index e68c97b3ca..eef9c81de9 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -5,10 +5,12 @@ #include <fs.h> #ifndef WIN32 +#include <cstring> #include <fcntl.h> #include <string> #include <sys/file.h> #include <sys/utsname.h> +#include <unistd.h> #else #ifndef NOMINMAX #define NOMINMAX @@ -31,7 +33,8 @@ FILE *fopen(const fs::path& p, const char *mode) #ifndef WIN32 -static std::string GetErrorReason() { +static std::string GetErrorReason() +{ return std::strerror(errno); } diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 5e78fd1d71..1e5ea2de83 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -7,10 +7,10 @@ #include <chainparamsbase.h> #include <compat.h> #include <netbase.h> +#include <node/ui_interface.h> #include <rpc/protocol.h> // For HTTP status codes #include <shutdown.h> #include <sync.h> -#include <ui_interface.h> #include <util/strencodings.h> #include <util/system.h> #include <util/threadnames.h> diff --git a/src/index/base.cpp b/src/index/base.cpp index 1d09f2e577..f587205a28 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -4,9 +4,9 @@ #include <chainparams.h> #include <index/base.h> +#include <node/ui_interface.h> #include <shutdown.h> #include <tinyformat.h> -#include <ui_interface.h> #include <util/system.h> #include <util/translation.h> #include <validation.h> @@ -17,15 +17,13 @@ constexpr char DB_BEST_BLOCK = 'B'; constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds -template<typename... Args> +template <typename... Args> static void FatalError(const char* fmt, const Args&... args) { std::string strMessage = tfm::format(fmt, args...); SetMiscWarning(Untranslated(strMessage)); LogPrintf("*** %s\n", strMessage); - uiInterface.ThreadSafeMessageBox( - Untranslated("Error: A fatal internal error occurred, see debug.log for details"), - "", CClientUIInterface::MSG_ERROR); + AbortError(_("A fatal internal error occurred, see debug.log for details")); StartShutdown(); } diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 59d1888fff..64472714cc 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -3,8 +3,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <index/txindex.h> +#include <node/ui_interface.h> #include <shutdown.h> -#include <ui_interface.h> #include <util/system.h> #include <util/translation.h> #include <validation.h> diff --git a/src/init.cpp b/src/init.cpp index fd7c8d0f80..1c89e2eeff 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -31,6 +31,7 @@ #include <net_processing.h> #include <netbase.h> #include <node/context.h> +#include <node/ui_interface.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/policy.h> @@ -48,8 +49,8 @@ #include <torcontrol.h> #include <txdb.h> #include <txmempool.h> -#include <ui_interface.h> #include <util/asmap.h> +#include <util/check.h> #include <util/moneystr.h> #include <util/string.h> #include <util/system.h> @@ -152,6 +153,8 @@ NODISCARD static bool CreatePidFile() static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; +static std::thread g_load_block; + static boost::thread_group threadGroup; void Interrupt(NodeContext& node) @@ -214,8 +217,9 @@ void Shutdown(NodeContext& node) StopTorControl(); // After everything has been shut down, but before things get flushed, stop the - // CScheduler/checkqueue threadGroup + // CScheduler/checkqueue, threadGroup and load block thread. if (node.scheduler) node.scheduler->stop(); + if (g_load_block.joinable()) g_load_block.join(); threadGroup.interrupt_all(); threadGroup.join_all(); @@ -392,8 +396,8 @@ void SetupServerArgs(NodeContext& node) gArgs.AddArg("-blocknotify=<cmd>", "Execute command when the best block changes (%s in cmd is replaced by block hash)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #endif gArgs.AddArg("-blockreconstructionextratxn=<n>", strprintf("Extra transactions to keep in memory for compact block reconstructions (default: %u)", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Automatic broadcast and rebroadcast of any transactions from inbound peers is disabled, unless '-whitelistforcerelay' is '1', in which case whitelisted peers' transactions will be relayed. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Automatic broadcast and rebroadcast of any transactions from inbound peers is disabled, unless the peer has the 'forcerelay' permission. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-conf=<file>", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); gArgs.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -414,6 +418,7 @@ void SetupServerArgs(NodeContext& node) "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-settings=<file>", strprintf("Specify path to dynamic settings data file. Can be disabled with -nosettings. File is written at runtime and not meant to be edited by users (use %s instead for custom settings). Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME, BITCOIN_SETTINGS_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #ifndef WIN32 gArgs.AddArg("-sysperms", "Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #else @@ -427,8 +432,7 @@ void SetupServerArgs(NodeContext& node) gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-asmap=<file>", strprintf("Specify asn mapping used for bucketing of the peers (default: %s). Relative paths will be prefixed by the net-specific datadir location.", DEFAULT_ASMAP_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-bantime=<n>", strprintf("Number of seconds to keep misbehaving peers from reconnecting (default: %u)", DEFAULT_MISBEHAVING_BANTIME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-bantime=<n>", strprintf("Default duration (in seconds) of manually configured bans (default: %u)", DEFAULT_MISBEHAVING_BANTIME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-bind=<addr>", "Bind to given address and always listen on it. Use [host]:port notation for IPv6", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-connect=<ip>", "Connect only to the specified node; -noconnect disables automatic connections (the rules for this peer are the same as for -addnode). This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-discover", "Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -442,7 +446,7 @@ void SetupServerArgs(NodeContext& node) gArgs.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by peers forward or backward by this amount. (default: %u seconds)", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h). Limit does not apply to peers with 'noban' permission. 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h). Limit does not apply to peers with 'download' permission. 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor hidden services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-onlynet=<net>", "Make outgoing connections only through network <net> (ipv4, ipv6 or onion). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -452,6 +456,7 @@ void SetupServerArgs(NodeContext& node) gArgs.AddArg("-proxy=<ip:port>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-proxyrandomize", strprintf("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)", DEFAULT_PROXYRANDOMIZE), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-networkactive", "Enable all P2P network activity (default: 1). Can be changed by the setnetworkactive RPC command", ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); gArgs.AddArg("-timeout=<n>", strprintf("Specify connection timeout in milliseconds (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-peertimeout=<n>", strprintf("Specify p2p connection timeout in seconds. This option determines the amount of time a peer may be inactive before the connection to it is dropped. (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -465,12 +470,12 @@ void SetupServerArgs(NodeContext& node) #else hidden_args.emplace_back("-upnp"); #endif - gArgs.AddArg("-whitebind=<[permissions@]addr>", "Bind to given address and whitelist peers connecting to it. " + gArgs.AddArg("-whitebind=<[permissions@]addr>", "Bind to the given address and add permission flags to the peers connecting to it. " "Use [host]:port notation for IPv6. Allowed permissions: " + Join(NET_PERMISSIONS_DOC, ", ") + ". " - "Specify multiple permissions separated by commas (default: noban,mempool,relay). Can be specified multiple times.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + "Specify multiple permissions separated by commas (default: download,noban,mempool,relay). Can be specified multiple times.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-whitelist=<[permissions@]IP address or network>", "Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) or " - "CIDR notated network(e.g. 1.2.3.0/24). Uses same permissions as " + gArgs.AddArg("-whitelist=<[permissions@]IP address or network>", "Add permission flags to the peers connecting from the given IP address (e.g. 1.2.3.4) or " + "CIDR-notated network (e.g. 1.2.3.0/24). Uses the same permissions as " "-whitebind. Can be specified multiple times." , ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); g_wallet_init_interface.AddWalletOptions(); @@ -680,7 +685,6 @@ static void CleanupBlockRevFiles() static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles) { const CChainParams& chainparams = Params(); - util::ThreadRename("loadblk"); ScheduleBatchPriority(); { @@ -772,13 +776,14 @@ static bool InitSanityCheck() return true; } -static bool AppInitServers(const util::Ref& context) +static bool AppInitServers(const util::Ref& context, NodeContext& node) { RPCServer::OnStarted(&OnRPCStarted); RPCServer::OnStopped(&OnRPCStopped); if (!InitHTTPServer()) return false; StartRPC(); + node.rpc_interruption_point = RpcInterruptionPoint; if (!StartHTTPRPC(context)) return false; if (gArgs.GetBoolArg("-rest", DEFAULT_REST_ENABLE)) StartREST(context); @@ -1315,8 +1320,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) node.scheduler = MakeUnique<CScheduler>(); // Start the lightweight task scheduler thread - CScheduler::Function serviceLoop = [&node]{ node.scheduler->serviceQueue(); }; - threadGroup.create_thread(std::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); + threadGroup.create_thread([&] { TraceThread("scheduler", [&] { node.scheduler->serviceQueue(); }); }); // Gather some entropy once per minute. node.scheduler->scheduleEvery([]{ @@ -1350,7 +1354,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) if (gArgs.GetBoolArg("-server", false)) { uiInterface.InitMessage_connect(SetRPCWarmupStatus); - if (!AppInitServers(context)) + if (!AppInitServers(context, node)) return InitError(_("Unable to start HTTP server. See debug log for details.")); } @@ -1370,16 +1374,16 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) assert(!node.banman); node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", &uiInterface, gArgs.GetArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); assert(!node.connman); - node.connman = std::unique_ptr<CConnman>(new CConnman(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()))); + node.connman = MakeUnique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()), gArgs.GetBoolArg("-networkactive", true)); // Make mempool generally available in the node context. For example the connection manager, wallet, or RPC threads, // which are all started after this, may use it from the node context. assert(!node.mempool); node.mempool = &::mempool; assert(!node.chainman); node.chainman = &g_chainman; - ChainstateManager& chainman = EnsureChainman(node); + ChainstateManager& chainman = *Assert(node.chainman); - node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), *node.scheduler, *node.chainman, *node.mempool)); + node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), *node.scheduler, chainman, *node.mempool)); RegisterValidationInterface(node.peer_logic.get()); // sanitize comments per BIP-0014, format user agent and check total size @@ -1530,7 +1534,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; - nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache + int64_t nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; LogPrintf("Cache configuration:\n"); LogPrintf("* Using %.1f MiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); @@ -1559,6 +1563,9 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) try { LOCK(cs_main); chainman.InitializeChainstate(); + chainman.m_total_coinstip_cache = nCoinCacheUsage; + chainman.m_total_coinsdb_cache = nCoinDBCache; + UnloadBlockIndex(); // new CBlockTreeDB tries to delete the existing file, which @@ -1587,7 +1594,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) // If the loaded chain has a wrong genesis, bail out immediately // (we're likely using a testnet datadir, or the other way around). - if (!::BlockIndex().empty() && + if (!chainman.BlockIndex().empty() && !LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) { return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); } @@ -1642,7 +1649,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) } // The on-disk coinsdb is now in a good state, create the cache - chainstate->InitCoinsCache(); + chainstate->InitCoinsCache(nCoinCacheUsage); assert(chainstate->CanFlushToDisk()); if (!is_coinsview_empty(chainstate)) { @@ -1842,7 +1849,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) vImportFiles.push_back(strFile); } - threadGroup.create_thread([=, &chainman] { ThreadImport(chainman, vImportFiles); }); + g_load_block = std::thread(&TraceThread<std::function<void()>>, "loadblk", [=, &chainman]{ ThreadImport(chainman, vImportFiles); }); // Wait for genesis block to be processed { @@ -1867,8 +1874,8 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) //// debug print { LOCK(cs_main); - LogPrintf("block tree size = %u\n", ::BlockIndex().size()); - chain_active_height = ::ChainActive().Height(); + LogPrintf("block tree size = %u\n", chainman.BlockIndex().size()); + chain_active_height = chainman.ActiveChain().Height(); } LogPrintf("nBestHeight = %d\n", chain_active_height); @@ -1886,7 +1893,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) connOptions.nLocalServices = nLocalServices; connOptions.nMaxConnections = nMaxConnections; connOptions.m_max_outbound_full_relay = std::min(MAX_OUTBOUND_FULL_RELAY_CONNECTIONS, connOptions.nMaxConnections); - connOptions.m_max_outbound_block_relay = std::min(MAX_BLOCKS_ONLY_CONNECTIONS, connOptions.nMaxConnections-connOptions.m_max_outbound_full_relay); + connOptions.m_max_outbound_block_relay = std::min(MAX_BLOCK_RELAY_ONLY_CONNECTIONS, connOptions.nMaxConnections-connOptions.m_max_outbound_full_relay); connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; connOptions.nMaxFeeler = MAX_FEELER_CONNECTIONS; connOptions.nBestHeight = chain_active_height; diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index d1e04b114d..d49e4454af 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -13,6 +13,7 @@ #include <node/coin.h> #include <node/context.h> #include <node/transaction.h> +#include <node/ui_interface.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -25,7 +26,6 @@ #include <sync.h> #include <timedata.h> #include <txmempool.h> -#include <ui_interface.h> #include <uint256.h> #include <univalue.h> #include <util/system.h> diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 61d7ddb934..bbeb0fa801 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -8,12 +8,14 @@ #include <optional.h> // For Optional and nullopt #include <primitives/transaction.h> // For CTransactionRef +#include <functional> #include <memory> #include <stddef.h> #include <stdint.h> #include <string> #include <vector> +class ArgsManager; class CBlock; class CFeeRate; class CRPCCommand; @@ -321,7 +323,7 @@ std::unique_ptr<Chain> MakeChain(NodeContext& node); //! analysis, or fee estimation. These clients need to expose their own //! MakeXXXClient functions returning their implementations of the ChainClient //! interface. -std::unique_ptr<ChainClient> MakeWalletClient(Chain& chain, std::vector<std::string> wallet_filenames); +std::unique_ptr<ChainClient> MakeWalletClient(Chain& chain, ArgsManager& args, std::vector<std::string> wallet_filenames); } // namespace interfaces diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index d420788dbe..33f0dac263 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -17,6 +17,7 @@ #include <netaddress.h> #include <netbase.h> #include <node/context.h> +#include <node/ui_interface.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/settings.h> @@ -26,7 +27,6 @@ #include <support/allocators/secure.h> #include <sync.h> #include <txmempool.h> -#include <ui_interface.h> #include <util/ref.h> #include <util/system.h> #include <util/translation.h> @@ -66,6 +66,7 @@ public: bool softSetArg(const std::string& arg, const std::string& value) override { return gArgs.SoftSetArg(arg, value); } bool softSetBoolArg(const std::string& arg, bool value) override { return gArgs.SoftSetBoolArg(arg, value); } void selectParams(const std::string& network) override { SelectParams(network); } + bool initSettings(std::string& error) override { return gArgs.InitSettings(error); } uint64_t getAssumedBlockchainSize() override { return Params().AssumedBlockchainSize(); } uint64_t getAssumedChainStateSize() override { return Params().AssumedChainStateSize(); } std::string getNetwork() override { return Params().NetworkIDString(); } @@ -146,10 +147,10 @@ public: } return false; } - bool ban(const CNetAddr& net_addr, BanReason reason, int64_t ban_time_offset) override + bool ban(const CNetAddr& net_addr, int64_t ban_time_offset) override { if (m_context.banman) { - m_context.banman->Ban(net_addr, reason, ban_time_offset); + m_context.banman->Ban(net_addr, ban_time_offset); return true; } return false; diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 877a40568f..a9680c42b5 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -66,6 +66,11 @@ public: //! Choose network parameters. virtual void selectParams(const std::string& network) = 0; + //! Read and update <datadir>/settings.json file with saved settings. This + //! needs to be called after selectParams() because the settings file + //! location is network-specific. + virtual bool initSettings(std::string& error) = 0; + //! Get the (assumed) blockchain size. virtual uint64_t getAssumedBlockchainSize() = 0; @@ -122,7 +127,7 @@ public: virtual bool getBanned(banmap_t& banmap) = 0; //! Ban node. - virtual bool ban(const CNetAddr& net_addr, BanReason reason, int64_t ban_time_offset) = 0; + virtual bool ban(const CNetAddr& net_addr, int64_t ban_time_offset) = 0; //! Unban node. virtual bool unban(const CSubNet& ip) = 0; diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 397403d308..7fd24425cf 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -13,11 +13,11 @@ #include <script/standard.h> #include <support/allocators/secure.h> #include <sync.h> -#include <ui_interface.h> #include <uint256.h> #include <util/check.h> #include <util/ref.h> #include <util/system.h> +#include <util/ui_change_type.h> #include <wallet/context.h> #include <wallet/feebumper.h> #include <wallet/fees.h> @@ -335,9 +335,10 @@ public: bool sign, bool bip32derivs, PartiallySignedTransaction& psbtx, - bool& complete) override + bool& complete, + size_t* n_signed) override { - return m_wallet->FillPSBT(psbtx, complete, sighash_type, sign, bip32derivs); + return m_wallet->FillPSBT(psbtx, complete, sighash_type, sign, bip32derivs, n_signed); } WalletBalances getBalances() override { @@ -437,7 +438,6 @@ public: bool canGetAddresses() override { return m_wallet->CanGetAddresses(); } bool privateKeysDisabled() override { return m_wallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); } OutputType getDefaultAddressType() override { return m_wallet->m_default_address_type; } - OutputType getDefaultChangeType() override { return m_wallet->m_default_change_type; } CAmount getDefaultMaxTxFee() override { return m_wallet->m_default_max_tx_fee; } void remove() override { @@ -483,10 +483,11 @@ public: class WalletClientImpl : public ChainClient { public: - WalletClientImpl(Chain& chain, std::vector<std::string> wallet_filenames) + WalletClientImpl(Chain& chain, ArgsManager& args, std::vector<std::string> wallet_filenames) : m_wallet_filenames(std::move(wallet_filenames)) { m_context.chain = &chain; + m_context.args = &args; } void registerRpcs() override { @@ -499,7 +500,7 @@ public: } bool verify() override { return VerifyWallets(*m_context.chain, m_wallet_filenames); } bool load() override { return LoadWallets(*m_context.chain, m_wallet_filenames); } - void start(CScheduler& scheduler) override { return StartWallets(scheduler); } + void start(CScheduler& scheduler) override { return StartWallets(scheduler, *Assert(m_context.args)); } void flush() override { return FlushWallets(); } void stop() override { return StopWallets(); } void setMockTime(int64_t time) override { return SetMockTime(time); } @@ -514,7 +515,7 @@ public: ~WalletClientImpl() override { UnloadWallets(); } WalletContext m_context; - std::vector<std::string> m_wallet_filenames; + const std::vector<std::string> m_wallet_filenames; std::vector<std::unique_ptr<Handler>> m_rpc_handlers; std::list<CRPCCommand> m_rpc_commands; }; @@ -523,9 +524,9 @@ public: std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet) { return wallet ? MakeUnique<WalletImpl>(wallet) : nullptr; } -std::unique_ptr<ChainClient> MakeWalletClient(Chain& chain, std::vector<std::string> wallet_filenames) +std::unique_ptr<ChainClient> MakeWalletClient(Chain& chain, ArgsManager& args, std::vector<std::string> wallet_filenames) { - return MakeUnique<WalletClientImpl>(chain, std::move(wallet_filenames)); + return MakeUnique<WalletClientImpl>(chain, args, std::move(wallet_filenames)); } } // namespace interfaces diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 67569a3e55..3cdadbc72e 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -9,8 +9,8 @@ #include <pubkey.h> // For CKeyID and CScriptID (definitions needed in CTxDestination instantiation) #include <script/standard.h> // For CTxDestination #include <support/allocators/secure.h> // For SecureString -#include <ui_interface.h> // For ChangeType #include <util/message.h> +#include <util/ui_change_type.h> #include <functional> #include <map> @@ -197,7 +197,8 @@ public: bool sign, bool bip32derivs, PartiallySignedTransaction& psbtx, - bool& complete) = 0; + bool& complete, + size_t* n_signed) = 0; //! Get balances. virtual WalletBalances getBalances() = 0; @@ -255,9 +256,6 @@ public: // Get default address type. virtual OutputType getDefaultAddressType() = 0; - // Get default change type. - virtual OutputType getDefaultChangeType() = 0; - //! Get max tx fee. virtual CAmount getDefaultMaxTxFee() = 0; diff --git a/src/key.cpp b/src/key.cpp index b6ed29e8e3..7eecc6e083 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -31,46 +31,46 @@ static secp256k1_context* secp256k1_context_sign = nullptr; * * out32 must point to an output buffer of length at least 32 bytes. */ -static int ec_privkey_import_der(const secp256k1_context* ctx, unsigned char *out32, const unsigned char *privkey, size_t privkeylen) { - const unsigned char *end = privkey + privkeylen; +static int ec_seckey_import_der(const secp256k1_context* ctx, unsigned char *out32, const unsigned char *seckey, size_t seckeylen) { + const unsigned char *end = seckey + seckeylen; memset(out32, 0, 32); /* sequence header */ - if (end - privkey < 1 || *privkey != 0x30u) { + if (end - seckey < 1 || *seckey != 0x30u) { return 0; } - privkey++; + seckey++; /* sequence length constructor */ - if (end - privkey < 1 || !(*privkey & 0x80u)) { + if (end - seckey < 1 || !(*seckey & 0x80u)) { return 0; } - ptrdiff_t lenb = *privkey & ~0x80u; privkey++; + ptrdiff_t lenb = *seckey & ~0x80u; seckey++; if (lenb < 1 || lenb > 2) { return 0; } - if (end - privkey < lenb) { + if (end - seckey < lenb) { return 0; } /* sequence length */ - ptrdiff_t len = privkey[lenb-1] | (lenb > 1 ? privkey[lenb-2] << 8 : 0u); - privkey += lenb; - if (end - privkey < len) { + ptrdiff_t len = seckey[lenb-1] | (lenb > 1 ? seckey[lenb-2] << 8 : 0u); + seckey += lenb; + if (end - seckey < len) { return 0; } /* sequence element 0: version number (=1) */ - if (end - privkey < 3 || privkey[0] != 0x02u || privkey[1] != 0x01u || privkey[2] != 0x01u) { + if (end - seckey < 3 || seckey[0] != 0x02u || seckey[1] != 0x01u || seckey[2] != 0x01u) { return 0; } - privkey += 3; + seckey += 3; /* sequence element 1: octet string, up to 32 bytes */ - if (end - privkey < 2 || privkey[0] != 0x04u) { + if (end - seckey < 2 || seckey[0] != 0x04u) { return 0; } - ptrdiff_t oslen = privkey[1]; - privkey += 2; - if (oslen > 32 || end - privkey < oslen) { + ptrdiff_t oslen = seckey[1]; + seckey += 2; + if (oslen > 32 || end - seckey < oslen) { return 0; } - memcpy(out32 + (32 - oslen), privkey, oslen); + memcpy(out32 + (32 - oslen), seckey, oslen); if (!secp256k1_ec_seckey_verify(ctx, out32)) { memset(out32, 0, 32); return 0; @@ -83,17 +83,17 @@ static int ec_privkey_import_der(const secp256k1_context* ctx, unsigned char *ou * <http://www.secg.org/sec1-v2.pdf>. The optional parameters and publicKey fields are * included. * - * privkey must point to an output buffer of length at least CKey::SIZE bytes. - * privkeylen must initially be set to the size of the privkey buffer. Upon return it + * seckey must point to an output buffer of length at least CKey::SIZE bytes. + * seckeylen must initially be set to the size of the seckey buffer. Upon return it * will be set to the number of bytes used in the buffer. * key32 must point to a 32-byte raw private key. */ -static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *privkey, size_t *privkeylen, const unsigned char *key32, bool compressed) { - assert(*privkeylen >= CKey::SIZE); +static int ec_seckey_export_der(const secp256k1_context *ctx, unsigned char *seckey, size_t *seckeylen, const unsigned char *key32, bool compressed) { + assert(*seckeylen >= CKey::SIZE); secp256k1_pubkey pubkey; size_t pubkeylen = 0; if (!secp256k1_ec_pubkey_create(ctx, &pubkey, key32)) { - *privkeylen = 0; + *seckeylen = 0; return 0; } if (compressed) { @@ -111,15 +111,15 @@ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *pr 0xFF,0xFF,0xFF,0xFF,0xFE,0xBA,0xAE,0xDC,0xE6,0xAF,0x48,0xA0,0x3B,0xBF,0xD2,0x5E, 0x8C,0xD0,0x36,0x41,0x41,0x02,0x01,0x01,0xA1,0x24,0x03,0x22,0x00 }; - unsigned char *ptr = privkey; + unsigned char *ptr = seckey; memcpy(ptr, begin, sizeof(begin)); ptr += sizeof(begin); memcpy(ptr, key32, 32); ptr += 32; memcpy(ptr, middle, sizeof(middle)); ptr += sizeof(middle); pubkeylen = CPubKey::COMPRESSED_SIZE; secp256k1_ec_pubkey_serialize(ctx, ptr, &pubkeylen, &pubkey, SECP256K1_EC_COMPRESSED); ptr += pubkeylen; - *privkeylen = ptr - privkey; - assert(*privkeylen == CKey::COMPRESSED_SIZE); + *seckeylen = ptr - seckey; + assert(*seckeylen == CKey::COMPRESSED_SIZE); } else { static const unsigned char begin[] = { 0x30,0x82,0x01,0x13,0x02,0x01,0x01,0x04,0x20 @@ -137,15 +137,15 @@ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *pr 0xFF,0xFF,0xFF,0xFF,0xFE,0xBA,0xAE,0xDC,0xE6,0xAF,0x48,0xA0,0x3B,0xBF,0xD2,0x5E, 0x8C,0xD0,0x36,0x41,0x41,0x02,0x01,0x01,0xA1,0x44,0x03,0x42,0x00 }; - unsigned char *ptr = privkey; + unsigned char *ptr = seckey; memcpy(ptr, begin, sizeof(begin)); ptr += sizeof(begin); memcpy(ptr, key32, 32); ptr += 32; memcpy(ptr, middle, sizeof(middle)); ptr += sizeof(middle); pubkeylen = CPubKey::SIZE; secp256k1_ec_pubkey_serialize(ctx, ptr, &pubkeylen, &pubkey, SECP256K1_EC_UNCOMPRESSED); ptr += pubkeylen; - *privkeylen = ptr - privkey; - assert(*privkeylen == CKey::SIZE); + *seckeylen = ptr - seckey; + assert(*seckeylen == CKey::SIZE); } return 1; } @@ -165,20 +165,20 @@ void CKey::MakeNewKey(bool fCompressedIn) { bool CKey::Negate() { assert(fValid); - return secp256k1_ec_privkey_negate(secp256k1_context_sign, keydata.data()); + return secp256k1_ec_seckey_negate(secp256k1_context_sign, keydata.data()); } CPrivKey CKey::GetPrivKey() const { assert(fValid); - CPrivKey privkey; + CPrivKey seckey; int ret; - size_t privkeylen; - privkey.resize(SIZE); - privkeylen = SIZE; - ret = ec_privkey_export_der(secp256k1_context_sign, privkey.data(), &privkeylen, begin(), fCompressed); + size_t seckeylen; + seckey.resize(SIZE); + seckeylen = SIZE; + ret = ec_seckey_export_der(secp256k1_context_sign, seckey.data(), &seckeylen, begin(), fCompressed); assert(ret); - privkey.resize(privkeylen); - return privkey; + seckey.resize(seckeylen); + return seckey; } CPubKey CKey::GetPubKey() const { @@ -258,8 +258,8 @@ bool CKey::SignCompact(const uint256 &hash, std::vector<unsigned char>& vchSig) return true; } -bool CKey::Load(const CPrivKey &privkey, const CPubKey &vchPubKey, bool fSkipCheck=false) { - if (!ec_privkey_import_der(secp256k1_context_sign, (unsigned char*)begin(), privkey.data(), privkey.size())) +bool CKey::Load(const CPrivKey &seckey, const CPubKey &vchPubKey, bool fSkipCheck=false) { + if (!ec_seckey_import_der(secp256k1_context_sign, (unsigned char*)begin(), seckey.data(), seckey.size())) return false; fCompressed = vchPubKey.IsCompressed(); fValid = true; @@ -284,7 +284,7 @@ bool CKey::Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const } memcpy(ccChild.begin(), vout.data()+32, 32); memcpy((unsigned char*)keyChild.begin(), begin(), 32); - bool ret = secp256k1_ec_privkey_tweak_add(secp256k1_context_sign, (unsigned char*)keyChild.begin(), vout.data()); + bool ret = secp256k1_ec_seckey_tweak_add(secp256k1_context_sign, (unsigned char*)keyChild.begin(), vout.data()); keyChild.fCompressed = true; keyChild.fValid = ret; return ret; diff --git a/src/miner.cpp b/src/miner.cpp index d9dcbe8a70..41a835f70a 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -109,7 +109,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc if(!pblocktemplate.get()) return nullptr; - pblock = &pblocktemplate->block; // pointer for convenience + CBlock* const pblock = &pblocktemplate->block; // pointer for convenience // Add dummy coinbase tx as first transaction pblock->vtx.emplace_back(); @@ -226,7 +226,7 @@ bool BlockAssembler::TestPackageTransactions(const CTxMemPool::setEntries& packa void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) { - pblock->vtx.emplace_back(iter->GetSharedTx()); + pblocktemplate->block.vtx.emplace_back(iter->GetSharedTx()); pblocktemplate->vTxFees.push_back(iter->GetFee()); pblocktemplate->vTxSigOpsCost.push_back(iter->GetSigOpCost()); nBlockWeight += iter->GetTxWeight(); diff --git a/src/miner.h b/src/miner.h index 69296f9078..096585dfe4 100644 --- a/src/miner.h +++ b/src/miner.h @@ -128,8 +128,6 @@ class BlockAssembler private: // The constructed block template std::unique_ptr<CBlockTemplate> pblocktemplate; - // A convenience pointer that always refers to the CBlock in pblocktemplate - CBlock* pblock; // Configuration parameters for the block size bool fIncludeWitness; diff --git a/src/net.cpp b/src/net.cpp index f3b7e8dab1..0c56cddbdc 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -14,12 +14,12 @@ #include <clientversion.h> #include <consensus/consensus.h> #include <crypto/sha256.h> -#include <netbase.h> #include <net_permissions.h> +#include <netbase.h> +#include <node/ui_interface.h> #include <protocol.h> #include <random.h> #include <scheduler.h> -#include <ui_interface.h> #include <util/strencodings.h> #include <util/translation.h> @@ -42,6 +42,7 @@ static_assert(MINIUPNPC_API_VERSION >= 10, "miniUPnPc API version >= 10 assumed"); #endif +#include <cstdint> #include <unordered_map> #include <math.h> @@ -110,9 +111,9 @@ void CConnman::AddOneShot(const std::string& strDest) vOneShots.push_back(strDest); } -unsigned short GetListenPort() +uint16_t GetListenPort() { - return (unsigned short)(gArgs.GetArg("-port", Params().GetDefaultPort())); + return (uint16_t)(gArgs.GetArg("-port", Params().GetDefaultPort())); } // find 'best' local address for a particular peer @@ -563,15 +564,15 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) // since pingtime does not update until the ping is complete, which might take a while. // So, if a ping is taking an unusually long time in flight, // the caller can immediately detect that this is happening. - int64_t nPingUsecWait = 0; - if ((0 != nPingNonceSent) && (0 != nPingUsecStart)) { - nPingUsecWait = GetTimeMicros() - nPingUsecStart; + std::chrono::microseconds ping_wait{0}; + if ((0 != nPingNonceSent) && (0 != m_ping_start.load().count())) { + ping_wait = GetTime<std::chrono::microseconds>() - m_ping_start.load(); } // Raw ping time is in microseconds, but show it to user as whole seconds (Bitcoin users should be well used to small numbers with many decimal places by now :) stats.m_ping_usec = nPingUsecTime; stats.m_min_ping_usec = nMinPingUsecTime; - stats.m_ping_wait_usec = nPingUsecWait; + stats.m_ping_wait_usec = count_microseconds(ping_wait); // Leave string empty if addrLocal invalid (not filled in yet) CService addrLocalUnlocked = GetAddrLocal(); @@ -582,9 +583,9 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete) { complete = false; - int64_t nTimeMicros = GetTimeMicros(); + const auto time = GetTime<std::chrono::microseconds>(); LOCK(cs_vRecv); - nLastRecv = nTimeMicros / 1000000; + nLastRecv = std::chrono::duration_cast<std::chrono::seconds>(time).count(); nRecvBytes += nBytes; while (nBytes > 0) { // absorb network data @@ -596,7 +597,7 @@ bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete if (m_deserializer->Complete()) { // decompose a transport agnostic CNetMessage from the deserializer - CNetMessage msg = m_deserializer->GetMessage(Params().MessageStart(), nTimeMicros); + CNetMessage msg = m_deserializer->GetMessage(Params().MessageStart(), time); //store received bytes per message command //to prevent a memory DOS, only allow valid commands @@ -699,7 +700,8 @@ const uint256& V1TransportDeserializer::GetMessageHash() const return data_hash; } -CNetMessage V1TransportDeserializer::GetMessage(const CMessageHeader::MessageStartChars& message_start, int64_t time) { +CNetMessage V1TransportDeserializer::GetMessage(const CMessageHeader::MessageStartChars& message_start, const std::chrono::microseconds time) +{ // decompose a single CNetMessage from the TransportDeserializer CNetMessage msg(std::move(vRecv)); @@ -737,7 +739,7 @@ void V1TransportSerializer::prepareForTransport(CSerializedNetMsg& msg, std::vec uint256 hash = Hash(msg.data.begin(), msg.data.end()); // create header - CMessageHeader hdr(Params().MessageStart(), msg.command.c_str(), msg.data.size()); + CMessageHeader hdr(Params().MessageStart(), msg.m_type.c_str(), msg.data.size()); memcpy(hdr.pchChecksum, hash.begin(), CMessageHeader::CHECKSUM_SIZE); // serialize header @@ -1010,17 +1012,24 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { // on all platforms. Set it again here just to be sure. SetSocketNoDelay(hSocket); - int bannedlevel = m_banman ? m_banman->IsBannedLevel(addr) : 0; - - // Don't accept connections from banned peers, but if our inbound slots aren't almost full, accept - // if the only banning reason was an automatic misbehavior ban. - if (!NetPermissions::HasFlag(permissionFlags, NetPermissionFlags::PF_NOBAN) && bannedlevel > ((nInbound + 1 < nMaxInbound) ? 1 : 0)) + // Don't accept connections from banned peers. + bool banned = m_banman && m_banman->IsBanned(addr); + if (!NetPermissions::HasFlag(permissionFlags, NetPermissionFlags::PF_NOBAN) && banned) { LogPrint(BCLog::NET, "connection from %s dropped (banned)\n", addr.ToString()); CloseSocket(hSocket); return; } + // Only accept connections from discouraged peers if our inbound slots aren't (almost) full. + bool discouraged = m_banman && m_banman->IsDiscouraged(addr); + if (!NetPermissions::HasFlag(permissionFlags, NetPermissionFlags::PF_NOBAN) && nInbound + 1 >= nMaxInbound && discouraged) + { + LogPrint(BCLog::NET, "connection from %s dropped (discouraged)\n", addr.ToString()); + CloseSocket(hSocket); + return; + } + if (nInbound >= nMaxInbound) { if (!AttemptToEvictConnection()) { @@ -1044,7 +1053,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { pnode->m_permissionFlags = permissionFlags; // If this flag is present, the user probably expect that RPC and QT report it as whitelisted (backward compatibility) pnode->m_legacyWhitelisted = legacyWhitelisted; - pnode->m_prefer_evict = bannedlevel > 0; + pnode->m_prefer_evict = discouraged; m_msgproc->InitializeNode(pnode); LogPrint(BCLog::NET, "connection from %s accepted\n", addr.ToString()); @@ -1103,12 +1112,9 @@ void CConnman::DisconnectNodes() if (pnode->GetRefCount() <= 0) { bool fDelete = false; { - TRY_LOCK(pnode->cs_inventory, lockInv); - if (lockInv) { - TRY_LOCK(pnode->cs_vSend, lockSend); - if (lockSend) { - fDelete = true; - } + TRY_LOCK(pnode->cs_vSend, lockSend); + if (lockSend) { + fDelete = true; } } if (fDelete) { @@ -1154,9 +1160,9 @@ void CConnman::InactivityCheck(CNode *pnode) LogPrintf("socket receive timeout: %is\n", nTime - pnode->nLastRecv); pnode->fDisconnect = true; } - else if (pnode->nPingNonceSent && pnode->nPingUsecStart + TIMEOUT_INTERVAL * 1000000 < GetTimeMicros()) + else if (pnode->nPingNonceSent && pnode->m_ping_start.load() + std::chrono::seconds{TIMEOUT_INTERVAL} < GetTime<std::chrono::microseconds>()) { - LogPrintf("ping timeout: %fs\n", 0.000001 * (GetTimeMicros() - pnode->nPingUsecStart)); + LogPrintf("ping timeout: %fs\n", 0.000001 * count_microseconds(GetTime<std::chrono::microseconds>() - pnode->m_ping_start.load())); pnode->fDisconnect = true; } else if (!pnode->fSuccessfullyConnected) @@ -2045,10 +2051,10 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai return; } if (!pszDest) { - if (IsLocal(addrConnect) || - FindNode(static_cast<CNetAddr>(addrConnect)) || (m_banman && m_banman->IsBanned(addrConnect)) || - FindNode(addrConnect.ToStringIPPort())) + bool banned_or_discouraged = m_banman && (m_banman->IsDiscouraged(addrConnect) || m_banman->IsBanned(addrConnect)); + if (IsLocal(addrConnect) || FindNode(static_cast<CNetAddr>(addrConnect)) || banned_or_discouraged || FindNode(addrConnect.ToStringIPPort())) { return; + } } else if (FindNode(std::string(pszDest))) return; @@ -2247,7 +2253,7 @@ void Discover() void CConnman::SetNetworkActive(bool active) { - LogPrint(BCLog::NET, "SetNetworkActive: %s\n", active); + LogPrintf("%s: %s\n", __func__, active); if (fNetworkActive == active) { return; @@ -2258,12 +2264,14 @@ void CConnman::SetNetworkActive(bool active) uiInterface.NotifyNetworkActiveChanged(fNetworkActive); } -CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In) : nSeed0(nSeed0In), nSeed1(nSeed1In) +CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In, bool network_active) + : nSeed0(nSeed0In), nSeed1(nSeed1In) { SetTryNewOutboundPeer(false); Options connOptions; Init(connOptions); + SetNetworkActive(network_active); } NodeId CConnman::GetNewNodeId() @@ -2505,11 +2513,6 @@ CConnman::~CConnman() Stop(); } -size_t CConnman::GetAddressCount() const -{ - return addrman.size(); -} - void CConnman::SetServices(const CService &addr, ServiceFlags nServices) { addrman.SetServices(addr, nServices); @@ -2639,7 +2642,7 @@ void CConnman::RecordBytesSent(uint64_t bytes) nMaxOutboundTotalBytesSentInCycle = 0; } - // TODO, exclude peers with noban permission + // TODO, exclude peers with download permission nMaxOutboundTotalBytesSentInCycle += bytes; } @@ -2794,7 +2797,7 @@ bool CConnman::NodeFullyConnected(const CNode* pnode) void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { size_t nMessageSize = msg.data.size(); - LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command), nMessageSize, pnode->GetId()); + LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.m_type), nMessageSize, pnode->GetId()); // make sure we use the appropriate network transport format std::vector<unsigned char> serializedHeader; @@ -2806,8 +2809,8 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) LOCK(pnode->cs_vSend); bool optimisticSend(pnode->vSendMsg.empty()); - //log total amount of bytes per command - pnode->mapSendBytesPerMsgCmd[msg.command] += nTotalSize; + //log total amount of bytes per message type + pnode->mapSendBytesPerMsgCmd[msg.m_type] += nTotalSize; pnode->nSendSize += nTotalSize; if (pnode->nSendSize > nSendBufferMaxSize) @@ -25,8 +25,8 @@ #include <uint256.h> #include <atomic> +#include <cstdint> #include <deque> -#include <stdint.h> #include <thread> #include <memory> #include <condition_variable> @@ -61,7 +61,7 @@ static const int MAX_OUTBOUND_FULL_RELAY_CONNECTIONS = 8; /** Maximum number of addnode outgoing nodes */ static const int MAX_ADDNODE_CONNECTIONS = 8; /** Maximum number of block-relay-only outgoing connections */ -static const int MAX_BLOCKS_ONLY_CONNECTIONS = 2; +static const int MAX_BLOCK_RELAY_ONLY_CONNECTIONS = 2; /** Maximum number of feeler connections */ static const int MAX_FEELER_CONNECTIONS = 1; /** -listen default */ @@ -110,7 +110,7 @@ struct CSerializedNetMsg CSerializedNetMsg& operator=(const CSerializedNetMsg&) = delete; std::vector<unsigned char> data; - std::string command; + std::string m_type; }; @@ -181,7 +181,7 @@ public: } } - CConnman(uint64_t seed0, uint64_t seed1); + CConnman(uint64_t seed0, uint64_t seed1, bool network_active = true); ~CConnman(); bool Start(CScheduler& scheduler, const Options& options); @@ -247,7 +247,6 @@ public: }; // Addrman functions - size_t GetAddressCount() const; void SetServices(const CService &addr, ServiceFlags nServices); void MarkAddressGood(const CAddress& addr); void AddNewAddresses(const std::vector<CAddress>& vAddr, const CAddress& addrFrom, int64_t nTimePenalty = 0); @@ -448,6 +447,7 @@ private: std::atomic<int> nBestHeight; CClientUIInterface* clientInterface; NetEventsInterface* m_msgproc; + /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ BanMan* m_banman; /** SipHasher seeds for deterministic randomness */ @@ -482,7 +482,7 @@ void Discover(); void StartMapPort(); void InterruptMapPort(); void StopMapPort(); -unsigned short GetListenPort(); +uint16_t GetListenPort(); struct CombinerAll { @@ -612,13 +612,13 @@ public: */ class CNetMessage { public: - CDataStream m_recv; // received message data - int64_t m_time = 0; // time (in microseconds) of message receipt. + CDataStream m_recv; //!< received message data + std::chrono::microseconds m_time{0}; //!< time of message receipt bool m_valid_netmagic = false; bool m_valid_header = false; bool m_valid_checksum = false; - uint32_t m_message_size = 0; // size of the payload - uint32_t m_raw_message_size = 0; // used wire size of the message (including header/checksum) + uint32_t m_message_size{0}; //!< size of the payload + uint32_t m_raw_message_size{0}; //!< used wire size of the message (including header/checksum) std::string m_command; CNetMessage(CDataStream&& recv_in) : m_recv(std::move(recv_in)) {} @@ -642,7 +642,7 @@ public: // read and deserialize data virtual int Read(const char *data, unsigned int bytes) = 0; // decomposes a message from the context - virtual CNetMessage GetMessage(const CMessageHeader::MessageStartChars& message_start, int64_t time) = 0; + virtual CNetMessage GetMessage(const CMessageHeader::MessageStartChars& message_start, std::chrono::microseconds time) = 0; virtual ~TransportDeserializer() {} }; @@ -695,7 +695,7 @@ public: if (ret < 0) Reset(); return ret; } - CNetMessage GetMessage(const CMessageHeader::MessageStartChars& message_start, int64_t time) override; + CNetMessage GetMessage(const CMessageHeader::MessageStartChars& message_start, std::chrono::microseconds time) override; }; /** The TransportSerializer prepares messages for the network transport @@ -803,7 +803,7 @@ public: // There is no final sorting before sending, as they are always sent immediately // and in the order requested. std::vector<uint256> vInventoryBlockToSend GUARDED_BY(cs_inventory); - RecursiveMutex cs_inventory; + Mutex cs_inventory; struct TxRelay { mutable RecursiveMutex cs_filter; @@ -845,8 +845,8 @@ public: // Ping time measurement: // The pong reply we're expecting, or 0 if no pong expected. std::atomic<uint64_t> nPingNonceSent{0}; - // Time (in usec) the last ping was sent, or 0 if no ping was ever sent. - std::atomic<int64_t> nPingUsecStart{0}; + /** When the last ping was sent, or 0 if no ping was ever sent */ + std::atomic<std::chrono::microseconds> m_ping_start{std::chrono::microseconds{0}}; // Last measured round-trip time. std::atomic<int64_t> nPingUsecTime{0}; // Best measured round-trip time. @@ -965,33 +965,23 @@ public: } - void AddInventoryKnown(const CInv& inv) + void AddKnownTx(const uint256& hash) { if (m_tx_relay != nullptr) { LOCK(m_tx_relay->cs_tx_inventory); - m_tx_relay->filterInventoryKnown.insert(inv.hash); + m_tx_relay->filterInventoryKnown.insert(hash); } } - void PushInventory(const CInv& inv) + void PushTxInventory(const uint256& hash) { - if (inv.type == MSG_TX && m_tx_relay != nullptr) { - LOCK(m_tx_relay->cs_tx_inventory); - if (!m_tx_relay->filterInventoryKnown.contains(inv.hash)) { - m_tx_relay->setInventoryTxToSend.insert(inv.hash); - } - } else if (inv.type == MSG_BLOCK) { - LOCK(cs_inventory); - vInventoryBlockToSend.push_back(inv.hash); + if (m_tx_relay == nullptr) return; + LOCK(m_tx_relay->cs_tx_inventory); + if (!m_tx_relay->filterInventoryKnown.contains(hash)) { + m_tx_relay->setInventoryTxToSend.insert(hash); } } - void PushBlockHash(const uint256 &hash) - { - LOCK(cs_inventory); - vBlockHashesToAnnounce.push_back(hash); - } - void CloseSocketDisconnect(); void copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap); diff --git a/src/net_permissions.cpp b/src/net_permissions.cpp index da09149856..a75838307c 100644 --- a/src/net_permissions.cpp +++ b/src/net_permissions.cpp @@ -10,10 +10,11 @@ const std::vector<std::string> NET_PERMISSIONS_DOC{ "bloomfilter (allow requesting BIP37 filtered blocks and transactions)", - "noban (do not ban for misbehavior)", + "noban (do not ban for misbehavior; implies download)", "forcerelay (relay transactions that are already in the mempool; implies relay)", "relay (relay even in -blocksonly mode)", "mempool (allow requesting BIP35 mempool contents)", + "download (allow getheaders during IBD, no disconnect after maxuploadtarget limit)", }; namespace { @@ -46,6 +47,7 @@ bool TryParsePermissionFlags(const std::string str, NetPermissionFlags& output, else if (permission == "noban") NetPermissions::AddFlag(flags, PF_NOBAN); else if (permission == "forcerelay") NetPermissions::AddFlag(flags, PF_FORCERELAY); else if (permission == "mempool") NetPermissions::AddFlag(flags, PF_MEMPOOL); + else if (permission == "download") NetPermissions::AddFlag(flags, PF_DOWNLOAD); else if (permission == "all") NetPermissions::AddFlag(flags, PF_ALL); else if (permission == "relay") NetPermissions::AddFlag(flags, PF_RELAY); else if (permission.length() == 0); // Allow empty entries @@ -72,6 +74,7 @@ std::vector<std::string> NetPermissions::ToStrings(NetPermissionFlags flags) if (NetPermissions::HasFlag(flags, PF_FORCERELAY)) strings.push_back("forcerelay"); if (NetPermissions::HasFlag(flags, PF_RELAY)) strings.push_back("relay"); if (NetPermissions::HasFlag(flags, PF_MEMPOOL)) strings.push_back("mempool"); + if (NetPermissions::HasFlag(flags, PF_DOWNLOAD)) strings.push_back("download"); return strings; } diff --git a/src/net_permissions.h b/src/net_permissions.h index e004067e75..a9633ee2ae 100644 --- a/src/net_permissions.h +++ b/src/net_permissions.h @@ -14,8 +14,7 @@ struct bilingual_str; extern const std::vector<std::string> NET_PERMISSIONS_DOC; -enum NetPermissionFlags -{ +enum NetPermissionFlags { PF_NONE = 0, // Can query bloomfilter even if -peerbloomfilters is false PF_BLOOMFILTER = (1U << 1), @@ -24,14 +23,16 @@ enum NetPermissionFlags // Always relay transactions from this peer, even if already in mempool // Keep parameter interaction: forcerelay implies relay PF_FORCERELAY = (1U << 2) | PF_RELAY, - // Can't be banned for misbehavior - PF_NOBAN = (1U << 4), + // Allow getheaders during IBD and block-download after maxuploadtarget limit + PF_DOWNLOAD = (1U << 6), + // Can't be banned/disconnected/discouraged for misbehavior + PF_NOBAN = (1U << 4) | PF_DOWNLOAD, // Can query the mempool PF_MEMPOOL = (1U << 5), // True if the user did not specifically set fine grained permissions PF_ISIMPLICIT = (1U << 31), - PF_ALL = PF_BLOOMFILTER | PF_FORCERELAY | PF_RELAY | PF_NOBAN | PF_MEMPOOL, + PF_ALL = PF_BLOOMFILTER | PF_FORCERELAY | PF_RELAY | PF_NOBAN | PF_MEMPOOL | PF_DOWNLOAD, }; class NetPermissions diff --git a/src/net_processing.cpp b/src/net_processing.cpp index d48745aef2..ca701a7e5b 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -13,10 +13,9 @@ #include <consensus/validation.h> #include <hash.h> #include <index/blockfilterindex.h> -#include <validation.h> #include <merkleblock.h> -#include <netmessagemaker.h> #include <netbase.h> +#include <netmessagemaker.h> #include <policy/fees.h> #include <policy/policy.h> #include <primitives/block.h> @@ -26,22 +25,22 @@ #include <scheduler.h> #include <tinyformat.h> #include <txmempool.h> -#include <util/system.h> +#include <util/check.h> // For NDEBUG compile time check #include <util/strencodings.h> +#include <util/system.h> +#include <validation.h> #include <memory> #include <typeinfo> -#if defined(NDEBUG) -# error "Bitcoin cannot be compiled without assertions." -#endif - /** Expiration time for orphan transactions in seconds */ static constexpr int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60; /** Minimum time between orphan transactions expire time checks in seconds */ static constexpr int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; /** How long to cache transactions in mapRelay for normal relay */ -static constexpr std::chrono::seconds RELAY_TX_CACHE_TIME{15 * 60}; +static constexpr std::chrono::seconds RELAY_TX_CACHE_TIME = std::chrono::minutes{15}; +/** How long a transaction has to be in the mempool before it can unconditionally be relayed (even when not in mapRelay). */ +static constexpr std::chrono::seconds UNCONDITIONAL_RELAY_DELAY = std::chrono::minutes{2}; /** Headers download timeout expressed in microseconds * Timeout = base + per_header * (expected number of headers) */ static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_BASE = 15 * 60 * 1000000; // 15 minutes @@ -66,8 +65,8 @@ static constexpr int STALE_RELAY_AGE_LIMIT = 30 * 24 * 60 * 60; /// Age after which a block is considered historical for purposes of rate /// limiting block relay. Set to one week, denominated in seconds. static constexpr int HISTORICAL_BLOCK_AGE = 7 * 24 * 60 * 60; -/** Time between pings automatically sent out for latency probing and keepalive (in seconds). */ -static const int PING_INTERVAL = 2 * 60; +/** Time between pings automatically sent out for latency probing and keepalive */ +static constexpr std::chrono::minutes PING_INTERVAL{2}; /** The maximum number of entries in a locator */ static const unsigned int MAX_LOCATOR_SZ = 101; /** The maximum number of entries in an 'inv' protocol message */ @@ -76,6 +75,8 @@ static const unsigned int MAX_INV_SZ = 50000; static constexpr int32_t MAX_PEER_TX_IN_FLIGHT = 100; /** Maximum number of announced transactions from a peer */ static constexpr int32_t MAX_PEER_TX_ANNOUNCEMENTS = 2 * MAX_INV_SZ; +/** How many microseconds to delay requesting transactions via txids, if we have wtxid-relaying peers */ +static constexpr std::chrono::microseconds TXID_RELAY_DELAY{std::chrono::seconds{2}}; /** How many microseconds to delay requesting transactions from inbound peers */ static constexpr std::chrono::microseconds INBOUND_PEER_TX_DELAY{std::chrono::seconds{2}}; /** How long to wait (in microseconds) before downloading a transaction from an additional peer */ @@ -120,11 +121,20 @@ static constexpr std::chrono::hours AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL{24}; /** Average delay between peer address broadcasts */ static constexpr std::chrono::seconds AVG_ADDRESS_BROADCAST_INTERVAL{30}; /** Average delay between trickled inventory transmissions in seconds. - * Blocks and whitelisted receivers bypass this, outbound peers get half this delay. */ + * Blocks and peers with noban permission bypass this, outbound peers get half this delay. */ static const unsigned int INVENTORY_BROADCAST_INTERVAL = 5; -/** Maximum number of inventory items to send per transmission. +/** Maximum rate of inventory items to send per second. * Limits the impact of low-fee transaction floods. */ -static constexpr unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_INTERVAL; +static constexpr unsigned int INVENTORY_BROADCAST_PER_SECOND = 7; +/** Maximum number of inventory items to send per transmission. */ +static constexpr unsigned int INVENTORY_BROADCAST_MAX = INVENTORY_BROADCAST_PER_SECOND * INVENTORY_BROADCAST_INTERVAL; +/** The number of most recently announced transactions a peer can request. */ +static constexpr unsigned int INVENTORY_MAX_RECENT_RELAY = 3500; +/** Verify that INVENTORY_MAX_RECENT_RELAY is enough to cache everything typically + * relayed before unconditional relay from the mempool kicks in. This is only a + * lower bound, and it should be larger to account for higher inv rate to outbound + * peers, and random variations in the broadcast mechanism. */ +static_assert(INVENTORY_MAX_RECENT_RELAY >= INVENTORY_BROADCAST_PER_SECOND * UNCONDITIONAL_RELAY_DELAY / std::chrono::seconds{1}, "INVENTORY_RELAY_MAX too low"); /** Average delay between feefilter broadcasts in seconds. */ static constexpr unsigned int AVG_FEEFILTER_BROADCAST_INTERVAL = 10 * 60; /** Maximum feefilter broadcast delay after significant change. */ @@ -143,12 +153,10 @@ struct COrphanTx { }; RecursiveMutex g_cs_orphans; std::map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(g_cs_orphans); +std::map<uint256, std::map<uint256, COrphanTx>::iterator> g_orphans_by_wtxid GUARDED_BY(g_cs_orphans); void EraseOrphansFor(NodeId peer); -/** Increase a node's misbehavior score. */ -void Misbehaving(NodeId nodeid, int howmuch, const std::string& message="") EXCLUSIVE_LOCKS_REQUIRED(cs_main); - // Internal stuff namespace { /** Number of nodes with fSyncStarted. */ @@ -179,6 +187,15 @@ namespace { * million to make it highly unlikely for users to have issues with this * filter. * + * We only need to add wtxids to this filter. For non-segwit + * transactions, the txid == wtxid, so this only prevents us from + * re-downloading non-segwit transactions when communicating with + * non-wtxidrelay peers -- which is important for avoiding malleation + * attacks that could otherwise interfere with transaction relay from + * non-wtxidrelay peers. For communicating with wtxidrelay peers, having + * the reject filter store wtxids is exactly what we want to avoid + * redownload of a rejected transaction. + * * Memory used: 1.3 MB */ std::unique_ptr<CRollingBloomFilter> recentRejects GUARDED_BY(cs_main); @@ -189,7 +206,7 @@ namespace { * We use this to avoid requesting transactions that have already been * confirnmed. */ - RecursiveMutex g_cs_recent_confirmed_transactions; + Mutex g_cs_recent_confirmed_transactions; std::unique_ptr<CRollingBloomFilter> g_recent_confirmed_transactions GUARDED_BY(g_cs_recent_confirmed_transactions); /** Blocks that are in flight, and that are in the queue to be downloaded. */ @@ -210,6 +227,9 @@ namespace { /** Number of peers from which we're downloading blocks. */ int nPeersWithValidatedDownloads GUARDED_BY(cs_main) = 0; + /** Number of peers with wtxid relay. */ + int g_wtxid_relay_peers GUARDED_BY(cs_main) = 0; + /** Number of outbound peers with m_chain_sync.m_protect. */ int g_outbound_peers_with_protect_from_disconnect GUARDED_BY(cs_main) = 0; @@ -252,8 +272,8 @@ struct CNodeState { bool fCurrentlyConnected; //! Accumulated misbehaviour score for this peer. int nMisbehavior; - //! Whether this peer should be disconnected and banned (unless whitelisted). - bool fShouldBan; + //! Whether this peer should be disconnected and marked as discouraged (unless it has the noban permission). + bool m_should_discourage; //! String name of this peer (debugging/logging purposes). const std::string name; //! The best known block we know this peer has announced. @@ -398,13 +418,19 @@ struct CNodeState { //! Whether this peer is a manual connection bool m_is_manual_connection; + //! A rolling bloom filter of all announced tx CInvs to this peer. + CRollingBloomFilter m_recently_announced_invs = CRollingBloomFilter{INVENTORY_MAX_RECENT_RELAY, 0.000001}; + + //! Whether this peer relays txs via wtxid + bool m_wtxid_relay{false}; + CNodeState(CAddress addrIn, std::string addrNameIn, bool is_inbound, bool is_manual) : address(addrIn), name(std::move(addrNameIn)), m_is_inbound(is_inbound), m_is_manual_connection (is_manual) { fCurrentlyConnected = false; nMisbehavior = 0; - fShouldBan = false; + m_should_discourage = false; pindexBestKnownBlock = nullptr; hashLastUnknownBlock.SetNull(); pindexLastCommonBlock = nullptr; @@ -425,6 +451,7 @@ struct CNodeState { fSupportsDesiredCmpctVersion = false; m_chain_sync = { 0, nullptr, false, false }; m_last_block_announcement = 0; + m_recently_announced_invs.reset(); } }; @@ -451,7 +478,7 @@ static void UpdatePreferredDownload(const CNode& node, CNodeState* state) EXCLUS nPreferredDownload += state->fPreferredDownload; } -static void PushNodeVersion(CNode& pnode, CConnman* connman, int64_t nTime) +static void PushNodeVersion(CNode& pnode, CConnman& connman, int64_t nTime) { // Note that pnode->GetLocalServices() is a reflection of the local // services we were offering when the CNode object was created for this @@ -465,7 +492,7 @@ static void PushNodeVersion(CNode& pnode, CConnman* connman, int64_t nTime) CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService(), addr.nServices)); CAddress addrMe = CAddress(CService(), nLocalNodeServices); - connman->PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, + connman.PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, nonce, strSubVersion, nNodeStartingHeight, ::g_relay_txes && pnode.m_tx_relay != nullptr)); if (fLogIPs) { @@ -576,7 +603,7 @@ static void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) EXCLUSIV * lNodesAnnouncingHeaderAndIDs, and keeping that list under a certain size by * removing the first element if necessary. */ -static void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman* connman) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman& connman) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(cs_main); CNodeState* nodestate = State(nodeid); @@ -592,20 +619,20 @@ static void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman* connma return; } } - connman->ForNode(nodeid, [connman](CNode* pfrom){ + connman.ForNode(nodeid, [&connman](CNode* pfrom){ AssertLockHeld(cs_main); uint64_t nCMPCTBLOCKVersion = (pfrom->GetLocalServices() & NODE_WITNESS) ? 2 : 1; if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { // As per BIP152, we only get 3 of our peers to announce // blocks using compact encodings. - connman->ForNode(lNodesAnnouncingHeaderAndIDs.front(), [connman, nCMPCTBLOCKVersion](CNode* pnodeStop){ + connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, nCMPCTBLOCKVersion](CNode* pnodeStop){ AssertLockHeld(cs_main); - connman->PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/false, nCMPCTBLOCKVersion)); + connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/false, nCMPCTBLOCKVersion)); return true; }); lNodesAnnouncingHeaderAndIDs.pop_front(); } - connman->PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/true, nCMPCTBLOCKVersion)); + connman.PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/true, nCMPCTBLOCKVersion)); lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); return true; }); @@ -748,7 +775,7 @@ void UpdateTxRequestTime(const uint256& txid, std::chrono::microseconds request_ } } -std::chrono::microseconds CalculateTxGetDataTime(const uint256& txid, std::chrono::microseconds current_time, bool use_inbound_delay) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +std::chrono::microseconds CalculateTxGetDataTime(const uint256& txid, std::chrono::microseconds current_time, bool use_inbound_delay, bool use_txid_delay) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { std::chrono::microseconds process_time; const auto last_request_time = GetTxRequestTime(txid); @@ -764,6 +791,9 @@ std::chrono::microseconds CalculateTxGetDataTime(const uint256& txid, std::chron // We delay processing announcements from inbound peers if (use_inbound_delay) process_time += INBOUND_PEER_TX_DELAY; + // We delay processing announcements from peers that use txid-relay (instead of wtxid) + if (use_txid_delay) process_time += TXID_RELAY_DELAY; + return process_time; } @@ -781,7 +811,7 @@ void RequestTx(CNodeState* state, const uint256& txid, std::chrono::microseconds // Calculate the time to try requesting this transaction. Use // fPreferredDownload as a proxy for outbound peers. - const auto process_time = CalculateTxGetDataTime(txid, current_time, !state->fPreferredDownload); + const auto process_time = CalculateTxGetDataTime(txid, current_time, !state->fPreferredDownload, !state->m_wtxid_relay && g_wtxid_relay_peers > 0); peer_download_state.m_tx_process_time.emplace(process_time, txid); } @@ -813,19 +843,20 @@ void PeerLogicValidation::InitializeNode(CNode *pnode) { mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName), pnode->fInbound, pnode->m_manual_connection)); } if(!pnode->fInbound) - PushNodeVersion(*pnode, connman, GetTime()); + PushNodeVersion(*pnode, *connman, GetTime()); } void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const { - std::set<uint256> unbroadcast_txids = m_mempool.GetUnbroadcastTxs(); + std::map<uint256, uint256> unbroadcast_txids = m_mempool.GetUnbroadcastTxs(); - for (const uint256& txid : unbroadcast_txids) { + for (const auto& elem : unbroadcast_txids) { // Sanity check: all unbroadcast txns should exist in the mempool - if (m_mempool.exists(txid)) { - RelayTransaction(txid, *connman); + if (m_mempool.exists(elem.first)) { + LOCK(cs_main); + RelayTransaction(elem.first, elem.second, *connman); } else { - m_mempool.RemoveUnbroadcastTx(txid, true); + m_mempool.RemoveUnbroadcastTx(elem.first, true); } } @@ -857,6 +888,8 @@ void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTim assert(nPeersWithValidatedDownloads >= 0); g_outbound_peers_with_protect_from_disconnect -= state->m_chain_sync.m_protect; assert(g_outbound_peers_with_protect_from_disconnect >= 0); + g_wtxid_relay_peers -= state->m_wtxid_relay; + assert(g_wtxid_relay_peers >= 0); mapNodeState.erase(nodeid); @@ -866,6 +899,7 @@ void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTim assert(nPreferredDownload == 0); assert(nPeersWithValidatedDownloads == 0); assert(g_outbound_peers_with_protect_from_disconnect == 0); + assert(g_wtxid_relay_peers == 0); } LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); } @@ -924,6 +958,8 @@ bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRE auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME, g_orphan_list.size()}); assert(ret.second); g_orphan_list.push_back(ret.first); + // Allow for lookups in the orphan pool by wtxid, as well as txid + g_orphans_by_wtxid.emplace(tx->GetWitnessHash(), ret.first); for (const CTxIn& txin : tx->vin) { mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first); } @@ -960,6 +996,7 @@ int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) it_last->second.list_pos = old_pos; } g_orphan_list.pop_back(); + g_orphans_by_wtxid.erase(it->second.tx->GetWitnessHash()); mapOrphanTransactions.erase(it); return 1; @@ -1019,30 +1056,29 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) } /** - * Mark a misbehaving peer to be banned depending upon the value of `-banscore`. + * Increment peer's misbehavior score. If the new value >= DISCOURAGEMENT_THRESHOLD, mark the node + * to be discouraged, meaning the peer might be disconnected and added to the discouragement filter. */ -void Misbehaving(NodeId pnode, int howmuch, const std::string& message) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +void Misbehaving(const NodeId pnode, const int howmuch, const std::string& message) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - if (howmuch == 0) - return; + assert(howmuch > 0); - CNodeState *state = State(pnode); - if (state == nullptr) - return; + CNodeState* const state = State(pnode); + if (state == nullptr) return; state->nMisbehavior += howmuch; - int banscore = gArgs.GetArg("-banscore", DEFAULT_BANSCORE_THRESHOLD); - std::string message_prefixed = message.empty() ? "" : (": " + message); - if (state->nMisbehavior >= banscore && state->nMisbehavior - howmuch < banscore) + const std::string message_prefixed = message.empty() ? "" : (": " + message); + if (state->nMisbehavior >= DISCOURAGEMENT_THRESHOLD && state->nMisbehavior - howmuch < DISCOURAGEMENT_THRESHOLD) { - LogPrint(BCLog::NET, "%s: %s peer=%d (%d -> %d) BAN THRESHOLD EXCEEDED%s\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior, message_prefixed); - state->fShouldBan = true; - } else - LogPrint(BCLog::NET, "%s: %s peer=%d (%d -> %d)%s\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior, message_prefixed); + LogPrint(BCLog::NET, "Misbehaving: peer=%d (%d -> %d) DISCOURAGE THRESHOLD EXCEEDED%s\n", pnode, state->nMisbehavior - howmuch, state->nMisbehavior, message_prefixed); + state->m_should_discourage = true; + } else { + LogPrint(BCLog::NET, "Misbehaving: peer=%d (%d -> %d)%s\n", pnode, state->nMisbehavior - howmuch, state->nMisbehavior, message_prefixed); + } } /** - * Potentially ban a node based on the contents of a BlockValidationState object + * Potentially mark a node discouraged based on the contents of a BlockValidationState object * * @param[in] via_compact_block this bool is passed in because net_processing should * punish peers differently depending on whether the data was provided in a compact @@ -1072,7 +1108,7 @@ static bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& s break; } - // Ban outbound (but not inbound) peers if on an invalid chain. + // Discourage outbound (but not inbound) peers if on an invalid chain. // Exempt HB compact block peers and manual connections. if (!via_compact_block && !node_state->m_is_inbound && !node_state->m_is_manual_connection) { Misbehaving(nodeid, 100, message); @@ -1107,7 +1143,7 @@ static bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& s } /** - * Potentially ban a node based on the contents of a TxValidationState object + * Potentially disconnect and discourage a node based on the contents of a TxValidationState object * * @return Returns true if the peer was punished (probably disconnected) */ @@ -1129,6 +1165,7 @@ static bool MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, case TxValidationResult::TX_MISSING_INPUTS: case TxValidationResult::TX_PREMATURE_SPEND: case TxValidationResult::TX_WITNESS_MUTATED: + case TxValidationResult::TX_WITNESS_STRIPPED: case TxValidationResult::TX_CONFLICT: case TxValidationResult::TX_MEMPOOL_POLICY: break; @@ -1169,14 +1206,15 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CS recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); // Blocks don't typically have more than 4000 transactions, so this should - // be at least six blocks (~1 hr) worth of transactions that we can store. + // be at least six blocks (~1 hr) worth of transactions that we can store, + // inserting both a txid and wtxid for every observed transaction. // If the number of transactions appearing in a block goes up, or if we are // seeing getdata requests more than an hour after initial announcement, we // can increase this number. // The false positive rate of 1/1M should come out to less than 1 // transaction per day that would be inadvertently ignored (which is the // same probability that we have in the reject filter). - g_recent_confirmed_transactions.reset(new CRollingBloomFilter(24000, 0.000001)); + g_recent_confirmed_transactions.reset(new CRollingBloomFilter(48000, 0.000001)); const Consensus::Params& consensusParams = Params().GetConsensus(); // Stale tip checking and peer eviction are on two different timers, but we @@ -1232,6 +1270,9 @@ void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pb LOCK(g_cs_recent_confirmed_transactions); for (const auto& ptx : pblock->vtx) { g_recent_confirmed_transactions->insert(ptx->GetHash()); + if (ptx->GetHash() != ptx->GetWitnessHash()) { + g_recent_confirmed_transactions->insert(ptx->GetWitnessHash()); + } } } } @@ -1328,9 +1369,10 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB } // Relay inventory, but don't relay old inventory during initial block download. connman->ForEachNode([nNewHeight, &vHashes](CNode* pnode) { + LOCK(pnode->cs_inventory); if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : 0)) { for (const uint256& hash : reverse_iterate(vHashes)) { - pnode->PushBlockHash(hash); + pnode->vBlockHashesToAnnounce.push_back(hash); } } }); @@ -1339,7 +1381,7 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB } /** - * Handle invalid block rejection and consequent peer banning, maintain which + * Handle invalid block rejection and consequent peer discouragement, maintain which * peers announce compact blocks. */ void PeerLogicValidation::BlockChecked(const CBlock& block, const BlockValidationState& state) { @@ -1365,7 +1407,7 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const BlockValidatio !::ChainstateActive().IsInitialBlockDownload() && mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) { if (it != mapBlockSource.end()) { - MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, connman); + MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, *connman); } } if (it != mapBlockSource.end()) @@ -1384,6 +1426,7 @@ bool static AlreadyHave(const CInv& inv, const CTxMemPool& mempool) EXCLUSIVE_LO { case MSG_TX: case MSG_WITNESS_TX: + case MSG_WTX: { assert(recentRejects); if (::ChainActive().Tip()->GetBlockHash() != hashRecentRejectsChainTip) @@ -1398,7 +1441,11 @@ bool static AlreadyHave(const CInv& inv, const CTxMemPool& mempool) EXCLUSIVE_LO { LOCK(g_cs_orphans); - if (mapOrphanTransactions.count(inv.hash)) return true; + if (inv.type != MSG_WTX && mapOrphanTransactions.count(inv.hash)) { + return true; + } else if (inv.type == MSG_WTX && g_orphans_by_wtxid.count(inv.hash)) { + return true; + } } { @@ -1406,8 +1453,8 @@ bool static AlreadyHave(const CInv& inv, const CTxMemPool& mempool) EXCLUSIVE_LO if (g_recent_confirmed_transactions->contains(inv.hash)) return true; } - return recentRejects->contains(inv.hash) || - mempool.exists(inv.hash); + const bool by_wtxid = (inv.type == MSG_WTX); + return recentRejects->contains(inv.hash) || mempool.exists(inv.hash, by_wtxid); } case MSG_BLOCK: case MSG_WITNESS_BLOCK: @@ -1417,12 +1464,17 @@ bool static AlreadyHave(const CInv& inv, const CTxMemPool& mempool) EXCLUSIVE_LO return true; } -void RelayTransaction(const uint256& txid, const CConnman& connman) +void RelayTransaction(const uint256& txid, const uint256& wtxid, const CConnman& connman) { - CInv inv(MSG_TX, txid); - connman.ForEachNode([&inv](CNode* pnode) + connman.ForEachNode([&txid, &wtxid](CNode* pnode) { - pnode->PushInventory(inv); + AssertLockHeld(cs_main); + CNodeState &state = *State(pnode->GetId()); + if (state.m_wtxid_relay) { + pnode->PushTxInventory(wtxid); + } else { + pnode->PushTxInventory(txid); + } }); } @@ -1441,7 +1493,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& assert(nRelayNodes <= best.size()); auto sortfunc = [&best, &hasher, nRelayNodes](CNode* pnode) { - if (pnode->nVersion >= CADDR_TIME_VERSION && pnode->IsAddrRelayPeer()) { + if (pnode->IsAddrRelayPeer()) { uint64_t hashKey = CSipHasher(hasher).Write(pnode->GetId()).Finalize(); for (unsigned int i = 0; i < nRelayNodes; i++) { if (hashKey > best[i].first) { @@ -1462,7 +1514,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& connman.ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); } -void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, const CInv& inv, CConnman* connman) +void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, const CInv& inv, CConnman& connman) { bool send = false; std::shared_ptr<const CBlock> a_recent_block; @@ -1510,9 +1562,9 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); // disconnect node in case we have reached the outbound limit for serving historical blocks if (send && - connman->OutboundTargetReached(true) && + connman.OutboundTargetReached(true) && (((pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - pindex->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.type == MSG_FILTERED_BLOCK) && - !pfrom.HasPermission(PF_NOBAN) // never disconnect nodes with the noban permission + !pfrom.HasPermission(PF_DOWNLOAD) // nodes with the download permission may exceed target ) { LogPrint(BCLog::NET, "historical block serving limit reached, disconnect peer=%d\n", pfrom.GetId()); @@ -1544,7 +1596,7 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c if (!ReadRawBlockFromDisk(block_data, pindex, chainparams.MessageStart())) { assert(!"cannot load block from disk"); } - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, MakeSpan(block_data))); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, MakeSpan(block_data))); // Don't set pblock as we've sent the block } else { // Send block from disk @@ -1555,9 +1607,9 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c } if (pblock) { if (inv.type == MSG_BLOCK) - connman->PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); + connman.PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_WITNESS_BLOCK) - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_FILTERED_BLOCK) { bool sendMerkleBlock = false; @@ -1570,7 +1622,7 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c } } if (sendMerkleBlock) { - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see // This avoids hurting performance by pointlessly requiring a round-trip // Note that there is currently no way for a node to request any single transactions we didn't send here - @@ -1579,7 +1631,7 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; for (PairType& pair : merkleBlock.vMatchedTxn) - connman->PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); + connman.PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); } // else // no response @@ -1594,13 +1646,13 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c int nSendFlags = fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; if (CanDirectFetch(consensusParams) && pindex->nHeight >= ::ChainActive().Height() - MAX_CMPCTBLOCK_DEPTH) { if ((fPeerWantsWitness || !fWitnessesPresentInARecentCompactBlock) && a_recent_compact_block && a_recent_compact_block->header.GetHash() == pindex->GetBlockHash()) { - connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); + connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); } else { CBlockHeaderAndShortTxIDs cmpctblock(*pblock, fPeerWantsWitness); - connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); } } else { - connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, *pblock)); + connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, *pblock)); } } } @@ -1608,49 +1660,46 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c // Trigger the peer node to send a getblocks request for the next batch of inventory if (inv.hash == pfrom.hashContinue) { - // Bypass PushInventory, this must send even if redundant, + // Send immediately. This must send even if redundant, // and we want it right after the last block so they don't // wait for other stuff first. std::vector<CInv> vInv; vInv.push_back(CInv(MSG_BLOCK, ::ChainActive().Tip()->GetBlockHash())); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::INV, vInv)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::INV, vInv)); pfrom.hashContinue.SetNull(); } } } //! Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). -CTransactionRef static FindTxForGetData(CNode& peer, const uint256& txid, const std::chrono::seconds mempool_req, const std::chrono::seconds longlived_mempool_time) LOCKS_EXCLUDED(cs_main) +CTransactionRef static FindTxForGetData(const CNode& peer, const uint256& txid_or_wtxid, bool use_wtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) LOCKS_EXCLUDED(cs_main) { - // Check if the requested transaction is so recent that we're just - // about to announce it to the peer; if so, they certainly shouldn't - // know we already have it. - { - LOCK(peer.m_tx_relay->cs_tx_inventory); - if (peer.m_tx_relay->setInventoryTxToSend.count(txid)) return {}; + auto txinfo = mempool.info(txid_or_wtxid, use_wtxid); + if (txinfo.tx) { + // If a TX could have been INVed in reply to a MEMPOOL request, + // or is older than UNCONDITIONAL_RELAY_DELAY, permit the request + // unconditionally. + if ((mempool_req.count() && txinfo.m_time <= mempool_req) || txinfo.m_time <= now - UNCONDITIONAL_RELAY_DELAY) { + return std::move(txinfo.tx); + } } { LOCK(cs_main); - // Look up transaction in relay pool - auto mi = mapRelay.find(txid); - if (mi != mapRelay.end()) return mi->second; - } - - auto txinfo = mempool.info(txid); - if (txinfo.tx) { - // To protect privacy, do not answer getdata using the mempool when - // that TX couldn't have been INVed in reply to a MEMPOOL request, - // or when it's too recent to have expired from mapRelay. - if ((mempool_req.count() && txinfo.m_time <= mempool_req) || txinfo.m_time <= longlived_mempool_time) { - return txinfo.tx; + // Otherwise, the transaction must have been announced recently. + if (State(peer.GetId())->m_recently_announced_invs.contains(txid_or_wtxid)) { + // If it was, it can be relayed from either the mempool... + if (txinfo.tx) return std::move(txinfo.tx); + // ... or the relay pool. + auto mi = mapRelay.find(txid_or_wtxid); + if (mi != mapRelay.end()) return mi->second; } } return {}; } -void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnman* connman, CTxMemPool& mempool, const std::atomic<bool>& interruptMsgProc) LOCKS_EXCLUDED(cs_main) +void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnman& connman, CTxMemPool& mempool, const std::atomic<bool>& interruptMsgProc) LOCKS_EXCLUDED(cs_main) { AssertLockNotHeld(cs_main); @@ -1658,8 +1707,7 @@ void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnm std::vector<CInv> vNotFound; const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); - // mempool entries added before this time have likely expired from mapRelay - const std::chrono::seconds longlived_mempool_time = GetTime<std::chrono::seconds>() - RELAY_TX_CACHE_TIME; + const std::chrono::seconds now = GetTime<std::chrono::seconds>(); // Get last mempool request time const std::chrono::seconds mempool_req = pfrom.m_tx_relay != nullptr ? pfrom.m_tx_relay->m_last_mempool_req.load() : std::chrono::seconds::min(); @@ -1667,7 +1715,7 @@ void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnm // Process as many TX items from the front of the getdata queue as // possible, since they're common and it's efficient to batch process // them. - while (it != pfrom.vRecvGetData.end() && (it->type == MSG_TX || it->type == MSG_WITNESS_TX)) { + while (it != pfrom.vRecvGetData.end() && (it->type == MSG_TX || it->type == MSG_WITNESS_TX || it->type == MSG_WTX)) { if (interruptMsgProc) return; // The send buffer provides backpressure. If there's no space in // the buffer, pause processing until the next call. @@ -1680,11 +1728,23 @@ void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnm continue; } - CTransactionRef tx = FindTxForGetData(pfrom, inv.hash, mempool_req, longlived_mempool_time); + CTransactionRef tx = FindTxForGetData(pfrom, inv.hash, inv.type == MSG_WTX, mempool_req, now); if (tx) { + // WTX and WITNESS_TX imply we serialize with witness int nSendFlags = (inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); - connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); - mempool.RemoveUnbroadcastTx(inv.hash); + connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); + mempool.RemoveUnbroadcastTx(tx->GetHash()); + // As we're going to send tx, make sure its unconfirmed parents are made requestable. + for (const auto& txin : tx->vin) { + auto txinfo = mempool.info(txin.prevout.hash); + if (txinfo.tx && txinfo.m_time > now - UNCONDITIONAL_RELAY_DELAY) { + // Relaying a transaction with a recent but unconfirmed parent. + if (WITH_LOCK(pfrom.m_tx_relay->cs_tx_inventory, return !pfrom.m_tx_relay->filterInventoryKnown.contains(txin.prevout.hash))) { + LOCK(cs_main); + State(pfrom.GetId())->m_recently_announced_invs.insert(txin.prevout.hash); + } + } + } } else { vNotFound.push_back(inv); } @@ -1718,7 +1778,7 @@ void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnm // In normal operation, we often send NOTFOUND messages for parents of // transactions that we relay; if a peer is missing a parent, they may // assume we have them and request the parents from us. - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); } } @@ -1730,12 +1790,12 @@ static uint32_t GetFetchFlags(const CNode& pfrom) EXCLUSIVE_LOCKS_REQUIRED(cs_ma return nFetchFlags; } -inline void static SendBlockTransactions(const CBlock& block, const BlockTransactionsRequest& req, CNode& pfrom, CConnman* connman) { +inline void static SendBlockTransactions(const CBlock& block, const BlockTransactionsRequest& req, CNode& pfrom, CConnman& connman) { BlockTransactions resp(req); for (size_t i = 0; i < req.indexes.size(); i++) { if (req.indexes[i] >= block.vtx.size()) { LOCK(cs_main); - Misbehaving(pfrom.GetId(), 100, strprintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom.GetId())); + Misbehaving(pfrom.GetId(), 100, "getblocktxn with out-of-bounds tx indices"); return; } resp.txn[i] = block.vtx[req.indexes[i]]; @@ -1743,17 +1803,17 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac LOCK(cs_main); const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); int nSendFlags = State(pfrom.GetId())->fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; - connman->PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); + connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); } -bool static ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateManager& chainman, CTxMemPool& mempool, const std::vector<CBlockHeader>& headers, const CChainParams& chainparams, bool via_compact_block) +static void ProcessHeadersMessage(CNode& pfrom, CConnman& connman, ChainstateManager& chainman, CTxMemPool& mempool, const std::vector<CBlockHeader>& headers, const CChainParams& chainparams, bool via_compact_block) { const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); size_t nCount = headers.size(); if (nCount == 0) { // Nothing interesting. Stop asking this peers for more headers. - return true; + return; } bool received_new_header = false; @@ -1772,7 +1832,7 @@ bool static ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateMan // nUnconnectingHeaders gets reset back to 0. if (!LookupBlockIndex(headers[0].hashPrevBlock) && nCount < MAX_BLOCKS_TO_ANNOUNCE) { nodestate->nUnconnectingHeaders++; - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), @@ -1784,16 +1844,16 @@ bool static ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateMan UpdateBlockAvailability(pfrom.GetId(), headers.back().GetHash()); if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { - Misbehaving(pfrom.GetId(), 20); + Misbehaving(pfrom.GetId(), 20, strprintf("%d non-connecting headers", nodestate->nUnconnectingHeaders)); } - return true; + return; } uint256 hashLastBlock; for (const CBlockHeader& header : headers) { if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { Misbehaving(pfrom.GetId(), 20, "non-continuous headers sequence"); - return false; + return; } hashLastBlock = header.GetHash(); } @@ -1809,7 +1869,7 @@ bool static ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateMan if (!chainman.ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { if (state.IsInvalid()) { MaybePunishNodeForBlock(pfrom.GetId(), state, via_compact_block, "invalid header received"); - return false; + return; } } @@ -1837,7 +1897,7 @@ bool static ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateMan // TODO: optimize: if pindexLast is an ancestor of ::ChainActive().Tip or pindexBestHeader, continue // from there instead. LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom.GetId(), pfrom.nStartingHeight); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexLast), uint256())); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexLast), uint256())); } bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); @@ -1887,7 +1947,7 @@ bool static ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateMan // In any case, we want to download using a compact block, not a regular one vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); } - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); } } } @@ -1898,8 +1958,8 @@ bool static ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateMan // headers to fetch from this peer. if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { // This peer has too little work on their headers chain to help - // us sync -- disconnect if using an outbound slot (unless - // whitelisted or addnode). + // us sync -- disconnect if it is an outbound disconnection + // candidate. // Note: We compare their tip to nMinimumChainWork (rather than // ::ChainActive().Tip()) because we won't start block download // until we have a headers chain that has at least @@ -1925,10 +1985,10 @@ bool static ProcessHeadersMessage(CNode& pfrom, CConnman* connman, ChainstateMan } } - return true; + return; } -void static ProcessOrphanTx(CConnman* connman, CTxMemPool& mempool, std::set<uint256>& orphan_work_set, std::list<CTransactionRef>& removed_txn) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans) +void static ProcessOrphanTx(CConnman& connman, CTxMemPool& mempool, std::set<uint256>& orphan_work_set, std::list<CTransactionRef>& removed_txn) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans) { AssertLockHeld(cs_main); AssertLockHeld(g_cs_orphans); @@ -1952,7 +2012,7 @@ void static ProcessOrphanTx(CConnman* connman, CTxMemPool& mempool, std::set<uin if (setMisbehaving.count(fromPeer)) continue; if (AcceptToMemoryPool(mempool, orphan_state, porphanTx, &removed_txn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); - RelayTransaction(orphanHash, *connman); + RelayTransaction(orphanHash, porphanTx->GetWitnessHash(), connman); for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(orphanHash, i)); if (it_by_prev != mapOrphanTransactionsByPrev.end()) { @@ -1969,17 +2029,30 @@ void static ProcessOrphanTx(CConnman* connman, CTxMemPool& mempool, std::set<uin if (MaybePunishNodeForTx(fromPeer, orphan_state)) { setMisbehaving.insert(fromPeer); } - LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s\n", orphanHash.ToString()); + LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s from peer=%d. %s\n", + orphanHash.ToString(), + fromPeer, + orphan_state.ToString()); } // Has inputs but not accepted to mempool // Probably non-standard or insufficient fee LogPrint(BCLog::MEMPOOL, " removed orphan tx %s\n", orphanHash.ToString()); - if (!orphanTx.HasWitness() && orphan_state.GetResult() != TxValidationResult::TX_WITNESS_MUTATED) { - // Do not use rejection cache for witness transactions or - // witness-stripped transactions, as they can have been malleated. - // See https://github.com/bitcoin/bitcoin/issues/8279 for details. + if (orphan_state.GetResult() != TxValidationResult::TX_WITNESS_STRIPPED) { + // We can add the wtxid of this transaction to our reject filter. + // Do not add txids of witness transactions or witness-stripped + // transactions to the filter, as they can have been malleated; + // adding such txids to the reject filter would potentially + // interfere with relay of valid transactions from peers that + // do not support wtxid-based relay. See + // https://github.com/bitcoin/bitcoin/issues/8279 for details. + // We can remove this restriction (and always add wtxids to + // the filter even for witness stripped transactions) once + // wtxid-based relay is broadly deployed. + // See also comments in https://github.com/bitcoin/bitcoin/pull/18044#discussion_r443419034 + // for concerns around weakening security of unupgraded nodes + // if we start doing this too early. assert(recentRejects); - recentRejects->insert(orphanHash); + recentRejects->insert(orphanTx.GetWitnessHash()); } EraseOrphanTx(orphanHash); done = true; @@ -2205,37 +2278,33 @@ static void ProcessGetCFCheckPt(CNode& pfrom, CDataStream& vRecv, const CChainPa connman.PushMessage(&pfrom, std::move(msg)); } -bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, ChainstateManager& chainman, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) +void ProcessMessage( + CNode& pfrom, + const std::string& msg_type, + CDataStream& vRecv, + const std::chrono::microseconds time_received, + const CChainParams& chainparams, + ChainstateManager& chainman, + CTxMemPool& mempool, + CConnman& connman, + BanMan* banman, + const std::atomic<bool>& interruptMsgProc) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(msg_type), vRecv.size(), pfrom.GetId()); if (gArgs.IsArgSet("-dropmessagestest") && GetRand(gArgs.GetArg("-dropmessagestest", 0)) == 0) { LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); - return true; + return; } - if (!(pfrom.GetLocalServices() & NODE_BLOOM) && - (msg_type == NetMsgType::FILTERLOAD || - msg_type == NetMsgType::FILTERADD)) - { - if (pfrom.nVersion >= NO_BLOOM_VERSION) { - LOCK(cs_main); - Misbehaving(pfrom.GetId(), 100); - return false; - } else { - pfrom.fDisconnect = true; - return false; - } - } - if (msg_type == NetMsgType::VERSION) { // Each connection can only send one version message if (pfrom.nVersion != 0) { LOCK(cs_main); - Misbehaving(pfrom.GetId(), 1); - return false; + Misbehaving(pfrom.GetId(), 1, "redundant version message"); + return; } int64_t nTime; @@ -2255,20 +2324,20 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec nServices = ServiceFlags(nServiceInt); if (!pfrom.fInbound) { - connman->SetServices(pfrom.addr, nServices); + connman.SetServices(pfrom.addr, nServices); } if (!pfrom.fInbound && !pfrom.fFeeler && !pfrom.m_manual_connection && !HasAllDesirableServiceFlags(nServices)) { LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom.GetId(), nServices, GetDesirableServiceFlags(nServices)); pfrom.fDisconnect = true; - return false; + return; } if (nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version LogPrint(BCLog::NET, "peer=%d using obsolete version %i; disconnecting\n", pfrom.GetId(), nVersion); pfrom.fDisconnect = true; - return false; + return; } if (!vRecv.empty()) @@ -2284,11 +2353,11 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec if (!vRecv.empty()) vRecv >> fRelay; // Disconnect if we connected to ourself - if (pfrom.fInbound && !connman->CheckIncomingNonce(nNonce)) + if (pfrom.fInbound && !connman.CheckIncomingNonce(nNonce)) { LogPrintf("connected to self at %s, disconnecting\n", pfrom.addr.ToString()); pfrom.fDisconnect = true; - return true; + return; } if (pfrom.fInbound && addrMe.IsRoutable()) @@ -2300,7 +2369,11 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec if (pfrom.fInbound) PushNodeVersion(pfrom, connman, GetAdjustedTime()); - connman->PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); + if (nVersion >= WTXID_RELAY_VERSION) { + connman.PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::WTXIDRELAY)); + } + + connman.PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); pfrom.nServices = nServices; pfrom.SetAddrLocal(addrMe); @@ -2356,12 +2429,9 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec } // Get recent addresses - if (pfrom.fOneShot || pfrom.nVersion >= CADDR_TIME_VERSION || connman->GetAddressCount() < 1000) - { - connman->PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); - pfrom.fGetAddr = true; - } - connman->MarkAddressGood(pfrom.addr); + connman.PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); + pfrom.fGetAddr = true; + connman.MarkAddressGood(pfrom.addr); } std::string remoteAddr; @@ -2380,7 +2450,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // If the peer is old enough to have the old alert system, send it the final alert. if (pfrom.nVersion <= 70012) { CDataStream finalAlert(ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50"), SER_NETWORK, PROTOCOL_VERSION); - connman->PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make("alert", finalAlert)); + connman.PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make("alert", finalAlert)); } // Feeler connections exist only to verify if address is online. @@ -2388,14 +2458,14 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec assert(pfrom.fInbound == false); pfrom.fDisconnect = true; } - return true; + return; } if (pfrom.nVersion == 0) { // Must have a version message before anything else LOCK(cs_main); - Misbehaving(pfrom.GetId(), 1); - return false; + Misbehaving(pfrom.GetId(), 1, "non-version message before version handshake"); + return; } // At this point, the outgoing message serialization version can't change. @@ -2420,7 +2490,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // We send this to non-NODE NETWORK peers as well, because even // non-NODE NETWORK peers can announce blocks (such as pruning // nodes) - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); } if (pfrom.nVersion >= SHORT_IDS_BLOCKS_VERSION) { // Tell our peer we are willing to provide version 1 or 2 cmpctblocks @@ -2431,36 +2501,52 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 2; if (pfrom.GetLocalServices() & NODE_WITNESS) - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); nCMPCTBLOCKVersion = 1; - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); } pfrom.fSuccessfullyConnected = true; - return true; + return; + } + + // Feature negotiation of wtxidrelay should happen between VERSION and + // VERACK, to avoid relay problems from switching after a connection is up + if (msg_type == NetMsgType::WTXIDRELAY) { + if (pfrom.fSuccessfullyConnected) { + // Disconnect peers that send wtxidrelay message after VERACK; this + // must be negotiated between VERSION and VERACK. + pfrom.fDisconnect = true; + return; + } + if (pfrom.nVersion >= WTXID_RELAY_VERSION) { + LOCK(cs_main); + if (!State(pfrom.GetId())->m_wtxid_relay) { + State(pfrom.GetId())->m_wtxid_relay = true; + g_wtxid_relay_peers++; + } + } + return; } if (!pfrom.fSuccessfullyConnected) { // Must have a verack message before anything else LOCK(cs_main); - Misbehaving(pfrom.GetId(), 1); - return false; + Misbehaving(pfrom.GetId(), 1, "non-verack message before version handshake"); + return; } if (msg_type == NetMsgType::ADDR) { std::vector<CAddress> vAddr; vRecv >> vAddr; - // Don't want addr from older versions unless seeding - if (pfrom.nVersion < CADDR_TIME_VERSION && connman->GetAddressCount() > 1000) - return true; if (!pfrom.IsAddrRelayPeer()) { - return true; + return; } if (vAddr.size() > 1000) { LOCK(cs_main); - Misbehaving(pfrom.GetId(), 20, strprintf("message addr size() = %u", vAddr.size())); - return false; + Misbehaving(pfrom.GetId(), 20, strprintf("addr message size = %u", vAddr.size())); + return; } // Store the new addresses @@ -2470,7 +2556,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec for (CAddress& addr : vAddr) { if (interruptMsgProc) - return true; + return; // We only bother storing full nodes, though this may include // things which we would not make an outbound connection to, in @@ -2481,29 +2567,32 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; pfrom.AddAddressKnown(addr); - if (banman->IsBanned(addr)) continue; // Do not process banned addresses beyond remembering we received them + if (banman && (banman->IsDiscouraged(addr) || banman->IsBanned(addr))) { + // Do not process banned/discouraged addresses beyond remembering we received them + continue; + } bool fReachable = IsReachable(addr); if (addr.nTime > nSince && !pfrom.fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { // Relay to a limited number of other nodes - RelayAddress(addr, fReachable, *connman); + RelayAddress(addr, fReachable, connman); } // Do not store addresses outside our network if (fReachable) vAddrOk.push_back(addr); } - connman->AddNewAddresses(vAddrOk, pfrom.addr, 2 * 60 * 60); + connman.AddNewAddresses(vAddrOk, pfrom.addr, 2 * 60 * 60); if (vAddr.size() < 1000) pfrom.fGetAddr = false; if (pfrom.fOneShot) pfrom.fDisconnect = true; - return true; + return; } if (msg_type == NetMsgType::SENDHEADERS) { LOCK(cs_main); State(pfrom.GetId())->fPreferHeaders = true; - return true; + return; } if (msg_type == NetMsgType::SENDCMPCT) { @@ -2526,7 +2615,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec State(pfrom.GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 1); } } - return true; + return; } if (msg_type == NetMsgType::INV) { @@ -2535,17 +2624,18 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec if (vInv.size() > MAX_INV_SZ) { LOCK(cs_main); - Misbehaving(pfrom.GetId(), 20, strprintf("message inv size() = %u", vInv.size())); - return false; + Misbehaving(pfrom.GetId(), 20, strprintf("inv message size = %u", vInv.size())); + return; } // We won't accept tx inv's if we're in blocks-only mode, or this is a // block-relay-only peer bool fBlocksOnly = !g_relay_txes || (pfrom.m_tx_relay == nullptr); - // Allow whitelisted peers to send data other than blocks in blocks only mode if whitelistrelay is true - if (pfrom.HasPermission(PF_RELAY)) + // Allow peers with relay permission to send data other than blocks in blocks only mode + if (pfrom.HasPermission(PF_RELAY)) { fBlocksOnly = false; + } LOCK(cs_main); @@ -2556,7 +2646,14 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec for (CInv &inv : vInv) { if (interruptMsgProc) - return true; + return; + + // ignore INVs that don't match wtxidrelay setting + if (State(pfrom.GetId())->m_wtxid_relay) { + if (inv.type == MSG_TX) continue; + } else { + if (inv.type == MSG_WTX) continue; + } bool fAlreadyHave = AlreadyHave(inv, mempool); LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); @@ -2576,23 +2673,23 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec best_block = &inv.hash; } } else { - pfrom.AddInventoryKnown(inv); + pfrom.AddKnownTx(inv.hash); if (fBlocksOnly) { LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); pfrom.fDisconnect = true; - return true; - } else if (!fAlreadyHave && !fImporting && !fReindex && !::ChainstateActive().IsInitialBlockDownload()) { + return; + } else if (!fAlreadyHave && !chainman.ActiveChainstate().IsInitialBlockDownload()) { RequestTx(State(pfrom.GetId()), inv.hash, current_time); } } } if (best_block != nullptr) { - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), *best_block)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), *best_block)); LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, best_block->ToString(), pfrom.GetId()); } - return true; + return; } if (msg_type == NetMsgType::GETDATA) { @@ -2601,8 +2698,8 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec if (vInv.size() > MAX_INV_SZ) { LOCK(cs_main); - Misbehaving(pfrom.GetId(), 20, strprintf("message getdata size() = %u", vInv.size())); - return false; + Misbehaving(pfrom.GetId(), 20, strprintf("getdata message size = %u", vInv.size())); + return; } LogPrint(BCLog::NET, "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom.GetId()); @@ -2613,7 +2710,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec pfrom.vRecvGetData.insert(pfrom.vRecvGetData.end(), vInv.begin(), vInv.end()); ProcessGetData(pfrom, chainparams, connman, mempool, interruptMsgProc); - return true; + return; } if (msg_type == NetMsgType::GETBLOCKS) { @@ -2624,7 +2721,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec if (locator.vHave.size() > MAX_LOCATOR_SZ) { LogPrint(BCLog::NET, "getblocks locator size %lld > %d, disconnect peer=%d\n", locator.vHave.size(), MAX_LOCATOR_SZ, pfrom.GetId()); pfrom.fDisconnect = true; - return true; + return; } // We might have announced the currently-being-connected tip using a @@ -2671,7 +2768,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec LogPrint(BCLog::NET, " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); break; } - pfrom.PushInventory(CInv(MSG_BLOCK, pindex->GetBlockHash())); + WITH_LOCK(pfrom.cs_inventory, pfrom.vInventoryBlockToSend.push_back(pindex->GetBlockHash())); if (--nLimit <= 0) { // When this block is requested, we'll send an inv that'll @@ -2681,7 +2778,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec break; } } - return true; + return; } if (msg_type == NetMsgType::GETBLOCKTXN) { @@ -2697,7 +2794,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec } if (recent_block) { SendBlockTransactions(*recent_block, req, pfrom, connman); - return true; + return; } LOCK(cs_main); @@ -2705,7 +2802,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec const CBlockIndex* pindex = LookupBlockIndex(req.blockhash); if (!pindex || !(pindex->nStatus & BLOCK_HAVE_DATA)) { LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block we don't have\n", pfrom.GetId()); - return true; + return; } if (pindex->nHeight < ::ChainActive().Height() - MAX_BLOCKTXN_DEPTH) { @@ -2722,7 +2819,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec inv.hash = req.blockhash; pfrom.vRecvGetData.push_back(inv); // The message processing loop will go around again (without pausing) and we'll respond then (without cs_main) - return true; + return; } CBlock block; @@ -2730,7 +2827,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec assert(ret); SendBlockTransactions(block, req, pfrom, connman); - return true; + return; } if (msg_type == NetMsgType::GETHEADERS) { @@ -2741,13 +2838,13 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec if (locator.vHave.size() > MAX_LOCATOR_SZ) { LogPrint(BCLog::NET, "getheaders locator size %lld > %d, disconnect peer=%d\n", locator.vHave.size(), MAX_LOCATOR_SZ, pfrom.GetId()); pfrom.fDisconnect = true; - return true; + return; } LOCK(cs_main); - if (::ChainstateActive().IsInitialBlockDownload() && !pfrom.HasPermission(PF_NOBAN)) { + if (::ChainstateActive().IsInitialBlockDownload() && !pfrom.HasPermission(PF_DOWNLOAD)) { LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom.GetId()); - return true; + return; } CNodeState *nodestate = State(pfrom.GetId()); @@ -2757,12 +2854,12 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // If locator is null, return the hashStop block pindex = LookupBlockIndex(hashStop); if (!pindex) { - return true; + return; } if (!BlockRequestAllowed(pindex, chainparams.GetConsensus())) { LogPrint(BCLog::NET, "%s: ignoring request from peer=%i for old block header that isn't in the main chain\n", __func__, pfrom.GetId()); - return true; + return; } } else @@ -2796,8 +2893,8 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // will re-announce the new block via headers (or compact blocks again) // in the SendMessages logic. nodestate->pindexBestHeaderSent = pindex ? pindex : ::ChainActive().Tip(); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); - return true; + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + return; } if (msg_type == NetMsgType::TX) { @@ -2808,33 +2905,57 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec { LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom.GetId()); pfrom.fDisconnect = true; - return true; + return; } CTransactionRef ptx; vRecv >> ptx; const CTransaction& tx = *ptx; - CInv inv(MSG_TX, tx.GetHash()); - pfrom.AddInventoryKnown(inv); + const uint256& txid = ptx->GetHash(); + const uint256& wtxid = ptx->GetWitnessHash(); LOCK2(cs_main, g_cs_orphans); + CNodeState* nodestate = State(pfrom.GetId()); + + const uint256& hash = nodestate->m_wtxid_relay ? wtxid : txid; + pfrom.AddKnownTx(hash); + if (nodestate->m_wtxid_relay && txid != wtxid) { + // Insert txid into filterInventoryKnown, even for + // wtxidrelay peers. This prevents re-adding of + // unconfirmed parents to the recently_announced + // filter, when a child tx is requested. See + // ProcessGetData(). + pfrom.AddKnownTx(txid); + } + TxValidationState state; - CNodeState* nodestate = State(pfrom.GetId()); - nodestate->m_tx_download.m_tx_announced.erase(inv.hash); - nodestate->m_tx_download.m_tx_in_flight.erase(inv.hash); - EraseTxRequest(inv.hash); + nodestate->m_tx_download.m_tx_announced.erase(hash); + nodestate->m_tx_download.m_tx_in_flight.erase(hash); + EraseTxRequest(hash); std::list<CTransactionRef> lRemovedTxn; - if (!AlreadyHave(inv, mempool) && + // We do the AlreadyHave() check using wtxid, rather than txid - in the + // absence of witness malleation, this is strictly better, because the + // recent rejects filter may contain the wtxid but will never contain + // the txid of a segwit transaction that has been rejected. + // In the presence of witness malleation, it's possible that by only + // doing the check with wtxid, we could overlook a transaction which + // was confirmed with a different witness, or exists in our mempool + // with a different witness, but this has limited downside: + // mempool validation does its own lookup of whether we have the txid + // already; and an adversary can already relay us old transactions + // (older than our recency filter) if trying to DoS us, without any need + // for witness malleation. + if (!AlreadyHave(CInv(MSG_WTX, wtxid), mempool) && AcceptToMemoryPool(mempool, state, ptx, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { mempool.check(&::ChainstateActive().CoinsTip()); - RelayTransaction(tx.GetHash(), *connman); + RelayTransaction(tx.GetHash(), tx.GetWitnessHash(), connman); for (unsigned int i = 0; i < tx.vout.size(); i++) { - auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(inv.hash, i)); + auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(txid, i)); if (it_by_prev != mapOrphanTransactionsByPrev.end()) { for (const auto& elem : it_by_prev->second) { pfrom.orphan_work_set.insert(elem->first); @@ -2865,10 +2986,17 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec uint32_t nFetchFlags = GetFetchFlags(pfrom); const auto current_time = GetTime<std::chrono::microseconds>(); - for (const CTxIn& txin : tx.vin) { - CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); - pfrom.AddInventoryKnown(_inv); - if (!AlreadyHave(_inv, mempool)) RequestTx(State(pfrom.GetId()), _inv.hash, current_time); + if (!State(pfrom.GetId())->m_wtxid_relay) { + for (const CTxIn& txin : tx.vin) { + // Here, we only have the txid (and not wtxid) of the + // inputs, so we only request parents from + // non-wtxid-relay peers. + // Eventually we should replace this with an improved + // protocol for getting all unconfirmed parents. + CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); + pfrom.AddKnownTx(txin.prevout.hash); + if (!AlreadyHave(_inv, mempool)) RequestTx(State(pfrom.GetId()), _inv.hash, current_time); + } } AddOrphanTx(ptx, pfrom.GetId()); @@ -2882,15 +3010,30 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec LogPrint(BCLog::MEMPOOL, "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); // We will continue to reject this tx since it has rejected // parents so avoid re-requesting it from other peers. + // Here we add both the txid and the wtxid, as we know that + // regardless of what witness is provided, we will not accept + // this, so we don't need to allow for redownload of this txid + // from any of our non-wtxidrelay peers. recentRejects->insert(tx.GetHash()); + recentRejects->insert(tx.GetWitnessHash()); } } else { - if (!tx.HasWitness() && state.GetResult() != TxValidationResult::TX_WITNESS_MUTATED) { - // Do not use rejection cache for witness transactions or - // witness-stripped transactions, as they can have been malleated. - // See https://github.com/bitcoin/bitcoin/issues/8279 for details. + if (state.GetResult() != TxValidationResult::TX_WITNESS_STRIPPED) { + // We can add the wtxid of this transaction to our reject filter. + // Do not add txids of witness transactions or witness-stripped + // transactions to the filter, as they can have been malleated; + // adding such txids to the reject filter would potentially + // interfere with relay of valid transactions from peers that + // do not support wtxid-based relay. See + // https://github.com/bitcoin/bitcoin/issues/8279 for details. + // We can remove this restriction (and always add wtxids to + // the filter even for witness stripped transactions) once + // wtxid-based relay is broadly deployed. + // See also comments in https://github.com/bitcoin/bitcoin/pull/18044#discussion_r443419034 + // for concerns around weakening security of unupgraded nodes + // if we start doing this too early. assert(recentRejects); - recentRejects->insert(tx.GetHash()); + recentRejects->insert(tx.GetWitnessHash()); if (RecursiveDynamicUsage(*ptx) < 100000) { AddToCompactExtraTransactions(ptx); } @@ -2899,15 +3042,15 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec } if (pfrom.HasPermission(PF_FORCERELAY)) { - // Always relay transactions received from whitelisted peers, even + // Always relay transactions received from peers with forcerelay permission, even // if they were already in the mempool, // allowing the node to function as a gateway for // nodes hidden behind it. if (!mempool.exists(tx.GetHash())) { - LogPrintf("Not relaying non-mempool transaction %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); + LogPrintf("Not relaying non-mempool transaction %s from forcerelay peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); } else { - LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); - RelayTransaction(tx.GetHash(), *connman); + LogPrintf("Force relaying tx %s from peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); + RelayTransaction(tx.GetHash(), tx.GetWitnessHash(), connman); } } } @@ -2932,14 +3075,13 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // peer simply for relaying a tx that our recentRejects has caught, // regardless of false positives. - if (state.IsInvalid()) - { + if (state.IsInvalid()) { LogPrint(BCLog::MEMPOOLREJ, "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), pfrom.GetId(), state.ToString()); MaybePunishNodeForTx(pfrom.GetId(), state); } - return true; + return; } if (msg_type == NetMsgType::CMPCTBLOCK) @@ -2947,7 +3089,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // Ignore cmpctblock received while importing if (fImporting || fReindex) { LogPrint(BCLog::NET, "Unexpected cmpctblock message received from peer %d\n", pfrom.GetId()); - return true; + return; } CBlockHeaderAndShortTxIDs cmpctblock; @@ -2961,8 +3103,8 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec if (!LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers if (!::ChainstateActive().IsInitialBlockDownload()) - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); - return true; + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); + return; } if (!LookupBlockIndex(cmpctblock.header.GetHash())) { @@ -2975,7 +3117,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec if (!chainman.ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { if (state.IsInvalid()) { MaybePunishNodeForBlock(pfrom.GetId(), state, /*via_compact_block*/ true, "invalid header via cmpctblock"); - return true; + return; } } @@ -3013,7 +3155,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); if (pindex->nStatus & BLOCK_HAVE_DATA) // Nothing to do here - return true; + return; if (pindex->nChainWork <= ::ChainActive().Tip()->nChainWork || // We know something better pindex->nTx != 0) { // We had this block at some point, but pruned it @@ -3022,19 +3164,19 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // so we just grab the block via normal getdata std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } - return true; + return; } // If we're not close to tip yet, give up and let parallel block fetch work its magic if (!fAlreadyInFlight && !CanDirectFetch(chainparams.GetConsensus())) - return true; + return; if (IsWitnessEnabled(pindex->pprev, chainparams.GetConsensus()) && !nodestate->fSupportsDesiredCmpctVersion) { // Don't bother trying to process compact blocks from v1 peers // after segwit activates. - return true; + return; } // We want to be a bit conservative just to be extra careful about DoS @@ -3049,22 +3191,22 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec else { // The block was already in flight using compact blocks from the same peer LogPrint(BCLog::NET, "Peer sent us compact block we were already syncing!\n"); - return true; + return; } } PartiallyDownloadedBlock& partialBlock = *(*queuedBlockIt)->partialBlock; ReadStatus status = partialBlock.InitData(cmpctblock, vExtraTxnForCompact); if (status == READ_STATUS_INVALID) { - MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist - Misbehaving(pfrom.GetId(), 100, strprintf("Peer %d sent us invalid compact block\n", pfrom.GetId())); - return true; + MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case Misbehaving does not result in a disconnect + Misbehaving(pfrom.GetId(), 100, "invalid compact block"); + return; } else if (status == READ_STATUS_FAILED) { // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); - return true; + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + return; } BlockTransactionsRequest req; @@ -3080,7 +3222,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec fProcessBLOCKTXN = true; } else { req.blockhash = pindex->GetBlockHash(); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); } } else { // This block is either already in flight from a different @@ -3092,7 +3234,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec ReadStatus status = tempBlock.InitData(cmpctblock, vExtraTxnForCompact); if (status != READ_STATUS_OK) { // TODO: don't ignore failures - return true; + return; } std::vector<CTransactionRef> dummy; status = tempBlock.FillBlock(*pblock, dummy); @@ -3106,8 +3248,8 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); - return true; + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + return; } else { // If this was an announce-cmpctblock, we want the same treatment as a header message fRevertToHeaderProcessing = true; @@ -3116,14 +3258,14 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec } // cs_main if (fProcessBLOCKTXN) - return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, chainman, mempool, connman, banman, interruptMsgProc); + return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, time_received, chainparams, chainman, mempool, connman, banman, interruptMsgProc); if (fRevertToHeaderProcessing) { // Headers received from HB compact block peers are permitted to be // relayed before full validation (see BIP 152), so we don't want to disconnect // the peer if the header turns out to be for an invalid block. // Note that if a peer tries to build on an invalid chain, that - // will be detected and the peer will be banned. + // will be detected and the peer will be disconnected/discouraged. return ProcessHeadersMessage(pfrom, connman, chainman, mempool, {cmpctblock.header}, chainparams, /*via_compact_block=*/true); } @@ -3160,7 +3302,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec MarkBlockAsReceived(pblock->GetHash()); } } - return true; + return; } if (msg_type == NetMsgType::BLOCKTXN) @@ -3168,7 +3310,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // Ignore blocktxn received while importing if (fImporting || fReindex) { LogPrint(BCLog::NET, "Unexpected blocktxn message received from peer %d\n", pfrom.GetId()); - return true; + return; } BlockTransactions resp; @@ -3183,20 +3325,20 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || it->second.first != pfrom.GetId()) { LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom.GetId()); - return true; + return; } PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; ReadStatus status = partialBlock.FillBlock(*pblock, resp.txn); if (status == READ_STATUS_INVALID) { - MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist - Misbehaving(pfrom.GetId(), 100, strprintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom.GetId())); - return true; + MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case Misbehaving does not result in a disconnect + Misbehaving(pfrom.GetId(), 100, "invalid compact block/non-matching block transactions"); + return; } else if (status == READ_STATUS_FAILED) { // Might have collided, fall back to getdata now :( std::vector<CInv> invs; invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom), resp.blockhash)); - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { // Block is either okay, or possibly we received // READ_STATUS_CHECKBLOCK_FAILED. @@ -3209,7 +3351,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // 3. the block is otherwise invalid (eg invalid coinbase, // block is too big, too many legacy sigops, etc). // So if CheckBlock failed, #3 is the only possibility. - // Under BIP 152, we don't DoS-ban unless proof of work is + // Under BIP 152, we don't discourage the peer unless proof of work is // invalid (we don't require all the stateless checks to have // been run). This is handled below, so just treat this as // though the block was successfully read, and rely on the @@ -3242,7 +3384,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec mapBlockSource.erase(pblock->GetHash()); } } - return true; + return; } if (msg_type == NetMsgType::HEADERS) @@ -3250,7 +3392,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // Ignore headers received while importing if (fImporting || fReindex) { LogPrint(BCLog::NET, "Unexpected headers message received from peer %d\n", pfrom.GetId()); - return true; + return; } std::vector<CBlockHeader> headers; @@ -3260,7 +3402,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec if (nCount > MAX_HEADERS_RESULTS) { LOCK(cs_main); Misbehaving(pfrom.GetId(), 20, strprintf("headers message size = %u", nCount)); - return false; + return; } headers.resize(nCount); for (unsigned int n = 0; n < nCount; n++) { @@ -3276,7 +3418,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // Ignore block received while importing if (fImporting || fReindex) { LogPrint(BCLog::NET, "Unexpected block message received from peer %d\n", pfrom.GetId()); - return true; + return; } std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); @@ -3304,7 +3446,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec LOCK(cs_main); mapBlockSource.erase(pblock->GetHash()); } - return true; + return; } if (msg_type == NetMsgType::GETADDR) { @@ -3315,30 +3457,31 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // the getaddr message mitigates the attack. if (!pfrom.fInbound) { LogPrint(BCLog::NET, "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom.GetId()); - return true; + return; } if (!pfrom.IsAddrRelayPeer()) { LogPrint(BCLog::NET, "Ignoring \"getaddr\" from block-relay-only connection. peer=%d\n", pfrom.GetId()); - return true; + return; } // Only send one GetAddr response per connection to reduce resource waste // and discourage addr stamping of INV announcements. if (pfrom.fSentAddr) { LogPrint(BCLog::NET, "Ignoring repeated \"getaddr\". peer=%d\n", pfrom.GetId()); - return true; + return; } pfrom.fSentAddr = true; pfrom.vAddrToSend.clear(); - std::vector<CAddress> vAddr = connman->GetAddresses(); + std::vector<CAddress> vAddr = connman.GetAddresses(); FastRandomContext insecure_rand; for (const CAddress &addr : vAddr) { - if (!banman->IsBanned(addr)) { + bool banned_or_discouraged = banman && (banman->IsDiscouraged(addr) || banman->IsBanned(addr)); + if (!banned_or_discouraged) { pfrom.PushAddress(addr, insecure_rand); } } - return true; + return; } if (msg_type == NetMsgType::MEMPOOL) { @@ -3349,24 +3492,24 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec LogPrint(BCLog::NET, "mempool request with bloom filters disabled, disconnect peer=%d\n", pfrom.GetId()); pfrom.fDisconnect = true; } - return true; + return; } - if (connman->OutboundTargetReached(false) && !pfrom.HasPermission(PF_MEMPOOL)) + if (connman.OutboundTargetReached(false) && !pfrom.HasPermission(PF_MEMPOOL)) { if (!pfrom.HasPermission(PF_NOBAN)) { LogPrint(BCLog::NET, "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom.GetId()); pfrom.fDisconnect = true; } - return true; + return; } if (pfrom.m_tx_relay != nullptr) { LOCK(pfrom.m_tx_relay->cs_tx_inventory); pfrom.m_tx_relay->fSendMempool = true; } - return true; + return; } if (msg_type == NetMsgType::PING) { @@ -3385,13 +3528,13 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec // it, if the remote node sends a ping once per second and this node takes 5 // seconds to respond to each, the 5th ping the remote sends would appear to // return very quickly. - connman->PushMessage(&pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); } - return true; + return; } if (msg_type == NetMsgType::PONG) { - int64_t pingUsecEnd = nTimeReceived; + const auto ping_end = time_received; uint64_t nonce = 0; size_t nAvail = vRecv.in_avail(); bool bPingFinished = false; @@ -3405,11 +3548,11 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec if (nonce == pfrom.nPingNonceSent) { // Matching pong received, this ping is no longer outstanding bPingFinished = true; - int64_t pingUsecTime = pingUsecEnd - pfrom.nPingUsecStart; - if (pingUsecTime > 0) { + const auto ping_time = ping_end - pfrom.m_ping_start.load(); + if (ping_time.count() > 0) { // Successful ping time measurement, replace previous - pfrom.nPingUsecTime = pingUsecTime; - pfrom.nMinPingUsecTime = std::min(pfrom.nMinPingUsecTime.load(), pingUsecTime); + pfrom.nPingUsecTime = count_microseconds(ping_time); + pfrom.nMinPingUsecTime = std::min(pfrom.nMinPingUsecTime.load(), count_microseconds(ping_time)); } else { // This should never happen sProblem = "Timing mishap"; @@ -3443,10 +3586,14 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec if (bPingFinished) { pfrom.nPingNonceSent = 0; } - return true; + return; } if (msg_type == NetMsgType::FILTERLOAD) { + if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + pfrom.fDisconnect = true; + return; + } CBloomFilter filter; vRecv >> filter; @@ -3454,7 +3601,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec { // There is no excuse for sending a too-large filter LOCK(cs_main); - Misbehaving(pfrom.GetId(), 100); + Misbehaving(pfrom.GetId(), 100, "too-large bloom filter"); } else if (pfrom.m_tx_relay != nullptr) { @@ -3462,10 +3609,14 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec pfrom.m_tx_relay->pfilter.reset(new CBloomFilter(filter)); pfrom.m_tx_relay->fRelayTxes = true; } - return true; + return; } if (msg_type == NetMsgType::FILTERADD) { + if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + pfrom.fDisconnect = true; + return; + } std::vector<unsigned char> vData; vRecv >> vData; @@ -3484,21 +3635,23 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec } if (bad) { LOCK(cs_main); - Misbehaving(pfrom.GetId(), 100); + Misbehaving(pfrom.GetId(), 100, "bad filteradd message"); } - return true; + return; } if (msg_type == NetMsgType::FILTERCLEAR) { + if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + pfrom.fDisconnect = true; + return; + } if (pfrom.m_tx_relay == nullptr) { - return true; + return; } LOCK(pfrom.m_tx_relay->cs_filter); - if (pfrom.GetLocalServices() & NODE_BLOOM) { - pfrom.m_tx_relay->pfilter = nullptr; - } + pfrom.m_tx_relay->pfilter = nullptr; pfrom.m_tx_relay->fRelayTxes = true; - return true; + return; } if (msg_type == NetMsgType::FEEFILTER) { @@ -3511,22 +3664,22 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec } LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom.GetId()); } - return true; + return; } if (msg_type == NetMsgType::GETCFILTERS) { - ProcessGetCFilters(pfrom, vRecv, chainparams, *connman); - return true; + ProcessGetCFilters(pfrom, vRecv, chainparams, connman); + return; } if (msg_type == NetMsgType::GETCFHEADERS) { - ProcessGetCFHeaders(pfrom, vRecv, chainparams, *connman); - return true; + ProcessGetCFHeaders(pfrom, vRecv, chainparams, connman); + return; } if (msg_type == NetMsgType::GETCFCHECKPT) { - ProcessGetCFCheckPt(pfrom, vRecv, chainparams, *connman); - return true; + ProcessGetCFCheckPt(pfrom, vRecv, chainparams, connman); + return; } if (msg_type == NetMsgType::NOTFOUND) { @@ -3537,7 +3690,7 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec vRecv >> vInv; if (vInv.size() <= MAX_PEER_TX_IN_FLIGHT + MAX_BLOCKS_IN_TRANSIT_PER_PEER) { for (CInv &inv : vInv) { - if (inv.type == MSG_TX || inv.type == MSG_WITNESS_TX) { + if (inv.type == MSG_TX || inv.type == MSG_WITNESS_TX || inv.type == MSG_WTX) { // If we receive a NOTFOUND message for a txid we requested, erase // it from our data structures for this peer. auto in_flight_it = state->m_tx_download.m_tx_in_flight.find(inv.hash); @@ -3551,39 +3704,57 @@ bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRec } } } - return true; + return; } // Ignore unknown commands for extensibility LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(msg_type), pfrom.GetId()); - return true; + return; } -bool PeerLogicValidation::CheckIfBanned(CNode& pnode) +/** Maybe disconnect a peer and discourage future connections from its address. + * + * @param[in] pnode The node to check. + * @return True if the peer was marked for disconnection in this function + */ +bool PeerLogicValidation::MaybeDiscourageAndDisconnect(CNode& pnode) { - AssertLockHeld(cs_main); - CNodeState &state = *State(pnode.GetId()); - - if (state.fShouldBan) { - state.fShouldBan = false; - if (pnode.HasPermission(PF_NOBAN)) - LogPrintf("Warning: not punishing whitelisted peer %s!\n", pnode.addr.ToString()); - else if (pnode.m_manual_connection) - LogPrintf("Warning: not punishing manually-connected peer %s!\n", pnode.addr.ToString()); - else if (pnode.addr.IsLocal()) { - // Disconnect but don't ban _this_ local node - LogPrintf("Warning: disconnecting but not banning local peer %s!\n", pnode.addr.ToString()); - pnode.fDisconnect = true; - } else { - // Disconnect and ban all nodes sharing the address - if (m_banman) { - m_banman->Ban(pnode.addr, BanReasonNodeMisbehaving); - } - connman->DisconnectNode(pnode.addr); - } + const NodeId peer_id{pnode.GetId()}; + { + LOCK(cs_main); + CNodeState& state = *State(peer_id); + + // There's nothing to do if the m_should_discourage flag isn't set + if (!state.m_should_discourage) return false; + + state.m_should_discourage = false; + } // cs_main + + if (pnode.HasPermission(PF_NOBAN)) { + // We never disconnect or discourage peers for bad behavior if they have the NOBAN permission flag + LogPrintf("Warning: not punishing noban peer %d!\n", peer_id); + return false; + } + + if (pnode.m_manual_connection) { + // We never disconnect or discourage manual peers for bad behavior + LogPrintf("Warning: not punishing manually connected peer %d!\n", peer_id); + return false; + } + + if (pnode.addr.IsLocal()) { + // We disconnect local peers for bad behavior but don't discourage (since that would discourage + // all peers on the same local address) + LogPrintf("Warning: disconnecting but not discouraging local peer %d!\n", peer_id); + pnode.fDisconnect = true; return true; } - return false; + + // Normal case: Disconnect the peer and discourage all nodes sharing the address + LogPrintf("Disconnecting and discouraging peer %d!\n", peer_id); + if (m_banman) m_banman->Discourage(pnode.addr); + connman->DisconnectNode(pnode.addr); + return true; } bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& interruptMsgProc) @@ -3600,12 +3771,12 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter bool fMoreWork = false; if (!pfrom->vRecvGetData.empty()) - ProcessGetData(*pfrom, chainparams, connman, m_mempool, interruptMsgProc); + ProcessGetData(*pfrom, chainparams, *connman, m_mempool, interruptMsgProc); if (!pfrom->orphan_work_set.empty()) { std::list<CTransactionRef> removed_txn; LOCK2(cs_main, g_cs_orphans); - ProcessOrphanTx(connman, m_mempool, pfrom->orphan_work_set, removed_txn); + ProcessOrphanTx(*connman, m_mempool, pfrom->orphan_work_set, removed_txn); for (const CTransactionRef& removedTx : removed_txn) { AddToCompactExtraTransactions(removedTx); } @@ -3664,11 +3835,8 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter return fMoreWork; } - // Process message - bool fRet = false; - try - { - fRet = ProcessMessage(*pfrom, msg_type, vRecv, msg.m_time, chainparams, m_chainman, m_mempool, connman, m_banman, interruptMsgProc); + try { + ProcessMessage(*pfrom, msg_type, vRecv, msg.m_time, chainparams, m_chainman, m_mempool, *connman, m_banman, interruptMsgProc); if (interruptMsgProc) return false; if (!pfrom->vRecvGetData.empty()) @@ -3679,13 +3847,6 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter LogPrint(BCLog::NET, "%s(%s, %u bytes): Unknown exception caught\n", __func__, SanitizeString(msg_type), nMessageSize); } - if (!fRet) { - LogPrint(BCLog::NET, "%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(msg_type), nMessageSize, pfrom->GetId()); - } - - LOCK(cs_main); - CheckIfBanned(*pfrom); - return fMoreWork; } @@ -3828,17 +3989,19 @@ namespace { class CompareInvMempoolOrder { CTxMemPool *mp; + bool m_wtxid_relay; public: - explicit CompareInvMempoolOrder(CTxMemPool *_mempool) + explicit CompareInvMempoolOrder(CTxMemPool *_mempool, bool use_wtxid) { mp = _mempool; + m_wtxid_relay = use_wtxid; } bool operator()(std::set<uint256>::iterator a, std::set<uint256>::iterator b) { /* As std::make_heap produces a max-heap, we want the entries with the * fewest ancestors/highest fee to sort later. */ - return mp->CompareDepthAndScore(*b, *a); + return mp->CompareDepthAndScore(*b, *a, m_wtxid_relay); } }; } @@ -3846,48 +4009,49 @@ public: bool PeerLogicValidation::SendMessages(CNode* pto) { const Consensus::Params& consensusParams = Params().GetConsensus(); - { - // Don't send anything until the version handshake is complete - if (!pto->fSuccessfullyConnected || pto->fDisconnect) - return true; - // If we get here, the outgoing message serialization version is set and can't change. - const CNetMsgMaker msgMaker(pto->GetSendVersion()); + // We must call MaybeDiscourageAndDisconnect first, to ensure that we'll + // disconnect misbehaving peers even before the version handshake is complete. + if (MaybeDiscourageAndDisconnect(*pto)) return true; - // - // Message: ping - // - bool pingSend = false; - if (pto->fPingQueued) { - // RPC ping request by user - pingSend = true; - } - if (pto->nPingNonceSent == 0 && pto->nPingUsecStart + PING_INTERVAL * 1000000 < GetTimeMicros()) { - // Ping automatically sent as a latency probe & keepalive. - pingSend = true; - } - if (pingSend) { - uint64_t nonce = 0; - while (nonce == 0) { - GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); - } - pto->fPingQueued = false; - pto->nPingUsecStart = GetTimeMicros(); - if (pto->nVersion > BIP0031_VERSION) { - pto->nPingNonceSent = nonce; - connman->PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); - } else { - // Peer is too old to support ping command with nonce, pong will never arrive. - pto->nPingNonceSent = 0; - connman->PushMessage(pto, msgMaker.Make(NetMsgType::PING)); - } - } + // Don't send anything until the version handshake is complete + if (!pto->fSuccessfullyConnected || pto->fDisconnect) + return true; - TRY_LOCK(cs_main, lockMain); - if (!lockMain) - return true; + // If we get here, the outgoing message serialization version is set and can't change. + const CNetMsgMaker msgMaker(pto->GetSendVersion()); + + // + // Message: ping + // + bool pingSend = false; + if (pto->fPingQueued) { + // RPC ping request by user + pingSend = true; + } + if (pto->nPingNonceSent == 0 && pto->m_ping_start.load() + PING_INTERVAL < GetTime<std::chrono::microseconds>()) { + // Ping automatically sent as a latency probe & keepalive. + pingSend = true; + } + if (pingSend) { + uint64_t nonce = 0; + while (nonce == 0) { + GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); + } + pto->fPingQueued = false; + pto->m_ping_start = GetTime<std::chrono::microseconds>(); + if (pto->nVersion > BIP0031_VERSION) { + pto->nPingNonceSent = nonce; + connman->PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); + } else { + // Peer is too old to support ping command with nonce, pong will never arrive. + pto->nPingNonceSent = 0; + connman->PushMessage(pto, msgMaker.Make(NetMsgType::PING)); + } + } - if (CheckIfBanned(*pto)) return true; + { + LOCK(cs_main); CNodeState &state = *State(pto->GetId()); @@ -4085,7 +4249,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // If the peer's chain has this block, don't inv it back. if (!PeerHasHeader(&state, pindex)) { - pto->PushInventory(CInv(MSG_BLOCK, hashToAnnounce)); + pto->vInventoryBlockToSend.push_back(hashToAnnounce); LogPrint(BCLog::NET, "%s: sending inv peer=%d hash=%s\n", __func__, pto->GetId(), hashToAnnounce.ToString()); } @@ -4145,8 +4309,8 @@ bool PeerLogicValidation::SendMessages(CNode* pto) LOCK(pto->m_tx_relay->cs_filter); for (const auto& txinfo : vtxinfo) { - const uint256& hash = txinfo.tx->GetHash(); - CInv inv(MSG_TX, hash); + const uint256& hash = state.m_wtxid_relay ? txinfo.tx->GetWitnessHash() : txinfo.tx->GetHash(); + CInv inv(state.m_wtxid_relay ? MSG_WTX : MSG_TX, hash); pto->m_tx_relay->setInventoryTxToSend.erase(hash); // Don't send transactions that peers will not put into their mempool if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { @@ -4156,6 +4320,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) if (!pto->m_tx_relay->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; } pto->m_tx_relay->filterInventoryKnown.insert(hash); + // Responses to MEMPOOL requests bypass the m_recently_announced_invs filter. vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); @@ -4180,7 +4345,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) } // Topologically and fee-rate sort the inventory we send for privacy and priority reasons. // A heap is used so that not all items need sorting if only a few are being sent. - CompareInvMempoolOrder compareInvMempoolOrder(&m_mempool); + CompareInvMempoolOrder compareInvMempoolOrder(&m_mempool, state.m_wtxid_relay); std::make_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); // No reason to drain out at many times the network's capacity, // especially since we have many peers and some will draw much shorter delays. @@ -4199,17 +4364,20 @@ bool PeerLogicValidation::SendMessages(CNode* pto) continue; } // Not in the mempool anymore? don't bother sending it. - auto txinfo = m_mempool.info(hash); + auto txinfo = m_mempool.info(hash, state.m_wtxid_relay); if (!txinfo.tx) { continue; } + auto txid = txinfo.tx->GetHash(); + auto wtxid = txinfo.tx->GetWitnessHash(); // Peer told you to not send transactions at that feerate? Don't bother sending it. if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { continue; } if (pto->m_tx_relay->pfilter && !pto->m_tx_relay->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; // Send - vInv.push_back(CInv(MSG_TX, hash)); + State(pto->GetId())->m_recently_announced_invs.insert(hash); + vInv.push_back(CInv(state.m_wtxid_relay ? MSG_WTX : MSG_TX, hash)); nRelayedTransactions++; { // Expire old relay messages @@ -4219,9 +4387,14 @@ bool PeerLogicValidation::SendMessages(CNode* pto) vRelayExpiration.pop_front(); } - auto ret = mapRelay.insert(std::make_pair(hash, std::move(txinfo.tx))); + auto ret = mapRelay.emplace(txid, std::move(txinfo.tx)); if (ret.second) { - vRelayExpiration.push_back(std::make_pair(nNow + std::chrono::microseconds{RELAY_TX_CACHE_TIME}.count(), ret.first)); + vRelayExpiration.emplace_back(nNow + std::chrono::microseconds{RELAY_TX_CACHE_TIME}.count(), ret.first); + } + // Add wtxid-based lookup into mapRelay as well, so that peers can request by wtxid + auto ret2 = mapRelay.emplace(wtxid, ret.first->second); + if (ret2.second) { + vRelayExpiration.emplace_back(nNow + std::chrono::microseconds{RELAY_TX_CACHE_TIME}.count(), ret2.first); } } if (vInv.size() == MAX_INV_SZ) { @@ -4229,6 +4402,14 @@ bool PeerLogicValidation::SendMessages(CNode* pto) vInv.clear(); } pto->m_tx_relay->filterInventoryKnown.insert(hash); + if (hash != txid) { + // Insert txid into filterInventoryKnown, even for + // wtxidrelay peers. This prevents re-adding of + // unconfirmed parents to the recently_announced + // filter, when a child tx is requested. See + // ProcessGetData(). + pto->m_tx_relay->filterInventoryKnown.insert(txid); + } } } } @@ -4266,9 +4447,9 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // Check for headers sync timeouts if (state.fSyncStarted && state.nHeadersSyncTimeout < std::numeric_limits<int64_t>::max()) { // Detect whether this is a stalling initial-headers-sync peer - if (pindexBestHeader->GetBlockTime() <= GetAdjustedTime() - 24*60*60) { + if (pindexBestHeader->GetBlockTime() <= GetAdjustedTime() - 24 * 60 * 60) { if (nNow > state.nHeadersSyncTimeout && nSyncStarted == 1 && (nPreferredDownload - state.fPreferredDownload >= 1)) { - // Disconnect a (non-whitelisted) peer if it is our only sync peer, + // Disconnect a peer (without the noban permission) if it is our only sync peer, // and we have others we could be using instead. // Note: If all our peers are inbound, then we won't // disconnect our sync peer for stalling; we have bigger @@ -4278,7 +4459,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) pto->fDisconnect = true; return true; } else { - LogPrintf("Timeout downloading headers from whitelisted peer=%d, not disconnecting\n", pto->GetId()); + LogPrintf("Timeout downloading headers from noban peer=%d, not disconnecting\n", pto->GetId()); // Reset the headers sync state so that we have a // chance to try downloading from a different peer. // Note: this will also result in at least one more @@ -4353,7 +4534,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // Erase this entry from tx_process_time (it may be added back for // processing at a later time, see below) tx_process_time.erase(tx_process_time.begin()); - CInv inv(MSG_TX | GetFetchFlags(*pto), txid); + CInv inv(state.m_wtxid_relay ? MSG_WTX : (MSG_TX | GetFetchFlags(*pto)), txid); if (!AlreadyHave(inv, m_mempool)) { // If this transaction was last requested more than 1 minute ago, // then request. @@ -4372,7 +4553,15 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // up processing to happen after the download times out // (with a slight delay for inbound peers, to prefer // requests to outbound peers). - const auto next_process_time = CalculateTxGetDataTime(txid, current_time, !state.fPreferredDownload); + // Don't apply the txid-delay to re-requests of a + // transaction; the heuristic of delaying requests to + // txid-relay peers is to save bandwidth on initial + // announcement of a transaction, and doesn't make sense + // for a followup request if our first peer times out (and + // would open us up to an attacker using inbound + // wtxid-relay to prevent us from requesting transactions + // from outbound txid-relay peers). + const auto next_process_time = CalculateTxGetDataTime(txid, current_time, !state.fPreferredDownload, false); tx_process_time.emplace(next_process_time, txid); } } else { @@ -4389,15 +4578,26 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // // Message: feefilter // - // We don't want white listed peers to filter txs to us if we have -whitelistforcerelay if (pto->m_tx_relay != nullptr && pto->nVersion >= FEEFILTER_VERSION && gArgs.GetBoolArg("-feefilter", DEFAULT_FEEFILTER) && - !pto->HasPermission(PF_FORCERELAY)) { + !pto->HasPermission(PF_FORCERELAY) // peers with the forcerelay permission should not filter txs to us + ) { CAmount currentFilter = m_mempool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); int64_t timeNow = GetTimeMicros(); + static FeeFilterRounder g_filter_rounder{CFeeRate{DEFAULT_MIN_RELAY_TX_FEE}}; + if (m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + // Received tx-inv messages are discarded when the active + // chainstate is in IBD, so tell the peer to not send them. + currentFilter = MAX_MONEY; + } else { + static const CAmount MAX_FILTER{g_filter_rounder.round(MAX_MONEY)}; + if (pto->m_tx_relay->lastSentFeeFilter == MAX_FILTER) { + // Send the current filter if we sent MAX_FILTER previously + // and made it out of IBD. + pto->m_tx_relay->nextSendTimeFeeFilter = timeNow - 1; + } + } if (timeNow > pto->m_tx_relay->nextSendTimeFeeFilter) { - static CFeeRate default_feerate(DEFAULT_MIN_RELAY_TX_FEE); - static FeeFilterRounder filterRounder(default_feerate); - CAmount filterToSend = filterRounder.round(currentFilter); + CAmount filterToSend = g_filter_rounder.round(currentFilter); // We always have a fee filter of at least minRelayTxFee filterToSend = std::max(filterToSend, ::minRelayTxFee.GetFeePerK()); if (filterToSend != pto->m_tx_relay->lastSentFeeFilter) { @@ -4413,7 +4613,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) pto->m_tx_relay->nextSendTimeFeeFilter = timeNow + GetRandInt(MAX_FEEFILTER_CHANGE_DELAY) * 1000000; } } - } + } // release cs_main return true; } @@ -4425,6 +4625,7 @@ public: // orphan transactions mapOrphanTransactions.clear(); mapOrphanTransactionsByPrev.clear(); + g_orphans_by_wtxid.clear(); } }; static CNetProcessingCleanup instance_of_cnetprocessingcleanup; diff --git a/src/net_processing.h b/src/net_processing.h index 19beca0cc4..2d98714122 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -23,15 +23,18 @@ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; static const bool DEFAULT_PEERBLOOMFILTERS = false; static const bool DEFAULT_PEERBLOCKFILTERS = false; +/** Threshold for marking a node to be discouraged, e.g. disconnected and added to the discouragement filter. */ +static const int DISCOURAGEMENT_THRESHOLD{100}; class PeerLogicValidation final : public CValidationInterface, public NetEventsInterface { private: CConnman* const connman; + /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ BanMan* const m_banman; ChainstateManager& m_chainman; CTxMemPool& m_mempool; - bool CheckIfBanned(CNode& pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool MaybeDiscourageAndDisconnect(CNode& pnode); public: PeerLogicValidation(CConnman* connman, BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool); @@ -97,6 +100,6 @@ struct CNodeStateStats { bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats); /** Relay transaction to every node */ -void RelayTransaction(const uint256&, const CConnman& connman); +void RelayTransaction(const uint256& txid, const uint256& wtxid, const CConnman& connman) EXCLUSIVE_LOCKS_REQUIRED(cs_main); #endif // BITCOIN_NET_PROCESSING_H diff --git a/src/netaddress.cpp b/src/netaddress.cpp index f79425a52e..0874b8dcea 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <cstdint> #include <netaddress.h> #include <hash.h> #include <util/strencodings.h> @@ -27,19 +28,35 @@ CNetAddr::CNetAddr() void CNetAddr::SetIP(const CNetAddr& ipIn) { + m_net = ipIn.m_net; memcpy(ip, ipIn.ip, sizeof(ip)); } +void CNetAddr::SetLegacyIPv6(const uint8_t ipv6[16]) +{ + if (memcmp(ipv6, pchIPv4, sizeof(pchIPv4)) == 0) { + m_net = NET_IPV4; + } else if (memcmp(ipv6, pchOnionCat, sizeof(pchOnionCat)) == 0) { + m_net = NET_ONION; + } else if (memcmp(ipv6, g_internal_prefix, sizeof(g_internal_prefix)) == 0) { + m_net = NET_INTERNAL; + } else { + m_net = NET_IPV6; + } + memcpy(ip, ipv6, 16); +} + void CNetAddr::SetRaw(Network network, const uint8_t *ip_in) { switch(network) { case NET_IPV4: + m_net = NET_IPV4; memcpy(ip, pchIPv4, 12); memcpy(ip+12, ip_in, 4); break; case NET_IPV6: - memcpy(ip, ip_in, 16); + SetLegacyIPv6(ip_in); break; default: assert(!"invalid network"); @@ -65,6 +82,7 @@ bool CNetAddr::SetInternal(const std::string &name) if (name.empty()) { return false; } + m_net = NET_INTERNAL; unsigned char hash[32] = {}; CSHA256().Write((const unsigned char*)name.data(), name.size()).Finalize(hash); memcpy(ip, g_internal_prefix, sizeof(g_internal_prefix)); @@ -88,6 +106,7 @@ bool CNetAddr::SetSpecial(const std::string &strName) std::vector<unsigned char> vchAddr = DecodeBase32(strName.substr(0, strName.size() - 6).c_str()); if (vchAddr.size() != 16-sizeof(pchOnionCat)) return false; + m_net = NET_ONION; memcpy(ip, pchOnionCat, sizeof(pchOnionCat)); for (unsigned int i=0; i<16-sizeof(pchOnionCat); i++) ip[i + sizeof(pchOnionCat)] = vchAddr[i]; @@ -122,15 +141,9 @@ bool CNetAddr::IsBindAny() const return true; } -bool CNetAddr::IsIPv4() const -{ - return (memcmp(ip, pchIPv4, sizeof(pchIPv4)) == 0); -} +bool CNetAddr::IsIPv4() const { return m_net == NET_IPV4; } -bool CNetAddr::IsIPv6() const -{ - return (!IsIPv4() && !IsTor() && !IsInternal()); -} +bool CNetAddr::IsIPv6() const { return m_net == NET_IPV6; } bool CNetAddr::IsRFC1918() const { @@ -164,50 +177,54 @@ bool CNetAddr::IsRFC5737() const bool CNetAddr::IsRFC3849() const { - return GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x0D && GetByte(12) == 0xB8; + return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && + GetByte(13) == 0x0D && GetByte(12) == 0xB8; } bool CNetAddr::IsRFC3964() const { - return (GetByte(15) == 0x20 && GetByte(14) == 0x02); + return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x02; } bool CNetAddr::IsRFC6052() const { static const unsigned char pchRFC6052[] = {0,0x64,0xFF,0x9B,0,0,0,0,0,0,0,0}; - return (memcmp(ip, pchRFC6052, sizeof(pchRFC6052)) == 0); + return IsIPv6() && memcmp(ip, pchRFC6052, sizeof(pchRFC6052)) == 0; } bool CNetAddr::IsRFC4380() const { - return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0 && GetByte(12) == 0); + return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0 && + GetByte(12) == 0; } bool CNetAddr::IsRFC4862() const { static const unsigned char pchRFC4862[] = {0xFE,0x80,0,0,0,0,0,0}; - return (memcmp(ip, pchRFC4862, sizeof(pchRFC4862)) == 0); + return IsIPv6() && memcmp(ip, pchRFC4862, sizeof(pchRFC4862)) == 0; } bool CNetAddr::IsRFC4193() const { - return ((GetByte(15) & 0xFE) == 0xFC); + return IsIPv6() && (GetByte(15) & 0xFE) == 0xFC; } bool CNetAddr::IsRFC6145() const { static const unsigned char pchRFC6145[] = {0,0,0,0,0,0,0,0,0xFF,0xFF,0,0}; - return (memcmp(ip, pchRFC6145, sizeof(pchRFC6145)) == 0); + return IsIPv6() && memcmp(ip, pchRFC6145, sizeof(pchRFC6145)) == 0; } bool CNetAddr::IsRFC4843() const { - return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x10); + return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && + GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x10; } bool CNetAddr::IsRFC7343() const { - return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x20); + return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && + GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x20; } bool CNetAddr::IsHeNet() const @@ -221,10 +238,7 @@ bool CNetAddr::IsHeNet() const * * @see CNetAddr::SetSpecial(const std::string &) */ -bool CNetAddr::IsTor() const -{ - return (memcmp(ip, pchOnionCat, sizeof(pchOnionCat)) == 0); -} +bool CNetAddr::IsTor() const { return m_net == NET_ONION; } bool CNetAddr::IsLocal() const { @@ -234,7 +248,7 @@ bool CNetAddr::IsLocal() const // IPv6 loopback (::1/128) static const unsigned char pchLocal[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1}; - if (memcmp(ip, pchLocal, 16) == 0) + if (IsIPv6() && memcmp(ip, pchLocal, 16) == 0) return true; return false; @@ -258,12 +272,12 @@ bool CNetAddr::IsValid() const // header20 vectorlen3 addr26 addr26 addr26 header20 vectorlen3 addr26 addr26 addr26... // so if the first length field is garbled, it reads the second batch // of addr misaligned by 3 bytes. - if (memcmp(ip, pchIPv4+3, sizeof(pchIPv4)-3) == 0) + if (IsIPv6() && memcmp(ip, pchIPv4+3, sizeof(pchIPv4)-3) == 0) return false; // unspecified IPv6 address (::/128) unsigned char ipNone6[16] = {}; - if (memcmp(ip, ipNone6, 16) == 0) + if (IsIPv6() && memcmp(ip, ipNone6, 16) == 0) return false; // documentation IPv6 address @@ -310,7 +324,7 @@ bool CNetAddr::IsRoutable() const */ bool CNetAddr::IsInternal() const { - return memcmp(ip, g_internal_prefix, sizeof(g_internal_prefix)) == 0; + return m_net == NET_INTERNAL; } enum Network CNetAddr::GetNetwork() const @@ -321,13 +335,7 @@ enum Network CNetAddr::GetNetwork() const if (!IsRoutable()) return NET_UNROUTABLE; - if (IsIPv4()) - return NET_IPV4; - - if (IsTor()) - return NET_ONION; - - return NET_IPV6; + return m_net; } std::string CNetAddr::ToStringIP() const @@ -361,12 +369,12 @@ std::string CNetAddr::ToString() const bool operator==(const CNetAddr& a, const CNetAddr& b) { - return (memcmp(a.ip, b.ip, 16) == 0); + return a.m_net == b.m_net && memcmp(a.ip, b.ip, 16) == 0; } bool operator<(const CNetAddr& a, const CNetAddr& b) { - return (memcmp(a.ip, b.ip, 16) < 0); + return a.m_net < b.m_net || (a.m_net == b.m_net && memcmp(a.ip, b.ip, 16) < 0); } /** @@ -627,15 +635,15 @@ CService::CService() : port(0) { } -CService::CService(const CNetAddr& cip, unsigned short portIn) : CNetAddr(cip), port(portIn) +CService::CService(const CNetAddr& cip, uint16_t portIn) : CNetAddr(cip), port(portIn) { } -CService::CService(const struct in_addr& ipv4Addr, unsigned short portIn) : CNetAddr(ipv4Addr), port(portIn) +CService::CService(const struct in_addr& ipv4Addr, uint16_t portIn) : CNetAddr(ipv4Addr), port(portIn) { } -CService::CService(const struct in6_addr& ipv6Addr, unsigned short portIn) : CNetAddr(ipv6Addr), port(portIn) +CService::CService(const struct in6_addr& ipv6Addr, uint16_t portIn) : CNetAddr(ipv6Addr), port(portIn) { } @@ -663,7 +671,7 @@ bool CService::SetSockAddr(const struct sockaddr *paddr) } } -unsigned short CService::GetPort() const +uint16_t CService::GetPort() const { return port; } @@ -725,12 +733,10 @@ bool CService::GetSockAddr(struct sockaddr* paddr, socklen_t *addrlen) const */ std::vector<unsigned char> CService::GetKey() const { - std::vector<unsigned char> vKey; - vKey.resize(18); - memcpy(vKey.data(), ip, 16); - vKey[16] = port / 0x100; // most significant byte of our port - vKey[17] = port & 0x0FF; // least significant byte of our port - return vKey; + auto key = GetAddrBytes(); + key.push_back(port / 0x100); // most significant byte of our port + key.push_back(port & 0x0FF); // least significant byte of our port + return key; } std::string CService::ToStringPort() const @@ -814,7 +820,7 @@ CSubNet::CSubNet(const CNetAddr &addr): */ bool CSubNet::Match(const CNetAddr &addr) const { - if (!valid || !addr.IsValid()) + if (!valid || !addr.IsValid() || network.m_net != addr.m_net) return false; for(int x=0; x<16; ++x) if ((addr.ip[x] & netmask[x]) != network.ip[x]) diff --git a/src/netaddress.h b/src/netaddress.h index e640c07d32..0365907d44 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -12,25 +12,54 @@ #include <compat.h> #include <serialize.h> -#include <stdint.h> +#include <cstdint> #include <string> #include <vector> +/** + * A network type. + * @note An address may belong to more than one network, for example `10.0.0.1` + * belongs to both `NET_UNROUTABLE` and `NET_IPV4`. + * Keep these sequential starting from 0 and `NET_MAX` as the last entry. + * We have loops like `for (int i = 0; i < NET_MAX; i++)` that expect to iterate + * over all enum values and also `GetExtNetwork()` "extends" this enum by + * introducing standalone constants starting from `NET_MAX`. + */ enum Network { + /// Addresses from these networks are not publicly routable on the global Internet. NET_UNROUTABLE = 0, + + /// IPv4 NET_IPV4, + + /// IPv6 NET_IPV6, + + /// TORv2 NET_ONION, + + /// A set of dummy addresses that map a name to an IPv6 address. These + /// addresses belong to RFC4193's fc00::/7 subnet (unique-local addresses). + /// We use them to map a string or FQDN to an IPv6 address in CAddrMan to + /// keep track of which DNS seeds were used. NET_INTERNAL, + /// Dummy value to indicate the number of NET_* constants. NET_MAX, }; -/** IP address (IPv6, or IPv4 using mapped IPv6 range (::FFFF:0:0/96)) */ +/** + * Network address. + */ class CNetAddr { protected: + /** + * Network to which this address belongs. + */ + Network m_net{NET_IPV6}; + unsigned char ip[16]; // in network byte order uint32_t scopeId{0}; // for scoped/link-local ipv6 addresses @@ -40,6 +69,14 @@ class CNetAddr void SetIP(const CNetAddr& ip); /** + * Set from a legacy IPv6 address. + * Legacy IPv6 address may be a normal IPv6 address, or another address + * (e.g. IPv4) disguised as IPv6. This encoding is used in the legacy + * `addr` encoding. + */ + void SetLegacyIPv6(const uint8_t ipv6[16]); + + /** * Set raw IPv4 or IPv6 address (in network byte order) * @note Only NET_IPV4 and NET_IPV6 are allowed for network. */ @@ -90,6 +127,7 @@ class CNetAddr uint32_t GetMappedAS(const std::vector<bool> &asmap) const; std::vector<unsigned char> GetGroup(const std::vector<bool> &asmap) const; + std::vector<unsigned char> GetAddrBytes() const { return {std::begin(ip), std::end(ip)}; } int GetReachabilityFrom(const CNetAddr *paddrPartner = nullptr) const; explicit CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); @@ -99,7 +137,27 @@ class CNetAddr friend bool operator!=(const CNetAddr& a, const CNetAddr& b) { return !(a == b); } friend bool operator<(const CNetAddr& a, const CNetAddr& b); - SERIALIZE_METHODS(CNetAddr, obj) { READWRITE(obj.ip); } + /** + * Serialize to a stream. + */ + template <typename Stream> + void Serialize(Stream& s) const + { + s << ip; + } + + /** + * Unserialize from a stream. + */ + template <typename Stream> + void Unserialize(Stream& s) + { + unsigned char ip_temp[sizeof(ip)]; + s >> ip_temp; + // Use SetLegacyIPv6() so that m_net is set correctly. For example + // ::FFFF:0102:0304 should be set as m_net=NET_IPV4 (1.2.3.4). + SetLegacyIPv6(ip_temp); + } friend class CSubNet; }; @@ -142,10 +200,10 @@ class CService : public CNetAddr public: CService(); - CService(const CNetAddr& ip, unsigned short port); - CService(const struct in_addr& ipv4Addr, unsigned short port); + CService(const CNetAddr& ip, uint16_t port); + CService(const struct in_addr& ipv4Addr, uint16_t port); explicit CService(const struct sockaddr_in& addr); - unsigned short GetPort() const; + uint16_t GetPort() const; bool GetSockAddr(struct sockaddr* paddr, socklen_t *addrlen) const; bool SetSockAddr(const struct sockaddr* paddr); friend bool operator==(const CService& a, const CService& b); @@ -156,10 +214,14 @@ class CService : public CNetAddr std::string ToStringPort() const; std::string ToStringIPPort() const; - CService(const struct in6_addr& ipv6Addr, unsigned short port); + CService(const struct in6_addr& ipv6Addr, uint16_t port); explicit CService(const struct sockaddr_in6& addr); - SERIALIZE_METHODS(CService, obj) { READWRITE(obj.ip, Using<BigEndianFormatter<2>>(obj.port)); } + SERIALIZE_METHODS(CService, obj) + { + READWRITEAS(CNetAddr, obj); + READWRITE(Using<BigEndianFormatter<2>>(obj.port)); + } }; bool SanityCheckASMap(const std::vector<bool>& asmap); diff --git a/src/netbase.cpp b/src/netbase.cpp index 9fe03c6a24..3a3b5f3e66 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -12,6 +12,7 @@ #include <util/system.h> #include <atomic> +#include <cstdint> #ifndef WIN32 #include <fcntl.h> @@ -798,11 +799,11 @@ bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int ProxyCredentials random_auth; static std::atomic_int counter(0); random_auth.username = random_auth.password = strprintf("%i", counter++); - if (!Socks5(strDest, (unsigned short)port, &random_auth, hSocket)) { + if (!Socks5(strDest, (uint16_t)port, &random_auth, hSocket)) { return false; } } else { - if (!Socks5(strDest, (unsigned short)port, 0, hSocket)) { + if (!Socks5(strDest, (uint16_t)port, 0, hSocket)) { return false; } } diff --git a/src/netmessagemaker.h b/src/netmessagemaker.h index 2efb384a7b..ffb3fe2f29 100644 --- a/src/netmessagemaker.h +++ b/src/netmessagemaker.h @@ -15,18 +15,18 @@ public: explicit CNetMsgMaker(int nVersionIn) : nVersion(nVersionIn){} template <typename... Args> - CSerializedNetMsg Make(int nFlags, std::string sCommand, Args&&... args) const + CSerializedNetMsg Make(int nFlags, std::string msg_type, Args&&... args) const { CSerializedNetMsg msg; - msg.command = std::move(sCommand); + msg.m_type = std::move(msg_type); CVectorWriter{ SER_NETWORK, nFlags | nVersion, msg.data, 0, std::forward<Args>(args)... }; return msg; } template <typename... Args> - CSerializedNetMsg Make(std::string sCommand, Args&&... args) const + CSerializedNetMsg Make(std::string msg_type, Args&&... args) const { - return Make(0, std::move(sCommand), std::forward<Args>(args)...); + return Make(0, std::move(msg_type), std::forward<Args>(args)...); } private: diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index e3c4c828b6..fb46ea1731 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -8,13 +8,23 @@ #include <coins.h> #include <hash.h> #include <serialize.h> -#include <validation.h> #include <uint256.h> #include <util/system.h> +#include <validation.h> #include <map> -static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +static uint64_t GetBogoSize(const CScript& scriptPubKey) +{ + return 32 /* txid */ + + 4 /* vout index */ + + 4 /* height + coinbase */ + + 8 /* amount */ + + 2 /* scriptPubKey len */ + + scriptPubKey.size() /* scriptPubKey */; +} + +static void ApplyStats(CCoinsStats& stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) { assert(!outputs.empty()); ss << hash; @@ -26,26 +36,38 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, ss << VARINT_MODE(output.second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); stats.nTransactionOutputs++; stats.nTotalAmount += output.second.out.nValue; - stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + - 2 /* scriptPubKey len */ + output.second.out.scriptPubKey.size() /* scriptPubKey */; + stats.nBogoSize += GetBogoSize(output.second.out.scriptPubKey); } ss << VARINT(0u); } +static void ApplyStats(CCoinsStats& stats, std::nullptr_t, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +{ + assert(!outputs.empty()); + stats.nTransactions++; + for (const auto& output : outputs) { + stats.nTransactionOutputs++; + stats.nTotalAmount += output.second.out.nValue; + stats.nBogoSize += GetBogoSize(output.second.out.scriptPubKey); + } +} + //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void()>& interruption_point) +template <typename T> +static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj, const std::function<void()>& interruption_point) { stats = CCoinsStats(); std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); assert(pcursor); - CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); stats.hashBlock = pcursor->GetBestBlock(); { LOCK(cs_main); stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight; } - ss << stats.hashBlock; + + PrepareHash(hash_obj, stats); + uint256 prevkey; std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { @@ -54,7 +76,7 @@ bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { if (!outputs.empty() && key.hash != prevkey) { - ApplyStats(stats, ss, prevkey, outputs); + ApplyStats(stats, hash_obj, prevkey, outputs); outputs.clear(); } prevkey = key.hash; @@ -66,9 +88,38 @@ bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void pcursor->Next(); } if (!outputs.empty()) { - ApplyStats(stats, ss, prevkey, outputs); + ApplyStats(stats, hash_obj, prevkey, outputs); } - stats.hashSerialized = ss.GetHash(); + + FinalizeHash(hash_obj, stats); + stats.nDiskSize = view->EstimateSize(); return true; } + +bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, CoinStatsHashType hash_type, const std::function<void()>& interruption_point) +{ + switch (hash_type) { + case(CoinStatsHashType::HASH_SERIALIZED): { + CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); + return GetUTXOStats(view, stats, ss, interruption_point); + } + case(CoinStatsHashType::NONE): { + return GetUTXOStats(view, stats, nullptr, interruption_point); + } + } // no default case, so the compiler can warn about missing cases + assert(false); +} + +// The legacy hash serializes the hashBlock +static void PrepareHash(CHashWriter& ss, CCoinsStats& stats) +{ + ss << stats.hashBlock; +} +static void PrepareHash(std::nullptr_t, CCoinsStats& stats) {} + +static void FinalizeHash(CHashWriter& ss, CCoinsStats& stats) +{ + stats.hashSerialized = ss.GetHash(); +} +static void FinalizeHash(std::nullptr_t, CCoinsStats& stats) {} diff --git a/src/node/coinstats.h b/src/node/coinstats.h index d9cdaa3036..2a7441c10e 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -14,6 +14,11 @@ class CCoinsView; +enum class CoinStatsHashType { + HASH_SERIALIZED, + NONE, +}; + struct CCoinsStats { int nHeight{0}; @@ -30,6 +35,6 @@ struct CCoinsStats }; //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void()>& interruption_point = {}); +bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const CoinStatsHashType hash_type, const std::function<void()>& interruption_point = {}); #endif // BITCOIN_NODE_COINSTATS_H diff --git a/src/node/context.h b/src/node/context.h index c45d9e6689..be568cba36 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -6,6 +6,7 @@ #define BITCOIN_NODE_CONTEXT_H #include <cassert> +#include <functional> #include <memory> #include <vector> @@ -41,6 +42,7 @@ struct NodeContext { std::unique_ptr<interfaces::Chain> chain; std::vector<std::unique_ptr<interfaces::ChainClient>> chain_clients; std::unique_ptr<CScheduler> scheduler; + std::function<void()> rpc_interruption_point = [] {}; //! Declare default constructor and destructor that are not inline, so code //! instantiating the NodeContext struct doesn't need to #include class @@ -49,10 +51,4 @@ struct NodeContext { ~NodeContext(); }; -inline ChainstateManager& EnsureChainman(const NodeContext& node) -{ - assert(node.chainman); - return *node.chainman; -} - #endif // BITCOIN_NODE_CONTEXT_H diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 3841d8687d..5633abe817 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -80,9 +80,10 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t if (relay) { // the mempool tracks locally submitted transactions to make a // best-effort of initial broadcast - node.mempool->AddUnbroadcastTx(hashTx); + node.mempool->AddUnbroadcastTx(hashTx, tx->GetWitnessHash()); - RelayTransaction(hashTx, *node.connman); + LOCK(cs_main); + RelayTransaction(hashTx, tx->GetWitnessHash(), *node.connman); } return TransactionError::OK; diff --git a/src/ui_interface.cpp b/src/node/ui_interface.cpp index 15795bd67f..8d3665975d 100644 --- a/src/ui_interface.cpp +++ b/src/node/ui_interface.cpp @@ -2,18 +2,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <ui_interface.h> +#include <node/ui_interface.h> #include <util/translation.h> -#include <boost/signals2/last_value.hpp> +#include <boost/signals2/optional_last_value.hpp> #include <boost/signals2/signal.hpp> CClientUIInterface uiInterface; struct UISignals { - boost::signals2::signal<CClientUIInterface::ThreadSafeMessageBoxSig, boost::signals2::last_value<bool>> ThreadSafeMessageBox; - boost::signals2::signal<CClientUIInterface::ThreadSafeQuestionSig, boost::signals2::last_value<bool>> ThreadSafeQuestion; + boost::signals2::signal<CClientUIInterface::ThreadSafeMessageBoxSig, boost::signals2::optional_last_value<bool>> ThreadSafeMessageBox; + boost::signals2::signal<CClientUIInterface::ThreadSafeQuestionSig, boost::signals2::optional_last_value<bool>> ThreadSafeQuestion; boost::signals2::signal<CClientUIInterface::InitMessageSig> InitMessage; boost::signals2::signal<CClientUIInterface::NotifyNumConnectionsChangedSig> NotifyNumConnectionsChanged; boost::signals2::signal<CClientUIInterface::NotifyNetworkActiveChangedSig> NotifyNetworkActiveChanged; @@ -42,8 +42,8 @@ ADD_SIGNALS_IMPL_WRAPPER(NotifyBlockTip); ADD_SIGNALS_IMPL_WRAPPER(NotifyHeaderTip); ADD_SIGNALS_IMPL_WRAPPER(BannedListChanged); -bool CClientUIInterface::ThreadSafeMessageBox(const bilingual_str& message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeMessageBox(message, caption, style); } -bool CClientUIInterface::ThreadSafeQuestion(const bilingual_str& message, const std::string& non_interactive_message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeQuestion(message, non_interactive_message, caption, style); } +bool CClientUIInterface::ThreadSafeMessageBox(const bilingual_str& message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeMessageBox(message, caption, style).value_or(false);} +bool CClientUIInterface::ThreadSafeQuestion(const bilingual_str& message, const std::string& non_interactive_message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeQuestion(message, non_interactive_message, caption, style).value_or(false);} void CClientUIInterface::InitMessage(const std::string& message) { return g_ui_signals.InitMessage(message); } void CClientUIInterface::NotifyNumConnectionsChanged(int newNumConnections) { return g_ui_signals.NotifyNumConnectionsChanged(newNumConnections); } void CClientUIInterface::NotifyNetworkActiveChanged(bool networkActive) { return g_ui_signals.NotifyNetworkActiveChanged(networkActive); } diff --git a/src/ui_interface.h b/src/node/ui_interface.h index d45811178f..d574ab879f 100644 --- a/src/ui_interface.h +++ b/src/node/ui_interface.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_UI_INTERFACE_H -#define BITCOIN_UI_INTERFACE_H +#ifndef BITCOIN_NODE_UI_INTERFACE_H +#define BITCOIN_NODE_UI_INTERFACE_H #include <functional> #include <memory> @@ -20,14 +20,6 @@ class connection; } } // namespace boost -/** General change type (added, updated, removed). */ -enum ChangeType -{ - CT_NEW, - CT_UPDATED, - CT_DELETED -}; - /** Signals for UI communication. */ class CClientUIInterface { @@ -67,9 +59,6 @@ public: /** Force blocking, modal message box dialog (not just OS notification) */ MODAL = 0x10000000U, - /** Do not prepend error/warning prefix */ - MSG_NOPREFIX = 0x20000000U, - /** Do not print contents of message to debug log */ SECURE = 0x40000000U, @@ -125,7 +114,8 @@ void InitWarning(const bilingual_str& str); /** Show error message **/ bool InitError(const bilingual_str& str); +constexpr auto AbortError = InitError; extern CClientUIInterface uiInterface; -#endif // BITCOIN_UI_INTERFACE_H +#endif // BITCOIN_NODE_UI_INTERFACE_H diff --git a/src/noui.cpp b/src/noui.cpp index ddb3a50ff7..3c82512fac 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -6,7 +6,7 @@ #include <noui.h> #include <logging.h> -#include <ui_interface.h> +#include <node/ui_interface.h> #include <util/translation.h> #include <string> @@ -23,24 +23,20 @@ bool noui_ThreadSafeMessageBox(const bilingual_str& message, const std::string& { bool fSecure = style & CClientUIInterface::SECURE; style &= ~CClientUIInterface::SECURE; - bool prefix = !(style & CClientUIInterface::MSG_NOPREFIX); - style &= ~CClientUIInterface::MSG_NOPREFIX; std::string strCaption; - if (prefix) { - switch (style) { - case CClientUIInterface::MSG_ERROR: - strCaption = "Error: "; - break; - case CClientUIInterface::MSG_WARNING: - strCaption = "Warning: "; - break; - case CClientUIInterface::MSG_INFORMATION: - strCaption = "Information: "; - break; - default: - strCaption = caption + ": "; // Use supplied caption (can be empty) - } + switch (style) { + case CClientUIInterface::MSG_ERROR: + strCaption = "Error: "; + break; + case CClientUIInterface::MSG_WARNING: + strCaption = "Warning: "; + break; + case CClientUIInterface::MSG_INFORMATION: + strCaption = "Information: "; + break; + default: + strCaption = caption + ": "; // Use supplied caption (can be empty) } if (!fSecure) { diff --git a/src/outputtype.cpp b/src/outputtype.cpp index ea7a86d6d6..e978852826 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -42,8 +42,8 @@ const std::string& FormatOutputType(OutputType type) case OutputType::LEGACY: return OUTPUT_TYPE_STRING_LEGACY; case OutputType::P2SH_SEGWIT: return OUTPUT_TYPE_STRING_P2SH_SEGWIT; case OutputType::BECH32: return OUTPUT_TYPE_STRING_BECH32; - default: assert(false); - } + } // no default case, so the compiler can warn about missing cases + assert(false); } CTxDestination GetDestinationForKey(const CPubKey& key, OutputType type) @@ -53,7 +53,7 @@ CTxDestination GetDestinationForKey(const CPubKey& key, OutputType type) case OutputType::P2SH_SEGWIT: case OutputType::BECH32: { if (!key.IsCompressed()) return PKHash(key); - CTxDestination witdest = WitnessV0KeyHash(PKHash(key)); + CTxDestination witdest = WitnessV0KeyHash(key); CScript witprog = GetScriptForDestination(witdest); if (type == OutputType::P2SH_SEGWIT) { return ScriptHash(witprog); @@ -61,8 +61,8 @@ CTxDestination GetDestinationForKey(const CPubKey& key, OutputType type) return witdest; } } - default: assert(false); - } + } // no default case, so the compiler can warn about missing cases + assert(false); } std::vector<CTxDestination> GetAllDestinationsForKey(const CPubKey& key) @@ -100,6 +100,6 @@ CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, return ScriptHash(witprog); } } - default: assert(false); - } + } // no default case, so the compiler can warn about missing cases + assert(false); } diff --git a/src/outputtype.h b/src/outputtype.h index 1438f65844..77a16b1d05 100644 --- a/src/outputtype.h +++ b/src/outputtype.h @@ -18,14 +18,6 @@ enum class OutputType { LEGACY, P2SH_SEGWIT, BECH32, - - /** - * Special output type for change outputs only. Automatically choose type - * based on address type setting and the types other of non-change outputs - * (see -changetype option documentation and implementation in - * CWallet::TransactionChangeType for details). - */ - CHANGE_AUTO, }; extern const std::array<OutputType, 3> OUTPUT_TYPES; diff --git a/src/policy/feerate.cpp b/src/policy/feerate.cpp index 14be6192fe..a01e259731 100644 --- a/src/policy/feerate.cpp +++ b/src/policy/feerate.cpp @@ -7,8 +7,6 @@ #include <tinyformat.h> -const std::string CURRENCY_UNIT = "BTC"; - CFeeRate::CFeeRate(const CAmount& nFeePaid, size_t nBytes_) { assert(nBytes_ <= uint64_t(std::numeric_limits<int64_t>::max())); @@ -37,7 +35,10 @@ CAmount CFeeRate::GetFee(size_t nBytes_) const return nFee; } -std::string CFeeRate::ToString() const +std::string CFeeRate::ToString(const FeeEstimateMode& fee_estimate_mode) const { - return strprintf("%d.%08d %s/kB", nSatoshisPerK / COIN, nSatoshisPerK % COIN, CURRENCY_UNIT); + switch (fee_estimate_mode) { + case FeeEstimateMode::SAT_B: return strprintf("%d.%03d %s/B", nSatoshisPerK / 1000, nSatoshisPerK % 1000, CURRENCY_ATOM); + default: return strprintf("%d.%08d %s/kB", nSatoshisPerK / COIN, nSatoshisPerK % COIN, CURRENCY_UNIT); + } } diff --git a/src/policy/feerate.h b/src/policy/feerate.h index 61fa80c130..883940f73c 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -11,7 +11,17 @@ #include <string> -extern const std::string CURRENCY_UNIT; +const std::string CURRENCY_UNIT = "BTC"; // One formatted unit +const std::string CURRENCY_ATOM = "sat"; // One indivisible minimum value unit + +/* Used to determine type of fee estimation requested */ +enum class FeeEstimateMode { + UNSET, //!< Use default settings based on other criteria + ECONOMICAL, //!< Force estimateSmartFee to use non-conservative estimates + CONSERVATIVE, //!< Force estimateSmartFee to use conservative estimates + BTC_KB, //!< Use explicit BTC/kB fee given in coin control + SAT_B, //!< Use explicit sat/B fee given in coin control +}; /** * Fee rate in satoshis per kilobyte: CAmount / kB @@ -46,7 +56,7 @@ public: friend bool operator>=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK >= b.nSatoshisPerK; } friend bool operator!=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK != b.nSatoshisPerK; } CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } - std::string ToString() const; + std::string ToString(const FeeEstimateMode& fee_estimate_mode = FeeEstimateMode::BTC_KB) const; SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); } }; diff --git a/src/policy/fees.h b/src/policy/fees.h index 6ee6e0d547..e79dbc9868 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -45,13 +45,6 @@ enum class FeeReason { REQUIRED, }; -/* Used to determine type of fee estimation requested */ -enum class FeeEstimateMode { - UNSET, //!< Use default settings based on other criteria - ECONOMICAL, //!< Force estimateSmartFee to use non-conservative estimates - CONSERVATIVE, //!< Force estimateSmartFee to use conservative estimates -}; - /* Used to return detailed information about a feerate bucket */ struct EstimatorBucket { @@ -280,7 +273,7 @@ public: /** Create new FeeFilterRounder */ explicit FeeFilterRounder(const CFeeRate& minIncrementalFee); - /** Quantize a minimum fee for privacy purpose before broadcast **/ + /** Quantize a minimum fee for privacy purpose before broadcast. Not thread-safe due to use of FastRandomContext */ CAmount round(CAmount currentMinFee); private: diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 07d51c0088..c56abaf6c9 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -50,14 +50,14 @@ bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) return (txout.nValue < GetDustThreshold(txout, dustRelayFeeIn)); } -bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType) +bool IsStandard(const CScript& scriptPubKey, TxoutType& whichType) { std::vector<std::vector<unsigned char> > vSolutions; whichType = Solver(scriptPubKey, vSolutions); - if (whichType == TX_NONSTANDARD) { + if (whichType == TxoutType::NONSTANDARD) { return false; - } else if (whichType == TX_MULTISIG) { + } else if (whichType == TxoutType::MULTISIG) { unsigned char m = vSolutions.front()[0]; unsigned char n = vSolutions.back()[0]; // Support up to x-of-3 multisig txns as standard @@ -65,7 +65,7 @@ bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType) return false; if (m < 1 || m > n) return false; - } else if (whichType == TX_NULL_DATA && + } else if (whichType == TxoutType::NULL_DATA && (!fAcceptDatacarrier || scriptPubKey.size() > nMaxDatacarrierBytes)) { return false; } @@ -110,16 +110,16 @@ bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeR } unsigned int nDataOut = 0; - txnouttype whichType; + TxoutType whichType; for (const CTxOut& txout : tx.vout) { if (!::IsStandard(txout.scriptPubKey, whichType)) { reason = "scriptpubkey"; return false; } - if (whichType == TX_NULL_DATA) + if (whichType == TxoutType::NULL_DATA) nDataOut++; - else if ((whichType == TX_MULTISIG) && (!permit_bare_multisig)) { + else if ((whichType == TxoutType::MULTISIG) && (!permit_bare_multisig)) { reason = "bare-multisig"; return false; } else if (IsDust(txout, dust_relay_fee)) { @@ -163,10 +163,10 @@ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) const CTxOut& prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; std::vector<std::vector<unsigned char> > vSolutions; - txnouttype whichType = Solver(prev.scriptPubKey, vSolutions); - if (whichType == TX_NONSTANDARD) { + TxoutType whichType = Solver(prev.scriptPubKey, vSolutions); + if (whichType == TxoutType::NONSTANDARD) { return false; - } else if (whichType == TX_SCRIPTHASH) { + } else if (whichType == TxoutType::SCRIPTHASH) { std::vector<std::vector<unsigned char> > stack; // convert the scriptSig into a stack, so we can inspect the redeemScript if (!EvalScript(stack, tx.vin[i].scriptSig, SCRIPT_VERIFY_NONE, BaseSignatureChecker(), SigVersion::BASE)) diff --git a/src/policy/policy.h b/src/policy/policy.h index 1561a41c5e..7f168ee20f 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -81,7 +81,7 @@ CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFee); bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFee); -bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType); +bool IsStandard(const CScript& scriptPubKey, TxoutType& whichType); /** * Check for standard transaction types * @return True if all outputs (scriptPubKeys) use only standard transaction forms diff --git a/src/protocol.cpp b/src/protocol.cpp index 2dfe4bee74..ee77ca3b94 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -46,6 +46,7 @@ const char *GETCFHEADERS="getcfheaders"; const char *CFHEADERS="cfheaders"; const char *GETCFCHECKPT="getcfcheckpt"; const char *CFCHECKPT="cfcheckpt"; +const char *WTXIDRELAY="wtxidrelay"; } // namespace NetMsgType /** All known message types. Keep this in the same order as the list of @@ -83,6 +84,7 @@ const static std::string allNetMessageTypes[] = { NetMsgType::CFHEADERS, NetMsgType::GETCFCHECKPT, NetMsgType::CFCHECKPT, + NetMsgType::WTXIDRELAY, }; const static std::vector<std::string> allNetMessageTypesVec(allNetMessageTypes, allNetMessageTypes+ARRAYLEN(allNetMessageTypes)); @@ -177,6 +179,8 @@ std::string CInv::GetCommand() const switch (masked) { case MSG_TX: return cmd.append(NetMsgType::TX); + // WTX is not a message type, just an inv type + case MSG_WTX: return cmd.append("wtx"); case MSG_BLOCK: return cmd.append(NetMsgType::BLOCK); case MSG_FILTERED_BLOCK: return cmd.append(NetMsgType::MERKLEBLOCK); case MSG_CMPCT_BLOCK: return cmd.append(NetMsgType::CMPCTBLOCK); diff --git a/src/protocol.h b/src/protocol.h index 985f44640b..d83da2034a 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -261,6 +261,12 @@ extern const char* GETCFCHECKPT; * evenly spaced filter headers for blocks on the requested chain. */ extern const char* CFCHECKPT; +/** + * Indicates that a node prefers to relay transactions via wtxid, rather than + * txid. + * @since protocol version 70016 as described by BIP 339. + */ +extern const char *WTXIDRELAY; }; // namespace NetMsgType /* Get a vector of all valid message types (see above) */ @@ -371,7 +377,13 @@ public: READWRITE(nVersion); } if ((s.GetType() & SER_DISK) || - (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) { + (nVersion != INIT_PROTO_VERSION && !(s.GetType() & SER_GETHASH))) { + // The only time we serialize a CAddress object without nTime is in + // the initial VERSION messages which contain two CAddress records. + // At that point, the serialization version is INIT_PROTO_VERSION. + // After the version handshake, serialization version is >= + // MIN_PEER_PROTO_VERSION and all ADDR messages are serialized with + // nTime. READWRITE(obj.nTime); } READWRITE(Using<CustomUintFormatter<8>>(obj.nServices)); @@ -391,11 +403,12 @@ const uint32_t MSG_TYPE_MASK = 0xffffffff >> 2; * These numbers are defined by the protocol. When adding a new value, be sure * to mention it in the respective BIP. */ -enum GetDataMsg { +enum GetDataMsg : uint32_t { UNDEFINED = 0, MSG_TX = 1, MSG_BLOCK = 2, - // The following can only occur in getdata. Invs always use TX or BLOCK. + MSG_WTX = 5, //!< Defined in BIP 339 + // The following can only occur in getdata. Invs always use TX/WTX or BLOCK. MSG_FILTERED_BLOCK = 3, //!< Defined in BIP37 MSG_CMPCT_BLOCK = 4, //!< Defined in BIP152 MSG_WITNESS_BLOCK = MSG_BLOCK | MSG_WITNESS_FLAG, //!< Defined in BIP144 diff --git a/src/psbt.cpp b/src/psbt.cpp index ef9781817a..3fb743e5db 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -35,14 +35,6 @@ bool PartiallySignedTransaction::Merge(const PartiallySignedTransaction& psbt) return true; } -bool PartiallySignedTransaction::IsSane() const -{ - for (PSBTInput input : inputs) { - if (!input.IsSane()) return false; - } - return true; -} - bool PartiallySignedTransaction::AddInput(const CTxIn& txin, PSBTInput& psbtin) { if (std::find(tx->vin.begin(), tx->vin.end(), txin) != tx->vin.end()) { @@ -144,8 +136,8 @@ void PSBTInput::Merge(const PSBTInput& input) { if (!non_witness_utxo && input.non_witness_utxo) non_witness_utxo = input.non_witness_utxo; if (witness_utxo.IsNull() && !input.witness_utxo.IsNull()) { + // TODO: For segwit v1, we will want to clear out the non-witness utxo when setting a witness one. For v0 and non-segwit, this is not safe witness_utxo = input.witness_utxo; - non_witness_utxo = nullptr; // Clear out any non-witness utxo when we set a witness one. } partial_sigs.insert(input.partial_sigs.begin(), input.partial_sigs.end()); @@ -158,18 +150,6 @@ void PSBTInput::Merge(const PSBTInput& input) if (final_script_witness.IsNull() && !input.final_script_witness.IsNull()) final_script_witness = input.final_script_witness; } -bool PSBTInput::IsSane() const -{ - // Cannot have both witness and non-witness utxos - if (!witness_utxo.IsNull() && non_witness_utxo) return false; - - // If we have a witness_script or a scriptWitness, we must also have a witness utxo - if (!witness_script.empty() && witness_utxo.IsNull()) return false; - if (!final_script_witness.IsNull() && witness_utxo.IsNull()) return false; - - return true; -} - void PSBTOutput::FillSignatureData(SignatureData& sigdata) const { if (!redeem_script.empty()) { @@ -214,6 +194,17 @@ bool PSBTInputSigned(const PSBTInput& input) return !input.final_script_sig.empty() || !input.final_script_witness.IsNull(); } +size_t CountPSBTUnsignedInputs(const PartiallySignedTransaction& psbt) { + size_t count = 0; + for (const auto& input : psbt.inputs) { + if (!PSBTInputSigned(input)) { + count++; + } + } + + return count; +} + void UpdatePSBTOutput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index) { const CTxOut& out = psbt.tx->vout.at(index); @@ -250,11 +241,6 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& bool require_witness_sig = false; CTxOut utxo; - // Verify input sanity, which checks that at most one of witness or non-witness utxos is provided. - if (!input.IsSane()) { - return false; - } - if (input.non_witness_utxo) { // If we're taking our information from a non-witness UTXO, verify that it matches the prevout. COutPoint prevout = tx.vin[index].prevout; @@ -288,10 +274,11 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& if (require_witness_sig && !sigdata.witness) return false; input.FromSignatureData(sigdata); - // If we have a witness signature, use the smaller witness UTXO. + // If we have a witness signature, put a witness UTXO. + // TODO: For segwit v1, we should remove the non_witness_utxo if (sigdata.witness) { input.witness_utxo = utxo; - input.non_witness_utxo = nullptr; + // input.non_witness_utxo = nullptr; } // Fill in the missing info @@ -345,10 +332,6 @@ TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector return TransactionError::PSBT_MISMATCH; } } - if (!out.IsSane()) { - return TransactionError::INVALID_PSBT; - } - return TransactionError::OK; } diff --git a/src/psbt.h b/src/psbt.h index 888e0fd119..0951b76f83 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -62,18 +62,17 @@ struct PSBTInput void FillSignatureData(SignatureData& sigdata) const; void FromSignatureData(const SignatureData& sigdata); void Merge(const PSBTInput& input); - bool IsSane() const; PSBTInput() {} template <typename Stream> inline void Serialize(Stream& s) const { // Write the utxo - // If there is a non-witness utxo, then don't add the witness one. if (non_witness_utxo) { SerializeToVector(s, PSBT_IN_NON_WITNESS_UTXO); OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS); SerializeToVector(os, non_witness_utxo); - } else if (!witness_utxo.IsNull()) { + } + if (!witness_utxo.IsNull()) { SerializeToVector(s, PSBT_IN_WITNESS_UTXO); SerializeToVector(s, witness_utxo); } @@ -284,7 +283,6 @@ struct PSBTOutput void FillSignatureData(SignatureData& sigdata) const; void FromSignatureData(const SignatureData& sigdata); void Merge(const PSBTOutput& output); - bool IsSane() const; PSBTOutput() {} template <typename Stream> @@ -401,7 +399,6 @@ struct PartiallySignedTransaction /** Merge psbt into this. The two psbts must have the same underlying CTransaction (i.e. the * same actual Bitcoin transaction.) Returns true if the merge succeeded, false otherwise. */ NODISCARD bool Merge(const PartiallySignedTransaction& psbt); - bool IsSane() const; bool AddInput(const CTxIn& txin, PSBTInput& psbtin); bool AddOutput(const CTxOut& txout, const PSBTOutput& psbtout); PartiallySignedTransaction() {} @@ -551,10 +548,6 @@ struct PartiallySignedTransaction if (outputs.size() != tx->vout.size()) { throw std::ios_base::failure("Outputs provided does not match the number of outputs in transaction."); } - // Sanity check - if (!IsSane()) { - throw std::ios_base::failure("PSBT is not sane."); - } } template <typename Stream> @@ -579,6 +572,9 @@ bool PSBTInputSigned(const PSBTInput& input); /** Signs a PSBTInput, verifying that all provided data matches what is being signed. */ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, int sighash = SIGHASH_ALL, SignatureData* out_sigdata = nullptr, bool use_dummy = false); +/** Counts the unsigned inputs of a PSBT. */ +size_t CountPSBTUnsignedInputs(const PartiallySignedTransaction& psbt); + /** Updates a PSBTOutput with information from provider. * * This fills in the redeem_script, witness_script, and hd_keypaths where possible. diff --git a/src/pubkey.h b/src/pubkey.h index 261842b7f7..4c28af4a4d 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -142,6 +142,9 @@ public: unsigned int len = ::ReadCompactSize(s); if (len <= SIZE) { s.read((char*)vch, len); + if (len != size()) { + Invalidate(); + } } else { // invalid pubkey, skip available data char dummy; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index e0b9345a32..ecb753a306 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -30,11 +30,10 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <noui.h> -#include <ui_interface.h> #include <uint256.h> #include <util/system.h> -#include <util/translation.h> #include <util/threadnames.h> +#include <util/translation.h> #include <validation.h> #include <memory> @@ -529,6 +528,11 @@ int GuiMain(int argc, char* argv[]) // Parse URIs on command line -- this can affect Params() PaymentServer::ipcParseCommandLine(*node, argc, argv); #endif + if (!node->initSettings(error)) { + node->initError(Untranslated(error)); + QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error initializing settings: %1").arg(QString::fromStdString(error))); + return EXIT_FAILURE; + } QScopedPointer<const NetworkStyle> networkStyle(NetworkStyle::instantiate(Params().NetworkIDString())); assert(!networkStyle.isNull()); @@ -557,6 +561,8 @@ int GuiMain(int argc, char* argv[]) /// 9. Main GUI initialization // Install global event filter that makes sure that long tooltips can be word-wrapped app.installEventFilter(new GUIUtil::ToolTipToRichTextFilter(TOOLTIP_WRAP_THRESHOLD, &app)); + // Install global event filter that makes sure that out-of-focus labels do not contain text cursor. + app.installEventFilter(new GUIUtil::LabelOutOfFocusEventFilter(&app)); #if defined(Q_OS_WIN) // Install global event filter for processing Windows session related Windows messages (WM_QUERYENDSESSION and WM_ENDSESSION) qApp->installNativeEventFilter(new WinShutdownMonitor()); diff --git a/src/qt/bitcoin.qrc b/src/qt/bitcoin.qrc index 037b23e4b2..7115459808 100644 --- a/src/qt/bitcoin.qrc +++ b/src/qt/bitcoin.qrc @@ -45,42 +45,42 @@ <file alias="network_disabled">res/icons/network_disabled.png</file> <file alias="proxy">res/icons/proxy.png</file> </qresource> - <qresource prefix="/movies"> - <file alias="spinner-000">res/movies/spinner-000.png</file> - <file alias="spinner-001">res/movies/spinner-001.png</file> - <file alias="spinner-002">res/movies/spinner-002.png</file> - <file alias="spinner-003">res/movies/spinner-003.png</file> - <file alias="spinner-004">res/movies/spinner-004.png</file> - <file alias="spinner-005">res/movies/spinner-005.png</file> - <file alias="spinner-006">res/movies/spinner-006.png</file> - <file alias="spinner-007">res/movies/spinner-007.png</file> - <file alias="spinner-008">res/movies/spinner-008.png</file> - <file alias="spinner-009">res/movies/spinner-009.png</file> - <file alias="spinner-010">res/movies/spinner-010.png</file> - <file alias="spinner-011">res/movies/spinner-011.png</file> - <file alias="spinner-012">res/movies/spinner-012.png</file> - <file alias="spinner-013">res/movies/spinner-013.png</file> - <file alias="spinner-014">res/movies/spinner-014.png</file> - <file alias="spinner-015">res/movies/spinner-015.png</file> - <file alias="spinner-016">res/movies/spinner-016.png</file> - <file alias="spinner-017">res/movies/spinner-017.png</file> - <file alias="spinner-018">res/movies/spinner-018.png</file> - <file alias="spinner-019">res/movies/spinner-019.png</file> - <file alias="spinner-020">res/movies/spinner-020.png</file> - <file alias="spinner-021">res/movies/spinner-021.png</file> - <file alias="spinner-022">res/movies/spinner-022.png</file> - <file alias="spinner-023">res/movies/spinner-023.png</file> - <file alias="spinner-024">res/movies/spinner-024.png</file> - <file alias="spinner-025">res/movies/spinner-025.png</file> - <file alias="spinner-026">res/movies/spinner-026.png</file> - <file alias="spinner-027">res/movies/spinner-027.png</file> - <file alias="spinner-028">res/movies/spinner-028.png</file> - <file alias="spinner-029">res/movies/spinner-029.png</file> - <file alias="spinner-030">res/movies/spinner-030.png</file> - <file alias="spinner-031">res/movies/spinner-031.png</file> - <file alias="spinner-032">res/movies/spinner-032.png</file> - <file alias="spinner-033">res/movies/spinner-033.png</file> - <file alias="spinner-034">res/movies/spinner-034.png</file> - <file alias="spinner-035">res/movies/spinner-035.png</file> + <qresource prefix="/animation"> + <file alias="spinner-000">res/animation/spinner-000.png</file> + <file alias="spinner-001">res/animation/spinner-001.png</file> + <file alias="spinner-002">res/animation/spinner-002.png</file> + <file alias="spinner-003">res/animation/spinner-003.png</file> + <file alias="spinner-004">res/animation/spinner-004.png</file> + <file alias="spinner-005">res/animation/spinner-005.png</file> + <file alias="spinner-006">res/animation/spinner-006.png</file> + <file alias="spinner-007">res/animation/spinner-007.png</file> + <file alias="spinner-008">res/animation/spinner-008.png</file> + <file alias="spinner-009">res/animation/spinner-009.png</file> + <file alias="spinner-010">res/animation/spinner-010.png</file> + <file alias="spinner-011">res/animation/spinner-011.png</file> + <file alias="spinner-012">res/animation/spinner-012.png</file> + <file alias="spinner-013">res/animation/spinner-013.png</file> + <file alias="spinner-014">res/animation/spinner-014.png</file> + <file alias="spinner-015">res/animation/spinner-015.png</file> + <file alias="spinner-016">res/animation/spinner-016.png</file> + <file alias="spinner-017">res/animation/spinner-017.png</file> + <file alias="spinner-018">res/animation/spinner-018.png</file> + <file alias="spinner-019">res/animation/spinner-019.png</file> + <file alias="spinner-020">res/animation/spinner-020.png</file> + <file alias="spinner-021">res/animation/spinner-021.png</file> + <file alias="spinner-022">res/animation/spinner-022.png</file> + <file alias="spinner-023">res/animation/spinner-023.png</file> + <file alias="spinner-024">res/animation/spinner-024.png</file> + <file alias="spinner-025">res/animation/spinner-025.png</file> + <file alias="spinner-026">res/animation/spinner-026.png</file> + <file alias="spinner-027">res/animation/spinner-027.png</file> + <file alias="spinner-028">res/animation/spinner-028.png</file> + <file alias="spinner-029">res/animation/spinner-029.png</file> + <file alias="spinner-030">res/animation/spinner-030.png</file> + <file alias="spinner-031">res/animation/spinner-031.png</file> + <file alias="spinner-032">res/animation/spinner-032.png</file> + <file alias="spinner-033">res/animation/spinner-033.png</file> + <file alias="spinner-034">res/animation/spinner-034.png</file> + <file alias="spinner-035">res/animation/spinner-035.png</file> </qresource> </RCC> diff --git a/src/qt/bitcoinamountfield.cpp b/src/qt/bitcoinamountfield.cpp index 4c57f1e352..a953c991bc 100644 --- a/src/qt/bitcoinamountfield.cpp +++ b/src/qt/bitcoinamountfield.cpp @@ -56,7 +56,7 @@ public: if (valid) { val = qBound(m_min_amount, val, m_max_amount); - input = BitcoinUnits::format(currentUnit, val, false, BitcoinUnits::separatorAlways); + input = BitcoinUnits::format(currentUnit, val, false, BitcoinUnits::SeparatorStyle::ALWAYS); lineEdit()->setText(input); } } @@ -68,7 +68,7 @@ public: void setValue(const CAmount& value) { - lineEdit()->setText(BitcoinUnits::format(currentUnit, value, false, BitcoinUnits::separatorAlways)); + lineEdit()->setText(BitcoinUnits::format(currentUnit, value, false, BitcoinUnits::SeparatorStyle::ALWAYS)); Q_EMIT valueChanged(); } @@ -102,7 +102,7 @@ public: CAmount val = value(&valid); currentUnit = unit; - lineEdit()->setPlaceholderText(BitcoinUnits::format(currentUnit, m_min_amount, false, BitcoinUnits::separatorAlways)); + lineEdit()->setPlaceholderText(BitcoinUnits::format(currentUnit, m_min_amount, false, BitcoinUnits::SeparatorStyle::ALWAYS)); if(valid) setValue(val); else @@ -122,7 +122,7 @@ public: const QFontMetrics fm(fontMetrics()); int h = lineEdit()->minimumSizeHint().height(); - int w = GUIUtil::TextWidth(fm, BitcoinUnits::format(BitcoinUnits::BTC, BitcoinUnits::maxMoney(), false, BitcoinUnits::separatorAlways)); + int w = GUIUtil::TextWidth(fm, BitcoinUnits::format(BitcoinUnits::BTC, BitcoinUnits::maxMoney(), false, BitcoinUnits::SeparatorStyle::ALWAYS)); w += 2; // cursor blinking space QStyleOptionSpinBox opt; diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 195d80d47c..ebcc04a5eb 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -34,7 +34,7 @@ #include <chainparams.h> #include <interfaces/handler.h> #include <interfaces/node.h> -#include <ui_interface.h> +#include <node/ui_interface.h> #include <util/system.h> #include <util/translation.h> #include <validation.h> @@ -112,6 +112,8 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty Q_EMIT consoleShown(rpcConsole); } + modalOverlay = new ModalOverlay(enableWallet, this->centralWidget()); + // Accept D&D of URIs setAcceptDrops(true); @@ -201,7 +203,6 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty openOptionsDialogWithTab(OptionsDialog::TAB_NETWORK); }); - modalOverlay = new ModalOverlay(enableWallet, this->centralWidget()); connect(labelBlocksIcon, &GUIUtil::ClickableLabel::clicked, this, &BitcoinGUI::showModalOverlay); connect(progressBar, &GUIUtil::ClickableProgressBar::clicked, this, &BitcoinGUI::showModalOverlay); #ifdef ENABLE_WALLET @@ -238,6 +239,7 @@ BitcoinGUI::~BitcoinGUI() void BitcoinGUI::createActions() { QActionGroup *tabGroup = new QActionGroup(this); + connect(modalOverlay, &ModalOverlay::triggered, tabGroup, &QActionGroup::setEnabled); overviewAction = new QAction(platformStyle->SingleColorIcon(":/icons/overview"), tr("&Overview"), this); overviewAction->setStatusTip(tr("Show general overview of wallet")); @@ -321,8 +323,10 @@ void BitcoinGUI::createActions() signMessageAction->setStatusTip(tr("Sign messages with your Bitcoin addresses to prove you own them")); verifyMessageAction = new QAction(tr("&Verify message..."), this); verifyMessageAction->setStatusTip(tr("Verify messages to ensure they were signed with specified Bitcoin addresses")); - m_load_psbt_action = new QAction(tr("Load PSBT..."), this); + m_load_psbt_action = new QAction(tr("&Load PSBT from file..."), this); m_load_psbt_action->setStatusTip(tr("Load Partially Signed Bitcoin Transaction")); + m_load_psbt_clipboard_action = new QAction(tr("Load PSBT from clipboard..."), this); + m_load_psbt_clipboard_action->setStatusTip(tr("Load Partially Signed Bitcoin Transaction from clipboard")); openRPCConsoleAction = new QAction(tr("Node window"), this); openRPCConsoleAction->setStatusTip(tr("Open node debugging and diagnostic console")); @@ -381,6 +385,7 @@ void BitcoinGUI::createActions() connect(signMessageAction, &QAction::triggered, [this]{ showNormalIfMinimized(); }); connect(signMessageAction, &QAction::triggered, [this]{ gotoSignMessageTab(); }); connect(m_load_psbt_action, &QAction::triggered, [this]{ gotoLoadPSBT(); }); + connect(m_load_psbt_clipboard_action, &QAction::triggered, [this]{ gotoLoadPSBT(true); }); connect(verifyMessageAction, &QAction::triggered, [this]{ showNormalIfMinimized(); }); connect(verifyMessageAction, &QAction::triggered, [this]{ gotoVerifyMessageTab(); }); connect(usedSendingAddressesAction, &QAction::triggered, walletFrame, &WalletFrame::usedSendingAddresses); @@ -459,6 +464,7 @@ void BitcoinGUI::createMenuBar() file->addAction(signMessageAction); file->addAction(verifyMessageAction); file->addAction(m_load_psbt_action); + file->addAction(m_load_psbt_clipboard_action); file->addSeparator(); } file->addAction(quitAction); @@ -679,6 +685,7 @@ void BitcoinGUI::removeWallet(WalletModel* walletModel) m_wallet_selector->removeItem(index); if (m_wallet_selector->count() == 0) { setWalletActionsEnabled(false); + overviewAction->setChecked(true); } else if (m_wallet_selector->count() == 1) { m_wallet_selector_label_action->setVisible(false); m_wallet_selector_action->setVisible(false); @@ -878,9 +885,9 @@ void BitcoinGUI::gotoVerifyMessageTab(QString addr) { if (walletFrame) walletFrame->gotoVerifyMessageTab(addr); } -void BitcoinGUI::gotoLoadPSBT() +void BitcoinGUI::gotoLoadPSBT(bool from_clipboard) { - if (walletFrame) walletFrame->gotoLoadPSBT(); + if (walletFrame) walletFrame->gotoLoadPSBT(from_clipboard); } #endif // ENABLE_WALLET @@ -1033,7 +1040,7 @@ void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVer if(count != prevBlocks) { labelBlocksIcon->setPixmap(platformStyle->SingleColorIcon(QString( - ":/movies/spinner-%1").arg(spinnerFrame, 3, 10, QChar('0'))) + ":/animation/spinner-%1").arg(spinnerFrame, 3, 10, QChar('0'))) .pixmap(STATUSBAR_ICONSIZE, STATUSBAR_ICONSIZE)); spinnerFrame = (spinnerFrame + 1) % SPINNER_FRAMES; } @@ -1069,9 +1076,6 @@ void BitcoinGUI::message(const QString& title, QString message, unsigned int sty int nMBoxIcon = QMessageBox::Information; int nNotifyIcon = Notificator::Information; - bool prefix = !(style & CClientUIInterface::MSG_NOPREFIX); - style &= ~CClientUIInterface::MSG_NOPREFIX; - QString msgType; if (!title.isEmpty()) { msgType = title; @@ -1079,11 +1083,11 @@ void BitcoinGUI::message(const QString& title, QString message, unsigned int sty switch (style) { case CClientUIInterface::MSG_ERROR: msgType = tr("Error"); - if (prefix) message = tr("Error: %1").arg(message); + message = tr("Error: %1").arg(message); break; case CClientUIInterface::MSG_WARNING: msgType = tr("Warning"); - if (prefix) message = tr("Warning: %1").arg(message); + message = tr("Warning: %1").arg(message); break; case CClientUIInterface::MSG_INFORMATION: msgType = tr("Information"); diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index b009e279b6..697e83e772 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -139,6 +139,7 @@ private: QAction* signMessageAction = nullptr; QAction* verifyMessageAction = nullptr; QAction* m_load_psbt_action = nullptr; + QAction* m_load_psbt_clipboard_action = nullptr; QAction* aboutAction = nullptr; QAction* receiveCoinsAction = nullptr; QAction* receiveCoinsMenuAction = nullptr; @@ -278,8 +279,8 @@ public Q_SLOTS: void gotoSignMessageTab(QString addr = ""); /** Show Sign/Verify Message dialog and switch to verify message tab */ void gotoVerifyMessageTab(QString addr = ""); - /** Show load Partially Signed Bitcoin Transaction dialog */ - void gotoLoadPSBT(); + /** Load Partially Signed Bitcoin Transaction from file or clipboard */ + void gotoLoadPSBT(bool from_clipboard = false); /** Show open dialog */ void openClicked(); diff --git a/src/qt/bitcoinunits.cpp b/src/qt/bitcoinunits.cpp index 318a6dcbfd..fd55c547fc 100644 --- a/src/qt/bitcoinunits.cpp +++ b/src/qt/bitcoinunits.cpp @@ -114,7 +114,7 @@ QString BitcoinUnits::format(int unit, const CAmount& nIn, bool fPlus, Separator // confused with the decimal marker. QChar thin_sp(THIN_SP_CP); int q_size = quotient_str.size(); - if (separators == separatorAlways || (separators == separatorStandard && q_size > 4)) + if (separators == SeparatorStyle::ALWAYS || (separators == SeparatorStyle::STANDARD && q_size > 4)) for (int i = 3; i < q_size; i += 3) quotient_str.insert(q_size - i, thin_sp); diff --git a/src/qt/bitcoinunits.h b/src/qt/bitcoinunits.h index dac5484393..e22ba0a938 100644 --- a/src/qt/bitcoinunits.h +++ b/src/qt/bitcoinunits.h @@ -46,11 +46,11 @@ public: SAT }; - enum SeparatorStyle + enum class SeparatorStyle { - separatorNever, - separatorStandard, - separatorAlways + NEVER, + STANDARD, + ALWAYS }; //! @name Static API @@ -72,11 +72,11 @@ public: //! Number of decimals left static int decimals(int unit); //! Format as string - static QString format(int unit, const CAmount& amount, bool plussign = false, SeparatorStyle separators = separatorStandard, bool justify = false); + static QString format(int unit, const CAmount& amount, bool plussign = false, SeparatorStyle separators = SeparatorStyle::STANDARD, bool justify = false); //! Format as string (with unit) - static QString formatWithUnit(int unit, const CAmount& amount, bool plussign=false, SeparatorStyle separators=separatorStandard); + static QString formatWithUnit(int unit, const CAmount& amount, bool plussign=false, SeparatorStyle separators=SeparatorStyle::STANDARD); //! Format as HTML string (with unit) - static QString formatHtmlWithUnit(int unit, const CAmount& amount, bool plussign=false, SeparatorStyle separators=separatorStandard); + static QString formatHtmlWithUnit(int unit, const CAmount& amount, bool plussign=false, SeparatorStyle separators=SeparatorStyle::STANDARD); //! Format as string (with unit) of fixed length to preserve privacy, if it is set. static QString formatWithPrivacy(int unit, const CAmount& amount, SeparatorStyle separators, bool privacy); //! Parse string to coin amount diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index db77c17df0..7c72858501 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -400,7 +400,6 @@ void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel * // nPayAmount CAmount nPayAmount = 0; bool fDust = false; - CMutableTransaction txDummy; for (const CAmount &amount : CoinControlDialog::payAmounts) { nPayAmount += amount; @@ -409,7 +408,6 @@ void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel * { // Assumes a p2pkh script size CTxOut txout(amount, CScript() << std::vector<unsigned char>(24, 0)); - txDummy.vout.push_back(txout); fDust |= IsDust(txout, model->node().getDustRelayFee()); } } @@ -458,7 +456,7 @@ void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel * { CPubKey pubkey; PKHash *pkhash = boost::get<PKHash>(&address); - if (pkhash && model->wallet().getPubKey(out.txout.scriptPubKey, CKeyID(*pkhash), pubkey)) + if (pkhash && model->wallet().getPubKey(out.txout.scriptPubKey, ToKeyID(*pkhash), pubkey)) { nBytesInputs += (pubkey.IsCompressed() ? 148 : 180); } diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index 8b70800838..93840b4169 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -1078,22 +1078,16 @@ <height>426</height> </rect> </property> - <property name="minimumSize"> - <size> - <width>300</width> - <height>0</height> - </size> - </property> <layout class="QGridLayout" name="gridLayout_2" columnstretch="0,1"> <item row="0" column="0"> <widget class="QLabel" name="label_30"> <property name="text"> - <string>Whitelisted</string> + <string>Permissions</string> </property> </widget> </item> <item row="0" column="1"> - <widget class="QLabel" name="peerWhitelisted"> + <widget class="QLabel" name="peerPermissions"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> </property> @@ -1270,36 +1264,13 @@ </widget> </item> <item row="8" column="0"> - <widget class="QLabel" name="label_24"> - <property name="text"> - <string>Ban Score</string> - </property> - </widget> - </item> - <item row="8" column="1"> - <widget class="QLabel" name="peerBanScore"> - <property name="cursor"> - <cursorShape>IBeamCursor</cursorShape> - </property> - <property name="text"> - <string>N/A</string> - </property> - <property name="textFormat"> - <enum>Qt::PlainText</enum> - </property> - <property name="textInteractionFlags"> - <set>Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse</set> - </property> - </widget> - </item> - <item row="9" column="0"> <widget class="QLabel" name="label_22"> <property name="text"> <string>Connection Time</string> </property> </widget> </item> - <item row="9" column="1"> + <item row="8" column="1"> <widget class="QLabel" name="peerConnTime"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1315,14 +1286,14 @@ </property> </widget> </item> - <item row="10" column="0"> + <item row="9" column="0"> <widget class="QLabel" name="label_15"> <property name="text"> <string>Last Send</string> </property> </widget> </item> - <item row="10" column="1"> + <item row="9" column="1"> <widget class="QLabel" name="peerLastSend"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1338,14 +1309,14 @@ </property> </widget> </item> - <item row="11" column="0"> + <item row="10" column="0"> <widget class="QLabel" name="label_19"> <property name="text"> <string>Last Receive</string> </property> </widget> </item> - <item row="11" column="1"> + <item row="10" column="1"> <widget class="QLabel" name="peerLastRecv"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1361,14 +1332,14 @@ </property> </widget> </item> - <item row="12" column="0"> + <item row="11" column="0"> <widget class="QLabel" name="label_18"> <property name="text"> <string>Sent</string> </property> </widget> </item> - <item row="12" column="1"> + <item row="11" column="1"> <widget class="QLabel" name="peerBytesSent"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1384,14 +1355,14 @@ </property> </widget> </item> - <item row="13" column="0"> + <item row="12" column="0"> <widget class="QLabel" name="label_20"> <property name="text"> <string>Received</string> </property> </widget> </item> - <item row="13" column="1"> + <item row="12" column="1"> <widget class="QLabel" name="peerBytesRecv"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1407,14 +1378,14 @@ </property> </widget> </item> - <item row="14" column="0"> + <item row="13" column="0"> <widget class="QLabel" name="label_26"> <property name="text"> <string>Ping Time</string> </property> </widget> </item> - <item row="14" column="1"> + <item row="13" column="1"> <widget class="QLabel" name="peerPingTime"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1430,7 +1401,7 @@ </property> </widget> </item> - <item row="15" column="0"> + <item row="14" column="0"> <widget class="QLabel" name="peerPingWaitLabel"> <property name="toolTip"> <string>The duration of a currently outstanding ping.</string> @@ -1440,7 +1411,7 @@ </property> </widget> </item> - <item row="15" column="1"> + <item row="14" column="1"> <widget class="QLabel" name="peerPingWait"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1456,14 +1427,14 @@ </property> </widget> </item> - <item row="16" column="0"> + <item row="15" column="0"> <widget class="QLabel" name="peerMinPingLabel"> <property name="text"> <string>Min Ping</string> </property> </widget> </item> - <item row="16" column="1"> + <item row="15" column="1"> <widget class="QLabel" name="peerMinPing"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1479,14 +1450,14 @@ </property> </widget> </item> - <item row="17" column="0"> + <item row="16" column="0"> <widget class="QLabel" name="label_timeoffset"> <property name="text"> <string>Time Offset</string> </property> </widget> </item> - <item row="17" column="1"> + <item row="16" column="1"> <widget class="QLabel" name="timeoffset"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1502,7 +1473,7 @@ </property> </widget> </item> - <item row="18" column="0"> + <item row="17" column="0"> <widget class="QLabel" name="peerMappedASLabel"> <property name="toolTip"> <string>The mapped Autonomous System used for diversifying peer selection.</string> @@ -1512,7 +1483,7 @@ </property> </widget> </item> - <item row="18" column="1"> + <item row="17" column="1"> <widget class="QLabel" name="peerMappedAS"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1528,7 +1499,7 @@ </property> </widget> </item> - <item row="19" column="0"> + <item row="18" column="0"> <spacer name="verticalSpacer_3"> <property name="orientation"> <enum>Qt::Vertical</enum> diff --git a/src/qt/forms/psbtoperationsdialog.ui b/src/qt/forms/psbtoperationsdialog.ui new file mode 100644 index 0000000000..c2e2f5035b --- /dev/null +++ b/src/qt/forms/psbtoperationsdialog.ui @@ -0,0 +1,148 @@ +<?xml version="1.0" encoding="UTF-8"?> +<ui version="4.0"> + <class>PSBTOperationsDialog</class> + <widget class="QDialog" name="PSBTOperationsDialog"> + <property name="geometry"> + <rect> + <x>0</x> + <y>0</y> + <width>585</width> + <height>327</height> + </rect> + </property> + <property name="windowTitle"> + <string>Dialog</string> + </property> + <layout class="QVBoxLayout" name="verticalLayout"> + <property name="spacing"> + <number>12</number> + </property> + <property name="sizeConstraint"> + <enum>QLayout::SetDefaultConstraint</enum> + </property> + <property name="bottomMargin"> + <number>12</number> + </property> + <item> + <layout class="QVBoxLayout" name="mainDialogLayout"> + <property name="spacing"> + <number>5</number> + </property> + <property name="topMargin"> + <number>0</number> + </property> + <property name="bottomMargin"> + <number>0</number> + </property> + <item> + <widget class="QLabel" name="statusBar"> + <property name="font"> + <font> + <weight>75</weight> + <bold>true</bold> + </font> + </property> + <property name="autoFillBackground"> + <bool>false</bool> + </property> + <property name="styleSheet"> + <string notr="true"/> + </property> + <property name="text"> + <string/> + </property> + </widget> + </item> + <item> + <widget class="QTextEdit" name="transactionDescription"> + <property name="undoRedoEnabled"> + <bool>false</bool> + </property> + <property name="readOnly"> + <bool>true</bool> + </property> + </widget> + </item> + <item> + <layout class="QHBoxLayout" name="buttonRowLayout"> + <property name="spacing"> + <number>5</number> + </property> + <item> + <widget class="QPushButton" name="signTransactionButton"> + <property name="sizePolicy"> + <sizepolicy hsizetype="Minimum" vsizetype="Fixed"> + <horstretch>0</horstretch> + <verstretch>0</verstretch> + </sizepolicy> + </property> + <property name="font"> + <font> + <weight>50</weight> + <bold>false</bold> + </font> + </property> + <property name="text"> + <string>Sign Tx</string> + </property> + <property name="autoDefault"> + <bool>true</bool> + </property> + <property name="default"> + <bool>false</bool> + </property> + <property name="flat"> + <bool>false</bool> + </property> + </widget> + </item> + <item> + <widget class="QPushButton" name="broadcastTransactionButton"> + <property name="text"> + <string>Broadcast Tx</string> + </property> + </widget> + </item> + <item> + <spacer name="horizontalSpacer"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>40</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + <item> + <widget class="QPushButton" name="copyToClipboardButton"> + <property name="text"> + <string>Copy to Clipboard</string> + </property> + </widget> + </item> + <item> + <widget class="QPushButton" name="saveButton"> + <property name="text"> + <string>Save...</string> + </property> + </widget> + </item> + <item> + <widget class="QPushButton" name="closeButton"> + <property name="text"> + <string>Close</string> + </property> + </widget> + </item> + </layout> + </item> + </layout> + </item> + </layout> + </widget> + <resources/> + <connections/> +</ui> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index ce44d4f3a5..7f439fa45e 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -188,7 +188,7 @@ QString formatBitcoinURI(const SendCoinsRecipient &info) if (info.amount) { - ret += QString("?amount=%1").arg(BitcoinUnits::format(BitcoinUnits::BTC, info.amount, false, BitcoinUnits::separatorNever)); + ret += QString("?amount=%1").arg(BitcoinUnits::format(BitcoinUnits::BTC, info.amount, false, BitcoinUnits::SeparatorStyle::NEVER)); paramCount++; } @@ -450,6 +450,28 @@ bool ToolTipToRichTextFilter::eventFilter(QObject *obj, QEvent *evt) return QObject::eventFilter(obj, evt); } +LabelOutOfFocusEventFilter::LabelOutOfFocusEventFilter(QObject* parent) + : QObject(parent) +{ +} + +bool LabelOutOfFocusEventFilter::eventFilter(QObject* watched, QEvent* event) +{ + if (event->type() == QEvent::FocusOut) { + auto focus_out = static_cast<QFocusEvent*>(event); + if (focus_out->reason() != Qt::PopupFocusReason) { + auto label = qobject_cast<QLabel*>(watched); + if (label) { + auto flags = label->textInteractionFlags(); + label->setTextInteractionFlags(Qt::NoTextInteraction); + label->setTextInteractionFlags(flags); + } + } + } + + return QObject::eventFilter(watched, event); +} + void TableViewLastColumnResizingFixer::connectViewHeadersSignals() { connect(tableView->horizontalHeader(), &QHeaderView::sectionResized, this, &TableViewLastColumnResizingFixer::on_sectionResized); diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 8741d90102..2bd94b5eb3 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -162,6 +162,21 @@ namespace GUIUtil }; /** + * Qt event filter that intercepts QEvent::FocusOut events for QLabel objects, and + * resets their `textInteractionFlags' property to get rid of the visible cursor. + * + * This is a temporary fix of QTBUG-59514. + */ + class LabelOutOfFocusEventFilter : public QObject + { + Q_OBJECT + + public: + explicit LabelOutOfFocusEventFilter(QObject* parent); + bool eventFilter(QObject* watched, QEvent* event) override; + }; + + /** * Makes a QTableView last column feel as if it was being resized from its left border. * Also makes sure the column widths are never larger than the table's viewport. * In Qt, all columns are resizable from the right, but it's not intuitive resizing the last column from the right. diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index 0ba1beaf3e..8070aa627c 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -171,6 +171,8 @@ void ModalOverlay::showHide(bool hide, bool userRequested) if ( (layerIsVisible && !hide) || (!layerIsVisible && hide) || (!hide && userClosed && !userRequested)) return; + Q_EMIT triggered(hide); + if (!isVisible() && !hide) setVisible(true); diff --git a/src/qt/modaloverlay.h b/src/qt/modaloverlay.h index 1d84046d3d..7b07777641 100644 --- a/src/qt/modaloverlay.h +++ b/src/qt/modaloverlay.h @@ -25,16 +25,20 @@ public: explicit ModalOverlay(bool enable_wallet, QWidget *parent); ~ModalOverlay(); -public Q_SLOTS: void tipUpdate(int count, const QDateTime& blockDate, double nVerificationProgress); void setKnownBestHeight(int count, const QDateTime& blockDate); - void toggleVisibility(); // will show or hide the modal layer void showHide(bool hide = false, bool userRequested = false); - void closeClicked(); bool isLayerVisible() const { return layerIsVisible; } +public Q_SLOTS: + void toggleVisibility(); + void closeClicked(); + +Q_SIGNALS: + void triggered(bool hidden); + protected: bool eventFilter(QObject * obj, QEvent * ev) override; bool event(QEvent* ev) override; diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 6ca5ac9d75..14fdf9046e 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -6,6 +6,7 @@ #define BITCOIN_QT_OPTIONSMODEL_H #include <amount.h> +#include <cstdint> #include <qt/guiconstants.h> #include <QAbstractListModel> @@ -15,7 +16,7 @@ class Node; } extern const char *DEFAULT_GUI_PROXY_HOST; -static constexpr unsigned short DEFAULT_GUI_PROXY_PORT = 9050; +static constexpr uint16_t DEFAULT_GUI_PROXY_PORT = 9050; /** * Convert configured prune target MiB to displayed GB. Round up to avoid underestimating max disk usage. diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 0af70f2735..b536567c8b 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -88,7 +88,7 @@ public: foreground = option.palette.color(QPalette::Text); } painter->setPen(foreground); - QString amountText = BitcoinUnits::formatWithUnit(unit, amount, true, BitcoinUnits::separatorAlways); + QString amountText = BitcoinUnits::formatWithUnit(unit, amount, true, BitcoinUnits::SeparatorStyle::ALWAYS); if(!confirmed) { amountText = QString("[") + amountText + QString("]"); @@ -180,25 +180,25 @@ void OverviewPage::setBalance(const interfaces::WalletBalances& balances) m_balances = balances; if (walletModel->wallet().isLegacy()) { if (walletModel->wallet().privateKeysDisabled()) { - ui->labelBalance->setText(BitcoinUnits::formatWithPrivacy(unit, balances.watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); - ui->labelUnconfirmed->setText(BitcoinUnits::formatWithPrivacy(unit, balances.unconfirmed_watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); - ui->labelImmature->setText(BitcoinUnits::formatWithPrivacy(unit, balances.immature_watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); - ui->labelTotal->setText(BitcoinUnits::formatWithPrivacy(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelBalance->setText(BitcoinUnits::formatWithPrivacy(unit, balances.watch_only_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); + ui->labelUnconfirmed->setText(BitcoinUnits::formatWithPrivacy(unit, balances.unconfirmed_watch_only_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); + ui->labelImmature->setText(BitcoinUnits::formatWithPrivacy(unit, balances.immature_watch_only_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); + ui->labelTotal->setText(BitcoinUnits::formatWithPrivacy(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); } else { - ui->labelBalance->setText(BitcoinUnits::formatWithPrivacy(unit, balances.balance, BitcoinUnits::separatorAlways, m_privacy)); - ui->labelUnconfirmed->setText(BitcoinUnits::formatWithPrivacy(unit, balances.unconfirmed_balance, BitcoinUnits::separatorAlways, m_privacy)); - ui->labelImmature->setText(BitcoinUnits::formatWithPrivacy(unit, balances.immature_balance, BitcoinUnits::separatorAlways, m_privacy)); - ui->labelTotal->setText(BitcoinUnits::formatWithPrivacy(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, BitcoinUnits::separatorAlways, m_privacy)); - ui->labelWatchAvailable->setText(BitcoinUnits::formatWithPrivacy(unit, balances.watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); - ui->labelWatchPending->setText(BitcoinUnits::formatWithPrivacy(unit, balances.unconfirmed_watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); - ui->labelWatchImmature->setText(BitcoinUnits::formatWithPrivacy(unit, balances.immature_watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); - ui->labelWatchTotal->setText(BitcoinUnits::formatWithPrivacy(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelBalance->setText(BitcoinUnits::formatWithPrivacy(unit, balances.balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); + ui->labelUnconfirmed->setText(BitcoinUnits::formatWithPrivacy(unit, balances.unconfirmed_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); + ui->labelImmature->setText(BitcoinUnits::formatWithPrivacy(unit, balances.immature_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); + ui->labelTotal->setText(BitcoinUnits::formatWithPrivacy(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); + ui->labelWatchAvailable->setText(BitcoinUnits::formatWithPrivacy(unit, balances.watch_only_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); + ui->labelWatchPending->setText(BitcoinUnits::formatWithPrivacy(unit, balances.unconfirmed_watch_only_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); + ui->labelWatchImmature->setText(BitcoinUnits::formatWithPrivacy(unit, balances.immature_watch_only_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); + ui->labelWatchTotal->setText(BitcoinUnits::formatWithPrivacy(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); } } else { - ui->labelBalance->setText(BitcoinUnits::formatWithPrivacy(unit, balances.balance, BitcoinUnits::separatorAlways, m_privacy)); - ui->labelUnconfirmed->setText(BitcoinUnits::formatWithPrivacy(unit, balances.unconfirmed_balance, BitcoinUnits::separatorAlways, m_privacy)); - ui->labelImmature->setText(BitcoinUnits::formatWithPrivacy(unit, balances.immature_balance, BitcoinUnits::separatorAlways, m_privacy)); - ui->labelTotal->setText(BitcoinUnits::formatWithPrivacy(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, BitcoinUnits::separatorAlways, m_privacy)); + ui->labelBalance->setText(BitcoinUnits::formatWithPrivacy(unit, balances.balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); + ui->labelUnconfirmed->setText(BitcoinUnits::formatWithPrivacy(unit, balances.unconfirmed_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); + ui->labelImmature->setText(BitcoinUnits::formatWithPrivacy(unit, balances.immature_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); + ui->labelTotal->setText(BitcoinUnits::formatWithPrivacy(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, BitcoinUnits::SeparatorStyle::ALWAYS, m_privacy)); } // only show immature (newly mined) balance if it's non-zero, so as not to complicate things // for the non-mining users diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index beca78a021..a1da85bda7 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -14,9 +14,9 @@ #include <chainparams.h> #include <interfaces/node.h> -#include <policy/policy.h> #include <key_io.h> -#include <ui_interface.h> +#include <node/ui_interface.h> +#include <policy/policy.h> #include <util/system.h> #include <wallet/wallet.h> diff --git a/src/qt/psbtoperationsdialog.cpp b/src/qt/psbtoperationsdialog.cpp new file mode 100644 index 0000000000..58167d4bb4 --- /dev/null +++ b/src/qt/psbtoperationsdialog.cpp @@ -0,0 +1,268 @@ +// Copyright (c) 2011-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <qt/psbtoperationsdialog.h> + +#include <core_io.h> +#include <interfaces/node.h> +#include <key_io.h> +#include <node/psbt.h> +#include <policy/policy.h> +#include <qt/bitcoinunits.h> +#include <qt/forms/ui_psbtoperationsdialog.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <util/strencodings.h> + +#include <iostream> + + +PSBTOperationsDialog::PSBTOperationsDialog( + QWidget* parent, WalletModel* wallet_model, ClientModel* client_model) : QDialog(parent), + m_ui(new Ui::PSBTOperationsDialog), + m_wallet_model(wallet_model), + m_client_model(client_model) +{ + m_ui->setupUi(this); + setWindowTitle("PSBT Operations"); + + connect(m_ui->signTransactionButton, &QPushButton::clicked, this, &PSBTOperationsDialog::signTransaction); + connect(m_ui->broadcastTransactionButton, &QPushButton::clicked, this, &PSBTOperationsDialog::broadcastTransaction); + connect(m_ui->copyToClipboardButton, &QPushButton::clicked, this, &PSBTOperationsDialog::copyToClipboard); + connect(m_ui->saveButton, &QPushButton::clicked, this, &PSBTOperationsDialog::saveTransaction); + + connect(m_ui->closeButton, &QPushButton::clicked, this, &PSBTOperationsDialog::close); + + m_ui->signTransactionButton->setEnabled(false); + m_ui->broadcastTransactionButton->setEnabled(false); +} + +PSBTOperationsDialog::~PSBTOperationsDialog() +{ + delete m_ui; +} + +void PSBTOperationsDialog::openWithPSBT(PartiallySignedTransaction psbtx) +{ + m_transaction_data = psbtx; + + bool complete; + size_t n_could_sign; + FinalizePSBT(psbtx); // Make sure all existing signatures are fully combined before checking for completeness. + TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, m_transaction_data, complete, &n_could_sign); + if (err != TransactionError::OK) { + showStatus(tr("Failed to load transaction: %1") + .arg(QString::fromStdString(TransactionErrorString(err).translated)), StatusLevel::ERR); + return; + } + + m_ui->broadcastTransactionButton->setEnabled(complete); + m_ui->signTransactionButton->setEnabled(!complete && !m_wallet_model->wallet().privateKeysDisabled() && n_could_sign > 0); + + updateTransactionDisplay(); +} + +void PSBTOperationsDialog::signTransaction() +{ + bool complete; + size_t n_signed; + TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, true /* sign */, true /* bip32derivs */, m_transaction_data, complete, &n_signed); + + if (err != TransactionError::OK) { + showStatus(tr("Failed to sign transaction: %1") + .arg(QString::fromStdString(TransactionErrorString(err).translated)), StatusLevel::ERR); + return; + } + + updateTransactionDisplay(); + + if (!complete && n_signed < 1) { + showStatus(tr("Could not sign any more inputs."), StatusLevel::WARN); + } else if (!complete) { + showStatus(tr("Signed %1 inputs, but more signatures are still required.").arg(n_signed), + StatusLevel::INFO); + } else { + showStatus(tr("Signed transaction successfully. Transaction is ready to broadcast."), + StatusLevel::INFO); + m_ui->broadcastTransactionButton->setEnabled(true); + } +} + +void PSBTOperationsDialog::broadcastTransaction() +{ + CMutableTransaction mtx; + if (!FinalizeAndExtractPSBT(m_transaction_data, mtx)) { + // This is never expected to fail unless we were given a malformed PSBT + // (e.g. with an invalid signature.) + showStatus(tr("Unknown error processing transaction."), StatusLevel::ERR); + return; + } + + CTransactionRef tx = MakeTransactionRef(mtx); + std::string err_string; + TransactionError error = BroadcastTransaction( + *m_client_model->node().context(), tx, err_string, DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK(), /* relay */ true, /* await_callback */ false); + + if (error == TransactionError::OK) { + showStatus(tr("Transaction broadcast successfully! Transaction ID: %1") + .arg(QString::fromStdString(tx->GetHash().GetHex())), StatusLevel::INFO); + } else { + showStatus(tr("Transaction broadcast failed: %1") + .arg(QString::fromStdString(TransactionErrorString(error).translated)), StatusLevel::ERR); + } +} + +void PSBTOperationsDialog::copyToClipboard() { + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << m_transaction_data; + GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); + showStatus(tr("PSBT copied to clipboard."), StatusLevel::INFO); +} + +void PSBTOperationsDialog::saveTransaction() { + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << m_transaction_data; + + QString selected_filter; + QString filename_suggestion = ""; + bool first = true; + for (const CTxOut& out : m_transaction_data.tx->vout) { + if (!first) { + filename_suggestion.append("-"); + } + CTxDestination address; + ExtractDestination(out.scriptPubKey, address); + QString amount = BitcoinUnits::format(m_wallet_model->getOptionsModel()->getDisplayUnit(), out.nValue); + QString address_str = QString::fromStdString(EncodeDestination(address)); + filename_suggestion.append(address_str + "-" + amount); + first = false; + } + filename_suggestion.append(".psbt"); + QString filename = GUIUtil::getSaveFileName(this, + tr("Save Transaction Data"), filename_suggestion, + tr("Partially Signed Transaction (Binary) (*.psbt)"), &selected_filter); + if (filename.isEmpty()) { + return; + } + std::ofstream out(filename.toLocal8Bit().data()); + out << ssTx.str(); + out.close(); + showStatus(tr("PSBT saved to disk."), StatusLevel::INFO); +} + +void PSBTOperationsDialog::updateTransactionDisplay() { + m_ui->transactionDescription->setText(QString::fromStdString(renderTransaction(m_transaction_data))); + showTransactionStatus(m_transaction_data); +} + +std::string PSBTOperationsDialog::renderTransaction(const PartiallySignedTransaction &psbtx) +{ + QString tx_description = ""; + CAmount totalAmount = 0; + for (const CTxOut& out : psbtx.tx->vout) { + CTxDestination address; + ExtractDestination(out.scriptPubKey, address); + totalAmount += out.nValue; + tx_description.append(tr(" * Sends %1 to %2") + .arg(BitcoinUnits::formatWithUnit(BitcoinUnits::BTC, out.nValue)) + .arg(QString::fromStdString(EncodeDestination(address)))); + tx_description.append("<br>"); + } + + PSBTAnalysis analysis = AnalyzePSBT(psbtx); + tx_description.append(" * "); + if (!*analysis.fee) { + // This happens if the transaction is missing input UTXO information. + tx_description.append(tr("Unable to calculate transaction fee or total transaction amount.")); + } else { + tx_description.append(tr("Pays transaction fee: ")); + tx_description.append(BitcoinUnits::formatWithUnit(BitcoinUnits::BTC, *analysis.fee)); + + // add total amount in all subdivision units + tx_description.append("<hr />"); + QStringList alternativeUnits; + for (const BitcoinUnits::Unit u : BitcoinUnits::availableUnits()) + { + if(u != m_client_model->getOptionsModel()->getDisplayUnit()) { + alternativeUnits.append(BitcoinUnits::formatHtmlWithUnit(u, totalAmount)); + } + } + tx_description.append(QString("<b>%1</b>: <b>%2</b>").arg(tr("Total Amount")) + .arg(BitcoinUnits::formatHtmlWithUnit(m_client_model->getOptionsModel()->getDisplayUnit(), totalAmount))); + tx_description.append(QString("<br /><span style='font-size:10pt; font-weight:normal;'>(=%1)</span>") + .arg(alternativeUnits.join(" " + tr("or") + " "))); + } + + size_t num_unsigned = CountPSBTUnsignedInputs(psbtx); + if (num_unsigned > 0) { + tx_description.append("<br><br>"); + tx_description.append(tr("Transaction has %1 unsigned inputs.").arg(QString::number(num_unsigned))); + } + + return tx_description.toStdString(); +} + +void PSBTOperationsDialog::showStatus(const QString &msg, StatusLevel level) { + m_ui->statusBar->setText(msg); + switch (level) { + case StatusLevel::INFO: { + m_ui->statusBar->setStyleSheet("QLabel { background-color : lightgreen }"); + break; + } + case StatusLevel::WARN: { + m_ui->statusBar->setStyleSheet("QLabel { background-color : orange }"); + break; + } + case StatusLevel::ERR: { + m_ui->statusBar->setStyleSheet("QLabel { background-color : red }"); + break; + } + } + m_ui->statusBar->show(); +} + +size_t PSBTOperationsDialog::couldSignInputs(const PartiallySignedTransaction &psbtx) { + size_t n_signed; + bool complete; + TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, false /* sign */, false /* bip32derivs */, m_transaction_data, complete, &n_signed); + + if (err != TransactionError::OK) { + return 0; + } + return n_signed; +} + +void PSBTOperationsDialog::showTransactionStatus(const PartiallySignedTransaction &psbtx) { + PSBTAnalysis analysis = AnalyzePSBT(psbtx); + size_t n_could_sign = couldSignInputs(psbtx); + + switch (analysis.next) { + case PSBTRole::UPDATER: { + showStatus(tr("Transaction is missing some information about inputs."), StatusLevel::WARN); + break; + } + case PSBTRole::SIGNER: { + QString need_sig_text = tr("Transaction still needs signature(s)."); + StatusLevel level = StatusLevel::INFO; + if (m_wallet_model->wallet().privateKeysDisabled()) { + need_sig_text += " " + tr("(But this wallet cannot sign transactions.)"); + level = StatusLevel::WARN; + } else if (n_could_sign < 1) { + need_sig_text += " " + tr("(But this wallet does not have the right keys.)"); // XXX wording + level = StatusLevel::WARN; + } + showStatus(need_sig_text, level); + break; + } + case PSBTRole::FINALIZER: + case PSBTRole::EXTRACTOR: { + showStatus(tr("Transaction is fully signed and ready for broadcast."), StatusLevel::INFO); + break; + } + default: { + showStatus(tr("Transaction status is unknown."), StatusLevel::ERR); + break; + } + } +} diff --git a/src/qt/psbtoperationsdialog.h b/src/qt/psbtoperationsdialog.h new file mode 100644 index 0000000000..f37bdbe39a --- /dev/null +++ b/src/qt/psbtoperationsdialog.h @@ -0,0 +1,54 @@ +// Copyright (c) 2011-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_QT_PSBTOPERATIONSDIALOG_H +#define BITCOIN_QT_PSBTOPERATIONSDIALOG_H + +#include <QDialog> + +#include <psbt.h> +#include <qt/clientmodel.h> +#include <qt/walletmodel.h> + +namespace Ui { +class PSBTOperationsDialog; +} + +/** Dialog showing transaction details. */ +class PSBTOperationsDialog : public QDialog +{ + Q_OBJECT + +public: + explicit PSBTOperationsDialog(QWidget* parent, WalletModel* walletModel, ClientModel* clientModel); + ~PSBTOperationsDialog(); + + void openWithPSBT(PartiallySignedTransaction psbtx); + +public Q_SLOTS: + void signTransaction(); + void broadcastTransaction(); + void copyToClipboard(); + void saveTransaction(); + +private: + Ui::PSBTOperationsDialog* m_ui; + PartiallySignedTransaction m_transaction_data; + WalletModel* m_wallet_model; + ClientModel* m_client_model; + + enum class StatusLevel { + INFO, + WARN, + ERR + }; + + size_t couldSignInputs(const PartiallySignedTransaction &psbtx); + void updateTransactionDisplay(); + std::string renderTransaction(const PartiallySignedTransaction &psbtx); + void showStatus(const QString &msg, StatusLevel level); + void showTransactionStatus(const PartiallySignedTransaction &psbtx); +}; + +#endif // BITCOIN_QT_PSBTOPERATIONSDIALOG_H diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 7419297a96..3e20368a36 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -82,7 +82,7 @@ QVariant RecentRequestsTableModel::data(const QModelIndex &index, int role) cons if (rec->recipient.amount == 0 && role == Qt::DisplayRole) return tr("(no amount requested)"); else if (role == Qt::EditRole) - return BitcoinUnits::format(walletModel->getOptionsModel()->getDisplayUnit(), rec->recipient.amount, false, BitcoinUnits::separatorNever); + return BitcoinUnits::format(walletModel->getOptionsModel()->getDisplayUnit(), rec->recipient.amount, false, BitcoinUnits::SeparatorStyle::NEVER); else return BitcoinUnits::format(walletModel->getOptionsModel()->getDisplayUnit(), rec->recipient.amount); } diff --git a/src/qt/res/movies/makespinner.sh b/src/qt/res/animation/makespinner.sh index 4fa8dadf86..4fa8dadf86 100755 --- a/src/qt/res/movies/makespinner.sh +++ b/src/qt/res/animation/makespinner.sh diff --git a/src/qt/res/movies/spinner-000.png b/src/qt/res/animation/spinner-000.png Binary files differindex 0dc48d0d8c..0dc48d0d8c 100644 --- a/src/qt/res/movies/spinner-000.png +++ b/src/qt/res/animation/spinner-000.png diff --git a/src/qt/res/movies/spinner-001.png b/src/qt/res/animation/spinner-001.png Binary files differindex d167f20541..d167f20541 100644 --- a/src/qt/res/movies/spinner-001.png +++ b/src/qt/res/animation/spinner-001.png diff --git a/src/qt/res/movies/spinner-002.png b/src/qt/res/animation/spinner-002.png Binary files differindex 4a1f1f8e56..4a1f1f8e56 100644 --- a/src/qt/res/movies/spinner-002.png +++ b/src/qt/res/animation/spinner-002.png diff --git a/src/qt/res/movies/spinner-003.png b/src/qt/res/animation/spinner-003.png Binary files differindex fb1c2cd4ad..fb1c2cd4ad 100644 --- a/src/qt/res/movies/spinner-003.png +++ b/src/qt/res/animation/spinner-003.png diff --git a/src/qt/res/movies/spinner-004.png b/src/qt/res/animation/spinner-004.png Binary files differindex 4df2132344..4df2132344 100644 --- a/src/qt/res/movies/spinner-004.png +++ b/src/qt/res/animation/spinner-004.png diff --git a/src/qt/res/movies/spinner-005.png b/src/qt/res/animation/spinner-005.png Binary files differindex 5d6f41e0dc..5d6f41e0dc 100644 --- a/src/qt/res/movies/spinner-005.png +++ b/src/qt/res/animation/spinner-005.png diff --git a/src/qt/res/movies/spinner-006.png b/src/qt/res/animation/spinner-006.png Binary files differindex c1f7d18899..c1f7d18899 100644 --- a/src/qt/res/movies/spinner-006.png +++ b/src/qt/res/animation/spinner-006.png diff --git a/src/qt/res/movies/spinner-007.png b/src/qt/res/animation/spinner-007.png Binary files differindex 1e794b2626..1e794b2626 100644 --- a/src/qt/res/movies/spinner-007.png +++ b/src/qt/res/animation/spinner-007.png diff --git a/src/qt/res/movies/spinner-008.png b/src/qt/res/animation/spinner-008.png Binary files differindex df12ea8719..df12ea8719 100644 --- a/src/qt/res/movies/spinner-008.png +++ b/src/qt/res/animation/spinner-008.png diff --git a/src/qt/res/movies/spinner-009.png b/src/qt/res/animation/spinner-009.png Binary files differindex 18fc3a7d16..18fc3a7d16 100644 --- a/src/qt/res/movies/spinner-009.png +++ b/src/qt/res/animation/spinner-009.png diff --git a/src/qt/res/movies/spinner-010.png b/src/qt/res/animation/spinner-010.png Binary files differindex a79c845fe8..a79c845fe8 100644 --- a/src/qt/res/movies/spinner-010.png +++ b/src/qt/res/animation/spinner-010.png diff --git a/src/qt/res/movies/spinner-011.png b/src/qt/res/animation/spinner-011.png Binary files differindex 57baf66895..57baf66895 100644 --- a/src/qt/res/movies/spinner-011.png +++ b/src/qt/res/animation/spinner-011.png diff --git a/src/qt/res/movies/spinner-012.png b/src/qt/res/animation/spinner-012.png Binary files differindex 9deae7853a..9deae7853a 100644 --- a/src/qt/res/movies/spinner-012.png +++ b/src/qt/res/animation/spinner-012.png diff --git a/src/qt/res/movies/spinner-013.png b/src/qt/res/animation/spinner-013.png Binary files differindex 0659d48dec..0659d48dec 100644 --- a/src/qt/res/movies/spinner-013.png +++ b/src/qt/res/animation/spinner-013.png diff --git a/src/qt/res/movies/spinner-014.png b/src/qt/res/animation/spinner-014.png Binary files differindex bc1ef51bde..bc1ef51bde 100644 --- a/src/qt/res/movies/spinner-014.png +++ b/src/qt/res/animation/spinner-014.png diff --git a/src/qt/res/movies/spinner-015.png b/src/qt/res/animation/spinner-015.png Binary files differindex 24b57b62c2..24b57b62c2 100644 --- a/src/qt/res/movies/spinner-015.png +++ b/src/qt/res/animation/spinner-015.png diff --git a/src/qt/res/movies/spinner-016.png b/src/qt/res/animation/spinner-016.png Binary files differindex d622872651..d622872651 100644 --- a/src/qt/res/movies/spinner-016.png +++ b/src/qt/res/animation/spinner-016.png diff --git a/src/qt/res/movies/spinner-017.png b/src/qt/res/animation/spinner-017.png Binary files differindex f48f688db2..f48f688db2 100644 --- a/src/qt/res/movies/spinner-017.png +++ b/src/qt/res/animation/spinner-017.png diff --git a/src/qt/res/movies/spinner-018.png b/src/qt/res/animation/spinner-018.png Binary files differindex a2c8f38b1d..a2c8f38b1d 100644 --- a/src/qt/res/movies/spinner-018.png +++ b/src/qt/res/animation/spinner-018.png diff --git a/src/qt/res/movies/spinner-019.png b/src/qt/res/animation/spinner-019.png Binary files differindex 9d7cc35d82..9d7cc35d82 100644 --- a/src/qt/res/movies/spinner-019.png +++ b/src/qt/res/animation/spinner-019.png diff --git a/src/qt/res/movies/spinner-020.png b/src/qt/res/animation/spinner-020.png Binary files differindex 1a07acc454..1a07acc454 100644 --- a/src/qt/res/movies/spinner-020.png +++ b/src/qt/res/animation/spinner-020.png diff --git a/src/qt/res/movies/spinner-021.png b/src/qt/res/animation/spinner-021.png Binary files differindex 9cea8f2543..9cea8f2543 100644 --- a/src/qt/res/movies/spinner-021.png +++ b/src/qt/res/animation/spinner-021.png diff --git a/src/qt/res/movies/spinner-022.png b/src/qt/res/animation/spinner-022.png Binary files differindex 60250f6dea..60250f6dea 100644 --- a/src/qt/res/movies/spinner-022.png +++ b/src/qt/res/animation/spinner-022.png diff --git a/src/qt/res/movies/spinner-023.png b/src/qt/res/animation/spinner-023.png Binary files differindex fc290a0cf2..fc290a0cf2 100644 --- a/src/qt/res/movies/spinner-023.png +++ b/src/qt/res/animation/spinner-023.png diff --git a/src/qt/res/movies/spinner-024.png b/src/qt/res/animation/spinner-024.png Binary files differindex c5dcf1eae9..c5dcf1eae9 100644 --- a/src/qt/res/movies/spinner-024.png +++ b/src/qt/res/animation/spinner-024.png diff --git a/src/qt/res/movies/spinner-025.png b/src/qt/res/animation/spinner-025.png Binary files differindex 7f3577a4de..7f3577a4de 100644 --- a/src/qt/res/movies/spinner-025.png +++ b/src/qt/res/animation/spinner-025.png diff --git a/src/qt/res/movies/spinner-026.png b/src/qt/res/animation/spinner-026.png Binary files differindex 1663ddf44c..1663ddf44c 100644 --- a/src/qt/res/movies/spinner-026.png +++ b/src/qt/res/animation/spinner-026.png diff --git a/src/qt/res/movies/spinner-027.png b/src/qt/res/animation/spinner-027.png Binary files differindex d0e6da4503..d0e6da4503 100644 --- a/src/qt/res/movies/spinner-027.png +++ b/src/qt/res/animation/spinner-027.png diff --git a/src/qt/res/movies/spinner-028.png b/src/qt/res/animation/spinner-028.png Binary files differindex 2a7aba50e2..2a7aba50e2 100644 --- a/src/qt/res/movies/spinner-028.png +++ b/src/qt/res/animation/spinner-028.png diff --git a/src/qt/res/movies/spinner-029.png b/src/qt/res/animation/spinner-029.png Binary files differindex c8ca15c1e1..c8ca15c1e1 100644 --- a/src/qt/res/movies/spinner-029.png +++ b/src/qt/res/animation/spinner-029.png diff --git a/src/qt/res/movies/spinner-030.png b/src/qt/res/animation/spinner-030.png Binary files differindex c847c99a93..c847c99a93 100644 --- a/src/qt/res/movies/spinner-030.png +++ b/src/qt/res/animation/spinner-030.png diff --git a/src/qt/res/movies/spinner-031.png b/src/qt/res/animation/spinner-031.png Binary files differindex 403443144e..403443144e 100644 --- a/src/qt/res/movies/spinner-031.png +++ b/src/qt/res/animation/spinner-031.png diff --git a/src/qt/res/movies/spinner-032.png b/src/qt/res/animation/spinner-032.png Binary files differindex f9db080567..f9db080567 100644 --- a/src/qt/res/movies/spinner-032.png +++ b/src/qt/res/animation/spinner-032.png diff --git a/src/qt/res/movies/spinner-033.png b/src/qt/res/animation/spinner-033.png Binary files differindex 43f57719e7..43f57719e7 100644 --- a/src/qt/res/movies/spinner-033.png +++ b/src/qt/res/animation/spinner-033.png diff --git a/src/qt/res/movies/spinner-034.png b/src/qt/res/animation/spinner-034.png Binary files differindex c26656ff17..c26656ff17 100644 --- a/src/qt/res/movies/spinner-034.png +++ b/src/qt/res/animation/spinner-034.png diff --git a/src/qt/res/movies/spinner-035.png b/src/qt/res/animation/spinner-035.png Binary files differindex e471f950a3..e471f950a3 100644 --- a/src/qt/res/movies/spinner-035.png +++ b/src/qt/res/animation/spinner-035.png diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 0f89d4e6fe..821a337a62 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -466,6 +466,7 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty // Install event filter for up and down arrow ui->lineEdit->installEventFilter(this); + ui->lineEdit->setMaxLength(16 * 1024 * 1024); ui->messagesWidget->installEventFilter(this); connect(ui->clearButton, &QPushButton::clicked, this, &RPCConsole::clear); @@ -1118,15 +1119,20 @@ void RPCConsole::updateNodeDetail(const CNodeCombinedStats *stats) ui->peerSubversion->setText(QString::fromStdString(stats->nodeStats.cleanSubVer)); ui->peerDirection->setText(stats->nodeStats.fInbound ? tr("Inbound") : tr("Outbound")); ui->peerHeight->setText(QString::number(stats->nodeStats.nStartingHeight)); - ui->peerWhitelisted->setText(stats->nodeStats.m_legacyWhitelisted ? tr("Yes") : tr("No")); + if (stats->nodeStats.m_permissionFlags == PF_NONE) { + ui->peerPermissions->setText(tr("N/A")); + } else { + QStringList permissions; + for (const auto& permission : NetPermissions::ToStrings(stats->nodeStats.m_permissionFlags)) { + permissions.append(QString::fromStdString(permission)); + } + ui->peerPermissions->setText(permissions.join(" & ")); + } ui->peerMappedAS->setText(stats->nodeStats.m_mapped_as != 0 ? QString::number(stats->nodeStats.m_mapped_as) : tr("N/A")); // This check fails for example if the lock was busy and // nodeStateStats couldn't be fetched. if (stats->fNodeStateStatsAvailable) { - // Ban score is init to 0 - ui->peerBanScore->setText(QString("%1").arg(stats->nodeStateStats.nMisbehavior)); - // Sync height is init to -1 if (stats->nodeStateStats.nSyncHeight > -1) ui->peerSyncHeight->setText(QString("%1").arg(stats->nodeStateStats.nSyncHeight)); @@ -1217,7 +1223,7 @@ void RPCConsole::banSelectedNode(int bantime) // Find possible nodes, ban it and clear the selected node const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(detailNodeRow); if (stats) { - m_node.ban(stats->nodeStats.addr, BanReasonManuallyAdded, bantime); + m_node.ban(stats->nodeStats.addr, bantime); m_node.disconnectByAddress(stats->nodeStats.addr); } } diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 9e23fe78d8..97fb88d71c 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -21,9 +21,9 @@ #include <chainparams.h> #include <interfaces/node.h> #include <key_io.h> +#include <node/ui_interface.h> #include <policy/fees.h> #include <txmempool.h> -#include <ui_interface.h> #include <wallet/coincontrol.h> #include <wallet/fees.h> #include <wallet/wallet.h> @@ -392,7 +392,7 @@ void SendCoinsDialog::on_sendButton_clicked() CMutableTransaction mtx = CMutableTransaction{*(m_current_transaction->getWtx())}; PartiallySignedTransaction psbtx(mtx); bool complete = false; - const TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, psbtx, complete); + const TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, psbtx, complete, nullptr); assert(!complete); assert(err == TransactionError::OK); // Serialize the PSBT diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index ced6a299d5..6e6b2b8466 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -14,7 +14,6 @@ #include <interfaces/wallet.h> #include <qt/guiutil.h> #include <qt/networkstyle.h> -#include <ui_interface.h> #include <util/system.h> #include <util/translation.h> diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 476128520c..9347ff9e42 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -59,7 +59,7 @@ void EditAddressAndSubmit( void TestAddAddressesToSendBook(interfaces::Node& node) { TestChain100Setup test; - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), WalletLocation(), CreateMockWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); bool firstRun; wallet->LoadWallet(firstRun); diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index f88d57c716..443e2d047d 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -62,9 +62,10 @@ void AppTests::appTests() } #endif - BasicTestingSetup test{CBaseChainParams::REGTEST}; // Create a temp data directory to backup the gui settings to - ECC_Stop(); // Already started by the common test setup, so stop it to avoid interference - LogInstance().DisconnectTestLogger(); + fs::create_directories([] { + BasicTestingSetup test{CBaseChainParams::REGTEST}; // Create a temp data directory to backup the gui settings to + return GetDataDir() / "blocks"; + }()); m_app.parameterSetup(); m_app.createOptionsModel(true /* reset settings */); @@ -80,6 +81,7 @@ void AppTests::appTests() m_app.exec(); // Reset global state to avoid interfering with later tests. + LogInstance().DisconnectTestLogger(); AbortShutdown(); UnloadBlockIndex(); WITH_LOCK(::cs_main, g_chainman.Reset()); diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index aefdcd2716..12efca2503 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -40,7 +40,7 @@ Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin); const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; // This is all you need to run all the tests -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { // Initialize persistent globals with the testing setup state for sanity. // E.g. -datadir in gArgs is set to a temp directory dummy value (instead @@ -70,6 +70,8 @@ int main(int argc, char *argv[]) BitcoinApplication app(*node); app.setApplicationName("Bitcoin-Qt-test"); + node->setupServerArgs(); // Make gArgs available in the NodeContext + node->context()->args->ClearArgs(); // Clear added args again AppTests app_tests(app); if (QTest::qExec(&app_tests) != 0) { fInvalid = true; diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 8da0250e57..6648029bae 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -140,7 +140,7 @@ void TestGUI(interfaces::Node& node) } node.context()->connman = std::move(test.m_node.connman); node.context()->mempool = std::move(test.m_node.mempool); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), WalletLocation(), CreateMockWalletDatabase()); bool firstRun; wallet->LoadWallet(firstRun); { @@ -178,7 +178,7 @@ void TestGUI(interfaces::Node& node) QString balanceText = balanceLabel->text(); int unit = walletModel.getOptionsModel()->getDisplayUnit(); CAmount balance = walletModel.wallet().getBalance(); - QString balanceComparison = BitcoinUnits::formatWithUnit(unit, balance, false, BitcoinUnits::separatorAlways); + QString balanceComparison = BitcoinUnits::formatWithUnit(unit, balance, false, BitcoinUnits::SeparatorStyle::ALWAYS); QCOMPARE(balanceText, balanceComparison); } @@ -204,7 +204,7 @@ void TestGUI(interfaces::Node& node) QString balanceText = balanceLabel->text().trimmed(); int unit = walletModel.getOptionsModel()->getDisplayUnit(); CAmount balance = walletModel.wallet().getBalance(); - QString balanceComparison = BitcoinUnits::formatWithUnit(unit, balance, false, BitcoinUnits::separatorAlways); + QString balanceComparison = BitcoinUnits::formatWithUnit(unit, balance, false, BitcoinUnits::SeparatorStyle::ALWAYS); QCOMPARE(balanceText, balanceComparison); // Check Request Payment button diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index 01dff8069c..632a18de5c 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -47,7 +47,6 @@ QList<TransactionRecord> TransactionRecord::decomposeTransaction(const interface if(mine) { TransactionRecord sub(hash, nTime); - CTxDestination address; sub.idx = i; // vout index sub.credit = txout.nValue; sub.involvesWatchAddress = mine & ISMINE_WATCH_ONLY; @@ -235,6 +234,7 @@ void TransactionRecord::updateStatus(const interfaces::WalletTxStatus& wtx, cons bool TransactionRecord::statusUpdateNeeded(const uint256& block_hash) const { + assert(!block_hash.IsNull()); return status.m_cur_block_hash != block_hash || status.needsUpdate; } diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 04eb1ae706..c560dc58e7 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -178,21 +178,16 @@ public: TransactionRecord* index(interfaces::Wallet& wallet, const uint256& cur_block_hash, const int idx) { - if(idx >= 0 && idx < cachedWallet.size()) - { + if (idx >= 0 && idx < cachedWallet.size()) { TransactionRecord *rec = &cachedWallet[idx]; - // Get required locks upfront. This avoids the GUI from getting - // stuck if the core is holding the locks for a longer time - for - // example, during a wallet rescan. - // // If a status update is needed (blocks came in since last check), - // update the status of this transaction from the wallet. Otherwise, - // simply re-use the cached status. + // try to update the status of this transaction from the wallet. + // Otherwise, simply re-use the cached status. interfaces::WalletTxStatus wtx; int numBlocks; int64_t block_time; - if (rec->statusUpdateNeeded(cur_block_hash) && wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time)) { + if (!cur_block_hash.IsNull() && rec->statusUpdateNeeded(cur_block_hash) && wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time)) { rec->updateStatus(wtx, cur_block_hash, numBlocks, block_time); } return rec; @@ -524,7 +519,7 @@ QVariant TransactionTableModel::data(const QModelIndex &index, int role) const case ToAddress: return formatTxToAddress(rec, false); case Amount: - return formatTxAmount(rec, true, BitcoinUnits::separatorAlways); + return formatTxAmount(rec, true, BitcoinUnits::SeparatorStyle::ALWAYS); } break; case Qt::EditRole: @@ -614,14 +609,14 @@ QVariant TransactionTableModel::data(const QModelIndex &index, int role) const details.append(QString::fromStdString(rec->address)); details.append(" "); } - details.append(formatTxAmount(rec, false, BitcoinUnits::separatorNever)); + details.append(formatTxAmount(rec, false, BitcoinUnits::SeparatorStyle::NEVER)); return details; } case ConfirmedRole: return rec->status.status == TransactionStatus::Status::Confirming || rec->status.status == TransactionStatus::Status::Confirmed; case FormattedAmountRole: // Used for copy/export, so don't include separators - return formatTxAmount(rec, false, BitcoinUnits::separatorNever); + return formatTxAmount(rec, false, BitcoinUnits::SeparatorStyle::NEVER); case StatusRole: return rec->status.status; } @@ -664,7 +659,7 @@ QVariant TransactionTableModel::headerData(int section, Qt::Orientation orientat QModelIndex TransactionTableModel::index(int row, int column, const QModelIndex &parent) const { Q_UNUSED(parent); - TransactionRecord* data = priv->index(walletModel->wallet(), walletModel->clientModel().getBestBlockHash(), row); + TransactionRecord* data = priv->index(walletModel->wallet(), walletModel->getLastBlockProcessed(), row); if(data) { return createIndex(row, column, data); diff --git a/src/qt/transactiontablemodel.h b/src/qt/transactiontablemodel.h index f06f0ea15f..4b699d4d7d 100644 --- a/src/qt/transactiontablemodel.h +++ b/src/qt/transactiontablemodel.h @@ -101,7 +101,7 @@ private: QString formatTxDate(const TransactionRecord *wtx) const; QString formatTxType(const TransactionRecord *wtx) const; QString formatTxToAddress(const TransactionRecord *wtx, bool tooltip) const; - QString formatTxAmount(const TransactionRecord *wtx, bool showUnconfirmed=true, BitcoinUnits::SeparatorStyle separators=BitcoinUnits::separatorStandard) const; + QString formatTxAmount(const TransactionRecord *wtx, bool showUnconfirmed=true, BitcoinUnits::SeparatorStyle separators=BitcoinUnits::SeparatorStyle::STANDARD) const; QString formatTooltip(const TransactionRecord *rec) const; QVariant txStatusDecoration(const TransactionRecord *wtx) const; QVariant txWatchonlyDecoration(const TransactionRecord *wtx) const; diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 3df81807f0..54ecfc38ec 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -17,7 +17,7 @@ #include <qt/transactiontablemodel.h> #include <qt/walletmodel.h> -#include <ui_interface.h> +#include <node/ui_interface.h> #include <QApplication> #include <QComboBox> diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index f1438e3439..3aed98e0e8 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -262,7 +262,7 @@ void CreateWalletActivity::finish() { destroyProgressDialog(); - if (!m_error_message.original.empty()) { + if (!m_error_message.empty()) { QMessageBox::critical(m_parent_widget, tr("Create wallet failed"), QString::fromStdString(m_error_message.translated)); } else if (!m_warning_message.empty()) { QMessageBox::warning(m_parent_widget, tr("Create wallet warning"), QString::fromStdString(Join(m_warning_message, Untranslated("\n")).translated)); @@ -303,7 +303,7 @@ void OpenWalletActivity::finish() { destroyProgressDialog(); - if (!m_error_message.original.empty()) { + if (!m_error_message.empty()) { QMessageBox::critical(m_parent_widget, tr("Open wallet failed"), QString::fromStdString(m_error_message.translated)); } else if (!m_warning_message.empty()) { QMessageBox::warning(m_parent_widget, tr("Open wallet warning"), QString::fromStdString(Join(m_warning_message, Untranslated("\n")).translated)); diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 5e68ee4f93..ec56f2755f 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -165,11 +165,11 @@ void WalletFrame::gotoVerifyMessageTab(QString addr) walletView->gotoVerifyMessageTab(addr); } -void WalletFrame::gotoLoadPSBT() +void WalletFrame::gotoLoadPSBT(bool from_clipboard) { WalletView *walletView = currentWalletView(); if (walletView) { - walletView->gotoLoadPSBT(); + walletView->gotoLoadPSBT(from_clipboard); } } diff --git a/src/qt/walletframe.h b/src/qt/walletframe.h index d90ade5005..2b5f263468 100644 --- a/src/qt/walletframe.h +++ b/src/qt/walletframe.h @@ -79,7 +79,7 @@ public Q_SLOTS: void gotoVerifyMessageTab(QString addr = ""); /** Load Partially Signed Bitcoin Transaction */ - void gotoLoadPSBT(); + void gotoLoadPSBT(bool from_clipboard = false); /** Encrypt the wallet */ void encryptWallet(bool status); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 671b5e1ce6..e374dd191c 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -21,8 +21,8 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <key_io.h> +#include <node/ui_interface.h> #include <psbt.h> -#include <ui_interface.h> #include <util/system.h> // for GetBoolArg #include <util/translation.h> #include <wallet/coincontrol.h> @@ -87,7 +87,7 @@ void WalletModel::pollBalanceChanged() { // Avoid recomputing wallet balances unless a TransactionChanged or // BlockTip notification was received. - if (!fForceCheckBalanceChanged && m_cached_last_update_tip == m_client_model->getBestBlockHash()) return; + if (!fForceCheckBalanceChanged && m_cached_last_update_tip == getLastBlockProcessed()) return; // Try to get balances and return early if locks can't be acquired. This // avoids the GUI from getting stuck on periodical polls if the core is @@ -536,7 +536,7 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) if (create_psbt) { PartiallySignedTransaction psbtx(mtx); bool complete = false; - const TransactionError err = wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, psbtx, complete); + const TransactionError err = wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, psbtx, complete, nullptr); if (err != TransactionError::OK || complete) { QMessageBox::critical(nullptr, tr("Fee bump error"), tr("Can't draft transaction.")); return false; @@ -588,3 +588,8 @@ void WalletModel::refresh(bool pk_hash_only) { addressTableModel = new AddressTableModel(this, pk_hash_only); } + +uint256 WalletModel::getLastBlockProcessed() const +{ + return m_client_model ? m_client_model->getBestBlockHash() : uint256{}; +} diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 38e8a14556..fd52db2da3 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -155,6 +155,9 @@ public: AddressTableModel* getAddressTableModel() const { return addressTableModel; } void refresh(bool pk_hash_only = false); + + uint256 getLastBlockProcessed() const; + private: std::unique_ptr<interfaces::Wallet> m_wallet; std::unique_ptr<interfaces::Handler> m_handler_unload; diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index 861d1c5f4a..2fc883a5f5 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -4,13 +4,11 @@ #include <qt/walletview.h> -#include <node/psbt.h> -#include <node/transaction.h> -#include <policy/policy.h> #include <qt/addressbookpage.h> #include <qt/askpassphrasedialog.h> #include <qt/clientmodel.h> #include <qt/guiutil.h> +#include <qt/psbtoperationsdialog.h> #include <qt/optionsmodel.h> #include <qt/overviewpage.h> #include <qt/platformstyle.h> @@ -22,11 +20,14 @@ #include <qt/walletmodel.h> #include <interfaces/node.h> -#include <ui_interface.h> +#include <node/ui_interface.h> +#include <psbt.h> #include <util/strencodings.h> #include <QAction> #include <QActionGroup> +#include <QApplication> +#include <QClipboard> #include <QFileDialog> #include <QHBoxLayout> #include <QProgressDialog> @@ -204,78 +205,42 @@ void WalletView::gotoVerifyMessageTab(QString addr) signVerifyMessageDialog->setAddress_VM(addr); } -void WalletView::gotoLoadPSBT() +void WalletView::gotoLoadPSBT(bool from_clipboard) { - QString filename = GUIUtil::getOpenFileName(this, - tr("Load Transaction Data"), QString(), - tr("Partially Signed Transaction (*.psbt)"), nullptr); - if (filename.isEmpty()) return; - if (GetFileSize(filename.toLocal8Bit().data(), MAX_FILE_SIZE_PSBT) == MAX_FILE_SIZE_PSBT) { - Q_EMIT message(tr("Error"), tr("PSBT file must be smaller than 100 MiB"), CClientUIInterface::MSG_ERROR); - return; + std::string data; + + if (from_clipboard) { + std::string raw = QApplication::clipboard()->text().toStdString(); + bool invalid; + data = DecodeBase64(raw, &invalid); + if (invalid) { + Q_EMIT message(tr("Error"), tr("Unable to decode PSBT from clipboard (invalid base64)"), CClientUIInterface::MSG_ERROR); + return; + } + } else { + QString filename = GUIUtil::getOpenFileName(this, + tr("Load Transaction Data"), QString(), + tr("Partially Signed Transaction (*.psbt)"), nullptr); + if (filename.isEmpty()) return; + if (GetFileSize(filename.toLocal8Bit().data(), MAX_FILE_SIZE_PSBT) == MAX_FILE_SIZE_PSBT) { + Q_EMIT message(tr("Error"), tr("PSBT file must be smaller than 100 MiB"), CClientUIInterface::MSG_ERROR); + return; + } + std::ifstream in(filename.toLocal8Bit().data(), std::ios::binary); + data = std::string(std::istreambuf_iterator<char>{in}, {}); } - std::ifstream in(filename.toLocal8Bit().data(), std::ios::binary); - std::string data(std::istreambuf_iterator<char>{in}, {}); std::string error; PartiallySignedTransaction psbtx; if (!DecodeRawPSBT(psbtx, data, error)) { - Q_EMIT message(tr("Error"), tr("Unable to decode PSBT file") + "\n" + QString::fromStdString(error), CClientUIInterface::MSG_ERROR); + Q_EMIT message(tr("Error"), tr("Unable to decode PSBT") + "\n" + QString::fromStdString(error), CClientUIInterface::MSG_ERROR); return; } - CMutableTransaction mtx; - bool complete = false; - PSBTAnalysis analysis = AnalyzePSBT(psbtx); - QMessageBox msgBox; - msgBox.setText("PSBT"); - switch (analysis.next) { - case PSBTRole::CREATOR: - case PSBTRole::UPDATER: - msgBox.setInformativeText("PSBT is incomplete. Copy to clipboard for manual inspection?"); - break; - case PSBTRole::SIGNER: - msgBox.setInformativeText("Transaction needs more signatures. Copy to clipboard?"); - break; - case PSBTRole::FINALIZER: - case PSBTRole::EXTRACTOR: - complete = FinalizeAndExtractPSBT(psbtx, mtx); - if (complete) { - msgBox.setInformativeText(tr("Would you like to send this transaction?")); - } else { - // The analyzer missed something, e.g. if there are final_scriptSig/final_scriptWitness - // but with invalid signatures. - msgBox.setInformativeText(tr("There was an unexpected problem processing the PSBT. Copy to clipboard for manual inspection?")); - } - } - - msgBox.setStandardButtons(QMessageBox::Yes | QMessageBox::Cancel); - switch (msgBox.exec()) { - case QMessageBox::Yes: { - if (complete) { - std::string err_string; - CTransactionRef tx = MakeTransactionRef(mtx); - - TransactionError result = BroadcastTransaction(*clientModel->node().context(), tx, err_string, DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK(), /* relay */ true, /* wait_callback */ false); - if (result == TransactionError::OK) { - Q_EMIT message(tr("Success"), tr("Broadcasted transaction successfully."), CClientUIInterface::MSG_INFORMATION | CClientUIInterface::MODAL); - } else { - Q_EMIT message(tr("Error"), QString::fromStdString(err_string), CClientUIInterface::MSG_ERROR); - } - } else { - // Serialize the PSBT - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << psbtx; - GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); - Q_EMIT message(tr("PSBT copied"), "Copied to clipboard", CClientUIInterface::MSG_INFORMATION); - return; - } - } - case QMessageBox::Cancel: - break; - default: - assert(false); - } + PSBTOperationsDialog* dlg = new PSBTOperationsDialog(this, walletModel, clientModel); + dlg->openWithPSBT(psbtx); + dlg->setAttribute(Qt::WA_DeleteOnClose); + dlg->exec(); } bool WalletView::handlePaymentRequest(const SendCoinsRecipient& recipient) diff --git a/src/qt/walletview.h b/src/qt/walletview.h index fd09456baa..f186554758 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -84,7 +84,7 @@ public Q_SLOTS: /** Show Sign/Verify Message dialog and switch to verify message tab */ void gotoVerifyMessageTab(QString addr = ""); /** Load Partially Signed Bitcoin Transaction */ - void gotoLoadPSBT(); + void gotoLoadPSBT(bool from_clipboard = false); /** Show incoming transaction notification for new transactions. diff --git a/src/rest.cpp b/src/rest.cpp index cde8b472d3..7130625d5c 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -68,13 +68,32 @@ static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string me } /** - * Get the node context mempool. + * Get the node context. * - * Set the HTTP error and return nullptr if node context - * mempool is not found. + * @param[in] req The HTTP request, whose status code will be set if node + * context is not found. + * @returns Pointer to the node context or nullptr if not found. + */ +static NodeContext* GetNodeContext(const util::Ref& context, HTTPRequest* req) +{ + NodeContext* node = context.Has<NodeContext>() ? &context.Get<NodeContext>() : nullptr; + if (!node) { + RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, + strprintf("%s:%d (%s)\n" + "Internal bug detected: Node context not found!\n" + "You may report this issue here: %s\n", + __FILE__, __LINE__, __func__, PACKAGE_BUGREPORT)); + return nullptr; + } + return node; +} + +/** + * Get the node context mempool. * - * @param[in] req the HTTP request - * return pointer to the mempool or nullptr if no mempool found + * @param[in] req The HTTP request, whose status code will be set if node + * context mempool is not found. + * @returns Pointer to the mempool or nullptr if no mempool found. */ static CTxMemPool* GetMemPool(const util::Ref& context, HTTPRequest* req) { @@ -188,7 +207,7 @@ static bool rest_headers(const util::Ref& context, ssHeader << pindex->GetBlockHeader(); } - std::string strHex = HexStr(ssHeader.begin(), ssHeader.end()) + "\n"; + std::string strHex = HexStr(ssHeader) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; @@ -253,7 +272,7 @@ static bool rest_block(HTTPRequest* req, case RetFormat::HEX: { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; - std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()) + "\n"; + std::string strHex = HexStr(ssBlock) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; @@ -371,10 +390,13 @@ static bool rest_tx(const util::Ref& context, HTTPRequest* req, const std::strin g_txindex->BlockUntilSyncedToCurrentChain(); } - CTransactionRef tx; + const NodeContext* const node = GetNodeContext(context, req); + if (!node) return false; uint256 hashBlock = uint256(); - if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock)) + const CTransactionRef tx = GetTransaction(/* block_index */ nullptr, node->mempool, hash, Params().GetConsensus(), hashBlock); + if (!tx) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); + } switch (rf) { case RetFormat::BINARY: { @@ -391,7 +413,7 @@ static bool rest_tx(const util::Ref& context, HTTPRequest* req, const std::strin CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssTx << tx; - std::string strHex = HexStr(ssTx.begin(), ssTx.end()) + "\n"; + std::string strHex = HexStr(ssTx) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; @@ -556,7 +578,7 @@ static bool rest_getutxos(const util::Ref& context, HTTPRequest* req, const std: case RetFormat::HEX: { CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); ssGetUTXOResponse << ::ChainActive().Height() << ::ChainActive().Tip()->GetBlockHash() << bitmap << outs; - std::string strHex = HexStr(ssGetUTXOResponse.begin(), ssGetUTXOResponse.end()) + "\n"; + std::string strHex = HexStr(ssGetUTXOResponse) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 8252af3ee6..2afc9a3d4a 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -75,7 +75,10 @@ CTxMemPool& EnsureMemPool(const util::Ref& context) ChainstateManager& EnsureChainman(const util::Ref& context) { NodeContext& node = EnsureNodeContext(context); - return EnsureChainman(node); + if (!node.chainman) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Node chainman not found"); + } + return *node.chainman; } /* Calculate the difficulty for a given block index. @@ -779,7 +782,7 @@ static UniValue getblockheader(const JSONRPCRequest& request) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); ssBlock << pblockindex->GetBlockHeader(); - std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); + std::string strHex = HexStr(ssBlock); return strHex; } @@ -795,10 +798,8 @@ static CBlock GetBlockChecked(const CBlockIndex* pblockindex) if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { // Block not found on disk. This could be because we have the block - // header in our index but don't have the block (for example if a - // non-whitelisted node sends us an unrequested long chain of valid - // blocks, we add the headers to our index, but don't accept the - // block). + // header in our index but not yet have the block or did not accept the + // block. throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); } @@ -905,7 +906,7 @@ static UniValue getblock(const JSONRPCRequest& request) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; - std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); + std::string strHex = HexStr(ssBlock); return strHex; } @@ -972,7 +973,9 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) RPCHelpMan{"gettxoutsetinfo", "\nReturns statistics about the unspent transaction output set.\n" "Note this call may take some time.\n", - {}, + { + {"hash_type", RPCArg::Type::STR, /* default */ "hash_serialized_2", "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'none'."}, + }, RPCResult{ RPCResult::Type::OBJ, "", "", { @@ -981,7 +984,7 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs"}, {RPCResult::Type::NUM, "txouts", "The number of unspent transaction outputs"}, {RPCResult::Type::NUM, "bogosize", "A meaningless metric for UTXO set size"}, - {RPCResult::Type::STR_HEX, "hash_serialized_2", "The serialized hash"}, + {RPCResult::Type::STR_HEX, "hash_serialized_2", "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"}, {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk"}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount"}, }}, @@ -996,14 +999,19 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) CCoinsStats stats; ::ChainstateActive().ForceFlushStateToDisk(); + const CoinStatsHashType hash_type = ParseHashType(request.params[0], CoinStatsHashType::HASH_SERIALIZED); + CCoinsView* coins_view = WITH_LOCK(cs_main, return &ChainstateActive().CoinsDB()); - if (GetUTXOStats(coins_view, stats, RpcInterruptionPoint)) { + NodeContext& node = EnsureNodeContext(request.context); + if (GetUTXOStats(coins_view, stats, hash_type, node.rpc_interruption_point)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); ret.pushKV("transactions", (int64_t)stats.nTransactions); ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs); ret.pushKV("bogosize", (int64_t)stats.nBogoSize); - ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); + if (hash_type == CoinStatsHashType::HASH_SERIALIZED) { + ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); + } ret.pushKV("disk_size", stats.nDiskSize); ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { @@ -1325,50 +1333,48 @@ static UniValue getchaintips(const JSONRPCRequest& request) }, }.Check(request); + ChainstateManager& chainman = EnsureChainman(request.context); LOCK(cs_main); /* - * Idea: the set of chain tips is ::ChainActive().tip, plus orphan blocks which do not have another orphan building off of them. + * Idea: The set of chain tips is the active chain tip, plus orphan blocks which do not have another orphan building off of them. * Algorithm: * - Make one pass through BlockIndex(), picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. * - Iterate through the orphan blocks. If the block isn't pointed to by another orphan, it is a chain tip. - * - add ::ChainActive().Tip() + * - Add the active chain tip */ std::set<const CBlockIndex*, CompareBlocksByHeight> setTips; std::set<const CBlockIndex*> setOrphans; std::set<const CBlockIndex*> setPrevs; - for (const std::pair<const uint256, CBlockIndex*>& item : ::BlockIndex()) - { - if (!::ChainActive().Contains(item.second)) { + for (const std::pair<const uint256, CBlockIndex*>& item : chainman.BlockIndex()) { + if (!chainman.ActiveChain().Contains(item.second)) { setOrphans.insert(item.second); setPrevs.insert(item.second->pprev); } } - for (std::set<const CBlockIndex*>::iterator it = setOrphans.begin(); it != setOrphans.end(); ++it) - { + for (std::set<const CBlockIndex*>::iterator it = setOrphans.begin(); it != setOrphans.end(); ++it) { if (setPrevs.erase(*it) == 0) { setTips.insert(*it); } } // Always report the currently active tip. - setTips.insert(::ChainActive().Tip()); + setTips.insert(chainman.ActiveChain().Tip()); /* Construct the output array. */ UniValue res(UniValue::VARR); - for (const CBlockIndex* block : setTips) - { + for (const CBlockIndex* block : setTips) { UniValue obj(UniValue::VOBJ); obj.pushKV("height", block->nHeight); obj.pushKV("hash", block->phashBlock->GetHex()); - const int branchLen = block->nHeight - ::ChainActive().FindFork(block)->nHeight; + const int branchLen = block->nHeight - chainman.ActiveChain().FindFork(block)->nHeight; obj.pushKV("branchlen", branchLen); std::string status; - if (::ChainActive().Contains(block)) { + if (chainman.ActiveChain().Contains(block)) { // This block is part of the currently active chain. status = "active"; } else if (block->nStatus & BLOCK_FAILED_MASK) { @@ -1967,8 +1973,10 @@ static UniValue savemempool(const JSONRPCRequest& request) return NullUniValue; } +namespace { //! Search for a given set of pubkey scripts -bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& should_abort, int64_t& count, CCoinsViewCursor* cursor, const std::set<CScript>& needles, std::map<COutPoint, Coin>& out_results) { +bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& should_abort, int64_t& count, CCoinsViewCursor* cursor, const std::set<CScript>& needles, std::map<COutPoint, Coin>& out_results, std::function<void()>& interruption_point) +{ scan_progress = 0; count = 0; while (cursor->Valid()) { @@ -1976,7 +1984,7 @@ bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& Coin coin; if (!cursor->GetKey(key) || !cursor->GetValue(coin)) return false; if (++count % 8192 == 0) { - RpcInterruptionPoint(); + interruption_point(); if (should_abort) { // allow to abort the scan via the abort reference return false; @@ -1995,6 +2003,7 @@ bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& scan_progress = 100; return true; } +} // namespace /** RAII object to prevent concurrency issue when scanning the txout set */ static std::atomic<int> g_scan_progress; @@ -2143,7 +2152,8 @@ UniValue scantxoutset(const JSONRPCRequest& request) tip = ::ChainActive().Tip(); CHECK_NONFATAL(tip); } - bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins); + NodeContext& node = EnsureNodeContext(request.context); + bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins, node.rpc_interruption_point); result.pushKV("success", res); result.pushKV("txouts", count); result.pushKV("height", tip->nHeight); @@ -2159,7 +2169,7 @@ UniValue scantxoutset(const JSONRPCRequest& request) UniValue unspent(UniValue::VOBJ); unspent.pushKV("txid", outpoint.hash.GetHex()); unspent.pushKV("vout", (int32_t)outpoint.n); - unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey.begin(), txo.scriptPubKey.end())); + unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey)); unspent.pushKV("desc", descriptors[txo.scriptPubKey]); unspent.pushKV("amount", ValueFromAmount(txo.nValue)); unspent.pushKV("height", (int32_t)coin.nHeight); @@ -2298,6 +2308,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) std::unique_ptr<CCoinsViewCursor> pcursor; CCoinsStats stats; CBlockIndex* tip; + NodeContext& node = EnsureNodeContext(request.context); { // We need to lock cs_main to ensure that the coinsdb isn't written to @@ -2316,7 +2327,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) ::ChainstateActive().ForceFlushStateToDisk(); - if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats, RpcInterruptionPoint)) { + if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats, CoinStatsHashType::NONE, node.rpc_interruption_point)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } @@ -2334,7 +2345,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) unsigned int iter{0}; while (pcursor->Valid()) { - if (iter % 5000 == 0) RpcInterruptionPoint(); + if (iter % 5000 == 0) node.rpc_interruption_point(); ++iter; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { afile << key; @@ -2377,7 +2388,7 @@ static const CRPCCommand commands[] = { "blockchain", "getmempoolinfo", &getmempoolinfo, {} }, { "blockchain", "getrawmempool", &getrawmempool, {"verbose"} }, { "blockchain", "gettxout", &gettxout, {"txid","n","include_mempool"} }, - { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, {} }, + { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, {"hash_type"} }, { "blockchain", "pruneblockchain", &pruneblockchain, {"height"} }, { "blockchain", "savemempool", &savemempool, {} }, { "blockchain", "verifychain", &verifychain, {"checklevel","nblocks"} }, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 3045a74d7a..66ace7263a 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -217,7 +217,7 @@ UniValue ParseNonRFCJSONValue(const std::string& strVal) UniValue jVal; if (!jVal.read(std::string("[")+strVal+std::string("]")) || !jVal.isArray() || jVal.size()!=1) - throw std::runtime_error(std::string("Error parsing JSON:")+strVal); + throw std::runtime_error(std::string("Error parsing JSON: ") + strVal); return jVal[0]; } diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index a2de175c85..fee6a893eb 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -17,6 +17,7 @@ #include <policy/fees.h> #include <pow.h> #include <rpc/blockchain.h> +#include <rpc/mining.h> #include <rpc/server.h> #include <rpc/util.h> #include <script/descriptor.h> @@ -207,7 +208,7 @@ static UniValue generatetodescriptor(const JSONRPCRequest& request) { {"num_blocks", RPCArg::Type::NUM, RPCArg::Optional::NO, "How many blocks are generated immediately."}, {"descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The descriptor to send the newly generated bitcoin to."}, - {"maxtries", RPCArg::Type::NUM, /* default */ "1000000", "How many iterations to try."}, + {"maxtries", RPCArg::Type::NUM, /* default */ ToString(DEFAULT_MAX_TRIES), "How many iterations to try."}, }, RPCResult{ RPCResult::Type::ARR, "", "hashes of blocks generated", @@ -221,7 +222,7 @@ static UniValue generatetodescriptor(const JSONRPCRequest& request) .Check(request); const int num_blocks{request.params[0].get_int()}; - const int64_t max_tries{request.params[2].isNull() ? 1000000 : request.params[2].get_int()}; + const uint64_t max_tries{request.params[2].isNull() ? DEFAULT_MAX_TRIES : request.params[2].get_int()}; CScript coinbase_script; std::string error; @@ -242,7 +243,7 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) { {"nblocks", RPCArg::Type::NUM, RPCArg::Optional::NO, "How many blocks are generated immediately."}, {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The address to send the newly generated bitcoin to."}, - {"maxtries", RPCArg::Type::NUM, /* default */ "1000000", "How many iterations to try."}, + {"maxtries", RPCArg::Type::NUM, /* default */ ToString(DEFAULT_MAX_TRIES), "How many iterations to try."}, }, RPCResult{ RPCResult::Type::ARR, "", "hashes of blocks generated", @@ -252,16 +253,13 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) RPCExamples{ "\nGenerate 11 blocks to myaddress\n" + HelpExampleCli("generatetoaddress", "11 \"myaddress\"") - + "If you are running the bitcoin core wallet, you can get a new address to send the newly generated bitcoin to with:\n" + + "If you are using the " PACKAGE_NAME " wallet, you can get a new address to send the newly generated bitcoin to with:\n" + HelpExampleCli("getnewaddress", "") }, }.Check(request); - int nGenerate = request.params[0].get_int(); - uint64_t nMaxTries = 1000000; - if (!request.params[2].isNull()) { - nMaxTries = request.params[2].get_int(); - } + const int num_blocks{request.params[0].get_int()}; + const uint64_t max_tries{request.params[2].isNull() ? DEFAULT_MAX_TRIES : request.params[2].get_int()}; CTxDestination destination = DecodeDestination(request.params[1].get_str()); if (!IsValidDestination(destination)) { @@ -273,7 +271,7 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) CScript coinbase_script = GetScriptForDestination(destination); - return generateBlocks(chainman, mempool, coinbase_script, nGenerate, nMaxTries); + return generateBlocks(chainman, mempool, coinbase_script, num_blocks, max_tries); } static UniValue generateblock(const JSONRPCRequest& request) @@ -371,7 +369,7 @@ static UniValue generateblock(const JSONRPCRequest& request) } uint256 block_hash; - uint64_t max_tries{1000000}; + uint64_t max_tries{DEFAULT_MAX_TRIES}; unsigned int extra_nonce{0}; if (!GenerateBlock(EnsureChainman(request.context), block, max_tries, extra_nonce, block_hash) || block_hash.IsNull()) { @@ -875,7 +873,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) result.pushKV("height", (int64_t)(pindexPrev->nHeight+1)); if (!pblocktemplate->vchCoinbaseCommitment.empty()) { - result.pushKV("default_witness_commitment", HexStr(pblocktemplate->vchCoinbaseCommitment.begin(), pblocktemplate->vchCoinbaseCommitment.end())); + result.pushKV("default_witness_commitment", HexStr(pblocktemplate->vchCoinbaseCommitment)); } return result; diff --git a/src/rpc/mining.h b/src/rpc/mining.h new file mode 100644 index 0000000000..acc74e1dcc --- /dev/null +++ b/src/rpc/mining.h @@ -0,0 +1,11 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RPC_MINING_H +#define BITCOIN_RPC_MINING_H + +/** Default max iterations to try in RPC generatetodescriptor, generatetoaddress, and generateblock. */ +static const uint64_t DEFAULT_MAX_TRIES{1000000}; + +#endif // BITCOIN_RPC_MINING_H diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index ce98a7c937..53d38f4e11 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -63,7 +63,7 @@ static UniValue validateaddress(const JSONRPCRequest& request) ret.pushKV("address", currentAddress); CScript scriptPubKey = GetScriptForDestination(dest); - ret.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); + ret.pushKV("scriptPubKey", HexStr(scriptPubKey)); UniValue detail = DescribeAddress(dest); ret.pushKVs(detail); @@ -131,7 +131,7 @@ static UniValue createmultisig(const JSONRPCRequest& request) UniValue result(UniValue::VOBJ); result.pushKV("address", EncodeDestination(dest)); - result.pushKV("redeemScript", HexStr(inner.begin(), inner.end())); + result.pushKV("redeemScript", HexStr(inner)); result.pushKV("descriptor", descriptor->ToString()); return result; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index df1e0fe623..9981ea35df 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -112,7 +112,7 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) {RPCResult::Type::BOOL, "inbound", "Inbound (true) or Outbound (false)"}, {RPCResult::Type::BOOL, "addnode", "Whether connection was due to addnode/-connect or if it was an automatic/inbound connection"}, {RPCResult::Type::NUM, "startingheight", "The starting height (block) of the peer"}, - {RPCResult::Type::NUM, "banscore", "The ban score"}, + {RPCResult::Type::NUM, "banscore", "The ban score (DEPRECATED, returned only if config option -deprecatedrpc=banscore is passed)"}, {RPCResult::Type::NUM, "synced_headers", "The last header we have in common with this peer"}, {RPCResult::Type::NUM, "synced_blocks", "The last block we have in common with this peer"}, {RPCResult::Type::ARR, "inflight", "", @@ -191,7 +191,10 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) obj.pushKV("addnode", stats.m_manual_connection); obj.pushKV("startingheight", stats.nStartingHeight); if (fStateStats) { - obj.pushKV("banscore", statestats.nMisbehavior); + if (IsDeprecatedRPCEnabled("banscore")) { + // banscore is deprecated in v0.21 for removal in v0.22 + obj.pushKV("banscore", statestats.nMisbehavior); + } obj.pushKV("synced_headers", statestats.nSyncHeight); obj.pushKV("synced_blocks", statestats.nCommonHeight); UniValue heights(UniValue::VARR); @@ -614,12 +617,12 @@ static UniValue setban(const JSONRPCRequest& request) absolute = true; if (isSubnet) { - node.banman->Ban(subNet, BanReasonManuallyAdded, banTime, absolute); + node.banman->Ban(subNet, banTime, absolute); if (node.connman) { node.connman->DisconnectNode(subNet); } } else { - node.banman->Ban(netAddr, BanReasonManuallyAdded, banTime, absolute); + node.banman->Ban(netAddr, banTime, absolute); if (node.connman) { node.connman->DisconnectNode(netAddr); } @@ -628,7 +631,7 @@ static UniValue setban(const JSONRPCRequest& request) else if(strCommand == "remove") { if (!( isSubnet ? node.banman->Unban(subNet) : node.banman->Unban(netAddr) )) { - throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Unban failed. Requested address/subnet was not previously banned."); + throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Unban failed. Requested address/subnet was not previously manually banned."); } } return NullUniValue; @@ -637,7 +640,7 @@ static UniValue setban(const JSONRPCRequest& request) static UniValue listbanned(const JSONRPCRequest& request) { RPCHelpMan{"listbanned", - "\nList all banned IPs/Subnets.\n", + "\nList all manually banned IPs/Subnets.\n", {}, RPCResult{RPCResult::Type::ARR, "", "", { @@ -646,7 +649,6 @@ static UniValue listbanned(const JSONRPCRequest& request) {RPCResult::Type::STR, "address", ""}, {RPCResult::Type::NUM_TIME, "banned_until", ""}, {RPCResult::Type::NUM_TIME, "ban_created", ""}, - {RPCResult::Type::STR, "ban_reason", ""}, }}, }}, RPCExamples{ @@ -671,7 +673,6 @@ static UniValue listbanned(const JSONRPCRequest& request) rec.pushKV("address", entry.first.ToString()); rec.pushKV("banned_until", banEntry.nBanUntil); rec.pushKV("ban_created", banEntry.nCreateTime); - rec.pushKV("ban_reason", banEntry.banReasonToString()); bannedAddresses.push_back(rec); } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index e14217c307..70caf6009a 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -28,6 +28,7 @@ #include <script/signingprovider.h> #include <script/standard.h> #include <uint256.h> +#include <util/bip32.h> #include <util/moneystr.h> #include <util/strencodings.h> #include <util/string.h> @@ -156,6 +157,8 @@ static UniValue getrawtransaction(const JSONRPCRequest& request) }, }.Check(request); + const NodeContext& node = EnsureNodeContext(request.context); + bool in_active_chain = true; uint256 hash = ParseHashV(request.params[0], "parameter 1"); CBlockIndex* blockindex = nullptr; @@ -187,9 +190,9 @@ static UniValue getrawtransaction(const JSONRPCRequest& request) f_txindex_ready = g_txindex->BlockUntilSyncedToCurrentChain(); } - CTransactionRef tx; uint256 hash_block; - if (!GetTransaction(hash, tx, Params().GetConsensus(), hash_block, blockindex)) { + const CTransactionRef tx = GetTransaction(blockindex, node.mempool, hash, Params().GetConsensus(), hash_block); + if (!tx) { std::string errmsg; if (blockindex) { if (!(blockindex->nStatus & BLOCK_HAVE_DATA)) { @@ -244,10 +247,11 @@ static UniValue gettxoutproof(const JSONRPCRequest& request) for (unsigned int idx = 0; idx < txids.size(); idx++) { const UniValue& txid = txids[idx]; uint256 hash(ParseHashV(txid, "txid")); - if (setTxids.count(hash)) - throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated txid: ")+txid.get_str()); - setTxids.insert(hash); - oneTxid = hash; + if (setTxids.count(hash)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated txid: ") + txid.get_str()); + } + setTxids.insert(hash); + oneTxid = hash; } CBlockIndex* pblockindex = nullptr; @@ -280,11 +284,11 @@ static UniValue gettxoutproof(const JSONRPCRequest& request) LOCK(cs_main); - if (pblockindex == nullptr) - { - CTransactionRef tx; - if (!GetTransaction(oneTxid, tx, Params().GetConsensus(), hashBlock) || hashBlock.IsNull()) + if (pblockindex == nullptr) { + const CTransactionRef tx = GetTransaction(/* block_index */ nullptr, /* mempool */ nullptr, oneTxid, Params().GetConsensus(), hashBlock); + if (!tx || hashBlock.IsNull()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); + } pblockindex = LookupBlockIndex(hashBlock); if (!pblockindex) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Transaction index corrupt"); @@ -292,20 +296,24 @@ static UniValue gettxoutproof(const JSONRPCRequest& request) } CBlock block; - if(!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) + if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); + } unsigned int ntxFound = 0; - for (const auto& tx : block.vtx) - if (setTxids.count(tx->GetHash())) + for (const auto& tx : block.vtx) { + if (setTxids.count(tx->GetHash())) { ntxFound++; - if (ntxFound != setTxids.size()) + } + } + if (ntxFound != setTxids.size()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Not all transactions found in specified or retrieved block"); + } CDataStream ssMB(SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); CMerkleBlock mb(block, setTxids); ssMB << mb; - std::string strHex = HexStr(ssMB.begin(), ssMB.end()); + std::string strHex = HexStr(ssMB); return strHex; } @@ -511,12 +519,12 @@ static UniValue decoderawtransaction(const JSONRPCRequest& request) static std::string GetAllOutputTypes() { - std::string ret; - for (int i = TX_NONSTANDARD; i <= TX_WITNESS_UNKNOWN; ++i) { - if (i != TX_NONSTANDARD) ret += ", "; - ret += GetTxnOutputType(static_cast<txnouttype>(i)); + std::vector<std::string> ret; + using U = std::underlying_type<TxoutType>::type; + for (U i = (U)TxoutType::NONSTANDARD; i <= (U)TxoutType::WITNESS_UNKNOWN; ++i) { + ret.emplace_back(GetTxnOutputType(static_cast<TxoutType>(i))); } - return ret; + return Join(ret, ", "); } static UniValue decodescript(const JSONRPCRequest& request) @@ -580,10 +588,10 @@ static UniValue decodescript(const JSONRPCRequest& request) // is a witness program, don't return addresses for a segwit programs. if (type.get_str() == "pubkey" || type.get_str() == "pubkeyhash" || type.get_str() == "multisig" || type.get_str() == "nonstandard") { std::vector<std::vector<unsigned char>> solutions_data; - txnouttype which_type = Solver(script, solutions_data); + TxoutType which_type = Solver(script, solutions_data); // Uncompressed pubkeys cannot be used with segwit checksigs. // If the script contains an uncompressed pubkey, skip encoding of a segwit program. - if ((which_type == TX_PUBKEY) || (which_type == TX_MULTISIG)) { + if ((which_type == TxoutType::PUBKEY) || (which_type == TxoutType::MULTISIG)) { for (const auto& solution : solutions_data) { if ((solution.size() != 1) && !CPubKey(solution).IsCompressed()) { return r; @@ -592,10 +600,10 @@ static UniValue decodescript(const JSONRPCRequest& request) } UniValue sr(UniValue::VOBJ); CScript segwitScr; - if (which_type == TX_PUBKEY) { + if (which_type == TxoutType::PUBKEY) { segwitScr = GetScriptForDestination(WitnessV0KeyHash(Hash160(solutions_data[0].begin(), solutions_data[0].end()))); - } else if (which_type == TX_PUBKEYHASH) { - segwitScr = GetScriptForDestination(WitnessV0KeyHash(solutions_data[0])); + } else if (which_type == TxoutType::PUBKEYHASH) { + segwitScr = GetScriptForDestination(WitnessV0KeyHash(uint160{solutions_data[0]})); } else { // Scripts that are not fit for P2WPKH are encoded as P2WSH. // Newer segwit program versions should be considered when then become available. @@ -938,25 +946,6 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) return result; } -static std::string WriteHDKeypath(std::vector<uint32_t>& keypath) -{ - std::string keypath_str = "m"; - for (uint32_t num : keypath) { - keypath_str += "/"; - bool hardened = false; - if (num & 0x80000000) { - hardened = true; - num &= ~0x80000000; - } - - keypath_str += ToString(num); - if (hardened) { - keypath_str += "'"; - } - } - return keypath_str; -} - UniValue decodepsbt(const JSONRPCRequest& request) { RPCHelpMan{"decodepsbt", @@ -1104,30 +1093,34 @@ UniValue decodepsbt(const JSONRPCRequest& request) const PSBTInput& input = psbtx.inputs[i]; UniValue in(UniValue::VOBJ); // UTXOs + bool have_a_utxo = false; + CTxOut txout; if (!input.witness_utxo.IsNull()) { - const CTxOut& txout = input.witness_utxo; - - UniValue out(UniValue::VOBJ); - - out.pushKV("amount", ValueFromAmount(txout.nValue)); - if (MoneyRange(txout.nValue) && MoneyRange(total_in + txout.nValue)) { - total_in += txout.nValue; - } else { - // Hack to just not show fee later - have_all_utxos = false; - } + txout = input.witness_utxo; UniValue o(UniValue::VOBJ); ScriptToUniv(txout.scriptPubKey, o, true); + + UniValue out(UniValue::VOBJ); + out.pushKV("amount", ValueFromAmount(txout.nValue)); out.pushKV("scriptPubKey", o); + in.pushKV("witness_utxo", out); - } else if (input.non_witness_utxo) { + + have_a_utxo = true; + } + if (input.non_witness_utxo) { + txout = input.non_witness_utxo->vout[psbtx.tx->vin[i].prevout.n]; + UniValue non_wit(UniValue::VOBJ); TxToUniv(*input.non_witness_utxo, uint256(), non_wit, false); in.pushKV("non_witness_utxo", non_wit); - CAmount utxo_val = input.non_witness_utxo->vout[psbtx.tx->vin[i].prevout.n].nValue; - if (MoneyRange(utxo_val) && MoneyRange(total_in + utxo_val)) { - total_in += utxo_val; + + have_a_utxo = true; + } + if (have_a_utxo) { + if (MoneyRange(txout.nValue) && MoneyRange(total_in + txout.nValue)) { + total_in += txout.nValue; } else { // Hack to just not show fee later have_all_utxos = false; @@ -1186,7 +1179,7 @@ UniValue decodepsbt(const JSONRPCRequest& request) if (!input.final_script_witness.IsNull()) { UniValue txinwitness(UniValue::VARR); for (const auto& item : input.final_script_witness.stack) { - txinwitness.push_back(HexStr(item.begin(), item.end())); + txinwitness.push_back(HexStr(item)); } in.pushKV("final_scriptwitness", txinwitness); } @@ -1628,7 +1621,7 @@ UniValue joinpsbts(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "At least two PSBTs are required to join PSBTs."); } - int32_t best_version = 1; + uint32_t best_version = 1; uint32_t best_locktime = 0xffffffff; for (unsigned int i = 0; i < txs.size(); ++i) { PartiallySignedTransaction psbtx; @@ -1638,8 +1631,8 @@ UniValue joinpsbts(const JSONRPCRequest& request) } psbtxs.push_back(psbtx); // Choose the highest version number - if (psbtx.tx->nVersion > best_version) { - best_version = psbtx.tx->nVersion; + if (static_cast<uint32_t>(psbtx.tx->nVersion) > best_version) { + best_version = static_cast<uint32_t>(psbtx.tx->nVersion); } // Choose the lowest lock time if (psbtx.tx->nLockTime < best_locktime) { @@ -1650,7 +1643,7 @@ UniValue joinpsbts(const JSONRPCRequest& request) // Create a blank psbt where everything will be added PartiallySignedTransaction merged_psbt; merged_psbt.tx = CMutableTransaction(); - merged_psbt.tx->nVersion = best_version; + merged_psbt.tx->nVersion = static_cast<int32_t>(best_version); merged_psbt.tx->nLockTime = best_locktime; // Merge diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index bd82773bf2..1031716b4a 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -138,10 +138,10 @@ static void TxInErrorToJSON(const CTxIn& txin, UniValue& vErrorsRet, const std:: entry.pushKV("vout", (uint64_t)txin.prevout.n); UniValue witness(UniValue::VARR); for (unsigned int i = 0; i < txin.scriptWitness.stack.size(); i++) { - witness.push_back(HexStr(txin.scriptWitness.stack[i].begin(), txin.scriptWitness.stack[i].end())); + witness.push_back(HexStr(txin.scriptWitness.stack[i])); } entry.pushKV("witness", witness); - entry.pushKV("scriptSig", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())); + entry.pushKV("scriptSig", HexStr(txin.scriptSig)); entry.pushKV("sequence", (uint64_t)txin.nSequence); entry.pushKV("error", strMessage); vErrorsRet.push_back(entry); diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 844f62cbc6..e5f6b1b9f1 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -20,10 +20,10 @@ #include <mutex> #include <unordered_map> -static RecursiveMutex cs_rpcWarmup; +static Mutex g_rpc_warmup_mutex; static std::atomic<bool> g_rpc_running{false}; -static bool fRPCInWarmup GUARDED_BY(cs_rpcWarmup) = true; -static std::string rpcWarmupStatus GUARDED_BY(cs_rpcWarmup) = "RPC server started"; +static bool fRPCInWarmup GUARDED_BY(g_rpc_warmup_mutex) = true; +static std::string rpcWarmupStatus GUARDED_BY(g_rpc_warmup_mutex) = "RPC server started"; /* Timer-creating functions */ static RPCTimerInterface* timerInterface = nullptr; /* Map of name to timer. */ @@ -130,11 +130,9 @@ std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest& return strRet; } -UniValue help(const JSONRPCRequest& jsonRequest) +static RPCHelpMan help() { - if (jsonRequest.fHelp || jsonRequest.params.size() > 1) - throw std::runtime_error( - RPCHelpMan{"help", + return RPCHelpMan{"help", "\nList all commands, or get help for a specified command.\n", { {"command", RPCArg::Type::STR, /* default */ "all commands", "The command to get help on"}, @@ -143,32 +141,32 @@ UniValue help(const JSONRPCRequest& jsonRequest) RPCResult::Type::STR, "", "The help text" }, RPCExamples{""}, - }.ToString() - ); - + [&](const RPCHelpMan& self, const JSONRPCRequest& jsonRequest) -> UniValue +{ std::string strCommand; if (jsonRequest.params.size() > 0) strCommand = jsonRequest.params[0].get_str(); return tableRPC.help(strCommand, jsonRequest); +}, + }; } - -UniValue stop(const JSONRPCRequest& jsonRequest) +static RPCHelpMan stop() { static const std::string RESULT{PACKAGE_NAME " stopping"}; - // Accept the deprecated and ignored 'detach' boolean argument + return RPCHelpMan{"stop", // Also accept the hidden 'wait' integer argument (milliseconds) // For instance, 'stop 1000' makes the call wait 1 second before returning // to the client (intended for testing) - if (jsonRequest.fHelp || jsonRequest.params.size() > 1) - throw std::runtime_error( - RPCHelpMan{"stop", "\nRequest a graceful shutdown of " PACKAGE_NAME ".", - {}, + { + {"wait", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "how long to wait in ms", "", {}, /* hidden */ true}, + }, RPCResult{RPCResult::Type::STR, "", "A string with the content '" + RESULT + "'"}, RPCExamples{""}, - }.ToString()); + [&](const RPCHelpMan& self, const JSONRPCRequest& jsonRequest) -> UniValue +{ // Event loop will exit after current HTTP requests have been handled, so // this reply will get back to the client. StartShutdown(); @@ -176,11 +174,13 @@ UniValue stop(const JSONRPCRequest& jsonRequest) UninterruptibleSleep(std::chrono::milliseconds{jsonRequest.params[0].get_int()}); } return RESULT; +}, + }; } -static UniValue uptime(const JSONRPCRequest& jsonRequest) +static RPCHelpMan uptime() { - RPCHelpMan{"uptime", + return RPCHelpMan{"uptime", "\nReturns the total uptime of the server.\n", {}, RPCResult{ @@ -190,14 +190,16 @@ static UniValue uptime(const JSONRPCRequest& jsonRequest) HelpExampleCli("uptime", "") + HelpExampleRpc("uptime", "") }, - }.Check(jsonRequest); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ return GetTime() - GetStartupTime(); } + }; +} -static UniValue getrpcinfo(const JSONRPCRequest& request) +static RPCHelpMan getrpcinfo() { - RPCHelpMan{"getrpcinfo", + return RPCHelpMan{"getrpcinfo", "\nReturns details of the RPC server.\n", {}, RPCResult{ @@ -217,8 +219,8 @@ static UniValue getrpcinfo(const JSONRPCRequest& request) RPCExamples{ HelpExampleCli("getrpcinfo", "") + HelpExampleRpc("getrpcinfo", "")}, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ LOCK(g_rpc_server_info.mutex); UniValue active_commands(UniValue::VARR); for (const RPCCommandExecutionInfo& info : g_rpc_server_info.active_commands) { @@ -237,6 +239,8 @@ static UniValue getrpcinfo(const JSONRPCRequest& request) return result; } + }; +} // clang-format off static const CRPCCommand vRPCCommands[] = @@ -327,20 +331,20 @@ void RpcInterruptionPoint() void SetRPCWarmupStatus(const std::string& newStatus) { - LOCK(cs_rpcWarmup); + LOCK(g_rpc_warmup_mutex); rpcWarmupStatus = newStatus; } void SetRPCWarmupFinished() { - LOCK(cs_rpcWarmup); + LOCK(g_rpc_warmup_mutex); assert(fRPCInWarmup); fRPCInWarmup = false; } bool RPCIsInWarmup(std::string *outStatus) { - LOCK(cs_rpcWarmup); + LOCK(g_rpc_warmup_mutex); if (outStatus) *outStatus = rpcWarmupStatus; return fRPCInWarmup; @@ -439,7 +443,7 @@ UniValue CRPCTable::execute(const JSONRPCRequest &request) const { // Return immediately if in warmup { - LOCK(cs_rpcWarmup); + LOCK(g_rpc_warmup_mutex); if (fRPCInWarmup) throw JSONRPCError(RPC_IN_WARMUP, rpcWarmupStatus); } diff --git a/src/rpc/server.h b/src/rpc/server.h index d7a04ff6e8..6da3e94ea2 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -8,6 +8,7 @@ #include <amount.h> #include <rpc/request.h> +#include <rpc/util.h> #include <functional> #include <map> @@ -85,6 +86,7 @@ void RPCUnsetTimerInterface(RPCTimerInterface *iface); void RPCRunLater(const std::string& name, std::function<void()> func, int64_t nSeconds); typedef UniValue(*rpcfn_type)(const JSONRPCRequest& jsonRequest); +typedef RPCHelpMan (*RpcMethodFnType)(); class CRPCCommand { @@ -101,6 +103,19 @@ public: { } + //! Simplified constructor taking plain RpcMethodFnType function pointer. + CRPCCommand(std::string category, std::string name_in, RpcMethodFnType fn, std::vector<std::string> args_in) + : CRPCCommand( + category, + fn().m_name, + [fn](const JSONRPCRequest& request, UniValue& result, bool) { result = fn().HandleRequest(request); return true; }, + fn().GetArgNames(), + intptr_t(fn)) + { + CHECK_NONFATAL(fn().m_name == name_in); + CHECK_NONFATAL(fn().GetArgNames() == args_in); + } + //! Simplified constructor taking plain rpcfn_type function pointer. CRPCCommand(const char* category, const char* name, rpcfn_type fn, std::initializer_list<const char*> args) : CRPCCommand(category, name, @@ -117,7 +132,7 @@ public: }; /** - * Bitcoin RPC command dispatcher. + * RPC command dispatcher. */ class CRPCTable { diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index e7afef4cac..9f4c7bee9c 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -113,6 +113,23 @@ std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey) return ParseHexV(find_value(o, strKey), strKey); } +CoinStatsHashType ParseHashType(const UniValue& param, const CoinStatsHashType default_type) +{ + if (param.isNull()) { + return default_type; + } else { + std::string hash_type_input = param.get_str(); + + if (hash_type_input == "hash_serialized_2") { + return CoinStatsHashType::HASH_SERIALIZED; + } else if (hash_type_input == "none") { + return CoinStatsHashType::NONE; + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%d is not a valid hash_type", hash_type_input)); + } + } +} + std::string HelpExampleCli(const std::string& methodname, const std::string& args) { return "> bitcoin-cli " + methodname + " " + args + "\n"; @@ -224,7 +241,7 @@ public: obj.pushKV("isscript", false); obj.pushKV("iswitness", true); obj.pushKV("witness_version", 0); - obj.pushKV("witness_program", HexStr(id.begin(), id.end())); + obj.pushKV("witness_program", HexStr(id)); return obj; } @@ -234,7 +251,7 @@ public: obj.pushKV("isscript", true); obj.pushKV("iswitness", true); obj.pushKV("witness_version", 0); - obj.pushKV("witness_program", HexStr(id.begin(), id.end())); + obj.pushKV("witness_program", HexStr(id)); return obj; } @@ -368,9 +385,7 @@ struct Sections { PushSection({indent + "]" + (outer_type != OuterType::NONE ? "," : ""), ""}); break; } - - // no default case, so the compiler can warn about missing cases - } + } // no default case, so the compiler can warn about missing cases } /** @@ -381,6 +396,9 @@ struct Sections { std::string ret; const size_t pad = m_max_pad + 4; for (const auto& s : m_sections) { + // The left part of a section is assumed to be a single line, usually it is the name of the JSON struct or a + // brace like {, }, [, or ] + CHECK_NONFATAL(s.m_left.find('\n') == std::string::npos); if (s.m_right.empty()) { ret += s.m_left; ret += "\n"; @@ -415,7 +433,11 @@ struct Sections { }; RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples) + : RPCHelpMan{std::move(name), std::move(description), std::move(args), std::move(results), std::move(examples), nullptr} {} + +RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples, RPCMethodImpl fun) : m_name{std::move(name)}, + m_fun{std::move(fun)}, m_description{std::move(description)}, m_args{std::move(args)}, m_results{std::move(results)}, @@ -464,6 +486,16 @@ bool RPCHelpMan::IsValidNumArgs(size_t num_args) const } return num_required_args <= num_args && num_args <= m_args.size(); } + +std::vector<std::string> RPCHelpMan::GetArgNames() const +{ + std::vector<std::string> ret; + for (const auto& arg : m_args) { + ret.emplace_back(arg.m_names); + } + return ret; +} + std::string RPCHelpMan::ToString() const { std::string ret; @@ -472,6 +504,7 @@ std::string RPCHelpMan::ToString() const ret += m_name; bool was_optional{false}; for (const auto& arg : m_args) { + if (arg.m_hidden) continue; const bool optional = arg.IsOptional(); ret += " "; if (optional) { @@ -493,6 +526,7 @@ std::string RPCHelpMan::ToString() const Sections sections; for (size_t i{0}; i < m_args.size(); ++i) { const auto& arg = m_args.at(i); + if (arg.m_hidden) continue; if (i == 0) ret += "\nArguments:\n"; @@ -572,9 +606,7 @@ std::string RPCArg::ToDescriptionString() const ret += "json array"; break; } - - // no default case, so the compiler can warn about missing cases - } + } // no default case, so the compiler can warn about missing cases } if (m_fallback.which() == 1) { ret += ", optional, default=" + boost::get<std::string>(m_fallback); @@ -592,9 +624,7 @@ std::string RPCArg::ToDescriptionString() const ret += ", required"; break; } - - // no default case, so the compiler can warn about missing cases - } + } // no default case, so the compiler can warn about missing cases } ret += ")"; ret += m_description.empty() ? "" : " " + m_description; @@ -689,10 +719,7 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const sections.PushSection({indent + "}" + maybe_separator, ""}); return; } - - // no default case, so the compiler can warn about missing cases - } - + } // no default case, so the compiler can warn about missing cases CHECK_NONFATAL(false); } @@ -729,9 +756,7 @@ std::string RPCArg::ToStringObj(const bool oneline) const case Type::OBJ_USER_KEYS: // Currently unused, so avoid writing dead code CHECK_NONFATAL(false); - - // no default case, so the compiler can warn about missing cases - } + } // no default case, so the compiler can warn about missing cases CHECK_NONFATAL(false); } @@ -766,9 +791,7 @@ std::string RPCArg::ToString(const bool oneline) const } return "[" + res + "...]"; } - - // no default case, so the compiler can warn about missing cases - } + } // no default case, so the compiler can warn about missing cases CHECK_NONFATAL(false); } diff --git a/src/rpc/util.h b/src/rpc/util.h index 53dce2c397..45b0bb0c7e 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_RPC_UTIL_H #define BITCOIN_RPC_UTIL_H +#include <node/coinstats.h> #include <node/transaction.h> #include <outputtype.h> #include <protocol.h> @@ -77,6 +78,8 @@ extern uint256 ParseHashO(const UniValue& o, std::string strKey); extern std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName); extern std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey); +CoinStatsHashType ParseHashType(const UniValue& param, const CoinStatsHashType default_type); + extern CAmount AmountFromValue(const UniValue& value); extern std::string HelpExampleCli(const std::string& methodname, const std::string& args); extern std::string HelpExampleRpc(const std::string& methodname, const std::string& args); @@ -144,6 +147,7 @@ struct RPCArg { using Fallback = boost::variant<Optional, /* default value for optional args */ std::string>; const std::string m_names; //!< The name of the arg (can be empty for inner args, can contain multiple aliases separated by | for named request arguments) const Type m_type; + const bool m_hidden; const std::vector<RPCArg> m_inner; //!< Only used for arrays or dicts const Fallback m_fallback; const std::string m_description; @@ -156,9 +160,11 @@ struct RPCArg { const Fallback fallback, const std::string description, const std::string oneline_description = "", - const std::vector<std::string> type_str = {}) + const std::vector<std::string> type_str = {}, + const bool hidden = false) : m_names{std::move(name)}, m_type{std::move(type)}, + m_hidden{hidden}, m_fallback{std::move(fallback)}, m_description{std::move(description)}, m_oneline_description{std::move(oneline_description)}, @@ -177,6 +183,7 @@ struct RPCArg { const std::vector<std::string> type_str = {}) : m_names{std::move(name)}, m_type{std::move(type)}, + m_hidden{false}, m_inner{std::move(inner)}, m_fallback{std::move(fallback)}, m_description{std::move(description)}, @@ -326,8 +333,15 @@ class RPCHelpMan { public: RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples); + using RPCMethodImpl = std::function<UniValue(const RPCHelpMan&, const JSONRPCRequest&)>; + RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples, RPCMethodImpl fun); std::string ToString() const; + UniValue HandleRequest(const JSONRPCRequest& request) + { + Check(request); + return m_fun(*this, request); + } /** If the supplied number of args is neither too small nor too high */ bool IsValidNumArgs(size_t num_args) const; /** @@ -340,8 +354,12 @@ public: } } -private: + std::vector<std::string> GetArgNames() const; + const std::string m_name; + +private: + const RPCMethodImpl m_fun; const std::string m_description; const std::vector<RPCArg> m_args; const RPCResults m_results; diff --git a/src/scheduler.cpp b/src/scheduler.cpp index c4bd47310b..7c361bf26f 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -30,9 +30,6 @@ void CScheduler::serviceQueue() // is called. while (!shouldStop()) { try { - if (!shouldStop() && taskQueue.empty()) { - REVERSE_LOCK(lock); - } while (!shouldStop() && taskQueue.empty()) { // Wait until there is something to do. newTaskScheduled.wait(lock); @@ -71,18 +68,6 @@ void CScheduler::serviceQueue() newTaskScheduled.notify_one(); } -void CScheduler::stop(bool drain) -{ - { - LOCK(newTaskMutex); - if (drain) - stopWhenEmpty = true; - else - stopRequested = true; - } - newTaskScheduled.notify_all(); -} - void CScheduler::schedule(CScheduler::Function f, std::chrono::system_clock::time_point t) { { @@ -125,8 +110,8 @@ void CScheduler::scheduleEvery(CScheduler::Function f, std::chrono::milliseconds scheduleFromNow([=] { Repeat(*this, f, delta); }, delta); } -size_t CScheduler::getQueueInfo(std::chrono::system_clock::time_point &first, - std::chrono::system_clock::time_point &last) const +size_t CScheduler::getQueueInfo(std::chrono::system_clock::time_point& first, + std::chrono::system_clock::time_point& last) const { LOCK(newTaskMutex); size_t result = taskQueue.size(); @@ -137,13 +122,15 @@ size_t CScheduler::getQueueInfo(std::chrono::system_clock::time_point &first, return result; } -bool CScheduler::AreThreadsServicingQueue() const { +bool CScheduler::AreThreadsServicingQueue() const +{ LOCK(newTaskMutex); return nThreadsServicingQueue; } -void SingleThreadedSchedulerClient::MaybeScheduleProcessQueue() { +void SingleThreadedSchedulerClient::MaybeScheduleProcessQueue() +{ { LOCK(m_cs_callbacks_pending); // Try to avoid scheduling too many copies here, but if we @@ -155,8 +142,9 @@ void SingleThreadedSchedulerClient::MaybeScheduleProcessQueue() { m_pscheduler->schedule(std::bind(&SingleThreadedSchedulerClient::ProcessQueue, this), std::chrono::system_clock::now()); } -void SingleThreadedSchedulerClient::ProcessQueue() { - std::function<void ()> callback; +void SingleThreadedSchedulerClient::ProcessQueue() +{ + std::function<void()> callback; { LOCK(m_cs_callbacks_pending); if (m_are_callbacks_running) return; @@ -172,7 +160,8 @@ void SingleThreadedSchedulerClient::ProcessQueue() { struct RAIICallbacksRunning { SingleThreadedSchedulerClient* instance; explicit RAIICallbacksRunning(SingleThreadedSchedulerClient* _instance) : instance(_instance) {} - ~RAIICallbacksRunning() { + ~RAIICallbacksRunning() + { { LOCK(instance->m_cs_callbacks_pending); instance->m_are_callbacks_running = false; @@ -184,7 +173,8 @@ void SingleThreadedSchedulerClient::ProcessQueue() { callback(); } -void SingleThreadedSchedulerClient::AddToProcessQueue(std::function<void ()> func) { +void SingleThreadedSchedulerClient::AddToProcessQueue(std::function<void()> func) +{ assert(m_pscheduler); { @@ -194,7 +184,8 @@ void SingleThreadedSchedulerClient::AddToProcessQueue(std::function<void ()> fun MaybeScheduleProcessQueue(); } -void SingleThreadedSchedulerClient::EmptyQueue() { +void SingleThreadedSchedulerClient::EmptyQueue() +{ assert(!m_pscheduler->AreThreadsServicingQueue()); bool should_continue = true; while (should_continue) { @@ -204,7 +195,8 @@ void SingleThreadedSchedulerClient::EmptyQueue() { } } -size_t SingleThreadedSchedulerClient::CallbacksPending() { +size_t SingleThreadedSchedulerClient::CallbacksPending() +{ LOCK(m_cs_callbacks_pending); return m_callbacks_pending.size(); } diff --git a/src/scheduler.h b/src/scheduler.h index 1e64195484..d7fe00d1b4 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -5,11 +5,6 @@ #ifndef BITCOIN_SCHEDULER_H #define BITCOIN_SCHEDULER_H -// -// NOTE: -// boost::thread should be ported to std::thread -// when we support C++11. -// #include <condition_variable> #include <functional> #include <list> @@ -17,24 +12,23 @@ #include <sync.h> -// -// Simple class for background tasks that should be run -// periodically or once "after a while" -// -// Usage: -// -// CScheduler* s = new CScheduler(); -// s->scheduleFromNow(doSomething, std::chrono::milliseconds{11}); // Assuming a: void doSomething() { } -// s->scheduleFromNow([=] { this->func(argument); }, std::chrono::milliseconds{3}); -// boost::thread* t = new boost::thread(std::bind(CScheduler::serviceQueue, s)); -// -// ... then at program shutdown, make sure to call stop() to clean up the thread(s) running serviceQueue: -// s->stop(); -// t->join(); -// delete t; -// delete s; // Must be done after thread is interrupted/joined. -// - +/** + * Simple class for background tasks that should be run + * periodically or once "after a while" + * + * Usage: + * + * CScheduler* s = new CScheduler(); + * s->scheduleFromNow(doSomething, std::chrono::milliseconds{11}); // Assuming a: void doSomething() { } + * s->scheduleFromNow([=] { this->func(argument); }, std::chrono::milliseconds{3}); + * std::thread* t = new std::thread([&] { s->serviceQueue(); }); + * + * ... then at program shutdown, make sure to call stop() to clean up the thread(s) running serviceQueue: + * s->stop(); + * t->join(); + * delete t; + * delete s; // Must be done after thread is interrupted/joined. + */ class CScheduler { public: @@ -43,7 +37,7 @@ public: typedef std::function<void()> Function; - // Call func at/after time t + /** Call func at/after time t */ void schedule(Function f, std::chrono::system_clock::time_point t); /** Call f once after the delta has passed */ @@ -67,23 +61,33 @@ public: */ void MockForward(std::chrono::seconds delta_seconds); - // To keep things as simple as possible, there is no unschedule. - - // Services the queue 'forever'. Should be run in a thread, - // and interrupted using boost::interrupt_thread + /** + * Services the queue 'forever'. Should be run in a thread, + * and interrupted using boost::interrupt_thread + */ void serviceQueue(); - // Tell any threads running serviceQueue to stop as soon as they're - // done servicing whatever task they're currently servicing (drain=false) - // or when there is no work left to be done (drain=true) - void stop(bool drain=false); + /** Tell any threads running serviceQueue to stop as soon as the current task is done */ + void stop() + { + WITH_LOCK(newTaskMutex, stopRequested = true); + newTaskScheduled.notify_all(); + } + /** Tell any threads running serviceQueue to stop when there is no work left to be done */ + void StopWhenDrained() + { + WITH_LOCK(newTaskMutex, stopWhenEmpty = true); + newTaskScheduled.notify_all(); + } - // Returns number of tasks waiting to be serviced, - // and first and last task times - size_t getQueueInfo(std::chrono::system_clock::time_point &first, - std::chrono::system_clock::time_point &last) const; + /** + * Returns number of tasks waiting to be serviced, + * and first and last task times + */ + size_t getQueueInfo(std::chrono::system_clock::time_point& first, + std::chrono::system_clock::time_point& last) const; - // Returns true if there are threads actively running in serviceQueue() + /** Returns true if there are threads actively running in serviceQueue() */ bool AreThreadsServicingQueue() const; private: @@ -106,19 +110,20 @@ private: * B() will be able to observe all of the effects of callback A() which executed * before it. */ -class SingleThreadedSchedulerClient { +class SingleThreadedSchedulerClient +{ private: - CScheduler *m_pscheduler; + CScheduler* m_pscheduler; RecursiveMutex m_cs_callbacks_pending; - std::list<std::function<void ()>> m_callbacks_pending GUARDED_BY(m_cs_callbacks_pending); + std::list<std::function<void()>> m_callbacks_pending GUARDED_BY(m_cs_callbacks_pending); bool m_are_callbacks_running GUARDED_BY(m_cs_callbacks_pending) = false; void MaybeScheduleProcessQueue(); void ProcessQueue(); public: - explicit SingleThreadedSchedulerClient(CScheduler *pschedulerIn) : m_pscheduler(pschedulerIn) {} + explicit SingleThreadedSchedulerClient(CScheduler* pschedulerIn) : m_pscheduler(pschedulerIn) {} /** * Add a callback to be executed. Callbacks are executed serially @@ -126,10 +131,12 @@ public: * Practically, this means that callbacks can behave as if they are executed * in order by a single thread. */ - void AddToProcessQueue(std::function<void ()> func); + void AddToProcessQueue(std::function<void()> func); - // Processes all remaining queue members on the calling thread, blocking until queue is empty - // Must be called after the CScheduler has no remaining processing threads! + /** + * Processes all remaining queue members on the calling thread, blocking until queue is empty + * Must be called after the CScheduler has no remaining processing threads! + */ void EmptyQueue(); size_t CallbacksPending(); diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index ed0175bb10..9978d084d5 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -139,7 +139,7 @@ std::string DescriptorChecksum(const Span<const char>& span) return ret; } -std::string AddChecksum(const std::string& str) { return str + "#" + DescriptorChecksum(MakeSpan(str)); } +std::string AddChecksum(const std::string& str) { return str + "#" + DescriptorChecksum(str); } //////////////////////////////////////////////////////////////////////////// // Internal representation // @@ -235,7 +235,7 @@ public: } bool IsRange() const override { return false; } size_t GetSize() const override { return m_pubkey.size(); } - std::string ToString() const override { return HexStr(m_pubkey.begin(), m_pubkey.end()); } + std::string ToString() const override { return HexStr(m_pubkey); } bool ToPrivateString(const SigningProvider& arg, std::string& ret) const override { CKey key; @@ -583,7 +583,7 @@ class RawDescriptor final : public DescriptorImpl { const CScript m_script; protected: - std::string ToStringExtra() const override { return HexStr(m_script.begin(), m_script.end()); } + std::string ToStringExtra() const override { return HexStr(m_script); } std::vector<CScript> MakeScripts(const std::vector<CPubKey>&, const CScript*, FlatSigningProvider&) const override { return Vector(m_script); } public: RawDescriptor(CScript script) : DescriptorImpl({}, {}, "raw"), m_script(std::move(script)) {} @@ -825,8 +825,9 @@ std::unique_ptr<PubkeyProvider> ParsePubkey(uint32_t key_exp_index, const Span<c return nullptr; } if (origin_split.size() == 1) return ParsePubkeyInner(key_exp_index, origin_split[0], permit_uncompressed, out, error); - if (origin_split[0].size() < 1 || origin_split[0][0] != '[') { - error = strprintf("Key origin start '[ character expected but not found, got '%c' instead", origin_split[0][0]); + if (origin_split[0].empty() || origin_split[0][0] != '[') { + error = strprintf("Key origin start '[ character expected but not found, got '%c' instead", + origin_split[0].empty() ? /** empty, implies split char */ ']' : origin_split[0][0]); return nullptr; } auto slash_split = Split(origin_split[0].subspan(1), '/'); @@ -896,7 +897,7 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t key_exp_index, Span<const c providers.emplace_back(std::move(pk)); key_exp_index++; } - if (providers.size() < 1 || providers.size() > 16) { + if (providers.empty() || providers.size() > 16) { error = strprintf("Cannot have %u keys in multisig; must have between 1 and 16 keys, inclusive", providers.size()); return nullptr; } else if (thres < 1) { @@ -985,15 +986,15 @@ std::unique_ptr<PubkeyProvider> InferPubkey(const CPubKey& pubkey, ParseScriptCo std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptContext ctx, const SigningProvider& provider) { std::vector<std::vector<unsigned char>> data; - txnouttype txntype = Solver(script, data); + TxoutType txntype = Solver(script, data); - if (txntype == TX_PUBKEY) { + if (txntype == TxoutType::PUBKEY) { CPubKey pubkey(data[0].begin(), data[0].end()); if (pubkey.IsValid()) { return MakeUnique<PKDescriptor>(InferPubkey(pubkey, ctx, provider)); } } - if (txntype == TX_PUBKEYHASH) { + if (txntype == TxoutType::PUBKEYHASH) { uint160 hash(data[0]); CKeyID keyid(hash); CPubKey pubkey; @@ -1001,7 +1002,7 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo return MakeUnique<PKHDescriptor>(InferPubkey(pubkey, ctx, provider)); } } - if (txntype == TX_WITNESS_V0_KEYHASH && ctx != ParseScriptContext::P2WSH) { + if (txntype == TxoutType::WITNESS_V0_KEYHASH && ctx != ParseScriptContext::P2WSH) { uint160 hash(data[0]); CKeyID keyid(hash); CPubKey pubkey; @@ -1009,7 +1010,7 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo return MakeUnique<WPKHDescriptor>(InferPubkey(pubkey, ctx, provider)); } } - if (txntype == TX_MULTISIG) { + if (txntype == TxoutType::MULTISIG) { std::vector<std::unique_ptr<PubkeyProvider>> providers; for (size_t i = 1; i + 1 < data.size(); ++i) { CPubKey pubkey(data[i].begin(), data[i].end()); @@ -1017,7 +1018,7 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo } return MakeUnique<MultisigDescriptor>((int)data[0][0], std::move(providers)); } - if (txntype == TX_SCRIPTHASH && ctx == ParseScriptContext::TOP) { + if (txntype == TxoutType::SCRIPTHASH && ctx == ParseScriptContext::TOP) { uint160 hash(data[0]); CScriptID scriptid(hash); CScript subscript; @@ -1026,7 +1027,7 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo if (sub) return MakeUnique<SHDescriptor>(std::move(sub)); } } - if (txntype == TX_WITNESS_V0_SCRIPTHASH && ctx != ParseScriptContext::P2WSH) { + if (txntype == TxoutType::WITNESS_V0_SCRIPTHASH && ctx != ParseScriptContext::P2WSH) { CScriptID scriptid; CRIPEMD160().Write(data[0].data(), data[0].size()).Finalize(scriptid.begin()); CScript subscript; @@ -1087,7 +1088,7 @@ bool CheckChecksum(Span<const char>& sp, bool require_checksum, std::string& err std::unique_ptr<Descriptor> Parse(const std::string& descriptor, FlatSigningProvider& out, std::string& error, bool require_checksum) { - Span<const char> sp(descriptor.data(), descriptor.size()); + Span<const char> sp{descriptor}; if (!CheckChecksum(sp, require_checksum, error)) return nullptr; auto ret = ParseScript(0, sp, ParseScriptContext::TOP, out, error); if (sp.size() == 0 && ret) return std::unique_ptr<Descriptor>(std::move(ret)); @@ -1098,7 +1099,7 @@ std::string GetDescriptorChecksum(const std::string& descriptor) { std::string ret; std::string error; - Span<const char> sp(descriptor.data(), descriptor.size()); + Span<const char> sp{descriptor}; if (!CheckChecksum(sp, false, error, &ret)) return ""; return ret; } diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 23d5b72a5c..9415bba585 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1522,7 +1522,7 @@ static bool ExecuteWitnessScript(const Span<const valtype>& stack_span, const CS static bool VerifyWitnessProgram(const CScriptWitness& witness, int witversion, const std::vector<unsigned char>& program, unsigned int flags, const BaseSignatureChecker& checker, ScriptError* serror) { CScript scriptPubKey; - Span<const valtype> stack = MakeSpan(witness.stack); + Span<const valtype> stack{witness.stack}; if (witversion == 0) { if (program.size() == WITNESS_V0_SCRIPTHASH_SIZE) { diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 1e00afcf89..f425215549 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -92,11 +92,11 @@ static bool CreateSig(const BaseSignatureCreator& creator, SignatureData& sigdat /** * Sign scriptPubKey using signature made with creator. * Signatures are returned in scriptSigRet (or returns false if scriptPubKey can't be signed), - * unless whichTypeRet is TX_SCRIPTHASH, in which case scriptSigRet is the redemption script. + * unless whichTypeRet is TxoutType::SCRIPTHASH, in which case scriptSigRet is the redemption script. * Returns false if scriptPubKey could not be completely satisfied. */ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator& creator, const CScript& scriptPubKey, - std::vector<valtype>& ret, txnouttype& whichTypeRet, SigVersion sigversion, SignatureData& sigdata) + std::vector<valtype>& ret, TxoutType& whichTypeRet, SigVersion sigversion, SignatureData& sigdata) { CScript scriptRet; uint160 h160; @@ -108,15 +108,15 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator switch (whichTypeRet) { - case TX_NONSTANDARD: - case TX_NULL_DATA: - case TX_WITNESS_UNKNOWN: + case TxoutType::NONSTANDARD: + case TxoutType::NULL_DATA: + case TxoutType::WITNESS_UNKNOWN: return false; - case TX_PUBKEY: + case TxoutType::PUBKEY: if (!CreateSig(creator, sigdata, provider, sig, CPubKey(vSolutions[0]), scriptPubKey, sigversion)) return false; ret.push_back(std::move(sig)); return true; - case TX_PUBKEYHASH: { + case TxoutType::PUBKEYHASH: { CKeyID keyID = CKeyID(uint160(vSolutions[0])); CPubKey pubkey; if (!GetPubKey(provider, sigdata, keyID, pubkey)) { @@ -129,9 +129,9 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator ret.push_back(ToByteVector(pubkey)); return true; } - case TX_SCRIPTHASH: + case TxoutType::SCRIPTHASH: h160 = uint160(vSolutions[0]); - if (GetCScript(provider, sigdata, h160, scriptRet)) { + if (GetCScript(provider, sigdata, CScriptID{h160}, scriptRet)) { ret.push_back(std::vector<unsigned char>(scriptRet.begin(), scriptRet.end())); return true; } @@ -139,7 +139,7 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator sigdata.missing_redeem_script = h160; return false; - case TX_MULTISIG: { + case TxoutType::MULTISIG: { size_t required = vSolutions.front()[0]; ret.push_back(valtype()); // workaround CHECKMULTISIG bug for (size_t i = 1; i < vSolutions.size() - 1; ++i) { @@ -159,13 +159,13 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator } return ok; } - case TX_WITNESS_V0_KEYHASH: + case TxoutType::WITNESS_V0_KEYHASH: ret.push_back(vSolutions[0]); return true; - case TX_WITNESS_V0_SCRIPTHASH: + case TxoutType::WITNESS_V0_SCRIPTHASH: CRIPEMD160().Write(&vSolutions[0][0], vSolutions[0].size()).Finalize(h160.begin()); - if (GetCScript(provider, sigdata, h160, scriptRet)) { + if (GetCScript(provider, sigdata, CScriptID{h160}, scriptRet)) { ret.push_back(std::vector<unsigned char>(scriptRet.begin(), scriptRet.end())); return true; } @@ -198,44 +198,44 @@ bool ProduceSignature(const SigningProvider& provider, const BaseSignatureCreato if (sigdata.complete) return true; std::vector<valtype> result; - txnouttype whichType; + TxoutType whichType; bool solved = SignStep(provider, creator, fromPubKey, result, whichType, SigVersion::BASE, sigdata); bool P2SH = false; CScript subscript; sigdata.scriptWitness.stack.clear(); - if (solved && whichType == TX_SCRIPTHASH) + if (solved && whichType == TxoutType::SCRIPTHASH) { // Solver returns the subscript that needs to be evaluated; // the final scriptSig is the signatures from that // and then the serialized subscript: subscript = CScript(result[0].begin(), result[0].end()); sigdata.redeem_script = subscript; - solved = solved && SignStep(provider, creator, subscript, result, whichType, SigVersion::BASE, sigdata) && whichType != TX_SCRIPTHASH; + solved = solved && SignStep(provider, creator, subscript, result, whichType, SigVersion::BASE, sigdata) && whichType != TxoutType::SCRIPTHASH; P2SH = true; } - if (solved && whichType == TX_WITNESS_V0_KEYHASH) + if (solved && whichType == TxoutType::WITNESS_V0_KEYHASH) { CScript witnessscript; witnessscript << OP_DUP << OP_HASH160 << ToByteVector(result[0]) << OP_EQUALVERIFY << OP_CHECKSIG; - txnouttype subType; + TxoutType subType; solved = solved && SignStep(provider, creator, witnessscript, result, subType, SigVersion::WITNESS_V0, sigdata); sigdata.scriptWitness.stack = result; sigdata.witness = true; result.clear(); } - else if (solved && whichType == TX_WITNESS_V0_SCRIPTHASH) + else if (solved && whichType == TxoutType::WITNESS_V0_SCRIPTHASH) { CScript witnessscript(result[0].begin(), result[0].end()); sigdata.witness_script = witnessscript; - txnouttype subType; - solved = solved && SignStep(provider, creator, witnessscript, result, subType, SigVersion::WITNESS_V0, sigdata) && subType != TX_SCRIPTHASH && subType != TX_WITNESS_V0_SCRIPTHASH && subType != TX_WITNESS_V0_KEYHASH; + TxoutType subType; + solved = solved && SignStep(provider, creator, witnessscript, result, subType, SigVersion::WITNESS_V0, sigdata) && subType != TxoutType::SCRIPTHASH && subType != TxoutType::WITNESS_V0_SCRIPTHASH && subType != TxoutType::WITNESS_V0_KEYHASH; result.push_back(std::vector<unsigned char>(witnessscript.begin(), witnessscript.end())); sigdata.scriptWitness.stack = result; sigdata.witness = true; result.clear(); - } else if (solved && whichType == TX_WITNESS_UNKNOWN) { + } else if (solved && whichType == TxoutType::WITNESS_UNKNOWN) { sigdata.witness = true; } @@ -301,11 +301,11 @@ SignatureData DataFromTransaction(const CMutableTransaction& tx, unsigned int nI // Get scripts std::vector<std::vector<unsigned char>> solutions; - txnouttype script_type = Solver(txout.scriptPubKey, solutions); + TxoutType script_type = Solver(txout.scriptPubKey, solutions); SigVersion sigversion = SigVersion::BASE; CScript next_script = txout.scriptPubKey; - if (script_type == TX_SCRIPTHASH && !stack.script.empty() && !stack.script.back().empty()) { + if (script_type == TxoutType::SCRIPTHASH && !stack.script.empty() && !stack.script.back().empty()) { // Get the redeemScript CScript redeem_script(stack.script.back().begin(), stack.script.back().end()); data.redeem_script = redeem_script; @@ -315,7 +315,7 @@ SignatureData DataFromTransaction(const CMutableTransaction& tx, unsigned int nI script_type = Solver(next_script, solutions); stack.script.pop_back(); } - if (script_type == TX_WITNESS_V0_SCRIPTHASH && !stack.witness.empty() && !stack.witness.back().empty()) { + if (script_type == TxoutType::WITNESS_V0_SCRIPTHASH && !stack.witness.empty() && !stack.witness.back().empty()) { // Get the witnessScript CScript witness_script(stack.witness.back().begin(), stack.witness.back().end()); data.witness_script = witness_script; @@ -328,7 +328,7 @@ SignatureData DataFromTransaction(const CMutableTransaction& tx, unsigned int nI stack.witness.clear(); sigversion = SigVersion::WITNESS_V0; } - if (script_type == TX_MULTISIG && !stack.script.empty()) { + if (script_type == TxoutType::MULTISIG && !stack.script.empty()) { // Build a map of pubkey -> signature by matching sigs to pubkeys: assert(solutions.size() > 1); unsigned int num_pubkeys = solutions.size()-2; @@ -454,13 +454,13 @@ bool IsSegWitOutput(const SigningProvider& provider, const CScript& script) { std::vector<valtype> solutions; auto whichtype = Solver(script, solutions); - if (whichtype == TX_WITNESS_V0_SCRIPTHASH || whichtype == TX_WITNESS_V0_KEYHASH || whichtype == TX_WITNESS_UNKNOWN) return true; - if (whichtype == TX_SCRIPTHASH) { + if (whichtype == TxoutType::WITNESS_V0_SCRIPTHASH || whichtype == TxoutType::WITNESS_V0_KEYHASH || whichtype == TxoutType::WITNESS_UNKNOWN) return true; + if (whichtype == TxoutType::SCRIPTHASH) { auto h160 = uint160(solutions[0]); CScript subscript; - if (provider.GetCScript(h160, subscript)) { + if (provider.GetCScript(CScriptID{h160}, subscript)) { whichtype = Solver(subscript, solutions); - if (whichtype == TX_WITNESS_V0_SCRIPTHASH || whichtype == TX_WITNESS_V0_KEYHASH || whichtype == TX_WITNESS_UNKNOWN) return true; + if (whichtype == TxoutType::WITNESS_V0_SCRIPTHASH || whichtype == TxoutType::WITNESS_V0_KEYHASH || whichtype == TxoutType::WITNESS_UNKNOWN) return true; } } return false; diff --git a/src/script/signingprovider.cpp b/src/script/signingprovider.cpp index 01757e2f65..2d8dc7d471 100644 --- a/src/script/signingprovider.cpp +++ b/src/script/signingprovider.cpp @@ -180,10 +180,10 @@ CKeyID GetKeyForDestination(const SigningProvider& store, const CTxDestination& // Only supports destinations which map to single public keys, i.e. P2PKH, // P2WPKH, and P2SH-P2WPKH. if (auto id = boost::get<PKHash>(&dest)) { - return CKeyID(*id); + return ToKeyID(*id); } if (auto witness_id = boost::get<WitnessV0KeyHash>(&dest)) { - return CKeyID(*witness_id); + return ToKeyID(*witness_id); } if (auto script_hash = boost::get<ScriptHash>(&dest)) { CScript script; @@ -191,7 +191,7 @@ CKeyID GetKeyForDestination(const SigningProvider& store, const CTxDestination& CTxDestination inner_dest; if (store.GetCScript(script_id, script) && ExtractDestination(script, inner_dest)) { if (auto inner_witness_id = boost::get<WitnessV0KeyHash>(&inner_dest)) { - return CKeyID(*inner_witness_id); + return ToKeyID(*inner_witness_id); } } } diff --git a/src/script/standard.cpp b/src/script/standard.cpp index c90c2c24a0..1c4990791c 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -16,31 +16,47 @@ typedef std::vector<unsigned char> valtype; bool fAcceptDatacarrier = DEFAULT_ACCEPT_DATACARRIER; unsigned nMaxDatacarrierBytes = MAX_OP_RETURN_RELAY; -CScriptID::CScriptID(const CScript& in) : uint160(Hash160(in.begin(), in.end())) {} +CScriptID::CScriptID(const CScript& in) : BaseHash(Hash160(in.begin(), in.end())) {} +CScriptID::CScriptID(const ScriptHash& in) : BaseHash(static_cast<uint160>(in)) {} -ScriptHash::ScriptHash(const CScript& in) : uint160(Hash160(in.begin(), in.end())) {} +ScriptHash::ScriptHash(const CScript& in) : BaseHash(Hash160(in.begin(), in.end())) {} +ScriptHash::ScriptHash(const CScriptID& in) : BaseHash(static_cast<uint160>(in)) {} -PKHash::PKHash(const CPubKey& pubkey) : uint160(pubkey.GetID()) {} +PKHash::PKHash(const CPubKey& pubkey) : BaseHash(pubkey.GetID()) {} +PKHash::PKHash(const CKeyID& pubkey_id) : BaseHash(pubkey_id) {} + +WitnessV0KeyHash::WitnessV0KeyHash(const CPubKey& pubkey) : BaseHash(pubkey.GetID()) {} +WitnessV0KeyHash::WitnessV0KeyHash(const PKHash& pubkey_hash) : BaseHash(static_cast<uint160>(pubkey_hash)) {} + +CKeyID ToKeyID(const PKHash& key_hash) +{ + return CKeyID{static_cast<uint160>(key_hash)}; +} + +CKeyID ToKeyID(const WitnessV0KeyHash& key_hash) +{ + return CKeyID{static_cast<uint160>(key_hash)}; +} WitnessV0ScriptHash::WitnessV0ScriptHash(const CScript& in) { CSHA256().Write(in.data(), in.size()).Finalize(begin()); } -std::string GetTxnOutputType(txnouttype t) +std::string GetTxnOutputType(TxoutType t) { switch (t) { - case TX_NONSTANDARD: return "nonstandard"; - case TX_PUBKEY: return "pubkey"; - case TX_PUBKEYHASH: return "pubkeyhash"; - case TX_SCRIPTHASH: return "scripthash"; - case TX_MULTISIG: return "multisig"; - case TX_NULL_DATA: return "nulldata"; - case TX_WITNESS_V0_KEYHASH: return "witness_v0_keyhash"; - case TX_WITNESS_V0_SCRIPTHASH: return "witness_v0_scripthash"; - case TX_WITNESS_UNKNOWN: return "witness_unknown"; - } + case TxoutType::NONSTANDARD: return "nonstandard"; + case TxoutType::PUBKEY: return "pubkey"; + case TxoutType::PUBKEYHASH: return "pubkeyhash"; + case TxoutType::SCRIPTHASH: return "scripthash"; + case TxoutType::MULTISIG: return "multisig"; + case TxoutType::NULL_DATA: return "nulldata"; + case TxoutType::WITNESS_V0_KEYHASH: return "witness_v0_keyhash"; + case TxoutType::WITNESS_V0_SCRIPTHASH: return "witness_v0_scripthash"; + case TxoutType::WITNESS_UNKNOWN: return "witness_unknown"; + } // no default case, so the compiler can warn about missing cases assert(false); } @@ -90,7 +106,7 @@ static bool MatchMultisig(const CScript& script, unsigned int& required, std::ve return (it + 1 == script.end()); } -txnouttype Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned char>>& vSolutionsRet) +TxoutType Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned char>>& vSolutionsRet) { vSolutionsRet.clear(); @@ -100,7 +116,7 @@ txnouttype Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned { std::vector<unsigned char> hashBytes(scriptPubKey.begin()+2, scriptPubKey.begin()+22); vSolutionsRet.push_back(hashBytes); - return TX_SCRIPTHASH; + return TxoutType::SCRIPTHASH; } int witnessversion; @@ -108,18 +124,18 @@ txnouttype Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned if (scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { if (witnessversion == 0 && witnessprogram.size() == WITNESS_V0_KEYHASH_SIZE) { vSolutionsRet.push_back(witnessprogram); - return TX_WITNESS_V0_KEYHASH; + return TxoutType::WITNESS_V0_KEYHASH; } if (witnessversion == 0 && witnessprogram.size() == WITNESS_V0_SCRIPTHASH_SIZE) { vSolutionsRet.push_back(witnessprogram); - return TX_WITNESS_V0_SCRIPTHASH; + return TxoutType::WITNESS_V0_SCRIPTHASH; } if (witnessversion != 0) { vSolutionsRet.push_back(std::vector<unsigned char>{(unsigned char)witnessversion}); vSolutionsRet.push_back(std::move(witnessprogram)); - return TX_WITNESS_UNKNOWN; + return TxoutType::WITNESS_UNKNOWN; } - return TX_NONSTANDARD; + return TxoutType::NONSTANDARD; } // Provably prunable, data-carrying output @@ -128,18 +144,18 @@ txnouttype Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned // byte passes the IsPushOnly() test we don't care what exactly is in the // script. if (scriptPubKey.size() >= 1 && scriptPubKey[0] == OP_RETURN && scriptPubKey.IsPushOnly(scriptPubKey.begin()+1)) { - return TX_NULL_DATA; + return TxoutType::NULL_DATA; } std::vector<unsigned char> data; if (MatchPayToPubkey(scriptPubKey, data)) { vSolutionsRet.push_back(std::move(data)); - return TX_PUBKEY; + return TxoutType::PUBKEY; } if (MatchPayToPubkeyHash(scriptPubKey, data)) { vSolutionsRet.push_back(std::move(data)); - return TX_PUBKEYHASH; + return TxoutType::PUBKEYHASH; } unsigned int required; @@ -148,19 +164,19 @@ txnouttype Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned vSolutionsRet.push_back({static_cast<unsigned char>(required)}); // safe as required is in range 1..16 vSolutionsRet.insert(vSolutionsRet.end(), keys.begin(), keys.end()); vSolutionsRet.push_back({static_cast<unsigned char>(keys.size())}); // safe as size is in range 1..16 - return TX_MULTISIG; + return TxoutType::MULTISIG; } vSolutionsRet.clear(); - return TX_NONSTANDARD; + return TxoutType::NONSTANDARD; } bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) { std::vector<valtype> vSolutions; - txnouttype whichType = Solver(scriptPubKey, vSolutions); + TxoutType whichType = Solver(scriptPubKey, vSolutions); - if (whichType == TX_PUBKEY) { + if (whichType == TxoutType::PUBKEY) { CPubKey pubKey(vSolutions[0]); if (!pubKey.IsValid()) return false; @@ -168,26 +184,26 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) addressRet = PKHash(pubKey); return true; } - else if (whichType == TX_PUBKEYHASH) + else if (whichType == TxoutType::PUBKEYHASH) { addressRet = PKHash(uint160(vSolutions[0])); return true; } - else if (whichType == TX_SCRIPTHASH) + else if (whichType == TxoutType::SCRIPTHASH) { addressRet = ScriptHash(uint160(vSolutions[0])); return true; - } else if (whichType == TX_WITNESS_V0_KEYHASH) { + } else if (whichType == TxoutType::WITNESS_V0_KEYHASH) { WitnessV0KeyHash hash; std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); addressRet = hash; return true; - } else if (whichType == TX_WITNESS_V0_SCRIPTHASH) { + } else if (whichType == TxoutType::WITNESS_V0_SCRIPTHASH) { WitnessV0ScriptHash hash; std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); addressRet = hash; return true; - } else if (whichType == TX_WITNESS_UNKNOWN) { + } else if (whichType == TxoutType::WITNESS_UNKNOWN) { WitnessUnknown unk; unk.version = vSolutions[0][0]; std::copy(vSolutions[1].begin(), vSolutions[1].end(), unk.program); @@ -199,19 +215,19 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) return false; } -bool ExtractDestinations(const CScript& scriptPubKey, txnouttype& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet) +bool ExtractDestinations(const CScript& scriptPubKey, TxoutType& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet) { addressRet.clear(); std::vector<valtype> vSolutions; typeRet = Solver(scriptPubKey, vSolutions); - if (typeRet == TX_NONSTANDARD) { + if (typeRet == TxoutType::NONSTANDARD) { return false; - } else if (typeRet == TX_NULL_DATA) { + } else if (typeRet == TxoutType::NULL_DATA) { // This is data, not addresses return false; } - if (typeRet == TX_MULTISIG) + if (typeRet == TxoutType::MULTISIG) { nRequiredRet = vSolutions.front()[0]; for (unsigned int i = 1; i < vSolutions.size()-1; i++) @@ -241,59 +257,44 @@ bool ExtractDestinations(const CScript& scriptPubKey, txnouttype& typeRet, std:: namespace { -class CScriptVisitor : public boost::static_visitor<bool> +class CScriptVisitor : public boost::static_visitor<CScript> { -private: - CScript *script; public: - explicit CScriptVisitor(CScript *scriptin) { script = scriptin; } - - bool operator()(const CNoDestination &dest) const { - script->clear(); - return false; + CScript operator()(const CNoDestination& dest) const + { + return CScript(); } - bool operator()(const PKHash &keyID) const { - script->clear(); - *script << OP_DUP << OP_HASH160 << ToByteVector(keyID) << OP_EQUALVERIFY << OP_CHECKSIG; - return true; + CScript operator()(const PKHash& keyID) const + { + return CScript() << OP_DUP << OP_HASH160 << ToByteVector(keyID) << OP_EQUALVERIFY << OP_CHECKSIG; } - bool operator()(const ScriptHash &scriptID) const { - script->clear(); - *script << OP_HASH160 << ToByteVector(scriptID) << OP_EQUAL; - return true; + CScript operator()(const ScriptHash& scriptID) const + { + return CScript() << OP_HASH160 << ToByteVector(scriptID) << OP_EQUAL; } - bool operator()(const WitnessV0KeyHash& id) const + CScript operator()(const WitnessV0KeyHash& id) const { - script->clear(); - *script << OP_0 << ToByteVector(id); - return true; + return CScript() << OP_0 << ToByteVector(id); } - bool operator()(const WitnessV0ScriptHash& id) const + CScript operator()(const WitnessV0ScriptHash& id) const { - script->clear(); - *script << OP_0 << ToByteVector(id); - return true; + return CScript() << OP_0 << ToByteVector(id); } - bool operator()(const WitnessUnknown& id) const + CScript operator()(const WitnessUnknown& id) const { - script->clear(); - *script << CScript::EncodeOP_N(id.version) << std::vector<unsigned char>(id.program, id.program + id.length); - return true; + return CScript() << CScript::EncodeOP_N(id.version) << std::vector<unsigned char>(id.program, id.program + id.length); } }; } // namespace CScript GetScriptForDestination(const CTxDestination& dest) { - CScript script; - - boost::apply_visitor(CScriptVisitor(&script), dest); - return script; + return boost::apply_visitor(CScriptVisitor(), dest); } CScript GetScriptForRawPubKey(const CPubKey& pubKey) @@ -315,11 +316,11 @@ CScript GetScriptForMultisig(int nRequired, const std::vector<CPubKey>& keys) CScript GetScriptForWitness(const CScript& redeemscript) { std::vector<std::vector<unsigned char> > vSolutions; - txnouttype typ = Solver(redeemscript, vSolutions); - if (typ == TX_PUBKEY) { + TxoutType typ = Solver(redeemscript, vSolutions); + if (typ == TxoutType::PUBKEY) { return GetScriptForDestination(WitnessV0KeyHash(Hash160(vSolutions[0].begin(), vSolutions[0].end()))); - } else if (typ == TX_PUBKEYHASH) { - return GetScriptForDestination(WitnessV0KeyHash(vSolutions[0])); + } else if (typ == TxoutType::PUBKEYHASH) { + return GetScriptForDestination(WitnessV0KeyHash(uint160{vSolutions[0]})); } return GetScriptForDestination(WitnessV0ScriptHash(redeemscript)); } diff --git a/src/script/standard.h b/src/script/standard.h index 2929425670..fd29353886 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -18,14 +18,77 @@ static const bool DEFAULT_ACCEPT_DATACARRIER = true; class CKeyID; class CScript; +struct ScriptHash; + +template<typename HashType> +class BaseHash +{ +protected: + HashType m_hash; + +public: + BaseHash() : m_hash() {} + BaseHash(const HashType& in) : m_hash(in) {} + + unsigned char* begin() + { + return m_hash.begin(); + } + + const unsigned char* begin() const + { + return m_hash.begin(); + } + + unsigned char* end() + { + return m_hash.end(); + } + + const unsigned char* end() const + { + return m_hash.end(); + } + + operator std::vector<unsigned char>() const + { + return std::vector<unsigned char>{m_hash.begin(), m_hash.end()}; + } + + std::string ToString() const + { + return m_hash.ToString(); + } + + bool operator==(const BaseHash<HashType>& other) const noexcept + { + return m_hash == other.m_hash; + } + + bool operator!=(const BaseHash<HashType>& other) const noexcept + { + return !(m_hash == other.m_hash); + } + + bool operator<(const BaseHash<HashType>& other) const noexcept + { + return m_hash < other.m_hash; + } + + size_t size() const + { + return m_hash.size(); + } +}; /** A reference to a CScript: the Hash160 of its serialization (see script.h) */ -class CScriptID : public uint160 +class CScriptID : public BaseHash<uint160> { public: - CScriptID() : uint160() {} + CScriptID() : BaseHash() {} explicit CScriptID(const CScript& in); - CScriptID(const uint160& in) : uint160(in) {} + explicit CScriptID(const uint160& in) : BaseHash(in) {} + explicit CScriptID(const ScriptHash& in); }; /** @@ -36,11 +99,11 @@ static const unsigned int MAX_OP_RETURN_RELAY = 83; /** * A data carrying output is an unspendable output containing data. The script - * type is designated as TX_NULL_DATA. + * type is designated as TxoutType::NULL_DATA. */ extern bool fAcceptDatacarrier; -/** Maximum size of TX_NULL_DATA scripts that this node considers standard. */ +/** Maximum size of TxoutType::NULL_DATA scripts that this node considers standard. */ extern unsigned nMaxDatacarrierBytes; /** @@ -53,18 +116,17 @@ extern unsigned nMaxDatacarrierBytes; */ static const unsigned int MANDATORY_SCRIPT_VERIFY_FLAGS = SCRIPT_VERIFY_P2SH; -enum txnouttype -{ - TX_NONSTANDARD, +enum class TxoutType { + NONSTANDARD, // 'standard' transaction types: - TX_PUBKEY, - TX_PUBKEYHASH, - TX_SCRIPTHASH, - TX_MULTISIG, - TX_NULL_DATA, //!< unspendable OP_RETURN script that carries data - TX_WITNESS_V0_SCRIPTHASH, - TX_WITNESS_V0_KEYHASH, - TX_WITNESS_UNKNOWN, //!< Only for Witness versions not already defined above + PUBKEY, + PUBKEYHASH, + SCRIPTHASH, + MULTISIG, + NULL_DATA, //!< unspendable OP_RETURN script that carries data + WITNESS_V0_SCRIPTHASH, + WITNESS_V0_KEYHASH, + WITNESS_UNKNOWN, //!< Only for Witness versions not already defined above }; class CNoDestination { @@ -73,41 +135,44 @@ public: friend bool operator<(const CNoDestination &a, const CNoDestination &b) { return true; } }; -struct PKHash : public uint160 +struct PKHash : public BaseHash<uint160> { - PKHash() : uint160() {} - explicit PKHash(const uint160& hash) : uint160(hash) {} + PKHash() : BaseHash() {} + explicit PKHash(const uint160& hash) : BaseHash(hash) {} explicit PKHash(const CPubKey& pubkey); - using uint160::uint160; + explicit PKHash(const CKeyID& pubkey_id); }; +CKeyID ToKeyID(const PKHash& key_hash); struct WitnessV0KeyHash; -struct ScriptHash : public uint160 +struct ScriptHash : public BaseHash<uint160> { - ScriptHash() : uint160() {} + ScriptHash() : BaseHash() {} // These don't do what you'd expect. // Use ScriptHash(GetScriptForDestination(...)) instead. explicit ScriptHash(const WitnessV0KeyHash& hash) = delete; explicit ScriptHash(const PKHash& hash) = delete; - explicit ScriptHash(const uint160& hash) : uint160(hash) {} + + explicit ScriptHash(const uint160& hash) : BaseHash(hash) {} explicit ScriptHash(const CScript& script); - using uint160::uint160; + explicit ScriptHash(const CScriptID& script); }; -struct WitnessV0ScriptHash : public uint256 +struct WitnessV0ScriptHash : public BaseHash<uint256> { - WitnessV0ScriptHash() : uint256() {} - explicit WitnessV0ScriptHash(const uint256& hash) : uint256(hash) {} + WitnessV0ScriptHash() : BaseHash() {} + explicit WitnessV0ScriptHash(const uint256& hash) : BaseHash(hash) {} explicit WitnessV0ScriptHash(const CScript& script); - using uint256::uint256; }; -struct WitnessV0KeyHash : public uint160 +struct WitnessV0KeyHash : public BaseHash<uint160> { - WitnessV0KeyHash() : uint160() {} - explicit WitnessV0KeyHash(const uint160& hash) : uint160(hash) {} - using uint160::uint160; + WitnessV0KeyHash() : BaseHash() {} + explicit WitnessV0KeyHash(const uint160& hash) : BaseHash(hash) {} + explicit WitnessV0KeyHash(const CPubKey& pubkey); + explicit WitnessV0KeyHash(const PKHash& pubkey_hash); }; +CKeyID ToKeyID(const WitnessV0KeyHash& key_hash); //! CTxDestination subtype to encode any future Witness version struct WitnessUnknown @@ -134,11 +199,11 @@ struct WitnessUnknown /** * A txout script template with a specific destination. It is either: * * CNoDestination: no destination set - * * PKHash: TX_PUBKEYHASH destination (P2PKH) - * * ScriptHash: TX_SCRIPTHASH destination (P2SH) - * * WitnessV0ScriptHash: TX_WITNESS_V0_SCRIPTHASH destination (P2WSH) - * * WitnessV0KeyHash: TX_WITNESS_V0_KEYHASH destination (P2WPKH) - * * WitnessUnknown: TX_WITNESS_UNKNOWN destination (P2W???) + * * PKHash: TxoutType::PUBKEYHASH destination (P2PKH) + * * ScriptHash: TxoutType::SCRIPTHASH destination (P2SH) + * * WitnessV0ScriptHash: TxoutType::WITNESS_V0_SCRIPTHASH destination (P2WSH) + * * WitnessV0KeyHash: TxoutType::WITNESS_V0_KEYHASH destination (P2WPKH) + * * WitnessUnknown: TxoutType::WITNESS_UNKNOWN destination (P2W???) * A CTxDestination is the internal data type encoded in a bitcoin address */ typedef boost::variant<CNoDestination, PKHash, ScriptHash, WitnessV0ScriptHash, WitnessV0KeyHash, WitnessUnknown> CTxDestination; @@ -146,8 +211,8 @@ typedef boost::variant<CNoDestination, PKHash, ScriptHash, WitnessV0ScriptHash, /** Check whether a CTxDestination is a CNoDestination. */ bool IsValidDestination(const CTxDestination& dest); -/** Get the name of a txnouttype as a C string, or nullptr if unknown. */ -std::string GetTxnOutputType(txnouttype t); +/** Get the name of a TxoutType as a string */ +std::string GetTxnOutputType(TxoutType t); /** * Parse a scriptPubKey and identify script type for standard scripts. If @@ -157,9 +222,9 @@ std::string GetTxnOutputType(txnouttype t); * * @param[in] scriptPubKey Script to parse * @param[out] vSolutionsRet Vector of parsed pubkeys and hashes - * @return The script type. TX_NONSTANDARD represents a failed solve. + * @return The script type. TxoutType::NONSTANDARD represents a failed solve. */ -txnouttype Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned char>>& vSolutionsRet); +TxoutType Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned char>>& vSolutionsRet); /** * Parse a standard scriptPubKey for the destination address. Assigns result to @@ -180,7 +245,7 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) * encodable as an address) with key identifiers (of keys involved in a * CScript), and its use should be phased out. */ -bool ExtractDestinations(const CScript& scriptPubKey, txnouttype& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet); +bool ExtractDestinations(const CScript& scriptPubKey, TxoutType& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet); /** * Generate a Bitcoin scriptPubKey for the given CTxDestination. Returns a P2PKH diff --git a/src/secp256k1/.gitignore b/src/secp256k1/.gitignore index 55d325aeef..cb4331aa90 100644 --- a/src/secp256k1/.gitignore +++ b/src/secp256k1/.gitignore @@ -9,6 +9,7 @@ bench_internal tests exhaustive_tests gen_context +valgrind_ctime_test *.exe *.so *.a diff --git a/src/secp256k1/.travis.yml b/src/secp256k1/.travis.yml index 74f658f4d1..a6ad6fb27e 100644 --- a/src/secp256k1/.travis.yml +++ b/src/secp256k1/.travis.yml @@ -1,18 +1,23 @@ language: c -os: linux +os: + - linux + - osx + +dist: bionic +# Valgrind currently supports upto macOS 10.13, the latest xcode of that version is 10.1 +osx_image: xcode10.1 addons: apt: - packages: libgmp-dev + packages: + - libgmp-dev + - valgrind + - libtool-bin compiler: - clang - gcc -cache: - directories: - - src/java/guava/ env: global: - - FIELD=auto BIGNUM=auto SCALAR=auto ENDOMORPHISM=no STATICPRECOMPUTATION=yes ASM=no BUILD=check EXTRAFLAGS= HOST= ECDH=no RECOVERY=no EXPERIMENTAL=no JNI=no - - GUAVA_URL=https://search.maven.org/remotecontent?filepath=com/google/guava/guava/18.0/guava-18.0.jar GUAVA_JAR=src/java/guava/guava-18.0.jar + - FIELD=auto BIGNUM=auto SCALAR=auto ENDOMORPHISM=no STATICPRECOMPUTATION=yes ECMULTGENPRECISION=auto ASM=no BUILD=check EXTRAFLAGS= HOST= ECDH=no RECOVERY=no EXPERIMENTAL=no CTIMETEST=yes BENCH=yes ITERS=2 matrix: - SCALAR=32bit RECOVERY=yes - SCALAR=32bit FIELD=32bit ECDH=yes EXPERIMENTAL=yes @@ -26,43 +31,78 @@ env: - BIGNUM=no - BIGNUM=no ENDOMORPHISM=yes RECOVERY=yes EXPERIMENTAL=yes - BIGNUM=no STATICPRECOMPUTATION=no - - BUILD=distcheck - - EXTRAFLAGS=CPPFLAGS=-DDETERMINISTIC - - EXTRAFLAGS=CFLAGS=-O0 - - BUILD=check-java JNI=yes ECDH=yes EXPERIMENTAL=yes + - BUILD=distcheck CTIMETEST= BENCH= + - CPPFLAGS=-DDETERMINISTIC + - CFLAGS=-O0 CTIMETEST= + - ECMULTGENPRECISION=2 + - ECMULTGENPRECISION=8 + - VALGRIND=yes ENDOMORPHISM=yes BIGNUM=no ASM=x86_64 EXPERIMENTAL=yes ECDH=yes RECOVERY=yes EXTRAFLAGS="--disable-openssl-tests" CPPFLAGS=-DVALGRIND BUILD= + - VALGRIND=yes BIGNUM=no ASM=x86_64 EXPERIMENTAL=yes ECDH=yes RECOVERY=yes EXTRAFLAGS="--disable-openssl-tests" CPPFLAGS=-DVALGRIND BUILD= matrix: fast_finish: true include: - compiler: clang + os: linux env: HOST=i686-linux-gnu ENDOMORPHISM=yes addons: apt: packages: - gcc-multilib - libgmp-dev:i386 + - valgrind + - libtool-bin + - libc6-dbg:i386 - compiler: clang env: HOST=i686-linux-gnu + os: linux addons: apt: packages: - gcc-multilib + - valgrind + - libtool-bin + - libc6-dbg:i386 - compiler: gcc env: HOST=i686-linux-gnu ENDOMORPHISM=yes + os: linux addons: apt: packages: - gcc-multilib + - valgrind + - libtool-bin + - libc6-dbg:i386 - compiler: gcc + os: linux env: HOST=i686-linux-gnu addons: apt: packages: - gcc-multilib - libgmp-dev:i386 -before_install: mkdir -p `dirname $GUAVA_JAR` -install: if [ ! -f $GUAVA_JAR ]; then wget $GUAVA_URL -O $GUAVA_JAR; fi + - valgrind + - libtool-bin + - libc6-dbg:i386 + +# We use this to install macOS dependencies instead of the built in `homebrew` plugin, +# because in xcode earlier than 11 they have a bug requiring updating the system which overall takes ~8 minutes. +# https://travis-ci.community/t/macos-build-fails-because-of-homebrew-bundle-unknown-command/7296 +before_install: + - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then HOMEBREW_NO_AUTO_UPDATE=1 brew install gmp valgrind gcc@9; fi + before_script: ./autogen.sh + +# travis auto terminates jobs that go for 10 minutes without printing to stdout, but travis_wait doesn't work well with forking programs like valgrind (https://docs.travis-ci.com/user/common-build-problems/#build-times-out-because-no-output-was-received https://github.com/bitcoin-core/secp256k1/pull/750#issuecomment-623476860) script: - - if [ -n "$HOST" ]; then export USE_HOST="--host=$HOST"; fi - - if [ "x$HOST" = "xi686-linux-gnu" ]; then export CC="$CC -m32"; fi - - ./configure --enable-experimental=$EXPERIMENTAL --enable-endomorphism=$ENDOMORPHISM --with-field=$FIELD --with-bignum=$BIGNUM --with-scalar=$SCALAR --enable-ecmult-static-precomputation=$STATICPRECOMPUTATION --enable-module-ecdh=$ECDH --enable-module-recovery=$RECOVERY --enable-jni=$JNI $EXTRAFLAGS $USE_HOST && make -j2 $BUILD + - function keep_alive() { while true; do echo -en "\a"; sleep 60; done } + - keep_alive & + - ./contrib/travis.sh + - kill %keep_alive + +after_script: + - cat ./tests.log + - cat ./exhaustive_tests.log + - cat ./valgrind_ctime_test.log + - cat ./bench.log + - $CC --version + - valgrind --version diff --git a/src/secp256k1/Makefile.am b/src/secp256k1/Makefile.am index 9e5b7dcce0..d8c1c79e8c 100644 --- a/src/secp256k1/Makefile.am +++ b/src/secp256k1/Makefile.am @@ -1,13 +1,8 @@ ACLOCAL_AMFLAGS = -I build-aux/m4 lib_LTLIBRARIES = libsecp256k1.la -if USE_JNI -JNI_LIB = libsecp256k1_jni.la -noinst_LTLIBRARIES = $(JNI_LIB) -else -JNI_LIB = -endif include_HEADERS = include/secp256k1.h +include_HEADERS += include/secp256k1_preallocated.h noinst_HEADERS = noinst_HEADERS += src/scalar.h noinst_HEADERS += src/scalar_4x64.h @@ -39,8 +34,6 @@ noinst_HEADERS += src/field_5x52.h noinst_HEADERS += src/field_5x52_impl.h noinst_HEADERS += src/field_5x52_int128_impl.h noinst_HEADERS += src/field_5x52_asm_impl.h -noinst_HEADERS += src/java/org_bitcoin_NativeSecp256k1.h -noinst_HEADERS += src/java/org_bitcoin_Secp256k1Context.h noinst_HEADERS += src/util.h noinst_HEADERS += src/scratch.h noinst_HEADERS += src/scratch_impl.h @@ -74,16 +67,19 @@ endif libsecp256k1_la_SOURCES = src/secp256k1.c libsecp256k1_la_CPPFLAGS = -DSECP256K1_BUILD -I$(top_srcdir)/include -I$(top_srcdir)/src $(SECP_INCLUDES) -libsecp256k1_la_LIBADD = $(JNI_LIB) $(SECP_LIBS) $(COMMON_LIB) +libsecp256k1_la_LIBADD = $(SECP_LIBS) $(COMMON_LIB) -libsecp256k1_jni_la_SOURCES = src/java/org_bitcoin_NativeSecp256k1.c src/java/org_bitcoin_Secp256k1Context.c -libsecp256k1_jni_la_CPPFLAGS = -DSECP256K1_BUILD $(JNI_INCLUDES) +if VALGRIND_ENABLED +libsecp256k1_la_CPPFLAGS += -DVALGRIND +endif noinst_PROGRAMS = if USE_BENCHMARK noinst_PROGRAMS += bench_verify bench_sign bench_internal bench_ecmult bench_verify_SOURCES = src/bench_verify.c bench_verify_LDADD = libsecp256k1.la $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) +# SECP_TEST_INCLUDES are only used here for CRYPTO_CPPFLAGS +bench_verify_CPPFLAGS = -DSECP256K1_BUILD $(SECP_TEST_INCLUDES) bench_sign_SOURCES = src/bench_sign.c bench_sign_LDADD = libsecp256k1.la $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) bench_internal_SOURCES = src/bench_internal.c @@ -99,6 +95,12 @@ if USE_TESTS noinst_PROGRAMS += tests tests_SOURCES = src/tests.c tests_CPPFLAGS = -DSECP256K1_BUILD -I$(top_srcdir)/src -I$(top_srcdir)/include $(SECP_INCLUDES) $(SECP_TEST_INCLUDES) +if VALGRIND_ENABLED +tests_CPPFLAGS += -DVALGRIND +noinst_PROGRAMS += valgrind_ctime_test +valgrind_ctime_test_SOURCES = src/valgrind_ctime_test.c +valgrind_ctime_test_LDADD = libsecp256k1.la $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) +endif if !ENABLE_COVERAGE tests_CPPFLAGS += -DVERIFY endif @@ -119,42 +121,12 @@ exhaustive_tests_LDFLAGS = -static TESTS += exhaustive_tests endif -JAVAROOT=src/java -JAVAORG=org/bitcoin -JAVA_GUAVA=$(srcdir)/$(JAVAROOT)/guava/guava-18.0.jar -CLASSPATH_ENV=CLASSPATH=$(JAVA_GUAVA) -JAVA_FILES= \ - $(JAVAROOT)/$(JAVAORG)/NativeSecp256k1.java \ - $(JAVAROOT)/$(JAVAORG)/NativeSecp256k1Test.java \ - $(JAVAROOT)/$(JAVAORG)/NativeSecp256k1Util.java \ - $(JAVAROOT)/$(JAVAORG)/Secp256k1Context.java - -if USE_JNI - -$(JAVA_GUAVA): - @echo Guava is missing. Fetch it via: \ - wget https://search.maven.org/remotecontent?filepath=com/google/guava/guava/18.0/guava-18.0.jar -O $(@) - @false - -.stamp-java: $(JAVA_FILES) - @echo Compiling $^ - $(AM_V_at)$(CLASSPATH_ENV) javac $^ - @touch $@ - -if USE_TESTS - -check-java: libsecp256k1.la $(JAVA_GUAVA) .stamp-java - $(AM_V_at)java -Djava.library.path="./:./src:./src/.libs:.libs/" -cp "$(JAVA_GUAVA):$(JAVAROOT)" $(JAVAORG)/NativeSecp256k1Test - -endif -endif - if USE_ECMULT_STATIC_PRECOMPUTATION -CPPFLAGS_FOR_BUILD +=-I$(top_srcdir) +CPPFLAGS_FOR_BUILD +=-I$(top_srcdir) -I$(builddir)/src gen_context_OBJECTS = gen_context.o gen_context_BIN = gen_context$(BUILD_EXEEXT) -gen_%.o: src/gen_%.c +gen_%.o: src/gen_%.c src/libsecp256k1-config.h $(CC_FOR_BUILD) $(CPPFLAGS_FOR_BUILD) $(CFLAGS_FOR_BUILD) -c $< -o $@ $(gen_context_BIN): $(gen_context_OBJECTS) @@ -168,10 +140,10 @@ $(bench_ecmult_OBJECTS): src/ecmult_static_context.h src/ecmult_static_context.h: $(gen_context_BIN) ./$(gen_context_BIN) -CLEANFILES = $(gen_context_BIN) src/ecmult_static_context.h $(JAVAROOT)/$(JAVAORG)/*.class .stamp-java +CLEANFILES = $(gen_context_BIN) src/ecmult_static_context.h endif -EXTRA_DIST = autogen.sh src/gen_context.c src/basic-config.h $(JAVA_FILES) +EXTRA_DIST = autogen.sh src/gen_context.c src/basic-config.h if ENABLE_MODULE_ECDH include src/modules/ecdh/Makefile.am.include diff --git a/src/secp256k1/README.md b/src/secp256k1/README.md index 8cd344ea81..434178b372 100644 --- a/src/secp256k1/README.md +++ b/src/secp256k1/README.md @@ -3,17 +3,22 @@ libsecp256k1 [](https://travis-ci.org/bitcoin-core/secp256k1) -Optimized C library for EC operations on curve secp256k1. +Optimized C library for ECDSA signatures and secret/public key operations on curve secp256k1. -This library is a work in progress and is being used to research best practices. Use at your own risk. +This library is intended to be the highest quality publicly available library for cryptography on the secp256k1 curve. However, the primary focus of its development has been for usage in the Bitcoin system and usage unlike Bitcoin's may be less well tested, verified, or suffer from a less well thought out interface. Correct usage requires some care and consideration that the library is fit for your application's purpose. Features: * secp256k1 ECDSA signing/verification and key generation. -* Adding/multiplying private/public keys. -* Serialization/parsing of private keys, public keys, signatures. -* Constant time, constant memory access signing and pubkey generation. -* Derandomized DSA (via RFC6979 or with a caller provided function.) +* Additive and multiplicative tweaking of secret/public keys. +* Serialization/parsing of secret keys, public keys, signatures. +* Constant time, constant memory access signing and public key generation. +* Derandomized ECDSA (via RFC6979 or with a caller provided function.) * Very efficient implementation. +* Suitable for embedded systems. +* Optional module for public key recovery. +* Optional module for ECDH key exchange (experimental). + +Experimental features have not received enough scrutiny to satisfy the standard of quality of this library but are made available for testing and review by the community. The APIs of these features should not be considered stable. Implementation details ---------------------- @@ -23,11 +28,12 @@ Implementation details * Extensive testing infrastructure. * Structured to facilitate review and analysis. * Intended to be portable to any system with a C89 compiler and uint64_t support. + * No use of floating types. * Expose only higher level interfaces to minimize the API surface and improve application security. ("Be difficult to use insecurely.") * Field operations * Optimized implementation of arithmetic modulo the curve's field size (2^256 - 0x1000003D1). * Using 5 52-bit limbs (including hand-optimized assembly for x86_64, by Diederik Huys). - * Using 10 26-bit limbs. + * Using 10 26-bit limbs (including hand-optimized assembly for 32-bit ARM, by Wladimir J. van der Laan). * Field inverses and square roots using a sliding window over blocks of 1s (by Peter Dettman). * Scalar operations * Optimized implementation without data-dependent branches of arithmetic modulo the curve's order. @@ -45,9 +51,11 @@ Implementation details * Optionally (off by default) use secp256k1's efficiently-computable endomorphism to split the P multiplicand into 2 half-sized ones. * Point multiplication for signing * Use a precomputed table of multiples of powers of 16 multiplied with the generator, so general multiplication becomes a series of additions. - * Access the table with branch-free conditional moves so memory access is uniform. - * No data-dependent branches - * The precomputed tables add and eventually subtract points for which no known scalar (private key) is known, preventing even an attacker with control over the private key used to control the data internally. + * Intended to be completely free of timing sidechannels for secret-key operations (on reasonable hardware/toolchains) + * Access the table with branch-free conditional moves so memory access is uniform. + * No data-dependent branches + * Optional runtime blinding which attempts to frustrate differential power analysis. + * The precomputed tables add and eventually subtract points for which no known scalar (secret key) is known, preventing even an attacker with control over the secret key used to control the data internally. Build steps ----------- @@ -57,5 +65,40 @@ libsecp256k1 is built using autotools: $ ./autogen.sh $ ./configure $ make - $ ./tests + $ make check $ sudo make install # optional + +Exhaustive tests +----------- + + $ ./exhaustive_tests + +With valgrind, you might need to increase the max stack size: + + $ valgrind --max-stackframe=2500000 ./exhaustive_tests + +Test coverage +----------- + +This library aims to have full coverage of the reachable lines and branches. + +To create a test coverage report, configure with `--enable-coverage` (use of GCC is necessary): + + $ ./configure --enable-coverage + +Run the tests: + + $ make check + +To create a report, `gcovr` is recommended, as it includes branch coverage reporting: + + $ gcovr --exclude 'src/bench*' --print-summary + +To create a HTML report with coloured and annotated source code: + + $ gcovr --exclude 'src/bench*' --html --html-details -o coverage.html + +Reporting a vulnerability +------------ + +See [SECURITY.md](SECURITY.md) diff --git a/src/secp256k1/SECURITY.md b/src/secp256k1/SECURITY.md new file mode 100644 index 0000000000..0e4d588030 --- /dev/null +++ b/src/secp256k1/SECURITY.md @@ -0,0 +1,15 @@ +# Security Policy + +## Reporting a Vulnerability + +To report security issues send an email to secp256k1-security@bitcoincore.org (not for support). + +The following keys may be used to communicate sensitive information to developers: + +| Name | Fingerprint | +|------|-------------| +| Pieter Wuille | 133E AC17 9436 F14A 5CF1 B794 860F EB80 4E66 9320 | +| Andrew Poelstra | 699A 63EF C17A D3A9 A34C FFC0 7AD0 A91C 40BD 0091 | +| Tim Ruffing | 09E0 3F87 1092 E40E 106E 902B 33BC 86AB 80FF 5516 | + +You can import a key by running the following command with that individual’s fingerprint: `gpg --recv-keys "<fingerprint>"` Ensure that you put quotes around fingerprints containing spaces. diff --git a/src/secp256k1/build-aux/m4/ax_jni_include_dir.m4 b/src/secp256k1/build-aux/m4/ax_jni_include_dir.m4 deleted file mode 100644 index cdc78d87d4..0000000000 --- a/src/secp256k1/build-aux/m4/ax_jni_include_dir.m4 +++ /dev/null @@ -1,145 +0,0 @@ -# =========================================================================== -# https://www.gnu.org/software/autoconf-archive/ax_jni_include_dir.html -# =========================================================================== -# -# SYNOPSIS -# -# AX_JNI_INCLUDE_DIR -# -# DESCRIPTION -# -# AX_JNI_INCLUDE_DIR finds include directories needed for compiling -# programs using the JNI interface. -# -# JNI include directories are usually in the Java distribution. This is -# deduced from the value of $JAVA_HOME, $JAVAC, or the path to "javac", in -# that order. When this macro completes, a list of directories is left in -# the variable JNI_INCLUDE_DIRS. -# -# Example usage follows: -# -# AX_JNI_INCLUDE_DIR -# -# for JNI_INCLUDE_DIR in $JNI_INCLUDE_DIRS -# do -# CPPFLAGS="$CPPFLAGS -I$JNI_INCLUDE_DIR" -# done -# -# If you want to force a specific compiler: -# -# - at the configure.in level, set JAVAC=yourcompiler before calling -# AX_JNI_INCLUDE_DIR -# -# - at the configure level, setenv JAVAC -# -# Note: This macro can work with the autoconf M4 macros for Java programs. -# This particular macro is not part of the original set of macros. -# -# LICENSE -# -# Copyright (c) 2008 Don Anderson <dda@sleepycat.com> -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. This file is offered as-is, without any -# warranty. - -#serial 14 - -AU_ALIAS([AC_JNI_INCLUDE_DIR], [AX_JNI_INCLUDE_DIR]) -AC_DEFUN([AX_JNI_INCLUDE_DIR],[ - -JNI_INCLUDE_DIRS="" - -if test "x$JAVA_HOME" != x; then - _JTOPDIR="$JAVA_HOME" -else - if test "x$JAVAC" = x; then - JAVAC=javac - fi - AC_PATH_PROG([_ACJNI_JAVAC], [$JAVAC], [no]) - if test "x$_ACJNI_JAVAC" = xno; then - AC_MSG_WARN([cannot find JDK; try setting \$JAVAC or \$JAVA_HOME]) - fi - _ACJNI_FOLLOW_SYMLINKS("$_ACJNI_JAVAC") - _JTOPDIR=`echo "$_ACJNI_FOLLOWED" | sed -e 's://*:/:g' -e 's:/[[^/]]*$::'` -fi - -case "$host_os" in - darwin*) # Apple Java headers are inside the Xcode bundle. - macos_version=$(sw_vers -productVersion | sed -n -e 's/^@<:@0-9@:>@*.\(@<:@0-9@:>@*\).@<:@0-9@:>@*/\1/p') - if @<:@ "$macos_version" -gt "7" @:>@; then - _JTOPDIR="$(xcrun --show-sdk-path)/System/Library/Frameworks/JavaVM.framework" - _JINC="$_JTOPDIR/Headers" - else - _JTOPDIR="/System/Library/Frameworks/JavaVM.framework" - _JINC="$_JTOPDIR/Headers" - fi - ;; - *) _JINC="$_JTOPDIR/include";; -esac -_AS_ECHO_LOG([_JTOPDIR=$_JTOPDIR]) -_AS_ECHO_LOG([_JINC=$_JINC]) - -# On Mac OS X 10.6.4, jni.h is a symlink: -# /System/Library/Frameworks/JavaVM.framework/Versions/Current/Headers/jni.h -# -> ../../CurrentJDK/Headers/jni.h. -AC_CACHE_CHECK(jni headers, ac_cv_jni_header_path, -[ - if test -f "$_JINC/jni.h"; then - ac_cv_jni_header_path="$_JINC" - JNI_INCLUDE_DIRS="$JNI_INCLUDE_DIRS $ac_cv_jni_header_path" - else - _JTOPDIR=`echo "$_JTOPDIR" | sed -e 's:/[[^/]]*$::'` - if test -f "$_JTOPDIR/include/jni.h"; then - ac_cv_jni_header_path="$_JTOPDIR/include" - JNI_INCLUDE_DIRS="$JNI_INCLUDE_DIRS $ac_cv_jni_header_path" - else - ac_cv_jni_header_path=none - fi - fi -]) - -# get the likely subdirectories for system specific java includes -case "$host_os" in -bsdi*) _JNI_INC_SUBDIRS="bsdos";; -freebsd*) _JNI_INC_SUBDIRS="freebsd";; -darwin*) _JNI_INC_SUBDIRS="darwin";; -linux*) _JNI_INC_SUBDIRS="linux genunix";; -osf*) _JNI_INC_SUBDIRS="alpha";; -solaris*) _JNI_INC_SUBDIRS="solaris";; -mingw*) _JNI_INC_SUBDIRS="win32";; -cygwin*) _JNI_INC_SUBDIRS="win32";; -*) _JNI_INC_SUBDIRS="genunix";; -esac - -if test "x$ac_cv_jni_header_path" != "xnone"; then - # add any subdirectories that are present - for JINCSUBDIR in $_JNI_INC_SUBDIRS - do - if test -d "$_JTOPDIR/include/$JINCSUBDIR"; then - JNI_INCLUDE_DIRS="$JNI_INCLUDE_DIRS $_JTOPDIR/include/$JINCSUBDIR" - fi - done -fi -]) - -# _ACJNI_FOLLOW_SYMLINKS <path> -# Follows symbolic links on <path>, -# finally setting variable _ACJNI_FOLLOWED -# ---------------------------------------- -AC_DEFUN([_ACJNI_FOLLOW_SYMLINKS],[ -# find the include directory relative to the javac executable -_cur="$1" -while ls -ld "$_cur" 2>/dev/null | grep " -> " >/dev/null; do - AC_MSG_CHECKING([symlink for $_cur]) - _slink=`ls -ld "$_cur" | sed 's/.* -> //'` - case "$_slink" in - /*) _cur="$_slink";; - # 'X' avoids triggering unwanted echo options. - *) _cur=`echo "X$_cur" | sed -e 's/^X//' -e 's:[[^/]]*$::'`"$_slink";; - esac - AC_MSG_RESULT([$_cur]) -done -_ACJNI_FOLLOWED="$_cur" -])# _ACJNI diff --git a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 index 3b3975cbdd..1b2b71e6ab 100644 --- a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 +++ b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 @@ -38,6 +38,8 @@ AC_DEFUN([SECP_OPENSSL_CHECK],[ fi if test x"$has_libcrypto" = x"yes" && test x"$has_openssl_ec" = x; then AC_MSG_CHECKING(for EC functions in libcrypto) + CPPFLAGS_TEMP="$CPPFLAGS" + CPPFLAGS="$CRYPTO_CPPFLAGS $CPPFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <openssl/ec.h> #include <openssl/ecdsa.h> @@ -51,6 +53,7 @@ if test x"$has_libcrypto" = x"yes" && test x"$has_openssl_ec" = x; then ECDSA_SIG_free(sig_openssl); ]])],[has_openssl_ec=yes],[has_openssl_ec=no]) AC_MSG_RESULT([$has_openssl_ec]) + CPPFLAGS="$CPPFLAGS_TEMP" fi ]) diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index 3b7a328c8a..6021b760b5 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -7,6 +7,11 @@ AH_TOP([#ifndef LIBSECP256K1_CONFIG_H]) AH_TOP([#define LIBSECP256K1_CONFIG_H]) AH_BOTTOM([#endif /*LIBSECP256K1_CONFIG_H*/]) AM_INIT_AUTOMAKE([foreign subdir-objects]) + +# Set -g if CFLAGS are not already set, which matches the default autoconf +# behavior (see PROG_CC in the Autoconf manual) with the exception that we don't +# set -O2 here because we set it in any case (see further down). +: ${CFLAGS="-g"} LT_INIT dnl make the compilation flags quiet unless V=1 is used @@ -19,10 +24,6 @@ AC_PATH_TOOL(RANLIB, ranlib) AC_PATH_TOOL(STRIP, strip) AX_PROG_CC_FOR_BUILD -if test "x$CFLAGS" = "x"; then - CFLAGS="-g" -fi - AM_PROG_CC_C_O AC_PROG_CC_C89 @@ -45,6 +46,7 @@ case $host_os in if test x$openssl_prefix != x; then PKG_CONFIG_PATH="$openssl_prefix/lib/pkgconfig:$PKG_CONFIG_PATH" export PKG_CONFIG_PATH + CRYPTO_CPPFLAGS="-I$openssl_prefix/include" fi if test x$gmp_prefix != x; then GMP_CPPFLAGS="-I$gmp_prefix/include" @@ -63,11 +65,11 @@ case $host_os in ;; esac -CFLAGS="$CFLAGS -W" +CFLAGS="-W $CFLAGS" warn_CFLAGS="-std=c89 -pedantic -Wall -Wextra -Wcast-align -Wnested-externs -Wshadow -Wstrict-prototypes -Wno-unused-function -Wno-long-long -Wno-overlength-strings" saved_CFLAGS="$CFLAGS" -CFLAGS="$CFLAGS $warn_CFLAGS" +CFLAGS="$warn_CFLAGS $CFLAGS" AC_MSG_CHECKING([if ${CC} supports ${warn_CFLAGS}]) AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])], [ AC_MSG_RESULT([yes]) ], @@ -76,7 +78,7 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])], ]) saved_CFLAGS="$CFLAGS" -CFLAGS="$CFLAGS -fvisibility=hidden" +CFLAGS="-fvisibility=hidden $CFLAGS" AC_MSG_CHECKING([if ${CC} supports -fvisibility=hidden]) AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])], [ AC_MSG_RESULT([yes]) ], @@ -85,42 +87,42 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])], ]) AC_ARG_ENABLE(benchmark, - AS_HELP_STRING([--enable-benchmark],[compile benchmark (default is yes)]), + AS_HELP_STRING([--enable-benchmark],[compile benchmark [default=yes]]), [use_benchmark=$enableval], [use_benchmark=yes]) AC_ARG_ENABLE(coverage, - AS_HELP_STRING([--enable-coverage],[enable compiler flags to support kcov coverage analysis]), + AS_HELP_STRING([--enable-coverage],[enable compiler flags to support kcov coverage analysis [default=no]]), [enable_coverage=$enableval], [enable_coverage=no]) AC_ARG_ENABLE(tests, - AS_HELP_STRING([--enable-tests],[compile tests (default is yes)]), + AS_HELP_STRING([--enable-tests],[compile tests [default=yes]]), [use_tests=$enableval], [use_tests=yes]) AC_ARG_ENABLE(openssl_tests, - AS_HELP_STRING([--enable-openssl-tests],[enable OpenSSL tests, if OpenSSL is available (default is auto)]), + AS_HELP_STRING([--enable-openssl-tests],[enable OpenSSL tests [default=auto]]), [enable_openssl_tests=$enableval], [enable_openssl_tests=auto]) AC_ARG_ENABLE(experimental, - AS_HELP_STRING([--enable-experimental],[allow experimental configure options (default is no)]), + AS_HELP_STRING([--enable-experimental],[allow experimental configure options [default=no]]), [use_experimental=$enableval], [use_experimental=no]) AC_ARG_ENABLE(exhaustive_tests, - AS_HELP_STRING([--enable-exhaustive-tests],[compile exhaustive tests (default is yes)]), + AS_HELP_STRING([--enable-exhaustive-tests],[compile exhaustive tests [default=yes]]), [use_exhaustive_tests=$enableval], [use_exhaustive_tests=yes]) AC_ARG_ENABLE(endomorphism, - AS_HELP_STRING([--enable-endomorphism],[enable endomorphism (default is no)]), + AS_HELP_STRING([--enable-endomorphism],[enable endomorphism [default=no]]), [use_endomorphism=$enableval], [use_endomorphism=no]) AC_ARG_ENABLE(ecmult_static_precomputation, - AS_HELP_STRING([--enable-ecmult-static-precomputation],[enable precomputed ecmult table for signing (default is yes)]), + AS_HELP_STRING([--enable-ecmult-static-precomputation],[enable precomputed ecmult table for signing [default=auto]]), [use_ecmult_static_precomputation=$enableval], [use_ecmult_static_precomputation=auto]) @@ -130,35 +132,55 @@ AC_ARG_ENABLE(module_ecdh, [enable_module_ecdh=no]) AC_ARG_ENABLE(module_recovery, - AS_HELP_STRING([--enable-module-recovery],[enable ECDSA pubkey recovery module (default is no)]), + AS_HELP_STRING([--enable-module-recovery],[enable ECDSA pubkey recovery module [default=no]]), [enable_module_recovery=$enableval], [enable_module_recovery=no]) -AC_ARG_ENABLE(jni, - AS_HELP_STRING([--enable-jni],[enable libsecp256k1_jni (default is no)]), - [use_jni=$enableval], - [use_jni=no]) +AC_ARG_ENABLE(external_default_callbacks, + AS_HELP_STRING([--enable-external-default-callbacks],[enable external default callback functions [default=no]]), + [use_external_default_callbacks=$enableval], + [use_external_default_callbacks=no]) AC_ARG_WITH([field], [AS_HELP_STRING([--with-field=64bit|32bit|auto], -[Specify Field Implementation. Default is auto])],[req_field=$withval], [req_field=auto]) +[finite field implementation to use [default=auto]])],[req_field=$withval], [req_field=auto]) AC_ARG_WITH([bignum], [AS_HELP_STRING([--with-bignum=gmp|no|auto], -[Specify Bignum Implementation. Default is auto])],[req_bignum=$withval], [req_bignum=auto]) +[bignum implementation to use [default=auto]])],[req_bignum=$withval], [req_bignum=auto]) AC_ARG_WITH([scalar], [AS_HELP_STRING([--with-scalar=64bit|32bit|auto], -[Specify scalar implementation. Default is auto])],[req_scalar=$withval], [req_scalar=auto]) - -AC_ARG_WITH([asm], [AS_HELP_STRING([--with-asm=x86_64|arm|no|auto] -[Specify assembly optimizations to use. Default is auto (experimental: arm)])],[req_asm=$withval], [req_asm=auto]) +[scalar implementation to use [default=auto]])],[req_scalar=$withval], [req_scalar=auto]) + +AC_ARG_WITH([asm], [AS_HELP_STRING([--with-asm=x86_64|arm|no|auto], +[assembly optimizations to use (experimental: arm) [default=auto]])],[req_asm=$withval], [req_asm=auto]) + +AC_ARG_WITH([ecmult-window], [AS_HELP_STRING([--with-ecmult-window=SIZE|auto], +[window size for ecmult precomputation for verification, specified as integer in range [2..24].] +[Larger values result in possibly better performance at the cost of an exponentially larger precomputed table.] +[The table will store 2^(SIZE-2) * 64 bytes of data but can be larger in memory due to platform-specific padding and alignment.] +[If the endomorphism optimization is enabled, two tables of this size are used instead of only one.] +["auto" is a reasonable setting for desktop machines (currently 15). [default=auto]] +)], +[req_ecmult_window=$withval], [req_ecmult_window=auto]) + +AC_ARG_WITH([ecmult-gen-precision], [AS_HELP_STRING([--with-ecmult-gen-precision=2|4|8|auto], +[Precision bits to tune the precomputed table size for signing.] +[The size of the table is 32kB for 2 bits, 64kB for 4 bits, 512kB for 8 bits of precision.] +[A larger table size usually results in possible faster signing.] +["auto" is a reasonable setting for desktop machines (currently 4). [default=auto]] +)], +[req_ecmult_gen_precision=$withval], [req_ecmult_gen_precision=auto]) AC_CHECK_TYPES([__int128]) +AC_CHECK_HEADER([valgrind/memcheck.h], [enable_valgrind=yes], [enable_valgrind=no], []) +AM_CONDITIONAL([VALGRIND_ENABLED],[test "$enable_valgrind" = "yes"]) + if test x"$enable_coverage" = x"yes"; then AC_DEFINE(COVERAGE, 1, [Define this symbol to compile out all VERIFY code]) - CFLAGS="$CFLAGS -O0 --coverage" - LDFLAGS="--coverage" + CFLAGS="-O0 --coverage $CFLAGS" + LDFLAGS="--coverage $LDFLAGS" else - CFLAGS="$CFLAGS -O3" + CFLAGS="-O2 $CFLAGS" fi if test x"$use_ecmult_static_precomputation" != x"no"; then @@ -176,7 +198,7 @@ if test x"$use_ecmult_static_precomputation" != x"no"; then warn_CFLAGS_FOR_BUILD="-Wall -Wextra -Wno-unused-function" saved_CFLAGS="$CFLAGS" - CFLAGS="$CFLAGS $warn_CFLAGS_FOR_BUILD" + CFLAGS="$warn_CFLAGS_FOR_BUILD $CFLAGS" AC_MSG_CHECKING([if native ${CC_FOR_BUILD} supports ${warn_CFLAGS_FOR_BUILD}]) AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])], [ AC_MSG_RESULT([yes]) ], @@ -188,7 +210,7 @@ if test x"$use_ecmult_static_precomputation" != x"no"; then AC_RUN_IFELSE( [AC_LANG_PROGRAM([], [])], [working_native_cc=yes], - [working_native_cc=no],[dnl]) + [working_native_cc=no],[:]) CFLAGS_FOR_BUILD="$CFLAGS" @@ -387,12 +409,50 @@ case $set_scalar in ;; esac +#set ecmult window size +if test x"$req_ecmult_window" = x"auto"; then + set_ecmult_window=15 +else + set_ecmult_window=$req_ecmult_window +fi + +error_window_size=['window size for ecmult precomputation not an integer in range [2..24] or "auto"'] +case $set_ecmult_window in +''|*[[!0-9]]*) + # no valid integer + AC_MSG_ERROR($error_window_size) + ;; +*) + if test "$set_ecmult_window" -lt 2 -o "$set_ecmult_window" -gt 24 ; then + # not in range + AC_MSG_ERROR($error_window_size) + fi + AC_DEFINE_UNQUOTED(ECMULT_WINDOW_SIZE, $set_ecmult_window, [Set window size for ecmult precomputation]) + ;; +esac + +#set ecmult gen precision +if test x"$req_ecmult_gen_precision" = x"auto"; then + set_ecmult_gen_precision=4 +else + set_ecmult_gen_precision=$req_ecmult_gen_precision +fi + +case $set_ecmult_gen_precision in +2|4|8) + AC_DEFINE_UNQUOTED(ECMULT_GEN_PREC_BITS, $set_ecmult_gen_precision, [Set ecmult gen precision bits]) + ;; +*) + AC_MSG_ERROR(['ecmult gen precision not 2, 4, 8 or "auto"']) + ;; +esac + if test x"$use_tests" = x"yes"; then SECP_OPENSSL_CHECK if test x"$has_openssl_ec" = x"yes"; then if test x"$enable_openssl_tests" != x"no"; then AC_DEFINE(ENABLE_OPENSSL_TESTS, 1, [Define this symbol if OpenSSL EC functions are available]) - SECP_TEST_INCLUDES="$SSL_CFLAGS $CRYPTO_CFLAGS" + SECP_TEST_INCLUDES="$SSL_CFLAGS $CRYPTO_CFLAGS $CRYPTO_CPPFLAGS" SECP_TEST_LIBS="$CRYPTO_LIBS" case $host in @@ -412,29 +472,6 @@ else fi fi -if test x"$use_jni" != x"no"; then - AX_JNI_INCLUDE_DIR - have_jni_dependencies=yes - if test x"$enable_module_ecdh" = x"no"; then - have_jni_dependencies=no - fi - if test "x$JNI_INCLUDE_DIRS" = "x"; then - have_jni_dependencies=no - fi - if test "x$have_jni_dependencies" = "xno"; then - if test x"$use_jni" = x"yes"; then - AC_MSG_ERROR([jni support explicitly requested but headers/dependencies were not found. Enable ECDH and try again.]) - fi - AC_MSG_WARN([jni headers/dependencies not found. jni support disabled]) - use_jni=no - else - use_jni=yes - for JNI_INCLUDE_DIR in $JNI_INCLUDE_DIRS; do - JNI_INCLUDES="$JNI_INCLUDES -I$JNI_INCLUDE_DIR" - done - fi -fi - if test x"$set_bignum" = x"gmp"; then SECP_LIBS="$SECP_LIBS $GMP_LIBS" SECP_INCLUDES="$SECP_INCLUDES $GMP_CPPFLAGS" @@ -462,6 +499,10 @@ if test x"$use_external_asm" = x"yes"; then AC_DEFINE(USE_EXTERNAL_ASM, 1, [Define this symbol if an external (non-inline) assembly implementation is used]) fi +if test x"$use_external_default_callbacks" = x"yes"; then + AC_DEFINE(USE_EXTERNAL_DEFAULT_CALLBACKS, 1, [Define this symbol if an external implementation of the default callbacks is used]) +fi + if test x"$enable_experimental" = x"yes"; then AC_MSG_NOTICE([******]) AC_MSG_NOTICE([WARNING: experimental build]) @@ -479,7 +520,6 @@ fi AC_CONFIG_HEADERS([src/libsecp256k1-config.h]) AC_CONFIG_FILES([Makefile libsecp256k1.pc]) -AC_SUBST(JNI_INCLUDES) AC_SUBST(SECP_INCLUDES) AC_SUBST(SECP_LIBS) AC_SUBST(SECP_TEST_LIBS) @@ -491,7 +531,6 @@ AM_CONDITIONAL([USE_BENCHMARK], [test x"$use_benchmark" = x"yes"]) AM_CONDITIONAL([USE_ECMULT_STATIC_PRECOMPUTATION], [test x"$set_precomp" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_ECDH], [test x"$enable_module_ecdh" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_RECOVERY], [test x"$enable_module_recovery" = x"yes"]) -AM_CONDITIONAL([USE_JNI], [test x"$use_jni" = x"yes"]) AM_CONDITIONAL([USE_EXTERNAL_ASM], [test x"$use_external_asm" = x"yes"]) AM_CONDITIONAL([USE_ASM_ARM], [test x"$set_asm" = x"arm"]) @@ -504,21 +543,24 @@ AC_OUTPUT echo echo "Build Options:" -echo " with endomorphism = $use_endomorphism" -echo " with ecmult precomp = $set_precomp" -echo " with jni = $use_jni" -echo " with benchmarks = $use_benchmark" -echo " with coverage = $enable_coverage" -echo " module ecdh = $enable_module_ecdh" -echo " module recovery = $enable_module_recovery" +echo " with endomorphism = $use_endomorphism" +echo " with ecmult precomp = $set_precomp" +echo " with external callbacks = $use_external_default_callbacks" +echo " with benchmarks = $use_benchmark" +echo " with coverage = $enable_coverage" +echo " module ecdh = $enable_module_ecdh" +echo " module recovery = $enable_module_recovery" echo -echo " asm = $set_asm" -echo " bignum = $set_bignum" -echo " field = $set_field" -echo " scalar = $set_scalar" +echo " asm = $set_asm" +echo " bignum = $set_bignum" +echo " field = $set_field" +echo " scalar = $set_scalar" +echo " ecmult window size = $set_ecmult_window" +echo " ecmult gen prec. bits = $set_ecmult_gen_precision" echo -echo " CC = $CC" -echo " CFLAGS = $CFLAGS" -echo " CPPFLAGS = $CPPFLAGS" -echo " LDFLAGS = $LDFLAGS" +echo " valgrind = $enable_valgrind" +echo " CC = $CC" +echo " CFLAGS = $CFLAGS" +echo " CPPFLAGS = $CPPFLAGS" +echo " LDFLAGS = $LDFLAGS" echo diff --git a/src/secp256k1/contrib/lax_der_parsing.c b/src/secp256k1/contrib/lax_der_parsing.c index 5b141a9948..e177a0562d 100644 --- a/src/secp256k1/contrib/lax_der_parsing.c +++ b/src/secp256k1/contrib/lax_der_parsing.c @@ -32,7 +32,7 @@ int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_ lenbyte = input[pos++]; if (lenbyte & 0x80) { lenbyte -= 0x80; - if (pos + lenbyte > inputlen) { + if (lenbyte > inputlen - pos) { return 0; } pos += lenbyte; @@ -51,7 +51,7 @@ int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_ lenbyte = input[pos++]; if (lenbyte & 0x80) { lenbyte -= 0x80; - if (pos + lenbyte > inputlen) { + if (lenbyte > inputlen - pos) { return 0; } while (lenbyte > 0 && input[pos] == 0) { @@ -89,7 +89,7 @@ int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_ lenbyte = input[pos++]; if (lenbyte & 0x80) { lenbyte -= 0x80; - if (pos + lenbyte > inputlen) { + if (lenbyte > inputlen - pos) { return 0; } while (lenbyte > 0 && input[pos] == 0) { diff --git a/src/secp256k1/contrib/travis.sh b/src/secp256k1/contrib/travis.sh new file mode 100755 index 0000000000..3909d16a27 --- /dev/null +++ b/src/secp256k1/contrib/travis.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +set -e +set -x + +if [ -n "$HOST" ] +then + export USE_HOST="--host=$HOST" +fi +if [ "$HOST" = "i686-linux-gnu" ] +then + export CC="$CC -m32" +fi +if [ "$TRAVIS_OS_NAME" = "osx" ] && [ "$TRAVIS_COMPILER" = "gcc" ] +then + export CC="gcc-9" +fi + +./configure \ + --enable-experimental="$EXPERIMENTAL" --enable-endomorphism="$ENDOMORPHISM" \ + --with-field="$FIELD" --with-bignum="$BIGNUM" --with-asm="$ASM" --with-scalar="$SCALAR" \ + --enable-ecmult-static-precomputation="$STATICPRECOMPUTATION" --with-ecmult-gen-precision="$ECMULTGENPRECISION" \ + --enable-module-ecdh="$ECDH" --enable-module-recovery="$RECOVERY" "$EXTRAFLAGS" "$USE_HOST" + +if [ -n "$BUILD" ] +then + make -j2 "$BUILD" +fi +if [ -n "$VALGRIND" ] +then + make -j2 + # the `--error-exitcode` is required to make the test fail if valgrind found errors, otherwise it'll return 0 (http://valgrind.org/docs/manual/manual-core.html) + valgrind --error-exitcode=42 ./tests 16 + valgrind --error-exitcode=42 ./exhaustive_tests +fi +if [ -n "$BENCH" ] +then + if [ -n "$VALGRIND" ] + then + # Using the local `libtool` because on macOS the system's libtool has nothing to do with GNU libtool + EXEC='./libtool --mode=execute valgrind --error-exitcode=42' + else + EXEC= + fi + # This limits the iterations in the benchmarks below to ITER(set in .travis.yml) iterations. + export SECP256K1_BENCH_ITERS="$ITERS" + { + $EXEC ./bench_ecmult + $EXEC ./bench_internal + $EXEC ./bench_sign + $EXEC ./bench_verify + } >> bench.log 2>&1 + if [ "$RECOVERY" = "yes" ] + then + $EXEC ./bench_recover >> bench.log 2>&1 + fi + if [ "$ECDH" = "yes" ] + then + $EXEC ./bench_ecdh >> bench.log 2>&1 + fi +fi +if [ -n "$CTIMETEST" ] +then + ./libtool --mode=execute valgrind --error-exitcode=42 ./valgrind_ctime_test > valgrind_ctime_test.log 2>&1 +fi diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index 43af09c330..2ba2dca388 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -14,7 +14,7 @@ extern "C" { * 2. Array lengths always immediately the follow the argument whose length * they describe, even if this violates rule 1. * 3. Within the OUT/OUTIN/IN groups, pointers to data that is typically generated - * later go first. This means: signatures, public nonces, private nonces, + * later go first. This means: signatures, public nonces, secret nonces, * messages, public keys, secret keys, tweaks. * 4. Arguments that are not data pointers go last, from more complex to less * complex: function pointers, algorithm names, messages, void pointers, @@ -33,9 +33,10 @@ extern "C" { * verification). * * A constructed context can safely be used from multiple threads - * simultaneously, but API call that take a non-const pointer to a context + * simultaneously, but API calls that take a non-const pointer to a context * need exclusive access to it. In particular this is the case for - * secp256k1_context_destroy and secp256k1_context_randomize. + * secp256k1_context_destroy, secp256k1_context_preallocated_destroy, + * and secp256k1_context_randomize. * * Regarding randomization, either do it once at creation time (in which case * you do not need any locking for the other calls), or use a read-write lock. @@ -161,14 +162,17 @@ typedef int (*secp256k1_nonce_function)( /** The higher bits contain the actual data. Do not use directly. */ #define SECP256K1_FLAGS_BIT_CONTEXT_VERIFY (1 << 8) #define SECP256K1_FLAGS_BIT_CONTEXT_SIGN (1 << 9) +#define SECP256K1_FLAGS_BIT_CONTEXT_DECLASSIFY (1 << 10) #define SECP256K1_FLAGS_BIT_COMPRESSION (1 << 8) -/** Flags to pass to secp256k1_context_create. */ +/** Flags to pass to secp256k1_context_create, secp256k1_context_preallocated_size, and + * secp256k1_context_preallocated_create. */ #define SECP256K1_CONTEXT_VERIFY (SECP256K1_FLAGS_TYPE_CONTEXT | SECP256K1_FLAGS_BIT_CONTEXT_VERIFY) #define SECP256K1_CONTEXT_SIGN (SECP256K1_FLAGS_TYPE_CONTEXT | SECP256K1_FLAGS_BIT_CONTEXT_SIGN) +#define SECP256K1_CONTEXT_DECLASSIFY (SECP256K1_FLAGS_TYPE_CONTEXT | SECP256K1_FLAGS_BIT_CONTEXT_DECLASSIFY) #define SECP256K1_CONTEXT_NONE (SECP256K1_FLAGS_TYPE_CONTEXT) -/** Flag to pass to secp256k1_ec_pubkey_serialize and secp256k1_ec_privkey_export. */ +/** Flag to pass to secp256k1_ec_pubkey_serialize. */ #define SECP256K1_EC_COMPRESSED (SECP256K1_FLAGS_TYPE_COMPRESSION | SECP256K1_FLAGS_BIT_COMPRESSION) #define SECP256K1_EC_UNCOMPRESSED (SECP256K1_FLAGS_TYPE_COMPRESSION) @@ -186,7 +190,11 @@ typedef int (*secp256k1_nonce_function)( */ SECP256K1_API extern const secp256k1_context *secp256k1_context_no_precomp; -/** Create a secp256k1 context object. +/** Create a secp256k1 context object (in dynamically allocated memory). + * + * This function uses malloc to allocate memory. It is guaranteed that malloc is + * called at most once for every call of this function. If you need to avoid dynamic + * memory allocation entirely, see the functions in secp256k1_preallocated.h. * * Returns: a newly created context object. * In: flags: which parts of the context to initialize. @@ -197,7 +205,11 @@ SECP256K1_API secp256k1_context* secp256k1_context_create( unsigned int flags ) SECP256K1_WARN_UNUSED_RESULT; -/** Copies a secp256k1 context object. +/** Copy a secp256k1 context object (into dynamically allocated memory). + * + * This function uses malloc to allocate memory. It is guaranteed that malloc is + * called at most once for every call of this function. If you need to avoid dynamic + * memory allocation entirely, see the functions in secp256k1_preallocated.h. * * Returns: a newly created context object. * Args: ctx: an existing context to copy (cannot be NULL) @@ -206,10 +218,18 @@ SECP256K1_API secp256k1_context* secp256k1_context_clone( const secp256k1_context* ctx ) SECP256K1_ARG_NONNULL(1) SECP256K1_WARN_UNUSED_RESULT; -/** Destroy a secp256k1 context object. +/** Destroy a secp256k1 context object (created in dynamically allocated memory). * * The context pointer may not be used afterwards. - * Args: ctx: an existing context to destroy (cannot be NULL) + * + * The context to destroy must have been created using secp256k1_context_create + * or secp256k1_context_clone. If the context has instead been created using + * secp256k1_context_preallocated_create or secp256k1_context_preallocated_clone, the + * behaviour is undefined. In that case, secp256k1_context_preallocated_destroy must + * be used instead. + * + * Args: ctx: an existing context to destroy, constructed using + * secp256k1_context_create or secp256k1_context_clone */ SECP256K1_API void secp256k1_context_destroy( secp256k1_context* ctx @@ -229,11 +249,28 @@ SECP256K1_API void secp256k1_context_destroy( * to cause a crash, though its return value and output arguments are * undefined. * + * When this function has not been called (or called with fn==NULL), then the + * default handler will be used. The library provides a default handler which + * writes the message to stderr and calls abort. This default handler can be + * replaced at link time if the preprocessor macro + * USE_EXTERNAL_DEFAULT_CALLBACKS is defined, which is the case if the build + * has been configured with --enable-external-default-callbacks. Then the + * following two symbols must be provided to link against: + * - void secp256k1_default_illegal_callback_fn(const char* message, void* data); + * - void secp256k1_default_error_callback_fn(const char* message, void* data); + * The library can call these default handlers even before a proper callback data + * pointer could have been set using secp256k1_context_set_illegal_callback or + * secp256k1_context_set_error_callback, e.g., when the creation of a context + * fails. In this case, the corresponding default handler will be called with + * the data pointer argument set to NULL. + * * Args: ctx: an existing context object (cannot be NULL) * In: fun: a pointer to a function to call when an illegal argument is - * passed to the API, taking a message and an opaque pointer - * (NULL restores a default handler that calls abort). + * passed to the API, taking a message and an opaque pointer. + * (NULL restores the default handler.) * data: the opaque pointer to pass to fun above. + * + * See also secp256k1_context_set_error_callback. */ SECP256K1_API void secp256k1_context_set_illegal_callback( secp256k1_context* ctx, @@ -253,9 +290,12 @@ SECP256K1_API void secp256k1_context_set_illegal_callback( * * Args: ctx: an existing context object (cannot be NULL) * In: fun: a pointer to a function to call when an internal error occurs, - * taking a message and an opaque pointer (NULL restores a default - * handler that calls abort). + * taking a message and an opaque pointer (NULL restores the + * default handler, see secp256k1_context_set_illegal_callback + * for details). * data: the opaque pointer to pass to fun above. + * + * See also secp256k1_context_set_illegal_callback. */ SECP256K1_API void secp256k1_context_set_error_callback( secp256k1_context* ctx, @@ -267,21 +307,24 @@ SECP256K1_API void secp256k1_context_set_error_callback( * * Returns: a newly created scratch space. * Args: ctx: an existing context object (cannot be NULL) - * In: max_size: maximum amount of memory to allocate + * In: size: amount of memory to be available as scratch space. Some extra + * (<100 bytes) will be allocated for extra accounting. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT secp256k1_scratch_space* secp256k1_scratch_space_create( const secp256k1_context* ctx, - size_t max_size + size_t size ) SECP256K1_ARG_NONNULL(1); /** Destroy a secp256k1 scratch space. * * The pointer may not be used afterwards. - * Args: scratch: space to destroy + * Args: ctx: a secp256k1 context object. + * scratch: space to destroy */ SECP256K1_API void secp256k1_scratch_space_destroy( + const secp256k1_context* ctx, secp256k1_scratch_space* scratch -); +) SECP256K1_ARG_NONNULL(1); /** Parse a variable-length public key into the pubkey object. * @@ -488,7 +531,7 @@ SECP256K1_API extern const secp256k1_nonce_function secp256k1_nonce_function_def /** Create an ECDSA signature. * * Returns: 1: signature created - * 0: the nonce generation function failed, or the private key was invalid. + * 0: the nonce generation function failed, or the secret key was invalid. * Args: ctx: pointer to a context object, initialized for signing (cannot be NULL) * Out: sig: pointer to an array where the signature will be placed (cannot be NULL) * In: msg32: the 32-byte message hash being signed (cannot be NULL) @@ -510,6 +553,11 @@ SECP256K1_API int secp256k1_ecdsa_sign( /** Verify an ECDSA secret key. * + * A secret key is valid if it is not 0 and less than the secp256k1 curve order + * when interpreted as an integer (most significant byte first). The + * probability of choosing a 32-byte string uniformly at random which is an + * invalid secret key is negligible. + * * Returns: 1: secret key is valid * 0: secret key is invalid * Args: ctx: pointer to a context object (cannot be NULL) @@ -526,7 +574,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_verify( * 0: secret was invalid, try again * Args: ctx: pointer to a context object, initialized for signing (cannot be NULL) * Out: pubkey: pointer to the created public key (cannot be NULL) - * In: seckey: pointer to a 32-byte private key (cannot be NULL) + * In: seckey: pointer to a 32-byte secret key (cannot be NULL) */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_create( const secp256k1_context* ctx, @@ -534,12 +582,24 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_create( const unsigned char *seckey ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); -/** Negates a private key in place. +/** Negates a secret key in place. * - * Returns: 1 always - * Args: ctx: pointer to a context object - * In/Out: seckey: pointer to the 32-byte private key to be negated (cannot be NULL) + * Returns: 0 if the given secret key is invalid according to + * secp256k1_ec_seckey_verify. 1 otherwise + * Args: ctx: pointer to a context object + * In/Out: seckey: pointer to the 32-byte secret key to be negated. If the + * secret key is invalid according to + * secp256k1_ec_seckey_verify, this function returns 0 and + * seckey will be set to some unspecified value. (cannot be + * NULL) */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_negate( + const secp256k1_context* ctx, + unsigned char *seckey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); + +/** Same as secp256k1_ec_seckey_negate, but DEPRECATED. Will be removed in + * future versions. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_negate( const secp256k1_context* ctx, unsigned char *seckey @@ -556,15 +616,29 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_negate( secp256k1_pubkey *pubkey ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); -/** Tweak a private key by adding tweak to it. - * Returns: 0 if the tweak was out of range (chance of around 1 in 2^128 for - * uniformly random 32-byte arrays, or if the resulting private key - * would be invalid (only when the tweak is the complement of the - * private key). 1 otherwise. - * Args: ctx: pointer to a context object (cannot be NULL). - * In/Out: seckey: pointer to a 32-byte private key. - * In: tweak: pointer to a 32-byte tweak. - */ +/** Tweak a secret key by adding tweak to it. + * + * Returns: 0 if the arguments are invalid or the resulting secret key would be + * invalid (only when the tweak is the negation of the secret key). 1 + * otherwise. + * Args: ctx: pointer to a context object (cannot be NULL). + * In/Out: seckey: pointer to a 32-byte secret key. If the secret key is + * invalid according to secp256k1_ec_seckey_verify, this + * function returns 0. seckey will be set to some unspecified + * value if this function returns 0. (cannot be NULL) + * In: tweak: pointer to a 32-byte tweak. If the tweak is invalid according to + * secp256k1_ec_seckey_verify, this function returns 0. For + * uniformly random 32-byte arrays the chance of being invalid + * is negligible (around 1 in 2^128) (cannot be NULL). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_tweak_add( + const secp256k1_context* ctx, + unsigned char *seckey, + const unsigned char *tweak +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Same as secp256k1_ec_seckey_tweak_add, but DEPRECATED. Will be removed in + * future versions. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_add( const secp256k1_context* ctx, unsigned char *seckey, @@ -572,14 +646,18 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_add( ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); /** Tweak a public key by adding tweak times the generator to it. - * Returns: 0 if the tweak was out of range (chance of around 1 in 2^128 for - * uniformly random 32-byte arrays, or if the resulting public key - * would be invalid (only when the tweak is the complement of the - * corresponding private key). 1 otherwise. - * Args: ctx: pointer to a context object initialized for validation + * + * Returns: 0 if the arguments are invalid or the resulting public key would be + * invalid (only when the tweak is the negation of the corresponding + * secret key). 1 otherwise. + * Args: ctx: pointer to a context object initialized for validation * (cannot be NULL). - * In/Out: pubkey: pointer to a public key object. - * In: tweak: pointer to a 32-byte tweak. + * In/Out: pubkey: pointer to a public key object. pubkey will be set to an + * invalid value if this function returns 0 (cannot be NULL). + * In: tweak: pointer to a 32-byte tweak. If the tweak is invalid according to + * secp256k1_ec_seckey_verify, this function returns 0. For + * uniformly random 32-byte arrays the chance of being invalid + * is negligible (around 1 in 2^128) (cannot be NULL). */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_add( const secp256k1_context* ctx, @@ -587,13 +665,27 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_add( const unsigned char *tweak ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); -/** Tweak a private key by multiplying it by a tweak. - * Returns: 0 if the tweak was out of range (chance of around 1 in 2^128 for - * uniformly random 32-byte arrays, or equal to zero. 1 otherwise. - * Args: ctx: pointer to a context object (cannot be NULL). - * In/Out: seckey: pointer to a 32-byte private key. - * In: tweak: pointer to a 32-byte tweak. +/** Tweak a secret key by multiplying it by a tweak. + * + * Returns: 0 if the arguments are invalid. 1 otherwise. + * Args: ctx: pointer to a context object (cannot be NULL). + * In/Out: seckey: pointer to a 32-byte secret key. If the secret key is + * invalid according to secp256k1_ec_seckey_verify, this + * function returns 0. seckey will be set to some unspecified + * value if this function returns 0. (cannot be NULL) + * In: tweak: pointer to a 32-byte tweak. If the tweak is invalid according to + * secp256k1_ec_seckey_verify, this function returns 0. For + * uniformly random 32-byte arrays the chance of being invalid + * is negligible (around 1 in 2^128) (cannot be NULL). */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_tweak_mul( + const secp256k1_context* ctx, + unsigned char *seckey, + const unsigned char *tweak +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Same as secp256k1_ec_seckey_tweak_mul, but DEPRECATED. Will be removed in + * future versions. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_mul( const secp256k1_context* ctx, unsigned char *seckey, @@ -601,12 +693,16 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_mul( ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); /** Tweak a public key by multiplying it by a tweak value. - * Returns: 0 if the tweak was out of range (chance of around 1 in 2^128 for - * uniformly random 32-byte arrays, or equal to zero. 1 otherwise. - * Args: ctx: pointer to a context object initialized for validation - * (cannot be NULL). - * In/Out: pubkey: pointer to a public key obkect. - * In: tweak: pointer to a 32-byte tweak. + * + * Returns: 0 if the arguments are invalid. 1 otherwise. + * Args: ctx: pointer to a context object initialized for validation + * (cannot be NULL). + * In/Out: pubkey: pointer to a public key object. pubkey will be set to an + * invalid value if this function returns 0 (cannot be NULL). + * In: tweak: pointer to a 32-byte tweak. If the tweak is invalid according to + * secp256k1_ec_seckey_verify, this function returns 0. For + * uniformly random 32-byte arrays the chance of being invalid + * is negligible (around 1 in 2^128) (cannot be NULL). */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_mul( const secp256k1_context* ctx, @@ -636,7 +732,8 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_mul( * contexts not initialized for signing; then it will have no effect and return 1. * * You should call this after secp256k1_context_create or - * secp256k1_context_clone, and may call this repeatedly afterwards. + * secp256k1_context_clone (and secp256k1_context_preallocated_create or + * secp256k1_context_clone, resp.), and you may call this repeatedly afterwards. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_context_randomize( secp256k1_context* ctx, @@ -644,6 +741,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_context_randomize( ) SECP256K1_ARG_NONNULL(1); /** Add a number of public keys together. + * * Returns: 1: the sum of the public keys is valid. * 0: the sum of the public keys is not valid. * Args: ctx: pointer to a context object diff --git a/src/secp256k1/include/secp256k1_ecdh.h b/src/secp256k1/include/secp256k1_ecdh.h index df5fde235c..4058e9c043 100644 --- a/src/secp256k1/include/secp256k1_ecdh.h +++ b/src/secp256k1/include/secp256k1_ecdh.h @@ -7,43 +7,50 @@ extern "C" { #endif -/** A pointer to a function that applies hash function to a point +/** A pointer to a function that hashes an EC point to obtain an ECDH secret * - * Returns: 1 if a point was successfully hashed. 0 will cause ecdh to fail - * Out: output: pointer to an array to be filled by the function - * In: x: pointer to a 32-byte x coordinate - * y: pointer to a 32-byte y coordinate - * data: Arbitrary data pointer that is passed through + * Returns: 1 if the point was successfully hashed. + * 0 will cause secp256k1_ecdh to fail and return 0. + * Other return values are not allowed, and the behaviour of + * secp256k1_ecdh is undefined for other return values. + * Out: output: pointer to an array to be filled by the function + * In: x32: pointer to a 32-byte x coordinate + * y32: pointer to a 32-byte y coordinate + * data: arbitrary data pointer that is passed through */ typedef int (*secp256k1_ecdh_hash_function)( unsigned char *output, - const unsigned char *x, - const unsigned char *y, + const unsigned char *x32, + const unsigned char *y32, void *data ); -/** An implementation of SHA256 hash function that applies to compressed public key. */ +/** An implementation of SHA256 hash function that applies to compressed public key. + * Populates the output parameter with 32 bytes. */ SECP256K1_API extern const secp256k1_ecdh_hash_function secp256k1_ecdh_hash_function_sha256; -/** A default ecdh hash function (currently equal to secp256k1_ecdh_hash_function_sha256). */ +/** A default ECDH hash function (currently equal to secp256k1_ecdh_hash_function_sha256). + * Populates the output parameter with 32 bytes. */ SECP256K1_API extern const secp256k1_ecdh_hash_function secp256k1_ecdh_hash_function_default; /** Compute an EC Diffie-Hellman secret in constant time + * * Returns: 1: exponentiation was successful - * 0: scalar was invalid (zero or overflow) + * 0: scalar was invalid (zero or overflow) or hashfp returned 0 * Args: ctx: pointer to a context object (cannot be NULL) - * Out: output: pointer to an array to be filled by the function + * Out: output: pointer to an array to be filled by hashfp * In: pubkey: a pointer to a secp256k1_pubkey containing an * initialized public key - * privkey: a 32-byte scalar with which to multiply the point + * seckey: a 32-byte scalar with which to multiply the point * hashfp: pointer to a hash function. If NULL, secp256k1_ecdh_hash_function_sha256 is used - * data: Arbitrary data pointer that is passed through + * (in which case, 32 bytes will be written to output) + * data: arbitrary data pointer that is passed through to hashfp */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ecdh( const secp256k1_context* ctx, unsigned char *output, const secp256k1_pubkey *pubkey, - const unsigned char *privkey, + const unsigned char *seckey, secp256k1_ecdh_hash_function hashfp, void *data ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); diff --git a/src/secp256k1/include/secp256k1_preallocated.h b/src/secp256k1/include/secp256k1_preallocated.h new file mode 100644 index 0000000000..a9ae15d5ae --- /dev/null +++ b/src/secp256k1/include/secp256k1_preallocated.h @@ -0,0 +1,128 @@ +#ifndef SECP256K1_PREALLOCATED_H +#define SECP256K1_PREALLOCATED_H + +#include "secp256k1.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* The module provided by this header file is intended for settings in which it + * is not possible or desirable to rely on dynamic memory allocation. It provides + * functions for creating, cloning, and destroying secp256k1 context objects in a + * contiguous fixed-size block of memory provided by the caller. + * + * Context objects created by functions in this module can be used like contexts + * objects created by functions in secp256k1.h, i.e., they can be passed to any + * API function that expects a context object (see secp256k1.h for details). The + * only exception is that context objects created by functions in this module + * must be destroyed using secp256k1_context_preallocated_destroy (in this + * module) instead of secp256k1_context_destroy (in secp256k1.h). + * + * It is guaranteed that functions in this module will not call malloc or its + * friends realloc, calloc, and free. + */ + +/** Determine the memory size of a secp256k1 context object to be created in + * caller-provided memory. + * + * The purpose of this function is to determine how much memory must be provided + * to secp256k1_context_preallocated_create. + * + * Returns: the required size of the caller-provided memory block + * In: flags: which parts of the context to initialize. + */ +SECP256K1_API size_t secp256k1_context_preallocated_size( + unsigned int flags +) SECP256K1_WARN_UNUSED_RESULT; + +/** Create a secp256k1 context object in caller-provided memory. + * + * The caller must provide a pointer to a rewritable contiguous block of memory + * of size at least secp256k1_context_preallocated_size(flags) bytes, suitably + * aligned to hold an object of any type. + * + * The block of memory is exclusively owned by the created context object during + * the lifetime of this context object, which begins with the call to this + * function and ends when a call to secp256k1_context_preallocated_destroy + * (which destroys the context object again) returns. During the lifetime of the + * context object, the caller is obligated not to access this block of memory, + * i.e., the caller may not read or write the memory, e.g., by copying the memory + * contents to a different location or trying to create a second context object + * in the memory. In simpler words, the prealloc pointer (or any pointer derived + * from it) should not be used during the lifetime of the context object. + * + * Returns: a newly created context object. + * In: prealloc: a pointer to a rewritable contiguous block of memory of + * size at least secp256k1_context_preallocated_size(flags) + * bytes, as detailed above (cannot be NULL) + * flags: which parts of the context to initialize. + * + * See also secp256k1_context_randomize (in secp256k1.h) + * and secp256k1_context_preallocated_destroy. + */ +SECP256K1_API secp256k1_context* secp256k1_context_preallocated_create( + void* prealloc, + unsigned int flags +) SECP256K1_ARG_NONNULL(1) SECP256K1_WARN_UNUSED_RESULT; + +/** Determine the memory size of a secp256k1 context object to be copied into + * caller-provided memory. + * + * Returns: the required size of the caller-provided memory block. + * In: ctx: an existing context to copy (cannot be NULL) + */ +SECP256K1_API size_t secp256k1_context_preallocated_clone_size( + const secp256k1_context* ctx +) SECP256K1_ARG_NONNULL(1) SECP256K1_WARN_UNUSED_RESULT; + +/** Copy a secp256k1 context object into caller-provided memory. + * + * The caller must provide a pointer to a rewritable contiguous block of memory + * of size at least secp256k1_context_preallocated_size(flags) bytes, suitably + * aligned to hold an object of any type. + * + * The block of memory is exclusively owned by the created context object during + * the lifetime of this context object, see the description of + * secp256k1_context_preallocated_create for details. + * + * Returns: a newly created context object. + * Args: ctx: an existing context to copy (cannot be NULL) + * In: prealloc: a pointer to a rewritable contiguous block of memory of + * size at least secp256k1_context_preallocated_size(flags) + * bytes, as detailed above (cannot be NULL) + */ +SECP256K1_API secp256k1_context* secp256k1_context_preallocated_clone( + const secp256k1_context* ctx, + void* prealloc +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_WARN_UNUSED_RESULT; + +/** Destroy a secp256k1 context object that has been created in + * caller-provided memory. + * + * The context pointer may not be used afterwards. + * + * The context to destroy must have been created using + * secp256k1_context_preallocated_create or secp256k1_context_preallocated_clone. + * If the context has instead been created using secp256k1_context_create or + * secp256k1_context_clone, the behaviour is undefined. In that case, + * secp256k1_context_destroy must be used instead. + * + * If required, it is the responsibility of the caller to deallocate the block + * of memory properly after this function returns, e.g., by calling free on the + * preallocated pointer given to secp256k1_context_preallocated_create or + * secp256k1_context_preallocated_clone. + * + * Args: ctx: an existing context to destroy, constructed using + * secp256k1_context_preallocated_create or + * secp256k1_context_preallocated_clone (cannot be NULL) + */ +SECP256K1_API void secp256k1_context_preallocated_destroy( + secp256k1_context* ctx +); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_PREALLOCATED_H */ diff --git a/src/secp256k1/include/secp256k1_recovery.h b/src/secp256k1/include/secp256k1_recovery.h index cf6c5ed7f5..f8ccaecd3d 100644 --- a/src/secp256k1/include/secp256k1_recovery.h +++ b/src/secp256k1/include/secp256k1_recovery.h @@ -70,7 +70,7 @@ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_serialize_compact( /** Create a recoverable ECDSA signature. * * Returns: 1: signature created - * 0: the nonce generation function failed, or the private key was invalid. + * 0: the nonce generation function failed, or the secret key was invalid. * Args: ctx: pointer to a context object, initialized for signing (cannot be NULL) * Out: sig: pointer to an array where the signature will be placed (cannot be NULL) * In: msg32: the 32-byte message hash being signed (cannot be NULL) diff --git a/src/secp256k1/src/asm/field_10x26_arm.s b/src/secp256k1/src/asm/field_10x26_arm.s index 5a9cc3ffcf..9a5bd06721 100644 --- a/src/secp256k1/src/asm/field_10x26_arm.s +++ b/src/secp256k1/src/asm/field_10x26_arm.s @@ -16,15 +16,9 @@ Note: */ .syntax unified - .arch armv7-a @ eabi attributes - see readelf -A - .eabi_attribute 8, 1 @ Tag_ARM_ISA_use = yes - .eabi_attribute 9, 0 @ Tag_Thumb_ISA_use = no - .eabi_attribute 10, 0 @ Tag_FP_arch = none .eabi_attribute 24, 1 @ Tag_ABI_align_needed = 8-byte .eabi_attribute 25, 1 @ Tag_ABI_align_preserved = 8-byte, except leaf SP - .eabi_attribute 30, 2 @ Tag_ABI_optimization_goals = Aggressive Speed - .eabi_attribute 34, 1 @ Tag_CPU_unaligned_access = v6 .text @ Field constants diff --git a/src/secp256k1/src/basic-config.h b/src/secp256k1/src/basic-config.h index fc588061ca..e9be39d4ca 100644 --- a/src/secp256k1/src/basic-config.h +++ b/src/secp256k1/src/basic-config.h @@ -10,7 +10,10 @@ #ifdef USE_BASIC_CONFIG #undef USE_ASM_X86_64 +#undef USE_ECMULT_STATIC_PRECOMPUTATION #undef USE_ENDOMORPHISM +#undef USE_EXTERNAL_ASM +#undef USE_EXTERNAL_DEFAULT_CALLBACKS #undef USE_FIELD_10X26 #undef USE_FIELD_5X52 #undef USE_FIELD_INV_BUILTIN @@ -21,12 +24,14 @@ #undef USE_SCALAR_8X32 #undef USE_SCALAR_INV_BUILTIN #undef USE_SCALAR_INV_NUM +#undef ECMULT_WINDOW_SIZE #define USE_NUM_NONE 1 #define USE_FIELD_INV_BUILTIN 1 #define USE_SCALAR_INV_BUILTIN 1 #define USE_FIELD_10X26 1 #define USE_SCALAR_8X32 1 +#define ECMULT_WINDOW_SIZE 15 #endif /* USE_BASIC_CONFIG */ diff --git a/src/secp256k1/src/bench.h b/src/secp256k1/src/bench.h index 5b59783f68..9bfed903e0 100644 --- a/src/secp256k1/src/bench.h +++ b/src/secp256k1/src/bench.h @@ -7,45 +7,87 @@ #ifndef SECP256K1_BENCH_H #define SECP256K1_BENCH_H +#include <stdint.h> #include <stdio.h> #include <string.h> -#include <math.h> #include "sys/time.h" -static double gettimedouble(void) { +static int64_t gettime_i64(void) { struct timeval tv; gettimeofday(&tv, NULL); - return tv.tv_usec * 0.000001 + tv.tv_sec; + return (int64_t)tv.tv_usec + (int64_t)tv.tv_sec * 1000000LL; } -void print_number(double x) { - double y = x; - int c = 0; - if (y < 0.0) { - y = -y; +#define FP_EXP (6) +#define FP_MULT (1000000LL) + +/* Format fixed point number. */ +void print_number(const int64_t x) { + int64_t x_abs, y; + int c, i, rounding; + size_t ptr; + char buffer[30]; + + if (x == INT64_MIN) { + /* Prevent UB. */ + printf("ERR"); + return; } - while (y > 0 && y < 100.0) { - y *= 10.0; + x_abs = x < 0 ? -x : x; + + /* Determine how many decimals we want to show (more than FP_EXP makes no + * sense). */ + y = x_abs; + c = 0; + while (y > 0LL && y < 100LL * FP_MULT && c < FP_EXP) { + y *= 10LL; c++; } - printf("%.*f", c, x); + + /* Round to 'c' decimals. */ + y = x_abs; + rounding = 0; + for (i = c; i < FP_EXP; ++i) { + rounding = (y % 10) >= 5; + y /= 10; + } + y += rounding; + + /* Format and print the number. */ + ptr = sizeof(buffer) - 1; + buffer[ptr] = 0; + if (c != 0) { + for (i = 0; i < c; ++i) { + buffer[--ptr] = '0' + (y % 10); + y /= 10; + } + buffer[--ptr] = '.'; + } + do { + buffer[--ptr] = '0' + (y % 10); + y /= 10; + } while (y != 0); + if (x < 0) { + buffer[--ptr] = '-'; + } + printf("%s", &buffer[ptr]); } -void run_benchmark(char *name, void (*benchmark)(void*), void (*setup)(void*), void (*teardown)(void*), void* data, int count, int iter) { +void run_benchmark(char *name, void (*benchmark)(void*, int), void (*setup)(void*), void (*teardown)(void*, int), void* data, int count, int iter) { int i; - double min = HUGE_VAL; - double sum = 0.0; - double max = 0.0; + int64_t min = INT64_MAX; + int64_t sum = 0; + int64_t max = 0; for (i = 0; i < count; i++) { - double begin, total; + int64_t begin, total; if (setup != NULL) { setup(data); } - begin = gettimedouble(); - benchmark(data); - total = gettimedouble() - begin; + begin = gettime_i64(); + benchmark(data, iter); + total = gettime_i64() - begin; if (teardown != NULL) { - teardown(data); + teardown(data, iter); } if (total < min) { min = total; @@ -56,11 +98,11 @@ void run_benchmark(char *name, void (*benchmark)(void*), void (*setup)(void*), v sum += total; } printf("%s: min ", name); - print_number(min * 1000000.0 / iter); + print_number(min * FP_MULT / iter); printf("us / avg "); - print_number((sum / count) * 1000000.0 / iter); + print_number(((sum * FP_MULT) / count) / iter); printf("us / max "); - print_number(max * 1000000.0 / iter); + print_number(max * FP_MULT / iter); printf("us\n"); } @@ -79,4 +121,13 @@ int have_flag(int argc, char** argv, char *flag) { return 0; } +int get_iters(int default_iters) { + char* env = getenv("SECP256K1_BENCH_ITERS"); + if (env) { + return strtol(env, NULL, 0); + } else { + return default_iters; + } +} + #endif /* SECP256K1_BENCH_H */ diff --git a/src/secp256k1/src/bench_ecdh.c b/src/secp256k1/src/bench_ecdh.c index c1dd5a6ac9..f099d33884 100644 --- a/src/secp256k1/src/bench_ecdh.c +++ b/src/secp256k1/src/bench_ecdh.c @@ -28,20 +28,18 @@ static void bench_ecdh_setup(void* arg) { 0xa2, 0xba, 0xd1, 0x84, 0xf8, 0x83, 0xc6, 0x9f }; - /* create a context with no capabilities */ - data->ctx = secp256k1_context_create(SECP256K1_FLAGS_TYPE_CONTEXT); for (i = 0; i < 32; i++) { data->scalar[i] = i + 1; } CHECK(secp256k1_ec_pubkey_parse(data->ctx, &data->point, point, sizeof(point)) == 1); } -static void bench_ecdh(void* arg) { +static void bench_ecdh(void* arg, int iters) { int i; unsigned char res[32]; bench_ecdh_data *data = (bench_ecdh_data*)arg; - for (i = 0; i < 20000; i++) { + for (i = 0; i < iters; i++) { CHECK(secp256k1_ecdh(data->ctx, res, &data->point, data->scalar, NULL, NULL) == 1); } } @@ -49,6 +47,13 @@ static void bench_ecdh(void* arg) { int main(void) { bench_ecdh_data data; - run_benchmark("ecdh", bench_ecdh, bench_ecdh_setup, NULL, &data, 10, 20000); + int iters = get_iters(20000); + + /* create a context with no capabilities */ + data.ctx = secp256k1_context_create(SECP256K1_FLAGS_TYPE_CONTEXT); + + run_benchmark("ecdh", bench_ecdh, bench_ecdh_setup, NULL, &data, 10, iters); + + secp256k1_context_destroy(data.ctx); return 0; } diff --git a/src/secp256k1/src/bench_ecmult.c b/src/secp256k1/src/bench_ecmult.c index 6d0ed1f436..facd07ef31 100644 --- a/src/secp256k1/src/bench_ecmult.c +++ b/src/secp256k1/src/bench_ecmult.c @@ -18,7 +18,6 @@ #include "secp256k1.c" #define POINTS 32768 -#define ITERS 10000 typedef struct { /* Setup once in advance */ @@ -55,16 +54,16 @@ static int bench_callback(secp256k1_scalar* sc, secp256k1_ge* ge, size_t idx, vo return 1; } -static void bench_ecmult(void* arg) { +static void bench_ecmult(void* arg, int iters) { bench_data* data = (bench_data*)arg; - size_t count = data->count; int includes_g = data->includes_g; - size_t iters = 1 + ITERS / count; - size_t iter; + int iter; + int count = data->count; + iters = iters / data->count; for (iter = 0; iter < iters; ++iter) { - data->ecmult_multi(&data->ctx->ecmult_ctx, data->scratch, &data->output[iter], data->includes_g ? &data->scalars[data->offset1] : NULL, bench_callback, arg, count - includes_g); + data->ecmult_multi(&data->ctx->error_callback, &data->ctx->ecmult_ctx, data->scratch, &data->output[iter], data->includes_g ? &data->scalars[data->offset1] : NULL, bench_callback, arg, count - includes_g); data->offset1 = (data->offset1 + count) % POINTS; data->offset2 = (data->offset2 + count - 1) % POINTS; } @@ -76,10 +75,10 @@ static void bench_ecmult_setup(void* arg) { data->offset2 = (data->count * 0x7f6f537b + 0x6a1a8f49) % POINTS; } -static void bench_ecmult_teardown(void* arg) { +static void bench_ecmult_teardown(void* arg, int iters) { bench_data* data = (bench_data*)arg; - size_t iters = 1 + ITERS / data->count; - size_t iter; + int iter; + iters = iters / data->count; /* Verify the results in teardown, to avoid doing comparisons while benchmarking. */ for (iter = 0; iter < iters; ++iter) { secp256k1_gej tmp; @@ -104,10 +103,10 @@ static void generate_scalar(uint32_t num, secp256k1_scalar* scalar) { CHECK(!overflow); } -static void run_test(bench_data* data, size_t count, int includes_g) { +static void run_test(bench_data* data, size_t count, int includes_g, int num_iters) { char str[32]; static const secp256k1_scalar zero = SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 0); - size_t iters = 1 + ITERS / count; + size_t iters = 1 + num_iters / count; size_t iter; data->count = count; @@ -130,7 +129,7 @@ static void run_test(bench_data* data, size_t count, int includes_g) { /* Run the benchmark. */ sprintf(str, includes_g ? "ecmult_%ig" : "ecmult_%i", (int)count); - run_benchmark(str, bench_ecmult, bench_ecmult_setup, bench_ecmult_teardown, data, 10, count * (1 + ITERS / count)); + run_benchmark(str, bench_ecmult, bench_ecmult_setup, bench_ecmult_teardown, data, 10, count * iters); } int main(int argc, char **argv) { @@ -139,6 +138,8 @@ int main(int argc, char **argv) { secp256k1_gej* pubkeys_gej; size_t scratch_size; + int iters = get_iters(10000); + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); scratch_size = secp256k1_strauss_scratch_size(POINTS) + STRAUSS_SCRATCH_OBJECTS*16; data.scratch = secp256k1_scratch_space_create(data.ctx, scratch_size); @@ -154,7 +155,7 @@ int main(int argc, char **argv) { } else if(have_flag(argc, argv, "simple")) { printf("Using simple algorithm:\n"); data.ecmult_multi = secp256k1_ecmult_multi_var; - secp256k1_scratch_space_destroy(data.scratch); + secp256k1_scratch_space_destroy(data.ctx, data.scratch); data.scratch = NULL; } else { fprintf(stderr, "%s: unrecognized argument '%s'.\n", argv[0], argv[1]); @@ -167,8 +168,8 @@ int main(int argc, char **argv) { data.scalars = malloc(sizeof(secp256k1_scalar) * POINTS); data.seckeys = malloc(sizeof(secp256k1_scalar) * POINTS); data.pubkeys = malloc(sizeof(secp256k1_ge) * POINTS); - data.expected_output = malloc(sizeof(secp256k1_gej) * (ITERS + 1)); - data.output = malloc(sizeof(secp256k1_gej) * (ITERS + 1)); + data.expected_output = malloc(sizeof(secp256k1_gej) * (iters + 1)); + data.output = malloc(sizeof(secp256k1_gej) * (iters + 1)); /* Generate a set of scalars, and private/public keypairs. */ pubkeys_gej = malloc(sizeof(secp256k1_gej) * POINTS); @@ -185,18 +186,24 @@ int main(int argc, char **argv) { free(pubkeys_gej); for (i = 1; i <= 8; ++i) { - run_test(&data, i, 1); + run_test(&data, i, 1, iters); } - for (p = 0; p <= 11; ++p) { - for (i = 9; i <= 16; ++i) { - run_test(&data, i << p, 1); + /* This is disabled with low count of iterations because the loop runs 77 times even with iters=1 + * and the higher it goes the longer the computation takes(more points) + * So we don't run this benchmark with low iterations to prevent slow down */ + if (iters > 2) { + for (p = 0; p <= 11; ++p) { + for (i = 9; i <= 16; ++i) { + run_test(&data, i << p, 1, iters); + } } } - secp256k1_context_destroy(data.ctx); + if (data.scratch != NULL) { - secp256k1_scratch_space_destroy(data.scratch); + secp256k1_scratch_space_destroy(data.ctx, data.scratch); } + secp256k1_context_destroy(data.ctx); free(data.scalars); free(data.pubkeys); free(data.seckeys); diff --git a/src/secp256k1/src/bench_internal.c b/src/secp256k1/src/bench_internal.c index 9071724331..20759127d3 100644 --- a/src/secp256k1/src/bench_internal.c +++ b/src/secp256k1/src/bench_internal.c @@ -56,263 +56,272 @@ void bench_setup(void* arg) { memcpy(data->data + 32, init_y, 32); } -void bench_scalar_add(void* arg) { - int i; +void bench_scalar_add(void* arg, int iters) { + int i, j = 0; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 2000000; i++) { - secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); + for (i = 0; i < iters; i++) { + j += secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); } + CHECK(j <= iters); } -void bench_scalar_negate(void* arg) { +void bench_scalar_negate(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 2000000; i++) { + for (i = 0; i < iters; i++) { secp256k1_scalar_negate(&data->scalar_x, &data->scalar_x); } } -void bench_scalar_sqr(void* arg) { +void bench_scalar_sqr(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 200000; i++) { + for (i = 0; i < iters; i++) { secp256k1_scalar_sqr(&data->scalar_x, &data->scalar_x); } } -void bench_scalar_mul(void* arg) { +void bench_scalar_mul(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 200000; i++) { + for (i = 0; i < iters; i++) { secp256k1_scalar_mul(&data->scalar_x, &data->scalar_x, &data->scalar_y); } } #ifdef USE_ENDOMORPHISM -void bench_scalar_split(void* arg) { - int i; +void bench_scalar_split(void* arg, int iters) { + int i, j = 0; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 20000; i++) { - secp256k1_scalar l, r; - secp256k1_scalar_split_lambda(&l, &r, &data->scalar_x); - secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); + for (i = 0; i < iters; i++) { + secp256k1_scalar_split_lambda(&data->scalar_x, &data->scalar_y, &data->scalar_x); + j += secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); } + CHECK(j <= iters); } #endif -void bench_scalar_inverse(void* arg) { - int i; +void bench_scalar_inverse(void* arg, int iters) { + int i, j = 0; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 2000; i++) { + for (i = 0; i < iters; i++) { secp256k1_scalar_inverse(&data->scalar_x, &data->scalar_x); - secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); + j += secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); } + CHECK(j <= iters); } -void bench_scalar_inverse_var(void* arg) { - int i; +void bench_scalar_inverse_var(void* arg, int iters) { + int i, j = 0; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 2000; i++) { + for (i = 0; i < iters; i++) { secp256k1_scalar_inverse_var(&data->scalar_x, &data->scalar_x); - secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); + j += secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); } + CHECK(j <= iters); } -void bench_field_normalize(void* arg) { +void bench_field_normalize(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 2000000; i++) { + for (i = 0; i < iters; i++) { secp256k1_fe_normalize(&data->fe_x); } } -void bench_field_normalize_weak(void* arg) { +void bench_field_normalize_weak(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 2000000; i++) { + for (i = 0; i < iters; i++) { secp256k1_fe_normalize_weak(&data->fe_x); } } -void bench_field_mul(void* arg) { +void bench_field_mul(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 200000; i++) { + for (i = 0; i < iters; i++) { secp256k1_fe_mul(&data->fe_x, &data->fe_x, &data->fe_y); } } -void bench_field_sqr(void* arg) { +void bench_field_sqr(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 200000; i++) { + for (i = 0; i < iters; i++) { secp256k1_fe_sqr(&data->fe_x, &data->fe_x); } } -void bench_field_inverse(void* arg) { +void bench_field_inverse(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 20000; i++) { + for (i = 0; i < iters; i++) { secp256k1_fe_inv(&data->fe_x, &data->fe_x); secp256k1_fe_add(&data->fe_x, &data->fe_y); } } -void bench_field_inverse_var(void* arg) { +void bench_field_inverse_var(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 20000; i++) { + for (i = 0; i < iters; i++) { secp256k1_fe_inv_var(&data->fe_x, &data->fe_x); secp256k1_fe_add(&data->fe_x, &data->fe_y); } } -void bench_field_sqrt(void* arg) { - int i; +void bench_field_sqrt(void* arg, int iters) { + int i, j = 0; bench_inv *data = (bench_inv*)arg; secp256k1_fe t; - for (i = 0; i < 20000; i++) { + for (i = 0; i < iters; i++) { t = data->fe_x; - secp256k1_fe_sqrt(&data->fe_x, &t); + j += secp256k1_fe_sqrt(&data->fe_x, &t); secp256k1_fe_add(&data->fe_x, &data->fe_y); } + CHECK(j <= iters); } -void bench_group_double_var(void* arg) { +void bench_group_double_var(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 200000; i++) { + for (i = 0; i < iters; i++) { secp256k1_gej_double_var(&data->gej_x, &data->gej_x, NULL); } } -void bench_group_add_var(void* arg) { +void bench_group_add_var(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 200000; i++) { + for (i = 0; i < iters; i++) { secp256k1_gej_add_var(&data->gej_x, &data->gej_x, &data->gej_y, NULL); } } -void bench_group_add_affine(void* arg) { +void bench_group_add_affine(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 200000; i++) { + for (i = 0; i < iters; i++) { secp256k1_gej_add_ge(&data->gej_x, &data->gej_x, &data->ge_y); } } -void bench_group_add_affine_var(void* arg) { +void bench_group_add_affine_var(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 200000; i++) { + for (i = 0; i < iters; i++) { secp256k1_gej_add_ge_var(&data->gej_x, &data->gej_x, &data->ge_y, NULL); } } -void bench_group_jacobi_var(void* arg) { - int i; +void bench_group_jacobi_var(void* arg, int iters) { + int i, j = 0; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 20000; i++) { - secp256k1_gej_has_quad_y_var(&data->gej_x); + for (i = 0; i < iters; i++) { + j += secp256k1_gej_has_quad_y_var(&data->gej_x); } + CHECK(j == iters); } -void bench_ecmult_wnaf(void* arg) { - int i; +void bench_ecmult_wnaf(void* arg, int iters) { + int i, bits = 0, overflow = 0; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 20000; i++) { - secp256k1_ecmult_wnaf(data->wnaf, 256, &data->scalar_x, WINDOW_A); - secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); + for (i = 0; i < iters; i++) { + bits += secp256k1_ecmult_wnaf(data->wnaf, 256, &data->scalar_x, WINDOW_A); + overflow += secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); } + CHECK(overflow >= 0); + CHECK(bits <= 256*iters); } -void bench_wnaf_const(void* arg) { - int i; +void bench_wnaf_const(void* arg, int iters) { + int i, bits = 0, overflow = 0; bench_inv *data = (bench_inv*)arg; - for (i = 0; i < 20000; i++) { - secp256k1_wnaf_const(data->wnaf, data->scalar_x, WINDOW_A, 256); - secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); + for (i = 0; i < iters; i++) { + bits += secp256k1_wnaf_const(data->wnaf, &data->scalar_x, WINDOW_A, 256); + overflow += secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); } + CHECK(overflow >= 0); + CHECK(bits <= 256*iters); } -void bench_sha256(void* arg) { +void bench_sha256(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; secp256k1_sha256 sha; - for (i = 0; i < 20000; i++) { + for (i = 0; i < iters; i++) { secp256k1_sha256_initialize(&sha); secp256k1_sha256_write(&sha, data->data, 32); secp256k1_sha256_finalize(&sha, data->data); } } -void bench_hmac_sha256(void* arg) { +void bench_hmac_sha256(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; secp256k1_hmac_sha256 hmac; - for (i = 0; i < 20000; i++) { + for (i = 0; i < iters; i++) { secp256k1_hmac_sha256_initialize(&hmac, data->data, 32); secp256k1_hmac_sha256_write(&hmac, data->data, 32); secp256k1_hmac_sha256_finalize(&hmac, data->data); } } -void bench_rfc6979_hmac_sha256(void* arg) { +void bench_rfc6979_hmac_sha256(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; secp256k1_rfc6979_hmac_sha256 rng; - for (i = 0; i < 20000; i++) { + for (i = 0; i < iters; i++) { secp256k1_rfc6979_hmac_sha256_initialize(&rng, data->data, 64); secp256k1_rfc6979_hmac_sha256_generate(&rng, data->data, 32); } } -void bench_context_verify(void* arg) { +void bench_context_verify(void* arg, int iters) { int i; (void)arg; - for (i = 0; i < 20; i++) { + for (i = 0; i < iters; i++) { secp256k1_context_destroy(secp256k1_context_create(SECP256K1_CONTEXT_VERIFY)); } } -void bench_context_sign(void* arg) { +void bench_context_sign(void* arg, int iters) { int i; (void)arg; - for (i = 0; i < 200; i++) { + for (i = 0; i < iters; i++) { secp256k1_context_destroy(secp256k1_context_create(SECP256K1_CONTEXT_SIGN)); } } #ifndef USE_NUM_NONE -void bench_num_jacobi(void* arg) { - int i; +void bench_num_jacobi(void* arg, int iters) { + int i, j = 0; bench_inv *data = (bench_inv*)arg; secp256k1_num nx, norder; @@ -320,50 +329,53 @@ void bench_num_jacobi(void* arg) { secp256k1_scalar_order_get_num(&norder); secp256k1_scalar_get_num(&norder, &data->scalar_y); - for (i = 0; i < 200000; i++) { - secp256k1_num_jacobi(&nx, &norder); + for (i = 0; i < iters; i++) { + j += secp256k1_num_jacobi(&nx, &norder); } + CHECK(j <= iters); } #endif int main(int argc, char **argv) { bench_inv data; - if (have_flag(argc, argv, "scalar") || have_flag(argc, argv, "add")) run_benchmark("scalar_add", bench_scalar_add, bench_setup, NULL, &data, 10, 2000000); - if (have_flag(argc, argv, "scalar") || have_flag(argc, argv, "negate")) run_benchmark("scalar_negate", bench_scalar_negate, bench_setup, NULL, &data, 10, 2000000); - if (have_flag(argc, argv, "scalar") || have_flag(argc, argv, "sqr")) run_benchmark("scalar_sqr", bench_scalar_sqr, bench_setup, NULL, &data, 10, 200000); - if (have_flag(argc, argv, "scalar") || have_flag(argc, argv, "mul")) run_benchmark("scalar_mul", bench_scalar_mul, bench_setup, NULL, &data, 10, 200000); + int iters = get_iters(20000); + + if (have_flag(argc, argv, "scalar") || have_flag(argc, argv, "add")) run_benchmark("scalar_add", bench_scalar_add, bench_setup, NULL, &data, 10, iters*100); + if (have_flag(argc, argv, "scalar") || have_flag(argc, argv, "negate")) run_benchmark("scalar_negate", bench_scalar_negate, bench_setup, NULL, &data, 10, iters*100); + if (have_flag(argc, argv, "scalar") || have_flag(argc, argv, "sqr")) run_benchmark("scalar_sqr", bench_scalar_sqr, bench_setup, NULL, &data, 10, iters*10); + if (have_flag(argc, argv, "scalar") || have_flag(argc, argv, "mul")) run_benchmark("scalar_mul", bench_scalar_mul, bench_setup, NULL, &data, 10, iters*10); #ifdef USE_ENDOMORPHISM - if (have_flag(argc, argv, "scalar") || have_flag(argc, argv, "split")) run_benchmark("scalar_split", bench_scalar_split, bench_setup, NULL, &data, 10, 20000); + if (have_flag(argc, argv, "scalar") || have_flag(argc, argv, "split")) run_benchmark("scalar_split", bench_scalar_split, bench_setup, NULL, &data, 10, iters); #endif if (have_flag(argc, argv, "scalar") || have_flag(argc, argv, "inverse")) run_benchmark("scalar_inverse", bench_scalar_inverse, bench_setup, NULL, &data, 10, 2000); if (have_flag(argc, argv, "scalar") || have_flag(argc, argv, "inverse")) run_benchmark("scalar_inverse_var", bench_scalar_inverse_var, bench_setup, NULL, &data, 10, 2000); - if (have_flag(argc, argv, "field") || have_flag(argc, argv, "normalize")) run_benchmark("field_normalize", bench_field_normalize, bench_setup, NULL, &data, 10, 2000000); - if (have_flag(argc, argv, "field") || have_flag(argc, argv, "normalize")) run_benchmark("field_normalize_weak", bench_field_normalize_weak, bench_setup, NULL, &data, 10, 2000000); - if (have_flag(argc, argv, "field") || have_flag(argc, argv, "sqr")) run_benchmark("field_sqr", bench_field_sqr, bench_setup, NULL, &data, 10, 200000); - if (have_flag(argc, argv, "field") || have_flag(argc, argv, "mul")) run_benchmark("field_mul", bench_field_mul, bench_setup, NULL, &data, 10, 200000); - if (have_flag(argc, argv, "field") || have_flag(argc, argv, "inverse")) run_benchmark("field_inverse", bench_field_inverse, bench_setup, NULL, &data, 10, 20000); - if (have_flag(argc, argv, "field") || have_flag(argc, argv, "inverse")) run_benchmark("field_inverse_var", bench_field_inverse_var, bench_setup, NULL, &data, 10, 20000); - if (have_flag(argc, argv, "field") || have_flag(argc, argv, "sqrt")) run_benchmark("field_sqrt", bench_field_sqrt, bench_setup, NULL, &data, 10, 20000); + if (have_flag(argc, argv, "field") || have_flag(argc, argv, "normalize")) run_benchmark("field_normalize", bench_field_normalize, bench_setup, NULL, &data, 10, iters*100); + if (have_flag(argc, argv, "field") || have_flag(argc, argv, "normalize")) run_benchmark("field_normalize_weak", bench_field_normalize_weak, bench_setup, NULL, &data, 10, iters*100); + if (have_flag(argc, argv, "field") || have_flag(argc, argv, "sqr")) run_benchmark("field_sqr", bench_field_sqr, bench_setup, NULL, &data, 10, iters*10); + if (have_flag(argc, argv, "field") || have_flag(argc, argv, "mul")) run_benchmark("field_mul", bench_field_mul, bench_setup, NULL, &data, 10, iters*10); + if (have_flag(argc, argv, "field") || have_flag(argc, argv, "inverse")) run_benchmark("field_inverse", bench_field_inverse, bench_setup, NULL, &data, 10, iters); + if (have_flag(argc, argv, "field") || have_flag(argc, argv, "inverse")) run_benchmark("field_inverse_var", bench_field_inverse_var, bench_setup, NULL, &data, 10, iters); + if (have_flag(argc, argv, "field") || have_flag(argc, argv, "sqrt")) run_benchmark("field_sqrt", bench_field_sqrt, bench_setup, NULL, &data, 10, iters); - if (have_flag(argc, argv, "group") || have_flag(argc, argv, "double")) run_benchmark("group_double_var", bench_group_double_var, bench_setup, NULL, &data, 10, 200000); - if (have_flag(argc, argv, "group") || have_flag(argc, argv, "add")) run_benchmark("group_add_var", bench_group_add_var, bench_setup, NULL, &data, 10, 200000); - if (have_flag(argc, argv, "group") || have_flag(argc, argv, "add")) run_benchmark("group_add_affine", bench_group_add_affine, bench_setup, NULL, &data, 10, 200000); - if (have_flag(argc, argv, "group") || have_flag(argc, argv, "add")) run_benchmark("group_add_affine_var", bench_group_add_affine_var, bench_setup, NULL, &data, 10, 200000); - if (have_flag(argc, argv, "group") || have_flag(argc, argv, "jacobi")) run_benchmark("group_jacobi_var", bench_group_jacobi_var, bench_setup, NULL, &data, 10, 20000); + if (have_flag(argc, argv, "group") || have_flag(argc, argv, "double")) run_benchmark("group_double_var", bench_group_double_var, bench_setup, NULL, &data, 10, iters*10); + if (have_flag(argc, argv, "group") || have_flag(argc, argv, "add")) run_benchmark("group_add_var", bench_group_add_var, bench_setup, NULL, &data, 10, iters*10); + if (have_flag(argc, argv, "group") || have_flag(argc, argv, "add")) run_benchmark("group_add_affine", bench_group_add_affine, bench_setup, NULL, &data, 10, iters*10); + if (have_flag(argc, argv, "group") || have_flag(argc, argv, "add")) run_benchmark("group_add_affine_var", bench_group_add_affine_var, bench_setup, NULL, &data, 10, iters*10); + if (have_flag(argc, argv, "group") || have_flag(argc, argv, "jacobi")) run_benchmark("group_jacobi_var", bench_group_jacobi_var, bench_setup, NULL, &data, 10, iters); - if (have_flag(argc, argv, "ecmult") || have_flag(argc, argv, "wnaf")) run_benchmark("wnaf_const", bench_wnaf_const, bench_setup, NULL, &data, 10, 20000); - if (have_flag(argc, argv, "ecmult") || have_flag(argc, argv, "wnaf")) run_benchmark("ecmult_wnaf", bench_ecmult_wnaf, bench_setup, NULL, &data, 10, 20000); + if (have_flag(argc, argv, "ecmult") || have_flag(argc, argv, "wnaf")) run_benchmark("wnaf_const", bench_wnaf_const, bench_setup, NULL, &data, 10, iters); + if (have_flag(argc, argv, "ecmult") || have_flag(argc, argv, "wnaf")) run_benchmark("ecmult_wnaf", bench_ecmult_wnaf, bench_setup, NULL, &data, 10, iters); - if (have_flag(argc, argv, "hash") || have_flag(argc, argv, "sha256")) run_benchmark("hash_sha256", bench_sha256, bench_setup, NULL, &data, 10, 20000); - if (have_flag(argc, argv, "hash") || have_flag(argc, argv, "hmac")) run_benchmark("hash_hmac_sha256", bench_hmac_sha256, bench_setup, NULL, &data, 10, 20000); - if (have_flag(argc, argv, "hash") || have_flag(argc, argv, "rng6979")) run_benchmark("hash_rfc6979_hmac_sha256", bench_rfc6979_hmac_sha256, bench_setup, NULL, &data, 10, 20000); + if (have_flag(argc, argv, "hash") || have_flag(argc, argv, "sha256")) run_benchmark("hash_sha256", bench_sha256, bench_setup, NULL, &data, 10, iters); + if (have_flag(argc, argv, "hash") || have_flag(argc, argv, "hmac")) run_benchmark("hash_hmac_sha256", bench_hmac_sha256, bench_setup, NULL, &data, 10, iters); + if (have_flag(argc, argv, "hash") || have_flag(argc, argv, "rng6979")) run_benchmark("hash_rfc6979_hmac_sha256", bench_rfc6979_hmac_sha256, bench_setup, NULL, &data, 10, iters); - if (have_flag(argc, argv, "context") || have_flag(argc, argv, "verify")) run_benchmark("context_verify", bench_context_verify, bench_setup, NULL, &data, 10, 20); - if (have_flag(argc, argv, "context") || have_flag(argc, argv, "sign")) run_benchmark("context_sign", bench_context_sign, bench_setup, NULL, &data, 10, 200); + if (have_flag(argc, argv, "context") || have_flag(argc, argv, "verify")) run_benchmark("context_verify", bench_context_verify, bench_setup, NULL, &data, 10, 1 + iters/1000); + if (have_flag(argc, argv, "context") || have_flag(argc, argv, "sign")) run_benchmark("context_sign", bench_context_sign, bench_setup, NULL, &data, 10, 1 + iters/100); #ifndef USE_NUM_NONE - if (have_flag(argc, argv, "num") || have_flag(argc, argv, "jacobi")) run_benchmark("num_jacobi", bench_num_jacobi, bench_setup, NULL, &data, 10, 200000); + if (have_flag(argc, argv, "num") || have_flag(argc, argv, "jacobi")) run_benchmark("num_jacobi", bench_num_jacobi, bench_setup, NULL, &data, 10, iters*10); #endif return 0; } diff --git a/src/secp256k1/src/bench_recover.c b/src/secp256k1/src/bench_recover.c index b806eed94e..e952ed1215 100644 --- a/src/secp256k1/src/bench_recover.c +++ b/src/secp256k1/src/bench_recover.c @@ -15,13 +15,13 @@ typedef struct { unsigned char sig[64]; } bench_recover_data; -void bench_recover(void* arg) { +void bench_recover(void* arg, int iters) { int i; bench_recover_data *data = (bench_recover_data*)arg; secp256k1_pubkey pubkey; unsigned char pubkeyc[33]; - for (i = 0; i < 20000; i++) { + for (i = 0; i < iters; i++) { int j; size_t pubkeylen = 33; secp256k1_ecdsa_recoverable_signature sig; @@ -51,9 +51,11 @@ void bench_recover_setup(void* arg) { int main(void) { bench_recover_data data; + int iters = get_iters(20000); + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); - run_benchmark("ecdsa_recover", bench_recover, bench_recover_setup, NULL, &data, 10, 20000); + run_benchmark("ecdsa_recover", bench_recover, bench_recover_setup, NULL, &data, 10, iters); secp256k1_context_destroy(data.ctx); return 0; diff --git a/src/secp256k1/src/bench_sign.c b/src/secp256k1/src/bench_sign.c index 544b43963c..c6b2942cc0 100644 --- a/src/secp256k1/src/bench_sign.c +++ b/src/secp256k1/src/bench_sign.c @@ -26,12 +26,12 @@ static void bench_sign_setup(void* arg) { } } -static void bench_sign_run(void* arg) { +static void bench_sign_run(void* arg, int iters) { int i; bench_sign *data = (bench_sign*)arg; unsigned char sig[74]; - for (i = 0; i < 20000; i++) { + for (i = 0; i < iters; i++) { size_t siglen = 74; int j; secp256k1_ecdsa_signature signature; @@ -47,9 +47,11 @@ static void bench_sign_run(void* arg) { int main(void) { bench_sign data; + int iters = get_iters(20000); + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); - run_benchmark("ecdsa_sign", bench_sign_run, bench_sign_setup, NULL, &data, 10, 20000); + run_benchmark("ecdsa_sign", bench_sign_run, bench_sign_setup, NULL, &data, 10, iters); secp256k1_context_destroy(data.ctx); return 0; diff --git a/src/secp256k1/src/bench_verify.c b/src/secp256k1/src/bench_verify.c index 418defa0aa..272d3e5cc4 100644 --- a/src/secp256k1/src/bench_verify.c +++ b/src/secp256k1/src/bench_verify.c @@ -17,6 +17,7 @@ #include <openssl/obj_mac.h> #endif + typedef struct { secp256k1_context *ctx; unsigned char msg[32]; @@ -30,11 +31,11 @@ typedef struct { #endif } benchmark_verify_t; -static void benchmark_verify(void* arg) { +static void benchmark_verify(void* arg, int iters) { int i; benchmark_verify_t* data = (benchmark_verify_t*)arg; - for (i = 0; i < 20000; i++) { + for (i = 0; i < iters; i++) { secp256k1_pubkey pubkey; secp256k1_ecdsa_signature sig; data->sig[data->siglen - 1] ^= (i & 0xFF); @@ -50,11 +51,11 @@ static void benchmark_verify(void* arg) { } #ifdef ENABLE_OPENSSL_TESTS -static void benchmark_verify_openssl(void* arg) { +static void benchmark_verify_openssl(void* arg, int iters) { int i; benchmark_verify_t* data = (benchmark_verify_t*)arg; - for (i = 0; i < 20000; i++) { + for (i = 0; i < iters; i++) { data->sig[data->siglen - 1] ^= (i & 0xFF); data->sig[data->siglen - 2] ^= ((i >> 8) & 0xFF); data->sig[data->siglen - 3] ^= ((i >> 16) & 0xFF); @@ -85,6 +86,8 @@ int main(void) { secp256k1_ecdsa_signature sig; benchmark_verify_t data; + int iters = get_iters(20000); + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); for (i = 0; i < 32; i++) { @@ -100,10 +103,10 @@ int main(void) { data.pubkeylen = 33; CHECK(secp256k1_ec_pubkey_serialize(data.ctx, data.pubkey, &data.pubkeylen, &pubkey, SECP256K1_EC_COMPRESSED) == 1); - run_benchmark("ecdsa_verify", benchmark_verify, NULL, NULL, &data, 10, 20000); + run_benchmark("ecdsa_verify", benchmark_verify, NULL, NULL, &data, 10, iters); #ifdef ENABLE_OPENSSL_TESTS data.ec_group = EC_GROUP_new_by_curve_name(NID_secp256k1); - run_benchmark("ecdsa_verify_openssl", benchmark_verify_openssl, NULL, NULL, &data, 10, 20000); + run_benchmark("ecdsa_verify_openssl", benchmark_verify_openssl, NULL, NULL, &data, 10, iters); EC_GROUP_free(data.ec_group); #endif diff --git a/src/secp256k1/src/ecdsa_impl.h b/src/secp256k1/src/ecdsa_impl.h index c3400042d8..5f54b59faa 100644 --- a/src/secp256k1/src/ecdsa_impl.h +++ b/src/secp256k1/src/ecdsa_impl.h @@ -46,68 +46,73 @@ static const secp256k1_fe secp256k1_ecdsa_const_p_minus_order = SECP256K1_FE_CON 0, 0, 0, 1, 0x45512319UL, 0x50B75FC4UL, 0x402DA172UL, 0x2FC9BAEEUL ); -static int secp256k1_der_read_len(const unsigned char **sigp, const unsigned char *sigend) { - int lenleft, b1; - size_t ret = 0; +static int secp256k1_der_read_len(size_t *len, const unsigned char **sigp, const unsigned char *sigend) { + size_t lenleft; + unsigned char b1; + VERIFY_CHECK(len != NULL); + *len = 0; if (*sigp >= sigend) { - return -1; + return 0; } b1 = *((*sigp)++); if (b1 == 0xFF) { /* X.690-0207 8.1.3.5.c the value 0xFF shall not be used. */ - return -1; + return 0; } if ((b1 & 0x80) == 0) { /* X.690-0207 8.1.3.4 short form length octets */ - return b1; + *len = b1; + return 1; } if (b1 == 0x80) { /* Indefinite length is not allowed in DER. */ - return -1; + return 0; } /* X.690-207 8.1.3.5 long form length octets */ - lenleft = b1 & 0x7F; - if (lenleft > sigend - *sigp) { - return -1; + lenleft = b1 & 0x7F; /* lenleft is at least 1 */ + if (lenleft > (size_t)(sigend - *sigp)) { + return 0; } if (**sigp == 0) { /* Not the shortest possible length encoding. */ - return -1; + return 0; } - if ((size_t)lenleft > sizeof(size_t)) { + if (lenleft > sizeof(size_t)) { /* The resulting length would exceed the range of a size_t, so * certainly longer than the passed array size. */ - return -1; + return 0; } while (lenleft > 0) { - ret = (ret << 8) | **sigp; - if (ret + lenleft > (size_t)(sigend - *sigp)) { - /* Result exceeds the length of the passed array. */ - return -1; - } + *len = (*len << 8) | **sigp; (*sigp)++; lenleft--; } - if (ret < 128) { + if (*len > (size_t)(sigend - *sigp)) { + /* Result exceeds the length of the passed array. */ + return 0; + } + if (*len < 128) { /* Not the shortest possible length encoding. */ - return -1; + return 0; } - return ret; + return 1; } static int secp256k1_der_parse_integer(secp256k1_scalar *r, const unsigned char **sig, const unsigned char *sigend) { int overflow = 0; unsigned char ra[32] = {0}; - int rlen; + size_t rlen; if (*sig == sigend || **sig != 0x02) { /* Not a primitive integer (X.690-0207 8.3.1). */ return 0; } (*sig)++; - rlen = secp256k1_der_read_len(sig, sigend); - if (rlen <= 0 || (*sig) + rlen > sigend) { + if (secp256k1_der_read_len(&rlen, sig, sigend) == 0) { + return 0; + } + if (rlen == 0 || *sig + rlen > sigend) { /* Exceeds bounds or not at least length 1 (X.690-0207 8.3.1). */ return 0; } @@ -123,8 +128,11 @@ static int secp256k1_der_parse_integer(secp256k1_scalar *r, const unsigned char /* Negative. */ overflow = 1; } - while (rlen > 0 && **sig == 0) { - /* Skip leading zero bytes */ + /* There is at most one leading zero byte: + * if there were two leading zero bytes, we would have failed and returned 0 + * because of excessive 0x00 padding already. */ + if (rlen > 0 && **sig == 0) { + /* Skip leading zero byte */ rlen--; (*sig)++; } @@ -144,18 +152,16 @@ static int secp256k1_der_parse_integer(secp256k1_scalar *r, const unsigned char static int secp256k1_ecdsa_sig_parse(secp256k1_scalar *rr, secp256k1_scalar *rs, const unsigned char *sig, size_t size) { const unsigned char *sigend = sig + size; - int rlen; + size_t rlen; if (sig == sigend || *(sig++) != 0x30) { /* The encoding doesn't start with a constructed sequence (X.690-0207 8.9.1). */ return 0; } - rlen = secp256k1_der_read_len(&sig, sigend); - if (rlen < 0 || sig + rlen > sigend) { - /* Tuple exceeds bounds */ + if (secp256k1_der_read_len(&rlen, &sig, sigend) == 0) { return 0; } - if (sig + rlen != sigend) { - /* Garbage after tuple. */ + if (rlen != (size_t)(sigend - sig)) { + /* Tuple exceeds bounds or garage after tuple. */ return 0; } @@ -274,6 +280,7 @@ static int secp256k1_ecdsa_sig_sign(const secp256k1_ecmult_gen_context *ctx, sec secp256k1_ge r; secp256k1_scalar n; int overflow = 0; + int high; secp256k1_ecmult_gen(ctx, &rp, nonce); secp256k1_ge_set_gej(&r, &rp); @@ -281,15 +288,11 @@ static int secp256k1_ecdsa_sig_sign(const secp256k1_ecmult_gen_context *ctx, sec secp256k1_fe_normalize(&r.y); secp256k1_fe_get_b32(b, &r.x); secp256k1_scalar_set_b32(sigr, b, &overflow); - /* These two conditions should be checked before calling */ - VERIFY_CHECK(!secp256k1_scalar_is_zero(sigr)); - VERIFY_CHECK(overflow == 0); - if (recid) { /* The overflow condition is cryptographically unreachable as hitting it requires finding the discrete log * of some P where P.x >= order, and only 1 in about 2^127 points meet this criteria. */ - *recid = (overflow ? 2 : 0) | (secp256k1_fe_is_odd(&r.y) ? 1 : 0); + *recid = (overflow << 1) | secp256k1_fe_is_odd(&r.y); } secp256k1_scalar_mul(&n, sigr, seckey); secp256k1_scalar_add(&n, &n, message); @@ -298,16 +301,15 @@ static int secp256k1_ecdsa_sig_sign(const secp256k1_ecmult_gen_context *ctx, sec secp256k1_scalar_clear(&n); secp256k1_gej_clear(&rp); secp256k1_ge_clear(&r); - if (secp256k1_scalar_is_zero(sigs)) { - return 0; - } - if (secp256k1_scalar_is_high(sigs)) { - secp256k1_scalar_negate(sigs, sigs); - if (recid) { - *recid ^= 1; - } + high = secp256k1_scalar_is_high(sigs); + secp256k1_scalar_cond_negate(sigs, high); + if (recid) { + *recid ^= high; } - return 1; + /* P.x = order is on the curve, so technically sig->r could end up being zero, which would be an invalid signature. + * This is cryptographically unreachable as hitting it requires finding the discrete log of P.x = N. + */ + return !secp256k1_scalar_is_zero(sigr) & !secp256k1_scalar_is_zero(sigs); } #endif /* SECP256K1_ECDSA_IMPL_H */ diff --git a/src/secp256k1/src/eckey_impl.h b/src/secp256k1/src/eckey_impl.h index 7c5b789325..e2e72d9303 100644 --- a/src/secp256k1/src/eckey_impl.h +++ b/src/secp256k1/src/eckey_impl.h @@ -54,10 +54,7 @@ static int secp256k1_eckey_pubkey_serialize(secp256k1_ge *elem, unsigned char *p static int secp256k1_eckey_privkey_tweak_add(secp256k1_scalar *key, const secp256k1_scalar *tweak) { secp256k1_scalar_add(key, key, tweak); - if (secp256k1_scalar_is_zero(key)) { - return 0; - } - return 1; + return !secp256k1_scalar_is_zero(key); } static int secp256k1_eckey_pubkey_tweak_add(const secp256k1_ecmult_context *ctx, secp256k1_ge *key, const secp256k1_scalar *tweak) { @@ -75,12 +72,11 @@ static int secp256k1_eckey_pubkey_tweak_add(const secp256k1_ecmult_context *ctx, } static int secp256k1_eckey_privkey_tweak_mul(secp256k1_scalar *key, const secp256k1_scalar *tweak) { - if (secp256k1_scalar_is_zero(tweak)) { - return 0; - } + int ret; + ret = !secp256k1_scalar_is_zero(tweak); secp256k1_scalar_mul(key, key, tweak); - return 1; + return ret; } static int secp256k1_eckey_pubkey_tweak_mul(const secp256k1_ecmult_context *ctx, secp256k1_ge *key, const secp256k1_scalar *tweak) { diff --git a/src/secp256k1/src/ecmult.h b/src/secp256k1/src/ecmult.h index 3d75a960f4..c9b198239d 100644 --- a/src/secp256k1/src/ecmult.h +++ b/src/secp256k1/src/ecmult.h @@ -20,10 +20,10 @@ typedef struct { #endif } secp256k1_ecmult_context; +static const size_t SECP256K1_ECMULT_CONTEXT_PREALLOCATED_SIZE; static void secp256k1_ecmult_context_init(secp256k1_ecmult_context *ctx); -static void secp256k1_ecmult_context_build(secp256k1_ecmult_context *ctx, const secp256k1_callback *cb); -static void secp256k1_ecmult_context_clone(secp256k1_ecmult_context *dst, - const secp256k1_ecmult_context *src, const secp256k1_callback *cb); +static void secp256k1_ecmult_context_build(secp256k1_ecmult_context *ctx, void **prealloc); +static void secp256k1_ecmult_context_finalize_memcpy(secp256k1_ecmult_context *dst, const secp256k1_ecmult_context *src); static void secp256k1_ecmult_context_clear(secp256k1_ecmult_context *ctx); static int secp256k1_ecmult_context_is_built(const secp256k1_ecmult_context *ctx); @@ -43,6 +43,6 @@ typedef int (secp256k1_ecmult_multi_callback)(secp256k1_scalar *sc, secp256k1_ge * 0 if there is not enough scratch space for a single point or * callback returns 0 */ -static int secp256k1_ecmult_multi_var(const secp256k1_ecmult_context *ctx, secp256k1_scratch *scratch, secp256k1_gej *r, const secp256k1_scalar *inp_g_sc, secp256k1_ecmult_multi_callback cb, void *cbdata, size_t n); +static int secp256k1_ecmult_multi_var(const secp256k1_callback* error_callback, const secp256k1_ecmult_context *ctx, secp256k1_scratch *scratch, secp256k1_gej *r, const secp256k1_scalar *inp_g_sc, secp256k1_ecmult_multi_callback cb, void *cbdata, size_t n); #endif /* SECP256K1_ECMULT_H */ diff --git a/src/secp256k1/src/ecmult_const.h b/src/secp256k1/src/ecmult_const.h index d4804b8b68..03bb33257d 100644 --- a/src/secp256k1/src/ecmult_const.h +++ b/src/secp256k1/src/ecmult_const.h @@ -10,8 +10,11 @@ #include "scalar.h" #include "group.h" -/* Here `bits` should be set to the maximum bitlength of the _absolute value_ of `q`, plus - * one because we internally sometimes add 2 to the number during the WNAF conversion. */ +/** + * Multiply: R = q*A (in constant-time) + * Here `bits` should be set to the maximum bitlength of the _absolute value_ of `q`, plus + * one because we internally sometimes add 2 to the number during the WNAF conversion. + */ static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, const secp256k1_scalar *q, int bits); #endif /* SECP256K1_ECMULT_CONST_H */ diff --git a/src/secp256k1/src/ecmult_const_impl.h b/src/secp256k1/src/ecmult_const_impl.h index 8411752eb0..6d6d354aa4 100644 --- a/src/secp256k1/src/ecmult_const_impl.h +++ b/src/secp256k1/src/ecmult_const_impl.h @@ -14,16 +14,22 @@ /* This is like `ECMULT_TABLE_GET_GE` but is constant time */ #define ECMULT_CONST_TABLE_GET_GE(r,pre,n,w) do { \ - int m; \ - int abs_n = (n) * (((n) > 0) * 2 - 1); \ - int idx_n = abs_n / 2; \ + int m = 0; \ + /* Extract the sign-bit for a constant time absolute-value. */ \ + int mask = (n) >> (sizeof(n) * CHAR_BIT - 1); \ + int abs_n = ((n) + mask) ^ mask; \ + int idx_n = abs_n >> 1; \ secp256k1_fe neg_y; \ VERIFY_CHECK(((n) & 1) == 1); \ VERIFY_CHECK((n) >= -((1 << ((w)-1)) - 1)); \ VERIFY_CHECK((n) <= ((1 << ((w)-1)) - 1)); \ VERIFY_SETUP(secp256k1_fe_clear(&(r)->x)); \ VERIFY_SETUP(secp256k1_fe_clear(&(r)->y)); \ - for (m = 0; m < ECMULT_TABLE_SIZE(w); m++) { \ + /* Unconditionally set r->x = (pre)[m].x. r->y = (pre)[m].y. because it's either the correct one \ + * or will get replaced in the later iterations, this is needed to make sure `r` is initialized. */ \ + (r)->x = (pre)[m].x; \ + (r)->y = (pre)[m].y; \ + for (m = 1; m < ECMULT_TABLE_SIZE(w); m++) { \ /* This loop is used to avoid secret data in array indices. See * the comment in ecmult_gen_impl.h for rationale. */ \ secp256k1_fe_cmov(&(r)->x, &(pre)[m].x, m == idx_n); \ @@ -44,11 +50,11 @@ * * Adapted from `The Width-w NAF Method Provides Small Memory and Fast Elliptic Scalar * Multiplications Secure against Side Channel Attacks`, Okeya and Tagaki. M. Joye (Ed.) - * CT-RSA 2003, LNCS 2612, pp. 328-443, 2003. Springer-Verlagy Berlin Heidelberg 2003 + * CT-RSA 2003, LNCS 2612, pp. 328-443, 2003. Springer-Verlag Berlin Heidelberg 2003 * * Numbers reference steps of `Algorithm SPA-resistant Width-w NAF with Odd Scalar` on pp. 335 */ -static int secp256k1_wnaf_const(int *wnaf, secp256k1_scalar s, int w, int size) { +static int secp256k1_wnaf_const(int *wnaf, const secp256k1_scalar *scalar, int w, int size) { int global_sign; int skew = 0; int word = 0; @@ -59,8 +65,12 @@ static int secp256k1_wnaf_const(int *wnaf, secp256k1_scalar s, int w, int size) int flip; int bit; - secp256k1_scalar neg_s; + secp256k1_scalar s; int not_neg_one; + + VERIFY_CHECK(w > 0); + VERIFY_CHECK(size > 0); + /* Note that we cannot handle even numbers by negating them to be odd, as is * done in other implementations, since if our scalars were specified to have * width < 256 for performance reasons, their negations would have width 256 @@ -75,12 +85,13 @@ static int secp256k1_wnaf_const(int *wnaf, secp256k1_scalar s, int w, int size) * {1, 2} we want to add to the scalar when ensuring that it's odd. Further * complicating things, -1 interacts badly with `secp256k1_scalar_cadd_bit` and * we need to special-case it in this logic. */ - flip = secp256k1_scalar_is_high(&s); + flip = secp256k1_scalar_is_high(scalar); /* We add 1 to even numbers, 2 to odd ones, noting that negation flips parity */ - bit = flip ^ !secp256k1_scalar_is_even(&s); + bit = flip ^ !secp256k1_scalar_is_even(scalar); /* We check for negative one, since adding 2 to it will cause an overflow */ - secp256k1_scalar_negate(&neg_s, &s); - not_neg_one = !secp256k1_scalar_is_one(&neg_s); + secp256k1_scalar_negate(&s, scalar); + not_neg_one = !secp256k1_scalar_is_one(&s); + s = *scalar; secp256k1_scalar_cadd_bit(&s, bit, not_neg_one); /* If we had negative one, flip == 1, s.d[0] == 0, bit == 1, so caller expects * that we added two to it and flipped it. In fact for -1 these operations are @@ -93,7 +104,7 @@ static int secp256k1_wnaf_const(int *wnaf, secp256k1_scalar s, int w, int size) /* 4 */ u_last = secp256k1_scalar_shr_int(&s, w); - while (word * w < size) { + do { int sign; int even; @@ -109,7 +120,7 @@ static int secp256k1_wnaf_const(int *wnaf, secp256k1_scalar s, int w, int size) wnaf[word++] = u_last * global_sign; u_last = u; - } + } while (word * w < size); wnaf[word] = u * global_sign; VERIFY_CHECK(secp256k1_scalar_is_zero(&s)); @@ -132,7 +143,6 @@ static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, cons int wnaf_1[1 + WNAF_SIZE(WINDOW_A - 1)]; int i; - secp256k1_scalar sc = *scalar; /* build wnaf representation for q. */ int rsize = size; @@ -140,13 +150,13 @@ static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, cons if (size > 128) { rsize = 128; /* split q into q_1 and q_lam (where q = q_1 + q_lam*lambda, and q_1 and q_lam are ~128 bit) */ - secp256k1_scalar_split_lambda(&q_1, &q_lam, &sc); - skew_1 = secp256k1_wnaf_const(wnaf_1, q_1, WINDOW_A - 1, 128); - skew_lam = secp256k1_wnaf_const(wnaf_lam, q_lam, WINDOW_A - 1, 128); + secp256k1_scalar_split_lambda(&q_1, &q_lam, scalar); + skew_1 = secp256k1_wnaf_const(wnaf_1, &q_1, WINDOW_A - 1, 128); + skew_lam = secp256k1_wnaf_const(wnaf_lam, &q_lam, WINDOW_A - 1, 128); } else #endif { - skew_1 = secp256k1_wnaf_const(wnaf_1, sc, WINDOW_A - 1, size); + skew_1 = secp256k1_wnaf_const(wnaf_1, scalar, WINDOW_A - 1, size); #ifdef USE_ENDOMORPHISM skew_lam = 0; #endif @@ -168,6 +178,7 @@ static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, cons for (i = 0; i < ECMULT_TABLE_SIZE(WINDOW_A); i++) { secp256k1_ge_mul_lambda(&pre_a_lam[i], &pre_a[i]); } + } #endif @@ -191,7 +202,7 @@ static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, cons int n; int j; for (j = 0; j < WINDOW_A - 1; ++j) { - secp256k1_gej_double_nonzero(r, r, NULL); + secp256k1_gej_double_nonzero(r, r); } n = wnaf_1[i]; diff --git a/src/secp256k1/src/ecmult_gen.h b/src/secp256k1/src/ecmult_gen.h index 7564b7015f..30815e5aa1 100644 --- a/src/secp256k1/src/ecmult_gen.h +++ b/src/secp256k1/src/ecmult_gen.h @@ -10,28 +10,35 @@ #include "scalar.h" #include "group.h" +#if ECMULT_GEN_PREC_BITS != 2 && ECMULT_GEN_PREC_BITS != 4 && ECMULT_GEN_PREC_BITS != 8 +# error "Set ECMULT_GEN_PREC_BITS to 2, 4 or 8." +#endif +#define ECMULT_GEN_PREC_B ECMULT_GEN_PREC_BITS +#define ECMULT_GEN_PREC_G (1 << ECMULT_GEN_PREC_B) +#define ECMULT_GEN_PREC_N (256 / ECMULT_GEN_PREC_B) + typedef struct { /* For accelerating the computation of a*G: * To harden against timing attacks, use the following mechanism: - * * Break up the multiplicand into groups of 4 bits, called n_0, n_1, n_2, ..., n_63. - * * Compute sum(n_i * 16^i * G + U_i, i=0..63), where: - * * U_i = U * 2^i (for i=0..62) - * * U_i = U * (1-2^63) (for i=63) - * where U is a point with no known corresponding scalar. Note that sum(U_i, i=0..63) = 0. - * For each i, and each of the 16 possible values of n_i, (n_i * 16^i * G + U_i) is - * precomputed (call it prec(i, n_i)). The formula now becomes sum(prec(i, n_i), i=0..63). + * * Break up the multiplicand into groups of PREC_B bits, called n_0, n_1, n_2, ..., n_(PREC_N-1). + * * Compute sum(n_i * (PREC_G)^i * G + U_i, i=0 ... PREC_N-1), where: + * * U_i = U * 2^i, for i=0 ... PREC_N-2 + * * U_i = U * (1-2^(PREC_N-1)), for i=PREC_N-1 + * where U is a point with no known corresponding scalar. Note that sum(U_i, i=0 ... PREC_N-1) = 0. + * For each i, and each of the PREC_G possible values of n_i, (n_i * (PREC_G)^i * G + U_i) is + * precomputed (call it prec(i, n_i)). The formula now becomes sum(prec(i, n_i), i=0 ... PREC_N-1). * None of the resulting prec group elements have a known scalar, and neither do any of * the intermediate sums while computing a*G. */ - secp256k1_ge_storage (*prec)[64][16]; /* prec[j][i] = 16^j * i * G + U_i */ + secp256k1_ge_storage (*prec)[ECMULT_GEN_PREC_N][ECMULT_GEN_PREC_G]; /* prec[j][i] = (PREC_G)^j * i * G + U_i */ secp256k1_scalar blind; secp256k1_gej initial; } secp256k1_ecmult_gen_context; +static const size_t SECP256K1_ECMULT_GEN_CONTEXT_PREALLOCATED_SIZE; static void secp256k1_ecmult_gen_context_init(secp256k1_ecmult_gen_context* ctx); -static void secp256k1_ecmult_gen_context_build(secp256k1_ecmult_gen_context* ctx, const secp256k1_callback* cb); -static void secp256k1_ecmult_gen_context_clone(secp256k1_ecmult_gen_context *dst, - const secp256k1_ecmult_gen_context* src, const secp256k1_callback* cb); +static void secp256k1_ecmult_gen_context_build(secp256k1_ecmult_gen_context* ctx, void **prealloc); +static void secp256k1_ecmult_gen_context_finalize_memcpy(secp256k1_ecmult_gen_context *dst, const secp256k1_ecmult_gen_context* src); static void secp256k1_ecmult_gen_context_clear(secp256k1_ecmult_gen_context* ctx); static int secp256k1_ecmult_gen_context_is_built(const secp256k1_ecmult_gen_context* ctx); diff --git a/src/secp256k1/src/ecmult_gen_impl.h b/src/secp256k1/src/ecmult_gen_impl.h index d64505dc00..30ac16518b 100644 --- a/src/secp256k1/src/ecmult_gen_impl.h +++ b/src/secp256k1/src/ecmult_gen_impl.h @@ -7,6 +7,7 @@ #ifndef SECP256K1_ECMULT_GEN_IMPL_H #define SECP256K1_ECMULT_GEN_IMPL_H +#include "util.h" #include "scalar.h" #include "group.h" #include "ecmult_gen.h" @@ -14,23 +15,32 @@ #ifdef USE_ECMULT_STATIC_PRECOMPUTATION #include "ecmult_static_context.h" #endif + +#ifndef USE_ECMULT_STATIC_PRECOMPUTATION + static const size_t SECP256K1_ECMULT_GEN_CONTEXT_PREALLOCATED_SIZE = ROUND_TO_ALIGN(sizeof(*((secp256k1_ecmult_gen_context*) NULL)->prec)); +#else + static const size_t SECP256K1_ECMULT_GEN_CONTEXT_PREALLOCATED_SIZE = 0; +#endif + static void secp256k1_ecmult_gen_context_init(secp256k1_ecmult_gen_context *ctx) { ctx->prec = NULL; } -static void secp256k1_ecmult_gen_context_build(secp256k1_ecmult_gen_context *ctx, const secp256k1_callback* cb) { +static void secp256k1_ecmult_gen_context_build(secp256k1_ecmult_gen_context *ctx, void **prealloc) { #ifndef USE_ECMULT_STATIC_PRECOMPUTATION - secp256k1_ge prec[1024]; + secp256k1_ge prec[ECMULT_GEN_PREC_N * ECMULT_GEN_PREC_G]; secp256k1_gej gj; secp256k1_gej nums_gej; int i, j; + size_t const prealloc_size = SECP256K1_ECMULT_GEN_CONTEXT_PREALLOCATED_SIZE; + void* const base = *prealloc; #endif if (ctx->prec != NULL) { return; } #ifndef USE_ECMULT_STATIC_PRECOMPUTATION - ctx->prec = (secp256k1_ge_storage (*)[64][16])checked_malloc(cb, sizeof(*ctx->prec)); + ctx->prec = (secp256k1_ge_storage (*)[ECMULT_GEN_PREC_N][ECMULT_GEN_PREC_G])manual_alloc(prealloc, prealloc_size, base, prealloc_size); /* get the generator */ secp256k1_gej_set_ge(&gj, &secp256k1_ge_const_g); @@ -54,39 +64,39 @@ static void secp256k1_ecmult_gen_context_build(secp256k1_ecmult_gen_context *ctx /* compute prec. */ { - secp256k1_gej precj[1024]; /* Jacobian versions of prec. */ + secp256k1_gej precj[ECMULT_GEN_PREC_N * ECMULT_GEN_PREC_G]; /* Jacobian versions of prec. */ secp256k1_gej gbase; secp256k1_gej numsbase; - gbase = gj; /* 16^j * G */ + gbase = gj; /* PREC_G^j * G */ numsbase = nums_gej; /* 2^j * nums. */ - for (j = 0; j < 64; j++) { - /* Set precj[j*16 .. j*16+15] to (numsbase, numsbase + gbase, ..., numsbase + 15*gbase). */ - precj[j*16] = numsbase; - for (i = 1; i < 16; i++) { - secp256k1_gej_add_var(&precj[j*16 + i], &precj[j*16 + i - 1], &gbase, NULL); + for (j = 0; j < ECMULT_GEN_PREC_N; j++) { + /* Set precj[j*PREC_G .. j*PREC_G+(PREC_G-1)] to (numsbase, numsbase + gbase, ..., numsbase + (PREC_G-1)*gbase). */ + precj[j*ECMULT_GEN_PREC_G] = numsbase; + for (i = 1; i < ECMULT_GEN_PREC_G; i++) { + secp256k1_gej_add_var(&precj[j*ECMULT_GEN_PREC_G + i], &precj[j*ECMULT_GEN_PREC_G + i - 1], &gbase, NULL); } - /* Multiply gbase by 16. */ - for (i = 0; i < 4; i++) { + /* Multiply gbase by PREC_G. */ + for (i = 0; i < ECMULT_GEN_PREC_B; i++) { secp256k1_gej_double_var(&gbase, &gbase, NULL); } /* Multiply numbase by 2. */ secp256k1_gej_double_var(&numsbase, &numsbase, NULL); - if (j == 62) { + if (j == ECMULT_GEN_PREC_N - 2) { /* In the last iteration, numsbase is (1 - 2^j) * nums instead. */ secp256k1_gej_neg(&numsbase, &numsbase); secp256k1_gej_add_var(&numsbase, &numsbase, &nums_gej, NULL); } } - secp256k1_ge_set_all_gej_var(prec, precj, 1024); + secp256k1_ge_set_all_gej_var(prec, precj, ECMULT_GEN_PREC_N * ECMULT_GEN_PREC_G); } - for (j = 0; j < 64; j++) { - for (i = 0; i < 16; i++) { - secp256k1_ge_to_storage(&(*ctx->prec)[j][i], &prec[j*16 + i]); + for (j = 0; j < ECMULT_GEN_PREC_N; j++) { + for (i = 0; i < ECMULT_GEN_PREC_G; i++) { + secp256k1_ge_to_storage(&(*ctx->prec)[j][i], &prec[j*ECMULT_GEN_PREC_G + i]); } } #else - (void)cb; - ctx->prec = (secp256k1_ge_storage (*)[64][16])secp256k1_ecmult_static_context; + (void)prealloc; + ctx->prec = (secp256k1_ge_storage (*)[ECMULT_GEN_PREC_N][ECMULT_GEN_PREC_G])secp256k1_ecmult_static_context; #endif secp256k1_ecmult_gen_blind(ctx, NULL); } @@ -95,27 +105,18 @@ static int secp256k1_ecmult_gen_context_is_built(const secp256k1_ecmult_gen_cont return ctx->prec != NULL; } -static void secp256k1_ecmult_gen_context_clone(secp256k1_ecmult_gen_context *dst, - const secp256k1_ecmult_gen_context *src, const secp256k1_callback* cb) { - if (src->prec == NULL) { - dst->prec = NULL; - } else { +static void secp256k1_ecmult_gen_context_finalize_memcpy(secp256k1_ecmult_gen_context *dst, const secp256k1_ecmult_gen_context *src) { #ifndef USE_ECMULT_STATIC_PRECOMPUTATION - dst->prec = (secp256k1_ge_storage (*)[64][16])checked_malloc(cb, sizeof(*dst->prec)); - memcpy(dst->prec, src->prec, sizeof(*dst->prec)); + if (src->prec != NULL) { + /* We cast to void* first to suppress a -Wcast-align warning. */ + dst->prec = (secp256k1_ge_storage (*)[ECMULT_GEN_PREC_N][ECMULT_GEN_PREC_G])(void*)((unsigned char*)dst + ((unsigned char*)src->prec - (unsigned char*)src)); + } #else - (void)cb; - dst->prec = src->prec; + (void)dst, (void)src; #endif - dst->initial = src->initial; - dst->blind = src->blind; - } } static void secp256k1_ecmult_gen_context_clear(secp256k1_ecmult_gen_context *ctx) { -#ifndef USE_ECMULT_STATIC_PRECOMPUTATION - free(ctx->prec); -#endif secp256k1_scalar_clear(&ctx->blind); secp256k1_gej_clear(&ctx->initial); ctx->prec = NULL; @@ -132,9 +133,9 @@ static void secp256k1_ecmult_gen(const secp256k1_ecmult_gen_context *ctx, secp25 /* Blind scalar/point multiplication by computing (n-b)G + bG instead of nG. */ secp256k1_scalar_add(&gnb, gn, &ctx->blind); add.infinity = 0; - for (j = 0; j < 64; j++) { - bits = secp256k1_scalar_get_bits(&gnb, j * 4, 4); - for (i = 0; i < 16; i++) { + for (j = 0; j < ECMULT_GEN_PREC_N; j++) { + bits = secp256k1_scalar_get_bits(&gnb, j * ECMULT_GEN_PREC_B, ECMULT_GEN_PREC_B); + for (i = 0; i < ECMULT_GEN_PREC_G; i++) { /** This uses a conditional move to avoid any secret data in array indexes. * _Any_ use of secret indexes has been demonstrated to result in timing * sidechannels, even when the cache-line access patterns are uniform. @@ -162,7 +163,7 @@ static void secp256k1_ecmult_gen_blind(secp256k1_ecmult_gen_context *ctx, const secp256k1_fe s; unsigned char nonce32[32]; secp256k1_rfc6979_hmac_sha256 rng; - int retry; + int overflow; unsigned char keydata[64] = {0}; if (seed32 == NULL) { /* When seed is NULL, reset the initial point and blinding value. */ @@ -182,21 +183,18 @@ static void secp256k1_ecmult_gen_blind(secp256k1_ecmult_gen_context *ctx, const } secp256k1_rfc6979_hmac_sha256_initialize(&rng, keydata, seed32 ? 64 : 32); memset(keydata, 0, sizeof(keydata)); - /* Retry for out of range results to achieve uniformity. */ - do { - secp256k1_rfc6979_hmac_sha256_generate(&rng, nonce32, 32); - retry = !secp256k1_fe_set_b32(&s, nonce32); - retry |= secp256k1_fe_is_zero(&s); - } while (retry); /* This branch true is cryptographically unreachable. Requires sha256_hmac output > Fp. */ + /* Accept unobservably small non-uniformity. */ + secp256k1_rfc6979_hmac_sha256_generate(&rng, nonce32, 32); + overflow = !secp256k1_fe_set_b32(&s, nonce32); + overflow |= secp256k1_fe_is_zero(&s); + secp256k1_fe_cmov(&s, &secp256k1_fe_one, overflow); /* Randomize the projection to defend against multiplier sidechannels. */ secp256k1_gej_rescale(&ctx->initial, &s); secp256k1_fe_clear(&s); - do { - secp256k1_rfc6979_hmac_sha256_generate(&rng, nonce32, 32); - secp256k1_scalar_set_b32(&b, nonce32, &retry); - /* A blinding value of 0 works, but would undermine the projection hardening. */ - retry |= secp256k1_scalar_is_zero(&b); - } while (retry); /* This branch true is cryptographically unreachable. Requires sha256_hmac output > order. */ + secp256k1_rfc6979_hmac_sha256_generate(&rng, nonce32, 32); + secp256k1_scalar_set_b32(&b, nonce32, NULL); + /* A blinding value of 0 works, but would undermine the projection hardening. */ + secp256k1_scalar_cmov(&b, &secp256k1_scalar_one, secp256k1_scalar_is_zero(&b)); secp256k1_rfc6979_hmac_sha256_finalize(&rng); memset(nonce32, 0, 32); secp256k1_ecmult_gen(ctx, &gb, &b); diff --git a/src/secp256k1/src/ecmult_impl.h b/src/secp256k1/src/ecmult_impl.h index 1986914a4f..f03fa9469d 100644 --- a/src/secp256k1/src/ecmult_impl.h +++ b/src/secp256k1/src/ecmult_impl.h @@ -10,6 +10,7 @@ #include <string.h> #include <stdint.h> +#include "util.h" #include "group.h" #include "scalar.h" #include "ecmult.h" @@ -30,16 +31,32 @@ # endif #else /* optimal for 128-bit and 256-bit exponents. */ -#define WINDOW_A 5 -/** larger numbers may result in slightly better performance, at the cost of - exponentially larger precomputed tables. */ -#ifdef USE_ENDOMORPHISM -/** Two tables for window size 15: 1.375 MiB. */ -#define WINDOW_G 15 -#else -/** One table for window size 16: 1.375 MiB. */ -#define WINDOW_G 16 +# define WINDOW_A 5 +/** Larger values for ECMULT_WINDOW_SIZE result in possibly better + * performance at the cost of an exponentially larger precomputed + * table. The exact table size is + * (1 << (WINDOW_G - 2)) * sizeof(secp256k1_ge_storage) bytes, + * where sizeof(secp256k1_ge_storage) is typically 64 bytes but can + * be larger due to platform-specific padding and alignment. + * If the endomorphism optimization is enabled (USE_ENDOMORMPHSIM) + * two tables of this size are used instead of only one. + */ +# define WINDOW_G ECMULT_WINDOW_SIZE #endif + +/* Noone will ever need more than a window size of 24. The code might + * be correct for larger values of ECMULT_WINDOW_SIZE but this is not + * not tested. + * + * The following limitations are known, and there are probably more: + * If WINDOW_G > 27 and size_t has 32 bits, then the code is incorrect + * because the size of the memory object that we allocate (in bytes) + * will not fit in a size_t. + * If WINDOW_G > 31 and int has 32 bits, then the code is incorrect + * because certain expressions will overflow. + */ +#if ECMULT_WINDOW_SIZE < 2 || ECMULT_WINDOW_SIZE > 24 +# error Set ECMULT_WINDOW_SIZE to an integer in range [2..24]. #endif #ifdef USE_ENDOMORPHISM @@ -121,7 +138,7 @@ static void secp256k1_ecmult_odd_multiples_table(int n, secp256k1_gej *prej, sec * It only operates on tables sized for WINDOW_A wnaf multiples. * - secp256k1_ecmult_odd_multiples_table_storage_var, which converts its * resulting point set to actually affine points, and stores those in pre. - * It operates on tables of any size, but uses heap-allocated temporaries. + * It operates on tables of any size. * * To compute a*P + b*G, we compute a table for P using the first function, * and for G using the second (which requires an inverse, but it only needs to @@ -294,6 +311,13 @@ static void secp256k1_ecmult_odd_multiples_table_storage_var(const int n, secp25 } \ } while(0) +static const size_t SECP256K1_ECMULT_CONTEXT_PREALLOCATED_SIZE = + ROUND_TO_ALIGN(sizeof((*((secp256k1_ecmult_context*) NULL)->pre_g)[0]) * ECMULT_TABLE_SIZE(WINDOW_G)) +#ifdef USE_ENDOMORPHISM + + ROUND_TO_ALIGN(sizeof((*((secp256k1_ecmult_context*) NULL)->pre_g_128)[0]) * ECMULT_TABLE_SIZE(WINDOW_G)) +#endif + ; + static void secp256k1_ecmult_context_init(secp256k1_ecmult_context *ctx) { ctx->pre_g = NULL; #ifdef USE_ENDOMORPHISM @@ -301,8 +325,10 @@ static void secp256k1_ecmult_context_init(secp256k1_ecmult_context *ctx) { #endif } -static void secp256k1_ecmult_context_build(secp256k1_ecmult_context *ctx, const secp256k1_callback *cb) { +static void secp256k1_ecmult_context_build(secp256k1_ecmult_context *ctx, void **prealloc) { secp256k1_gej gj; + void* const base = *prealloc; + size_t const prealloc_size = SECP256K1_ECMULT_CONTEXT_PREALLOCATED_SIZE; if (ctx->pre_g != NULL) { return; @@ -311,7 +337,12 @@ static void secp256k1_ecmult_context_build(secp256k1_ecmult_context *ctx, const /* get the generator */ secp256k1_gej_set_ge(&gj, &secp256k1_ge_const_g); - ctx->pre_g = (secp256k1_ge_storage (*)[])checked_malloc(cb, sizeof((*ctx->pre_g)[0]) * ECMULT_TABLE_SIZE(WINDOW_G)); + { + size_t size = sizeof((*ctx->pre_g)[0]) * ((size_t)ECMULT_TABLE_SIZE(WINDOW_G)); + /* check for overflow */ + VERIFY_CHECK(size / sizeof((*ctx->pre_g)[0]) == ((size_t)ECMULT_TABLE_SIZE(WINDOW_G))); + ctx->pre_g = (secp256k1_ge_storage (*)[])manual_alloc(prealloc, sizeof((*ctx->pre_g)[0]) * ECMULT_TABLE_SIZE(WINDOW_G), base, prealloc_size); + } /* precompute the tables with odd multiples */ secp256k1_ecmult_odd_multiples_table_storage_var(ECMULT_TABLE_SIZE(WINDOW_G), *ctx->pre_g, &gj); @@ -321,7 +352,10 @@ static void secp256k1_ecmult_context_build(secp256k1_ecmult_context *ctx, const secp256k1_gej g_128j; int i; - ctx->pre_g_128 = (secp256k1_ge_storage (*)[])checked_malloc(cb, sizeof((*ctx->pre_g_128)[0]) * ECMULT_TABLE_SIZE(WINDOW_G)); + size_t size = sizeof((*ctx->pre_g_128)[0]) * ((size_t) ECMULT_TABLE_SIZE(WINDOW_G)); + /* check for overflow */ + VERIFY_CHECK(size / sizeof((*ctx->pre_g_128)[0]) == ((size_t)ECMULT_TABLE_SIZE(WINDOW_G))); + ctx->pre_g_128 = (secp256k1_ge_storage (*)[])manual_alloc(prealloc, sizeof((*ctx->pre_g_128)[0]) * ECMULT_TABLE_SIZE(WINDOW_G), base, prealloc_size); /* calculate 2^128*generator */ g_128j = gj; @@ -333,22 +367,14 @@ static void secp256k1_ecmult_context_build(secp256k1_ecmult_context *ctx, const #endif } -static void secp256k1_ecmult_context_clone(secp256k1_ecmult_context *dst, - const secp256k1_ecmult_context *src, const secp256k1_callback *cb) { - if (src->pre_g == NULL) { - dst->pre_g = NULL; - } else { - size_t size = sizeof((*dst->pre_g)[0]) * ECMULT_TABLE_SIZE(WINDOW_G); - dst->pre_g = (secp256k1_ge_storage (*)[])checked_malloc(cb, size); - memcpy(dst->pre_g, src->pre_g, size); +static void secp256k1_ecmult_context_finalize_memcpy(secp256k1_ecmult_context *dst, const secp256k1_ecmult_context *src) { + if (src->pre_g != NULL) { + /* We cast to void* first to suppress a -Wcast-align warning. */ + dst->pre_g = (secp256k1_ge_storage (*)[])(void*)((unsigned char*)dst + ((unsigned char*)(src->pre_g) - (unsigned char*)src)); } #ifdef USE_ENDOMORPHISM - if (src->pre_g_128 == NULL) { - dst->pre_g_128 = NULL; - } else { - size_t size = sizeof((*dst->pre_g_128)[0]) * ECMULT_TABLE_SIZE(WINDOW_G); - dst->pre_g_128 = (secp256k1_ge_storage (*)[])checked_malloc(cb, size); - memcpy(dst->pre_g_128, src->pre_g_128, size); + if (src->pre_g_128 != NULL) { + dst->pre_g_128 = (secp256k1_ge_storage (*)[])(void*)((unsigned char*)dst + ((unsigned char*)(src->pre_g_128) - (unsigned char*)src)); } #endif } @@ -358,10 +384,6 @@ static int secp256k1_ecmult_context_is_built(const secp256k1_ecmult_context *ctx } static void secp256k1_ecmult_context_clear(secp256k1_ecmult_context *ctx) { - free(ctx->pre_g); -#ifdef USE_ENDOMORPHISM - free(ctx->pre_g_128); -#endif secp256k1_ecmult_context_init(ctx); } @@ -373,7 +395,7 @@ static void secp256k1_ecmult_context_clear(secp256k1_ecmult_context *ctx) { * than the number of bits in the (absolute value) of the input. */ static int secp256k1_ecmult_wnaf(int *wnaf, int len, const secp256k1_scalar *a, int w) { - secp256k1_scalar s = *a; + secp256k1_scalar s; int last_set_bit = -1; int bit = 0; int sign = 1; @@ -386,6 +408,7 @@ static int secp256k1_ecmult_wnaf(int *wnaf, int len, const secp256k1_scalar *a, memset(wnaf, 0, len * sizeof(wnaf[0])); + s = *a; if (secp256k1_scalar_get_bits(&s, 255, 1)) { secp256k1_scalar_negate(&s, &s); sign = -1; @@ -418,7 +441,7 @@ static int secp256k1_ecmult_wnaf(int *wnaf, int len, const secp256k1_scalar *a, CHECK(carry == 0); while (bit < 256) { CHECK(secp256k1_scalar_get_bits(&s, bit++, 1) == 0); - } + } #endif return last_set_bit + 1; } @@ -626,52 +649,55 @@ static size_t secp256k1_strauss_scratch_size(size_t n_points) { return n_points*point_size; } -static int secp256k1_ecmult_strauss_batch(const secp256k1_ecmult_context *ctx, secp256k1_scratch *scratch, secp256k1_gej *r, const secp256k1_scalar *inp_g_sc, secp256k1_ecmult_multi_callback cb, void *cbdata, size_t n_points, size_t cb_offset) { +static int secp256k1_ecmult_strauss_batch(const secp256k1_callback* error_callback, const secp256k1_ecmult_context *ctx, secp256k1_scratch *scratch, secp256k1_gej *r, const secp256k1_scalar *inp_g_sc, secp256k1_ecmult_multi_callback cb, void *cbdata, size_t n_points, size_t cb_offset) { secp256k1_gej* points; secp256k1_scalar* scalars; struct secp256k1_strauss_state state; size_t i; + const size_t scratch_checkpoint = secp256k1_scratch_checkpoint(error_callback, scratch); secp256k1_gej_set_infinity(r); if (inp_g_sc == NULL && n_points == 0) { return 1; } - if (!secp256k1_scratch_allocate_frame(scratch, secp256k1_strauss_scratch_size(n_points), STRAUSS_SCRATCH_OBJECTS)) { - return 0; - } - points = (secp256k1_gej*)secp256k1_scratch_alloc(scratch, n_points * sizeof(secp256k1_gej)); - scalars = (secp256k1_scalar*)secp256k1_scratch_alloc(scratch, n_points * sizeof(secp256k1_scalar)); - state.prej = (secp256k1_gej*)secp256k1_scratch_alloc(scratch, n_points * ECMULT_TABLE_SIZE(WINDOW_A) * sizeof(secp256k1_gej)); - state.zr = (secp256k1_fe*)secp256k1_scratch_alloc(scratch, n_points * ECMULT_TABLE_SIZE(WINDOW_A) * sizeof(secp256k1_fe)); + points = (secp256k1_gej*)secp256k1_scratch_alloc(error_callback, scratch, n_points * sizeof(secp256k1_gej)); + scalars = (secp256k1_scalar*)secp256k1_scratch_alloc(error_callback, scratch, n_points * sizeof(secp256k1_scalar)); + state.prej = (secp256k1_gej*)secp256k1_scratch_alloc(error_callback, scratch, n_points * ECMULT_TABLE_SIZE(WINDOW_A) * sizeof(secp256k1_gej)); + state.zr = (secp256k1_fe*)secp256k1_scratch_alloc(error_callback, scratch, n_points * ECMULT_TABLE_SIZE(WINDOW_A) * sizeof(secp256k1_fe)); #ifdef USE_ENDOMORPHISM - state.pre_a = (secp256k1_ge*)secp256k1_scratch_alloc(scratch, n_points * 2 * ECMULT_TABLE_SIZE(WINDOW_A) * sizeof(secp256k1_ge)); + state.pre_a = (secp256k1_ge*)secp256k1_scratch_alloc(error_callback, scratch, n_points * 2 * ECMULT_TABLE_SIZE(WINDOW_A) * sizeof(secp256k1_ge)); state.pre_a_lam = state.pre_a + n_points * ECMULT_TABLE_SIZE(WINDOW_A); #else - state.pre_a = (secp256k1_ge*)secp256k1_scratch_alloc(scratch, n_points * ECMULT_TABLE_SIZE(WINDOW_A) * sizeof(secp256k1_ge)); + state.pre_a = (secp256k1_ge*)secp256k1_scratch_alloc(error_callback, scratch, n_points * ECMULT_TABLE_SIZE(WINDOW_A) * sizeof(secp256k1_ge)); #endif - state.ps = (struct secp256k1_strauss_point_state*)secp256k1_scratch_alloc(scratch, n_points * sizeof(struct secp256k1_strauss_point_state)); + state.ps = (struct secp256k1_strauss_point_state*)secp256k1_scratch_alloc(error_callback, scratch, n_points * sizeof(struct secp256k1_strauss_point_state)); + + if (points == NULL || scalars == NULL || state.prej == NULL || state.zr == NULL || state.pre_a == NULL) { + secp256k1_scratch_apply_checkpoint(error_callback, scratch, scratch_checkpoint); + return 0; + } for (i = 0; i < n_points; i++) { secp256k1_ge point; if (!cb(&scalars[i], &point, i+cb_offset, cbdata)) { - secp256k1_scratch_deallocate_frame(scratch); + secp256k1_scratch_apply_checkpoint(error_callback, scratch, scratch_checkpoint); return 0; } secp256k1_gej_set_ge(&points[i], &point); } secp256k1_ecmult_strauss_wnaf(ctx, &state, r, n_points, points, scalars, inp_g_sc); - secp256k1_scratch_deallocate_frame(scratch); + secp256k1_scratch_apply_checkpoint(error_callback, scratch, scratch_checkpoint); return 1; } /* Wrapper for secp256k1_ecmult_multi_func interface */ -static int secp256k1_ecmult_strauss_batch_single(const secp256k1_ecmult_context *actx, secp256k1_scratch *scratch, secp256k1_gej *r, const secp256k1_scalar *inp_g_sc, secp256k1_ecmult_multi_callback cb, void *cbdata, size_t n) { - return secp256k1_ecmult_strauss_batch(actx, scratch, r, inp_g_sc, cb, cbdata, n, 0); +static int secp256k1_ecmult_strauss_batch_single(const secp256k1_callback* error_callback, const secp256k1_ecmult_context *actx, secp256k1_scratch *scratch, secp256k1_gej *r, const secp256k1_scalar *inp_g_sc, secp256k1_ecmult_multi_callback cb, void *cbdata, size_t n) { + return secp256k1_ecmult_strauss_batch(error_callback, actx, scratch, r, inp_g_sc, cb, cbdata, n, 0); } -static size_t secp256k1_strauss_max_points(secp256k1_scratch *scratch) { - return secp256k1_scratch_max_allocation(scratch, STRAUSS_SCRATCH_OBJECTS) / secp256k1_strauss_scratch_size(1); +static size_t secp256k1_strauss_max_points(const secp256k1_callback* error_callback, secp256k1_scratch *scratch) { + return secp256k1_scratch_max_allocation(error_callback, scratch, STRAUSS_SCRATCH_OBJECTS) / secp256k1_strauss_scratch_size(1); } /** Convert a number to WNAF notation. @@ -963,7 +989,8 @@ static size_t secp256k1_pippenger_scratch_size(size_t n_points, int bucket_windo return (sizeof(secp256k1_gej) << bucket_window) + sizeof(struct secp256k1_pippenger_state) + entries * entry_size; } -static int secp256k1_ecmult_pippenger_batch(const secp256k1_ecmult_context *ctx, secp256k1_scratch *scratch, secp256k1_gej *r, const secp256k1_scalar *inp_g_sc, secp256k1_ecmult_multi_callback cb, void *cbdata, size_t n_points, size_t cb_offset) { +static int secp256k1_ecmult_pippenger_batch(const secp256k1_callback* error_callback, const secp256k1_ecmult_context *ctx, secp256k1_scratch *scratch, secp256k1_gej *r, const secp256k1_scalar *inp_g_sc, secp256k1_ecmult_multi_callback cb, void *cbdata, size_t n_points, size_t cb_offset) { + const size_t scratch_checkpoint = secp256k1_scratch_checkpoint(error_callback, scratch); /* Use 2(n+1) with the endomorphism, n+1 without, when calculating batch * sizes. The reason for +1 is that we add the G scalar to the list of * other scalars. */ @@ -988,15 +1015,21 @@ static int secp256k1_ecmult_pippenger_batch(const secp256k1_ecmult_context *ctx, } bucket_window = secp256k1_pippenger_bucket_window(n_points); - if (!secp256k1_scratch_allocate_frame(scratch, secp256k1_pippenger_scratch_size(n_points, bucket_window), PIPPENGER_SCRATCH_OBJECTS)) { + points = (secp256k1_ge *) secp256k1_scratch_alloc(error_callback, scratch, entries * sizeof(*points)); + scalars = (secp256k1_scalar *) secp256k1_scratch_alloc(error_callback, scratch, entries * sizeof(*scalars)); + state_space = (struct secp256k1_pippenger_state *) secp256k1_scratch_alloc(error_callback, scratch, sizeof(*state_space)); + if (points == NULL || scalars == NULL || state_space == NULL) { + secp256k1_scratch_apply_checkpoint(error_callback, scratch, scratch_checkpoint); + return 0; + } + + state_space->ps = (struct secp256k1_pippenger_point_state *) secp256k1_scratch_alloc(error_callback, scratch, entries * sizeof(*state_space->ps)); + state_space->wnaf_na = (int *) secp256k1_scratch_alloc(error_callback, scratch, entries*(WNAF_SIZE(bucket_window+1)) * sizeof(int)); + buckets = (secp256k1_gej *) secp256k1_scratch_alloc(error_callback, scratch, (1<<bucket_window) * sizeof(*buckets)); + if (state_space->ps == NULL || state_space->wnaf_na == NULL || buckets == NULL) { + secp256k1_scratch_apply_checkpoint(error_callback, scratch, scratch_checkpoint); return 0; } - points = (secp256k1_ge *) secp256k1_scratch_alloc(scratch, entries * sizeof(*points)); - scalars = (secp256k1_scalar *) secp256k1_scratch_alloc(scratch, entries * sizeof(*scalars)); - state_space = (struct secp256k1_pippenger_state *) secp256k1_scratch_alloc(scratch, sizeof(*state_space)); - state_space->ps = (struct secp256k1_pippenger_point_state *) secp256k1_scratch_alloc(scratch, entries * sizeof(*state_space->ps)); - state_space->wnaf_na = (int *) secp256k1_scratch_alloc(scratch, entries*(WNAF_SIZE(bucket_window+1)) * sizeof(int)); - buckets = (secp256k1_gej *) secp256k1_scratch_alloc(scratch, sizeof(*buckets) << bucket_window); if (inp_g_sc != NULL) { scalars[0] = *inp_g_sc; @@ -1010,7 +1043,7 @@ static int secp256k1_ecmult_pippenger_batch(const secp256k1_ecmult_context *ctx, while (point_idx < n_points) { if (!cb(&scalars[idx], &points[idx], point_idx + cb_offset, cbdata)) { - secp256k1_scratch_deallocate_frame(scratch); + secp256k1_scratch_apply_checkpoint(error_callback, scratch, scratch_checkpoint); return 0; } idx++; @@ -1034,13 +1067,13 @@ static int secp256k1_ecmult_pippenger_batch(const secp256k1_ecmult_context *ctx, for(i = 0; i < 1<<bucket_window; i++) { secp256k1_gej_clear(&buckets[i]); } - secp256k1_scratch_deallocate_frame(scratch); + secp256k1_scratch_apply_checkpoint(error_callback, scratch, scratch_checkpoint); return 1; } /* Wrapper for secp256k1_ecmult_multi_func interface */ -static int secp256k1_ecmult_pippenger_batch_single(const secp256k1_ecmult_context *actx, secp256k1_scratch *scratch, secp256k1_gej *r, const secp256k1_scalar *inp_g_sc, secp256k1_ecmult_multi_callback cb, void *cbdata, size_t n) { - return secp256k1_ecmult_pippenger_batch(actx, scratch, r, inp_g_sc, cb, cbdata, n, 0); +static int secp256k1_ecmult_pippenger_batch_single(const secp256k1_callback* error_callback, const secp256k1_ecmult_context *actx, secp256k1_scratch *scratch, secp256k1_gej *r, const secp256k1_scalar *inp_g_sc, secp256k1_ecmult_multi_callback cb, void *cbdata, size_t n) { + return secp256k1_ecmult_pippenger_batch(error_callback, actx, scratch, r, inp_g_sc, cb, cbdata, n, 0); } /** @@ -1048,8 +1081,8 @@ static int secp256k1_ecmult_pippenger_batch_single(const secp256k1_ecmult_contex * a given scratch space. The function ensures that fewer points may also be * used. */ -static size_t secp256k1_pippenger_max_points(secp256k1_scratch *scratch) { - size_t max_alloc = secp256k1_scratch_max_allocation(scratch, PIPPENGER_SCRATCH_OBJECTS); +static size_t secp256k1_pippenger_max_points(const secp256k1_callback* error_callback, secp256k1_scratch *scratch) { + size_t max_alloc = secp256k1_scratch_max_allocation(error_callback, scratch, PIPPENGER_SCRATCH_OBJECTS); int bucket_window; size_t res = 0; @@ -1131,11 +1164,11 @@ static int secp256k1_ecmult_multi_batch_size_helper(size_t *n_batches, size_t *n return 1; } -typedef int (*secp256k1_ecmult_multi_func)(const secp256k1_ecmult_context*, secp256k1_scratch*, secp256k1_gej*, const secp256k1_scalar*, secp256k1_ecmult_multi_callback cb, void*, size_t); -static int secp256k1_ecmult_multi_var(const secp256k1_ecmult_context *ctx, secp256k1_scratch *scratch, secp256k1_gej *r, const secp256k1_scalar *inp_g_sc, secp256k1_ecmult_multi_callback cb, void *cbdata, size_t n) { +typedef int (*secp256k1_ecmult_multi_func)(const secp256k1_callback* error_callback, const secp256k1_ecmult_context*, secp256k1_scratch*, secp256k1_gej*, const secp256k1_scalar*, secp256k1_ecmult_multi_callback cb, void*, size_t); +static int secp256k1_ecmult_multi_var(const secp256k1_callback* error_callback, const secp256k1_ecmult_context *ctx, secp256k1_scratch *scratch, secp256k1_gej *r, const secp256k1_scalar *inp_g_sc, secp256k1_ecmult_multi_callback cb, void *cbdata, size_t n) { size_t i; - int (*f)(const secp256k1_ecmult_context*, secp256k1_scratch*, secp256k1_gej*, const secp256k1_scalar*, secp256k1_ecmult_multi_callback cb, void*, size_t, size_t); + int (*f)(const secp256k1_callback* error_callback, const secp256k1_ecmult_context*, secp256k1_scratch*, secp256k1_gej*, const secp256k1_scalar*, secp256k1_ecmult_multi_callback cb, void*, size_t, size_t); size_t n_batches; size_t n_batch_points; @@ -1152,16 +1185,18 @@ static int secp256k1_ecmult_multi_var(const secp256k1_ecmult_context *ctx, secp2 return secp256k1_ecmult_multi_simple_var(ctx, r, inp_g_sc, cb, cbdata, n); } - /* Compute the batch sizes for pippenger given a scratch space. If it's greater than a threshold - * use pippenger. Otherwise use strauss */ - if (!secp256k1_ecmult_multi_batch_size_helper(&n_batches, &n_batch_points, secp256k1_pippenger_max_points(scratch), n)) { - return 0; + /* Compute the batch sizes for Pippenger's algorithm given a scratch space. If it's greater than + * a threshold use Pippenger's algorithm. Otherwise use Strauss' algorithm. + * As a first step check if there's enough space for Pippenger's algo (which requires less space + * than Strauss' algo) and if not, use the simple algorithm. */ + if (!secp256k1_ecmult_multi_batch_size_helper(&n_batches, &n_batch_points, secp256k1_pippenger_max_points(error_callback, scratch), n)) { + return secp256k1_ecmult_multi_simple_var(ctx, r, inp_g_sc, cb, cbdata, n); } if (n_batch_points >= ECMULT_PIPPENGER_THRESHOLD) { f = secp256k1_ecmult_pippenger_batch; } else { - if (!secp256k1_ecmult_multi_batch_size_helper(&n_batches, &n_batch_points, secp256k1_strauss_max_points(scratch), n)) { - return 0; + if (!secp256k1_ecmult_multi_batch_size_helper(&n_batches, &n_batch_points, secp256k1_strauss_max_points(error_callback, scratch), n)) { + return secp256k1_ecmult_multi_simple_var(ctx, r, inp_g_sc, cb, cbdata, n); } f = secp256k1_ecmult_strauss_batch; } @@ -1169,7 +1204,7 @@ static int secp256k1_ecmult_multi_var(const secp256k1_ecmult_context *ctx, secp2 size_t nbp = n < n_batch_points ? n : n_batch_points; size_t offset = n_batch_points*i; secp256k1_gej tmp; - if (!f(ctx, scratch, &tmp, i == 0 ? inp_g_sc : NULL, cb, cbdata, nbp, offset)) { + if (!f(error_callback, ctx, scratch, &tmp, i == 0 ? inp_g_sc : NULL, cb, cbdata, nbp, offset)) { return 0; } secp256k1_gej_add_var(r, r, &tmp, NULL); diff --git a/src/secp256k1/src/field.h b/src/secp256k1/src/field.h index bb6692ad57..7993a1f11e 100644 --- a/src/secp256k1/src/field.h +++ b/src/secp256k1/src/field.h @@ -32,10 +32,12 @@ #include "util.h" -/** Normalize a field element. */ +/** Normalize a field element. This brings the field element to a canonical representation, reduces + * its magnitude to 1, and reduces it modulo field size `p`. + */ static void secp256k1_fe_normalize(secp256k1_fe *r); -/** Weakly normalize a field element: reduce it magnitude to 1, but don't fully normalize. */ +/** Weakly normalize a field element: reduce its magnitude to 1, but don't fully normalize. */ static void secp256k1_fe_normalize_weak(secp256k1_fe *r); /** Normalize a field element, without constant-time guarantee. */ @@ -123,10 +125,10 @@ static void secp256k1_fe_to_storage(secp256k1_fe_storage *r, const secp256k1_fe /** Convert a field element back from the storage type. */ static void secp256k1_fe_from_storage(secp256k1_fe *r, const secp256k1_fe_storage *a); -/** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. */ +/** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. Both *r and *a must be initialized.*/ static void secp256k1_fe_storage_cmov(secp256k1_fe_storage *r, const secp256k1_fe_storage *a, int flag); -/** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. */ +/** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. Both *r and *a must be initialized.*/ static void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag); #endif /* SECP256K1_FIELD_H */ diff --git a/src/secp256k1/src/field_10x26_impl.h b/src/secp256k1/src/field_10x26_impl.h index 4ae4fdcec8..651500ee8e 100644 --- a/src/secp256k1/src/field_10x26_impl.h +++ b/src/secp256k1/src/field_10x26_impl.h @@ -320,6 +320,7 @@ static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { } static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { + int ret; r->n[0] = (uint32_t)a[31] | ((uint32_t)a[30] << 8) | ((uint32_t)a[29] << 16) | ((uint32_t)(a[28] & 0x3) << 24); r->n[1] = (uint32_t)((a[28] >> 2) & 0x3f) | ((uint32_t)a[27] << 6) | ((uint32_t)a[26] << 14) | ((uint32_t)(a[25] & 0xf) << 22); r->n[2] = (uint32_t)((a[25] >> 4) & 0xf) | ((uint32_t)a[24] << 4) | ((uint32_t)a[23] << 12) | ((uint32_t)(a[22] & 0x3f) << 20); @@ -331,15 +332,17 @@ static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { r->n[8] = (uint32_t)a[5] | ((uint32_t)a[4] << 8) | ((uint32_t)a[3] << 16) | ((uint32_t)(a[2] & 0x3) << 24); r->n[9] = (uint32_t)((a[2] >> 2) & 0x3f) | ((uint32_t)a[1] << 6) | ((uint32_t)a[0] << 14); - if (r->n[9] == 0x3FFFFFUL && (r->n[8] & r->n[7] & r->n[6] & r->n[5] & r->n[4] & r->n[3] & r->n[2]) == 0x3FFFFFFUL && (r->n[1] + 0x40UL + ((r->n[0] + 0x3D1UL) >> 26)) > 0x3FFFFFFUL) { - return 0; - } + ret = !((r->n[9] == 0x3FFFFFUL) & ((r->n[8] & r->n[7] & r->n[6] & r->n[5] & r->n[4] & r->n[3] & r->n[2]) == 0x3FFFFFFUL) & ((r->n[1] + 0x40UL + ((r->n[0] + 0x3D1UL) >> 26)) > 0x3FFFFFFUL)); #ifdef VERIFY r->magnitude = 1; - r->normalized = 1; - secp256k1_fe_verify(r); + if (ret) { + r->normalized = 1; + secp256k1_fe_verify(r); + } else { + r->normalized = 0; + } #endif - return 1; + return ret; } /** Convert a field element to a 32-byte big endian value. Requires the input to be normalized */ @@ -1094,6 +1097,7 @@ static void secp256k1_fe_sqr(secp256k1_fe *r, const secp256k1_fe *a) { static SECP256K1_INLINE void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag) { uint32_t mask0, mask1; + VG_CHECK_VERIFY(r->n, sizeof(r->n)); mask0 = flag + ~((uint32_t)0); mask1 = ~mask0; r->n[0] = (r->n[0] & mask0) | (a->n[0] & mask1); @@ -1107,15 +1111,16 @@ static SECP256K1_INLINE void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_ r->n[8] = (r->n[8] & mask0) | (a->n[8] & mask1); r->n[9] = (r->n[9] & mask0) | (a->n[9] & mask1); #ifdef VERIFY - if (a->magnitude > r->magnitude) { + if (flag) { r->magnitude = a->magnitude; + r->normalized = a->normalized; } - r->normalized &= a->normalized; #endif } static SECP256K1_INLINE void secp256k1_fe_storage_cmov(secp256k1_fe_storage *r, const secp256k1_fe_storage *a, int flag) { uint32_t mask0, mask1; + VG_CHECK_VERIFY(r->n, sizeof(r->n)); mask0 = flag + ~((uint32_t)0); mask1 = ~mask0; r->n[0] = (r->n[0] & mask0) | (a->n[0] & mask1); diff --git a/src/secp256k1/src/field_5x52_impl.h b/src/secp256k1/src/field_5x52_impl.h index f4263320d5..71a38f915b 100644 --- a/src/secp256k1/src/field_5x52_impl.h +++ b/src/secp256k1/src/field_5x52_impl.h @@ -283,6 +283,7 @@ static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { } static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { + int ret; r->n[0] = (uint64_t)a[31] | ((uint64_t)a[30] << 8) | ((uint64_t)a[29] << 16) @@ -317,15 +318,17 @@ static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { | ((uint64_t)a[2] << 24) | ((uint64_t)a[1] << 32) | ((uint64_t)a[0] << 40); - if (r->n[4] == 0x0FFFFFFFFFFFFULL && (r->n[3] & r->n[2] & r->n[1]) == 0xFFFFFFFFFFFFFULL && r->n[0] >= 0xFFFFEFFFFFC2FULL) { - return 0; - } + ret = !((r->n[4] == 0x0FFFFFFFFFFFFULL) & ((r->n[3] & r->n[2] & r->n[1]) == 0xFFFFFFFFFFFFFULL) & (r->n[0] >= 0xFFFFEFFFFFC2FULL)); #ifdef VERIFY r->magnitude = 1; - r->normalized = 1; - secp256k1_fe_verify(r); + if (ret) { + r->normalized = 1; + secp256k1_fe_verify(r); + } else { + r->normalized = 0; + } #endif - return 1; + return ret; } /** Convert a field element to a 32-byte big endian value. Requires the input to be normalized */ @@ -446,6 +449,7 @@ static void secp256k1_fe_sqr(secp256k1_fe *r, const secp256k1_fe *a) { static SECP256K1_INLINE void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag) { uint64_t mask0, mask1; + VG_CHECK_VERIFY(r->n, sizeof(r->n)); mask0 = flag + ~((uint64_t)0); mask1 = ~mask0; r->n[0] = (r->n[0] & mask0) | (a->n[0] & mask1); @@ -454,15 +458,16 @@ static SECP256K1_INLINE void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_ r->n[3] = (r->n[3] & mask0) | (a->n[3] & mask1); r->n[4] = (r->n[4] & mask0) | (a->n[4] & mask1); #ifdef VERIFY - if (a->magnitude > r->magnitude) { + if (flag) { r->magnitude = a->magnitude; + r->normalized = a->normalized; } - r->normalized &= a->normalized; #endif } static SECP256K1_INLINE void secp256k1_fe_storage_cmov(secp256k1_fe_storage *r, const secp256k1_fe_storage *a, int flag) { uint64_t mask0, mask1; + VG_CHECK_VERIFY(r->n, sizeof(r->n)); mask0 = flag + ~((uint64_t)0); mask1 = ~mask0; r->n[0] = (r->n[0] & mask0) | (a->n[0] & mask1); diff --git a/src/secp256k1/src/field_impl.h b/src/secp256k1/src/field_impl.h index 6070caccfe..485921a60e 100644 --- a/src/secp256k1/src/field_impl.h +++ b/src/secp256k1/src/field_impl.h @@ -315,4 +315,6 @@ static int secp256k1_fe_is_quad_var(const secp256k1_fe *a) { #endif } +static const secp256k1_fe secp256k1_fe_one = SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 1); + #endif /* SECP256K1_FIELD_IMPL_H */ diff --git a/src/secp256k1/src/gen_context.c b/src/secp256k1/src/gen_context.c index 87d296ebf0..539f574bfd 100644 --- a/src/secp256k1/src/gen_context.c +++ b/src/secp256k1/src/gen_context.c @@ -4,10 +4,16 @@ * file COPYING or http://www.opensource.org/licenses/mit-license.php.* **********************************************************************/ +// Autotools creates libsecp256k1-config.h, of which ECMULT_GEN_PREC_BITS is needed. +// ifndef guard so downstream users can define their own if they do not use autotools. +#if !defined(ECMULT_GEN_PREC_BITS) +#include "libsecp256k1-config.h" +#endif #define USE_BASIC_CONFIG 1 - #include "basic-config.h" + #include "include/secp256k1.h" +#include "util.h" #include "field_impl.h" #include "scalar_impl.h" #include "group_impl.h" @@ -26,6 +32,7 @@ static const secp256k1_callback default_error_callback = { int main(int argc, char **argv) { secp256k1_ecmult_gen_context ctx; + void *prealloc, *base; int inner; int outer; FILE* fp; @@ -38,26 +45,31 @@ int main(int argc, char **argv) { fprintf(stderr, "Could not open src/ecmult_static_context.h for writing!\n"); return -1; } - + fprintf(fp, "#ifndef _SECP256K1_ECMULT_STATIC_CONTEXT_\n"); fprintf(fp, "#define _SECP256K1_ECMULT_STATIC_CONTEXT_\n"); fprintf(fp, "#include \"src/group.h\"\n"); fprintf(fp, "#define SC SECP256K1_GE_STORAGE_CONST\n"); - fprintf(fp, "static const secp256k1_ge_storage secp256k1_ecmult_static_context[64][16] = {\n"); + fprintf(fp, "#if ECMULT_GEN_PREC_N != %d || ECMULT_GEN_PREC_G != %d\n", ECMULT_GEN_PREC_N, ECMULT_GEN_PREC_G); + fprintf(fp, " #error configuration mismatch, invalid ECMULT_GEN_PREC_N, ECMULT_GEN_PREC_G. Try deleting ecmult_static_context.h before the build.\n"); + fprintf(fp, "#endif\n"); + fprintf(fp, "static const secp256k1_ge_storage secp256k1_ecmult_static_context[ECMULT_GEN_PREC_N][ECMULT_GEN_PREC_G] = {\n"); + base = checked_malloc(&default_error_callback, SECP256K1_ECMULT_GEN_CONTEXT_PREALLOCATED_SIZE); + prealloc = base; secp256k1_ecmult_gen_context_init(&ctx); - secp256k1_ecmult_gen_context_build(&ctx, &default_error_callback); - for(outer = 0; outer != 64; outer++) { + secp256k1_ecmult_gen_context_build(&ctx, &prealloc); + for(outer = 0; outer != ECMULT_GEN_PREC_N; outer++) { fprintf(fp,"{\n"); - for(inner = 0; inner != 16; inner++) { + for(inner = 0; inner != ECMULT_GEN_PREC_G; inner++) { fprintf(fp," SC(%uu, %uu, %uu, %uu, %uu, %uu, %uu, %uu, %uu, %uu, %uu, %uu, %uu, %uu, %uu, %uu)", SECP256K1_GE_STORAGE_CONST_GET((*ctx.prec)[outer][inner])); - if (inner != 15) { + if (inner != ECMULT_GEN_PREC_G - 1) { fprintf(fp,",\n"); } else { fprintf(fp,"\n"); } } - if (outer != 63) { + if (outer != ECMULT_GEN_PREC_N - 1) { fprintf(fp,"},\n"); } else { fprintf(fp,"}\n"); @@ -65,10 +77,11 @@ int main(int argc, char **argv) { } fprintf(fp,"};\n"); secp256k1_ecmult_gen_context_clear(&ctx); - + free(base); + fprintf(fp, "#undef SC\n"); fprintf(fp, "#endif\n"); fclose(fp); - + return 0; } diff --git a/src/secp256k1/src/group.h b/src/secp256k1/src/group.h index 8e122ab429..863644f0f0 100644 --- a/src/secp256k1/src/group.h +++ b/src/secp256k1/src/group.h @@ -95,14 +95,13 @@ static int secp256k1_gej_is_infinity(const secp256k1_gej *a); /** Check whether a group element's y coordinate is a quadratic residue. */ static int secp256k1_gej_has_quad_y_var(const secp256k1_gej *a); -/** Set r equal to the double of a. If rzr is not-NULL, r->z = a->z * *rzr (where infinity means an implicit z = 0). - * a may not be zero. Constant time. */ -static void secp256k1_gej_double_nonzero(secp256k1_gej *r, const secp256k1_gej *a, secp256k1_fe *rzr); +/** Set r equal to the double of a, a cannot be infinity. Constant time. */ +static void secp256k1_gej_double_nonzero(secp256k1_gej *r, const secp256k1_gej *a); -/** Set r equal to the double of a. If rzr is not-NULL, r->z = a->z * *rzr (where infinity means an implicit z = 0). */ +/** Set r equal to the double of a. If rzr is not-NULL this sets *rzr such that r->z == a->z * *rzr (where infinity means an implicit z = 0). */ static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, secp256k1_fe *rzr); -/** Set r equal to the sum of a and b. If rzr is non-NULL, r->z = a->z * *rzr (a cannot be infinity in that case). */ +/** Set r equal to the sum of a and b. If rzr is non-NULL this sets *rzr such that r->z == a->z * *rzr (a cannot be infinity in that case). */ static void secp256k1_gej_add_var(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_gej *b, secp256k1_fe *rzr); /** Set r equal to the sum of a and b (with b given in affine coordinates, and not infinity). */ @@ -110,7 +109,7 @@ static void secp256k1_gej_add_ge(secp256k1_gej *r, const secp256k1_gej *a, const /** Set r equal to the sum of a and b (with b given in affine coordinates). This is more efficient than secp256k1_gej_add_var. It is identical to secp256k1_gej_add_ge but without constant-time - guarantee, and b is allowed to be infinity. If rzr is non-NULL, r->z = a->z * *rzr (a cannot be infinity in that case). */ + guarantee, and b is allowed to be infinity. If rzr is non-NULL this sets *rzr such that r->z == a->z * *rzr (a cannot be infinity in that case). */ static void secp256k1_gej_add_ge_var(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_ge *b, secp256k1_fe *rzr); /** Set r equal to the sum of a and b (with the inverse of b's Z coordinate passed as bzinv). */ @@ -133,7 +132,7 @@ static void secp256k1_ge_to_storage(secp256k1_ge_storage *r, const secp256k1_ge /** Convert a group element back from the storage type. */ static void secp256k1_ge_from_storage(secp256k1_ge *r, const secp256k1_ge_storage *a); -/** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. */ +/** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. Both *r and *a must be initialized.*/ static void secp256k1_ge_storage_cmov(secp256k1_ge_storage *r, const secp256k1_ge_storage *a, int flag); /** Rescale a jacobian point by b which must be non-zero. Constant-time. */ diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index 9b93c39e92..43b039becf 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -303,7 +303,7 @@ static int secp256k1_ge_is_valid_var(const secp256k1_ge *a) { return secp256k1_fe_equal_var(&y2, &x3); } -static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, secp256k1_fe *rzr) { +static SECP256K1_INLINE void secp256k1_gej_double_nonzero(secp256k1_gej *r, const secp256k1_gej *a) { /* Operations: 3 mul, 4 sqr, 0 normalize, 12 mul_int/add/negate. * * Note that there is an implementation described at @@ -312,29 +312,9 @@ static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, s * mainly because it requires more normalizations. */ secp256k1_fe t1,t2,t3,t4; - /** For secp256k1, 2Q is infinity if and only if Q is infinity. This is because if 2Q = infinity, - * Q must equal -Q, or that Q.y == -(Q.y), or Q.y is 0. For a point on y^2 = x^3 + 7 to have - * y=0, x^3 must be -7 mod p. However, -7 has no cube root mod p. - * - * Having said this, if this function receives a point on a sextic twist, e.g. by - * a fault attack, it is possible for y to be 0. This happens for y^2 = x^3 + 6, - * since -6 does have a cube root mod p. For this point, this function will not set - * the infinity flag even though the point doubles to infinity, and the result - * point will be gibberish (z = 0 but infinity = 0). - */ - r->infinity = a->infinity; - if (r->infinity) { - if (rzr != NULL) { - secp256k1_fe_set_int(rzr, 1); - } - return; - } - if (rzr != NULL) { - *rzr = a->y; - secp256k1_fe_normalize_weak(rzr); - secp256k1_fe_mul_int(rzr, 2); - } + VERIFY_CHECK(!secp256k1_gej_is_infinity(a)); + r->infinity = 0; secp256k1_fe_mul(&r->z, &a->z, &a->y); secp256k1_fe_mul_int(&r->z, 2); /* Z' = 2*Y*Z (2) */ @@ -358,9 +338,32 @@ static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, s secp256k1_fe_add(&r->y, &t2); /* Y' = 36*X^3*Y^2 - 27*X^6 - 8*Y^4 (4) */ } -static SECP256K1_INLINE void secp256k1_gej_double_nonzero(secp256k1_gej *r, const secp256k1_gej *a, secp256k1_fe *rzr) { - VERIFY_CHECK(!secp256k1_gej_is_infinity(a)); - secp256k1_gej_double_var(r, a, rzr); +static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, secp256k1_fe *rzr) { + /** For secp256k1, 2Q is infinity if and only if Q is infinity. This is because if 2Q = infinity, + * Q must equal -Q, or that Q.y == -(Q.y), or Q.y is 0. For a point on y^2 = x^3 + 7 to have + * y=0, x^3 must be -7 mod p. However, -7 has no cube root mod p. + * + * Having said this, if this function receives a point on a sextic twist, e.g. by + * a fault attack, it is possible for y to be 0. This happens for y^2 = x^3 + 6, + * since -6 does have a cube root mod p. For this point, this function will not set + * the infinity flag even though the point doubles to infinity, and the result + * point will be gibberish (z = 0 but infinity = 0). + */ + if (a->infinity) { + r->infinity = 1; + if (rzr != NULL) { + secp256k1_fe_set_int(rzr, 1); + } + return; + } + + if (rzr != NULL) { + *rzr = a->y; + secp256k1_fe_normalize_weak(rzr); + secp256k1_fe_mul_int(rzr, 2); + } + + secp256k1_gej_double_nonzero(r, a); } static void secp256k1_gej_add_var(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_gej *b, secp256k1_fe *rzr) { diff --git a/src/secp256k1/src/hash_impl.h b/src/secp256k1/src/hash_impl.h index 009f26beba..782f97216c 100644 --- a/src/secp256k1/src/hash_impl.h +++ b/src/secp256k1/src/hash_impl.h @@ -131,7 +131,8 @@ static void secp256k1_sha256_transform(uint32_t* s, const uint32_t* chunk) { static void secp256k1_sha256_write(secp256k1_sha256 *hash, const unsigned char *data, size_t len) { size_t bufsize = hash->bytes & 0x3F; hash->bytes += len; - while (bufsize + len >= 64) { + VERIFY_CHECK(hash->bytes >= len); + while (len >= 64 - bufsize) { /* Fill the buffer, and process it. */ size_t chunk_len = 64 - bufsize; memcpy(((unsigned char*)hash->buf) + bufsize, data, chunk_len); diff --git a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1.java b/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1.java deleted file mode 100644 index 1c67802fba..0000000000 --- a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1.java +++ /dev/null @@ -1,446 +0,0 @@ -/* - * Copyright 2013 Google Inc. - * Copyright 2014-2016 the libsecp256k1 contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.bitcoin; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -import java.math.BigInteger; -import com.google.common.base.Preconditions; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import static org.bitcoin.NativeSecp256k1Util.*; - -/** - * <p>This class holds native methods to handle ECDSA verification.</p> - * - * <p>You can find an example library that can be used for this at https://github.com/bitcoin/secp256k1</p> - * - * <p>To build secp256k1 for use with bitcoinj, run - * `./configure --enable-jni --enable-experimental --enable-module-ecdh` - * and `make` then copy `.libs/libsecp256k1.so` to your system library path - * or point the JVM to the folder containing it with -Djava.library.path - * </p> - */ -public class NativeSecp256k1 { - - private static final ReentrantReadWriteLock rwl = new ReentrantReadWriteLock(); - private static final Lock r = rwl.readLock(); - private static final Lock w = rwl.writeLock(); - private static ThreadLocal<ByteBuffer> nativeECDSABuffer = new ThreadLocal<ByteBuffer>(); - /** - * Verifies the given secp256k1 signature in native code. - * Calling when enabled == false is undefined (probably library not loaded) - * - * @param data The data which was signed, must be exactly 32 bytes - * @param signature The signature - * @param pub The public key which did the signing - */ - public static boolean verify(byte[] data, byte[] signature, byte[] pub) throws AssertFailException{ - Preconditions.checkArgument(data.length == 32 && signature.length <= 520 && pub.length <= 520); - - ByteBuffer byteBuff = nativeECDSABuffer.get(); - if (byteBuff == null || byteBuff.capacity() < 520) { - byteBuff = ByteBuffer.allocateDirect(520); - byteBuff.order(ByteOrder.nativeOrder()); - nativeECDSABuffer.set(byteBuff); - } - byteBuff.rewind(); - byteBuff.put(data); - byteBuff.put(signature); - byteBuff.put(pub); - - byte[][] retByteArray; - - r.lock(); - try { - return secp256k1_ecdsa_verify(byteBuff, Secp256k1Context.getContext(), signature.length, pub.length) == 1; - } finally { - r.unlock(); - } - } - - /** - * libsecp256k1 Create an ECDSA signature. - * - * @param data Message hash, 32 bytes - * @param key Secret key, 32 bytes - * - * Return values - * @param sig byte array of signature - */ - public static byte[] sign(byte[] data, byte[] sec) throws AssertFailException{ - Preconditions.checkArgument(data.length == 32 && sec.length <= 32); - - ByteBuffer byteBuff = nativeECDSABuffer.get(); - if (byteBuff == null || byteBuff.capacity() < 32 + 32) { - byteBuff = ByteBuffer.allocateDirect(32 + 32); - byteBuff.order(ByteOrder.nativeOrder()); - nativeECDSABuffer.set(byteBuff); - } - byteBuff.rewind(); - byteBuff.put(data); - byteBuff.put(sec); - - byte[][] retByteArray; - - r.lock(); - try { - retByteArray = secp256k1_ecdsa_sign(byteBuff, Secp256k1Context.getContext()); - } finally { - r.unlock(); - } - - byte[] sigArr = retByteArray[0]; - int sigLen = new BigInteger(new byte[] { retByteArray[1][0] }).intValue(); - int retVal = new BigInteger(new byte[] { retByteArray[1][1] }).intValue(); - - assertEquals(sigArr.length, sigLen, "Got bad signature length."); - - return retVal == 0 ? new byte[0] : sigArr; - } - - /** - * libsecp256k1 Seckey Verify - returns 1 if valid, 0 if invalid - * - * @param seckey ECDSA Secret key, 32 bytes - */ - public static boolean secKeyVerify(byte[] seckey) { - Preconditions.checkArgument(seckey.length == 32); - - ByteBuffer byteBuff = nativeECDSABuffer.get(); - if (byteBuff == null || byteBuff.capacity() < seckey.length) { - byteBuff = ByteBuffer.allocateDirect(seckey.length); - byteBuff.order(ByteOrder.nativeOrder()); - nativeECDSABuffer.set(byteBuff); - } - byteBuff.rewind(); - byteBuff.put(seckey); - - r.lock(); - try { - return secp256k1_ec_seckey_verify(byteBuff,Secp256k1Context.getContext()) == 1; - } finally { - r.unlock(); - } - } - - - /** - * libsecp256k1 Compute Pubkey - computes public key from secret key - * - * @param seckey ECDSA Secret key, 32 bytes - * - * Return values - * @param pubkey ECDSA Public key, 33 or 65 bytes - */ - //TODO add a 'compressed' arg - public static byte[] computePubkey(byte[] seckey) throws AssertFailException{ - Preconditions.checkArgument(seckey.length == 32); - - ByteBuffer byteBuff = nativeECDSABuffer.get(); - if (byteBuff == null || byteBuff.capacity() < seckey.length) { - byteBuff = ByteBuffer.allocateDirect(seckey.length); - byteBuff.order(ByteOrder.nativeOrder()); - nativeECDSABuffer.set(byteBuff); - } - byteBuff.rewind(); - byteBuff.put(seckey); - - byte[][] retByteArray; - - r.lock(); - try { - retByteArray = secp256k1_ec_pubkey_create(byteBuff, Secp256k1Context.getContext()); - } finally { - r.unlock(); - } - - byte[] pubArr = retByteArray[0]; - int pubLen = new BigInteger(new byte[] { retByteArray[1][0] }).intValue(); - int retVal = new BigInteger(new byte[] { retByteArray[1][1] }).intValue(); - - assertEquals(pubArr.length, pubLen, "Got bad pubkey length."); - - return retVal == 0 ? new byte[0]: pubArr; - } - - /** - * libsecp256k1 Cleanup - This destroys the secp256k1 context object - * This should be called at the end of the program for proper cleanup of the context. - */ - public static synchronized void cleanup() { - w.lock(); - try { - secp256k1_destroy_context(Secp256k1Context.getContext()); - } finally { - w.unlock(); - } - } - - public static long cloneContext() { - r.lock(); - try { - return secp256k1_ctx_clone(Secp256k1Context.getContext()); - } finally { r.unlock(); } - } - - /** - * libsecp256k1 PrivKey Tweak-Mul - Tweak privkey by multiplying to it - * - * @param tweak some bytes to tweak with - * @param seckey 32-byte seckey - */ - public static byte[] privKeyTweakMul(byte[] privkey, byte[] tweak) throws AssertFailException{ - Preconditions.checkArgument(privkey.length == 32); - - ByteBuffer byteBuff = nativeECDSABuffer.get(); - if (byteBuff == null || byteBuff.capacity() < privkey.length + tweak.length) { - byteBuff = ByteBuffer.allocateDirect(privkey.length + tweak.length); - byteBuff.order(ByteOrder.nativeOrder()); - nativeECDSABuffer.set(byteBuff); - } - byteBuff.rewind(); - byteBuff.put(privkey); - byteBuff.put(tweak); - - byte[][] retByteArray; - r.lock(); - try { - retByteArray = secp256k1_privkey_tweak_mul(byteBuff,Secp256k1Context.getContext()); - } finally { - r.unlock(); - } - - byte[] privArr = retByteArray[0]; - - int privLen = (byte) new BigInteger(new byte[] { retByteArray[1][0] }).intValue() & 0xFF; - int retVal = new BigInteger(new byte[] { retByteArray[1][1] }).intValue(); - - assertEquals(privArr.length, privLen, "Got bad pubkey length."); - - assertEquals(retVal, 1, "Failed return value check."); - - return privArr; - } - - /** - * libsecp256k1 PrivKey Tweak-Add - Tweak privkey by adding to it - * - * @param tweak some bytes to tweak with - * @param seckey 32-byte seckey - */ - public static byte[] privKeyTweakAdd(byte[] privkey, byte[] tweak) throws AssertFailException{ - Preconditions.checkArgument(privkey.length == 32); - - ByteBuffer byteBuff = nativeECDSABuffer.get(); - if (byteBuff == null || byteBuff.capacity() < privkey.length + tweak.length) { - byteBuff = ByteBuffer.allocateDirect(privkey.length + tweak.length); - byteBuff.order(ByteOrder.nativeOrder()); - nativeECDSABuffer.set(byteBuff); - } - byteBuff.rewind(); - byteBuff.put(privkey); - byteBuff.put(tweak); - - byte[][] retByteArray; - r.lock(); - try { - retByteArray = secp256k1_privkey_tweak_add(byteBuff,Secp256k1Context.getContext()); - } finally { - r.unlock(); - } - - byte[] privArr = retByteArray[0]; - - int privLen = (byte) new BigInteger(new byte[] { retByteArray[1][0] }).intValue() & 0xFF; - int retVal = new BigInteger(new byte[] { retByteArray[1][1] }).intValue(); - - assertEquals(privArr.length, privLen, "Got bad pubkey length."); - - assertEquals(retVal, 1, "Failed return value check."); - - return privArr; - } - - /** - * libsecp256k1 PubKey Tweak-Add - Tweak pubkey by adding to it - * - * @param tweak some bytes to tweak with - * @param pubkey 32-byte seckey - */ - public static byte[] pubKeyTweakAdd(byte[] pubkey, byte[] tweak) throws AssertFailException{ - Preconditions.checkArgument(pubkey.length == 33 || pubkey.length == 65); - - ByteBuffer byteBuff = nativeECDSABuffer.get(); - if (byteBuff == null || byteBuff.capacity() < pubkey.length + tweak.length) { - byteBuff = ByteBuffer.allocateDirect(pubkey.length + tweak.length); - byteBuff.order(ByteOrder.nativeOrder()); - nativeECDSABuffer.set(byteBuff); - } - byteBuff.rewind(); - byteBuff.put(pubkey); - byteBuff.put(tweak); - - byte[][] retByteArray; - r.lock(); - try { - retByteArray = secp256k1_pubkey_tweak_add(byteBuff,Secp256k1Context.getContext(), pubkey.length); - } finally { - r.unlock(); - } - - byte[] pubArr = retByteArray[0]; - - int pubLen = (byte) new BigInteger(new byte[] { retByteArray[1][0] }).intValue() & 0xFF; - int retVal = new BigInteger(new byte[] { retByteArray[1][1] }).intValue(); - - assertEquals(pubArr.length, pubLen, "Got bad pubkey length."); - - assertEquals(retVal, 1, "Failed return value check."); - - return pubArr; - } - - /** - * libsecp256k1 PubKey Tweak-Mul - Tweak pubkey by multiplying to it - * - * @param tweak some bytes to tweak with - * @param pubkey 32-byte seckey - */ - public static byte[] pubKeyTweakMul(byte[] pubkey, byte[] tweak) throws AssertFailException{ - Preconditions.checkArgument(pubkey.length == 33 || pubkey.length == 65); - - ByteBuffer byteBuff = nativeECDSABuffer.get(); - if (byteBuff == null || byteBuff.capacity() < pubkey.length + tweak.length) { - byteBuff = ByteBuffer.allocateDirect(pubkey.length + tweak.length); - byteBuff.order(ByteOrder.nativeOrder()); - nativeECDSABuffer.set(byteBuff); - } - byteBuff.rewind(); - byteBuff.put(pubkey); - byteBuff.put(tweak); - - byte[][] retByteArray; - r.lock(); - try { - retByteArray = secp256k1_pubkey_tweak_mul(byteBuff,Secp256k1Context.getContext(), pubkey.length); - } finally { - r.unlock(); - } - - byte[] pubArr = retByteArray[0]; - - int pubLen = (byte) new BigInteger(new byte[] { retByteArray[1][0] }).intValue() & 0xFF; - int retVal = new BigInteger(new byte[] { retByteArray[1][1] }).intValue(); - - assertEquals(pubArr.length, pubLen, "Got bad pubkey length."); - - assertEquals(retVal, 1, "Failed return value check."); - - return pubArr; - } - - /** - * libsecp256k1 create ECDH secret - constant time ECDH calculation - * - * @param seckey byte array of secret key used in exponentiaion - * @param pubkey byte array of public key used in exponentiaion - */ - public static byte[] createECDHSecret(byte[] seckey, byte[] pubkey) throws AssertFailException{ - Preconditions.checkArgument(seckey.length <= 32 && pubkey.length <= 65); - - ByteBuffer byteBuff = nativeECDSABuffer.get(); - if (byteBuff == null || byteBuff.capacity() < 32 + pubkey.length) { - byteBuff = ByteBuffer.allocateDirect(32 + pubkey.length); - byteBuff.order(ByteOrder.nativeOrder()); - nativeECDSABuffer.set(byteBuff); - } - byteBuff.rewind(); - byteBuff.put(seckey); - byteBuff.put(pubkey); - - byte[][] retByteArray; - r.lock(); - try { - retByteArray = secp256k1_ecdh(byteBuff, Secp256k1Context.getContext(), pubkey.length); - } finally { - r.unlock(); - } - - byte[] resArr = retByteArray[0]; - int retVal = new BigInteger(new byte[] { retByteArray[1][0] }).intValue(); - - assertEquals(resArr.length, 32, "Got bad result length."); - assertEquals(retVal, 1, "Failed return value check."); - - return resArr; - } - - /** - * libsecp256k1 randomize - updates the context randomization - * - * @param seed 32-byte random seed - */ - public static synchronized boolean randomize(byte[] seed) throws AssertFailException{ - Preconditions.checkArgument(seed.length == 32 || seed == null); - - ByteBuffer byteBuff = nativeECDSABuffer.get(); - if (byteBuff == null || byteBuff.capacity() < seed.length) { - byteBuff = ByteBuffer.allocateDirect(seed.length); - byteBuff.order(ByteOrder.nativeOrder()); - nativeECDSABuffer.set(byteBuff); - } - byteBuff.rewind(); - byteBuff.put(seed); - - w.lock(); - try { - return secp256k1_context_randomize(byteBuff, Secp256k1Context.getContext()) == 1; - } finally { - w.unlock(); - } - } - - private static native long secp256k1_ctx_clone(long context); - - private static native int secp256k1_context_randomize(ByteBuffer byteBuff, long context); - - private static native byte[][] secp256k1_privkey_tweak_add(ByteBuffer byteBuff, long context); - - private static native byte[][] secp256k1_privkey_tweak_mul(ByteBuffer byteBuff, long context); - - private static native byte[][] secp256k1_pubkey_tweak_add(ByteBuffer byteBuff, long context, int pubLen); - - private static native byte[][] secp256k1_pubkey_tweak_mul(ByteBuffer byteBuff, long context, int pubLen); - - private static native void secp256k1_destroy_context(long context); - - private static native int secp256k1_ecdsa_verify(ByteBuffer byteBuff, long context, int sigLen, int pubLen); - - private static native byte[][] secp256k1_ecdsa_sign(ByteBuffer byteBuff, long context); - - private static native int secp256k1_ec_seckey_verify(ByteBuffer byteBuff, long context); - - private static native byte[][] secp256k1_ec_pubkey_create(ByteBuffer byteBuff, long context); - - private static native byte[][] secp256k1_ec_pubkey_parse(ByteBuffer byteBuff, long context, int inputLen); - - private static native byte[][] secp256k1_ecdh(ByteBuffer byteBuff, long context, int inputLen); - -} diff --git a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Test.java b/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Test.java deleted file mode 100644 index d766a1029c..0000000000 --- a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Test.java +++ /dev/null @@ -1,226 +0,0 @@ -package org.bitcoin; - -import com.google.common.io.BaseEncoding; -import java.util.Arrays; -import java.math.BigInteger; -import javax.xml.bind.DatatypeConverter; -import static org.bitcoin.NativeSecp256k1Util.*; - -/** - * This class holds test cases defined for testing this library. - */ -public class NativeSecp256k1Test { - - //TODO improve comments/add more tests - /** - * This tests verify() for a valid signature - */ - public static void testVerifyPos() throws AssertFailException{ - boolean result = false; - byte[] data = BaseEncoding.base16().lowerCase().decode("CF80CD8AED482D5D1527D7DC72FCEFF84E6326592848447D2DC0B0E87DFC9A90".toLowerCase()); //sha256hash of "testing" - byte[] sig = BaseEncoding.base16().lowerCase().decode("3044022079BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F817980220294F14E883B3F525B5367756C2A11EF6CF84B730B36C17CB0C56F0AAB2C98589".toLowerCase()); - byte[] pub = BaseEncoding.base16().lowerCase().decode("040A629506E1B65CD9D2E0BA9C75DF9C4FED0DB16DC9625ED14397F0AFC836FAE595DC53F8B0EFE61E703075BD9B143BAC75EC0E19F82A2208CAEB32BE53414C40".toLowerCase()); - - result = NativeSecp256k1.verify( data, sig, pub); - assertEquals( result, true , "testVerifyPos"); - } - - /** - * This tests verify() for a non-valid signature - */ - public static void testVerifyNeg() throws AssertFailException{ - boolean result = false; - byte[] data = BaseEncoding.base16().lowerCase().decode("CF80CD8AED482D5D1527D7DC72FCEFF84E6326592848447D2DC0B0E87DFC9A91".toLowerCase()); //sha256hash of "testing" - byte[] sig = BaseEncoding.base16().lowerCase().decode("3044022079BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F817980220294F14E883B3F525B5367756C2A11EF6CF84B730B36C17CB0C56F0AAB2C98589".toLowerCase()); - byte[] pub = BaseEncoding.base16().lowerCase().decode("040A629506E1B65CD9D2E0BA9C75DF9C4FED0DB16DC9625ED14397F0AFC836FAE595DC53F8B0EFE61E703075BD9B143BAC75EC0E19F82A2208CAEB32BE53414C40".toLowerCase()); - - result = NativeSecp256k1.verify( data, sig, pub); - //System.out.println(" TEST " + new BigInteger(1, resultbytes).toString(16)); - assertEquals( result, false , "testVerifyNeg"); - } - - /** - * This tests secret key verify() for a valid secretkey - */ - public static void testSecKeyVerifyPos() throws AssertFailException{ - boolean result = false; - byte[] sec = BaseEncoding.base16().lowerCase().decode("67E56582298859DDAE725F972992A07C6C4FB9F62A8FFF58CE3CA926A1063530".toLowerCase()); - - result = NativeSecp256k1.secKeyVerify( sec ); - //System.out.println(" TEST " + new BigInteger(1, resultbytes).toString(16)); - assertEquals( result, true , "testSecKeyVerifyPos"); - } - - /** - * This tests secret key verify() for an invalid secretkey - */ - public static void testSecKeyVerifyNeg() throws AssertFailException{ - boolean result = false; - byte[] sec = BaseEncoding.base16().lowerCase().decode("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF".toLowerCase()); - - result = NativeSecp256k1.secKeyVerify( sec ); - //System.out.println(" TEST " + new BigInteger(1, resultbytes).toString(16)); - assertEquals( result, false , "testSecKeyVerifyNeg"); - } - - /** - * This tests public key create() for a valid secretkey - */ - public static void testPubKeyCreatePos() throws AssertFailException{ - byte[] sec = BaseEncoding.base16().lowerCase().decode("67E56582298859DDAE725F972992A07C6C4FB9F62A8FFF58CE3CA926A1063530".toLowerCase()); - - byte[] resultArr = NativeSecp256k1.computePubkey( sec); - String pubkeyString = javax.xml.bind.DatatypeConverter.printHexBinary(resultArr); - assertEquals( pubkeyString , "04C591A8FF19AC9C4E4E5793673B83123437E975285E7B442F4EE2654DFFCA5E2D2103ED494718C697AC9AEBCFD19612E224DB46661011863ED2FC54E71861E2A6" , "testPubKeyCreatePos"); - } - - /** - * This tests public key create() for a invalid secretkey - */ - public static void testPubKeyCreateNeg() throws AssertFailException{ - byte[] sec = BaseEncoding.base16().lowerCase().decode("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF".toLowerCase()); - - byte[] resultArr = NativeSecp256k1.computePubkey( sec); - String pubkeyString = javax.xml.bind.DatatypeConverter.printHexBinary(resultArr); - assertEquals( pubkeyString, "" , "testPubKeyCreateNeg"); - } - - /** - * This tests sign() for a valid secretkey - */ - public static void testSignPos() throws AssertFailException{ - - byte[] data = BaseEncoding.base16().lowerCase().decode("CF80CD8AED482D5D1527D7DC72FCEFF84E6326592848447D2DC0B0E87DFC9A90".toLowerCase()); //sha256hash of "testing" - byte[] sec = BaseEncoding.base16().lowerCase().decode("67E56582298859DDAE725F972992A07C6C4FB9F62A8FFF58CE3CA926A1063530".toLowerCase()); - - byte[] resultArr = NativeSecp256k1.sign(data, sec); - String sigString = javax.xml.bind.DatatypeConverter.printHexBinary(resultArr); - assertEquals( sigString, "30440220182A108E1448DC8F1FB467D06A0F3BB8EA0533584CB954EF8DA112F1D60E39A202201C66F36DA211C087F3AF88B50EDF4F9BDAA6CF5FD6817E74DCA34DB12390C6E9" , "testSignPos"); - } - - /** - * This tests sign() for a invalid secretkey - */ - public static void testSignNeg() throws AssertFailException{ - byte[] data = BaseEncoding.base16().lowerCase().decode("CF80CD8AED482D5D1527D7DC72FCEFF84E6326592848447D2DC0B0E87DFC9A90".toLowerCase()); //sha256hash of "testing" - byte[] sec = BaseEncoding.base16().lowerCase().decode("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF".toLowerCase()); - - byte[] resultArr = NativeSecp256k1.sign(data, sec); - String sigString = javax.xml.bind.DatatypeConverter.printHexBinary(resultArr); - assertEquals( sigString, "" , "testSignNeg"); - } - - /** - * This tests private key tweak-add - */ - public static void testPrivKeyTweakAdd_1() throws AssertFailException { - byte[] sec = BaseEncoding.base16().lowerCase().decode("67E56582298859DDAE725F972992A07C6C4FB9F62A8FFF58CE3CA926A1063530".toLowerCase()); - byte[] data = BaseEncoding.base16().lowerCase().decode("3982F19BEF1615BCCFBB05E321C10E1D4CBA3DF0E841C2E41EEB6016347653C3".toLowerCase()); //sha256hash of "tweak" - - byte[] resultArr = NativeSecp256k1.privKeyTweakAdd( sec , data ); - String sigString = javax.xml.bind.DatatypeConverter.printHexBinary(resultArr); - assertEquals( sigString , "A168571E189E6F9A7E2D657A4B53AE99B909F7E712D1C23CED28093CD57C88F3" , "testPrivKeyAdd_1"); - } - - /** - * This tests private key tweak-mul - */ - public static void testPrivKeyTweakMul_1() throws AssertFailException { - byte[] sec = BaseEncoding.base16().lowerCase().decode("67E56582298859DDAE725F972992A07C6C4FB9F62A8FFF58CE3CA926A1063530".toLowerCase()); - byte[] data = BaseEncoding.base16().lowerCase().decode("3982F19BEF1615BCCFBB05E321C10E1D4CBA3DF0E841C2E41EEB6016347653C3".toLowerCase()); //sha256hash of "tweak" - - byte[] resultArr = NativeSecp256k1.privKeyTweakMul( sec , data ); - String sigString = javax.xml.bind.DatatypeConverter.printHexBinary(resultArr); - assertEquals( sigString , "97F8184235F101550F3C71C927507651BD3F1CDB4A5A33B8986ACF0DEE20FFFC" , "testPrivKeyMul_1"); - } - - /** - * This tests private key tweak-add uncompressed - */ - public static void testPrivKeyTweakAdd_2() throws AssertFailException { - byte[] pub = BaseEncoding.base16().lowerCase().decode("040A629506E1B65CD9D2E0BA9C75DF9C4FED0DB16DC9625ED14397F0AFC836FAE595DC53F8B0EFE61E703075BD9B143BAC75EC0E19F82A2208CAEB32BE53414C40".toLowerCase()); - byte[] data = BaseEncoding.base16().lowerCase().decode("3982F19BEF1615BCCFBB05E321C10E1D4CBA3DF0E841C2E41EEB6016347653C3".toLowerCase()); //sha256hash of "tweak" - - byte[] resultArr = NativeSecp256k1.pubKeyTweakAdd( pub , data ); - String sigString = javax.xml.bind.DatatypeConverter.printHexBinary(resultArr); - assertEquals( sigString , "0411C6790F4B663CCE607BAAE08C43557EDC1A4D11D88DFCB3D841D0C6A941AF525A268E2A863C148555C48FB5FBA368E88718A46E205FABC3DBA2CCFFAB0796EF" , "testPrivKeyAdd_2"); - } - - /** - * This tests private key tweak-mul uncompressed - */ - public static void testPrivKeyTweakMul_2() throws AssertFailException { - byte[] pub = BaseEncoding.base16().lowerCase().decode("040A629506E1B65CD9D2E0BA9C75DF9C4FED0DB16DC9625ED14397F0AFC836FAE595DC53F8B0EFE61E703075BD9B143BAC75EC0E19F82A2208CAEB32BE53414C40".toLowerCase()); - byte[] data = BaseEncoding.base16().lowerCase().decode("3982F19BEF1615BCCFBB05E321C10E1D4CBA3DF0E841C2E41EEB6016347653C3".toLowerCase()); //sha256hash of "tweak" - - byte[] resultArr = NativeSecp256k1.pubKeyTweakMul( pub , data ); - String sigString = javax.xml.bind.DatatypeConverter.printHexBinary(resultArr); - assertEquals( sigString , "04E0FE6FE55EBCA626B98A807F6CAF654139E14E5E3698F01A9A658E21DC1D2791EC060D4F412A794D5370F672BC94B722640B5F76914151CFCA6E712CA48CC589" , "testPrivKeyMul_2"); - } - - /** - * This tests seed randomization - */ - public static void testRandomize() throws AssertFailException { - byte[] seed = BaseEncoding.base16().lowerCase().decode("A441B15FE9A3CF56661190A0B93B9DEC7D04127288CC87250967CF3B52894D11".toLowerCase()); //sha256hash of "random" - boolean result = NativeSecp256k1.randomize(seed); - assertEquals( result, true, "testRandomize"); - } - - public static void testCreateECDHSecret() throws AssertFailException{ - - byte[] sec = BaseEncoding.base16().lowerCase().decode("67E56582298859DDAE725F972992A07C6C4FB9F62A8FFF58CE3CA926A1063530".toLowerCase()); - byte[] pub = BaseEncoding.base16().lowerCase().decode("040A629506E1B65CD9D2E0BA9C75DF9C4FED0DB16DC9625ED14397F0AFC836FAE595DC53F8B0EFE61E703075BD9B143BAC75EC0E19F82A2208CAEB32BE53414C40".toLowerCase()); - - byte[] resultArr = NativeSecp256k1.createECDHSecret(sec, pub); - String ecdhString = javax.xml.bind.DatatypeConverter.printHexBinary(resultArr); - assertEquals( ecdhString, "2A2A67007A926E6594AF3EB564FC74005B37A9C8AEF2033C4552051B5C87F043" , "testCreateECDHSecret"); - } - - public static void main(String[] args) throws AssertFailException{ - - - System.out.println("\n libsecp256k1 enabled: " + Secp256k1Context.isEnabled() + "\n"); - - assertEquals( Secp256k1Context.isEnabled(), true, "isEnabled" ); - - //Test verify() success/fail - testVerifyPos(); - testVerifyNeg(); - - //Test secKeyVerify() success/fail - testSecKeyVerifyPos(); - testSecKeyVerifyNeg(); - - //Test computePubkey() success/fail - testPubKeyCreatePos(); - testPubKeyCreateNeg(); - - //Test sign() success/fail - testSignPos(); - testSignNeg(); - - //Test privKeyTweakAdd() 1 - testPrivKeyTweakAdd_1(); - - //Test privKeyTweakMul() 2 - testPrivKeyTweakMul_1(); - - //Test privKeyTweakAdd() 3 - testPrivKeyTweakAdd_2(); - - //Test privKeyTweakMul() 4 - testPrivKeyTweakMul_2(); - - //Test randomize() - testRandomize(); - - //Test ECDH - testCreateECDHSecret(); - - NativeSecp256k1.cleanup(); - - System.out.println(" All tests passed." ); - - } -} diff --git a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Util.java b/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Util.java deleted file mode 100644 index 04732ba044..0000000000 --- a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Util.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2014-2016 the libsecp256k1 contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.bitcoin; - -public class NativeSecp256k1Util{ - - public static void assertEquals( int val, int val2, String message ) throws AssertFailException{ - if( val != val2 ) - throw new AssertFailException("FAIL: " + message); - } - - public static void assertEquals( boolean val, boolean val2, String message ) throws AssertFailException{ - if( val != val2 ) - throw new AssertFailException("FAIL: " + message); - else - System.out.println("PASS: " + message); - } - - public static void assertEquals( String val, String val2, String message ) throws AssertFailException{ - if( !val.equals(val2) ) - throw new AssertFailException("FAIL: " + message); - else - System.out.println("PASS: " + message); - } - - public static class AssertFailException extends Exception { - public AssertFailException(String message) { - super( message ); - } - } -} diff --git a/src/secp256k1/src/java/org/bitcoin/Secp256k1Context.java b/src/secp256k1/src/java/org/bitcoin/Secp256k1Context.java deleted file mode 100644 index 216c986a8b..0000000000 --- a/src/secp256k1/src/java/org/bitcoin/Secp256k1Context.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2014-2016 the libsecp256k1 contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.bitcoin; - -/** - * This class holds the context reference used in native methods - * to handle ECDSA operations. - */ -public class Secp256k1Context { - private static final boolean enabled; //true if the library is loaded - private static final long context; //ref to pointer to context obj - - static { //static initializer - boolean isEnabled = true; - long contextRef = -1; - try { - System.loadLibrary("secp256k1"); - contextRef = secp256k1_init_context(); - } catch (UnsatisfiedLinkError e) { - System.out.println("UnsatisfiedLinkError: " + e.toString()); - isEnabled = false; - } - enabled = isEnabled; - context = contextRef; - } - - public static boolean isEnabled() { - return enabled; - } - - public static long getContext() { - if(!enabled) return -1; //sanity check - return context; - } - - private static native long secp256k1_init_context(); -} diff --git a/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c b/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c deleted file mode 100644 index b50970b4f2..0000000000 --- a/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c +++ /dev/null @@ -1,379 +0,0 @@ -#include <stdlib.h> -#include <stdint.h> -#include <string.h> -#include "org_bitcoin_NativeSecp256k1.h" -#include "include/secp256k1.h" -#include "include/secp256k1_ecdh.h" -#include "include/secp256k1_recovery.h" - - -SECP256K1_API jlong JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ctx_1clone - (JNIEnv* env, jclass classObject, jlong ctx_l) -{ - const secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; - - jlong ctx_clone_l = (uintptr_t) secp256k1_context_clone(ctx); - - (void)classObject;(void)env; - - return ctx_clone_l; - -} - -SECP256K1_API jint JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1context_1randomize - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l) -{ - secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; - - const unsigned char* seed = (unsigned char*) (*env)->GetDirectBufferAddress(env, byteBufferObject); - - (void)classObject; - - return secp256k1_context_randomize(ctx, seed); - -} - -SECP256K1_API void JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1destroy_1context - (JNIEnv* env, jclass classObject, jlong ctx_l) -{ - secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; - - secp256k1_context_destroy(ctx); - - (void)classObject;(void)env; -} - -SECP256K1_API jint JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ecdsa_1verify - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l, jint siglen, jint publen) -{ - secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; - - unsigned char* data = (unsigned char*) (*env)->GetDirectBufferAddress(env, byteBufferObject); - const unsigned char* sigdata = { (unsigned char*) (data + 32) }; - const unsigned char* pubdata = { (unsigned char*) (data + siglen + 32) }; - - secp256k1_ecdsa_signature sig; - secp256k1_pubkey pubkey; - - int ret = secp256k1_ecdsa_signature_parse_der(ctx, &sig, sigdata, siglen); - - if( ret ) { - ret = secp256k1_ec_pubkey_parse(ctx, &pubkey, pubdata, publen); - - if( ret ) { - ret = secp256k1_ecdsa_verify(ctx, &sig, data, &pubkey); - } - } - - (void)classObject; - - return ret; -} - -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ecdsa_1sign - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l) -{ - secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; - unsigned char* data = (unsigned char*) (*env)->GetDirectBufferAddress(env, byteBufferObject); - unsigned char* secKey = (unsigned char*) (data + 32); - - jobjectArray retArray; - jbyteArray sigArray, intsByteArray; - unsigned char intsarray[2]; - - secp256k1_ecdsa_signature sig[72]; - - int ret = secp256k1_ecdsa_sign(ctx, sig, data, secKey, NULL, NULL); - - unsigned char outputSer[72]; - size_t outputLen = 72; - - if( ret ) { - int ret2 = secp256k1_ecdsa_signature_serialize_der(ctx,outputSer, &outputLen, sig ); (void)ret2; - } - - intsarray[0] = outputLen; - intsarray[1] = ret; - - retArray = (*env)->NewObjectArray(env, 2, - (*env)->FindClass(env, "[B"), - (*env)->NewByteArray(env, 1)); - - sigArray = (*env)->NewByteArray(env, outputLen); - (*env)->SetByteArrayRegion(env, sigArray, 0, outputLen, (jbyte*)outputSer); - (*env)->SetObjectArrayElement(env, retArray, 0, sigArray); - - intsByteArray = (*env)->NewByteArray(env, 2); - (*env)->SetByteArrayRegion(env, intsByteArray, 0, 2, (jbyte*)intsarray); - (*env)->SetObjectArrayElement(env, retArray, 1, intsByteArray); - - (void)classObject; - - return retArray; -} - -SECP256K1_API jint JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ec_1seckey_1verify - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l) -{ - secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; - unsigned char* secKey = (unsigned char*) (*env)->GetDirectBufferAddress(env, byteBufferObject); - - (void)classObject; - - return secp256k1_ec_seckey_verify(ctx, secKey); -} - -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ec_1pubkey_1create - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l) -{ - secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; - const unsigned char* secKey = (unsigned char*) (*env)->GetDirectBufferAddress(env, byteBufferObject); - - secp256k1_pubkey pubkey; - - jobjectArray retArray; - jbyteArray pubkeyArray, intsByteArray; - unsigned char intsarray[2]; - - int ret = secp256k1_ec_pubkey_create(ctx, &pubkey, secKey); - - unsigned char outputSer[65]; - size_t outputLen = 65; - - if( ret ) { - int ret2 = secp256k1_ec_pubkey_serialize(ctx,outputSer, &outputLen, &pubkey,SECP256K1_EC_UNCOMPRESSED );(void)ret2; - } - - intsarray[0] = outputLen; - intsarray[1] = ret; - - retArray = (*env)->NewObjectArray(env, 2, - (*env)->FindClass(env, "[B"), - (*env)->NewByteArray(env, 1)); - - pubkeyArray = (*env)->NewByteArray(env, outputLen); - (*env)->SetByteArrayRegion(env, pubkeyArray, 0, outputLen, (jbyte*)outputSer); - (*env)->SetObjectArrayElement(env, retArray, 0, pubkeyArray); - - intsByteArray = (*env)->NewByteArray(env, 2); - (*env)->SetByteArrayRegion(env, intsByteArray, 0, 2, (jbyte*)intsarray); - (*env)->SetObjectArrayElement(env, retArray, 1, intsByteArray); - - (void)classObject; - - return retArray; - -} - -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1privkey_1tweak_1add - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l) -{ - secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; - unsigned char* privkey = (unsigned char*) (*env)->GetDirectBufferAddress(env, byteBufferObject); - const unsigned char* tweak = (unsigned char*) (privkey + 32); - - jobjectArray retArray; - jbyteArray privArray, intsByteArray; - unsigned char intsarray[2]; - - int privkeylen = 32; - - int ret = secp256k1_ec_privkey_tweak_add(ctx, privkey, tweak); - - intsarray[0] = privkeylen; - intsarray[1] = ret; - - retArray = (*env)->NewObjectArray(env, 2, - (*env)->FindClass(env, "[B"), - (*env)->NewByteArray(env, 1)); - - privArray = (*env)->NewByteArray(env, privkeylen); - (*env)->SetByteArrayRegion(env, privArray, 0, privkeylen, (jbyte*)privkey); - (*env)->SetObjectArrayElement(env, retArray, 0, privArray); - - intsByteArray = (*env)->NewByteArray(env, 2); - (*env)->SetByteArrayRegion(env, intsByteArray, 0, 2, (jbyte*)intsarray); - (*env)->SetObjectArrayElement(env, retArray, 1, intsByteArray); - - (void)classObject; - - return retArray; -} - -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1privkey_1tweak_1mul - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l) -{ - secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; - unsigned char* privkey = (unsigned char*) (*env)->GetDirectBufferAddress(env, byteBufferObject); - const unsigned char* tweak = (unsigned char*) (privkey + 32); - - jobjectArray retArray; - jbyteArray privArray, intsByteArray; - unsigned char intsarray[2]; - - int privkeylen = 32; - - int ret = secp256k1_ec_privkey_tweak_mul(ctx, privkey, tweak); - - intsarray[0] = privkeylen; - intsarray[1] = ret; - - retArray = (*env)->NewObjectArray(env, 2, - (*env)->FindClass(env, "[B"), - (*env)->NewByteArray(env, 1)); - - privArray = (*env)->NewByteArray(env, privkeylen); - (*env)->SetByteArrayRegion(env, privArray, 0, privkeylen, (jbyte*)privkey); - (*env)->SetObjectArrayElement(env, retArray, 0, privArray); - - intsByteArray = (*env)->NewByteArray(env, 2); - (*env)->SetByteArrayRegion(env, intsByteArray, 0, 2, (jbyte*)intsarray); - (*env)->SetObjectArrayElement(env, retArray, 1, intsByteArray); - - (void)classObject; - - return retArray; -} - -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1pubkey_1tweak_1add - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l, jint publen) -{ - secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; -/* secp256k1_pubkey* pubkey = (secp256k1_pubkey*) (*env)->GetDirectBufferAddress(env, byteBufferObject);*/ - unsigned char* pkey = (*env)->GetDirectBufferAddress(env, byteBufferObject); - const unsigned char* tweak = (unsigned char*) (pkey + publen); - - jobjectArray retArray; - jbyteArray pubArray, intsByteArray; - unsigned char intsarray[2]; - unsigned char outputSer[65]; - size_t outputLen = 65; - - secp256k1_pubkey pubkey; - int ret = secp256k1_ec_pubkey_parse(ctx, &pubkey, pkey, publen); - - if( ret ) { - ret = secp256k1_ec_pubkey_tweak_add(ctx, &pubkey, tweak); - } - - if( ret ) { - int ret2 = secp256k1_ec_pubkey_serialize(ctx,outputSer, &outputLen, &pubkey,SECP256K1_EC_UNCOMPRESSED );(void)ret2; - } - - intsarray[0] = outputLen; - intsarray[1] = ret; - - retArray = (*env)->NewObjectArray(env, 2, - (*env)->FindClass(env, "[B"), - (*env)->NewByteArray(env, 1)); - - pubArray = (*env)->NewByteArray(env, outputLen); - (*env)->SetByteArrayRegion(env, pubArray, 0, outputLen, (jbyte*)outputSer); - (*env)->SetObjectArrayElement(env, retArray, 0, pubArray); - - intsByteArray = (*env)->NewByteArray(env, 2); - (*env)->SetByteArrayRegion(env, intsByteArray, 0, 2, (jbyte*)intsarray); - (*env)->SetObjectArrayElement(env, retArray, 1, intsByteArray); - - (void)classObject; - - return retArray; -} - -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1pubkey_1tweak_1mul - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l, jint publen) -{ - secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; - unsigned char* pkey = (*env)->GetDirectBufferAddress(env, byteBufferObject); - const unsigned char* tweak = (unsigned char*) (pkey + publen); - - jobjectArray retArray; - jbyteArray pubArray, intsByteArray; - unsigned char intsarray[2]; - unsigned char outputSer[65]; - size_t outputLen = 65; - - secp256k1_pubkey pubkey; - int ret = secp256k1_ec_pubkey_parse(ctx, &pubkey, pkey, publen); - - if ( ret ) { - ret = secp256k1_ec_pubkey_tweak_mul(ctx, &pubkey, tweak); - } - - if( ret ) { - int ret2 = secp256k1_ec_pubkey_serialize(ctx,outputSer, &outputLen, &pubkey,SECP256K1_EC_UNCOMPRESSED );(void)ret2; - } - - intsarray[0] = outputLen; - intsarray[1] = ret; - - retArray = (*env)->NewObjectArray(env, 2, - (*env)->FindClass(env, "[B"), - (*env)->NewByteArray(env, 1)); - - pubArray = (*env)->NewByteArray(env, outputLen); - (*env)->SetByteArrayRegion(env, pubArray, 0, outputLen, (jbyte*)outputSer); - (*env)->SetObjectArrayElement(env, retArray, 0, pubArray); - - intsByteArray = (*env)->NewByteArray(env, 2); - (*env)->SetByteArrayRegion(env, intsByteArray, 0, 2, (jbyte*)intsarray); - (*env)->SetObjectArrayElement(env, retArray, 1, intsByteArray); - - (void)classObject; - - return retArray; -} - -SECP256K1_API jlong JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ecdsa_1pubkey_1combine - (JNIEnv * env, jclass classObject, jobject byteBufferObject, jlong ctx_l, jint numkeys) -{ - (void)classObject;(void)env;(void)byteBufferObject;(void)ctx_l;(void)numkeys; - - return 0; -} - -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ecdh - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l, jint publen) -{ - secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; - const unsigned char* secdata = (*env)->GetDirectBufferAddress(env, byteBufferObject); - const unsigned char* pubdata = (const unsigned char*) (secdata + 32); - - jobjectArray retArray; - jbyteArray outArray, intsByteArray; - unsigned char intsarray[1]; - secp256k1_pubkey pubkey; - unsigned char nonce_res[32]; - size_t outputLen = 32; - - int ret = secp256k1_ec_pubkey_parse(ctx, &pubkey, pubdata, publen); - - if (ret) { - ret = secp256k1_ecdh( - ctx, - nonce_res, - &pubkey, - secdata, - NULL, - NULL - ); - } - - intsarray[0] = ret; - - retArray = (*env)->NewObjectArray(env, 2, - (*env)->FindClass(env, "[B"), - (*env)->NewByteArray(env, 1)); - - outArray = (*env)->NewByteArray(env, outputLen); - (*env)->SetByteArrayRegion(env, outArray, 0, 32, (jbyte*)nonce_res); - (*env)->SetObjectArrayElement(env, retArray, 0, outArray); - - intsByteArray = (*env)->NewByteArray(env, 1); - (*env)->SetByteArrayRegion(env, intsByteArray, 0, 1, (jbyte*)intsarray); - (*env)->SetObjectArrayElement(env, retArray, 1, intsByteArray); - - (void)classObject; - - return retArray; -} diff --git a/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h b/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h deleted file mode 100644 index fe613c9e9e..0000000000 --- a/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h +++ /dev/null @@ -1,119 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -#include "include/secp256k1.h" -/* Header for class org_bitcoin_NativeSecp256k1 */ - -#ifndef _Included_org_bitcoin_NativeSecp256k1 -#define _Included_org_bitcoin_NativeSecp256k1 -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_ctx_clone - * Signature: (J)J - */ -SECP256K1_API jlong JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ctx_1clone - (JNIEnv *, jclass, jlong); - -/* - * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_context_randomize - * Signature: (Ljava/nio/ByteBuffer;J)I - */ -SECP256K1_API jint JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1context_1randomize - (JNIEnv *, jclass, jobject, jlong); - -/* - * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_privkey_tweak_add - * Signature: (Ljava/nio/ByteBuffer;J)[[B - */ -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1privkey_1tweak_1add - (JNIEnv *, jclass, jobject, jlong); - -/* - * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_privkey_tweak_mul - * Signature: (Ljava/nio/ByteBuffer;J)[[B - */ -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1privkey_1tweak_1mul - (JNIEnv *, jclass, jobject, jlong); - -/* - * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_pubkey_tweak_add - * Signature: (Ljava/nio/ByteBuffer;JI)[[B - */ -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1pubkey_1tweak_1add - (JNIEnv *, jclass, jobject, jlong, jint); - -/* - * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_pubkey_tweak_mul - * Signature: (Ljava/nio/ByteBuffer;JI)[[B - */ -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1pubkey_1tweak_1mul - (JNIEnv *, jclass, jobject, jlong, jint); - -/* - * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_destroy_context - * Signature: (J)V - */ -SECP256K1_API void JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1destroy_1context - (JNIEnv *, jclass, jlong); - -/* - * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_ecdsa_verify - * Signature: (Ljava/nio/ByteBuffer;JII)I - */ -SECP256K1_API jint JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ecdsa_1verify - (JNIEnv *, jclass, jobject, jlong, jint, jint); - -/* - * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_ecdsa_sign - * Signature: (Ljava/nio/ByteBuffer;J)[[B - */ -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ecdsa_1sign - (JNIEnv *, jclass, jobject, jlong); - -/* - * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_ec_seckey_verify - * Signature: (Ljava/nio/ByteBuffer;J)I - */ -SECP256K1_API jint JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ec_1seckey_1verify - (JNIEnv *, jclass, jobject, jlong); - -/* - * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_ec_pubkey_create - * Signature: (Ljava/nio/ByteBuffer;J)[[B - */ -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ec_1pubkey_1create - (JNIEnv *, jclass, jobject, jlong); - -/* - * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_ec_pubkey_parse - * Signature: (Ljava/nio/ByteBuffer;JI)[[B - */ -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ec_1pubkey_1parse - (JNIEnv *, jclass, jobject, jlong, jint); - -/* - * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_ecdh - * Signature: (Ljava/nio/ByteBuffer;JI)[[B - */ -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ecdh - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l, jint publen); - - -#ifdef __cplusplus -} -#endif -#endif diff --git a/src/secp256k1/src/java/org_bitcoin_Secp256k1Context.c b/src/secp256k1/src/java/org_bitcoin_Secp256k1Context.c deleted file mode 100644 index a52939e7e7..0000000000 --- a/src/secp256k1/src/java/org_bitcoin_Secp256k1Context.c +++ /dev/null @@ -1,15 +0,0 @@ -#include <stdlib.h> -#include <stdint.h> -#include "org_bitcoin_Secp256k1Context.h" -#include "include/secp256k1.h" - -SECP256K1_API jlong JNICALL Java_org_bitcoin_Secp256k1Context_secp256k1_1init_1context - (JNIEnv* env, jclass classObject) -{ - secp256k1_context *ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); - - (void)classObject;(void)env; - - return (uintptr_t)ctx; -} - diff --git a/src/secp256k1/src/java/org_bitcoin_Secp256k1Context.h b/src/secp256k1/src/java/org_bitcoin_Secp256k1Context.h deleted file mode 100644 index 0d2bc84b7f..0000000000 --- a/src/secp256k1/src/java/org_bitcoin_Secp256k1Context.h +++ /dev/null @@ -1,22 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -#include "include/secp256k1.h" -/* Header for class org_bitcoin_Secp256k1Context */ - -#ifndef _Included_org_bitcoin_Secp256k1Context -#define _Included_org_bitcoin_Secp256k1Context -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: org_bitcoin_Secp256k1Context - * Method: secp256k1_init_context - * Signature: ()J - */ -SECP256K1_API jlong JNICALL Java_org_bitcoin_Secp256k1Context_secp256k1_1init_1context - (JNIEnv *, jclass); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/src/secp256k1/src/modules/ecdh/main_impl.h b/src/secp256k1/src/modules/ecdh/main_impl.h index 44cb68e750..07a25b80d4 100644 --- a/src/secp256k1/src/modules/ecdh/main_impl.h +++ b/src/secp256k1/src/modules/ecdh/main_impl.h @@ -10,14 +10,14 @@ #include "include/secp256k1_ecdh.h" #include "ecmult_const_impl.h" -static int ecdh_hash_function_sha256(unsigned char *output, const unsigned char *x, const unsigned char *y, void *data) { - unsigned char version = (y[31] & 0x01) | 0x02; +static int ecdh_hash_function_sha256(unsigned char *output, const unsigned char *x32, const unsigned char *y32, void *data) { + unsigned char version = (y32[31] & 0x01) | 0x02; secp256k1_sha256 sha; (void)data; secp256k1_sha256_initialize(&sha); secp256k1_sha256_write(&sha, &version, 1); - secp256k1_sha256_write(&sha, x, 32); + secp256k1_sha256_write(&sha, x32, 32); secp256k1_sha256_finalize(&sha, output); return 1; @@ -32,36 +32,40 @@ int secp256k1_ecdh(const secp256k1_context* ctx, unsigned char *output, const se secp256k1_gej res; secp256k1_ge pt; secp256k1_scalar s; + unsigned char x[32]; + unsigned char y[32]; + VERIFY_CHECK(ctx != NULL); ARG_CHECK(output != NULL); ARG_CHECK(point != NULL); ARG_CHECK(scalar != NULL); + if (hashfp == NULL) { hashfp = secp256k1_ecdh_hash_function_default; } secp256k1_pubkey_load(ctx, &pt, point); secp256k1_scalar_set_b32(&s, scalar, &overflow); - if (overflow || secp256k1_scalar_is_zero(&s)) { - ret = 0; - } else { - unsigned char x[32]; - unsigned char y[32]; - - secp256k1_ecmult_const(&res, &pt, &s, 256); - secp256k1_ge_set_gej(&pt, &res); - - /* Compute a hash of the point */ - secp256k1_fe_normalize(&pt.x); - secp256k1_fe_normalize(&pt.y); - secp256k1_fe_get_b32(x, &pt.x); - secp256k1_fe_get_b32(y, &pt.y); - - ret = hashfp(output, x, y, data); - } + overflow |= secp256k1_scalar_is_zero(&s); + secp256k1_scalar_cmov(&s, &secp256k1_scalar_one, overflow); + + secp256k1_ecmult_const(&res, &pt, &s, 256); + secp256k1_ge_set_gej(&pt, &res); + + /* Compute a hash of the point */ + secp256k1_fe_normalize(&pt.x); + secp256k1_fe_normalize(&pt.y); + secp256k1_fe_get_b32(x, &pt.x); + secp256k1_fe_get_b32(y, &pt.y); + + ret = hashfp(output, x, y, data); + + memset(x, 0, 32); + memset(y, 0, 32); secp256k1_scalar_clear(&s); - return ret; + + return !!ret & !overflow; } #endif /* SECP256K1_MODULE_ECDH_MAIN_H */ diff --git a/src/secp256k1/src/modules/recovery/main_impl.h b/src/secp256k1/src/modules/recovery/main_impl.h index 2f6691c5a1..e2576aa953 100755..100644 --- a/src/secp256k1/src/modules/recovery/main_impl.h +++ b/src/secp256k1/src/modules/recovery/main_impl.h @@ -122,48 +122,15 @@ static int secp256k1_ecdsa_sig_recover(const secp256k1_ecmult_context *ctx, cons int secp256k1_ecdsa_sign_recoverable(const secp256k1_context* ctx, secp256k1_ecdsa_recoverable_signature *signature, const unsigned char *msg32, const unsigned char *seckey, secp256k1_nonce_function noncefp, const void* noncedata) { secp256k1_scalar r, s; - secp256k1_scalar sec, non, msg; - int recid; - int ret = 0; - int overflow = 0; + int ret, recid; VERIFY_CHECK(ctx != NULL); ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); ARG_CHECK(msg32 != NULL); ARG_CHECK(signature != NULL); ARG_CHECK(seckey != NULL); - if (noncefp == NULL) { - noncefp = secp256k1_nonce_function_default; - } - secp256k1_scalar_set_b32(&sec, seckey, &overflow); - /* Fail if the secret key is invalid. */ - if (!overflow && !secp256k1_scalar_is_zero(&sec)) { - unsigned char nonce32[32]; - unsigned int count = 0; - secp256k1_scalar_set_b32(&msg, msg32, NULL); - while (1) { - ret = noncefp(nonce32, msg32, seckey, NULL, (void*)noncedata, count); - if (!ret) { - break; - } - secp256k1_scalar_set_b32(&non, nonce32, &overflow); - if (!secp256k1_scalar_is_zero(&non) && !overflow) { - if (secp256k1_ecdsa_sig_sign(&ctx->ecmult_gen_ctx, &r, &s, &sec, &msg, &non, &recid)) { - break; - } - } - count++; - } - memset(nonce32, 0, 32); - secp256k1_scalar_clear(&msg); - secp256k1_scalar_clear(&non); - secp256k1_scalar_clear(&sec); - } - if (ret) { - secp256k1_ecdsa_recoverable_signature_save(signature, &r, &s, recid); - } else { - memset(signature, 0, sizeof(*signature)); - } + ret = secp256k1_ecdsa_sign_inner(ctx, &r, &s, &recid, msg32, seckey, noncefp, noncedata); + secp256k1_ecdsa_recoverable_signature_save(signature, &r, &s, recid); return ret; } diff --git a/src/secp256k1/src/modules/recovery/tests_impl.h b/src/secp256k1/src/modules/recovery/tests_impl.h index 5c9bbe8610..38a533a755 100644 --- a/src/secp256k1/src/modules/recovery/tests_impl.h +++ b/src/secp256k1/src/modules/recovery/tests_impl.h @@ -215,7 +215,7 @@ void test_ecdsa_recovery_edge_cases(void) { }; const unsigned char sig64[64] = { /* Generated by signing the above message with nonce 'This is the nonce we will use...' - * and secret key 0 (which is not valid), resulting in recid 0. */ + * and secret key 0 (which is not valid), resulting in recid 1. */ 0x67, 0xCB, 0x28, 0x5F, 0x9C, 0xD1, 0x94, 0xE8, 0x40, 0xD6, 0x29, 0x39, 0x7A, 0xF5, 0x56, 0x96, 0x62, 0xFD, 0xE4, 0x46, 0x49, 0x99, 0x59, 0x63, diff --git a/src/secp256k1/src/scalar.h b/src/secp256k1/src/scalar.h index 59304cb66e..2a74703523 100644 --- a/src/secp256k1/src/scalar.h +++ b/src/secp256k1/src/scalar.h @@ -32,9 +32,17 @@ static unsigned int secp256k1_scalar_get_bits(const secp256k1_scalar *a, unsigne /** Access bits from a scalar. Not constant time. */ static unsigned int secp256k1_scalar_get_bits_var(const secp256k1_scalar *a, unsigned int offset, unsigned int count); -/** Set a scalar from a big endian byte array. */ +/** Set a scalar from a big endian byte array. The scalar will be reduced modulo group order `n`. + * In: bin: pointer to a 32-byte array. + * Out: r: scalar to be set. + * overflow: non-zero if the scalar was bigger or equal to `n` before reduction, zero otherwise (can be NULL). + */ static void secp256k1_scalar_set_b32(secp256k1_scalar *r, const unsigned char *bin, int *overflow); +/** Set a scalar from a big endian byte array and returns 1 if it is a valid + * seckey and 0 otherwise. */ +static int secp256k1_scalar_set_b32_seckey(secp256k1_scalar *r, const unsigned char *bin); + /** Set a scalar to an unsigned integer. */ static void secp256k1_scalar_set_int(secp256k1_scalar *r, unsigned int v); @@ -103,4 +111,7 @@ static void secp256k1_scalar_split_lambda(secp256k1_scalar *r1, secp256k1_scalar /** Multiply a and b (without taking the modulus!), divide by 2**shift, and round to the nearest integer. Shift must be at least 256. */ static void secp256k1_scalar_mul_shift_var(secp256k1_scalar *r, const secp256k1_scalar *a, const secp256k1_scalar *b, unsigned int shift); +/** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. Both *r and *a must be initialized.*/ +static void secp256k1_scalar_cmov(secp256k1_scalar *r, const secp256k1_scalar *a, int flag); + #endif /* SECP256K1_SCALAR_H */ diff --git a/src/secp256k1/src/scalar_4x64_impl.h b/src/secp256k1/src/scalar_4x64_impl.h index d378335d99..8f539c4bc6 100644 --- a/src/secp256k1/src/scalar_4x64_impl.h +++ b/src/secp256k1/src/scalar_4x64_impl.h @@ -946,4 +946,15 @@ SECP256K1_INLINE static void secp256k1_scalar_mul_shift_var(secp256k1_scalar *r, secp256k1_scalar_cadd_bit(r, 0, (l[(shift - 1) >> 6] >> ((shift - 1) & 0x3f)) & 1); } +static SECP256K1_INLINE void secp256k1_scalar_cmov(secp256k1_scalar *r, const secp256k1_scalar *a, int flag) { + uint64_t mask0, mask1; + VG_CHECK_VERIFY(r->d, sizeof(r->d)); + mask0 = flag + ~((uint64_t)0); + mask1 = ~mask0; + r->d[0] = (r->d[0] & mask0) | (a->d[0] & mask1); + r->d[1] = (r->d[1] & mask0) | (a->d[1] & mask1); + r->d[2] = (r->d[2] & mask0) | (a->d[2] & mask1); + r->d[3] = (r->d[3] & mask0) | (a->d[3] & mask1); +} + #endif /* SECP256K1_SCALAR_REPR_IMPL_H */ diff --git a/src/secp256k1/src/scalar_8x32_impl.h b/src/secp256k1/src/scalar_8x32_impl.h index 4f9ed61fea..3c372f34fe 100644 --- a/src/secp256k1/src/scalar_8x32_impl.h +++ b/src/secp256k1/src/scalar_8x32_impl.h @@ -718,4 +718,19 @@ SECP256K1_INLINE static void secp256k1_scalar_mul_shift_var(secp256k1_scalar *r, secp256k1_scalar_cadd_bit(r, 0, (l[(shift - 1) >> 5] >> ((shift - 1) & 0x1f)) & 1); } +static SECP256K1_INLINE void secp256k1_scalar_cmov(secp256k1_scalar *r, const secp256k1_scalar *a, int flag) { + uint32_t mask0, mask1; + VG_CHECK_VERIFY(r->d, sizeof(r->d)); + mask0 = flag + ~((uint32_t)0); + mask1 = ~mask0; + r->d[0] = (r->d[0] & mask0) | (a->d[0] & mask1); + r->d[1] = (r->d[1] & mask0) | (a->d[1] & mask1); + r->d[2] = (r->d[2] & mask0) | (a->d[2] & mask1); + r->d[3] = (r->d[3] & mask0) | (a->d[3] & mask1); + r->d[4] = (r->d[4] & mask0) | (a->d[4] & mask1); + r->d[5] = (r->d[5] & mask0) | (a->d[5] & mask1); + r->d[6] = (r->d[6] & mask0) | (a->d[6] & mask1); + r->d[7] = (r->d[7] & mask0) | (a->d[7] & mask1); +} + #endif /* SECP256K1_SCALAR_REPR_IMPL_H */ diff --git a/src/secp256k1/src/scalar_impl.h b/src/secp256k1/src/scalar_impl.h index fa790570ff..70cd73db06 100644 --- a/src/secp256k1/src/scalar_impl.h +++ b/src/secp256k1/src/scalar_impl.h @@ -7,8 +7,8 @@ #ifndef SECP256K1_SCALAR_IMPL_H #define SECP256K1_SCALAR_IMPL_H -#include "group.h" #include "scalar.h" +#include "util.h" #if defined HAVE_CONFIG_H #include "libsecp256k1-config.h" @@ -24,6 +24,9 @@ #error "Please select scalar implementation" #endif +static const secp256k1_scalar secp256k1_scalar_one = SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 1); +static const secp256k1_scalar secp256k1_scalar_zero = SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 0); + #ifndef USE_NUM_NONE static void secp256k1_scalar_get_num(secp256k1_num *r, const secp256k1_scalar *a) { unsigned char c[32]; @@ -52,6 +55,12 @@ static void secp256k1_scalar_order_get_num(secp256k1_num *r) { } #endif +static int secp256k1_scalar_set_b32_seckey(secp256k1_scalar *r, const unsigned char *bin) { + int overflow; + secp256k1_scalar_set_b32(r, bin, &overflow); + return (!overflow) & (!secp256k1_scalar_is_zero(r)); +} + static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar *x) { #if defined(EXHAUSTIVE_TEST_ORDER) int i; diff --git a/src/secp256k1/src/scalar_low.h b/src/secp256k1/src/scalar_low.h index 5836febc5b..2794a7f171 100644 --- a/src/secp256k1/src/scalar_low.h +++ b/src/secp256k1/src/scalar_low.h @@ -12,4 +12,6 @@ /** A scalar modulo the group order of the secp256k1 curve. */ typedef uint32_t secp256k1_scalar; +#define SECP256K1_SCALAR_CONST(d7, d6, d5, d4, d3, d2, d1, d0) (d0) + #endif /* SECP256K1_SCALAR_REPR_H */ diff --git a/src/secp256k1/src/scalar_low_impl.h b/src/secp256k1/src/scalar_low_impl.h index c80e70c5a2..b79cf1ff6c 100644 --- a/src/secp256k1/src/scalar_low_impl.h +++ b/src/secp256k1/src/scalar_low_impl.h @@ -38,8 +38,11 @@ static int secp256k1_scalar_add(secp256k1_scalar *r, const secp256k1_scalar *a, static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int flag) { if (flag && bit < 32) - *r += (1 << bit); + *r += ((uint32_t)1 << bit); #ifdef VERIFY + VERIFY_CHECK(bit < 32); + /* Verify that adding (1 << bit) will not overflow any in-range scalar *r by overflowing the underlying uint32_t. */ + VERIFY_CHECK(((uint32_t)1 << bit) - 1 <= UINT32_MAX - EXHAUSTIVE_TEST_ORDER); VERIFY_CHECK(secp256k1_scalar_check_overflow(r) == 0); #endif } @@ -111,4 +114,12 @@ SECP256K1_INLINE static int secp256k1_scalar_eq(const secp256k1_scalar *a, const return *a == *b; } +static SECP256K1_INLINE void secp256k1_scalar_cmov(secp256k1_scalar *r, const secp256k1_scalar *a, int flag) { + uint32_t mask0, mask1; + VG_CHECK_VERIFY(r, sizeof(*r)); + mask0 = flag + ~((uint32_t)0); + mask1 = ~mask0; + *r = (*r & mask0) | (*a & mask1); +} + #endif /* SECP256K1_SCALAR_REPR_IMPL_H */ diff --git a/src/secp256k1/src/scratch.h b/src/secp256k1/src/scratch.h index fef377af0d..77b35d126b 100644 --- a/src/secp256k1/src/scratch.h +++ b/src/secp256k1/src/scratch.h @@ -7,33 +7,36 @@ #ifndef _SECP256K1_SCRATCH_ #define _SECP256K1_SCRATCH_ -#define SECP256K1_SCRATCH_MAX_FRAMES 5 - /* The typedef is used internally; the struct name is used in the public API * (where it is exposed as a different typedef) */ typedef struct secp256k1_scratch_space_struct { - void *data[SECP256K1_SCRATCH_MAX_FRAMES]; - size_t offset[SECP256K1_SCRATCH_MAX_FRAMES]; - size_t frame_size[SECP256K1_SCRATCH_MAX_FRAMES]; - size_t frame; + /** guard against interpreting this object as other types */ + unsigned char magic[8]; + /** actual allocated data */ + void *data; + /** amount that has been allocated (i.e. `data + offset` is the next + * available pointer) */ + size_t alloc_size; + /** maximum size available to allocate */ size_t max_size; - const secp256k1_callback* error_callback; } secp256k1_scratch; static secp256k1_scratch* secp256k1_scratch_create(const secp256k1_callback* error_callback, size_t max_size); -static void secp256k1_scratch_destroy(secp256k1_scratch* scratch); +static void secp256k1_scratch_destroy(const secp256k1_callback* error_callback, secp256k1_scratch* scratch); -/** Attempts to allocate a new stack frame with `n` available bytes. Returns 1 on success, 0 on failure */ -static int secp256k1_scratch_allocate_frame(secp256k1_scratch* scratch, size_t n, size_t objects); +/** Returns an opaque object used to "checkpoint" a scratch space. Used + * with `secp256k1_scratch_apply_checkpoint` to undo allocations. */ +static size_t secp256k1_scratch_checkpoint(const secp256k1_callback* error_callback, const secp256k1_scratch* scratch); -/** Deallocates a stack frame */ -static void secp256k1_scratch_deallocate_frame(secp256k1_scratch* scratch); +/** Applies a check point received from `secp256k1_scratch_checkpoint`, + * undoing all allocations since that point. */ +static void secp256k1_scratch_apply_checkpoint(const secp256k1_callback* error_callback, secp256k1_scratch* scratch, size_t checkpoint); /** Returns the maximum allocation the scratch space will allow */ -static size_t secp256k1_scratch_max_allocation(const secp256k1_scratch* scratch, size_t n_objects); +static size_t secp256k1_scratch_max_allocation(const secp256k1_callback* error_callback, const secp256k1_scratch* scratch, size_t n_objects); /** Returns a pointer into the most recently allocated frame, or NULL if there is insufficient available space */ -static void *secp256k1_scratch_alloc(secp256k1_scratch* scratch, size_t n); +static void *secp256k1_scratch_alloc(const secp256k1_callback* error_callback, secp256k1_scratch* scratch, size_t n); #endif diff --git a/src/secp256k1/src/scratch_impl.h b/src/secp256k1/src/scratch_impl.h index abed713b21..4cee700001 100644 --- a/src/secp256k1/src/scratch_impl.h +++ b/src/secp256k1/src/scratch_impl.h @@ -7,78 +7,80 @@ #ifndef _SECP256K1_SCRATCH_IMPL_H_ #define _SECP256K1_SCRATCH_IMPL_H_ +#include "util.h" #include "scratch.h" -/* Using 16 bytes alignment because common architectures never have alignment - * requirements above 8 for any of the types we care about. In addition we - * leave some room because currently we don't care about a few bytes. - * TODO: Determine this at configure time. */ -#define ALIGNMENT 16 - -static secp256k1_scratch* secp256k1_scratch_create(const secp256k1_callback* error_callback, size_t max_size) { - secp256k1_scratch* ret = (secp256k1_scratch*)checked_malloc(error_callback, sizeof(*ret)); +static secp256k1_scratch* secp256k1_scratch_create(const secp256k1_callback* error_callback, size_t size) { + const size_t base_alloc = ((sizeof(secp256k1_scratch) + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT; + void *alloc = checked_malloc(error_callback, base_alloc + size); + secp256k1_scratch* ret = (secp256k1_scratch *)alloc; if (ret != NULL) { memset(ret, 0, sizeof(*ret)); - ret->max_size = max_size; - ret->error_callback = error_callback; + memcpy(ret->magic, "scratch", 8); + ret->data = (void *) ((char *) alloc + base_alloc); + ret->max_size = size; } return ret; } -static void secp256k1_scratch_destroy(secp256k1_scratch* scratch) { +static void secp256k1_scratch_destroy(const secp256k1_callback* error_callback, secp256k1_scratch* scratch) { if (scratch != NULL) { - VERIFY_CHECK(scratch->frame == 0); + VERIFY_CHECK(scratch->alloc_size == 0); /* all checkpoints should be applied */ + if (memcmp(scratch->magic, "scratch", 8) != 0) { + secp256k1_callback_call(error_callback, "invalid scratch space"); + return; + } + memset(scratch->magic, 0, sizeof(scratch->magic)); free(scratch); } } -static size_t secp256k1_scratch_max_allocation(const secp256k1_scratch* scratch, size_t objects) { - size_t i = 0; - size_t allocated = 0; - for (i = 0; i < scratch->frame; i++) { - allocated += scratch->frame_size[i]; - } - if (scratch->max_size - allocated <= objects * ALIGNMENT) { +static size_t secp256k1_scratch_checkpoint(const secp256k1_callback* error_callback, const secp256k1_scratch* scratch) { + if (memcmp(scratch->magic, "scratch", 8) != 0) { + secp256k1_callback_call(error_callback, "invalid scratch space"); return 0; } - return scratch->max_size - allocated - objects * ALIGNMENT; + return scratch->alloc_size; } -static int secp256k1_scratch_allocate_frame(secp256k1_scratch* scratch, size_t n, size_t objects) { - VERIFY_CHECK(scratch->frame < SECP256K1_SCRATCH_MAX_FRAMES); - - if (n <= secp256k1_scratch_max_allocation(scratch, objects)) { - n += objects * ALIGNMENT; - scratch->data[scratch->frame] = checked_malloc(scratch->error_callback, n); - if (scratch->data[scratch->frame] == NULL) { - return 0; - } - scratch->frame_size[scratch->frame] = n; - scratch->offset[scratch->frame] = 0; - scratch->frame++; - return 1; - } else { - return 0; +static void secp256k1_scratch_apply_checkpoint(const secp256k1_callback* error_callback, secp256k1_scratch* scratch, size_t checkpoint) { + if (memcmp(scratch->magic, "scratch", 8) != 0) { + secp256k1_callback_call(error_callback, "invalid scratch space"); + return; + } + if (checkpoint > scratch->alloc_size) { + secp256k1_callback_call(error_callback, "invalid checkpoint"); + return; } + scratch->alloc_size = checkpoint; } -static void secp256k1_scratch_deallocate_frame(secp256k1_scratch* scratch) { - VERIFY_CHECK(scratch->frame > 0); - scratch->frame -= 1; - free(scratch->data[scratch->frame]); +static size_t secp256k1_scratch_max_allocation(const secp256k1_callback* error_callback, const secp256k1_scratch* scratch, size_t objects) { + if (memcmp(scratch->magic, "scratch", 8) != 0) { + secp256k1_callback_call(error_callback, "invalid scratch space"); + return 0; + } + if (scratch->max_size - scratch->alloc_size <= objects * (ALIGNMENT - 1)) { + return 0; + } + return scratch->max_size - scratch->alloc_size - objects * (ALIGNMENT - 1); } -static void *secp256k1_scratch_alloc(secp256k1_scratch* scratch, size_t size) { +static void *secp256k1_scratch_alloc(const secp256k1_callback* error_callback, secp256k1_scratch* scratch, size_t size) { void *ret; - size_t frame = scratch->frame - 1; - size = ((size + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT; + size = ROUND_TO_ALIGN(size); + + if (memcmp(scratch->magic, "scratch", 8) != 0) { + secp256k1_callback_call(error_callback, "invalid scratch space"); + return NULL; + } - if (scratch->frame == 0 || size + scratch->offset[frame] > scratch->frame_size[frame]) { + if (size > scratch->max_size - scratch->alloc_size) { return NULL; } - ret = (void *) ((unsigned char *) scratch->data[frame] + scratch->offset[frame]); + ret = (void *) ((char *) scratch->data + scratch->alloc_size); memset(ret, 0, size); - scratch->offset[frame] += size; + scratch->alloc_size += size; return ret; } diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index 15981f46e2..b03a6e6345 100644 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -5,6 +5,7 @@ **********************************************************************/ #include "include/secp256k1.h" +#include "include/secp256k1_preallocated.h" #include "util.h" #include "num_impl.h" @@ -19,6 +20,10 @@ #include "hash_impl.h" #include "scratch_impl.h" +#if defined(VALGRIND) +# include <valgrind/memcheck.h> +#endif + #define ARG_CHECK(cond) do { \ if (EXPECT(!(cond), 0)) { \ secp256k1_callback_call(&ctx->illegal_callback, #cond); \ @@ -26,53 +31,101 @@ } \ } while(0) -static void default_illegal_callback_fn(const char* str, void* data) { +#define ARG_CHECK_NO_RETURN(cond) do { \ + if (EXPECT(!(cond), 0)) { \ + secp256k1_callback_call(&ctx->illegal_callback, #cond); \ + } \ +} while(0) + +#ifndef USE_EXTERNAL_DEFAULT_CALLBACKS +#include <stdlib.h> +#include <stdio.h> +static void secp256k1_default_illegal_callback_fn(const char* str, void* data) { (void)data; fprintf(stderr, "[libsecp256k1] illegal argument: %s\n", str); abort(); } - -static const secp256k1_callback default_illegal_callback = { - default_illegal_callback_fn, - NULL -}; - -static void default_error_callback_fn(const char* str, void* data) { +static void secp256k1_default_error_callback_fn(const char* str, void* data) { (void)data; fprintf(stderr, "[libsecp256k1] internal consistency check failed: %s\n", str); abort(); } +#else +void secp256k1_default_illegal_callback_fn(const char* str, void* data); +void secp256k1_default_error_callback_fn(const char* str, void* data); +#endif -static const secp256k1_callback default_error_callback = { - default_error_callback_fn, +static const secp256k1_callback default_illegal_callback = { + secp256k1_default_illegal_callback_fn, NULL }; +static const secp256k1_callback default_error_callback = { + secp256k1_default_error_callback_fn, + NULL +}; struct secp256k1_context_struct { secp256k1_ecmult_context ecmult_ctx; secp256k1_ecmult_gen_context ecmult_gen_ctx; secp256k1_callback illegal_callback; secp256k1_callback error_callback; + int declassify; }; static const secp256k1_context secp256k1_context_no_precomp_ = { { 0 }, { 0 }, - { default_illegal_callback_fn, 0 }, - { default_error_callback_fn, 0 } + { secp256k1_default_illegal_callback_fn, 0 }, + { secp256k1_default_error_callback_fn, 0 }, + 0 }; const secp256k1_context *secp256k1_context_no_precomp = &secp256k1_context_no_precomp_; -secp256k1_context* secp256k1_context_create(unsigned int flags) { - secp256k1_context* ret = (secp256k1_context*)checked_malloc(&default_error_callback, sizeof(secp256k1_context)); +size_t secp256k1_context_preallocated_size(unsigned int flags) { + size_t ret = ROUND_TO_ALIGN(sizeof(secp256k1_context)); + + if (EXPECT((flags & SECP256K1_FLAGS_TYPE_MASK) != SECP256K1_FLAGS_TYPE_CONTEXT, 0)) { + secp256k1_callback_call(&default_illegal_callback, + "Invalid flags"); + return 0; + } + + if (flags & SECP256K1_FLAGS_BIT_CONTEXT_SIGN) { + ret += SECP256K1_ECMULT_GEN_CONTEXT_PREALLOCATED_SIZE; + } + if (flags & SECP256K1_FLAGS_BIT_CONTEXT_VERIFY) { + ret += SECP256K1_ECMULT_CONTEXT_PREALLOCATED_SIZE; + } + return ret; +} + +size_t secp256k1_context_preallocated_clone_size(const secp256k1_context* ctx) { + size_t ret = ROUND_TO_ALIGN(sizeof(secp256k1_context)); + VERIFY_CHECK(ctx != NULL); + if (secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)) { + ret += SECP256K1_ECMULT_GEN_CONTEXT_PREALLOCATED_SIZE; + } + if (secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)) { + ret += SECP256K1_ECMULT_CONTEXT_PREALLOCATED_SIZE; + } + return ret; +} + +secp256k1_context* secp256k1_context_preallocated_create(void* prealloc, unsigned int flags) { + void* const base = prealloc; + size_t prealloc_size; + secp256k1_context* ret; + + VERIFY_CHECK(prealloc != NULL); + prealloc_size = secp256k1_context_preallocated_size(flags); + ret = (secp256k1_context*)manual_alloc(&prealloc, sizeof(secp256k1_context), base, prealloc_size); ret->illegal_callback = default_illegal_callback; ret->error_callback = default_error_callback; if (EXPECT((flags & SECP256K1_FLAGS_TYPE_MASK) != SECP256K1_FLAGS_TYPE_CONTEXT, 0)) { secp256k1_callback_call(&ret->illegal_callback, "Invalid flags"); - free(ret); return NULL; } @@ -80,47 +133,80 @@ secp256k1_context* secp256k1_context_create(unsigned int flags) { secp256k1_ecmult_gen_context_init(&ret->ecmult_gen_ctx); if (flags & SECP256K1_FLAGS_BIT_CONTEXT_SIGN) { - secp256k1_ecmult_gen_context_build(&ret->ecmult_gen_ctx, &ret->error_callback); + secp256k1_ecmult_gen_context_build(&ret->ecmult_gen_ctx, &prealloc); } if (flags & SECP256K1_FLAGS_BIT_CONTEXT_VERIFY) { - secp256k1_ecmult_context_build(&ret->ecmult_ctx, &ret->error_callback); + secp256k1_ecmult_context_build(&ret->ecmult_ctx, &prealloc); + } + ret->declassify = !!(flags & SECP256K1_FLAGS_BIT_CONTEXT_DECLASSIFY); + + return (secp256k1_context*) ret; +} + +secp256k1_context* secp256k1_context_create(unsigned int flags) { + size_t const prealloc_size = secp256k1_context_preallocated_size(flags); + secp256k1_context* ctx = (secp256k1_context*)checked_malloc(&default_error_callback, prealloc_size); + if (EXPECT(secp256k1_context_preallocated_create(ctx, flags) == NULL, 0)) { + free(ctx); + return NULL; } + return ctx; +} + +secp256k1_context* secp256k1_context_preallocated_clone(const secp256k1_context* ctx, void* prealloc) { + size_t prealloc_size; + secp256k1_context* ret; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(prealloc != NULL); + + prealloc_size = secp256k1_context_preallocated_clone_size(ctx); + ret = (secp256k1_context*)prealloc; + memcpy(ret, ctx, prealloc_size); + secp256k1_ecmult_gen_context_finalize_memcpy(&ret->ecmult_gen_ctx, &ctx->ecmult_gen_ctx); + secp256k1_ecmult_context_finalize_memcpy(&ret->ecmult_ctx, &ctx->ecmult_ctx); return ret; } secp256k1_context* secp256k1_context_clone(const secp256k1_context* ctx) { - secp256k1_context* ret = (secp256k1_context*)checked_malloc(&ctx->error_callback, sizeof(secp256k1_context)); - ret->illegal_callback = ctx->illegal_callback; - ret->error_callback = ctx->error_callback; - secp256k1_ecmult_context_clone(&ret->ecmult_ctx, &ctx->ecmult_ctx, &ctx->error_callback); - secp256k1_ecmult_gen_context_clone(&ret->ecmult_gen_ctx, &ctx->ecmult_gen_ctx, &ctx->error_callback); + secp256k1_context* ret; + size_t prealloc_size; + + VERIFY_CHECK(ctx != NULL); + prealloc_size = secp256k1_context_preallocated_clone_size(ctx); + ret = (secp256k1_context*)checked_malloc(&ctx->error_callback, prealloc_size); + ret = secp256k1_context_preallocated_clone(ctx, ret); return ret; } -void secp256k1_context_destroy(secp256k1_context* ctx) { - CHECK(ctx != secp256k1_context_no_precomp); +void secp256k1_context_preallocated_destroy(secp256k1_context* ctx) { + ARG_CHECK_NO_RETURN(ctx != secp256k1_context_no_precomp); if (ctx != NULL) { secp256k1_ecmult_context_clear(&ctx->ecmult_ctx); secp256k1_ecmult_gen_context_clear(&ctx->ecmult_gen_ctx); + } +} +void secp256k1_context_destroy(secp256k1_context* ctx) { + if (ctx != NULL) { + secp256k1_context_preallocated_destroy(ctx); free(ctx); } } void secp256k1_context_set_illegal_callback(secp256k1_context* ctx, void (*fun)(const char* message, void* data), const void* data) { - CHECK(ctx != secp256k1_context_no_precomp); + ARG_CHECK_NO_RETURN(ctx != secp256k1_context_no_precomp); if (fun == NULL) { - fun = default_illegal_callback_fn; + fun = secp256k1_default_illegal_callback_fn; } ctx->illegal_callback.fn = fun; ctx->illegal_callback.data = data; } void secp256k1_context_set_error_callback(secp256k1_context* ctx, void (*fun)(const char* message, void* data), const void* data) { - CHECK(ctx != secp256k1_context_no_precomp); + ARG_CHECK_NO_RETURN(ctx != secp256k1_context_no_precomp); if (fun == NULL) { - fun = default_error_callback_fn; + fun = secp256k1_default_error_callback_fn; } ctx->error_callback.fn = fun; ctx->error_callback.data = data; @@ -131,8 +217,23 @@ secp256k1_scratch_space* secp256k1_scratch_space_create(const secp256k1_context* return secp256k1_scratch_create(&ctx->error_callback, max_size); } -void secp256k1_scratch_space_destroy(secp256k1_scratch_space* scratch) { - secp256k1_scratch_destroy(scratch); +void secp256k1_scratch_space_destroy(const secp256k1_context *ctx, secp256k1_scratch_space* scratch) { + VERIFY_CHECK(ctx != NULL); + secp256k1_scratch_destroy(&ctx->error_callback, scratch); +} + +/* Mark memory as no-longer-secret for the purpose of analysing constant-time behaviour + * of the software. This is setup for use with valgrind but could be substituted with + * the appropriate instrumentation for other analysis tools. + */ +static SECP256K1_INLINE void secp256k1_declassify(const secp256k1_context* ctx, void *p, size_t len) { +#if defined(VALGRIND) + if (EXPECT(ctx->declassify,0)) VALGRIND_MAKE_MEM_DEFINED(p, len); +#else + (void)ctx; + (void)p; + (void)len; +#endif } static int secp256k1_pubkey_load(const secp256k1_context* ctx, secp256k1_ge* ge, const secp256k1_pubkey* pubkey) { @@ -366,61 +467,83 @@ static int nonce_function_rfc6979(unsigned char *nonce32, const unsigned char *m const secp256k1_nonce_function secp256k1_nonce_function_rfc6979 = nonce_function_rfc6979; const secp256k1_nonce_function secp256k1_nonce_function_default = nonce_function_rfc6979; -int secp256k1_ecdsa_sign(const secp256k1_context* ctx, secp256k1_ecdsa_signature *signature, const unsigned char *msg32, const unsigned char *seckey, secp256k1_nonce_function noncefp, const void* noncedata) { - secp256k1_scalar r, s; +static int secp256k1_ecdsa_sign_inner(const secp256k1_context* ctx, secp256k1_scalar* r, secp256k1_scalar* s, int* recid, const unsigned char *msg32, const unsigned char *seckey, secp256k1_nonce_function noncefp, const void* noncedata) { secp256k1_scalar sec, non, msg; int ret = 0; - int overflow = 0; - VERIFY_CHECK(ctx != NULL); - ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); - ARG_CHECK(msg32 != NULL); - ARG_CHECK(signature != NULL); - ARG_CHECK(seckey != NULL); + int is_sec_valid; + unsigned char nonce32[32]; + unsigned int count = 0; + /* Default initialization here is important so we won't pass uninit values to the cmov in the end */ + *r = secp256k1_scalar_zero; + *s = secp256k1_scalar_zero; + if (recid) { + *recid = 0; + } if (noncefp == NULL) { noncefp = secp256k1_nonce_function_default; } - secp256k1_scalar_set_b32(&sec, seckey, &overflow); /* Fail if the secret key is invalid. */ - if (!overflow && !secp256k1_scalar_is_zero(&sec)) { - unsigned char nonce32[32]; - unsigned int count = 0; - secp256k1_scalar_set_b32(&msg, msg32, NULL); - while (1) { - ret = noncefp(nonce32, msg32, seckey, NULL, (void*)noncedata, count); - if (!ret) { + is_sec_valid = secp256k1_scalar_set_b32_seckey(&sec, seckey); + secp256k1_scalar_cmov(&sec, &secp256k1_scalar_one, !is_sec_valid); + secp256k1_scalar_set_b32(&msg, msg32, NULL); + while (1) { + int is_nonce_valid; + ret = !!noncefp(nonce32, msg32, seckey, NULL, (void*)noncedata, count); + if (!ret) { + break; + } + is_nonce_valid = secp256k1_scalar_set_b32_seckey(&non, nonce32); + /* The nonce is still secret here, but it being invalid is is less likely than 1:2^255. */ + secp256k1_declassify(ctx, &is_nonce_valid, sizeof(is_nonce_valid)); + if (is_nonce_valid) { + ret = secp256k1_ecdsa_sig_sign(&ctx->ecmult_gen_ctx, r, s, &sec, &msg, &non, recid); + /* The final signature is no longer a secret, nor is the fact that we were successful or not. */ + secp256k1_declassify(ctx, &ret, sizeof(ret)); + if (ret) { break; } - secp256k1_scalar_set_b32(&non, nonce32, &overflow); - if (!overflow && !secp256k1_scalar_is_zero(&non)) { - if (secp256k1_ecdsa_sig_sign(&ctx->ecmult_gen_ctx, &r, &s, &sec, &msg, &non, NULL)) { - break; - } - } - count++; } - memset(nonce32, 0, 32); - secp256k1_scalar_clear(&msg); - secp256k1_scalar_clear(&non); - secp256k1_scalar_clear(&sec); + count++; } - if (ret) { - secp256k1_ecdsa_signature_save(signature, &r, &s); - } else { - memset(signature, 0, sizeof(*signature)); + /* We don't want to declassify is_sec_valid and therefore the range of + * seckey. As a result is_sec_valid is included in ret only after ret was + * used as a branching variable. */ + ret &= is_sec_valid; + memset(nonce32, 0, 32); + secp256k1_scalar_clear(&msg); + secp256k1_scalar_clear(&non); + secp256k1_scalar_clear(&sec); + secp256k1_scalar_cmov(r, &secp256k1_scalar_zero, !ret); + secp256k1_scalar_cmov(s, &secp256k1_scalar_zero, !ret); + if (recid) { + const int zero = 0; + secp256k1_int_cmov(recid, &zero, !ret); } return ret; } +int secp256k1_ecdsa_sign(const secp256k1_context* ctx, secp256k1_ecdsa_signature *signature, const unsigned char *msg32, const unsigned char *seckey, secp256k1_nonce_function noncefp, const void* noncedata) { + secp256k1_scalar r, s; + int ret; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); + ARG_CHECK(msg32 != NULL); + ARG_CHECK(signature != NULL); + ARG_CHECK(seckey != NULL); + + ret = secp256k1_ecdsa_sign_inner(ctx, &r, &s, NULL, msg32, seckey, noncefp, noncedata); + secp256k1_ecdsa_signature_save(signature, &r, &s); + return ret; +} + int secp256k1_ec_seckey_verify(const secp256k1_context* ctx, const unsigned char *seckey) { secp256k1_scalar sec; int ret; - int overflow; VERIFY_CHECK(ctx != NULL); ARG_CHECK(seckey != NULL); - secp256k1_scalar_set_b32(&sec, seckey, &overflow); - ret = !overflow && !secp256k1_scalar_is_zero(&sec); + ret = secp256k1_scalar_set_b32_seckey(&sec, seckey); secp256k1_scalar_clear(&sec); return ret; } @@ -429,7 +552,6 @@ int secp256k1_ec_pubkey_create(const secp256k1_context* ctx, secp256k1_pubkey *p secp256k1_gej pj; secp256k1_ge p; secp256k1_scalar sec; - int overflow; int ret = 0; VERIFY_CHECK(ctx != NULL); ARG_CHECK(pubkey != NULL); @@ -437,27 +559,35 @@ int secp256k1_ec_pubkey_create(const secp256k1_context* ctx, secp256k1_pubkey *p ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); ARG_CHECK(seckey != NULL); - secp256k1_scalar_set_b32(&sec, seckey, &overflow); - ret = (!overflow) & (!secp256k1_scalar_is_zero(&sec)); - if (ret) { - secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &pj, &sec); - secp256k1_ge_set_gej(&p, &pj); - secp256k1_pubkey_save(pubkey, &p); - } + ret = secp256k1_scalar_set_b32_seckey(&sec, seckey); + secp256k1_scalar_cmov(&sec, &secp256k1_scalar_one, !ret); + + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &pj, &sec); + secp256k1_ge_set_gej(&p, &pj); + secp256k1_pubkey_save(pubkey, &p); + memczero(pubkey, sizeof(*pubkey), !ret); + secp256k1_scalar_clear(&sec); return ret; } -int secp256k1_ec_privkey_negate(const secp256k1_context* ctx, unsigned char *seckey) { +int secp256k1_ec_seckey_negate(const secp256k1_context* ctx, unsigned char *seckey) { secp256k1_scalar sec; + int ret = 0; VERIFY_CHECK(ctx != NULL); ARG_CHECK(seckey != NULL); - secp256k1_scalar_set_b32(&sec, seckey, NULL); + ret = secp256k1_scalar_set_b32_seckey(&sec, seckey); + secp256k1_scalar_cmov(&sec, &secp256k1_scalar_zero, !ret); secp256k1_scalar_negate(&sec, &sec); secp256k1_scalar_get_b32(seckey, &sec); - return 1; + secp256k1_scalar_clear(&sec); + return ret; +} + +int secp256k1_ec_privkey_negate(const secp256k1_context* ctx, unsigned char *seckey) { + return secp256k1_ec_seckey_negate(ctx, seckey); } int secp256k1_ec_pubkey_negate(const secp256k1_context* ctx, secp256k1_pubkey *pubkey) { @@ -475,7 +605,7 @@ int secp256k1_ec_pubkey_negate(const secp256k1_context* ctx, secp256k1_pubkey *p return ret; } -int secp256k1_ec_privkey_tweak_add(const secp256k1_context* ctx, unsigned char *seckey, const unsigned char *tweak) { +int secp256k1_ec_seckey_tweak_add(const secp256k1_context* ctx, unsigned char *seckey, const unsigned char *tweak) { secp256k1_scalar term; secp256k1_scalar sec; int ret = 0; @@ -485,19 +615,21 @@ int secp256k1_ec_privkey_tweak_add(const secp256k1_context* ctx, unsigned char * ARG_CHECK(tweak != NULL); secp256k1_scalar_set_b32(&term, tweak, &overflow); - secp256k1_scalar_set_b32(&sec, seckey, NULL); + ret = secp256k1_scalar_set_b32_seckey(&sec, seckey); - ret = !overflow && secp256k1_eckey_privkey_tweak_add(&sec, &term); - memset(seckey, 0, 32); - if (ret) { - secp256k1_scalar_get_b32(seckey, &sec); - } + ret &= (!overflow) & secp256k1_eckey_privkey_tweak_add(&sec, &term); + secp256k1_scalar_cmov(&sec, &secp256k1_scalar_zero, !ret); + secp256k1_scalar_get_b32(seckey, &sec); secp256k1_scalar_clear(&sec); secp256k1_scalar_clear(&term); return ret; } +int secp256k1_ec_privkey_tweak_add(const secp256k1_context* ctx, unsigned char *seckey, const unsigned char *tweak) { + return secp256k1_ec_seckey_tweak_add(ctx, seckey, tweak); +} + int secp256k1_ec_pubkey_tweak_add(const secp256k1_context* ctx, secp256k1_pubkey *pubkey, const unsigned char *tweak) { secp256k1_ge p; secp256k1_scalar term; @@ -522,7 +654,7 @@ int secp256k1_ec_pubkey_tweak_add(const secp256k1_context* ctx, secp256k1_pubkey return ret; } -int secp256k1_ec_privkey_tweak_mul(const secp256k1_context* ctx, unsigned char *seckey, const unsigned char *tweak) { +int secp256k1_ec_seckey_tweak_mul(const secp256k1_context* ctx, unsigned char *seckey, const unsigned char *tweak) { secp256k1_scalar factor; secp256k1_scalar sec; int ret = 0; @@ -532,18 +664,20 @@ int secp256k1_ec_privkey_tweak_mul(const secp256k1_context* ctx, unsigned char * ARG_CHECK(tweak != NULL); secp256k1_scalar_set_b32(&factor, tweak, &overflow); - secp256k1_scalar_set_b32(&sec, seckey, NULL); - ret = !overflow && secp256k1_eckey_privkey_tweak_mul(&sec, &factor); - memset(seckey, 0, 32); - if (ret) { - secp256k1_scalar_get_b32(seckey, &sec); - } + ret = secp256k1_scalar_set_b32_seckey(&sec, seckey); + ret &= (!overflow) & secp256k1_eckey_privkey_tweak_mul(&sec, &factor); + secp256k1_scalar_cmov(&sec, &secp256k1_scalar_zero, !ret); + secp256k1_scalar_get_b32(seckey, &sec); secp256k1_scalar_clear(&sec); secp256k1_scalar_clear(&factor); return ret; } +int secp256k1_ec_privkey_tweak_mul(const secp256k1_context* ctx, unsigned char *seckey, const unsigned char *tweak) { + return secp256k1_ec_seckey_tweak_mul(ctx, seckey, tweak); +} + int secp256k1_ec_pubkey_tweak_mul(const secp256k1_context* ctx, secp256k1_pubkey *pubkey, const unsigned char *tweak) { secp256k1_ge p; secp256k1_scalar factor; diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index f1c4db929a..374ed7dc12 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -16,6 +16,7 @@ #include "secp256k1.c" #include "include/secp256k1.h" +#include "include/secp256k1_preallocated.h" #include "testrand_impl.h" #ifdef ENABLE_OPENSSL_TESTS @@ -31,17 +32,6 @@ void ECDSA_SIG_get0(const ECDSA_SIG *sig, const BIGNUM **pr, const BIGNUM **ps) #include "contrib/lax_der_parsing.c" #include "contrib/lax_der_privatekey_parsing.c" -#if !defined(VG_CHECK) -# if defined(VALGRIND) -# include <valgrind/memcheck.h> -# define VG_UNDEF(x,y) VALGRIND_MAKE_MEM_UNDEFINED((x),(y)) -# define VG_CHECK(x,y) VALGRIND_CHECK_MEM_IS_DEFINED((x),(y)) -# else -# define VG_UNDEF(x,y) -# define VG_CHECK(x,y) -# endif -#endif - static int count = 64; static secp256k1_context *ctx = NULL; @@ -82,7 +72,9 @@ void random_field_element_magnitude(secp256k1_fe *fe) { secp256k1_fe_negate(&zero, &zero, 0); secp256k1_fe_mul_int(&zero, n - 1); secp256k1_fe_add(fe, &zero); - VERIFY_CHECK(fe->magnitude == n); +#ifdef VERIFY + CHECK(fe->magnitude == n); +#endif } void random_group_element_test(secp256k1_ge *ge) { @@ -137,23 +129,53 @@ void random_scalar_order(secp256k1_scalar *num) { } while(1); } -void run_context_tests(void) { +void random_scalar_order_b32(unsigned char *b32) { + secp256k1_scalar num; + random_scalar_order(&num); + secp256k1_scalar_get_b32(b32, &num); +} + +void run_context_tests(int use_prealloc) { secp256k1_pubkey pubkey; secp256k1_pubkey zero_pubkey; secp256k1_ecdsa_signature sig; unsigned char ctmp[32]; int32_t ecount; int32_t ecount2; - secp256k1_context *none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); - secp256k1_context *sign = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); - secp256k1_context *vrfy = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); - secp256k1_context *both = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + secp256k1_context *none; + secp256k1_context *sign; + secp256k1_context *vrfy; + secp256k1_context *both; + void *none_prealloc = NULL; + void *sign_prealloc = NULL; + void *vrfy_prealloc = NULL; + void *both_prealloc = NULL; secp256k1_gej pubj; secp256k1_ge pub; secp256k1_scalar msg, key, nonce; secp256k1_scalar sigr, sigs; + if (use_prealloc) { + none_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_NONE)); + sign_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN)); + vrfy_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_VERIFY)); + both_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY)); + CHECK(none_prealloc != NULL); + CHECK(sign_prealloc != NULL); + CHECK(vrfy_prealloc != NULL); + CHECK(both_prealloc != NULL); + none = secp256k1_context_preallocated_create(none_prealloc, SECP256K1_CONTEXT_NONE); + sign = secp256k1_context_preallocated_create(sign_prealloc, SECP256K1_CONTEXT_SIGN); + vrfy = secp256k1_context_preallocated_create(vrfy_prealloc, SECP256K1_CONTEXT_VERIFY); + both = secp256k1_context_preallocated_create(both_prealloc, SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + } else { + none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); + sign = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + vrfy = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); + both = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + } + memset(&zero_pubkey, 0, sizeof(zero_pubkey)); ecount = 0; @@ -163,14 +185,57 @@ void run_context_tests(void) { secp256k1_context_set_error_callback(sign, counting_illegal_callback_fn, NULL); CHECK(vrfy->error_callback.fn != sign->error_callback.fn); + /* check if sizes for cloning are consistent */ + CHECK(secp256k1_context_preallocated_clone_size(none) == secp256k1_context_preallocated_size(SECP256K1_CONTEXT_NONE)); + CHECK(secp256k1_context_preallocated_clone_size(sign) == secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN)); + CHECK(secp256k1_context_preallocated_clone_size(vrfy) == secp256k1_context_preallocated_size(SECP256K1_CONTEXT_VERIFY)); + CHECK(secp256k1_context_preallocated_clone_size(both) == secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY)); + /*** clone and destroy all of them to make sure cloning was complete ***/ { secp256k1_context *ctx_tmp; - ctx_tmp = none; none = secp256k1_context_clone(none); secp256k1_context_destroy(ctx_tmp); - ctx_tmp = sign; sign = secp256k1_context_clone(sign); secp256k1_context_destroy(ctx_tmp); - ctx_tmp = vrfy; vrfy = secp256k1_context_clone(vrfy); secp256k1_context_destroy(ctx_tmp); - ctx_tmp = both; both = secp256k1_context_clone(both); secp256k1_context_destroy(ctx_tmp); + if (use_prealloc) { + /* clone into a non-preallocated context and then again into a new preallocated one. */ + ctx_tmp = none; none = secp256k1_context_clone(none); secp256k1_context_preallocated_destroy(ctx_tmp); + free(none_prealloc); none_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_NONE)); CHECK(none_prealloc != NULL); + ctx_tmp = none; none = secp256k1_context_preallocated_clone(none, none_prealloc); secp256k1_context_destroy(ctx_tmp); + + ctx_tmp = sign; sign = secp256k1_context_clone(sign); secp256k1_context_preallocated_destroy(ctx_tmp); + free(sign_prealloc); sign_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN)); CHECK(sign_prealloc != NULL); + ctx_tmp = sign; sign = secp256k1_context_preallocated_clone(sign, sign_prealloc); secp256k1_context_destroy(ctx_tmp); + + ctx_tmp = vrfy; vrfy = secp256k1_context_clone(vrfy); secp256k1_context_preallocated_destroy(ctx_tmp); + free(vrfy_prealloc); vrfy_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_VERIFY)); CHECK(vrfy_prealloc != NULL); + ctx_tmp = vrfy; vrfy = secp256k1_context_preallocated_clone(vrfy, vrfy_prealloc); secp256k1_context_destroy(ctx_tmp); + + ctx_tmp = both; both = secp256k1_context_clone(both); secp256k1_context_preallocated_destroy(ctx_tmp); + free(both_prealloc); both_prealloc = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY)); CHECK(both_prealloc != NULL); + ctx_tmp = both; both = secp256k1_context_preallocated_clone(both, both_prealloc); secp256k1_context_destroy(ctx_tmp); + } else { + /* clone into a preallocated context and then again into a new non-preallocated one. */ + void *prealloc_tmp; + + prealloc_tmp = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_NONE)); CHECK(prealloc_tmp != NULL); + ctx_tmp = none; none = secp256k1_context_preallocated_clone(none, prealloc_tmp); secp256k1_context_destroy(ctx_tmp); + ctx_tmp = none; none = secp256k1_context_clone(none); secp256k1_context_preallocated_destroy(ctx_tmp); + free(prealloc_tmp); + + prealloc_tmp = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN)); CHECK(prealloc_tmp != NULL); + ctx_tmp = sign; sign = secp256k1_context_preallocated_clone(sign, prealloc_tmp); secp256k1_context_destroy(ctx_tmp); + ctx_tmp = sign; sign = secp256k1_context_clone(sign); secp256k1_context_preallocated_destroy(ctx_tmp); + free(prealloc_tmp); + + prealloc_tmp = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_VERIFY)); CHECK(prealloc_tmp != NULL); + ctx_tmp = vrfy; vrfy = secp256k1_context_preallocated_clone(vrfy, prealloc_tmp); secp256k1_context_destroy(ctx_tmp); + ctx_tmp = vrfy; vrfy = secp256k1_context_clone(vrfy); secp256k1_context_preallocated_destroy(ctx_tmp); + free(prealloc_tmp); + + prealloc_tmp = malloc(secp256k1_context_preallocated_size(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY)); CHECK(prealloc_tmp != NULL); + ctx_tmp = both; both = secp256k1_context_preallocated_clone(both, prealloc_tmp); secp256k1_context_destroy(ctx_tmp); + ctx_tmp = both; both = secp256k1_context_clone(both); secp256k1_context_preallocated_destroy(ctx_tmp); + free(prealloc_tmp); + } } /* Verify that the error callback makes it across the clone. */ @@ -229,10 +294,6 @@ void run_context_tests(void) { secp256k1_context_set_illegal_callback(vrfy, NULL, NULL); secp256k1_context_set_illegal_callback(sign, NULL, NULL); - /* This shouldn't leak memory, due to already-set tests. */ - secp256k1_ecmult_gen_context_build(&sign->ecmult_gen_ctx, NULL); - secp256k1_ecmult_context_build(&vrfy->ecmult_ctx, NULL); - /* obtain a working nonce */ do { random_scalar_order_test(&nonce); @@ -247,49 +308,95 @@ void run_context_tests(void) { CHECK(secp256k1_ecdsa_sig_verify(&both->ecmult_ctx, &sigr, &sigs, &pub, &msg)); /* cleanup */ - secp256k1_context_destroy(none); - secp256k1_context_destroy(sign); - secp256k1_context_destroy(vrfy); - secp256k1_context_destroy(both); + if (use_prealloc) { + secp256k1_context_preallocated_destroy(none); + secp256k1_context_preallocated_destroy(sign); + secp256k1_context_preallocated_destroy(vrfy); + secp256k1_context_preallocated_destroy(both); + free(none_prealloc); + free(sign_prealloc); + free(vrfy_prealloc); + free(both_prealloc); + } else { + secp256k1_context_destroy(none); + secp256k1_context_destroy(sign); + secp256k1_context_destroy(vrfy); + secp256k1_context_destroy(both); + } /* Defined as no-op. */ secp256k1_context_destroy(NULL); + secp256k1_context_preallocated_destroy(NULL); + } void run_scratch_tests(void) { + const size_t adj_alloc = ((500 + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT; + int32_t ecount = 0; + size_t checkpoint; + size_t checkpoint_2; secp256k1_context *none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); secp256k1_scratch_space *scratch; + secp256k1_scratch_space local_scratch; /* Test public API */ secp256k1_context_set_illegal_callback(none, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(none, counting_illegal_callback_fn, &ecount); scratch = secp256k1_scratch_space_create(none, 1000); CHECK(scratch != NULL); CHECK(ecount == 0); /* Test internal API */ - CHECK(secp256k1_scratch_max_allocation(scratch, 0) == 1000); - CHECK(secp256k1_scratch_max_allocation(scratch, 1) < 1000); - - /* Allocating 500 bytes with no frame fails */ - CHECK(secp256k1_scratch_alloc(scratch, 500) == NULL); - CHECK(secp256k1_scratch_max_allocation(scratch, 0) == 1000); - - /* ...but pushing a new stack frame does affect the max allocation */ - CHECK(secp256k1_scratch_allocate_frame(scratch, 500, 1 == 1)); - CHECK(secp256k1_scratch_max_allocation(scratch, 1) < 500); /* 500 - ALIGNMENT */ - CHECK(secp256k1_scratch_alloc(scratch, 500) != NULL); - CHECK(secp256k1_scratch_alloc(scratch, 500) == NULL); - - CHECK(secp256k1_scratch_allocate_frame(scratch, 500, 1) == 0); + CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 0) == 1000); + CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 1) == 1000 - (ALIGNMENT - 1)); + CHECK(scratch->alloc_size == 0); + CHECK(scratch->alloc_size % ALIGNMENT == 0); + + /* Allocating 500 bytes succeeds */ + checkpoint = secp256k1_scratch_checkpoint(&none->error_callback, scratch); + CHECK(secp256k1_scratch_alloc(&none->error_callback, scratch, 500) != NULL); + CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 0) == 1000 - adj_alloc); + CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 1) == 1000 - adj_alloc - (ALIGNMENT - 1)); + CHECK(scratch->alloc_size != 0); + CHECK(scratch->alloc_size % ALIGNMENT == 0); + + /* Allocating another 500 bytes fails */ + CHECK(secp256k1_scratch_alloc(&none->error_callback, scratch, 500) == NULL); + CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 0) == 1000 - adj_alloc); + CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 1) == 1000 - adj_alloc - (ALIGNMENT - 1)); + CHECK(scratch->alloc_size != 0); + CHECK(scratch->alloc_size % ALIGNMENT == 0); + + /* ...but it succeeds once we apply the checkpoint to undo it */ + secp256k1_scratch_apply_checkpoint(&none->error_callback, scratch, checkpoint); + CHECK(scratch->alloc_size == 0); + CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 0) == 1000); + CHECK(secp256k1_scratch_alloc(&none->error_callback, scratch, 500) != NULL); + CHECK(scratch->alloc_size != 0); + + /* try to apply a bad checkpoint */ + checkpoint_2 = secp256k1_scratch_checkpoint(&none->error_callback, scratch); + secp256k1_scratch_apply_checkpoint(&none->error_callback, scratch, checkpoint); + CHECK(ecount == 0); + secp256k1_scratch_apply_checkpoint(&none->error_callback, scratch, checkpoint_2); /* checkpoint_2 is after checkpoint */ + CHECK(ecount == 1); + secp256k1_scratch_apply_checkpoint(&none->error_callback, scratch, (size_t) -1); /* this is just wildly invalid */ + CHECK(ecount == 2); - /* ...and this effect is undone by popping the frame */ - secp256k1_scratch_deallocate_frame(scratch); - CHECK(secp256k1_scratch_max_allocation(scratch, 0) == 1000); - CHECK(secp256k1_scratch_alloc(scratch, 500) == NULL); + /* try to use badly initialized scratch space */ + secp256k1_scratch_space_destroy(none, scratch); + memset(&local_scratch, 0, sizeof(local_scratch)); + scratch = &local_scratch; + CHECK(!secp256k1_scratch_max_allocation(&none->error_callback, scratch, 0)); + CHECK(ecount == 3); + CHECK(secp256k1_scratch_alloc(&none->error_callback, scratch, 500) == NULL); + CHECK(ecount == 4); + secp256k1_scratch_space_destroy(none, scratch); + CHECK(ecount == 5); /* cleanup */ - secp256k1_scratch_space_destroy(scratch); + secp256k1_scratch_space_destroy(none, NULL); /* no-op */ secp256k1_context_destroy(none); } @@ -965,11 +1072,31 @@ void scalar_test(void) { } +void run_scalar_set_b32_seckey_tests(void) { + unsigned char b32[32]; + secp256k1_scalar s1; + secp256k1_scalar s2; + + /* Usually set_b32 and set_b32_seckey give the same result */ + random_scalar_order_b32(b32); + secp256k1_scalar_set_b32(&s1, b32, NULL); + CHECK(secp256k1_scalar_set_b32_seckey(&s2, b32) == 1); + CHECK(secp256k1_scalar_eq(&s1, &s2) == 1); + + memset(b32, 0, sizeof(b32)); + CHECK(secp256k1_scalar_set_b32_seckey(&s2, b32) == 0); + memset(b32, 0xFF, sizeof(b32)); + CHECK(secp256k1_scalar_set_b32_seckey(&s2, b32) == 0); +} + void run_scalar_tests(void) { int i; for (i = 0; i < 128 * count; i++) { scalar_test(); } + for (i = 0; i < count; i++) { + run_scalar_set_b32_seckey_tests(); + } { /* (-1)+1 should be zero. */ @@ -985,16 +1112,43 @@ void run_scalar_tests(void) { #ifndef USE_NUM_NONE { - /* A scalar with value of the curve order should be 0. */ + /* Test secp256k1_scalar_set_b32 boundary conditions */ secp256k1_num order; - secp256k1_scalar zero; + secp256k1_scalar scalar; unsigned char bin[32]; + unsigned char bin_tmp[32]; int overflow = 0; + /* 2^256-1 - order */ + static const secp256k1_scalar all_ones_minus_order = SECP256K1_SCALAR_CONST( + 0x00000000UL, 0x00000000UL, 0x00000000UL, 0x00000001UL, + 0x45512319UL, 0x50B75FC4UL, 0x402DA173UL, 0x2FC9BEBEUL + ); + + /* A scalar set to 0s should be 0. */ + memset(bin, 0, 32); + secp256k1_scalar_set_b32(&scalar, bin, &overflow); + CHECK(overflow == 0); + CHECK(secp256k1_scalar_is_zero(&scalar)); + + /* A scalar with value of the curve order should be 0. */ secp256k1_scalar_order_get_num(&order); secp256k1_num_get_bin(bin, 32, &order); - secp256k1_scalar_set_b32(&zero, bin, &overflow); + secp256k1_scalar_set_b32(&scalar, bin, &overflow); + CHECK(overflow == 1); + CHECK(secp256k1_scalar_is_zero(&scalar)); + + /* A scalar with value of the curve order minus one should not overflow. */ + bin[31] -= 1; + secp256k1_scalar_set_b32(&scalar, bin, &overflow); + CHECK(overflow == 0); + secp256k1_scalar_get_b32(bin_tmp, &scalar); + CHECK(memcmp(bin, bin_tmp, 32) == 0); + + /* A scalar set to all 1s should overflow. */ + memset(bin, 0xFF, 32); + secp256k1_scalar_set_b32(&scalar, bin, &overflow); CHECK(overflow == 1); - CHECK(secp256k1_scalar_is_zero(&zero)); + CHECK(secp256k1_scalar_eq(&scalar, &all_ones_minus_order)); } #endif @@ -1709,24 +1863,32 @@ void run_field_misc(void) { /* Test fe conditional move; z is not normalized here. */ q = x; secp256k1_fe_cmov(&x, &z, 0); - VERIFY_CHECK(!x.normalized && x.magnitude == z.magnitude); +#ifdef VERIFY + CHECK(x.normalized && x.magnitude == 1); +#endif secp256k1_fe_cmov(&x, &x, 1); CHECK(fe_memcmp(&x, &z) != 0); CHECK(fe_memcmp(&x, &q) == 0); secp256k1_fe_cmov(&q, &z, 1); - VERIFY_CHECK(!q.normalized && q.magnitude == z.magnitude); +#ifdef VERIFY + CHECK(!q.normalized && q.magnitude == z.magnitude); +#endif CHECK(fe_memcmp(&q, &z) == 0); secp256k1_fe_normalize_var(&x); secp256k1_fe_normalize_var(&z); CHECK(!secp256k1_fe_equal_var(&x, &z)); secp256k1_fe_normalize_var(&q); secp256k1_fe_cmov(&q, &z, (i&1)); - VERIFY_CHECK(q.normalized && q.magnitude == 1); +#ifdef VERIFY + CHECK(q.normalized && q.magnitude == 1); +#endif for (j = 0; j < 6; j++) { secp256k1_fe_negate(&z, &z, j+1); secp256k1_fe_normalize_var(&q); secp256k1_fe_cmov(&q, &z, (j&1)); - VERIFY_CHECK(!q.normalized && q.magnitude == (j+2)); +#ifdef VERIFY + CHECK((q.normalized != (j&1)) && q.magnitude == ((j&1) ? z.magnitude : 1)); +#endif } secp256k1_fe_normalize_var(&z); /* Test storage conversion and conditional moves. */ @@ -2120,7 +2282,7 @@ void test_ge(void) { /* Test batch gej -> ge conversion with many infinities. */ for (i = 0; i < 4 * runs + 1; i++) { random_group_element_test(&ge[i]); - /* randomly set half the points to infinitiy */ + /* randomly set half the points to infinity */ if(secp256k1_fe_is_odd(&ge[i].x)) { secp256k1_ge_set_infinity(&ge[i]); } @@ -2572,14 +2734,13 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e secp256k1_gej r; secp256k1_gej r2; ecmult_multi_data data; - secp256k1_scratch *scratch_empty; data.sc = sc; data.pt = pt; secp256k1_scalar_set_int(&szero, 0); /* No points to multiply */ - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, NULL, ecmult_multi_callback, &data, 0)); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, NULL, ecmult_multi_callback, &data, 0)); /* Check 1- and 2-point multiplies against ecmult */ for (ncount = 0; ncount < count; ncount++) { @@ -2595,36 +2756,31 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e /* only G scalar */ secp256k1_ecmult(&ctx->ecmult_ctx, &r2, &ptgj, &szero, &sc[0]); - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &sc[0], ecmult_multi_callback, &data, 0)); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &sc[0], ecmult_multi_callback, &data, 0)); secp256k1_gej_neg(&r2, &r2); secp256k1_gej_add_var(&r, &r, &r2, NULL); CHECK(secp256k1_gej_is_infinity(&r)); /* 1-point */ secp256k1_ecmult(&ctx->ecmult_ctx, &r2, &ptgj, &sc[0], &szero); - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 1)); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 1)); secp256k1_gej_neg(&r2, &r2); secp256k1_gej_add_var(&r, &r, &r2, NULL); CHECK(secp256k1_gej_is_infinity(&r)); - /* Try to multiply 1 point, but scratch space is empty */ - scratch_empty = secp256k1_scratch_create(&ctx->error_callback, 0); - CHECK(!ecmult_multi(&ctx->ecmult_ctx, scratch_empty, &r, &szero, ecmult_multi_callback, &data, 1)); - secp256k1_scratch_destroy(scratch_empty); - /* Try to multiply 1 point, but callback returns false */ - CHECK(!ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_false_callback, &data, 1)); + CHECK(!ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_false_callback, &data, 1)); /* 2-point */ secp256k1_ecmult(&ctx->ecmult_ctx, &r2, &ptgj, &sc[0], &sc[1]); - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 2)); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 2)); secp256k1_gej_neg(&r2, &r2); secp256k1_gej_add_var(&r, &r, &r2, NULL); CHECK(secp256k1_gej_is_infinity(&r)); /* 2-point with G scalar */ secp256k1_ecmult(&ctx->ecmult_ctx, &r2, &ptgj, &sc[0], &sc[1]); - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &sc[1], ecmult_multi_callback, &data, 1)); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &sc[1], ecmult_multi_callback, &data, 1)); secp256k1_gej_neg(&r2, &r2); secp256k1_gej_add_var(&r, &r, &r2, NULL); CHECK(secp256k1_gej_is_infinity(&r)); @@ -2641,7 +2797,7 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e random_scalar_order(&sc[i]); secp256k1_ge_set_infinity(&pt[i]); } - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, sizes[j])); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, sizes[j])); CHECK(secp256k1_gej_is_infinity(&r)); } @@ -2651,7 +2807,7 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e pt[i] = ptg; secp256k1_scalar_set_int(&sc[i], 0); } - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, sizes[j])); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, sizes[j])); CHECK(secp256k1_gej_is_infinity(&r)); } @@ -2664,7 +2820,7 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e pt[2 * i + 1] = ptg; } - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, sizes[j])); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, sizes[j])); CHECK(secp256k1_gej_is_infinity(&r)); random_scalar_order(&sc[0]); @@ -2677,7 +2833,7 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e secp256k1_ge_neg(&pt[2*i+1], &pt[2*i]); } - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, sizes[j])); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, sizes[j])); CHECK(secp256k1_gej_is_infinity(&r)); } @@ -2692,7 +2848,7 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e secp256k1_scalar_negate(&sc[i], &sc[i]); } - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 32)); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 32)); CHECK(secp256k1_gej_is_infinity(&r)); } @@ -2711,7 +2867,7 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e } secp256k1_ecmult(&ctx->ecmult_ctx, &r2, &r, &sc[0], &szero); - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 20)); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 20)); secp256k1_gej_neg(&r2, &r2); secp256k1_gej_add_var(&r, &r, &r2, NULL); CHECK(secp256k1_gej_is_infinity(&r)); @@ -2734,7 +2890,7 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e secp256k1_gej_set_ge(&p0j, &pt[0]); secp256k1_ecmult(&ctx->ecmult_ctx, &r2, &p0j, &rs, &szero); - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 20)); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 20)); secp256k1_gej_neg(&r2, &r2); secp256k1_gej_add_var(&r, &r, &r2, NULL); CHECK(secp256k1_gej_is_infinity(&r)); @@ -2747,13 +2903,13 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e } secp256k1_scalar_clear(&sc[0]); - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 20)); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 20)); secp256k1_scalar_clear(&sc[1]); secp256k1_scalar_clear(&sc[2]); secp256k1_scalar_clear(&sc[3]); secp256k1_scalar_clear(&sc[4]); - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 6)); - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 5)); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 6)); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &szero, ecmult_multi_callback, &data, 5)); CHECK(secp256k1_gej_is_infinity(&r)); /* Run through s0*(t0*P) + s1*(t1*P) exhaustively for many small values of s0, s1, t0, t1 */ @@ -2798,7 +2954,7 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e secp256k1_scalar_add(&tmp1, &tmp1, &tmp2); secp256k1_ecmult(&ctx->ecmult_ctx, &expected, &ptgj, &tmp1, &szero); - CHECK(ecmult_multi(&ctx->ecmult_ctx, scratch, &actual, &szero, ecmult_multi_callback, &data, 2)); + CHECK(ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &actual, &szero, ecmult_multi_callback, &data, 2)); secp256k1_gej_neg(&expected, &expected); secp256k1_gej_add_var(&actual, &actual, &expected, NULL); CHECK(secp256k1_gej_is_infinity(&actual)); @@ -2809,6 +2965,24 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e } } +void test_ecmult_multi_batch_single(secp256k1_ecmult_multi_func ecmult_multi) { + secp256k1_scalar szero; + secp256k1_scalar sc[32]; + secp256k1_ge pt[32]; + secp256k1_gej r; + ecmult_multi_data data; + secp256k1_scratch *scratch_empty; + + data.sc = sc; + data.pt = pt; + secp256k1_scalar_set_int(&szero, 0); + + /* Try to multiply 1 point, but scratch space is empty.*/ + scratch_empty = secp256k1_scratch_create(&ctx->error_callback, 0); + CHECK(!ecmult_multi(&ctx->error_callback, &ctx->ecmult_ctx, scratch_empty, &r, &szero, ecmult_multi_callback, &data, 1)); + secp256k1_scratch_destroy(&ctx->error_callback, scratch_empty); +} + void test_secp256k1_pippenger_bucket_window_inv(void) { int i; @@ -2839,17 +3013,27 @@ void test_ecmult_multi_pippenger_max_points(void) { int bucket_window = 0; for(; scratch_size < max_size; scratch_size+=256) { + size_t i; + size_t total_alloc; + size_t checkpoint; scratch = secp256k1_scratch_create(&ctx->error_callback, scratch_size); CHECK(scratch != NULL); - n_points_supported = secp256k1_pippenger_max_points(scratch); + checkpoint = secp256k1_scratch_checkpoint(&ctx->error_callback, scratch); + n_points_supported = secp256k1_pippenger_max_points(&ctx->error_callback, scratch); if (n_points_supported == 0) { - secp256k1_scratch_destroy(scratch); + secp256k1_scratch_destroy(&ctx->error_callback, scratch); continue; } bucket_window = secp256k1_pippenger_bucket_window(n_points_supported); - CHECK(secp256k1_scratch_allocate_frame(scratch, secp256k1_pippenger_scratch_size(n_points_supported, bucket_window), PIPPENGER_SCRATCH_OBJECTS)); - secp256k1_scratch_deallocate_frame(scratch); - secp256k1_scratch_destroy(scratch); + /* allocate `total_alloc` bytes over `PIPPENGER_SCRATCH_OBJECTS` many allocations */ + total_alloc = secp256k1_pippenger_scratch_size(n_points_supported, bucket_window); + for (i = 0; i < PIPPENGER_SCRATCH_OBJECTS - 1; i++) { + CHECK(secp256k1_scratch_alloc(&ctx->error_callback, scratch, 1)); + total_alloc--; + } + CHECK(secp256k1_scratch_alloc(&ctx->error_callback, scratch, total_alloc)); + secp256k1_scratch_apply_checkpoint(&ctx->error_callback, scratch, checkpoint); + secp256k1_scratch_destroy(&ctx->error_callback, scratch); } CHECK(bucket_window == PIPPENGER_MAX_BUCKET_WINDOW); } @@ -2932,19 +3116,25 @@ void test_ecmult_multi_batching(void) { } data.sc = sc; data.pt = pt; + secp256k1_gej_neg(&r2, &r2); - /* Test with empty scratch space */ + /* Test with empty scratch space. It should compute the correct result using + * ecmult_mult_simple algorithm which doesn't require a scratch space. */ scratch = secp256k1_scratch_create(&ctx->error_callback, 0); - CHECK(!secp256k1_ecmult_multi_var(&ctx->ecmult_ctx, scratch, &r, &scG, ecmult_multi_callback, &data, 1)); - secp256k1_scratch_destroy(scratch); + CHECK(secp256k1_ecmult_multi_var(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &scG, ecmult_multi_callback, &data, n_points)); + secp256k1_gej_add_var(&r, &r, &r2, NULL); + CHECK(secp256k1_gej_is_infinity(&r)); + secp256k1_scratch_destroy(&ctx->error_callback, scratch); /* Test with space for 1 point in pippenger. That's not enough because - * ecmult_multi selects strauss which requires more memory. */ + * ecmult_multi selects strauss which requires more memory. It should + * therefore select the simple algorithm. */ scratch = secp256k1_scratch_create(&ctx->error_callback, secp256k1_pippenger_scratch_size(1, 1) + PIPPENGER_SCRATCH_OBJECTS*ALIGNMENT); - CHECK(!secp256k1_ecmult_multi_var(&ctx->ecmult_ctx, scratch, &r, &scG, ecmult_multi_callback, &data, 1)); - secp256k1_scratch_destroy(scratch); + CHECK(secp256k1_ecmult_multi_var(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &scG, ecmult_multi_callback, &data, n_points)); + secp256k1_gej_add_var(&r, &r, &r2, NULL); + CHECK(secp256k1_gej_is_infinity(&r)); + secp256k1_scratch_destroy(&ctx->error_callback, scratch); - secp256k1_gej_neg(&r2, &r2); for(i = 1; i <= n_points; i++) { if (i > ECMULT_PIPPENGER_THRESHOLD) { int bucket_window = secp256k1_pippenger_bucket_window(i); @@ -2954,10 +3144,10 @@ void test_ecmult_multi_batching(void) { size_t scratch_size = secp256k1_strauss_scratch_size(i); scratch = secp256k1_scratch_create(&ctx->error_callback, scratch_size + STRAUSS_SCRATCH_OBJECTS*ALIGNMENT); } - CHECK(secp256k1_ecmult_multi_var(&ctx->ecmult_ctx, scratch, &r, &scG, ecmult_multi_callback, &data, n_points)); + CHECK(secp256k1_ecmult_multi_var(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &r, &scG, ecmult_multi_callback, &data, n_points)); secp256k1_gej_add_var(&r, &r, &r2, NULL); CHECK(secp256k1_gej_is_infinity(&r)); - secp256k1_scratch_destroy(scratch); + secp256k1_scratch_destroy(&ctx->error_callback, scratch); } free(sc); free(pt); @@ -2972,13 +3162,15 @@ void run_ecmult_multi_tests(void) { test_ecmult_multi(scratch, secp256k1_ecmult_multi_var); test_ecmult_multi(NULL, secp256k1_ecmult_multi_var); test_ecmult_multi(scratch, secp256k1_ecmult_pippenger_batch_single); + test_ecmult_multi_batch_single(secp256k1_ecmult_pippenger_batch_single); test_ecmult_multi(scratch, secp256k1_ecmult_strauss_batch_single); - secp256k1_scratch_destroy(scratch); + test_ecmult_multi_batch_single(secp256k1_ecmult_strauss_batch_single); + secp256k1_scratch_destroy(&ctx->error_callback, scratch); /* Run test_ecmult_multi with space for exactly one point */ scratch = secp256k1_scratch_create(&ctx->error_callback, secp256k1_strauss_scratch_size(1) + STRAUSS_SCRATCH_OBJECTS*ALIGNMENT); test_ecmult_multi(scratch, secp256k1_ecmult_multi_var); - secp256k1_scratch_destroy(scratch); + secp256k1_scratch_destroy(&ctx->error_callback, scratch); test_ecmult_multi_batch_size_helper(); test_ecmult_multi_batching(); @@ -3050,7 +3242,7 @@ void test_constant_wnaf(const secp256k1_scalar *number, int w) { } bits = 128; #endif - skew = secp256k1_wnaf_const(wnaf, num, w, bits); + skew = secp256k1_wnaf_const(wnaf, &num, w, bits); for (i = WNAF_SIZE_BITS(bits, w); i >= 0; --i) { secp256k1_scalar t; @@ -3786,37 +3978,57 @@ void run_eckey_edge_case_test(void) { pubkey_negone = pubkey; /* Tweak of zero leaves the value unchanged. */ memset(ctmp2, 0, 32); - CHECK(secp256k1_ec_privkey_tweak_add(ctx, ctmp, ctmp2) == 1); + CHECK(secp256k1_ec_seckey_tweak_add(ctx, ctmp, ctmp2) == 1); CHECK(memcmp(orderc, ctmp, 31) == 0 && ctmp[31] == 0x40); memcpy(&pubkey2, &pubkey, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_tweak_add(ctx, &pubkey, ctmp2) == 1); CHECK(memcmp(&pubkey, &pubkey2, sizeof(pubkey)) == 0); /* Multiply tweak of zero zeroizes the output. */ - CHECK(secp256k1_ec_privkey_tweak_mul(ctx, ctmp, ctmp2) == 0); + CHECK(secp256k1_ec_seckey_tweak_mul(ctx, ctmp, ctmp2) == 0); CHECK(memcmp(zeros, ctmp, 32) == 0); CHECK(secp256k1_ec_pubkey_tweak_mul(ctx, &pubkey, ctmp2) == 0); CHECK(memcmp(&pubkey, zeros, sizeof(pubkey)) == 0); memcpy(&pubkey, &pubkey2, sizeof(pubkey)); - /* Overflowing key tweak zeroizes. */ + /* If seckey_tweak_add or seckey_tweak_mul are called with an overflowing + seckey, the seckey is zeroized. */ + memcpy(ctmp, orderc, 32); + memset(ctmp2, 0, 32); + ctmp2[31] = 0x01; + CHECK(secp256k1_ec_seckey_verify(ctx, ctmp2) == 1); + CHECK(secp256k1_ec_seckey_verify(ctx, ctmp) == 0); + CHECK(secp256k1_ec_seckey_tweak_add(ctx, ctmp, ctmp2) == 0); + CHECK(memcmp(zeros, ctmp, 32) == 0); + memcpy(ctmp, orderc, 32); + CHECK(secp256k1_ec_seckey_tweak_mul(ctx, ctmp, ctmp2) == 0); + CHECK(memcmp(zeros, ctmp, 32) == 0); + /* If seckey_tweak_add or seckey_tweak_mul are called with an overflowing + tweak, the seckey is zeroized. */ memcpy(ctmp, orderc, 32); ctmp[31] = 0x40; - CHECK(secp256k1_ec_privkey_tweak_add(ctx, ctmp, orderc) == 0); + CHECK(secp256k1_ec_seckey_tweak_add(ctx, ctmp, orderc) == 0); CHECK(memcmp(zeros, ctmp, 32) == 0); memcpy(ctmp, orderc, 32); ctmp[31] = 0x40; - CHECK(secp256k1_ec_privkey_tweak_mul(ctx, ctmp, orderc) == 0); + CHECK(secp256k1_ec_seckey_tweak_mul(ctx, ctmp, orderc) == 0); CHECK(memcmp(zeros, ctmp, 32) == 0); memcpy(ctmp, orderc, 32); ctmp[31] = 0x40; + /* If pubkey_tweak_add or pubkey_tweak_mul are called with an overflowing + tweak, the pubkey is zeroized. */ CHECK(secp256k1_ec_pubkey_tweak_add(ctx, &pubkey, orderc) == 0); CHECK(memcmp(&pubkey, zeros, sizeof(pubkey)) == 0); memcpy(&pubkey, &pubkey2, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_tweak_mul(ctx, &pubkey, orderc) == 0); CHECK(memcmp(&pubkey, zeros, sizeof(pubkey)) == 0); memcpy(&pubkey, &pubkey2, sizeof(pubkey)); - /* Private key tweaks results in a key of zero. */ + /* If the resulting key in secp256k1_ec_seckey_tweak_add and + * secp256k1_ec_pubkey_tweak_add is 0 the functions fail and in the latter + * case the pubkey is zeroized. */ + memcpy(ctmp, orderc, 32); + ctmp[31] = 0x40; + memset(ctmp2, 0, 32); ctmp2[31] = 1; - CHECK(secp256k1_ec_privkey_tweak_add(ctx, ctmp2, ctmp) == 0); + CHECK(secp256k1_ec_seckey_tweak_add(ctx, ctmp2, ctmp) == 0); CHECK(memcmp(zeros, ctmp2, 32) == 0); ctmp2[31] = 1; CHECK(secp256k1_ec_pubkey_tweak_add(ctx, &pubkey, ctmp2) == 0); @@ -3824,7 +4036,7 @@ void run_eckey_edge_case_test(void) { memcpy(&pubkey, &pubkey2, sizeof(pubkey)); /* Tweak computation wraps and results in a key of 1. */ ctmp2[31] = 2; - CHECK(secp256k1_ec_privkey_tweak_add(ctx, ctmp2, ctmp) == 1); + CHECK(secp256k1_ec_seckey_tweak_add(ctx, ctmp2, ctmp) == 1); CHECK(memcmp(ctmp2, zeros, 31) == 0 && ctmp2[31] == 1); ctmp2[31] = 2; CHECK(secp256k1_ec_pubkey_tweak_add(ctx, &pubkey, ctmp2) == 1); @@ -3872,16 +4084,16 @@ void run_eckey_edge_case_test(void) { CHECK(ecount == 2); ecount = 0; memset(ctmp2, 0, 32); - CHECK(secp256k1_ec_privkey_tweak_add(ctx, NULL, ctmp2) == 0); + CHECK(secp256k1_ec_seckey_tweak_add(ctx, NULL, ctmp2) == 0); CHECK(ecount == 1); - CHECK(secp256k1_ec_privkey_tweak_add(ctx, ctmp, NULL) == 0); + CHECK(secp256k1_ec_seckey_tweak_add(ctx, ctmp, NULL) == 0); CHECK(ecount == 2); ecount = 0; memset(ctmp2, 0, 32); ctmp2[31] = 1; - CHECK(secp256k1_ec_privkey_tweak_mul(ctx, NULL, ctmp2) == 0); + CHECK(secp256k1_ec_seckey_tweak_mul(ctx, NULL, ctmp2) == 0); CHECK(ecount == 1); - CHECK(secp256k1_ec_privkey_tweak_mul(ctx, ctmp, NULL) == 0); + CHECK(secp256k1_ec_seckey_tweak_mul(ctx, ctmp, NULL) == 0); CHECK(ecount == 2); ecount = 0; CHECK(secp256k1_ec_pubkey_create(ctx, NULL, ctmp) == 0); @@ -3954,6 +4166,41 @@ void run_eckey_edge_case_test(void) { secp256k1_context_set_illegal_callback(ctx, NULL, NULL); } +void run_eckey_negate_test(void) { + unsigned char seckey[32]; + unsigned char seckey_tmp[32]; + + random_scalar_order_b32(seckey); + memcpy(seckey_tmp, seckey, 32); + + /* Verify negation changes the key and changes it back */ + CHECK(secp256k1_ec_seckey_negate(ctx, seckey) == 1); + CHECK(memcmp(seckey, seckey_tmp, 32) != 0); + CHECK(secp256k1_ec_seckey_negate(ctx, seckey) == 1); + CHECK(memcmp(seckey, seckey_tmp, 32) == 0); + + /* Check that privkey alias gives same result */ + CHECK(secp256k1_ec_seckey_negate(ctx, seckey) == 1); + CHECK(secp256k1_ec_privkey_negate(ctx, seckey_tmp) == 1); + CHECK(memcmp(seckey, seckey_tmp, 32) == 0); + + /* Negating all 0s fails */ + memset(seckey, 0, 32); + memset(seckey_tmp, 0, 32); + CHECK(secp256k1_ec_seckey_negate(ctx, seckey) == 0); + /* Check that seckey is not modified */ + CHECK(memcmp(seckey, seckey_tmp, 32) == 0); + + /* Negating an overflowing seckey fails and the seckey is zeroed. In this + * test, the seckey has 16 random bytes to ensure that ec_seckey_negate + * doesn't just set seckey to a constant value in case of failure. */ + random_scalar_order_b32(seckey); + memset(seckey, 0xFF, 16); + memset(seckey_tmp, 0, 32); + CHECK(secp256k1_ec_seckey_negate(ctx, seckey) == 0); + CHECK(memcmp(seckey, seckey_tmp, 32) == 0); +} + void random_sign(secp256k1_scalar *sigr, secp256k1_scalar *sigs, const secp256k1_scalar *key, const secp256k1_scalar *msg, int *recid) { secp256k1_scalar nonce; do { @@ -4093,15 +4340,22 @@ void test_ecdsa_end_to_end(void) { if (secp256k1_rand_int(3) == 0) { int ret1; int ret2; + int ret3; unsigned char rnd[32]; + unsigned char privkey_tmp[32]; secp256k1_pubkey pubkey2; secp256k1_rand256_test(rnd); - ret1 = secp256k1_ec_privkey_tweak_add(ctx, privkey, rnd); + memcpy(privkey_tmp, privkey, 32); + ret1 = secp256k1_ec_seckey_tweak_add(ctx, privkey, rnd); ret2 = secp256k1_ec_pubkey_tweak_add(ctx, &pubkey, rnd); + /* Check that privkey alias gives same result */ + ret3 = secp256k1_ec_privkey_tweak_add(ctx, privkey_tmp, rnd); CHECK(ret1 == ret2); + CHECK(ret2 == ret3); if (ret1 == 0) { return; } + CHECK(memcmp(privkey, privkey_tmp, 32) == 0); CHECK(secp256k1_ec_pubkey_create(ctx, &pubkey2, privkey) == 1); CHECK(memcmp(&pubkey, &pubkey2, sizeof(pubkey)) == 0); } @@ -4110,15 +4364,22 @@ void test_ecdsa_end_to_end(void) { if (secp256k1_rand_int(3) == 0) { int ret1; int ret2; + int ret3; unsigned char rnd[32]; + unsigned char privkey_tmp[32]; secp256k1_pubkey pubkey2; secp256k1_rand256_test(rnd); - ret1 = secp256k1_ec_privkey_tweak_mul(ctx, privkey, rnd); + memcpy(privkey_tmp, privkey, 32); + ret1 = secp256k1_ec_seckey_tweak_mul(ctx, privkey, rnd); ret2 = secp256k1_ec_pubkey_tweak_mul(ctx, &pubkey, rnd); + /* Check that privkey alias gives same result */ + ret3 = secp256k1_ec_privkey_tweak_mul(ctx, privkey_tmp, rnd); CHECK(ret1 == ret2); + CHECK(ret2 == ret3); if (ret1 == 0) { return; } + CHECK(memcmp(privkey, privkey_tmp, 32) == 0); CHECK(secp256k1_ec_pubkey_create(ctx, &pubkey2, privkey) == 1); CHECK(memcmp(&pubkey, &pubkey2, sizeof(pubkey)) == 0); } @@ -4315,7 +4576,7 @@ int test_ecdsa_der_parse(const unsigned char *sig, size_t siglen, int certainly_ if (valid_der) { ret |= (!roundtrips_der_lax) << 12; ret |= (len_der != len_der_lax) << 13; - ret |= (memcmp(roundtrip_der_lax, roundtrip_der, len_der) != 0) << 14; + ret |= ((len_der != len_der_lax) || (memcmp(roundtrip_der_lax, roundtrip_der, len_der) != 0)) << 14; } ret |= (roundtrips_der != roundtrips_der_lax) << 15; if (parsed_der) { @@ -4356,7 +4617,7 @@ int test_ecdsa_der_parse(const unsigned char *sig, size_t siglen, int certainly_ ret |= (roundtrips_der != roundtrips_openssl) << 7; if (roundtrips_openssl) { ret |= (len_der != (size_t)len_openssl) << 8; - ret |= (memcmp(roundtrip_der, roundtrip_openssl, len_der) != 0) << 9; + ret |= ((len_der != (size_t)len_openssl) || (memcmp(roundtrip_der, roundtrip_openssl, len_der) != 0)) << 9; } #endif return ret; @@ -5016,9 +5277,188 @@ void run_ecdsa_openssl(void) { # include "modules/recovery/tests_impl.h" #endif +void run_memczero_test(void) { + unsigned char buf1[6] = {1, 2, 3, 4, 5, 6}; + unsigned char buf2[sizeof(buf1)]; + + /* memczero(..., ..., 0) is a noop. */ + memcpy(buf2, buf1, sizeof(buf1)); + memczero(buf1, sizeof(buf1), 0); + CHECK(memcmp(buf1, buf2, sizeof(buf1)) == 0); + + /* memczero(..., ..., 1) zeros the buffer. */ + memset(buf2, 0, sizeof(buf2)); + memczero(buf1, sizeof(buf1) , 1); + CHECK(memcmp(buf1, buf2, sizeof(buf1)) == 0); +} + +void int_cmov_test(void) { + int r = INT_MAX; + int a = 0; + + secp256k1_int_cmov(&r, &a, 0); + CHECK(r == INT_MAX); + + r = 0; a = INT_MAX; + secp256k1_int_cmov(&r, &a, 1); + CHECK(r == INT_MAX); + + a = 0; + secp256k1_int_cmov(&r, &a, 1); + CHECK(r == 0); + + a = 1; + secp256k1_int_cmov(&r, &a, 1); + CHECK(r == 1); + + r = 1; a = 0; + secp256k1_int_cmov(&r, &a, 0); + CHECK(r == 1); + +} + +void fe_cmov_test(void) { + static const secp256k1_fe zero = SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0); + static const secp256k1_fe one = SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 1); + static const secp256k1_fe max = SECP256K1_FE_CONST( + 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, + 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL + ); + secp256k1_fe r = max; + secp256k1_fe a = zero; + + secp256k1_fe_cmov(&r, &a, 0); + CHECK(memcmp(&r, &max, sizeof(r)) == 0); + + r = zero; a = max; + secp256k1_fe_cmov(&r, &a, 1); + CHECK(memcmp(&r, &max, sizeof(r)) == 0); + + a = zero; + secp256k1_fe_cmov(&r, &a, 1); + CHECK(memcmp(&r, &zero, sizeof(r)) == 0); + + a = one; + secp256k1_fe_cmov(&r, &a, 1); + CHECK(memcmp(&r, &one, sizeof(r)) == 0); + + r = one; a = zero; + secp256k1_fe_cmov(&r, &a, 0); + CHECK(memcmp(&r, &one, sizeof(r)) == 0); +} + +void fe_storage_cmov_test(void) { + static const secp256k1_fe_storage zero = SECP256K1_FE_STORAGE_CONST(0, 0, 0, 0, 0, 0, 0, 0); + static const secp256k1_fe_storage one = SECP256K1_FE_STORAGE_CONST(0, 0, 0, 0, 0, 0, 0, 1); + static const secp256k1_fe_storage max = SECP256K1_FE_STORAGE_CONST( + 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, + 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL + ); + secp256k1_fe_storage r = max; + secp256k1_fe_storage a = zero; + + secp256k1_fe_storage_cmov(&r, &a, 0); + CHECK(memcmp(&r, &max, sizeof(r)) == 0); + + r = zero; a = max; + secp256k1_fe_storage_cmov(&r, &a, 1); + CHECK(memcmp(&r, &max, sizeof(r)) == 0); + + a = zero; + secp256k1_fe_storage_cmov(&r, &a, 1); + CHECK(memcmp(&r, &zero, sizeof(r)) == 0); + + a = one; + secp256k1_fe_storage_cmov(&r, &a, 1); + CHECK(memcmp(&r, &one, sizeof(r)) == 0); + + r = one; a = zero; + secp256k1_fe_storage_cmov(&r, &a, 0); + CHECK(memcmp(&r, &one, sizeof(r)) == 0); +} + +void scalar_cmov_test(void) { + static const secp256k1_scalar zero = SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 0); + static const secp256k1_scalar one = SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 1); + static const secp256k1_scalar max = SECP256K1_SCALAR_CONST( + 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, + 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL + ); + secp256k1_scalar r = max; + secp256k1_scalar a = zero; + + secp256k1_scalar_cmov(&r, &a, 0); + CHECK(memcmp(&r, &max, sizeof(r)) == 0); + + r = zero; a = max; + secp256k1_scalar_cmov(&r, &a, 1); + CHECK(memcmp(&r, &max, sizeof(r)) == 0); + + a = zero; + secp256k1_scalar_cmov(&r, &a, 1); + CHECK(memcmp(&r, &zero, sizeof(r)) == 0); + + a = one; + secp256k1_scalar_cmov(&r, &a, 1); + CHECK(memcmp(&r, &one, sizeof(r)) == 0); + + r = one; a = zero; + secp256k1_scalar_cmov(&r, &a, 0); + CHECK(memcmp(&r, &one, sizeof(r)) == 0); +} + +void ge_storage_cmov_test(void) { + static const secp256k1_ge_storage zero = SECP256K1_GE_STORAGE_CONST(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + static const secp256k1_ge_storage one = SECP256K1_GE_STORAGE_CONST(0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1); + static const secp256k1_ge_storage max = SECP256K1_GE_STORAGE_CONST( + 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, + 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, + 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, + 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL + ); + secp256k1_ge_storage r = max; + secp256k1_ge_storage a = zero; + + secp256k1_ge_storage_cmov(&r, &a, 0); + CHECK(memcmp(&r, &max, sizeof(r)) == 0); + + r = zero; a = max; + secp256k1_ge_storage_cmov(&r, &a, 1); + CHECK(memcmp(&r, &max, sizeof(r)) == 0); + + a = zero; + secp256k1_ge_storage_cmov(&r, &a, 1); + CHECK(memcmp(&r, &zero, sizeof(r)) == 0); + + a = one; + secp256k1_ge_storage_cmov(&r, &a, 1); + CHECK(memcmp(&r, &one, sizeof(r)) == 0); + + r = one; a = zero; + secp256k1_ge_storage_cmov(&r, &a, 0); + CHECK(memcmp(&r, &one, sizeof(r)) == 0); +} + +void run_cmov_tests(void) { + int_cmov_test(); + fe_cmov_test(); + fe_storage_cmov_test(); + scalar_cmov_test(); + ge_storage_cmov_test(); +} + int main(int argc, char **argv) { unsigned char seed16[16] = {0}; unsigned char run32[32] = {0}; + + /* Disable buffering for stdout to improve reliability of getting + * diagnostic information. Happens right at the start of main because + * setbuf must be used before any other operation on the stream. */ + setbuf(stdout, NULL); + /* Also disable buffering for stderr because it's not guaranteed that it's + * unbuffered on all systems. */ + setbuf(stderr, NULL); + /* find iteration count */ if (argc > 1) { count = strtol(argv[1], NULL, 0); @@ -5030,7 +5470,7 @@ int main(int argc, char **argv) { const char* ch = argv[2]; while (pos < 16 && ch[0] != 0 && ch[1] != 0) { unsigned short sh; - if (sscanf(ch, "%2hx", &sh)) { + if ((sscanf(ch, "%2hx", &sh)) == 1) { seed16[pos] = sh; } else { break; @@ -5062,7 +5502,8 @@ int main(int argc, char **argv) { printf("random seed = %02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n", seed16[0], seed16[1], seed16[2], seed16[3], seed16[4], seed16[5], seed16[6], seed16[7], seed16[8], seed16[9], seed16[10], seed16[11], seed16[12], seed16[13], seed16[14], seed16[15]); /* initialize */ - run_context_tests(); + run_context_tests(0); + run_context_tests(1); run_scratch_tests(); ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); if (secp256k1_rand_bits(1)) { @@ -5119,6 +5560,9 @@ int main(int argc, char **argv) { /* EC key edge cases */ run_eckey_edge_case_test(); + /* EC key arithmetic test */ + run_eckey_negate_test(); + #ifdef ENABLE_MODULE_ECDH /* ecdh tests */ run_ecdh_tests(); @@ -5139,6 +5583,11 @@ int main(int argc, char **argv) { run_recovery_tests(); #endif + /* util tests */ + run_memczero_test(); + + run_cmov_tests(); + secp256k1_rand256(run32); printf("random run = %02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n", run32[0], run32[1], run32[2], run32[3], run32[4], run32[5], run32[6], run32[7], run32[8], run32[9], run32[10], run32[11], run32[12], run32[13], run32[14], run32[15]); diff --git a/src/secp256k1/src/tests_exhaustive.c b/src/secp256k1/src/tests_exhaustive.c index ab9779b02f..8cca1cef21 100644 --- a/src/secp256k1/src/tests_exhaustive.c +++ b/src/secp256k1/src/tests_exhaustive.c @@ -142,7 +142,7 @@ void test_exhaustive_addition(const secp256k1_ge *group, const secp256k1_gej *gr for (i = 0; i < order; i++) { secp256k1_gej tmp; if (i > 0) { - secp256k1_gej_double_nonzero(&tmp, &groupj[i], NULL); + secp256k1_gej_double_nonzero(&tmp, &groupj[i]); ge_equals_gej(&group[(2 * i) % order], &tmp); } secp256k1_gej_double_var(&tmp, &groupj[i], NULL); @@ -212,14 +212,14 @@ void test_exhaustive_ecmult_multi(const secp256k1_context *ctx, const secp256k1_ data.pt[0] = group[x]; data.pt[1] = group[y]; - secp256k1_ecmult_multi_var(&ctx->ecmult_ctx, scratch, &tmp, &g_sc, ecmult_multi_callback, &data, 2); + secp256k1_ecmult_multi_var(&ctx->error_callback, &ctx->ecmult_ctx, scratch, &tmp, &g_sc, ecmult_multi_callback, &data, 2); ge_equals_gej(&group[(i * x + j * y + k) % order], &tmp); } } } } } - secp256k1_scratch_destroy(scratch); + secp256k1_scratch_destroy(&ctx->error_callback, scratch); } void r_from_k(secp256k1_scalar *r, const secp256k1_ge *group, int k) { diff --git a/src/secp256k1/src/util.h b/src/secp256k1/src/util.h index e1f5b76452..8289e23e0c 100644 --- a/src/secp256k1/src/util.h +++ b/src/secp256k1/src/util.h @@ -14,6 +14,7 @@ #include <stdlib.h> #include <stdint.h> #include <stdio.h> +#include <limits.h> typedef struct { void (*fn)(const char *text, void* data); @@ -68,6 +69,25 @@ static SECP256K1_INLINE void secp256k1_callback_call(const secp256k1_callback * #define VERIFY_SETUP(stmt) #endif +/* Define `VG_UNDEF` and `VG_CHECK` when VALGRIND is defined */ +#if !defined(VG_CHECK) +# if defined(VALGRIND) +# include <valgrind/memcheck.h> +# define VG_UNDEF(x,y) VALGRIND_MAKE_MEM_UNDEFINED((x),(y)) +# define VG_CHECK(x,y) VALGRIND_CHECK_MEM_IS_DEFINED((x),(y)) +# else +# define VG_UNDEF(x,y) +# define VG_CHECK(x,y) +# endif +#endif + +/* Like `VG_CHECK` but on VERIFY only */ +#if defined(VERIFY) +#define VG_CHECK_VERIFY(x,y) VG_CHECK((x), (y)) +#else +#define VG_CHECK_VERIFY(x,y) +#endif + static SECP256K1_INLINE void *checked_malloc(const secp256k1_callback* cb, size_t size) { void *ret = malloc(size); if (ret == NULL) { @@ -84,6 +104,47 @@ static SECP256K1_INLINE void *checked_realloc(const secp256k1_callback* cb, void return ret; } +#if defined(__BIGGEST_ALIGNMENT__) +#define ALIGNMENT __BIGGEST_ALIGNMENT__ +#else +/* Using 16 bytes alignment because common architectures never have alignment + * requirements above 8 for any of the types we care about. In addition we + * leave some room because currently we don't care about a few bytes. */ +#define ALIGNMENT 16 +#endif + +#define ROUND_TO_ALIGN(size) (((size + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT) + +/* Assume there is a contiguous memory object with bounds [base, base + max_size) + * of which the memory range [base, *prealloc_ptr) is already allocated for usage, + * where *prealloc_ptr is an aligned pointer. In that setting, this functions + * reserves the subobject [*prealloc_ptr, *prealloc_ptr + alloc_size) of + * alloc_size bytes by increasing *prealloc_ptr accordingly, taking into account + * alignment requirements. + * + * The function returns an aligned pointer to the newly allocated subobject. + * + * This is useful for manual memory management: if we're simply given a block + * [base, base + max_size), the caller can use this function to allocate memory + * in this block and keep track of the current allocation state with *prealloc_ptr. + * + * It is VERIFY_CHECKed that there is enough space left in the memory object and + * *prealloc_ptr is aligned relative to base. + */ +static SECP256K1_INLINE void *manual_alloc(void** prealloc_ptr, size_t alloc_size, void* base, size_t max_size) { + size_t aligned_alloc_size = ROUND_TO_ALIGN(alloc_size); + void* ret; + VERIFY_CHECK(prealloc_ptr != NULL); + VERIFY_CHECK(*prealloc_ptr != NULL); + VERIFY_CHECK(base != NULL); + VERIFY_CHECK((unsigned char*)*prealloc_ptr >= (unsigned char*)base); + VERIFY_CHECK(((unsigned char*)*prealloc_ptr - (unsigned char*)base) % ALIGNMENT == 0); + VERIFY_CHECK((unsigned char*)*prealloc_ptr - (unsigned char*)base + aligned_alloc_size <= max_size); + ret = *prealloc_ptr; + *((unsigned char**)prealloc_ptr) += aligned_alloc_size; + return ret; +} + /* Macro for restrict, when available and not in a VERIFY build. */ #if defined(SECP256K1_BUILD) && defined(VERIFY) # define SECP256K1_RESTRICT @@ -118,4 +179,33 @@ static SECP256K1_INLINE void *checked_realloc(const secp256k1_callback* cb, void SECP256K1_GNUC_EXT typedef unsigned __int128 uint128_t; #endif +/* Zero memory if flag == 1. Flag must be 0 or 1. Constant time. */ +static SECP256K1_INLINE void memczero(void *s, size_t len, int flag) { + unsigned char *p = (unsigned char *)s; + /* Access flag with a volatile-qualified lvalue. + This prevents clang from figuring out (after inlining) that flag can + take only be 0 or 1, which leads to variable time code. */ + volatile int vflag = flag; + unsigned char mask = -(unsigned char) vflag; + while (len) { + *p &= ~mask; + p++; + len--; + } +} + +/** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. Both *r and *a must be initialized and non-negative.*/ +static SECP256K1_INLINE void secp256k1_int_cmov(int *r, const int *a, int flag) { + unsigned int mask0, mask1, r_masked, a_masked; + /* Casting a negative int to unsigned and back to int is implementation defined behavior */ + VERIFY_CHECK(*r >= 0 && *a >= 0); + + mask0 = (unsigned int)flag + ~0u; + mask1 = ~mask0; + r_masked = ((unsigned int)*r & mask0); + a_masked = ((unsigned int)*a & mask1); + + *r = (int)(r_masked | a_masked); +} + #endif /* SECP256K1_UTIL_H */ diff --git a/src/secp256k1/src/valgrind_ctime_test.c b/src/secp256k1/src/valgrind_ctime_test.c new file mode 100644 index 0000000000..60a82d599e --- /dev/null +++ b/src/secp256k1/src/valgrind_ctime_test.c @@ -0,0 +1,119 @@ +/********************************************************************** + * Copyright (c) 2020 Gregory Maxwell * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#include <valgrind/memcheck.h> +#include "include/secp256k1.h" +#include "util.h" + +#if ENABLE_MODULE_ECDH +# include "include/secp256k1_ecdh.h" +#endif + +#if ENABLE_MODULE_RECOVERY +# include "include/secp256k1_recovery.h" +#endif + +int main(void) { + secp256k1_context* ctx; + secp256k1_ecdsa_signature signature; + secp256k1_pubkey pubkey; + size_t siglen = 74; + size_t outputlen = 33; + int i; + int ret; + unsigned char msg[32]; + unsigned char key[32]; + unsigned char sig[74]; + unsigned char spubkey[33]; +#if ENABLE_MODULE_RECOVERY + secp256k1_ecdsa_recoverable_signature recoverable_signature; + int recid; +#endif + + if (!RUNNING_ON_VALGRIND) { + fprintf(stderr, "This test can only usefully be run inside valgrind.\n"); + fprintf(stderr, "Usage: libtool --mode=execute valgrind ./valgrind_ctime_test\n"); + exit(1); + } + + /** In theory, testing with a single secret input should be sufficient: + * If control flow depended on secrets the tool would generate an error. + */ + for (i = 0; i < 32; i++) { + key[i] = i + 65; + } + for (i = 0; i < 32; i++) { + msg[i] = i + 1; + } + + ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_DECLASSIFY); + + /* Test keygen. */ + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + ret = secp256k1_ec_pubkey_create(ctx, &pubkey, key); + VALGRIND_MAKE_MEM_DEFINED(&pubkey, sizeof(secp256k1_pubkey)); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret); + CHECK(secp256k1_ec_pubkey_serialize(ctx, spubkey, &outputlen, &pubkey, SECP256K1_EC_COMPRESSED) == 1); + + /* Test signing. */ + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + ret = secp256k1_ecdsa_sign(ctx, &signature, msg, key, NULL, NULL); + VALGRIND_MAKE_MEM_DEFINED(&signature, sizeof(secp256k1_ecdsa_signature)); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret); + CHECK(secp256k1_ecdsa_signature_serialize_der(ctx, sig, &siglen, &signature)); + +#if ENABLE_MODULE_ECDH + /* Test ECDH. */ + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + ret = secp256k1_ecdh(ctx, msg, &pubkey, key, NULL, NULL); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret == 1); +#endif + +#if ENABLE_MODULE_RECOVERY + /* Test signing a recoverable signature. */ + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + ret = secp256k1_ecdsa_sign_recoverable(ctx, &recoverable_signature, msg, key, NULL, NULL); + VALGRIND_MAKE_MEM_DEFINED(&recoverable_signature, sizeof(recoverable_signature)); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(ctx, sig, &recid, &recoverable_signature)); + CHECK(recid >= 0 && recid <= 3); +#endif + + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + ret = secp256k1_ec_seckey_verify(ctx, key); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret == 1); + + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + ret = secp256k1_ec_seckey_negate(ctx, key); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret == 1); + + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + VALGRIND_MAKE_MEM_UNDEFINED(msg, 32); + ret = secp256k1_ec_seckey_tweak_add(ctx, key, msg); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret == 1); + + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + VALGRIND_MAKE_MEM_UNDEFINED(msg, 32); + ret = secp256k1_ec_seckey_tweak_mul(ctx, key, msg); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret == 1); + + /* Test context randomisation. Do this last because it leaves the context tainted. */ + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + ret = secp256k1_context_randomize(ctx, key); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret); + + secp256k1_context_destroy(ctx); + return 0; +} diff --git a/src/serialize.h b/src/serialize.h index 71c2cfa164..7a94e704b2 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -9,13 +9,13 @@ #include <compat/endian.h> #include <algorithm> +#include <cstdint> #include <cstring> #include <ios> #include <limits> #include <map> #include <memory> #include <set> -#include <stdint.h> #include <string> #include <string.h> #include <utility> @@ -272,7 +272,7 @@ template<typename Stream> inline void Unserialize(Stream& s, bool& a) { char f=s inline unsigned int GetSizeOfCompactSize(uint64_t nSize) { if (nSize < 253) return sizeof(unsigned char); - else if (nSize <= std::numeric_limits<unsigned short>::max()) return sizeof(unsigned char) + sizeof(unsigned short); + else if (nSize <= std::numeric_limits<uint16_t>::max()) return sizeof(unsigned char) + sizeof(uint16_t); else if (nSize <= std::numeric_limits<unsigned int>::max()) return sizeof(unsigned char) + sizeof(unsigned int); else return sizeof(unsigned char) + sizeof(uint64_t); } @@ -286,7 +286,7 @@ void WriteCompactSize(Stream& os, uint64_t nSize) { ser_writedata8(os, nSize); } - else if (nSize <= std::numeric_limits<unsigned short>::max()) + else if (nSize <= std::numeric_limits<uint16_t>::max()) { ser_writedata8(os, 253); ser_writedata16(os, nSize); diff --git a/src/span.h b/src/span.h index 73b37e35f3..79f13c9203 100644 --- a/src/span.h +++ b/src/span.h @@ -10,20 +10,101 @@ #include <algorithm> #include <assert.h> +#ifdef DEBUG +#define CONSTEXPR_IF_NOT_DEBUG +#define ASSERT_IF_DEBUG(x) assert((x)) +#else +#define CONSTEXPR_IF_NOT_DEBUG constexpr +#define ASSERT_IF_DEBUG(x) +#endif + /** A Span is an object that can refer to a contiguous sequence of objects. * * It implements a subset of C++20's std::span. + * + * Things to be aware of when writing code that deals with Spans: + * + * - Similar to references themselves, Spans are subject to reference lifetime + * issues. The user is responsible for making sure the objects pointed to by + * a Span live as long as the Span is used. For example: + * + * std::vector<int> vec{1,2,3,4}; + * Span<int> sp(vec); + * vec.push_back(5); + * printf("%i\n", sp.front()); // UB! + * + * may exhibit undefined behavior, as increasing the size of a vector may + * invalidate references. + * + * - One particular pitfall is that Spans can be constructed from temporaries, + * but this is unsafe when the Span is stored in a variable, outliving the + * temporary. For example, this will compile, but exhibits undefined behavior: + * + * Span<const int> sp(std::vector<int>{1, 2, 3}); + * printf("%i\n", sp.front()); // UB! + * + * The lifetime of the vector ends when the statement it is created in ends. + * Thus the Span is left with a dangling reference, and using it is undefined. + * + * - Due to Span's automatic creation from range-like objects (arrays, and data + * types that expose a data() and size() member function), functions that + * accept a Span as input parameter can be called with any compatible + * range-like object. For example, this works: +* + * void Foo(Span<const int> arg); + * + * Foo(std::vector<int>{1, 2, 3}); // Works + * + * This is very useful in cases where a function truly does not care about the + * container, and only about having exactly a range of elements. However it + * may also be surprising to see automatic conversions in this case. + * + * When a function accepts a Span with a mutable element type, it will not + * accept temporaries; only variables or other references. For example: + * + * void FooMut(Span<int> arg); + * + * FooMut(std::vector<int>{1, 2, 3}); // Does not compile + * std::vector<int> baz{1, 2, 3}; + * FooMut(baz); // Works + * + * This is similar to how functions that take (non-const) lvalue references + * as input cannot accept temporaries. This does not work either: + * + * void FooVec(std::vector<int>& arg); + * FooVec(std::vector<int>{1, 2, 3}); // Does not compile + * + * The idea is that if a function accepts a mutable reference, a meaningful + * result will be present in that variable after the call. Passing a temporary + * is useless in that context. */ template<typename C> class Span { C* m_data; - std::ptrdiff_t m_size; + std::size_t m_size; public: constexpr Span() noexcept : m_data(nullptr), m_size(0) {} - constexpr Span(C* data, std::ptrdiff_t size) noexcept : m_data(data), m_size(size) {} - constexpr Span(C* data, C* end) noexcept : m_data(data), m_size(end - data) {} + + /** Construct a span from a begin pointer and a size. + * + * This implements a subset of the iterator-based std::span constructor in C++20, + * which is hard to implement without std::address_of. + */ + template <typename T, typename std::enable_if<std::is_convertible<T (*)[], C (*)[]>::value, int>::type = 0> + constexpr Span(T* begin, std::size_t size) noexcept : m_data(begin), m_size(size) {} + + /** Construct a span from a begin and end pointer. + * + * This implements a subset of the iterator-based std::span constructor in C++20, + * which is hard to implement without std::address_of. + */ + template <typename T, typename std::enable_if<std::is_convertible<T (*)[], C (*)[]>::value, int>::type = 0> + CONSTEXPR_IF_NOT_DEBUG Span(T* begin, T* end) noexcept : m_data(begin), m_size(end - begin) + { + ASSERT_IF_DEBUG(end >= begin); + } /** Implicit conversion of spans between compatible types. * @@ -42,18 +123,60 @@ public: /** Default assignment operator. */ Span& operator=(const Span& other) noexcept = default; + /** Construct a Span from an array. This matches the corresponding C++20 std::span constructor. */ + template <int N> + constexpr Span(C (&a)[N]) noexcept : m_data(a), m_size(N) {} + + /** Construct a Span for objects with .data() and .size() (std::string, std::array, std::vector, ...). + * + * This implements a subset of the functionality provided by the C++20 std::span range-based constructor. + * + * To prevent surprises, only Spans for constant value types are supported when passing in temporaries. + * Note that this restriction does not exist when converting arrays or other Spans (see above). + */ + template <typename V, typename std::enable_if<(std::is_const<C>::value || std::is_lvalue_reference<V>::value) && std::is_convertible<typename std::remove_pointer<decltype(std::declval<V&>().data())>::type (*)[], C (*)[]>::value && std::is_convertible<decltype(std::declval<V&>().size()), std::size_t>::value, int>::type = 0> + constexpr Span(V&& v) noexcept : m_data(v.data()), m_size(v.size()) {} + constexpr C* data() const noexcept { return m_data; } constexpr C* begin() const noexcept { return m_data; } constexpr C* end() const noexcept { return m_data + m_size; } - constexpr C& front() const noexcept { return m_data[0]; } - constexpr C& back() const noexcept { return m_data[m_size - 1]; } - constexpr std::ptrdiff_t size() const noexcept { return m_size; } - constexpr C& operator[](std::ptrdiff_t pos) const noexcept { return m_data[pos]; } - - constexpr Span<C> subspan(std::ptrdiff_t offset) const noexcept { return Span<C>(m_data + offset, m_size - offset); } - constexpr Span<C> subspan(std::ptrdiff_t offset, std::ptrdiff_t count) const noexcept { return Span<C>(m_data + offset, count); } - constexpr Span<C> first(std::ptrdiff_t count) const noexcept { return Span<C>(m_data, count); } - constexpr Span<C> last(std::ptrdiff_t count) const noexcept { return Span<C>(m_data + m_size - count, count); } + CONSTEXPR_IF_NOT_DEBUG C& front() const noexcept + { + ASSERT_IF_DEBUG(size() > 0); + return m_data[0]; + } + CONSTEXPR_IF_NOT_DEBUG C& back() const noexcept + { + ASSERT_IF_DEBUG(size() > 0); + return m_data[m_size - 1]; + } + constexpr std::size_t size() const noexcept { return m_size; } + constexpr bool empty() const noexcept { return size() == 0; } + CONSTEXPR_IF_NOT_DEBUG C& operator[](std::size_t pos) const noexcept + { + ASSERT_IF_DEBUG(size() > pos); + return m_data[pos]; + } + CONSTEXPR_IF_NOT_DEBUG Span<C> subspan(std::size_t offset) const noexcept + { + ASSERT_IF_DEBUG(size() >= offset); + return Span<C>(m_data + offset, m_size - offset); + } + CONSTEXPR_IF_NOT_DEBUG Span<C> subspan(std::size_t offset, std::size_t count) const noexcept + { + ASSERT_IF_DEBUG(size() >= offset + count); + return Span<C>(m_data + offset, count); + } + CONSTEXPR_IF_NOT_DEBUG Span<C> first(std::size_t count) const noexcept + { + ASSERT_IF_DEBUG(size() >= count); + return Span<C>(m_data, count); + } + CONSTEXPR_IF_NOT_DEBUG Span<C> last(std::size_t count) const noexcept + { + ASSERT_IF_DEBUG(size() >= count); + return Span<C>(m_data + m_size - count, count); + } friend constexpr bool operator==(const Span& a, const Span& b) noexcept { return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin()); } friend constexpr bool operator!=(const Span& a, const Span& b) noexcept { return !(a == b); } @@ -65,26 +188,20 @@ public: template <typename O> friend class Span; }; -/** Create a span to a container exposing data() and size(). - * - * This correctly deals with constness: the returned Span's element type will be - * whatever data() returns a pointer to. If either the passed container is const, - * or its element type is const, the resulting span will have a const element type. - * - * std::span will have a constructor that implements this functionality directly. - */ -template<typename A, int N> -constexpr Span<A> MakeSpan(A (&a)[N]) { return Span<A>(a, N); } - -template<typename V> -constexpr Span<typename std::remove_pointer<decltype(std::declval<V>().data())>::type> MakeSpan(V& v) { return Span<typename std::remove_pointer<decltype(std::declval<V>().data())>::type>(v.data(), v.size()); } +// MakeSpan helps constructing a Span of the right type automatically. +/** MakeSpan for arrays: */ +template <typename A, int N> Span<A> constexpr MakeSpan(A (&a)[N]) { return Span<A>(a, N); } +/** MakeSpan for temporaries / rvalue references, only supporting const output. */ +template <typename V> constexpr auto MakeSpan(V&& v) -> typename std::enable_if<!std::is_lvalue_reference<V>::value, Span<const typename std::remove_pointer<decltype(v.data())>::type>>::type { return std::forward<V>(v); } +/** MakeSpan for (lvalue) references, supporting mutable output. */ +template <typename V> constexpr auto MakeSpan(V& v) -> Span<typename std::remove_pointer<decltype(v.data())>::type> { return v; } /** Pop the last element off a span, and return a reference to that element. */ template <typename T> T& SpanPopBack(Span<T>& span) { size_t size = span.size(); - assert(size > 0); + ASSERT_IF_DEBUG(size > 0); T& back = span[size - 1]; span = Span<T>(span.data(), size - 1); return back; diff --git a/src/streams.h b/src/streams.h index e1d1b0eab2..6ce8065da8 100644 --- a/src/streams.h +++ b/src/streams.h @@ -814,18 +814,6 @@ public: return true; } - bool Seek(uint64_t nPos) { - long nLongPos = nPos; - if (nPos != (uint64_t)nLongPos) - return false; - if (fseek(src, nLongPos, SEEK_SET)) - return false; - nLongPos = ftell(src); - nSrcPos = nLongPos; - nReadPos = nLongPos; - return true; - } - //! prevent reading beyond a certain position //! no argument removes the limit bool SetLimit(uint64_t nPos = std::numeric_limits<uint64_t>::max()) { diff --git a/src/sync.cpp b/src/sync.cpp index 9abdedbed4..10f0483189 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -60,7 +60,7 @@ struct CLockLocation { std::string ToString() const { return strprintf( - "%s %s:%s%s (in thread %s)", + "'%s' in %s:%s%s (in thread '%s')", mutexName, sourceFile, sourceLine, (fTry ? " (TRY)" : ""), m_thread_name); } @@ -105,7 +105,7 @@ static void potential_deadlock_detected(const LockPair& mismatch, const LockStac { LogPrintf("POTENTIAL DEADLOCK DETECTED\n"); LogPrintf("Previous lock order was:\n"); - for (const LockStackItem& i : s2) { + for (const LockStackItem& i : s1) { if (i.first == mismatch.first) { LogPrintf(" (1)"); /* Continued */ } @@ -114,21 +114,25 @@ static void potential_deadlock_detected(const LockPair& mismatch, const LockStac } LogPrintf(" %s\n", i.second.ToString()); } + + std::string mutex_a, mutex_b; LogPrintf("Current lock order is:\n"); - for (const LockStackItem& i : s1) { + for (const LockStackItem& i : s2) { if (i.first == mismatch.first) { LogPrintf(" (1)"); /* Continued */ + mutex_a = i.second.Name(); } if (i.first == mismatch.second) { LogPrintf(" (2)"); /* Continued */ + mutex_b = i.second.Name(); } LogPrintf(" %s\n", i.second.ToString()); } if (g_debug_lockorder_abort) { - tfm::format(std::cerr, "Assertion failed: detected inconsistent lock order at %s:%i, details in debug log.\n", __FILE__, __LINE__); + tfm::format(std::cerr, "Assertion failed: detected inconsistent lock order for %s, details in debug log.\n", s2.back().second.ToString()); abort(); } - throw std::logic_error("potential deadlock detected"); + throw std::logic_error(strprintf("potential deadlock detected: %s -> %s -> %s", mutex_b, mutex_a, mutex_b)); } static void push_lock(void* c, const CLockLocation& locklocation) diff --git a/src/sync.h b/src/sync.h index 60e5a87aec..77327d8bfe 100644 --- a/src/sync.h +++ b/src/sync.h @@ -103,6 +103,12 @@ public: } using UniqueLock = std::unique_lock<PARENT>; +#ifdef __clang__ + //! For negative capabilities in the Clang Thread Safety Analysis. + //! A negative requirement uses the EXCLUSIVE_LOCKS_REQUIRED attribute, in conjunction + //! with the ! operator, to indicate that a mutex should not be held. + const AnnotatedMixin& operator!() const { return *this; } +#endif // __clang__ }; /** diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 7dff2e6e86..00c4bdc14e 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -94,7 +94,7 @@ bool BuildChainTestingSetup::BuildChain(const CBlockIndex* pindex, CBlockHeader header = block->GetBlockHeader(); BlockValidationState state; - if (!EnsureChainman(m_node).ProcessNewBlockHeaders({header}, state, Params(), &pindex)) { + if (!Assert(m_node.chainman)->ProcessNewBlockHeaders({header}, state, Params(), &pindex)) { return false; } } @@ -171,7 +171,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) uint256 chainA_last_header = last_header; for (size_t i = 0; i < 2; i++) { const auto& block = chainA[i]; - BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, nullptr)); } for (size_t i = 0; i < 2; i++) { const auto& block = chainA[i]; @@ -189,7 +189,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) uint256 chainB_last_header = last_header; for (size_t i = 0; i < 3; i++) { const auto& block = chainB[i]; - BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, nullptr)); } for (size_t i = 0; i < 3; i++) { const auto& block = chainB[i]; @@ -220,7 +220,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) // Reorg back to chain A. for (size_t i = 2; i < 4; i++) { const auto& block = chainA[i]; - BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, nullptr)); } // Check that chain A and B blocks can be retrieved. diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 60196c36a5..173ec5e3d9 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -538,7 +538,7 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) CDataStream tmp(SER_DISK, CLIENT_VERSION); uint64_t x = 3000000000ULL; tmp << VARINT(x); - BOOST_CHECK_EQUAL(HexStr(tmp.begin(), tmp.end()), "8a95c0bb00"); + BOOST_CHECK_EQUAL(HexStr(tmp), "8a95c0bb00"); CDataStream ss5(ParseHex("00008a95c0bb00"), SER_DISK, CLIENT_VERSION); try { Coin cc5; diff --git a/src/test/data/script_tests.json b/src/test/data/script_tests.json index c01ef307b7..724789bbf9 100644 --- a/src/test/data/script_tests.json +++ b/src/test/data/script_tests.json @@ -678,7 +678,7 @@ ["0 0x02 0x0000 0", "CHECKMULTISIGVERIFY 1", "", "OK"], ["While not really correctly DER encoded, the empty signature is allowed by"], -["STRICTENC to provide a compact way to provide a delibrately invalid signature."], +["STRICTENC to provide a compact way to provide a deliberately invalid signature."], ["0", "0x21 0x02865c40293a680cb9c020e7b1e106d8c1916d3cef99aa431a56d253e69256dac0 CHECKSIG NOT", "STRICTENC", "OK"], ["0 0", "1 0x21 0x02865c40293a680cb9c020e7b1e106d8c1916d3cef99aa431a56d253e69256dac0 1 CHECKMULTISIG NOT", "STRICTENC", "OK"], diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 348b170536..b1a635d9da 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -4,10 +4,12 @@ // Unit tests for denial-of-service detection/prevention code +#include <arith_uint256.h> #include <banman.h> #include <chainparams.h> #include <net.h> #include <net_processing.h> +#include <pubkey.h> #include <script/sign.h> #include <script/signingprovider.h> #include <script/standard.h> @@ -98,11 +100,11 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) // Test starts here { - LOCK2(cs_main, dummyNode1.cs_sendProcessing); + LOCK(dummyNode1.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); // should result in getheaders } { - LOCK2(cs_main, dummyNode1.cs_vSend); + LOCK(dummyNode1.cs_vSend); BOOST_CHECK(dummyNode1.vSendMsg.size() > 0); dummyNode1.vSendMsg.clear(); } @@ -111,17 +113,17 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) // Wait 21 minutes SetMockTime(nStartTime+21*60); { - LOCK2(cs_main, dummyNode1.cs_sendProcessing); + LOCK(dummyNode1.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); // should result in getheaders } { - LOCK2(cs_main, dummyNode1.cs_vSend); + LOCK(dummyNode1.cs_vSend); BOOST_CHECK(dummyNode1.vSendMsg.size() > 0); } // Wait 3 more minutes SetMockTime(nStartTime+24*60); { - LOCK2(cs_main, dummyNode1.cs_sendProcessing); + LOCK(dummyNode1.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); // should result in disconnect } BOOST_CHECK(dummyNode1.fDisconnect == true); @@ -217,7 +219,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) connman->ClearNodes(); } -BOOST_AUTO_TEST_CASE(DoS_banning) +BOOST_AUTO_TEST_CASE(peer_discouragement) { auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); @@ -232,14 +234,14 @@ BOOST_AUTO_TEST_CASE(DoS_banning) dummyNode1.fSuccessfullyConnected = true; { LOCK(cs_main); - Misbehaving(dummyNode1.GetId(), 100); // Should get banned + Misbehaving(dummyNode1.GetId(), DISCOURAGEMENT_THRESHOLD); // Should be discouraged } { - LOCK2(cs_main, dummyNode1.cs_sendProcessing); + LOCK(dummyNode1.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); } - BOOST_CHECK(banman->IsBanned(addr1)); - BOOST_CHECK(!banman->IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned + BOOST_CHECK(banman->IsDiscouraged(addr1)); + BOOST_CHECK(!banman->IsDiscouraged(ip(0xa0b0c001|0x0000ff00))); // Different IP, not discouraged CAddress addr2(ip(0xa0b0c002), NODE_NONE); CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, CAddress(), "", true); @@ -249,76 +251,30 @@ BOOST_AUTO_TEST_CASE(DoS_banning) dummyNode2.fSuccessfullyConnected = true; { LOCK(cs_main); - Misbehaving(dummyNode2.GetId(), 50); + Misbehaving(dummyNode2.GetId(), DISCOURAGEMENT_THRESHOLD - 1); } { - LOCK2(cs_main, dummyNode2.cs_sendProcessing); + LOCK(dummyNode2.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode2)); } - BOOST_CHECK(!banman->IsBanned(addr2)); // 2 not banned yet... - BOOST_CHECK(banman->IsBanned(addr1)); // ... but 1 still should be + BOOST_CHECK(!banman->IsDiscouraged(addr2)); // 2 not discouraged yet... + BOOST_CHECK(banman->IsDiscouraged(addr1)); // ... but 1 still should be { LOCK(cs_main); - Misbehaving(dummyNode2.GetId(), 50); + Misbehaving(dummyNode2.GetId(), 1); // 2 reaches discouragement threshold } { - LOCK2(cs_main, dummyNode2.cs_sendProcessing); + LOCK(dummyNode2.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode2)); } - BOOST_CHECK(banman->IsBanned(addr2)); + BOOST_CHECK(banman->IsDiscouraged(addr1)); // Expect both 1 and 2 + BOOST_CHECK(banman->IsDiscouraged(addr2)); // to be discouraged now bool dummy; peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); peerLogic->FinalizeNode(dummyNode2.GetId(), dummy); } -BOOST_AUTO_TEST_CASE(DoS_banscore) -{ - auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); - auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); - - banman->ClearBanned(); - gArgs.ForceSetArg("-banscore", "111"); // because 11 is my favorite number - CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, 1, CAddress(), "", true); - dummyNode1.SetSendVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(&dummyNode1); - dummyNode1.nVersion = 1; - dummyNode1.fSuccessfullyConnected = true; - { - LOCK(cs_main); - Misbehaving(dummyNode1.GetId(), 100); - } - { - LOCK2(cs_main, dummyNode1.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); - } - BOOST_CHECK(!banman->IsBanned(addr1)); - { - LOCK(cs_main); - Misbehaving(dummyNode1.GetId(), 10); - } - { - LOCK2(cs_main, dummyNode1.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); - } - BOOST_CHECK(!banman->IsBanned(addr1)); - { - LOCK(cs_main); - Misbehaving(dummyNode1.GetId(), 1); - } - { - LOCK2(cs_main, dummyNode1.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); - } - BOOST_CHECK(banman->IsBanned(addr1)); - gArgs.ForceSetArg("-banscore", ToString(DEFAULT_BANSCORE_THRESHOLD)); - - bool dummy; - peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); -} - BOOST_AUTO_TEST_CASE(DoS_bantime) { auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); @@ -338,19 +294,13 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) { LOCK(cs_main); - Misbehaving(dummyNode.GetId(), 100); + Misbehaving(dummyNode.GetId(), DISCOURAGEMENT_THRESHOLD); } { - LOCK2(cs_main, dummyNode.cs_sendProcessing); + LOCK(dummyNode.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode)); } - BOOST_CHECK(banman->IsBanned(addr)); - - SetMockTime(nStartTime+60*60); - BOOST_CHECK(banman->IsBanned(addr)); - - SetMockTime(nStartTime+60*60*24+1); - BOOST_CHECK(!banman->IsBanned(addr)); + BOOST_CHECK(banman->IsDiscouraged(addr)); bool dummy; peerLogic->FinalizeNode(dummyNode.GetId(), dummy); @@ -366,10 +316,26 @@ static CTransactionRef RandomOrphan() return it->second.tx; } +static void MakeNewKeyWithFastRandomContext(CKey& key) +{ + std::vector<unsigned char> keydata; + keydata = g_insecure_rand_ctx.randbytes(32); + key.Set(keydata.data(), keydata.data() + keydata.size(), /*fCompressedIn*/ true); + assert(key.IsValid()); +} + BOOST_AUTO_TEST_CASE(DoS_mapOrphans) { + // This test had non-deterministic coverage due to + // randomly selected seeds. + // This seed is chosen so that all branches of the function + // ecdsa_signature_parse_der_lax are executed during this test. + // Specifically branches that run only when an ECDSA + // signature's R and S values have leading zeros. + g_insecure_rand_ctx = FastRandomContext(ArithToUint256(arith_uint256(33))); + CKey key; - key.MakeNewKey(true); + MakeNewKeyWithFastRandomContext(key); FillableSigningProvider keystore; BOOST_CHECK(keystore.AddKey(key)); diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 5f9a78ceb2..20132d5782 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -135,7 +135,7 @@ void DoCheck(const std::string& prv, const std::string& pub, int flags, const st // When the descriptor is hardened, evaluate with access to the private keys inside. const FlatSigningProvider& key_provider = (flags & HARDENED) ? keys_priv : keys_pub; - // Evaluate the descriptor selected by `t` in poisition `i`. + // Evaluate the descriptor selected by `t` in position `i`. FlatSigningProvider script_provider, script_provider_cached; std::vector<CScript> spks, spks_cached; DescriptorCache desc_cache; @@ -216,7 +216,7 @@ void DoCheck(const std::string& prv, const std::string& pub, int flags, const st // For each of the produced scripts, verify solvability, and when possible, try to sign a transaction spending it. for (size_t n = 0; n < spks.size(); ++n) { - BOOST_CHECK_EQUAL(ref[n], HexStr(spks[n].begin(), spks[n].end())); + BOOST_CHECK_EQUAL(ref[n], HexStr(spks[n])); BOOST_CHECK_EQUAL(IsSolvable(Merge(key_provider, script_provider), spks[n]), (flags & UNSOLVABLE) == 0); if (flags & SIGNABLE) { diff --git a/src/test/fuzz/addrdb.cpp b/src/test/fuzz/addrdb.cpp index 524cea83fe..16b1cb755a 100644 --- a/src/test/fuzz/addrdb.cpp +++ b/src/test/fuzz/addrdb.cpp @@ -17,19 +17,13 @@ void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + // The point of this code is to exercise all CBanEntry constructors. const CBanEntry ban_entry = [&] { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 3)) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 2)) { case 0: return CBanEntry{fuzzed_data_provider.ConsumeIntegral<int64_t>()}; break; - case 1: - return CBanEntry{fuzzed_data_provider.ConsumeIntegral<int64_t>(), fuzzed_data_provider.PickValueInArray<BanReason>({ - BanReason::BanReasonUnknown, - BanReason::BanReasonNodeMisbehaving, - BanReason::BanReasonManuallyAdded, - })}; - break; - case 2: { + case 1: { const std::optional<CBanEntry> ban_entry = ConsumeDeserializable<CBanEntry>(fuzzed_data_provider); if (ban_entry) { return *ban_entry; @@ -39,5 +33,5 @@ void test_one_input(const std::vector<uint8_t>& buffer) } return CBanEntry{}; }(); - assert(!ban_entry.banReasonToString().empty()); + (void)ban_entry; // currently unused } diff --git a/src/test/fuzz/autofile.cpp b/src/test/fuzz/autofile.cpp new file mode 100644 index 0000000000..7ea0bdd2a7 --- /dev/null +++ b/src/test/fuzz/autofile.cpp @@ -0,0 +1,72 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <optional.h> +#include <streams.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <array> +#include <cstdint> +#include <iostream> +#include <optional> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); + CAutoFile auto_file = fuzzed_auto_file_provider.open(); + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 5)) { + case 0: { + std::array<uint8_t, 4096> arr{}; + try { + auto_file.read((char*)arr.data(), fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + } catch (const std::ios_base::failure&) { + } + break; + } + case 1: { + const std::array<uint8_t, 4096> arr{}; + try { + auto_file.write((const char*)arr.data(), fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + } catch (const std::ios_base::failure&) { + } + break; + } + case 2: { + try { + auto_file.ignore(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + } catch (const std::ios_base::failure&) { + } + break; + } + case 3: { + auto_file.fclose(); + break; + } + case 4: { + ReadFromStream(fuzzed_data_provider, auto_file); + break; + } + case 5: { + WriteToStream(fuzzed_data_provider, auto_file); + break; + } + } + } + (void)auto_file.Get(); + (void)auto_file.GetType(); + (void)auto_file.GetVersion(); + (void)auto_file.IsNull(); + if (fuzzed_data_provider.ConsumeBool()) { + FILE* f = auto_file.release(); + if (f != nullptr) { + fclose(f); + } + } +} diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp new file mode 100644 index 0000000000..fc4a1d9261 --- /dev/null +++ b/src/test/fuzz/banman.cpp @@ -0,0 +1,88 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <banman.h> +#include <fs.h> +#include <netaddress.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <util/system.h> + +#include <cstdint> +#include <limits> +#include <string> +#include <vector> + +namespace { +int64_t ConsumeBanTimeOffset(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + // Avoid signed integer overflow by capping to int32_t max: + // banman.cpp:137:73: runtime error: signed integer overflow: 1591700817 + 9223372036854775807 cannot be represented in type 'long' + return fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(std::numeric_limits<int64_t>::min(), std::numeric_limits<int32_t>::max()); +} +} // namespace + +void initialize() +{ + InitializeFuzzingContext(); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + const fs::path banlist_file = GetDataDir() / "fuzzed_banlist.dat"; + fs::remove(banlist_file); + { + BanMan ban_man{banlist_file, nullptr, ConsumeBanTimeOffset(fuzzed_data_provider)}; + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 11)) { + case 0: { + ban_man.Ban(ConsumeNetAddr(fuzzed_data_provider), + ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); + break; + } + case 1: { + ban_man.Ban(ConsumeSubNet(fuzzed_data_provider), + ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); + break; + } + case 2: { + ban_man.ClearBanned(); + break; + } + case 4: { + ban_man.IsBanned(ConsumeNetAddr(fuzzed_data_provider)); + break; + } + case 5: { + ban_man.IsBanned(ConsumeSubNet(fuzzed_data_provider)); + break; + } + case 6: { + ban_man.Unban(ConsumeNetAddr(fuzzed_data_provider)); + break; + } + case 7: { + ban_man.Unban(ConsumeSubNet(fuzzed_data_provider)); + break; + } + case 8: { + banmap_t banmap; + ban_man.GetBanned(banmap); + break; + } + case 9: { + ban_man.DumpBanlist(); + break; + } + case 11: { + ban_man.Discourage(ConsumeNetAddr(fuzzed_data_provider)); + break; + } + } + } + } + fs::remove(banlist_file); +} diff --git a/src/test/fuzz/buffered_file.cpp b/src/test/fuzz/buffered_file.cpp new file mode 100644 index 0000000000..e575640be5 --- /dev/null +++ b/src/test/fuzz/buffered_file.cpp @@ -0,0 +1,74 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <optional.h> +#include <streams.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <array> +#include <cstdint> +#include <iostream> +#include <optional> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + FuzzedFileProvider fuzzed_file_provider = ConsumeFile(fuzzed_data_provider); + std::optional<CBufferedFile> opt_buffered_file; + FILE* fuzzed_file = fuzzed_file_provider.open(); + try { + opt_buffered_file.emplace(fuzzed_file, fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096), fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096), fuzzed_data_provider.ConsumeIntegral<int>(), fuzzed_data_provider.ConsumeIntegral<int>()); + } catch (const std::ios_base::failure&) { + if (fuzzed_file != nullptr) { + fclose(fuzzed_file); + } + } + if (opt_buffered_file && fuzzed_file != nullptr) { + bool setpos_fail = false; + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 4)) { + case 0: { + std::array<uint8_t, 4096> arr{}; + try { + opt_buffered_file->read((char*)arr.data(), fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + } catch (const std::ios_base::failure&) { + } + break; + } + case 1: { + opt_buffered_file->SetLimit(fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096)); + break; + } + case 2: { + if (!opt_buffered_file->SetPos(fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096))) { + setpos_fail = true; + } + break; + } + case 3: { + if (setpos_fail) { + // Calling FindByte(...) after a failed SetPos(...) call may result in an infinite loop. + break; + } + try { + opt_buffered_file->FindByte(fuzzed_data_provider.ConsumeIntegral<char>()); + } catch (const std::ios_base::failure&) { + } + break; + } + case 4: { + ReadFromStream(fuzzed_data_provider, *opt_buffered_file); + break; + } + } + } + opt_buffered_file->GetPos(); + opt_buffered_file->GetType(); + opt_buffered_file->GetVersion(); + } +} diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 52dd62a145..c186bef7ae 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -278,7 +278,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) CCoinsStats stats; bool expected_code_path = false; try { - (void)GetUTXOStats(&coins_view_cache, stats); + (void)GetUTXOStats(&coins_view_cache, stats, CoinStatsHashType::HASH_SERIALIZED); } catch (const std::logic_error&) { expected_code_path = true; } diff --git a/src/test/fuzz/crypto.cpp b/src/test/fuzz/crypto.cpp new file mode 100644 index 0000000000..595cdf9abb --- /dev/null +++ b/src/test/fuzz/crypto.cpp @@ -0,0 +1,124 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/hmac_sha256.h> +#include <crypto/hmac_sha512.h> +#include <crypto/ripemd160.h> +#include <crypto/sha1.h> +#include <crypto/sha256.h> +#include <crypto/sha512.h> +#include <hash.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + std::vector<uint8_t> data = ConsumeRandomLengthByteVector(fuzzed_data_provider); + if (data.empty()) { + data.resize(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 4096), fuzzed_data_provider.ConsumeIntegral<uint8_t>()); + } + + CHash160 hash160; + CHash256 hash256; + CHMAC_SHA256 hmac_sha256{data.data(), data.size()}; + CHMAC_SHA512 hmac_sha512{data.data(), data.size()}; + CRIPEMD160 ripemd160; + CSHA1 sha1; + CSHA256 sha256; + CSHA512 sha512; + CSipHasher sip_hasher{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>()}; + + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 2)) { + case 0: { + if (fuzzed_data_provider.ConsumeBool()) { + data = ConsumeRandomLengthByteVector(fuzzed_data_provider); + if (data.empty()) { + data.resize(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 4096), fuzzed_data_provider.ConsumeIntegral<uint8_t>()); + } + } + + (void)hash160.Write(data.data(), data.size()); + (void)hash256.Write(data.data(), data.size()); + (void)hmac_sha256.Write(data.data(), data.size()); + (void)hmac_sha512.Write(data.data(), data.size()); + (void)ripemd160.Write(data.data(), data.size()); + (void)sha1.Write(data.data(), data.size()); + (void)sha256.Write(data.data(), data.size()); + (void)sha512.Write(data.data(), data.size()); + (void)sip_hasher.Write(data.data(), data.size()); + + (void)Hash(data.begin(), data.end()); + (void)Hash160(data); + (void)Hash160(data.begin(), data.end()); + (void)sha512.Size(); + break; + } + case 1: { + (void)hash160.Reset(); + (void)hash256.Reset(); + (void)ripemd160.Reset(); + (void)sha1.Reset(); + (void)sha256.Reset(); + (void)sha512.Reset(); + break; + } + case 2: { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 8)) { + case 0: { + data.resize(CHash160::OUTPUT_SIZE); + hash160.Finalize(data.data()); + break; + } + case 1: { + data.resize(CHash256::OUTPUT_SIZE); + hash256.Finalize(data.data()); + break; + } + case 2: { + data.resize(CHMAC_SHA256::OUTPUT_SIZE); + hmac_sha256.Finalize(data.data()); + break; + } + case 3: { + data.resize(CHMAC_SHA512::OUTPUT_SIZE); + hmac_sha512.Finalize(data.data()); + break; + } + case 4: { + data.resize(CRIPEMD160::OUTPUT_SIZE); + ripemd160.Finalize(data.data()); + break; + } + case 5: { + data.resize(CSHA1::OUTPUT_SIZE); + sha1.Finalize(data.data()); + break; + } + case 6: { + data.resize(CSHA256::OUTPUT_SIZE); + sha256.Finalize(data.data()); + break; + } + case 7: { + data.resize(CSHA512::OUTPUT_SIZE); + sha512.Finalize(data.data()); + break; + } + case 8: { + data.resize(1); + data[0] = sip_hasher.Finalize() % 256; + break; + } + } + break; + } + } + } +} diff --git a/src/test/fuzz/crypto_aes256.cpp b/src/test/fuzz/crypto_aes256.cpp new file mode 100644 index 0000000000..ae14073c96 --- /dev/null +++ b/src/test/fuzz/crypto_aes256.cpp @@ -0,0 +1,30 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/aes.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cassert> +#include <cstdint> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + const std::vector<uint8_t> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, AES256_KEYSIZE); + + AES256Encrypt encrypt{key.data()}; + AES256Decrypt decrypt{key.data()}; + + while (fuzzed_data_provider.ConsumeBool()) { + const std::vector<uint8_t> plaintext = ConsumeFixedLengthByteVector(fuzzed_data_provider, AES_BLOCKSIZE); + std::vector<uint8_t> ciphertext(AES_BLOCKSIZE); + encrypt.Encrypt(ciphertext.data(), plaintext.data()); + std::vector<uint8_t> decrypted_plaintext(AES_BLOCKSIZE); + decrypt.Decrypt(decrypted_plaintext.data(), ciphertext.data()); + assert(decrypted_plaintext == plaintext); + } +} diff --git a/src/test/fuzz/crypto_aes256cbc.cpp b/src/test/fuzz/crypto_aes256cbc.cpp new file mode 100644 index 0000000000..52983c7e79 --- /dev/null +++ b/src/test/fuzz/crypto_aes256cbc.cpp @@ -0,0 +1,34 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/aes.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cassert> +#include <cstdint> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + const std::vector<uint8_t> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, AES256_KEYSIZE); + const std::vector<uint8_t> iv = ConsumeFixedLengthByteVector(fuzzed_data_provider, AES_BLOCKSIZE); + const bool pad = fuzzed_data_provider.ConsumeBool(); + + AES256CBCEncrypt encrypt{key.data(), iv.data(), pad}; + AES256CBCDecrypt decrypt{key.data(), iv.data(), pad}; + + while (fuzzed_data_provider.ConsumeBool()) { + const std::vector<uint8_t> plaintext = ConsumeRandomLengthByteVector(fuzzed_data_provider); + std::vector<uint8_t> ciphertext(plaintext.size() + AES_BLOCKSIZE); + const int encrypt_ret = encrypt.Encrypt(plaintext.data(), plaintext.size(), ciphertext.data()); + ciphertext.resize(encrypt_ret); + std::vector<uint8_t> decrypted_plaintext(ciphertext.size()); + const int decrypt_ret = decrypt.Decrypt(ciphertext.data(), ciphertext.size(), decrypted_plaintext.data()); + decrypted_plaintext.resize(decrypt_ret); + assert(decrypted_plaintext == plaintext || (!pad && plaintext.size() % AES_BLOCKSIZE != 0 && encrypt_ret == 0 && decrypt_ret == 0)); + } +} diff --git a/src/test/fuzz/crypto_chacha20.cpp b/src/test/fuzz/crypto_chacha20.cpp new file mode 100644 index 0000000000..b7438d312d --- /dev/null +++ b/src/test/fuzz/crypto_chacha20.cpp @@ -0,0 +1,50 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/chacha20.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + ChaCha20 chacha20; + if (fuzzed_data_provider.ConsumeBool()) { + const std::vector<unsigned char> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, fuzzed_data_provider.ConsumeIntegralInRange<size_t>(16, 32)); + chacha20 = ChaCha20{key.data(), key.size()}; + } + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 4)) { + case 0: { + const std::vector<unsigned char> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, fuzzed_data_provider.ConsumeIntegralInRange<size_t>(16, 32)); + chacha20.SetKey(key.data(), key.size()); + break; + } + case 1: { + chacha20.SetIV(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + break; + } + case 2: { + chacha20.Seek(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + break; + } + case 3: { + std::vector<uint8_t> output(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + chacha20.Keystream(output.data(), output.size()); + break; + } + case 4: { + std::vector<uint8_t> output(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + const std::vector<uint8_t> input = ConsumeFixedLengthByteVector(fuzzed_data_provider, output.size()); + chacha20.Crypt(input.data(), output.data(), input.size()); + break; + } + } + } +} diff --git a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp new file mode 100644 index 0000000000..48e4263f27 --- /dev/null +++ b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp @@ -0,0 +1,72 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/chacha_poly_aead.h> +#include <crypto/poly1305.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cassert> +#include <cstdint> +#include <limits> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + const std::vector<uint8_t> k1 = ConsumeFixedLengthByteVector(fuzzed_data_provider, CHACHA20_POLY1305_AEAD_KEY_LEN); + const std::vector<uint8_t> k2 = ConsumeFixedLengthByteVector(fuzzed_data_provider, CHACHA20_POLY1305_AEAD_KEY_LEN); + + ChaCha20Poly1305AEAD aead(k1.data(), k1.size(), k2.data(), k2.size()); + uint64_t seqnr_payload = 0; + uint64_t seqnr_aad = 0; + int aad_pos = 0; + size_t buffer_size = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096); + std::vector<uint8_t> in(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); + std::vector<uint8_t> out(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); + bool is_encrypt = fuzzed_data_provider.ConsumeBool(); + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 6)) { + case 0: { + buffer_size = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(64, 4096); + in = std::vector<uint8_t>(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); + out = std::vector<uint8_t>(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); + break; + } + case 1: { + (void)aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffer_size, is_encrypt); + break; + } + case 2: { + uint32_t len = 0; + const bool ok = aead.GetLength(&len, seqnr_aad, aad_pos, in.data()); + assert(ok); + break; + } + case 3: { + seqnr_payload += 1; + aad_pos += CHACHA20_POLY1305_AEAD_AAD_LEN; + if (aad_pos + CHACHA20_POLY1305_AEAD_AAD_LEN > CHACHA20_ROUND_OUTPUT) { + aad_pos = 0; + seqnr_aad += 1; + } + break; + } + case 4: { + seqnr_payload = fuzzed_data_provider.ConsumeIntegral<int>(); + break; + } + case 5: { + seqnr_aad = fuzzed_data_provider.ConsumeIntegral<int>(); + break; + } + case 6: { + is_encrypt = fuzzed_data_provider.ConsumeBool(); + break; + } + } + } +} diff --git a/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp b/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp new file mode 100644 index 0000000000..e0a4e90c10 --- /dev/null +++ b/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp @@ -0,0 +1,25 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/hkdf_sha256_32.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + const std::vector<uint8_t> initial_key_material = ConsumeRandomLengthByteVector(fuzzed_data_provider); + + CHKDF_HMAC_SHA256_L32 hkdf_hmac_sha256_l32(initial_key_material.data(), initial_key_material.size(), fuzzed_data_provider.ConsumeRandomLengthString(1024)); + while (fuzzed_data_provider.ConsumeBool()) { + std::vector<uint8_t> out(32); + hkdf_hmac_sha256_l32.Expand32(fuzzed_data_provider.ConsumeRandomLengthString(128), out.data()); + } +} diff --git a/src/test/fuzz/crypto_poly1305.cpp b/src/test/fuzz/crypto_poly1305.cpp new file mode 100644 index 0000000000..5681e6a693 --- /dev/null +++ b/src/test/fuzz/crypto_poly1305.cpp @@ -0,0 +1,22 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/poly1305.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + const std::vector<uint8_t> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, POLY1305_KEYLEN); + const std::vector<uint8_t> in = ConsumeRandomLengthByteVector(fuzzed_data_provider); + + std::vector<uint8_t> tag_out(POLY1305_TAGLEN); + poly1305_auth(tag_out.data(), in.data(), in.size(), key.data()); +} diff --git a/src/test/fuzz/decode_tx.cpp b/src/test/fuzz/decode_tx.cpp index 09c4ff05df..0d89d4228a 100644 --- a/src/test/fuzz/decode_tx.cpp +++ b/src/test/fuzz/decode_tx.cpp @@ -14,7 +14,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) { - const std::string tx_hex = HexStr(std::string{buffer.begin(), buffer.end()}); + const std::string tx_hex = HexStr(buffer); CMutableTransaction mtx; const bool result_none = DecodeHexTx(mtx, tx_hex, false, false); const bool result_try_witness = DecodeHexTx(mtx, tx_hex, false, true); diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index 82e1d55c0b..1e1807d734 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -12,7 +12,16 @@ const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; -#if defined(__AFL_COMPILER) +// Decide if main(...) should be provided: +// * AFL needs main(...) regardless of platform. +// * macOS handles __attribute__((weak)) main(...) poorly when linking +// against libFuzzer. See https://github.com/bitcoin/bitcoin/pull/18008 +// for details. +#if defined(__AFL_COMPILER) || !defined(MAC_OSX) +#define PROVIDE_MAIN_FUNCTION +#endif + +#if defined(PROVIDE_MAIN_FUNCTION) static bool read_stdin(std::vector<uint8_t>& data) { uint8_t buffer[1024]; @@ -44,9 +53,8 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) return 0; } -// Generally, the fuzzer will provide main(), except for AFL -#if defined(__AFL_COMPILER) -int main(int argc, char** argv) +#if defined(PROVIDE_MAIN_FUNCTION) +__attribute__((weak)) int main(int argc, char** argv) { initialize(); #ifdef __AFL_INIT diff --git a/src/test/fuzz/http_request.cpp b/src/test/fuzz/http_request.cpp index ebf89749e9..36d44e361f 100644 --- a/src/test/fuzz/http_request.cpp +++ b/src/test/fuzz/http_request.cpp @@ -7,6 +7,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/strencodings.h> #include <event2/buffer.h> #include <event2/event.h> @@ -48,7 +49,14 @@ void test_one_input(const std::vector<uint8_t>& buffer) assert(evbuf != nullptr); const std::vector<uint8_t> http_buffer = ConsumeRandomLengthByteVector(fuzzed_data_provider, 4096); evbuffer_add(evbuf, http_buffer.data(), http_buffer.size()); - if (evhttp_parse_firstline_(evreq, evbuf) != 1 || evhttp_parse_headers_(evreq, evbuf) != 1) { + // Avoid constructing requests that will be interpreted by libevent as PROXY requests to avoid triggering + // a nullptr dereference. The dereference (req->evcon->http_server) takes place in evhttp_parse_request_line + // and is a consequence of our hacky but necessary use of the internal function evhttp_parse_firstline_ in + // this fuzzing harness. The workaround is not aesthetically pleasing, but it successfully avoids the troublesome + // code path. " http:// HTTP/1.1\n" was a crashing input prior to this workaround. + const std::string http_buffer_str = ToLower({http_buffer.begin(), http_buffer.end()}); + if (http_buffer_str.find(" http://") != std::string::npos || http_buffer_str.find(" https://") != std::string::npos || + evhttp_parse_firstline_(evreq, evbuf) != 1 || evhttp_parse_headers_(evreq, evbuf) != 1) { evbuffer_free(evbuf); evhttp_request_free(evreq); return; diff --git a/src/test/fuzz/key.cpp b/src/test/fuzz/key.cpp index 1919a5f881..c746374c61 100644 --- a/src/test/fuzz/key.cpp +++ b/src/test/fuzz/key.cpp @@ -108,7 +108,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) assert(pubkey.IsCompressed()); assert(pubkey.IsValid()); assert(pubkey.IsFullyValid()); - assert(HexToPubKey(HexStr(pubkey.begin(), pubkey.end())) == pubkey); + assert(HexToPubKey(HexStr(pubkey)) == pubkey); assert(GetAllDestinationsForKey(pubkey).size() == 3); } @@ -157,25 +157,25 @@ void test_one_input(const std::vector<uint8_t>& buffer) assert(ok_add_key_pubkey); assert(fillable_signing_provider_pub.HaveKey(pubkey.GetID())); - txnouttype which_type_tx_pubkey; + TxoutType which_type_tx_pubkey; const bool is_standard_tx_pubkey = IsStandard(tx_pubkey_script, which_type_tx_pubkey); assert(is_standard_tx_pubkey); - assert(which_type_tx_pubkey == txnouttype::TX_PUBKEY); + assert(which_type_tx_pubkey == TxoutType::PUBKEY); - txnouttype which_type_tx_multisig; + TxoutType which_type_tx_multisig; const bool is_standard_tx_multisig = IsStandard(tx_multisig_script, which_type_tx_multisig); assert(is_standard_tx_multisig); - assert(which_type_tx_multisig == txnouttype::TX_MULTISIG); + assert(which_type_tx_multisig == TxoutType::MULTISIG); std::vector<std::vector<unsigned char>> v_solutions_ret_tx_pubkey; - const txnouttype outtype_tx_pubkey = Solver(tx_pubkey_script, v_solutions_ret_tx_pubkey); - assert(outtype_tx_pubkey == txnouttype::TX_PUBKEY); + const TxoutType outtype_tx_pubkey = Solver(tx_pubkey_script, v_solutions_ret_tx_pubkey); + assert(outtype_tx_pubkey == TxoutType::PUBKEY); assert(v_solutions_ret_tx_pubkey.size() == 1); assert(v_solutions_ret_tx_pubkey[0].size() == 33); std::vector<std::vector<unsigned char>> v_solutions_ret_tx_multisig; - const txnouttype outtype_tx_multisig = Solver(tx_multisig_script, v_solutions_ret_tx_multisig); - assert(outtype_tx_multisig == txnouttype::TX_MULTISIG); + const TxoutType outtype_tx_multisig = Solver(tx_multisig_script, v_solutions_ret_tx_multisig); + assert(outtype_tx_multisig == TxoutType::MULTISIG); assert(v_solutions_ret_tx_multisig.size() == 3); assert(v_solutions_ret_tx_multisig[0].size() == 1); assert(v_solutions_ret_tx_multisig[1].size() == 33); diff --git a/src/test/fuzz/load_external_block_file.cpp b/src/test/fuzz/load_external_block_file.cpp new file mode 100644 index 0000000000..d9de9d9866 --- /dev/null +++ b/src/test/fuzz/load_external_block_file.cpp @@ -0,0 +1,31 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <flatfile.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <validation.h> + +#include <cstdint> +#include <vector> + +void initialize() +{ + InitializeFuzzingContext(); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + FuzzedFileProvider fuzzed_file_provider = ConsumeFile(fuzzed_data_provider); + FILE* fuzzed_block_file = fuzzed_file_provider.open(); + if (fuzzed_block_file == nullptr) { + return; + } + FlatFilePos flat_file_pos; + LoadExternalBlockFile(Params(), fuzzed_block_file, fuzzed_data_provider.ConsumeBool() ? &flat_file_pos : nullptr); +} diff --git a/src/test/fuzz/netaddress.cpp b/src/test/fuzz/netaddress.cpp index d8d53566c7..2901c704f6 100644 --- a/src/test/fuzz/netaddress.cpp +++ b/src/test/fuzz/netaddress.cpp @@ -5,41 +5,13 @@ #include <netaddress.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> #include <cassert> #include <cstdint> #include <netinet/in.h> #include <vector> -namespace { -CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); - if (network == Network::NET_IPV4) { - const in_addr v4_addr = { - .s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; - return CNetAddr{v4_addr}; - } else if (network == Network::NET_IPV6) { - if (fuzzed_data_provider.remaining_bytes() < 16) { - return CNetAddr{}; - } - in6_addr v6_addr = {}; - memcpy(v6_addr.s6_addr, fuzzed_data_provider.ConsumeBytes<uint8_t>(16).data(), 16); - return CNetAddr{v6_addr, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; - } else if (network == Network::NET_INTERNAL) { - CNetAddr net_addr; - net_addr.SetInternal(fuzzed_data_provider.ConsumeBytesAsString(32)); - return net_addr; - } else if (network == Network::NET_ONION) { - CNetAddr net_addr; - net_addr.SetSpecial(fuzzed_data_provider.ConsumeBytesAsString(32)); - return net_addr; - } else { - assert(false); - } -} -}; // namespace - void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/p2p_transport_deserializer.cpp b/src/test/fuzz/p2p_transport_deserializer.cpp index 57393fed45..6fba2bfaba 100644 --- a/src/test/fuzz/p2p_transport_deserializer.cpp +++ b/src/test/fuzz/p2p_transport_deserializer.cpp @@ -30,7 +30,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) pch += handled; n_bytes -= handled; if (deserializer.Complete()) { - const int64_t m_time = std::numeric_limits<int64_t>::max(); + const std::chrono::microseconds m_time{std::numeric_limits<int64_t>::max()}; const CNetMessage msg = deserializer.GetMessage(Params().MessageStart(), m_time); assert(msg.m_command.size() <= CMessageHeader::COMMAND_SIZE); assert(msg.m_raw_message_size <= buffer.size()); diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 1cbf9b347f..6c94a47f3c 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -14,6 +14,11 @@ #include <string> #include <vector> +void initialize() +{ + InitializeFuzzingContext(); +} + void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); @@ -66,4 +71,10 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)block_policy_estimator.estimateSmartFee(fuzzed_data_provider.ConsumeIntegral<int>(), fuzzed_data_provider.ConsumeBool() ? &fee_calculation : nullptr, fuzzed_data_provider.ConsumeBool()); (void)block_policy_estimator.HighestTargetTracked(fuzzed_data_provider.PickValueInArray({FeeEstimateHorizon::SHORT_HALFLIFE, FeeEstimateHorizon::MED_HALFLIFE, FeeEstimateHorizon::LONG_HALFLIFE})); } + { + FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); + CAutoFile fuzzed_auto_file = fuzzed_auto_file_provider.open(); + block_policy_estimator.Write(fuzzed_auto_file); + block_policy_estimator.Read(fuzzed_auto_file); + } } diff --git a/src/test/fuzz/policy_estimator_io.cpp b/src/test/fuzz/policy_estimator_io.cpp new file mode 100644 index 0000000000..0edcf201c7 --- /dev/null +++ b/src/test/fuzz/policy_estimator_io.cpp @@ -0,0 +1,28 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <policy/fees.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <vector> + +void initialize() +{ + InitializeFuzzingContext(); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); + CAutoFile fuzzed_auto_file = fuzzed_auto_file_provider.open(); + // Re-using block_policy_estimator across runs to avoid costly creation of CBlockPolicyEstimator object. + static CBlockPolicyEstimator block_policy_estimator; + if (block_policy_estimator.Read(fuzzed_auto_file)) { + block_policy_estimator.Write(fuzzed_auto_file); + } +} diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 211a84b5f2..9e40d5cd55 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -30,7 +30,17 @@ #include <string> #include <vector> -bool ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, ChainstateManager& chainman, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc); +void ProcessMessage( + CNode& pfrom, + const std::string& msg_type, + CDataStream& vRecv, + const std::chrono::microseconds time_received, + const CChainParams& chainparams, + ChainstateManager& chainman, + CTxMemPool& mempool, + CConnman& connman, + BanMan* banman, + const std::atomic<bool>& interruptMsgProc); namespace { @@ -77,7 +87,10 @@ void test_one_input(const std::vector<uint8_t>& buffer) connman.AddTestNode(p2p_node); g_setup->m_node.peer_logic->InitializeNode(&p2p_node); try { - (void)ProcessMessage(p2p_node, random_message_type, random_bytes_data_stream, GetTimeMillis(), Params(), *g_setup->m_node.chainman, *g_setup->m_node.mempool, g_setup->m_node.connman.get(), g_setup->m_node.banman.get(), std::atomic<bool>{false}); + ProcessMessage(p2p_node, random_message_type, random_bytes_data_stream, GetTime<std::chrono::microseconds>(), + Params(), *g_setup->m_node.chainman, *g_setup->m_node.mempool, + *g_setup->m_node.connman, g_setup->m_node.banman.get(), + std::atomic<bool>{false}); } catch (const std::ios_base::failure&) { } SyncWithValidationInterfaceQueue(); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index ad6c115a49..91ebf9fb1b 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -62,7 +62,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; CSerializedNetMsg net_msg; - net_msg.command = random_message_type; + net_msg.m_type = random_message_type; net_msg.data = ConsumeRandomLengthByteVector(fuzzed_data_provider); CNode& random_node = *peers.at(fuzzed_data_provider.ConsumeIntegralInRange<int>(0, peers.size() - 1)); diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp index 64328fb66e..908e2b16f2 100644 --- a/src/test/fuzz/psbt.cpp +++ b/src/test/fuzz/psbt.cpp @@ -39,7 +39,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) } (void)psbt.IsNull(); - (void)psbt.IsSane(); Optional<CMutableTransaction> tx = psbt.tx; if (tx) { @@ -50,7 +49,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) for (const PSBTInput& input : psbt.inputs) { (void)PSBTInputSigned(input); (void)input.IsNull(); - (void)input.IsSane(); } for (const PSBTOutput& output : psbt.outputs) { diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index 933cf9049d..85aac6ac7a 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -48,7 +48,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) if (CompressScript(script, compressed)) { const unsigned int size = compressed[0]; compressed.erase(compressed.begin()); - assert(size >= 0 && size <= 5); + assert(size <= 5); CScript decompressed_script; const bool ok = DecompressScript(decompressed_script, size, compressed); assert(ok); @@ -58,7 +58,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) CTxDestination address; (void)ExtractDestination(script, address); - txnouttype type_ret; + TxoutType type_ret; std::vector<CTxDestination> addresses; int required_ret; (void)ExtractDestinations(script, type_ret, addresses, required_ret); @@ -72,7 +72,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)IsSolvable(signing_provider, script); - txnouttype which_type; + TxoutType which_type; (void)IsStandard(script, which_type); (void)RecursiveDynamicUsage(script); diff --git a/src/test/fuzz/scriptnum_ops.cpp b/src/test/fuzz/scriptnum_ops.cpp index f4e079fb89..68c1ae58ca 100644 --- a/src/test/fuzz/scriptnum_ops.cpp +++ b/src/test/fuzz/scriptnum_ops.cpp @@ -33,7 +33,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) case 0: { const int64_t i = fuzzed_data_provider.ConsumeIntegral<int64_t>(); assert((script_num == i) != (script_num != i)); - assert((script_num <= i) != script_num > i); + assert((script_num <= i) != (script_num > i)); assert((script_num >= i) != (script_num < i)); // Avoid signed integer overflow: // script/script.h:264:93: runtime error: signed integer overflow: -2261405121394637306 + -9223372036854775802 cannot be represented in type 'long' diff --git a/src/test/fuzz/signature_checker.cpp b/src/test/fuzz/signature_checker.cpp index 4a8c7a63af..3aaeb66649 100644 --- a/src/test/fuzz/signature_checker.cpp +++ b/src/test/fuzz/signature_checker.cpp @@ -28,17 +28,17 @@ public: { } - virtual bool CheckSig(const std::vector<unsigned char>& scriptSig, const std::vector<unsigned char>& vchPubKey, const CScript& scriptCode, SigVersion sigversion) const + bool CheckSig(const std::vector<unsigned char>& scriptSig, const std::vector<unsigned char>& vchPubKey, const CScript& scriptCode, SigVersion sigversion) const override { return m_fuzzed_data_provider.ConsumeBool(); } - virtual bool CheckLockTime(const CScriptNum& nLockTime) const + bool CheckLockTime(const CScriptNum& nLockTime) const override { return m_fuzzed_data_provider.ConsumeBool(); } - virtual bool CheckSequence(const CScriptNum& nSequence) const + bool CheckSequence(const CScriptNum& nSequence) const override { return m_fuzzed_data_provider.ConsumeBool(); } diff --git a/src/test/fuzz/span.cpp b/src/test/fuzz/span.cpp index 4aea530ef2..f6b6e8f6f0 100644 --- a/src/test/fuzz/span.cpp +++ b/src/test/fuzz/span.cpp @@ -18,7 +18,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); std::string str = fuzzed_data_provider.ConsumeBytesAsString(32); - const Span<const char> span = MakeSpan(str); + const Span<const char> span{str}; (void)span.data(); (void)span.begin(); (void)span.end(); @@ -32,7 +32,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) } std::string another_str = fuzzed_data_provider.ConsumeBytesAsString(32); - const Span<const char> another_span = MakeSpan(another_str); + const Span<const char> another_span{another_str}; assert((span <= another_span) != (span > another_span)); assert((span == another_span) != (span != another_span)); assert((span >= another_span) != (span < another_span)); diff --git a/src/test/fuzz/spanparsing.cpp b/src/test/fuzz/spanparsing.cpp index 8e5e7dad11..e5bf5dd608 100644 --- a/src/test/fuzz/spanparsing.cpp +++ b/src/test/fuzz/spanparsing.cpp @@ -12,7 +12,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) const size_t query_size = fuzzed_data_provider.ConsumeIntegral<size_t>(); const std::string query = fuzzed_data_provider.ConsumeBytesAsString(std::min<size_t>(query_size, 1024 * 1024)); const std::string span_str = fuzzed_data_provider.ConsumeRemainingBytesAsString(); - const Span<const char> const_span = MakeSpan(span_str); + const Span<const char> const_span{span_str}; Span<const char> mut_span = const_span; (void)spanparsing::Const(query, mut_span); diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 1c1b2cd254..9f9552edb9 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -8,8 +8,11 @@ #include <amount.h> #include <arith_uint256.h> #include <attributes.h> +#include <chainparamsbase.h> #include <coins.h> #include <consensus/consensus.h> +#include <netaddress.h> +#include <netbase.h> #include <primitives/transaction.h> #include <script/script.h> #include <script/standard.h> @@ -17,12 +20,14 @@ #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/util/setup_common.h> #include <txmempool.h> #include <uint256.h> #include <version.h> #include <algorithm> #include <cstdint> +#include <cstdio> #include <optional> #include <string> #include <vector> @@ -228,4 +233,241 @@ NODISCARD inline std::vector<uint8_t> ConsumeFixedLengthByteVector(FuzzedDataPro return result; } +CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); + CNetAddr net_addr; + if (network == Network::NET_IPV4) { + const in_addr v4_addr = { + .s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + net_addr = CNetAddr{v4_addr}; + } else if (network == Network::NET_IPV6) { + if (fuzzed_data_provider.remaining_bytes() >= 16) { + in6_addr v6_addr = {}; + memcpy(v6_addr.s6_addr, fuzzed_data_provider.ConsumeBytes<uint8_t>(16).data(), 16); + net_addr = CNetAddr{v6_addr, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + } + } else if (network == Network::NET_INTERNAL) { + net_addr.SetInternal(fuzzed_data_provider.ConsumeBytesAsString(32)); + } else if (network == Network::NET_ONION) { + net_addr.SetSpecial(fuzzed_data_provider.ConsumeBytesAsString(32)); + } + return net_addr; +} + +CSubNet ConsumeSubNet(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<int32_t>()}; +} + +void InitializeFuzzingContext(const std::string& chain_name = CBaseChainParams::REGTEST) +{ + static const BasicTestingSetup basic_testing_setup{chain_name, {"-nodebuglogfile"}}; +} + +class FuzzedFileProvider +{ + FuzzedDataProvider& m_fuzzed_data_provider; + int64_t m_offset = 0; + +public: + FuzzedFileProvider(FuzzedDataProvider& fuzzed_data_provider) : m_fuzzed_data_provider{fuzzed_data_provider} + { + } + + FILE* open() + { + if (m_fuzzed_data_provider.ConsumeBool()) { + return nullptr; + } + std::string mode; + switch (m_fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 5)) { + case 0: { + mode = "r"; + break; + } + case 1: { + mode = "r+"; + break; + } + case 2: { + mode = "w"; + break; + } + case 3: { + mode = "w+"; + break; + } + case 4: { + mode = "a"; + break; + } + case 5: { + mode = "a+"; + break; + } + } +#ifdef _GNU_SOURCE + const cookie_io_functions_t io_hooks = { + FuzzedFileProvider::read, + FuzzedFileProvider::write, + FuzzedFileProvider::seek, + FuzzedFileProvider::close, + }; + return fopencookie(this, mode.c_str(), io_hooks); +#else + (void)mode; + return nullptr; +#endif + } + + static ssize_t read(void* cookie, char* buf, size_t size) + { + FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; + if (buf == nullptr || size == 0 || fuzzed_file->m_fuzzed_data_provider.ConsumeBool()) { + return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; + } + const std::vector<uint8_t> random_bytes = fuzzed_file->m_fuzzed_data_provider.ConsumeBytes<uint8_t>(size); + if (random_bytes.empty()) { + return 0; + } + std::memcpy(buf, random_bytes.data(), random_bytes.size()); + if (AdditionOverflow(fuzzed_file->m_offset, (int64_t)random_bytes.size())) { + return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; + } + fuzzed_file->m_offset += random_bytes.size(); + return random_bytes.size(); + } + + static ssize_t write(void* cookie, const char* buf, size_t size) + { + FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; + const ssize_t n = fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<ssize_t>(0, size); + if (AdditionOverflow(fuzzed_file->m_offset, (int64_t)n)) { + return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; + } + fuzzed_file->m_offset += n; + return n; + } + + static int seek(void* cookie, int64_t* offset, int whence) + { + assert(whence == SEEK_SET || whence == SEEK_CUR); // SEEK_END not implemented yet. + FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; + int64_t new_offset = 0; + if (whence == SEEK_SET) { + new_offset = *offset; + } else if (whence == SEEK_CUR) { + if (AdditionOverflow(fuzzed_file->m_offset, *offset)) { + return -1; + } + new_offset = fuzzed_file->m_offset + *offset; + } + if (new_offset < 0) { + return -1; + } + fuzzed_file->m_offset = new_offset; + *offset = new_offset; + return fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int>(-1, 0); + } + + static int close(void* cookie) + { + FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; + return fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int>(-1, 0); + } +}; + +NODISCARD inline FuzzedFileProvider ConsumeFile(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + return {fuzzed_data_provider}; +} + +class FuzzedAutoFileProvider +{ + FuzzedDataProvider& m_fuzzed_data_provider; + FuzzedFileProvider m_fuzzed_file_provider; + +public: + FuzzedAutoFileProvider(FuzzedDataProvider& fuzzed_data_provider) : m_fuzzed_data_provider{fuzzed_data_provider}, m_fuzzed_file_provider{fuzzed_data_provider} + { + } + + CAutoFile open() + { + return {m_fuzzed_file_provider.open(), m_fuzzed_data_provider.ConsumeIntegral<int>(), m_fuzzed_data_provider.ConsumeIntegral<int>()}; + } +}; + +NODISCARD inline FuzzedAutoFileProvider ConsumeAutoFile(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + return {fuzzed_data_provider}; +} + +#define WRITE_TO_STREAM_CASE(id, type, consume) \ + case id: { \ + type o = consume; \ + stream << o; \ + break; \ + } +template <typename Stream> +void WriteToStream(FuzzedDataProvider& fuzzed_data_provider, Stream& stream) noexcept +{ + while (fuzzed_data_provider.ConsumeBool()) { + try { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 13)) { + WRITE_TO_STREAM_CASE(0, bool, fuzzed_data_provider.ConsumeBool()) + WRITE_TO_STREAM_CASE(1, char, fuzzed_data_provider.ConsumeIntegral<char>()) + WRITE_TO_STREAM_CASE(2, int8_t, fuzzed_data_provider.ConsumeIntegral<int8_t>()) + WRITE_TO_STREAM_CASE(3, uint8_t, fuzzed_data_provider.ConsumeIntegral<uint8_t>()) + WRITE_TO_STREAM_CASE(4, int16_t, fuzzed_data_provider.ConsumeIntegral<int16_t>()) + WRITE_TO_STREAM_CASE(5, uint16_t, fuzzed_data_provider.ConsumeIntegral<uint16_t>()) + WRITE_TO_STREAM_CASE(6, int32_t, fuzzed_data_provider.ConsumeIntegral<int32_t>()) + WRITE_TO_STREAM_CASE(7, uint32_t, fuzzed_data_provider.ConsumeIntegral<uint32_t>()) + WRITE_TO_STREAM_CASE(8, int64_t, fuzzed_data_provider.ConsumeIntegral<int64_t>()) + WRITE_TO_STREAM_CASE(9, uint64_t, fuzzed_data_provider.ConsumeIntegral<uint64_t>()) + WRITE_TO_STREAM_CASE(10, float, fuzzed_data_provider.ConsumeFloatingPoint<float>()) + WRITE_TO_STREAM_CASE(11, double, fuzzed_data_provider.ConsumeFloatingPoint<double>()) + WRITE_TO_STREAM_CASE(12, std::string, fuzzed_data_provider.ConsumeRandomLengthString(32)) + WRITE_TO_STREAM_CASE(13, std::vector<char>, ConsumeRandomLengthIntegralVector<char>(fuzzed_data_provider)) + } + } catch (const std::ios_base::failure&) { + break; + } + } +} + +#define READ_FROM_STREAM_CASE(id, type) \ + case id: { \ + type o; \ + stream >> o; \ + break; \ + } +template <typename Stream> +void ReadFromStream(FuzzedDataProvider& fuzzed_data_provider, Stream& stream) noexcept +{ + while (fuzzed_data_provider.ConsumeBool()) { + try { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 13)) { + READ_FROM_STREAM_CASE(0, bool) + READ_FROM_STREAM_CASE(1, char) + READ_FROM_STREAM_CASE(2, int8_t) + READ_FROM_STREAM_CASE(3, uint8_t) + READ_FROM_STREAM_CASE(4, int16_t) + READ_FROM_STREAM_CASE(5, uint16_t) + READ_FROM_STREAM_CASE(6, int32_t) + READ_FROM_STREAM_CASE(7, uint32_t) + READ_FROM_STREAM_CASE(8, int64_t) + READ_FROM_STREAM_CASE(9, uint64_t) + READ_FROM_STREAM_CASE(10, float) + READ_FROM_STREAM_CASE(11, double) + READ_FROM_STREAM_CASE(12, std::string) + READ_FROM_STREAM_CASE(13, std::vector<char>) + } + } catch (const std::ios_base::failure&) { + break; + } + } +} + #endif // BITCOIN_TEST_FUZZ_UTIL_H diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index cf2bd03698..fd35537c77 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -5,6 +5,7 @@ #include <key.h> #include <key_io.h> +#include <streams.h> #include <test/util/setup_common.h> #include <uint256.h> #include <util/strencodings.h> @@ -220,4 +221,47 @@ BOOST_AUTO_TEST_CASE(key_key_negation) BOOST_CHECK(key.GetPubKey().data()[0] == 0x03); } +static CPubKey UnserializePubkey(const std::vector<uint8_t>& data) +{ + CDataStream stream{SER_NETWORK, INIT_PROTO_VERSION}; + stream << data; + CPubKey pubkey; + stream >> pubkey; + return pubkey; +} + +static unsigned int GetLen(unsigned char chHeader) +{ + if (chHeader == 2 || chHeader == 3) + return CPubKey::COMPRESSED_SIZE; + if (chHeader == 4 || chHeader == 6 || chHeader == 7) + return CPubKey::SIZE; + return 0; +} + +static void CmpSerializationPubkey(const CPubKey& pubkey) +{ + CDataStream stream{SER_NETWORK, INIT_PROTO_VERSION}; + stream << pubkey; + CPubKey pubkey2; + stream >> pubkey2; + BOOST_CHECK(pubkey == pubkey2); +} + +BOOST_AUTO_TEST_CASE(pubkey_unserialize) +{ + for (uint8_t i = 2; i <= 7; ++i) { + CPubKey key = UnserializePubkey({0x02}); + BOOST_CHECK(!key.IsValid()); + CmpSerializationPubkey(key); + key = UnserializePubkey(std::vector<uint8_t>(GetLen(i), i)); + CmpSerializationPubkey(key); + if (i == 5) { + BOOST_CHECK(!key.IsValid()); + } else { + BOOST_CHECK(key.IsValid()); + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 11ff7b833b..62a0dc4241 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -253,7 +253,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) pblock->nNonce = blockinfo[i].nonce; } std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); - BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); + BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); pblock->hashPrevBlock = pblock->GetHash(); } diff --git a/src/test/multisig_tests.cpp b/src/test/multisig_tests.cpp index dd2890c134..e14d2dd72d 100644 --- a/src/test/multisig_tests.cpp +++ b/src/test/multisig_tests.cpp @@ -141,7 +141,7 @@ BOOST_AUTO_TEST_CASE(multisig_IsStandard) for (int i = 0; i < 4; i++) key[i].MakeNewKey(true); - txnouttype whichType; + TxoutType whichType; CScript a_and_b; a_and_b << OP_2 << ToByteVector(key[0].GetPubKey()) << ToByteVector(key[1].GetPubKey()) << OP_2 << OP_CHECKMULTISIG; diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 84bf593497..ab42be21bd 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -6,6 +6,7 @@ #include <addrman.h> #include <chainparams.h> #include <clientversion.h> +#include <cstdint> #include <net.h> #include <netbase.h> #include <serialize.h> @@ -83,10 +84,10 @@ BOOST_FIXTURE_TEST_SUITE(net_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(cnode_listen_port) { // test default - unsigned short port = GetListenPort(); + uint16_t port = GetListenPort(); BOOST_CHECK(port == Params().GetDefaultPort()); // test set port - unsigned short altPort = 12345; + uint16_t altPort = 12345; BOOST_CHECK(gArgs.SoftSetArg("-port", ToString(altPort))); port = GetListenPort(); BOOST_CHECK(port == altPort); diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index d0ec401f9d..591b4ce49a 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -138,6 +138,14 @@ BOOST_AUTO_TEST_CASE(onioncat_test) } +BOOST_AUTO_TEST_CASE(embedded_test) +{ + CNetAddr addr1(ResolveIP("1.2.3.4")); + CNetAddr addr2(ResolveIP("::FFFF:0102:0304")); + BOOST_CHECK(addr2.IsIPv4()); + BOOST_CHECK_EQUAL(addr1.ToString(), addr2.ToString()); +} + BOOST_AUTO_TEST_CASE(subnet_test) { @@ -158,9 +166,13 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK(ResolveSubNet("1.2.2.1/24").Match(ResolveIP("1.2.2.4"))); BOOST_CHECK(ResolveSubNet("1.2.2.110/31").Match(ResolveIP("1.2.2.111"))); BOOST_CHECK(ResolveSubNet("1.2.2.20/26").Match(ResolveIP("1.2.2.63"))); - // All-Matching IPv6 Matches arbitrary IPv4 and IPv6 + // All-Matching IPv6 Matches arbitrary IPv6 BOOST_CHECK(ResolveSubNet("::/0").Match(ResolveIP("1:2:3:4:5:6:7:1234"))); - BOOST_CHECK(ResolveSubNet("::/0").Match(ResolveIP("1.2.3.4"))); + // But not `::` or `0.0.0.0` because they are considered invalid addresses + BOOST_CHECK(!ResolveSubNet("::/0").Match(ResolveIP("::"))); + BOOST_CHECK(!ResolveSubNet("::/0").Match(ResolveIP("0.0.0.0"))); + // Addresses from one network (IPv4) don't belong to subnets of another network (IPv6) + BOOST_CHECK(!ResolveSubNet("::/0").Match(ResolveIP("1.2.3.4"))); // All-Matching IPv4 does not Match IPv6 BOOST_CHECK(!ResolveSubNet("0.0.0.0/0").Match(ResolveIP("1:2:3:4:5:6:7:1234"))); // Invalid subnets Match nothing (not even invalid addresses) @@ -380,7 +392,7 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) BOOST_CHECK(!NetWhitebindPermissions::TryParse("bloom,forcerelay,oopsie@1.2.3.4:32", whitebindPermissions, error)); BOOST_CHECK(error.original.find("Invalid P2P permission") != std::string::npos); - // Check whitelist error + // Check netmask error BOOST_CHECK(!NetWhitelistPermissions::TryParse("bloom,forcerelay,noban@1.2.3.4:32", whitelistPermissions, error)); BOOST_CHECK(error.original.find("Invalid netmask specified in -whitelist") != std::string::npos); @@ -394,12 +406,13 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) BOOST_CHECK(NetWhitelistPermissions::TryParse("bloom,forcerelay,noban,relay,mempool@1.2.3.4/32", whitelistPermissions, error)); const auto strings = NetPermissions::ToStrings(PF_ALL); - BOOST_CHECK_EQUAL(strings.size(), 5U); + BOOST_CHECK_EQUAL(strings.size(), 6U); BOOST_CHECK(std::find(strings.begin(), strings.end(), "bloomfilter") != strings.end()); BOOST_CHECK(std::find(strings.begin(), strings.end(), "forcerelay") != strings.end()); BOOST_CHECK(std::find(strings.begin(), strings.end(), "relay") != strings.end()); BOOST_CHECK(std::find(strings.begin(), strings.end(), "noban") != strings.end()); BOOST_CHECK(std::find(strings.begin(), strings.end(), "mempool") != strings.end()); + BOOST_CHECK(std::find(strings.begin(), strings.end(), "download") != strings.end()); } BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) diff --git a/src/test/policy_fee_tests.cpp b/src/test/policy_fee_tests.cpp new file mode 100644 index 0000000000..6d8872b11e --- /dev/null +++ b/src/test/policy_fee_tests.cpp @@ -0,0 +1,34 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <amount.h> +#include <policy/fees.h> + +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(policy_fee_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(FeeRounder) +{ + FeeFilterRounder fee_rounder{CFeeRate{1000}}; + + // check that 1000 rounds to 974 or 1071 + std::set<CAmount> results; + while (results.size() < 2) { + results.emplace(fee_rounder.round(1000)); + } + BOOST_CHECK_EQUAL(*results.begin(), 974); + BOOST_CHECK_EQUAL(*++results.begin(), 1071); + + // check that negative amounts rounds to 0 + BOOST_CHECK_EQUAL(fee_rounder.round(-0), 0); + BOOST_CHECK_EQUAL(fee_rounder.round(-1), 0); + + // check that MAX_MONEY rounds to 9170997 + BOOST_CHECK_EQUAL(fee_rounder.round(MAX_MONEY), 9170997); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index fcee6a9b9d..2e5a7549b7 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -89,7 +89,7 @@ BOOST_AUTO_TEST_CASE(manythreads) } // Drain the task queue then exit threads - microTasks.stop(true); + microTasks.StopWhenDrained(); microThreads.join_all(); // ... wait until all the threads are done int counterSum = 0; @@ -155,7 +155,7 @@ BOOST_AUTO_TEST_CASE(singlethreadedscheduler_ordered) } // finish up - scheduler.stop(true); + scheduler.StopWhenDrained(); threads.join_all(); BOOST_CHECK_EQUAL(counter1, 100); @@ -186,7 +186,7 @@ BOOST_AUTO_TEST_CASE(mockforward) scheduler.MockForward(std::chrono::minutes{5}); // ensure scheduler has chance to process all tasks queued for before 1 ms from now. - scheduler.scheduleFromNow([&scheduler] { scheduler.stop(false); }, std::chrono::milliseconds{1}); + scheduler.scheduleFromNow([&scheduler] { scheduler.stop(); }, std::chrono::milliseconds{1}); scheduler_thread.join(); // check that the queue only has one job remaining diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index b185d3b4ac..77d748241b 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -31,35 +31,35 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_success) CScript s; std::vector<std::vector<unsigned char> > solutions; - // TX_PUBKEY + // TxoutType::PUBKEY s.clear(); s << ToByteVector(pubkeys[0]) << OP_CHECKSIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_PUBKEY); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::PUBKEY); BOOST_CHECK_EQUAL(solutions.size(), 1U); BOOST_CHECK(solutions[0] == ToByteVector(pubkeys[0])); - // TX_PUBKEYHASH + // TxoutType::PUBKEYHASH s.clear(); s << OP_DUP << OP_HASH160 << ToByteVector(pubkeys[0].GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_PUBKEYHASH); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::PUBKEYHASH); BOOST_CHECK_EQUAL(solutions.size(), 1U); BOOST_CHECK(solutions[0] == ToByteVector(pubkeys[0].GetID())); - // TX_SCRIPTHASH + // TxoutType::SCRIPTHASH CScript redeemScript(s); // initialize with leftover P2PKH script s.clear(); s << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_SCRIPTHASH); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::SCRIPTHASH); BOOST_CHECK_EQUAL(solutions.size(), 1U); BOOST_CHECK(solutions[0] == ToByteVector(CScriptID(redeemScript))); - // TX_MULTISIG + // TxoutType::MULTISIG s.clear(); s << OP_1 << ToByteVector(pubkeys[0]) << ToByteVector(pubkeys[1]) << OP_2 << OP_CHECKMULTISIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_MULTISIG); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::MULTISIG); BOOST_CHECK_EQUAL(solutions.size(), 4U); BOOST_CHECK(solutions[0] == std::vector<unsigned char>({1})); BOOST_CHECK(solutions[1] == ToByteVector(pubkeys[0])); @@ -72,7 +72,7 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_success) ToByteVector(pubkeys[1]) << ToByteVector(pubkeys[2]) << OP_3 << OP_CHECKMULTISIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_MULTISIG); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::MULTISIG); BOOST_CHECK_EQUAL(solutions.size(), 5U); BOOST_CHECK(solutions[0] == std::vector<unsigned char>({2})); BOOST_CHECK(solutions[1] == ToByteVector(pubkeys[0])); @@ -80,37 +80,37 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_success) BOOST_CHECK(solutions[3] == ToByteVector(pubkeys[2])); BOOST_CHECK(solutions[4] == std::vector<unsigned char>({3})); - // TX_NULL_DATA + // TxoutType::NULL_DATA s.clear(); s << OP_RETURN << std::vector<unsigned char>({0}) << std::vector<unsigned char>({75}) << std::vector<unsigned char>({255}); - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NULL_DATA); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NULL_DATA); BOOST_CHECK_EQUAL(solutions.size(), 0U); - // TX_WITNESS_V0_KEYHASH + // TxoutType::WITNESS_V0_KEYHASH s.clear(); s << OP_0 << ToByteVector(pubkeys[0].GetID()); - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_WITNESS_V0_KEYHASH); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::WITNESS_V0_KEYHASH); BOOST_CHECK_EQUAL(solutions.size(), 1U); BOOST_CHECK(solutions[0] == ToByteVector(pubkeys[0].GetID())); - // TX_WITNESS_V0_SCRIPTHASH + // TxoutType::WITNESS_V0_SCRIPTHASH uint256 scriptHash; CSHA256().Write(&redeemScript[0], redeemScript.size()) .Finalize(scriptHash.begin()); s.clear(); s << OP_0 << ToByteVector(scriptHash); - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_WITNESS_V0_SCRIPTHASH); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::WITNESS_V0_SCRIPTHASH); BOOST_CHECK_EQUAL(solutions.size(), 1U); BOOST_CHECK(solutions[0] == ToByteVector(scriptHash)); - // TX_NONSTANDARD + // TxoutType::NONSTANDARD s.clear(); s << OP_9 << OP_ADD << OP_11 << OP_EQUAL; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); } BOOST_AUTO_TEST_CASE(script_standard_Solver_failure) @@ -123,50 +123,50 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_failure) CScript s; std::vector<std::vector<unsigned char> > solutions; - // TX_PUBKEY with incorrectly sized pubkey + // TxoutType::PUBKEY with incorrectly sized pubkey s.clear(); s << std::vector<unsigned char>(30, 0x01) << OP_CHECKSIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_PUBKEYHASH with incorrectly sized key hash + // TxoutType::PUBKEYHASH with incorrectly sized key hash s.clear(); s << OP_DUP << OP_HASH160 << ToByteVector(pubkey) << OP_EQUALVERIFY << OP_CHECKSIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_SCRIPTHASH with incorrectly sized script hash + // TxoutType::SCRIPTHASH with incorrectly sized script hash s.clear(); s << OP_HASH160 << std::vector<unsigned char>(21, 0x01) << OP_EQUAL; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_MULTISIG 0/2 + // TxoutType::MULTISIG 0/2 s.clear(); s << OP_0 << ToByteVector(pubkey) << OP_1 << OP_CHECKMULTISIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_MULTISIG 2/1 + // TxoutType::MULTISIG 2/1 s.clear(); s << OP_2 << ToByteVector(pubkey) << OP_1 << OP_CHECKMULTISIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_MULTISIG n = 2 with 1 pubkey + // TxoutType::MULTISIG n = 2 with 1 pubkey s.clear(); s << OP_1 << ToByteVector(pubkey) << OP_2 << OP_CHECKMULTISIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_MULTISIG n = 1 with 0 pubkeys + // TxoutType::MULTISIG n = 1 with 0 pubkeys s.clear(); s << OP_1 << OP_1 << OP_CHECKMULTISIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_NULL_DATA with other opcodes + // TxoutType::NULL_DATA with other opcodes s.clear(); s << OP_RETURN << std::vector<unsigned char>({75}) << OP_ADD; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_WITNESS with incorrect program size + // TxoutType::WITNESS_UNKNOWN with incorrect program size s.clear(); s << OP_0 << std::vector<unsigned char>(19, 0x01); - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); } BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) @@ -179,21 +179,21 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) CScript s; CTxDestination address; - // TX_PUBKEY + // TxoutType::PUBKEY s.clear(); s << ToByteVector(pubkey) << OP_CHECKSIG; BOOST_CHECK(ExtractDestination(s, address)); BOOST_CHECK(boost::get<PKHash>(&address) && *boost::get<PKHash>(&address) == PKHash(pubkey)); - // TX_PUBKEYHASH + // TxoutType::PUBKEYHASH s.clear(); s << OP_DUP << OP_HASH160 << ToByteVector(pubkey.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; BOOST_CHECK(ExtractDestination(s, address)); BOOST_CHECK(boost::get<PKHash>(&address) && *boost::get<PKHash>(&address) == PKHash(pubkey)); - // TX_SCRIPTHASH + // TxoutType::SCRIPTHASH CScript redeemScript(s); // initialize with leftover P2PKH script s.clear(); s << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; @@ -201,17 +201,17 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) BOOST_CHECK(boost::get<ScriptHash>(&address) && *boost::get<ScriptHash>(&address) == ScriptHash(redeemScript)); - // TX_MULTISIG + // TxoutType::MULTISIG s.clear(); s << OP_1 << ToByteVector(pubkey) << OP_1 << OP_CHECKMULTISIG; BOOST_CHECK(!ExtractDestination(s, address)); - // TX_NULL_DATA + // TxoutType::NULL_DATA s.clear(); s << OP_RETURN << std::vector<unsigned char>({75}); BOOST_CHECK(!ExtractDestination(s, address)); - // TX_WITNESS_V0_KEYHASH + // TxoutType::WITNESS_V0_KEYHASH s.clear(); s << OP_0 << ToByteVector(pubkey.GetID()); BOOST_CHECK(ExtractDestination(s, address)); @@ -219,7 +219,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) CHash160().Write(pubkey.begin(), pubkey.size()).Finalize(keyhash.begin()); BOOST_CHECK(boost::get<WitnessV0KeyHash>(&address) && *boost::get<WitnessV0KeyHash>(&address) == keyhash); - // TX_WITNESS_V0_SCRIPTHASH + // TxoutType::WITNESS_V0_SCRIPTHASH s.clear(); WitnessV0ScriptHash scripthash; CSHA256().Write(redeemScript.data(), redeemScript.size()).Finalize(scripthash.begin()); @@ -227,7 +227,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) BOOST_CHECK(ExtractDestination(s, address)); BOOST_CHECK(boost::get<WitnessV0ScriptHash>(&address) && *boost::get<WitnessV0ScriptHash>(&address) == scripthash); - // TX_WITNESS with unknown version + // TxoutType::WITNESS_UNKNOWN with unknown version s.clear(); s << OP_1 << ToByteVector(pubkey); BOOST_CHECK(ExtractDestination(s, address)); @@ -248,49 +248,49 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) } CScript s; - txnouttype whichType; + TxoutType whichType; std::vector<CTxDestination> addresses; int nRequired; - // TX_PUBKEY + // TxoutType::PUBKEY s.clear(); s << ToByteVector(pubkeys[0]) << OP_CHECKSIG; BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TX_PUBKEY); + BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEY); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); BOOST_CHECK(boost::get<PKHash>(&addresses[0]) && *boost::get<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); - // TX_PUBKEYHASH + // TxoutType::PUBKEYHASH s.clear(); s << OP_DUP << OP_HASH160 << ToByteVector(pubkeys[0].GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TX_PUBKEYHASH); + BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEYHASH); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); BOOST_CHECK(boost::get<PKHash>(&addresses[0]) && *boost::get<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); - // TX_SCRIPTHASH + // TxoutType::SCRIPTHASH CScript redeemScript(s); // initialize with leftover P2PKH script s.clear(); s << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TX_SCRIPTHASH); + BOOST_CHECK_EQUAL(whichType, TxoutType::SCRIPTHASH); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); BOOST_CHECK(boost::get<ScriptHash>(&addresses[0]) && *boost::get<ScriptHash>(&addresses[0]) == ScriptHash(redeemScript)); - // TX_MULTISIG + // TxoutType::MULTISIG s.clear(); s << OP_2 << ToByteVector(pubkeys[0]) << ToByteVector(pubkeys[1]) << OP_2 << OP_CHECKMULTISIG; BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TX_MULTISIG); + BOOST_CHECK_EQUAL(whichType, TxoutType::MULTISIG); BOOST_CHECK_EQUAL(addresses.size(), 2U); BOOST_CHECK_EQUAL(nRequired, 2); BOOST_CHECK(boost::get<PKHash>(&addresses[0]) && @@ -298,7 +298,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK(boost::get<PKHash>(&addresses[1]) && *boost::get<PKHash>(&addresses[1]) == PKHash(pubkeys[1])); - // TX_NULL_DATA + // TxoutType::NULL_DATA s.clear(); s << OP_RETURN << std::vector<unsigned char>({75}); BOOST_CHECK(!ExtractDestinations(s, whichType, addresses, nRequired)); diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index fcd831ccda..1a2d775f49 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -12,10 +12,90 @@ #include <univalue.h> #include <util/strencodings.h> #include <util/string.h> +#include <util/system.h> #include <vector> +inline bool operator==(const util::SettingsValue& a, const util::SettingsValue& b) +{ + return a.write() == b.write(); +} + +inline std::ostream& operator<<(std::ostream& os, const util::SettingsValue& value) +{ + os << value.write(); + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const std::pair<std::string, util::SettingsValue>& kv) +{ + util::SettingsValue out(util::SettingsValue::VOBJ); + out.__pushKV(kv.first, kv.second); + os << out.write(); + return os; +} + +inline void WriteText(const fs::path& path, const std::string& text) +{ + fsbridge::ofstream file; + file.open(path); + file << text; +} + BOOST_FIXTURE_TEST_SUITE(settings_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(ReadWrite) +{ + fs::path path = GetDataDir() / "settings.json"; + + WriteText(path, R"({ + "string": "string", + "num": 5, + "bool": true, + "null": null + })"); + + std::map<std::string, util::SettingsValue> expected{ + {"string", "string"}, + {"num", 5}, + {"bool", true}, + {"null", {}}, + }; + + // Check file read. + std::map<std::string, util::SettingsValue> values; + std::vector<std::string> errors; + BOOST_CHECK(util::ReadSettings(path, values, errors)); + BOOST_CHECK_EQUAL_COLLECTIONS(values.begin(), values.end(), expected.begin(), expected.end()); + BOOST_CHECK(errors.empty()); + + // Check no errors if file doesn't exist. + fs::remove(path); + BOOST_CHECK(util::ReadSettings(path, values, errors)); + BOOST_CHECK(values.empty()); + BOOST_CHECK(errors.empty()); + + // Check duplicate keys not allowed + WriteText(path, R"({ + "dupe": "string", + "dupe": "dupe" + })"); + BOOST_CHECK(!util::ReadSettings(path, values, errors)); + std::vector<std::string> dup_keys = {strprintf("Found duplicate key dupe in settings file %s", path.string())}; + BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), dup_keys.begin(), dup_keys.end()); + + // Check non-kv json files not allowed + WriteText(path, R"("non-kv")"); + BOOST_CHECK(!util::ReadSettings(path, values, errors)); + std::vector<std::string> non_kv = {strprintf("Found non-object value \"non-kv\" in settings file %s", path.string())}; + BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), non_kv.begin(), non_kv.end()); + + // Check invalid json not allowed + WriteText(path, R"(invalid json)"); + BOOST_CHECK(!util::ReadSettings(path, values, errors)); + std::vector<std::string> fail_parse = {strprintf("Unable to parse settings file %s", path.string())}; + BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), fail_parse.begin(), fail_parse.end()); +} + //! Check settings struct contents against expected json strings. static void CheckValues(const util::Settings& settings, const std::string& single_val, const std::string& list_val) { diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 5ca136ea6e..c0bb92258b 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -141,7 +141,7 @@ BOOST_AUTO_TEST_CASE(sighash_test) ss << txTo; std::cout << "\t[\"" ; - std::cout << HexStr(ss.begin(), ss.end()) << "\", \""; + std::cout << HexStr(ss) << "\", \""; std::cout << HexStr(scriptCode) << "\", "; std::cout << nIn << ", "; std::cout << nHashType << ", \""; diff --git a/src/test/sync_tests.cpp b/src/test/sync_tests.cpp index 5c6c2ee38e..3ea8714f3a 100644 --- a/src/test/sync_tests.cpp +++ b/src/test/sync_tests.cpp @@ -18,7 +18,7 @@ void TestPotentialDeadLockDetected(MutexType& mutex1, MutexType& mutex2) try { LOCK2(mutex2, mutex1); } catch (const std::logic_error& e) { - BOOST_CHECK_EQUAL(e.what(), "potential deadlock detected"); + BOOST_CHECK_EQUAL(e.what(), "potential deadlock detected: mutex1 -> mutex2 -> mutex1"); error_thrown = true; } #ifdef DEBUG_LOCKORDER diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index ddbc68f8e2..4bf6e734ce 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -716,12 +716,12 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); BOOST_CHECK_EQUAL(reason, "scriptpubkey"); - // MAX_OP_RETURN_RELAY-byte TX_NULL_DATA (standard) + // MAX_OP_RETURN_RELAY-byte TxoutType::NULL_DATA (standard) t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); BOOST_CHECK_EQUAL(MAX_OP_RETURN_RELAY, t.vout[0].scriptPubKey.size()); BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); - // MAX_OP_RETURN_RELAY+1-byte TX_NULL_DATA (non-standard) + // MAX_OP_RETURN_RELAY+1-byte TxoutType::NULL_DATA (non-standard) t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3800"); BOOST_CHECK_EQUAL(MAX_OP_RETURN_RELAY + 1, t.vout[0].scriptPubKey.size()); reason.clear(); @@ -745,12 +745,12 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); BOOST_CHECK_EQUAL(reason, "scriptpubkey"); - // TX_NULL_DATA w/o PUSHDATA + // TxoutType::NULL_DATA w/o PUSHDATA t.vout.resize(1); t.vout[0].scriptPubKey = CScript() << OP_RETURN; BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); - // Only one TX_NULL_DATA permitted in all cases + // Only one TxoutType::NULL_DATA permitted in all cases t.vout.resize(2); t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); t.vout[1].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index dac7f1a07b..74536ae74c 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -11,6 +11,7 @@ #include <node/context.h> #include <pow.h> #include <script/standard.h> +#include <util/check.h> #include <validation.h> CTxIn generatetoaddress(const NodeContext& node, const std::string& address) @@ -31,7 +32,7 @@ CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) assert(block->nNonce); } - bool processed{EnsureChainman(node).ProcessNewBlock(Params(), block, true, nullptr)}; + bool processed{Assert(node.chainman)->ProcessNewBlock(Params(), block, true, nullptr)}; assert(processed); return CTxIn{block->vtx[0]->GetHash(), 0}; @@ -39,9 +40,8 @@ CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) { - assert(node.mempool); auto block = std::make_shared<CBlock>( - BlockAssembler{*node.mempool, Params()} + BlockAssembler{*Assert(node.mempool), Params()} .CreateNewBlock(coinbase_scriptPubKey) ->block); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 3b7a7c8d12..0cd991f453 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -19,6 +19,7 @@ #include <rpc/blockchain.h> #include <rpc/register.h> #include <rpc/server.h> +#include <scheduler.h> #include <script/sigcache.h> #include <streams.h> #include <txdb.h> @@ -74,11 +75,13 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve "dummy", "-printtoconsole=0", "-logtimemicros", + "-logthreadnames", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", }, extra_args); + util::ThreadRename("test"); fs::create_directories(m_path_root); gArgs.ForceSetArg("-datadir", m_path_root.string()); ClearDatadirCache(); @@ -129,7 +132,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const // We have to run a scheduler thread to prevent ActivateBestChain // from blocking due to queue overrun. - threadGroup.create_thread([&]{ m_node.scheduler->serviceQueue(); }); + threadGroup.create_thread([&] { TraceThread("scheduler", [&] { m_node.scheduler->serviceQueue(); }); }); GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); @@ -139,7 +142,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const ::ChainstateActive().InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); assert(!::ChainstateActive().CanFlushToDisk()); - ::ChainstateActive().InitCoinsCache(); + ::ChainstateActive().InitCoinsCache(1 << 23); assert(::ChainstateActive().CanFlushToDisk()); if (!LoadGenesisBlock(chainparams)) { throw std::runtime_error("LoadGenesisBlock failed."); @@ -228,7 +231,7 @@ CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransa while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); - EnsureChainman(m_node).ProcessNewBlock(chainparams, shared_pblock, true, nullptr); + Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr); CBlock result = block; return result; diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index d5cda8a95b..78b279e42a 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -11,8 +11,8 @@ #include <node/context.h> #include <pubkey.h> #include <random.h> -#include <scheduler.h> #include <txmempool.h> +#include <util/check.h> #include <util/string.h> #include <type_traits> diff --git a/src/test/util/transaction_utils.h b/src/test/util/transaction_utils.h index 1beddd334b..6f2faeec6c 100644 --- a/src/test/util/transaction_utils.h +++ b/src/test/util/transaction_utils.h @@ -22,8 +22,8 @@ CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int n CMutableTransaction BuildSpendingTransaction(const CScript& scriptSig, const CScriptWitness& scriptWitness, const CTransaction& txCredit); // Helper: create two dummy transactions, each with two outputs. -// The first has nValues[0] and nValues[1] outputs paid to a TX_PUBKEY, -// the second nValues[2] and nValues[3] outputs paid to a TX_PUBKEYHASH. +// The first has nValues[0] and nValues[1] outputs paid to a TxoutType::PUBKEY, +// the second nValues[2] and nValues[3] outputs paid to a TxoutType::PUBKEYHASH. std::vector<CMutableTransaction> SetupDummyInputs(FillableSigningProvider& keystoreRet, CCoinsViewCache& coinsRet, const std::array<CAmount,4>& nValues); #endif // BITCOIN_TEST_UTIL_TRANSACTION_UTILS_H diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index cf26ca3adb..a30e366028 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -9,6 +9,7 @@ #include <key.h> // For CKey #include <optional.h> #include <sync.h> +#include <test/util/logging.h> #include <test/util/setup_common.h> #include <test/util/str.h> #include <uint256.h> @@ -41,6 +42,16 @@ namespace BCLog { BOOST_FIXTURE_TEST_SUITE(util_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(util_check) +{ + // Check that Assert can forward + const std::unique_ptr<int> p_two = Assert(MakeUnique<int>(2)); + // Check that Assert works on lvalues and rvalues + const int two = *Assert(p_two); + Assert(two == 2); + Assert(true); +} + BOOST_AUTO_TEST_CASE(util_criticalsection) { RecursiveMutex cs; @@ -1128,6 +1139,28 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) BOOST_CHECK_EQUAL(out_sha_hex, "f0b3a3c29869edc765d579c928f7f1690a71fbb673b49ccf39cbc4de18156a0d"); } +BOOST_AUTO_TEST_CASE(util_ReadWriteSettings) +{ + // Test writing setting. + TestArgsManager args1; + args1.LockSettings([&](util::Settings& settings) { settings.rw_settings["name"] = "value"; }); + args1.WriteSettingsFile(); + + // Test reading setting. + TestArgsManager args2; + args2.ReadSettingsFile(); + args2.LockSettings([&](util::Settings& settings) { BOOST_CHECK_EQUAL(settings.rw_settings["name"].get_str(), "value"); }); + + // Test error logging, and remove previously written setting. + { + ASSERT_DEBUG_LOG("Failed renaming settings file"); + fs::remove(GetDataDir() / "settings.json"); + fs::create_directory(GetDataDir() / "settings.json"); + args2.WriteSettingsFile(); + fs::remove(GetDataDir() / "settings.json"); + } +} + BOOST_AUTO_TEST_CASE(util_FormatMoney) { BOOST_CHECK_EQUAL(FormatMoney(0), "0.00"); @@ -1829,7 +1862,7 @@ BOOST_AUTO_TEST_CASE(test_spanparsing) // Const(...): parse a constant, update span to skip it if successful input = "MilkToastHoney"; - sp = MakeSpan(input); + sp = input; success = Const("", sp); // empty BOOST_CHECK(success); BOOST_CHECK_EQUAL(SpanToStr(sp), "MilkToastHoney"); @@ -1854,7 +1887,7 @@ BOOST_AUTO_TEST_CASE(test_spanparsing) // Func(...): parse a function call, update span to argument if successful input = "Foo(Bar(xy,z()))"; - sp = MakeSpan(input); + sp = input; success = Func("FooBar", sp); BOOST_CHECK(!success); @@ -1877,31 +1910,31 @@ BOOST_AUTO_TEST_CASE(test_spanparsing) Span<const char> result; input = "(n*(n-1))/2"; - sp = MakeSpan(input); + sp = input; result = Expr(sp); BOOST_CHECK_EQUAL(SpanToStr(result), "(n*(n-1))/2"); BOOST_CHECK_EQUAL(SpanToStr(sp), ""); input = "foo,bar"; - sp = MakeSpan(input); + sp = input; result = Expr(sp); BOOST_CHECK_EQUAL(SpanToStr(result), "foo"); BOOST_CHECK_EQUAL(SpanToStr(sp), ",bar"); input = "(aaaaa,bbbbb()),c"; - sp = MakeSpan(input); + sp = input; result = Expr(sp); BOOST_CHECK_EQUAL(SpanToStr(result), "(aaaaa,bbbbb())"); BOOST_CHECK_EQUAL(SpanToStr(sp), ",c"); input = "xyz)foo"; - sp = MakeSpan(input); + sp = input; result = Expr(sp); BOOST_CHECK_EQUAL(SpanToStr(result), "xyz"); BOOST_CHECK_EQUAL(SpanToStr(sp), ")foo"); input = "((a),(b),(c)),xxx"; - sp = MakeSpan(input); + sp = input; result = Expr(sp); BOOST_CHECK_EQUAL(SpanToStr(result), "((a),(b),(c))"); BOOST_CHECK_EQUAL(SpanToStr(sp), ",xxx"); @@ -1910,7 +1943,7 @@ BOOST_AUTO_TEST_CASE(test_spanparsing) std::vector<Span<const char>> results; input = "xxx"; - results = Split(MakeSpan(input), 'x'); + results = Split(input, 'x'); BOOST_CHECK_EQUAL(results.size(), 4U); BOOST_CHECK_EQUAL(SpanToStr(results[0]), ""); BOOST_CHECK_EQUAL(SpanToStr(results[1]), ""); @@ -1918,19 +1951,19 @@ BOOST_AUTO_TEST_CASE(test_spanparsing) BOOST_CHECK_EQUAL(SpanToStr(results[3]), ""); input = "one#two#three"; - results = Split(MakeSpan(input), '-'); + results = Split(input, '-'); BOOST_CHECK_EQUAL(results.size(), 1U); BOOST_CHECK_EQUAL(SpanToStr(results[0]), "one#two#three"); input = "one#two#three"; - results = Split(MakeSpan(input), '#'); + results = Split(input, '#'); BOOST_CHECK_EQUAL(results.size(), 3U); BOOST_CHECK_EQUAL(SpanToStr(results[0]), "one"); BOOST_CHECK_EQUAL(SpanToStr(results[1]), "two"); BOOST_CHECK_EQUAL(SpanToStr(results[2]), "three"); input = "*foo*bar*"; - results = Split(MakeSpan(input), '*'); + results = Split(input, '*'); BOOST_CHECK_EQUAL(results.size(), 4U); BOOST_CHECK_EQUAL(SpanToStr(results[0]), ""); BOOST_CHECK_EQUAL(SpanToStr(results[1]), "foo"); diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index 4dcc080b2d..f3f9fb2bff 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -53,8 +53,6 @@ std::set<std::string> RenameEnMasse(int num_threads) */ BOOST_AUTO_TEST_CASE(util_threadnames_test_rename_threaded) { - BOOST_CHECK_EQUAL(util::ThreadGetInternalName(), ""); - #if !defined(HAVE_THREAD_LOCAL) // This test doesn't apply to platforms where we don't have thread_local. return; diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 45e0c5484e..8e85b7df3e 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -163,10 +163,10 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) std::transform(blocks.begin(), blocks.end(), std::back_inserter(headers), [](std::shared_ptr<const CBlock> b) { return b->GetBlockHeader(); }); // Process all the headers so we understand the toplogy of the chain - BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlockHeaders(headers, state, Params())); + BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlockHeaders(headers, state, Params())); // Connect the genesis block and drain any outstanding events - BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored)); + BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored)); SyncWithValidationInterfaceQueue(); // subscribe to events (this subscriber will validate event ordering) @@ -188,13 +188,13 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) FastRandomContext insecure; for (int i = 0; i < 1000; i++) { auto block = blocks[insecure.randrange(blocks.size() - 1)]; - EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, &ignored); + Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, &ignored); } // to make sure that eventually we process the full chain - do it here for (auto block : blocks) { if (block->vtx.size() == 1) { - bool processed = EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, &ignored); + bool processed = Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, &ignored); assert(processed); } } @@ -233,7 +233,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) { bool ignored; auto ProcessBlock = [&](std::shared_ptr<const CBlock> block) -> bool { - return EnsureChainman(m_node).ProcessNewBlock(Params(), block, /* fForceProcessing */ true, /* fNewBlock */ &ignored); + return Assert(m_node.chainman)->ProcessNewBlock(Params(), block, /* fForceProcessing */ true, /* fNewBlock */ &ignored); }; // Process all mined blocks diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp new file mode 100644 index 0000000000..f531b393b8 --- /dev/null +++ b/src/test/validation_chainstate_tests.cpp @@ -0,0 +1,76 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include <random.h> +#include <uint256.h> +#include <consensus/validation.h> +#include <sync.h> +#include <test/util/setup_common.h> +#include <validation.h> + +#include <vector> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(validation_chainstate_tests, TestingSetup) + +//! Test resizing coins-related CChainState caches during runtime. +//! +BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) +{ + ChainstateManager manager; + + //! Create and add a Coin with DynamicMemoryUsage of 80 bytes to the given view. + auto add_coin = [](CCoinsViewCache& coins_view) -> COutPoint { + Coin newcoin; + uint256 txid = InsecureRand256(); + COutPoint outp{txid, 0}; + newcoin.nHeight = 1; + newcoin.out.nValue = InsecureRand32(); + newcoin.out.scriptPubKey.assign((uint32_t)56, 1); + coins_view.AddCoin(outp, std::move(newcoin), false); + + return outp; + }; + + ENTER_CRITICAL_SECTION(cs_main); + CChainState& c1 = manager.InitializeChainstate(); + LEAVE_CRITICAL_SECTION(cs_main); + c1.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); + + // Add a coin to the in-memory cache, upsize once, then downsize. + { + LOCK(::cs_main); + auto outpoint = add_coin(c1.CoinsTip()); + + // Set a meaningless bestblock value in the coinsview cache - otherwise we won't + // flush during ResizecoinsCaches() and will subsequently hit an assertion. + c1.CoinsTip().SetBestBlock(InsecureRand256()); + + BOOST_CHECK(c1.CoinsTip().HaveCoinInCache(outpoint)); + + c1.ResizeCoinsCaches( + 1 << 24, // upsizing the coinsview cache + 1 << 22 // downsizing the coinsdb cache + ); + + // View should still have the coin cached, since we haven't destructed the cache on upsize. + BOOST_CHECK(c1.CoinsTip().HaveCoinInCache(outpoint)); + + c1.ResizeCoinsCaches( + 1 << 22, // downsizing the coinsview cache + 1 << 23 // upsizing the coinsdb cache + ); + + // The view cache should be empty since we had to destruct to downsize. + BOOST_CHECK(!c1.CoinsTip().HaveCoinInCache(outpoint)); + } + + // Avoid triggering the address sanitizer. + WITH_LOCK(::cs_main, manager.Unload()); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 0d149285ad..f99191b08f 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -28,13 +28,11 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a legacy (IBD) chainstate. // - ENTER_CRITICAL_SECTION(cs_main); - CChainState& c1 = manager.InitializeChainstate(); - LEAVE_CRITICAL_SECTION(cs_main); + CChainState& c1 = *WITH_LOCK(::cs_main, return &manager.InitializeChainstate()); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); - WITH_LOCK(::cs_main, c1.InitCoinsCache()); + WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); BOOST_CHECK(!manager.IsSnapshotActive()); BOOST_CHECK(!manager.IsSnapshotValidated()); @@ -57,12 +55,13 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a snapshot-based chainstate. // ENTER_CRITICAL_SECTION(cs_main); - CChainState& c2 = manager.InitializeChainstate(GetRandHash()); + CChainState& c2 = *WITH_LOCK(::cs_main, + return &manager.InitializeChainstate(GetRandHash())); LEAVE_CRITICAL_SECTION(cs_main); chainstates.push_back(&c2); c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); - WITH_LOCK(::cs_main, c2.InitCoinsCache()); + WITH_LOCK(::cs_main, c2.InitCoinsCache(1 << 23)); // Unlike c1, which doesn't have any blocks. Gets us different tip, height. c2.LoadGenesisBlock(chainparams); BlockValidationState _; @@ -104,4 +103,58 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) WITH_LOCK(::cs_main, manager.Unload()); } +//! Test rebalancing the caches associated with each chainstate. +BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) +{ + ChainstateManager manager; + size_t max_cache = 10000; + manager.m_total_coinsdb_cache = max_cache; + manager.m_total_coinstip_cache = max_cache; + + std::vector<CChainState*> chainstates; + + // Create a legacy (IBD) chainstate. + // + ENTER_CRITICAL_SECTION(cs_main); + CChainState& c1 = manager.InitializeChainstate(); + LEAVE_CRITICAL_SECTION(cs_main); + chainstates.push_back(&c1); + c1.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + + { + LOCK(::cs_main); + c1.InitCoinsCache(1 << 23); + c1.CoinsTip().SetBestBlock(InsecureRand256()); + manager.MaybeRebalanceCaches(); + } + + BOOST_CHECK_EQUAL(c1.m_coinstip_cache_size_bytes, max_cache); + BOOST_CHECK_EQUAL(c1.m_coinsdb_cache_size_bytes, max_cache); + + // Create a snapshot-based chainstate. + // + ENTER_CRITICAL_SECTION(cs_main); + CChainState& c2 = manager.InitializeChainstate(GetRandHash()); + LEAVE_CRITICAL_SECTION(cs_main); + chainstates.push_back(&c2); + c2.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + + { + LOCK(::cs_main); + c2.InitCoinsCache(1 << 23); + c2.CoinsTip().SetBestBlock(InsecureRand256()); + manager.MaybeRebalanceCaches(); + } + + // Since both chainstates are considered to be in initial block download, + // the snapshot chainstate should take priority. + BOOST_CHECK_CLOSE(c1.m_coinstip_cache_size_bytes, max_cache * 0.05, 1); + BOOST_CHECK_CLOSE(c1.m_coinsdb_cache_size_bytes, max_cache * 0.05, 1); + BOOST_CHECK_CLOSE(c2.m_coinstip_cache_size_bytes, max_cache * 0.95, 1); + BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1); + +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index a863e3a4d5..7111fc3380 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -21,7 +21,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) BlockManager blockman{}; CChainState chainstate{blockman}; chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false); - WITH_LOCK(::cs_main, chainstate.InitCoinsCache()); + WITH_LOCK(::cs_main, chainstate.InitCoinsCache(1 << 10)); CTxMemPool tx_pool{}; constexpr bool is_64_bit = sizeof(void*) == 8; diff --git a/src/threadsafety.h b/src/threadsafety.h index 942aa3fdcd..5f2c40bac6 100644 --- a/src/threadsafety.h +++ b/src/threadsafety.h @@ -60,6 +60,13 @@ // and should only be used when sync.h Mutex/LOCK/etc are not usable. class LOCKABLE StdMutex : public std::mutex { +public: +#ifdef __clang__ + //! For negative capabilities in the Clang Thread Safety Analysis. + //! A negative requirement uses the EXCLUSIVE_LOCKS_REQUIRED attribute, in conjunction + //! with the ! operator, to indicate that a mutex should not be held. + const StdMutex& operator!() const { return *this; } +#endif // __clang__ }; // StdLockGuard provides an annotated version of std::lock_guard for us, diff --git a/src/timedata.cpp b/src/timedata.cpp index 16dac24a48..6b3a79017b 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -9,8 +9,8 @@ #include <timedata.h> #include <netaddress.h> +#include <node/ui_interface.h> #include <sync.h> -#include <ui_interface.h> #include <util/system.h> #include <util/translation.h> #include <warnings.h> diff --git a/src/txdb.cpp b/src/txdb.cpp index 6f652c1375..72460e7c69 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -5,11 +5,12 @@ #include <txdb.h> +#include <node/ui_interface.h> #include <pow.h> #include <random.h> #include <shutdown.h> -#include <ui_interface.h> #include <uint256.h> +#include <util/memory.h> #include <util/system.h> #include <util/translation.h> #include <util/vector.h> @@ -39,35 +40,45 @@ struct CoinEntry { } -CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) : db(ldb_path, nCacheSize, fMemory, fWipe, true) +CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) : + m_db(MakeUnique<CDBWrapper>(ldb_path, nCacheSize, fMemory, fWipe, true)), + m_ldb_path(ldb_path), + m_is_memory(fMemory) { } + +void CCoinsViewDB::ResizeCache(size_t new_cache_size) { + // Have to do a reset first to get the original `m_db` state to release its + // filesystem lock. + m_db.reset(); + m_db = MakeUnique<CDBWrapper>( + m_ldb_path, new_cache_size, m_is_memory, /*fWipe*/ false, /*obfuscate*/ true); } bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { - return db.Read(CoinEntry(&outpoint), coin); + return m_db->Read(CoinEntry(&outpoint), coin); } bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const { - return db.Exists(CoinEntry(&outpoint)); + return m_db->Exists(CoinEntry(&outpoint)); } uint256 CCoinsViewDB::GetBestBlock() const { uint256 hashBestChain; - if (!db.Read(DB_BEST_BLOCK, hashBestChain)) + if (!m_db->Read(DB_BEST_BLOCK, hashBestChain)) return uint256(); return hashBestChain; } std::vector<uint256> CCoinsViewDB::GetHeadBlocks() const { std::vector<uint256> vhashHeadBlocks; - if (!db.Read(DB_HEAD_BLOCKS, vhashHeadBlocks)) { + if (!m_db->Read(DB_HEAD_BLOCKS, vhashHeadBlocks)) { return std::vector<uint256>(); } return vhashHeadBlocks; } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { - CDBBatch batch(db); + CDBBatch batch(*m_db); size_t count = 0; size_t changed = 0; size_t batch_size = (size_t)gArgs.GetArg("-dbbatchsize", nDefaultDbBatchSize); @@ -105,7 +116,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { mapCoins.erase(itOld); if (batch.SizeEstimate() > batch_size) { LogPrint(BCLog::COINDB, "Writing partial batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0)); - db.WriteBatch(batch); + m_db->WriteBatch(batch); batch.Clear(); if (crash_simulate) { static FastRandomContext rng; @@ -122,14 +133,14 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { batch.Write(DB_BEST_BLOCK, hashBlock); LogPrint(BCLog::COINDB, "Writing final batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0)); - bool ret = db.WriteBatch(batch); + bool ret = m_db->WriteBatch(batch); LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); return ret; } size_t CCoinsViewDB::EstimateSize() const { - return db.EstimateSize(DB_COIN, (char)(DB_COIN+1)); + return m_db->EstimateSize(DB_COIN, (char)(DB_COIN+1)); } CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { @@ -156,7 +167,7 @@ bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { CCoinsViewCursor *CCoinsViewDB::Cursor() const { - CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast<CDBWrapper&>(db).NewIterator(), GetBestBlock()); + CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast<CDBWrapper&>(*m_db).NewIterator(), GetBestBlock()); /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ @@ -335,7 +346,7 @@ public: * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout. */ bool CCoinsViewDB::Upgrade() { - std::unique_ptr<CDBIterator> pcursor(db.NewIterator()); + std::unique_ptr<CDBIterator> pcursor(m_db->NewIterator()); pcursor->Seek(std::make_pair(DB_COINS, uint256())); if (!pcursor->Valid()) { return true; @@ -346,7 +357,7 @@ bool CCoinsViewDB::Upgrade() { LogPrintf("[0%%]..."); /* Continued */ uiInterface.ShowProgress(_("Upgrading UTXO database").translated, 0, true); size_t batch_size = 1 << 24; - CDBBatch batch(db); + CDBBatch batch(*m_db); int reportDone = 0; std::pair<unsigned char, uint256> key; std::pair<unsigned char, uint256> prev_key = {DB_COINS, uint256()}; @@ -380,9 +391,9 @@ bool CCoinsViewDB::Upgrade() { } batch.Erase(key); if (batch.SizeEstimate() > batch_size) { - db.WriteBatch(batch); + m_db->WriteBatch(batch); batch.Clear(); - db.CompactRange(prev_key, key); + m_db->CompactRange(prev_key, key); prev_key = key; } pcursor->Next(); @@ -390,8 +401,8 @@ bool CCoinsViewDB::Upgrade() { break; } } - db.WriteBatch(batch); - db.CompactRange({DB_COINS, uint256()}, key); + m_db->WriteBatch(batch); + m_db->CompactRange({DB_COINS, uint256()}, key); uiInterface.ShowProgress("", 100, false); LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE"); return !ShutdownRequested(); diff --git a/src/txdb.h b/src/txdb.h index 488c24f935..0cf7e2f1b8 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -39,11 +39,16 @@ static const int64_t max_filter_index_cache = 1024; //! Max memory allocated to coin DB specific cache (MiB) static const int64_t nMaxCoinsDBCache = 8; +// Actually declared in validation.cpp; can't include because of circular dependency. +extern RecursiveMutex cs_main; + /** CCoinsView backed by the coin database (chainstate/) */ class CCoinsViewDB final : public CCoinsView { protected: - CDBWrapper db; + std::unique_ptr<CDBWrapper> m_db; + fs::path m_ldb_path; + bool m_is_memory; public: /** * @param[in] ldb_path Location in the filesystem where leveldb data will be stored. @@ -60,6 +65,9 @@ public: //! Attempt to update from an older database format. Returns whether an error occurred. bool Upgrade(); size_t EstimateSize() const override; + + //! Dynamically alter the underlying leveldb cache size. + void ResizeCache(size_t new_cache_size) EXCLUSIVE_LOCKS_REQUIRED(cs_main); }; /** Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB */ diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 7d8eb8a323..1d9f6a4a46 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -726,12 +726,12 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const assert(innerUsage == cachedInnerUsage); } -bool CTxMemPool::CompareDepthAndScore(const uint256& hasha, const uint256& hashb) +bool CTxMemPool::CompareDepthAndScore(const uint256& hasha, const uint256& hashb, bool wtxid) { LOCK(cs); - indexed_transaction_set::const_iterator i = mapTx.find(hasha); + indexed_transaction_set::const_iterator i = wtxid ? get_iter_from_wtxid(hasha) : mapTx.find(hasha); if (i == mapTx.end()) return false; - indexed_transaction_set::const_iterator j = mapTx.find(hashb); + indexed_transaction_set::const_iterator j = wtxid ? get_iter_from_wtxid(hashb) : mapTx.find(hashb); if (j == mapTx.end()) return true; uint64_t counta = i->GetCountWithAncestors(); uint64_t countb = j->GetCountWithAncestors(); @@ -811,10 +811,10 @@ CTransactionRef CTxMemPool::get(const uint256& hash) const return i->GetSharedTx(); } -TxMempoolInfo CTxMemPool::info(const uint256& hash) const +TxMempoolInfo CTxMemPool::info(const uint256& hash, bool wtxid) const { LOCK(cs); - indexed_transaction_set::const_iterator i = mapTx.find(hash); + indexed_transaction_set::const_iterator i = (wtxid ? get_iter_from_wtxid(hash) : mapTx.find(hash)); if (i == mapTx.end()) return TxMempoolInfo(); return GetInfo(i); @@ -917,8 +917,8 @@ bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); - // Estimate the overhead of mapTx to be 12 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. - return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 12 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; + // Estimate the overhead of mapTx to be 15 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. + return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; } void CTxMemPool::RemoveUnbroadcastTx(const uint256& txid, const bool unchecked) { diff --git a/src/txmempool.h b/src/txmempool.h index 583f7614b7..d4e9845942 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -198,6 +198,22 @@ struct mempoolentry_txid } }; +// extracts a transaction witness-hash from CTxMemPoolEntry or CTransactionRef +struct mempoolentry_wtxid +{ + typedef uint256 result_type; + result_type operator() (const CTxMemPoolEntry &entry) const + { + return entry.GetTx().GetWitnessHash(); + } + + result_type operator() (const CTransactionRef& tx) const + { + return tx->GetWitnessHash(); + } +}; + + /** \class CompareTxMemPoolEntryByDescendantScore * * Sort an entry by max(score/size of entry's tx, score/size with all descendants). @@ -318,6 +334,7 @@ public: struct descendant_score {}; struct entry_time {}; struct ancestor_score {}; +struct index_by_wtxid {}; class CBlockPolicyEstimator; @@ -383,8 +400,9 @@ public: * * CTxMemPool::mapTx, and CTxMemPoolEntry bookkeeping: * - * mapTx is a boost::multi_index that sorts the mempool on 4 criteria: - * - transaction hash + * mapTx is a boost::multi_index that sorts the mempool on 5 criteria: + * - transaction hash (txid) + * - witness-transaction hash (wtxid) * - descendant feerate [we use max(feerate of tx, feerate of tx with all descendants)] * - time in mempool * - ancestor feerate [we use min(feerate of tx, feerate of tx with all unconfirmed ancestors)] @@ -469,6 +487,12 @@ public: boost::multi_index::indexed_by< // sorted by txid boost::multi_index::hashed_unique<mempoolentry_txid, SaltedTxidHasher>, + // sorted by wtxid + boost::multi_index::hashed_unique< + boost::multi_index::tag<index_by_wtxid>, + mempoolentry_wtxid, + SaltedTxidHasher + >, // sorted by fee rate boost::multi_index::ordered_non_unique< boost::multi_index::tag<descendant_score>, @@ -549,8 +573,11 @@ private: std::vector<indexed_transaction_set::const_iterator> GetSortedDepthAndScore() const EXCLUSIVE_LOCKS_REQUIRED(cs); - /** track locally submitted transactions to periodically retry initial broadcast */ - std::set<uint256> m_unbroadcast_txids GUARDED_BY(cs); + /** + * track locally submitted transactions to periodically retry initial broadcast + * map of txid -> wtxid + */ + std::map<uint256, uint256> m_unbroadcast_txids GUARDED_BY(cs); public: indirectmap<COutPoint, const CTransaction*> mapNextTx GUARDED_BY(cs); @@ -586,7 +613,7 @@ public: void clear(); void _clear() EXCLUSIVE_LOCKS_REQUIRED(cs); //lock free - bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); + bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb, bool wtxid=false); void queryHashes(std::vector<uint256>& vtxid) const; bool isSpent(const COutPoint& outpoint) const; unsigned int GetTransactionsUpdated() const; @@ -689,24 +716,32 @@ public: return totalTxSize; } - bool exists(const uint256& hash) const + bool exists(const uint256& hash, bool wtxid=false) const { LOCK(cs); + if (wtxid) { + return (mapTx.get<index_by_wtxid>().count(hash) != 0); + } return (mapTx.count(hash) != 0); } CTransactionRef get(const uint256& hash) const; - TxMempoolInfo info(const uint256& hash) const; + txiter get_iter_from_wtxid(const uint256& wtxid) const EXCLUSIVE_LOCKS_REQUIRED(cs) + { + AssertLockHeld(cs); + return mapTx.project<0>(mapTx.get<index_by_wtxid>().find(wtxid)); + } + TxMempoolInfo info(const uint256& hash, bool wtxid=false) const; std::vector<TxMempoolInfo> infoAll() const; size_t DynamicMemoryUsage() const; /** Adds a transaction to the unbroadcast set */ - void AddUnbroadcastTx(const uint256& txid) { + void AddUnbroadcastTx(const uint256& txid, const uint256& wtxid) { LOCK(cs); // Sanity Check: the transaction should also be in the mempool if (exists(txid)) { - m_unbroadcast_txids.insert(txid); + m_unbroadcast_txids[txid] = wtxid; } } @@ -714,7 +749,7 @@ public: void RemoveUnbroadcastTx(const uint256& txid, const bool unchecked = false); /** Returns transactions in unbroadcast set */ - std::set<uint256> GetUnbroadcastTxs() const { + std::map<uint256, uint256> GetUnbroadcastTxs() const { LOCK(cs); return m_unbroadcast_txids; } diff --git a/src/util/check.h b/src/util/check.h index 5c0f32cf51..9edf394492 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -25,7 +25,7 @@ class NonFatalCheckError : public std::runtime_error * - where the condition is assumed to be true, not for error handling or validating user input * - where a failure to fulfill the condition is recoverable and does not abort the program * - * For example in RPC code, where it is undersirable to crash the whole program, this can be generally used to replace + * For example in RPC code, where it is undesirable to crash the whole program, this can be generally used to replace * asserts or recoverable logic errors. A NonFatalCheckError in RPC code is caught and passed as a string to the RPC * caller, which can then report the issue to the developers. */ @@ -42,4 +42,18 @@ class NonFatalCheckError : public std::runtime_error } \ } while (false) +#if defined(NDEBUG) +#error "Cannot compile without assertions!" +#endif + +/** Helper for Assert(). TODO remove in C++14 and replace `decltype(get_pure_r_value(val))` with `T` (templated lambda) */ +template <typename T> +T get_pure_r_value(T&& val) +{ + return std::forward<T>(val); +} + +/** Identity function. Abort if the value compares equal to zero */ +#define Assert(val) [&]() -> decltype(get_pure_r_value(val)) { auto&& check = (val); assert(#val && check); return std::forward<decltype(get_pure_r_value(val))>(check); }() + #endif // BITCOIN_UTIL_CHECK_H diff --git a/src/util/error.cpp b/src/util/error.cpp index c4d9ffd037..3e29083712 100644 --- a/src/util/error.cpp +++ b/src/util/error.cpp @@ -14,7 +14,7 @@ bilingual_str TransactionErrorString(const TransactionError err) case TransactionError::OK: return Untranslated("No error"); case TransactionError::MISSING_INPUTS: - return Untranslated("Missing inputs"); + return Untranslated("Inputs missing or spent"); case TransactionError::ALREADY_IN_CHAIN: return Untranslated("Transaction already in block chain"); case TransactionError::P2P_DISABLED: @@ -24,11 +24,11 @@ bilingual_str TransactionErrorString(const TransactionError err) case TransactionError::MEMPOOL_ERROR: return Untranslated("AcceptToMemoryPool failed"); case TransactionError::INVALID_PSBT: - return Untranslated("PSBT is not sane"); + return Untranslated("PSBT is not well-formed"); case TransactionError::PSBT_MISMATCH: return Untranslated("PSBTs not compatible (different transactions)"); case TransactionError::SIGHASH_MISMATCH: - return Untranslated("Specified sighash value does not match existing value"); + return Untranslated("Specified sighash value does not match value stored in PSBT"); case TransactionError::MAX_FEE_EXCEEDED: return Untranslated("Fee exceeds maximum configured by -maxtxfee"); // no default case, so the compiler can warn about missing cases diff --git a/src/util/fees.cpp b/src/util/fees.cpp index b335bfa666..6208a20a97 100644 --- a/src/util/fees.cpp +++ b/src/util/fees.cpp @@ -6,11 +6,16 @@ #include <util/fees.h> #include <policy/fees.h> +#include <util/strencodings.h> +#include <util/string.h> #include <map> #include <string> +#include <vector> +#include <utility> -std::string StringForFeeReason(FeeReason reason) { +std::string StringForFeeReason(FeeReason reason) +{ static const std::map<FeeReason, std::string> fee_reason_strings = { {FeeReason::NONE, "None"}, {FeeReason::HALF_ESTIMATE, "Half Target 60% Threshold"}, @@ -29,16 +34,31 @@ std::string StringForFeeReason(FeeReason reason) { return reason_string->second; } -bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode) { - static const std::map<std::string, FeeEstimateMode> fee_modes = { - {"UNSET", FeeEstimateMode::UNSET}, - {"ECONOMICAL", FeeEstimateMode::ECONOMICAL}, - {"CONSERVATIVE", FeeEstimateMode::CONSERVATIVE}, +const std::vector<std::pair<std::string, FeeEstimateMode>>& FeeModeMap() +{ + static const std::vector<std::pair<std::string, FeeEstimateMode>> FEE_MODES = { + {"unset", FeeEstimateMode::UNSET}, + {"economical", FeeEstimateMode::ECONOMICAL}, + {"conservative", FeeEstimateMode::CONSERVATIVE}, + {(CURRENCY_UNIT + "/kB"), FeeEstimateMode::BTC_KB}, + {(CURRENCY_ATOM + "/B"), FeeEstimateMode::SAT_B}, }; - auto mode = fee_modes.find(mode_string); + return FEE_MODES; +} - if (mode == fee_modes.end()) return false; +std::string FeeModes(const std::string& delimiter) +{ + return Join(FeeModeMap(), delimiter, [&](const std::pair<std::string, FeeEstimateMode>& i) { return i.first; }); +} - fee_estimate_mode = mode->second; - return true; +bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode) +{ + auto searchkey = ToUpper(mode_string); + for (const auto& pair : FeeModeMap()) { + if (ToUpper(pair.first) == searchkey) { + fee_estimate_mode = pair.second; + return true; + } + } + return false; } diff --git a/src/util/fees.h b/src/util/fees.h index a930c8935a..d52046a44c 100644 --- a/src/util/fees.h +++ b/src/util/fees.h @@ -12,5 +12,6 @@ enum class FeeReason; bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode); std::string StringForFeeReason(FeeReason reason); +std::string FeeModes(const std::string& delimiter); #endif // BITCOIN_UTIL_FEES_H diff --git a/src/util/settings.cpp b/src/util/settings.cpp index e4979df755..b92b1d30c3 100644 --- a/src/util/settings.cpp +++ b/src/util/settings.cpp @@ -4,6 +4,7 @@ #include <util/settings.h> +#include <tinyformat.h> #include <univalue.h> namespace util { @@ -12,12 +13,13 @@ namespace { enum class Source { FORCED, COMMAND_LINE, + RW_SETTINGS, CONFIG_FILE_NETWORK_SECTION, CONFIG_FILE_DEFAULT_SECTION }; //! Merge settings from multiple sources in precedence order: -//! Forced config > command line > config file network-specific section > config file default section +//! Forced config > command line > read-write settings file > config file network-specific section > config file default section //! //! This function is provided with a callback function fn that contains //! specific logic for how to merge the sources. @@ -32,6 +34,10 @@ static void MergeSettings(const Settings& settings, const std::string& section, if (auto* values = FindKey(settings.command_line_options, name)) { fn(SettingsSpan(*values), Source::COMMAND_LINE); } + // Merge in the read-write settings + if (const SettingsValue* value = FindKey(settings.rw_settings, name)) { + fn(SettingsSpan(*value), Source::RW_SETTINGS); + } // Merge in the network-specific section of the config file if (!section.empty()) { if (auto* map = FindKey(settings.ro_config, section)) { @@ -49,6 +55,62 @@ static void MergeSettings(const Settings& settings, const std::string& section, } } // namespace +bool ReadSettings(const fs::path& path, std::map<std::string, SettingsValue>& values, std::vector<std::string>& errors) +{ + values.clear(); + errors.clear(); + + fsbridge::ifstream file; + file.open(path); + if (!file.is_open()) return true; // Ok for file not to exist. + + SettingsValue in; + if (!in.read(std::string{std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>()})) { + errors.emplace_back(strprintf("Unable to parse settings file %s", path.string())); + return false; + } + + if (file.fail()) { + errors.emplace_back(strprintf("Failed reading settings file %s", path.string())); + return false; + } + file.close(); // Done with file descriptor. Release while copying data. + + if (!in.isObject()) { + errors.emplace_back(strprintf("Found non-object value %s in settings file %s", in.write(), path.string())); + return false; + } + + const std::vector<std::string>& in_keys = in.getKeys(); + const std::vector<SettingsValue>& in_values = in.getValues(); + for (size_t i = 0; i < in_keys.size(); ++i) { + auto inserted = values.emplace(in_keys[i], in_values[i]); + if (!inserted.second) { + errors.emplace_back(strprintf("Found duplicate key %s in settings file %s", in_keys[i], path.string())); + } + } + return errors.empty(); +} + +bool WriteSettings(const fs::path& path, + const std::map<std::string, SettingsValue>& values, + std::vector<std::string>& errors) +{ + SettingsValue out(SettingsValue::VOBJ); + for (const auto& value : values) { + out.__pushKV(value.first, value.second); + } + fsbridge::ofstream file; + file.open(path); + if (file.fail()) { + errors.emplace_back(strprintf("Error: Unable to open settings file %s for writing", path.string())); + return false; + } + file << out.write(/* prettyIndent= */ 1, /* indentLevel= */ 4) << std::endl; + file.close(); + return true; +} + SettingsValue GetSetting(const Settings& settings, const std::string& section, const std::string& name, diff --git a/src/util/settings.h b/src/util/settings.h index 1d03639fa2..ed36349232 100644 --- a/src/util/settings.h +++ b/src/util/settings.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_UTIL_SETTINGS_H #define BITCOIN_UTIL_SETTINGS_H +#include <fs.h> + #include <map> #include <string> #include <vector> @@ -24,19 +26,31 @@ namespace util { //! https://github.com/bitcoin/bitcoin/pull/15934/files#r337691812) using SettingsValue = UniValue; -//! Stored bitcoin settings. This struct combines settings from the command line -//! and a read-only configuration file. +//! Stored settings. This struct combines settings from the command line, a +//! read-only configuration file, and a read-write runtime settings file. struct Settings { //! Map of setting name to forced setting value. std::map<std::string, SettingsValue> forced_settings; //! Map of setting name to list of command line values. std::map<std::string, std::vector<SettingsValue>> command_line_options; + //! Map of setting name to read-write file setting value. + std::map<std::string, SettingsValue> rw_settings; //! Map of config section name and setting name to list of config file values. std::map<std::string, std::map<std::string, std::vector<SettingsValue>>> ro_config; }; +//! Read settings file. +bool ReadSettings(const fs::path& path, + std::map<std::string, SettingsValue>& values, + std::vector<std::string>& errors); + +//! Write settings file. +bool WriteSettings(const fs::path& path, + const std::map<std::string, SettingsValue>& values, + std::vector<std::string>& errors); + //! Get settings value from combined sources: forced settings, command line -//! arguments and the read-only config file. +//! arguments, runtime read-write settings, and the read-only config file. //! //! @param ignore_default_section_config - ignore values in the default section //! of the config file (part before any diff --git a/src/util/system.cpp b/src/util/system.cpp index 7e7ba840cd..8164e884b1 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -73,6 +73,7 @@ const int64_t nStartupTime = GetTime(); const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; +const char * const BITCOIN_SETTINGS_FILENAME = "settings.json"; ArgsManager gArgs; @@ -372,6 +373,84 @@ bool ArgsManager::IsArgSet(const std::string& strArg) const return !GetSetting(strArg).isNull(); } +bool ArgsManager::InitSettings(std::string& error) +{ + if (!GetSettingsPath()) { + return true; // Do nothing if settings file disabled. + } + + std::vector<std::string> errors; + if (!ReadSettingsFile(&errors)) { + error = strprintf("Failed loading settings file:\n- %s\n", Join(errors, "\n- ")); + return false; + } + if (!WriteSettingsFile(&errors)) { + error = strprintf("Failed saving settings file:\n- %s\n", Join(errors, "\n- ")); + return false; + } + return true; +} + +bool ArgsManager::GetSettingsPath(fs::path* filepath, bool temp) const +{ + if (IsArgNegated("-settings")) { + return false; + } + if (filepath) { + std::string settings = GetArg("-settings", BITCOIN_SETTINGS_FILENAME); + *filepath = fs::absolute(temp ? settings + ".tmp" : settings, GetDataDir(/* net_specific= */ true)); + } + return true; +} + +static void SaveErrors(const std::vector<std::string> errors, std::vector<std::string>* error_out) +{ + for (const auto& error : errors) { + if (error_out) { + error_out->emplace_back(error); + } else { + LogPrintf("%s\n", error); + } + } +} + +bool ArgsManager::ReadSettingsFile(std::vector<std::string>* errors) +{ + fs::path path; + if (!GetSettingsPath(&path, /* temp= */ false)) { + return true; // Do nothing if settings file disabled. + } + + LOCK(cs_args); + m_settings.rw_settings.clear(); + std::vector<std::string> read_errors; + if (!util::ReadSettings(path, m_settings.rw_settings, read_errors)) { + SaveErrors(read_errors, errors); + return false; + } + return true; +} + +bool ArgsManager::WriteSettingsFile(std::vector<std::string>* errors) const +{ + fs::path path, path_tmp; + if (!GetSettingsPath(&path, /* temp= */ false) || !GetSettingsPath(&path_tmp, /* temp= */ true)) { + throw std::logic_error("Attempt to write settings file when dynamic settings are disabled."); + } + + LOCK(cs_args); + std::vector<std::string> write_errors; + if (!util::WriteSettings(path_tmp, m_settings.rw_settings, write_errors)) { + SaveErrors(write_errors, errors); + return false; + } + if (!RenameOver(path_tmp, path)) { + SaveErrors({strprintf("Failed renaming settings file %s to %s\n", path_tmp.string(), path.string())}, errors); + return false; + } + return true; +} + bool ArgsManager::IsArgNegated(const std::string& strArg) const { return GetSetting(strArg).isFalse(); @@ -893,6 +972,9 @@ void ArgsManager::LogArgs() const for (const auto& section : m_settings.ro_config) { logArgsPrefix("Config file arg:", section.first, section.second); } + for (const auto& setting : m_settings.rw_settings) { + LogPrintf("Setting file arg: %s = %s\n", setting.first, setting.second.write()); + } logArgsPrefix("Command-line arg:", "", m_settings.command_line_options); } diff --git a/src/util/system.h b/src/util/system.h index a5eea5dfab..0bd14cc9ea 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -41,6 +41,7 @@ int64_t GetStartupTime(); extern const char * const BITCOIN_CONF_FILENAME; +extern const char * const BITCOIN_SETTINGS_FILENAME; void SetupEnvironment(); bool SetupNetworking(); @@ -334,6 +335,39 @@ public: Optional<unsigned int> GetArgFlags(const std::string& name) const; /** + * Read and update settings file with saved settings. This needs to be + * called after SelectParams() because the settings file location is + * network-specific. + */ + bool InitSettings(std::string& error); + + /** + * Get settings file path, or return false if read-write settings were + * disabled with -nosettings. + */ + bool GetSettingsPath(fs::path* filepath = nullptr, bool temp = false) const; + + /** + * Read settings file. Push errors to vector, or log them if null. + */ + bool ReadSettingsFile(std::vector<std::string>* errors = nullptr); + + /** + * Write settings file. Push errors to vector, or log them if null. + */ + bool WriteSettingsFile(std::vector<std::string>* errors = nullptr) const; + + /** + * Access settings with lock held. + */ + template <typename Fn> + void LockSettings(Fn&& fn) + { + LOCK(cs_args); + fn(m_settings); + } + + /** * Log the config file options and the command line arguments, * useful for troubleshooting. */ diff --git a/src/util/time.h b/src/util/time.h index b00c25f67c..af934e423b 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -15,10 +15,15 @@ void UninterruptibleSleep(const std::chrono::microseconds& n); /** * Helper to count the seconds of a duration. * - * All durations should be using std::chrono and calling this should generally be avoided in code. Though, it is still - * preferred to an inline t.count() to protect against a reliance on the exact type of t. + * All durations should be using std::chrono and calling this should generally + * be avoided in code. Though, it is still preferred to an inline t.count() to + * protect against a reliance on the exact type of t. + * + * This helper is used to convert durations before passing them over an + * interface that doesn't support std::chrono (e.g. RPC, debug log, or the GUI) */ inline int64_t count_seconds(std::chrono::seconds t) { return t.count(); } +inline int64_t count_microseconds(std::chrono::microseconds t) { return t.count(); } /** * DEPRECATED diff --git a/src/util/ui_change_type.h b/src/util/ui_change_type.h new file mode 100644 index 0000000000..1db761a18d --- /dev/null +++ b/src/util/ui_change_type.h @@ -0,0 +1,15 @@ +// Copyright (c) 2012-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_UI_CHANGE_TYPE_H +#define BITCOIN_UTIL_UI_CHANGE_TYPE_H + +/** General change type (added, updated, removed). */ +enum ChangeType { + CT_NEW, + CT_UPDATED, + CT_DELETED +}; + +#endif // BITCOIN_UTIL_UI_CHANGE_TYPE_H diff --git a/src/validation.cpp b/src/validation.cpp index cbe89443b8..545aa0f2d3 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -20,6 +20,7 @@ #include <index/txindex.h> #include <logging.h> #include <logging/timer.h> +#include <node/ui_interface.h> #include <optional.h> #include <policy/fees.h> #include <policy/policy.h> @@ -36,9 +37,9 @@ #include <tinyformat.h> #include <txdb.h> #include <txmempool.h> -#include <ui_interface.h> #include <uint256.h> #include <undo.h> +#include <util/check.h> // For NDEBUG compile time check #include <util/moneystr.h> #include <util/rbf.h> #include <util/strencodings.h> @@ -51,10 +52,6 @@ #include <boost/algorithm/string/replace.hpp> -#if defined(NDEBUG) -# error "Bitcoin cannot be compiled without assertions." -#endif - #define MICRO 0.000001 #define MILLI 0.001 @@ -142,7 +139,6 @@ bool fPruneMode = false; bool fRequireStandard = true; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; -size_t nCoinCacheUsage = 5000 * 300; uint64_t nPruneTarget = 0; int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE; @@ -576,8 +572,9 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) CAmount& nConflictingFees = ws.m_conflicting_fees; size_t& nConflictingSize = ws.m_conflicting_size; - if (!CheckTransaction(tx, state)) + if (!CheckTransaction(tx, state)) { return false; // state filled in by CheckTransaction + } // Coinbase is only valid in a block, not as a loose transaction if (tx.IsCoinBase()) @@ -687,7 +684,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) CAmount nFees = 0; if (!Consensus::CheckTxInputs(tx, state, m_view, GetSpendHeight(m_view), nFees)) { - return error("%s: Consensus::CheckTxInputs: %s, %s", __func__, tx.GetHash().ToString(), state.ToString()); + return false; // state filled in by CheckTxInputs } // Check for non-standard pay-to-script-hash in inputs @@ -941,7 +938,7 @@ bool MemPoolAccept::PolicyScriptChecks(ATMPArgs& args, Workspace& ws, Precompute if (!tx.HasWitness() && CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, false, txdata) && !CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, false, txdata)) { // Only the witness is missing, so the transaction itself may be fine. - state.Invalid(TxValidationResult::TX_WITNESS_MUTATED, + state.Invalid(TxValidationResult::TX_WITNESS_STRIPPED, state.GetRejectReason(), state.GetDebugMessage()); } return false; // state filled in by CheckInputScripts @@ -1091,45 +1088,33 @@ bool AcceptToMemoryPool(CTxMemPool& pool, TxValidationState &state, const CTrans return AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, GetTime(), plTxnReplaced, bypass_limits, nAbsurdFee, test_accept); } -/** - * Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock. - * If blockIndex is provided, the transaction is fetched from the corresponding block. - */ -bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus::Params& consensusParams, uint256& hashBlock, const CBlockIndex* const block_index) +CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock) { LOCK(cs_main); - if (!block_index) { - CTransactionRef ptx = mempool.get(hash); - if (ptx) { - txOut = ptx; - return true; - } - - if (g_txindex) { - return g_txindex->FindTx(hash, hashBlock, txOut); - } - } else { + if (block_index) { CBlock block; if (ReadBlockFromDisk(block, block_index, consensusParams)) { for (const auto& tx : block.vtx) { if (tx->GetHash() == hash) { - txOut = tx; hashBlock = block_index->GetBlockHash(); - return true; + return tx; } } } + return nullptr; } - - return false; + if (mempool) { + CTransactionRef ptx = mempool->get(hash); + if (ptx) return ptx; + } + if (g_txindex) { + CTransactionRef tx; + if (g_txindex->FindTx(hash, hashBlock, tx)) return tx; + } + return nullptr; } - - - - - ////////////////////////////////////////////////////////////////////////////// // // CBlock and CBlockIndex @@ -1286,9 +1271,10 @@ void CChainState::InitCoinsDB( leveldb_name, cache_size_bytes, in_memory, should_wipe); } -void CChainState::InitCoinsCache() +void CChainState::InitCoinsCache(size_t cache_size_bytes) { assert(m_coins_views != nullptr); + m_coinstip_cache_size_bytes = cache_size_bytes; m_coins_views->InitCache(); } @@ -1321,12 +1307,6 @@ bool CChainState::IsInitialBlockDownload() const static CBlockIndex *pindexBestForkTip = nullptr, *pindexBestForkBase = nullptr; -BlockMap& BlockIndex() -{ - LOCK(::cs_main); - return g_chainman.m_blockman.m_block_index; -} - static void AlertNotify(const std::string& strMessage) { uiInterface.NotifyAlertChanged(); @@ -1430,12 +1410,12 @@ void static InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(c pindexBestHeader = ::ChainActive().Tip(); } - LogPrintf("%s: invalid block=%s height=%d log2_work=%.8g date=%s\n", __func__, + LogPrintf("%s: invalid block=%s height=%d log2_work=%f date=%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, log(pindexNew->nChainWork.getdouble())/log(2.0), FormatISO8601DateTime(pindexNew->GetBlockTime())); CBlockIndex *tip = ::ChainActive().Tip(); assert (tip); - LogPrintf("%s: current best=%s height=%d log2_work=%.8g date=%s\n", __func__, + LogPrintf("%s: current best=%s height=%d log2_work=%f date=%s\n", __func__, tip->GetBlockHash().ToString(), ::ChainActive().Height(), log(tip->nChainWork.getdouble())/log(2.0), FormatISO8601DateTime(tip->GetBlockTime())); CheckForkWarningConditions(); @@ -1663,23 +1643,21 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) } /** Abort with a message */ -// TODO: AbortNode() should take bilingual_str userMessage parameter. -static bool AbortNode(const std::string& strMessage, const std::string& userMessage = "", unsigned int prefix = 0) +static bool AbortNode(const std::string& strMessage, bilingual_str user_message = bilingual_str()) { SetMiscWarning(Untranslated(strMessage)); LogPrintf("*** %s\n", strMessage); - if (!userMessage.empty()) { - uiInterface.ThreadSafeMessageBox(Untranslated(userMessage), "", CClientUIInterface::MSG_ERROR | prefix); - } else { - uiInterface.ThreadSafeMessageBox(_("Error: A fatal internal error occurred, see debug.log for details"), "", CClientUIInterface::MSG_ERROR | CClientUIInterface::MSG_NOPREFIX); + if (user_message.empty()) { + user_message = _("A fatal internal error occurred, see debug.log for details"); } + AbortError(user_message); StartShutdown(); return false; } -static bool AbortNode(BlockValidationState& state, const std::string& strMessage, const std::string& userMessage = "", unsigned int prefix = 0) +static bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = bilingual_str()) { - AbortNode(strMessage, userMessage, prefix); + AbortNode(strMessage, userMessage); return state.Error(strMessage); } @@ -2253,7 +2231,7 @@ CoinsCacheSizeState CChainState::GetCoinsCacheSizeState(const CTxMemPool& tx_poo { return this->GetCoinsCacheSizeState( tx_pool, - nCoinCacheUsage, + m_coinstip_cache_size_bytes, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); } @@ -2344,7 +2322,7 @@ bool CChainState::FlushStateToDisk( if (fDoFullFlush || fPeriodicWrite) { // Depend on nMinDiskSpace to ensure we can write block index if (!CheckDiskSpace(GetBlocksDir())) { - return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); + return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); } { LOG_TIME_MILLIS_WITH_CATEGORY("write block and undo data to disk", BCLog::BENCH); @@ -2392,7 +2370,7 @@ bool CChainState::FlushStateToDisk( // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. if (!CheckDiskSpace(GetDataDir(), 48 * 2 * 2 * CoinsTip().GetCacheSize())) { - return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); + return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); } // Flush the chainstate (which may refer to block index entries). if (!CoinsTip().Flush()) @@ -2487,7 +2465,7 @@ void static UpdateTip(const CBlockIndex* pindexNew, const CChainParams& chainPar if (nUpgraded > 0) AppendWarning(warning_messages, strprintf(_("%d of last 100 blocks have unexpected version"), nUpgraded)); } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n", __func__, + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, log(pindexNew->nChainWork.getdouble())/log(2.0), (unsigned long)pindexNew->nChainTx, FormatISO8601DateTime(pindexNew->GetBlockTime()), @@ -3299,7 +3277,7 @@ static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int n bool out_of_space; size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); if (out_of_space) { - return AbortNode("Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); + return AbortNode("Disk space is too low!", _("Disk space is too low!")); } if (bytes_allocated != 0 && fPruneMode) { fCheckForPruning = true; @@ -3323,7 +3301,7 @@ static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos bool out_of_space; size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); if (out_of_space) { - return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); + return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); } if (bytes_allocated != 0 && fPruneMode) { fCheckForPruning = true; @@ -3651,8 +3629,10 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS return true; } - if (!CheckBlockHeader(block, state, chainparams.GetConsensus())) - return error("%s: Consensus::CheckBlockHeader: %s, %s", __func__, hash.ToString(), state.ToString()); + if (!CheckBlockHeader(block, state, chainparams.GetConsensus())) { + LogPrint(BCLog::VALIDATION, "%s: Consensus::CheckBlockHeader: %s, %s\n", __func__, hash.ToString(), state.ToString()); + return false; + } // Get prev block index CBlockIndex* pindexPrev = nullptr; @@ -4326,7 +4306,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, } } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks - if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + ::ChainstateActive().CoinsTip().DynamicMemoryUsage()) <= nCoinCacheUsage) { + if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + ::ChainstateActive().CoinsTip().DynamicMemoryUsage()) <= ::ChainstateActive().m_coinstip_cache_size_bytes) { assert(coins.GetBestBlock() == pindex->GetBlockHash()); DisconnectResult res = ::ChainstateActive().DisconnectBlock(block, pindex, coins); if (res == DISCONNECT_FAILED) { @@ -4718,7 +4698,6 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi if (dbp) dbp->nPos = nBlockPos; blkdat.SetLimit(nBlockPos + nSize); - blkdat.SetPos(nBlockPos); std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); CBlock& block = *pblock; blkdat >> block; @@ -4990,6 +4969,39 @@ std::string CChainState::ToString() tip ? tip->nHeight : -1, tip ? tip->GetBlockHash().ToString() : "null"); } +bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) +{ + if (coinstip_size == m_coinstip_cache_size_bytes && + coinsdb_size == m_coinsdb_cache_size_bytes) { + // Cache sizes are unchanged, no need to continue. + return true; + } + size_t old_coinstip_size = m_coinstip_cache_size_bytes; + m_coinstip_cache_size_bytes = coinstip_size; + m_coinsdb_cache_size_bytes = coinsdb_size; + CoinsDB().ResizeCache(coinsdb_size); + + LogPrintf("[%s] resized coinsdb cache to %.1f MiB\n", + this->ToString(), coinsdb_size * (1.0 / 1024 / 1024)); + LogPrintf("[%s] resized coinstip cache to %.1f MiB\n", + this->ToString(), coinstip_size * (1.0 / 1024 / 1024)); + + BlockValidationState state; + const CChainParams& chainparams = Params(); + + bool ret; + + if (coinstip_size > old_coinstip_size) { + // Likely no need to flush if cache sizes have grown. + ret = FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED); + } else { + // Otherwise, flush state to disk and deallocate the in-memory coins map. + ret = FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS); + CoinsTip().ReallocateCache(); + } + return ret; +} + std::string CBlockFileInfo::ToString() const { return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); @@ -5093,19 +5105,22 @@ bool LoadMempool(CTxMemPool& pool) } // TODO: remove this try except in v0.22 + std::map<uint256, uint256> unbroadcast_txids; try { - std::set<uint256> unbroadcast_txids; file >> unbroadcast_txids; unbroadcast = unbroadcast_txids.size(); - - for (const auto& txid : unbroadcast_txids) { - pool.AddUnbroadcastTx(txid); - } } catch (const std::exception&) { // mempool.dat files created prior to v0.21 will not have an // unbroadcast set. No need to log a failure if parsing fails here. } - + for (const auto& elem : unbroadcast_txids) { + // Don't add unbroadcast transactions that didn't get back into the + // mempool. + const CTransactionRef& added_tx = pool.get(elem.first); + if (added_tx != nullptr) { + pool.AddUnbroadcastTx(elem.first, added_tx->GetWitnessHash()); + } + } } catch (const std::exception& e) { LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); return false; @@ -5121,7 +5136,7 @@ bool DumpMempool(const CTxMemPool& pool) std::map<uint256, CAmount> mapDeltas; std::vector<TxMempoolInfo> vinfo; - std::set<uint256> unbroadcast_txids; + std::map<uint256, uint256> unbroadcast_txids; static Mutex dump_mutex; LOCK(dump_mutex); @@ -5253,10 +5268,10 @@ CChainState& ChainstateManager::InitializeChainstate(const uint256& snapshot_blo return *to_modify; } -CChain& ChainstateManager::ActiveChain() const +CChainState& ChainstateManager::ActiveChainstate() const { assert(m_active_chainstate); - return m_active_chainstate->m_chain; + return *m_active_chainstate; } bool ChainstateManager::IsSnapshotActive() const @@ -5295,3 +5310,33 @@ void ChainstateManager::Reset() m_active_chainstate = nullptr; m_snapshot_validated = false; } + +void ChainstateManager::MaybeRebalanceCaches() +{ + if (m_ibd_chainstate && !m_snapshot_chainstate) { + LogPrintf("[snapshot] allocating all cache to the IBD chainstate\n"); + // Allocate everything to the IBD chainstate. + m_ibd_chainstate->ResizeCoinsCaches(m_total_coinstip_cache, m_total_coinsdb_cache); + } + else if (m_snapshot_chainstate && !m_ibd_chainstate) { + LogPrintf("[snapshot] allocating all cache to the snapshot chainstate\n"); + // Allocate everything to the snapshot chainstate. + m_snapshot_chainstate->ResizeCoinsCaches(m_total_coinstip_cache, m_total_coinsdb_cache); + } + else if (m_ibd_chainstate && m_snapshot_chainstate) { + // If both chainstates exist, determine who needs more cache based on IBD status. + // + // Note: shrink caches first so that we don't inadvertently overwhelm available memory. + if (m_snapshot_chainstate->IsInitialBlockDownload()) { + m_ibd_chainstate->ResizeCoinsCaches( + m_total_coinstip_cache * 0.05, m_total_coinsdb_cache * 0.05); + m_snapshot_chainstate->ResizeCoinsCaches( + m_total_coinstip_cache * 0.95, m_total_coinsdb_cache * 0.95); + } else { + m_snapshot_chainstate->ResizeCoinsCaches( + m_total_coinstip_cache * 0.05, m_total_coinsdb_cache * 0.05); + m_ibd_chainstate->ResizeCoinsCaches( + m_total_coinstip_cache * 0.95, m_total_coinsdb_cache * 0.95); + } + } +} diff --git a/src/validation.h b/src/validation.h index e403bcb51a..53503768aa 100644 --- a/src/validation.h +++ b/src/validation.h @@ -74,7 +74,6 @@ static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; -static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; /** Default for -persistmempool */ static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for using fee filter */ @@ -128,7 +127,6 @@ extern bool g_parallel_script_checks; extern bool fRequireStandard; extern bool fCheckBlockIndex; extern bool fCheckpointsEnabled; -extern size_t nCoinCacheUsage; /** A fee rate smaller than this is considered zero fee (for relaying, mining and transaction creation) */ extern CFeeRate minRelayTxFee; /** If the tip is older than this (in seconds), the node is considered to be in initial block download. */ @@ -165,8 +163,19 @@ bool LoadGenesisBlock(const CChainParams& chainparams); void UnloadBlockIndex(); /** Run an instance of the script checking thread */ void ThreadScriptCheck(int worker_num); -/** Retrieve a transaction (from memory pool, or from disk, if possible) */ -bool GetTransaction(const uint256& hash, CTransactionRef& tx, const Consensus::Params& params, uint256& hashBlock, const CBlockIndex* const blockIndex = nullptr); +/** + * Return transaction from the block at block_index. + * If block_index is not provided, fall back to mempool. + * If mempool is not provided or the tx couldn't be found in mempool, fall back to g_txindex. + * + * @param[in] block_index The block to read from disk, or nullptr + * @param[in] mempool If block_index is not provided, look in the mempool, if provided + * @param[in] hash The txid + * @param[in] consensusParams The params + * @param[out] hashBlock The hash of block_index, if the tx was found via block_index + * @returns The tx if found, otherwise nullptr + */ +CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock); /** * Find the best known block, and make it the tip of the block chain * @@ -522,7 +531,7 @@ public: //! Initialize the in-memory coins cache (to be done after the health of the on-disk database //! is verified). - void InitCoinsCache() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + void InitCoinsCache(size_t cache_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); //! @returns whether or not the CoinsViews object has been fully initialized and we can //! safely flush this object to disk. @@ -571,6 +580,17 @@ public: //! Destructs all objects related to accessing the UTXO set. void ResetCoinsViews() { m_coins_views.reset(); } + //! The cache size of the on-disk coins view. + size_t m_coinsdb_cache_size_bytes{0}; + + //! The cache size of the in-memory coins view. + size_t m_coinstip_cache_size_bytes{0}; + + //! Resize the CoinsViews caches dynamically and flush state to disk. + //! @returns true unless an error occurred during the flush. + bool ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** * Update the on-disk chain state. * The caches and indexes are flushed depending on the mode we're called with @@ -787,6 +807,14 @@ public: //! chainstate to avoid duplicating block metadata. BlockManager m_blockman GUARDED_BY(::cs_main); + //! The total number of bytes available for us to use across all in-memory + //! coins caches. This will be split somehow across chainstates. + int64_t m_total_coinstip_cache{0}; + // + //! The total number of bytes available for us to use across all leveldb + //! coins databases. This will be split somehow across chainstates. + int64_t m_total_coinsdb_cache{0}; + //! Instantiate a new chainstate and assign it based upon whether it is //! from a snapshot. //! @@ -799,7 +827,8 @@ public: std::vector<CChainState*> GetAll(); //! The most-work chain. - CChain& ActiveChain() const; + CChainState& ActiveChainstate() const; + CChain& ActiveChain() const { return ActiveChainstate().m_chain; } int ActiveHeight() const { return ActiveChain().Height(); } CBlockIndex* ActiveTip() const { return ActiveChain().Tip(); } @@ -844,7 +873,7 @@ public: * validationinterface callback. * * @param[in] pblock The block we want to process. - * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and whitelisted peers. + * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources. * @param[out] fNewBlock A boolean which is set to indicate if the block was first received via this call * @returns If the block was processed, independently of block validity */ @@ -874,20 +903,21 @@ public: //! Clear (deconstruct) chainstate data. void Reset(); + + //! Check to see if caches are out of balance and if so, call + //! ResizeCoinsCaches() as needed. + void MaybeRebalanceCaches() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); }; /** DEPRECATED! Please use node.chainman instead. May only be used in validation.cpp internally */ extern ChainstateManager g_chainman GUARDED_BY(::cs_main); -/** @returns the most-work valid chainstate. */ +/** Please prefer the identical ChainstateManager::ActiveChainstate */ CChainState& ChainstateActive(); -/** @returns the most-work chain. */ +/** Please prefer the identical ChainstateManager::ActiveChain */ CChain& ChainActive(); -/** @returns the global block index map. */ -BlockMap& BlockIndex(); - /** Global variable that points to the active block tree (protected by cs_main) */ extern std::unique_ptr<CBlockTreeDB> pblocktree; diff --git a/src/version.h b/src/version.h index d932b512d4..b5f379e1b8 100644 --- a/src/version.h +++ b/src/version.h @@ -9,20 +9,13 @@ * network protocol versioning */ -static const int PROTOCOL_VERSION = 70015; +static const int PROTOCOL_VERSION = 70016; //! initial proto version, to be increased after version/verack negotiation static const int INIT_PROTO_VERSION = 209; -//! In this version, 'getheaders' was introduced. -static const int GETHEADERS_VERSION = 31800; - //! disconnect from peers older than this proto version -static const int MIN_PEER_PROTO_VERSION = GETHEADERS_VERSION; - -//! nTime field added to CAddress, starting with this version; -//! if possible, avoid requesting addresses nodes older than this -static const int CADDR_TIME_VERSION = 31402; +static const int MIN_PEER_PROTO_VERSION = 31800; //! BIP 0031, pong message, is enabled for all versions AFTER this one static const int BIP0031_VERSION = 60000; @@ -42,4 +35,7 @@ static const int SHORT_IDS_BLOCKS_VERSION = 70014; //! not banning for invalid compact blocks starts with this version static const int INVALID_CB_NO_BAN_VERSION = 70015; +//! "wtxidrelay" command for wtxid-based relay starts with this version +static const int WTXID_RELAY_VERSION = 70016; + #endif // BITCOIN_VERSION_H diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp new file mode 100644 index 0000000000..a8719806ab --- /dev/null +++ b/src/wallet/bdb.cpp @@ -0,0 +1,845 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <wallet/bdb.h> +#include <wallet/db.h> + +#include <util/strencodings.h> +#include <util/translation.h> + +#include <stdint.h> + +#ifndef WIN32 +#include <sys/stat.h> +#endif + +namespace { + +//! Make sure database has a unique fileid within the environment. If it +//! doesn't, throw an error. BDB caches do not work properly when more than one +//! open database has the same fileid (values written to one database may show +//! up in reads to other databases). +//! +//! BerkeleyDB generates unique fileids by default +//! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), +//! so bitcoin should never create different databases with the same fileid, but +//! this error can be triggered if users manually copy database files. +void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filename, Db& db, WalletDatabaseFileId& fileid) +{ + if (env.IsMock()) return; + + int ret = db.get_mpf()->get_fileid(fileid.value); + if (ret != 0) { + throw std::runtime_error(strprintf("BerkeleyDatabase: Can't open database %s (get_fileid failed with %d)", filename, ret)); + } + + for (const auto& item : env.m_fileids) { + if (fileid == item.second && &fileid != &item.second) { + throw std::runtime_error(strprintf("BerkeleyDatabase: Can't open database %s (duplicates fileid %s from %s)", filename, + HexStr(std::begin(item.second.value), std::end(item.second.value)), item.first)); + } + } +} + +RecursiveMutex cs_db; +std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db); //!< Map from directory name to db environment. +} // namespace + +bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const +{ + return memcmp(value, &rhs.value, sizeof(value)) == 0; +} + +bool IsBDBWalletLoaded(const fs::path& wallet_path) +{ + fs::path env_directory; + std::string database_filename; + SplitWalletPath(wallet_path, env_directory, database_filename); + LOCK(cs_db); + auto env = g_dbenvs.find(env_directory.string()); + if (env == g_dbenvs.end()) return false; + auto database = env->second.lock(); + return database && database->IsDatabaseLoaded(database_filename); +} + +/** + * @param[in] wallet_path Path to wallet directory. Or (for backwards compatibility only) a path to a berkeley btree data file inside a wallet directory. + * @param[out] database_filename Filename of berkeley btree data file inside the wallet directory. + * @return A shared pointer to the BerkeleyEnvironment object for the wallet directory, never empty because ~BerkeleyEnvironment + * erases the weak pointer from the g_dbenvs map. + * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the directory path key was not already in the map. + */ +std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename) +{ + fs::path env_directory; + SplitWalletPath(wallet_path, env_directory, database_filename); + LOCK(cs_db); + auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr<BerkeleyEnvironment>()); + if (inserted.second) { + auto env = std::make_shared<BerkeleyEnvironment>(env_directory.string()); + inserted.first->second = env; + return env; + } + return inserted.first->second.lock(); +} + +// +// BerkeleyBatch +// + +void BerkeleyEnvironment::Close() +{ + if (!fDbEnvInit) + return; + + fDbEnvInit = false; + + for (auto& db : m_databases) { + BerkeleyDatabase& database = db.second.get(); + assert(database.m_refcount <= 0); + if (database.m_db) { + database.m_db->close(0); + database.m_db.reset(); + } + } + + FILE* error_file = nullptr; + dbenv->get_errfile(&error_file); + + int ret = dbenv->close(0); + if (ret != 0) + LogPrintf("BerkeleyEnvironment::Close: Error %d closing database environment: %s\n", ret, DbEnv::strerror(ret)); + if (!fMockDb) + DbEnv((u_int32_t)0).remove(strPath.c_str(), 0); + + if (error_file) fclose(error_file); + + UnlockDirectory(strPath, ".walletlock"); +} + +void BerkeleyEnvironment::Reset() +{ + dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS)); + fDbEnvInit = false; + fMockDb = false; +} + +BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(dir_path.string()) +{ + Reset(); +} + +BerkeleyEnvironment::~BerkeleyEnvironment() +{ + LOCK(cs_db); + g_dbenvs.erase(strPath); + Close(); +} + +bool BerkeleyEnvironment::Open(bilingual_str& err) +{ + if (fDbEnvInit) { + return true; + } + + fs::path pathIn = strPath; + TryCreateDirectories(pathIn); + if (!LockDirectory(pathIn, ".walletlock")) { + LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance of bitcoin may be using it.\n", strPath); + err = strprintf(_("Error initializing wallet database environment %s!"), Directory()); + return false; + } + + fs::path pathLogDir = pathIn / "database"; + TryCreateDirectories(pathLogDir); + fs::path pathErrorFile = pathIn / "db.log"; + LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string()); + + unsigned int nEnvFlags = 0; + if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) + nEnvFlags |= DB_PRIVATE; + + dbenv->set_lg_dir(pathLogDir.string().c_str()); + dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet + dbenv->set_lg_bsize(0x10000); + dbenv->set_lg_max(1048576); + dbenv->set_lk_max_locks(40000); + dbenv->set_lk_max_objects(40000); + dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); /// debug + dbenv->set_flags(DB_AUTO_COMMIT, 1); + dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1); + dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1); + int ret = dbenv->open(strPath.c_str(), + DB_CREATE | + DB_INIT_LOCK | + DB_INIT_LOG | + DB_INIT_MPOOL | + DB_INIT_TXN | + DB_THREAD | + DB_RECOVER | + nEnvFlags, + S_IRUSR | S_IWUSR); + if (ret != 0) { + LogPrintf("BerkeleyEnvironment::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret)); + int ret2 = dbenv->close(0); + if (ret2 != 0) { + LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed database environment: %s\n", ret2, DbEnv::strerror(ret2)); + } + Reset(); + err = strprintf(_("Error initializing wallet database environment %s!"), Directory()); + if (ret == DB_RUNRECOVERY) { + err += Untranslated(" ") + _("This error could occur if this wallet was not shutdown cleanly and was last loaded using a build with a newer version of Berkeley DB. If so, please use the software that last loaded this wallet"); + } + return false; + } + + fDbEnvInit = true; + fMockDb = false; + return true; +} + +//! Construct an in-memory mock Berkeley environment for testing +BerkeleyEnvironment::BerkeleyEnvironment() +{ + Reset(); + + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n"); + + dbenv->set_cachesize(1, 0, 1); + dbenv->set_lg_bsize(10485760 * 4); + dbenv->set_lg_max(10485760); + dbenv->set_lk_max_locks(10000); + dbenv->set_lk_max_objects(10000); + dbenv->set_flags(DB_AUTO_COMMIT, 1); + dbenv->log_set_config(DB_LOG_IN_MEMORY, 1); + int ret = dbenv->open(nullptr, + DB_CREATE | + DB_INIT_LOCK | + DB_INIT_LOG | + DB_INIT_MPOOL | + DB_INIT_TXN | + DB_THREAD | + DB_PRIVATE, + S_IRUSR | S_IWUSR); + if (ret > 0) { + throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock: Error %d opening database environment.", ret)); + } + + fDbEnvInit = true; + fMockDb = true; +} + +BerkeleyBatch::SafeDbt::SafeDbt() +{ + m_dbt.set_flags(DB_DBT_MALLOC); +} + +BerkeleyBatch::SafeDbt::SafeDbt(void* data, size_t size) + : m_dbt(data, size) +{ +} + +BerkeleyBatch::SafeDbt::~SafeDbt() +{ + if (m_dbt.get_data() != nullptr) { + // Clear memory, e.g. in case it was a private key + memory_cleanse(m_dbt.get_data(), m_dbt.get_size()); + // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be + // freed by the caller. + // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html + if (m_dbt.get_flags() & DB_DBT_MALLOC) { + free(m_dbt.get_data()); + } + } +} + +const void* BerkeleyBatch::SafeDbt::get_data() const +{ + return m_dbt.get_data(); +} + +u_int32_t BerkeleyBatch::SafeDbt::get_size() const +{ + return m_dbt.get_size(); +} + +BerkeleyBatch::SafeDbt::operator Dbt*() +{ + return &m_dbt; +} + +bool BerkeleyDatabase::Verify(bilingual_str& errorStr) +{ + fs::path walletDir = env->Directory(); + fs::path file_path = walletDir / strFile; + + LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion()); + LogPrintf("Using wallet %s\n", file_path.string()); + + if (!env->Open(errorStr)) { + return false; + } + + if (fs::exists(file_path)) + { + assert(m_refcount == 0); + + Db db(env->dbenv.get(), 0); + int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); + if (result != 0) { + errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), file_path); + return false; + } + } + // also return true if files does not exists + return true; +} + +void BerkeleyEnvironment::CheckpointLSN(const std::string& strFile) +{ + dbenv->txn_checkpoint(0, 0, 0); + if (fMockDb) + return; + dbenv->lsn_reset(strFile.c_str(), 0); +} + +BerkeleyDatabase::~BerkeleyDatabase() +{ + if (env) { + LOCK(cs_db); + env->CloseDb(strFile); + assert(!m_db); + size_t erased = env->m_databases.erase(strFile); + assert(erased == 1); + env->m_fileids.erase(strFile); + } +} + +BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr), m_cursor(nullptr), m_database(database) +{ + database.AddRef(); + database.Open(pszMode); + fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); + fFlushOnClose = fFlushOnCloseIn; + env = database.env.get(); + pdb = database.m_db.get(); + strFile = database.strFile; + bool fCreate = strchr(pszMode, 'c') != nullptr; + if (fCreate && !Exists(std::string("version"))) { + bool fTmp = fReadOnly; + fReadOnly = false; + Write(std::string("version"), CLIENT_VERSION); + fReadOnly = fTmp; + } +} + +void BerkeleyDatabase::Open(const char* pszMode) +{ + if (IsDummy()){ + return; + } + + bool fCreate = strchr(pszMode, 'c') != nullptr; + unsigned int nFlags = DB_THREAD; + if (fCreate) + nFlags |= DB_CREATE; + + { + LOCK(cs_db); + bilingual_str open_err; + if (!env->Open(open_err)) + throw std::runtime_error("BerkeleyDatabase: Failed to open database environment."); + + if (m_db == nullptr) { + int ret; + std::unique_ptr<Db> pdb_temp = MakeUnique<Db>(env->dbenv.get(), 0); + + bool fMockDb = env->IsMock(); + if (fMockDb) { + DbMpoolFile* mpf = pdb_temp->get_mpf(); + ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); + if (ret != 0) { + throw std::runtime_error(strprintf("BerkeleyDatabase: Failed to configure for no temp file backing for database %s", strFile)); + } + } + + ret = pdb_temp->open(nullptr, // Txn pointer + fMockDb ? nullptr : strFile.c_str(), // Filename + fMockDb ? strFile.c_str() : "main", // Logical db name + DB_BTREE, // Database type + nFlags, // Flags + 0); + + if (ret != 0) { + throw std::runtime_error(strprintf("BerkeleyDatabase: Error %d, can't open database %s", ret, strFile)); + } + m_file_path = (env->Directory() / strFile).string(); + + // Call CheckUniqueFileid on the containing BDB environment to + // avoid BDB data consistency bugs that happen when different data + // files in the same environment have the same fileid. + CheckUniqueFileid(*env, strFile, *pdb_temp, this->env->m_fileids[strFile]); + + m_db.reset(pdb_temp.release()); + + } + } +} + +void BerkeleyBatch::Flush() +{ + if (activeTxn) + return; + + // Flush database activity from memory pool to disk log + unsigned int nMinutes = 0; + if (fReadOnly) + nMinutes = 1; + + if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault + env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); + } +} + +void BerkeleyDatabase::IncrementUpdateCounter() +{ + ++nUpdateCounter; +} + +BerkeleyBatch::~BerkeleyBatch() +{ + Close(); + m_database.RemoveRef(); +} + +void BerkeleyBatch::Close() +{ + if (!pdb) + return; + if (activeTxn) + activeTxn->abort(); + activeTxn = nullptr; + pdb = nullptr; + CloseCursor(); + + if (fFlushOnClose) + Flush(); +} + +void BerkeleyEnvironment::CloseDb(const std::string& strFile) +{ + { + LOCK(cs_db); + auto it = m_databases.find(strFile); + assert(it != m_databases.end()); + BerkeleyDatabase& database = it->second.get(); + if (database.m_db) { + // Close the database handle + database.m_db->close(0); + database.m_db.reset(); + } + } +} + +void BerkeleyEnvironment::ReloadDbEnv() +{ + // Make sure that no Db's are in use + AssertLockNotHeld(cs_db); + std::unique_lock<RecursiveMutex> lock(cs_db); + m_db_in_use.wait(lock, [this](){ + for (auto& db : m_databases) { + if (db.second.get().m_refcount > 0) return false; + } + return true; + }); + + std::vector<std::string> filenames; + for (auto it : m_databases) { + filenames.push_back(it.first); + } + // Close the individual Db's + for (const std::string& filename : filenames) { + CloseDb(filename); + } + // Reset the environment + Flush(true); // This will flush and close the environment + Reset(); + bilingual_str open_err; + Open(open_err); +} + +bool BerkeleyDatabase::Rewrite(const char* pszSkip) +{ + if (IsDummy()) { + return true; + } + while (true) { + { + LOCK(cs_db); + if (m_refcount <= 0) { + // Flush log data to the dat file + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + m_refcount = -1; + + bool fSuccess = true; + LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile); + std::string strFileRes = strFile + ".rewrite"; + { // surround usage of db with extra {} + BerkeleyBatch db(*this, "r"); + std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0); + + int ret = pdbCopy->open(nullptr, // Txn pointer + strFileRes.c_str(), // Filename + "main", // Logical db name + DB_BTREE, // Database type + DB_CREATE, // Flags + 0); + if (ret > 0) { + LogPrintf("BerkeleyBatch::Rewrite: Can't create database file %s\n", strFileRes); + fSuccess = false; + } + + if (db.StartCursor()) { + while (fSuccess) { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + bool complete; + bool ret1 = db.ReadAtCursor(ssKey, ssValue, complete); + if (complete) { + break; + } else if (!ret1) { + fSuccess = false; + break; + } + if (pszSkip && + strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) + continue; + if (strncmp(ssKey.data(), "\x07version", 8) == 0) { + // Update version: + ssValue.clear(); + ssValue << CLIENT_VERSION; + } + Dbt datKey(ssKey.data(), ssKey.size()); + Dbt datValue(ssValue.data(), ssValue.size()); + int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE); + if (ret2 > 0) + fSuccess = false; + } + db.CloseCursor(); + } + if (fSuccess) { + db.Close(); + env->CloseDb(strFile); + if (pdbCopy->close(0)) + fSuccess = false; + } else { + pdbCopy->close(0); + } + } + if (fSuccess) { + Db dbA(env->dbenv.get(), 0); + if (dbA.remove(strFile.c_str(), nullptr, 0)) + fSuccess = false; + Db dbB(env->dbenv.get(), 0); + if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0)) + fSuccess = false; + } + if (!fSuccess) + LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite database file %s\n", strFileRes); + return fSuccess; + } + } + UninterruptibleSleep(std::chrono::milliseconds{100}); + } +} + + +void BerkeleyEnvironment::Flush(bool fShutdown) +{ + int64_t nStart = GetTimeMillis(); + // Flush log data to the actual data file on all files that are not in use + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); + if (!fDbEnvInit) + return; + { + LOCK(cs_db); + bool no_dbs_accessed = true; + for (auto& db_it : m_databases) { + std::string strFile = db_it.first; + int nRefCount = db_it.second.get().m_refcount; + if (nRefCount < 0) continue; + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); + if (nRefCount == 0) { + // Move log data to the dat file + CloseDb(strFile); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); + dbenv->txn_checkpoint(0, 0, 0); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); + if (!fMockDb) + dbenv->lsn_reset(strFile.c_str(), 0); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s closed\n", strFile); + nRefCount = -1; + } else { + no_dbs_accessed = false; + } + } + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); + if (fShutdown) { + char** listp; + if (no_dbs_accessed) { + dbenv->log_archive(&listp, DB_ARCH_REMOVE); + Close(); + if (!fMockDb) { + fs::remove_all(fs::path(strPath) / "database"); + } + } + } + } +} + +bool BerkeleyDatabase::PeriodicFlush() +{ + // There's nothing to do for dummy databases. Return true. + if (IsDummy()) return true; + + // Don't flush if we can't acquire the lock. + TRY_LOCK(cs_db, lockDb); + if (!lockDb) return false; + + // Don't flush if any databases are in use + for (auto& it : env->m_databases) { + if (it.second.get().m_refcount > 0) return false; + } + + // Don't flush if there haven't been any batch writes for this database. + if (m_refcount < 0) return false; + + LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); + int64_t nStart = GetTimeMillis(); + + // Flush wallet file so it's self contained + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + m_refcount = -1; + + LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); + + return true; +} + +bool BerkeleyDatabase::Backup(const std::string& strDest) const +{ + if (IsDummy()) { + return false; + } + while (true) + { + { + LOCK(cs_db); + if (m_refcount <= 0) + { + // Flush log data to the dat file + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + + // Copy wallet file + fs::path pathSrc = env->Directory() / strFile; + fs::path pathDest(strDest); + if (fs::is_directory(pathDest)) + pathDest /= strFile; + + try { + if (fs::equivalent(pathSrc, pathDest)) { + LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); + return false; + } + + fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); + LogPrintf("copied %s to %s\n", strFile, pathDest.string()); + return true; + } catch (const fs::filesystem_error& e) { + LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e)); + return false; + } + } + } + UninterruptibleSleep(std::chrono::milliseconds{100}); + } +} + +void BerkeleyDatabase::Flush() +{ + if (!IsDummy()) { + env->Flush(false); + } +} + +void BerkeleyDatabase::Close() +{ + if (!IsDummy()) { + env->Flush(true); + } +} + +void BerkeleyDatabase::ReloadDbEnv() +{ + if (!IsDummy()) { + env->ReloadDbEnv(); + } +} + +bool BerkeleyBatch::StartCursor() +{ + assert(!m_cursor); + if (!pdb) + return false; + int ret = pdb->cursor(nullptr, &m_cursor, 0); + return ret == 0; +} + +bool BerkeleyBatch::ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) +{ + complete = false; + if (m_cursor == nullptr) return false; + // Read at cursor + SafeDbt datKey; + SafeDbt datValue; + int ret = m_cursor->get(datKey, datValue, DB_NEXT); + if (ret == DB_NOTFOUND) { + complete = true; + } + if (ret != 0) + return false; + else if (datKey.get_data() == nullptr || datValue.get_data() == nullptr) + return false; + + // Convert to streams + ssKey.SetType(SER_DISK); + ssKey.clear(); + ssKey.write((char*)datKey.get_data(), datKey.get_size()); + ssValue.SetType(SER_DISK); + ssValue.clear(); + ssValue.write((char*)datValue.get_data(), datValue.get_size()); + return true; +} + +void BerkeleyBatch::CloseCursor() +{ + if (!m_cursor) return; + m_cursor->close(); + m_cursor = nullptr; +} + +bool BerkeleyBatch::TxnBegin() +{ + if (!pdb || activeTxn) + return false; + DbTxn* ptxn = env->TxnBegin(); + if (!ptxn) + return false; + activeTxn = ptxn; + return true; +} + +bool BerkeleyBatch::TxnCommit() +{ + if (!pdb || !activeTxn) + return false; + int ret = activeTxn->commit(0); + activeTxn = nullptr; + return (ret == 0); +} + +bool BerkeleyBatch::TxnAbort() +{ + if (!pdb || !activeTxn) + return false; + int ret = activeTxn->abort(); + activeTxn = nullptr; + return (ret == 0); +} + +std::string BerkeleyDatabaseVersion() +{ + return DbEnv::version(nullptr, nullptr, nullptr); +} + +bool BerkeleyBatch::ReadKey(CDataStream&& key, CDataStream& value) +{ + if (!pdb) + return false; + + SafeDbt datKey(key.data(), key.size()); + + SafeDbt datValue; + int ret = pdb->get(activeTxn, datKey, datValue, 0); + if (ret == 0 && datValue.get_data() != nullptr) { + value.write((char*)datValue.get_data(), datValue.get_size()); + return true; + } + return false; +} + +bool BerkeleyBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite) +{ + if (!pdb) + return true; + if (fReadOnly) + assert(!"Write called on database in read-only mode"); + + SafeDbt datKey(key.data(), key.size()); + + SafeDbt datValue(value.data(), value.size()); + + int ret = pdb->put(activeTxn, datKey, datValue, (overwrite ? 0 : DB_NOOVERWRITE)); + return (ret == 0); +} + +bool BerkeleyBatch::EraseKey(CDataStream&& key) +{ + if (!pdb) + return false; + if (fReadOnly) + assert(!"Erase called on database in read-only mode"); + + SafeDbt datKey(key.data(), key.size()); + + int ret = pdb->del(activeTxn, datKey, 0); + return (ret == 0 || ret == DB_NOTFOUND); +} + +bool BerkeleyBatch::HasKey(CDataStream&& key) +{ + if (!pdb) + return false; + + SafeDbt datKey(key.data(), key.size()); + + int ret = pdb->exists(activeTxn, datKey, 0); + return ret == 0; +} + +void BerkeleyDatabase::AddRef() +{ + LOCK(cs_db); + if (m_refcount < 0) { + m_refcount = 1; + } else { + m_refcount++; + } +} + +void BerkeleyDatabase::RemoveRef() +{ + LOCK(cs_db); + m_refcount--; + if (env) env->m_db_in_use.notify_all(); +} + +std::unique_ptr<DatabaseBatch> BerkeleyDatabase::MakeBatch(const char* mode, bool flush_on_close) +{ + return MakeUnique<BerkeleyBatch>(*this, mode, flush_on_close); +} diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h new file mode 100644 index 0000000000..982423f00e --- /dev/null +++ b/src/wallet/bdb.h @@ -0,0 +1,238 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_BDB_H +#define BITCOIN_WALLET_BDB_H + +#include <clientversion.h> +#include <fs.h> +#include <serialize.h> +#include <streams.h> +#include <util/system.h> +#include <wallet/db.h> + +#include <atomic> +#include <map> +#include <memory> +#include <string> +#include <unordered_map> +#include <vector> + +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsuggest-override" +#endif +#include <db_cxx.h> +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + +struct bilingual_str; + +static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; +static const bool DEFAULT_WALLET_PRIVDB = true; + +struct WalletDatabaseFileId { + u_int8_t value[DB_FILE_ID_LEN]; + bool operator==(const WalletDatabaseFileId& rhs) const; +}; + +class BerkeleyDatabase; + +class BerkeleyEnvironment +{ +private: + bool fDbEnvInit; + bool fMockDb; + // Don't change into fs::path, as that can result in + // shutdown problems/crashes caused by a static initialized internal pointer. + std::string strPath; + +public: + std::unique_ptr<DbEnv> dbenv; + std::map<std::string, std::reference_wrapper<BerkeleyDatabase>> m_databases; + std::unordered_map<std::string, WalletDatabaseFileId> m_fileids; + std::condition_variable_any m_db_in_use; + + BerkeleyEnvironment(const fs::path& env_directory); + BerkeleyEnvironment(); + ~BerkeleyEnvironment(); + void Reset(); + + bool IsMock() const { return fMockDb; } + bool IsInitialized() const { return fDbEnvInit; } + bool IsDatabaseLoaded(const std::string& db_filename) const { return m_databases.find(db_filename) != m_databases.end(); } + fs::path Directory() const { return strPath; } + + bool Open(bilingual_str& error); + void Close(); + void Flush(bool fShutdown); + void CheckpointLSN(const std::string& strFile); + + void CloseDb(const std::string& strFile); + void ReloadDbEnv(); + + DbTxn* TxnBegin(int flags = DB_TXN_WRITE_NOSYNC) + { + DbTxn* ptxn = nullptr; + int ret = dbenv->txn_begin(nullptr, &ptxn, flags); + if (!ptxn || ret != 0) + return nullptr; + return ptxn; + } +}; + +/** Get BerkeleyEnvironment and database filename given a wallet path. */ +std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename); + +/** Return whether a BDB wallet database is currently loaded. */ +bool IsBDBWalletLoaded(const fs::path& wallet_path); + +class BerkeleyBatch; + +/** An instance of this class represents one database. + * For BerkeleyDB this is just a (env, strFile) tuple. + **/ +class BerkeleyDatabase : public WalletDatabase +{ +public: + /** Create dummy DB handle */ + BerkeleyDatabase() : WalletDatabase(), env(nullptr) + { + } + + /** Create DB handle to real database */ + BerkeleyDatabase(std::shared_ptr<BerkeleyEnvironment> env, std::string filename) : + WalletDatabase(), env(std::move(env)), strFile(std::move(filename)) + { + auto inserted = this->env->m_databases.emplace(strFile, std::ref(*this)); + assert(inserted.second); + } + + ~BerkeleyDatabase() override; + + /** Open the database if it is not already opened. + * Dummy function, doesn't do anything right now, but is needed for class abstraction */ + void Open(const char* mode) override; + + /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero + */ + bool Rewrite(const char* pszSkip=nullptr) override; + + /** Indicate the a new database user has began using the database. */ + void AddRef() override; + /** Indicate that database user has stopped using the database and that it could be flushed or closed. */ + void RemoveRef() override; + + /** Back up the entire database to a file. + */ + bool Backup(const std::string& strDest) const override; + + /** Make sure all changes are flushed to database file. + */ + void Flush() override; + /** Flush to the database file and close the database. + * Also close the environment if no other databases are open in it. + */ + void Close() override; + /* flush the wallet passively (TRY_LOCK) + ideal to be called periodically */ + bool PeriodicFlush() override; + + void IncrementUpdateCounter() override; + + void ReloadDbEnv() override; + + /** Verifies the environment and database file */ + bool Verify(bilingual_str& error) override; + + /** + * Pointer to shared database environment. + * + * Normally there is only one BerkeleyDatabase object per + * BerkeleyEnvivonment, but in the special, backwards compatible case where + * multiple wallet BDB data files are loaded from the same directory, this + * will point to a shared instance that gets freed when the last data file + * is closed. + */ + std::shared_ptr<BerkeleyEnvironment> env; + + /** Database pointer. This is initialized lazily and reset during flushes, so it can be null. */ + std::unique_ptr<Db> m_db; + + std::string strFile; + + /** Make a BerkeleyBatch connected to this database */ + std::unique_ptr<DatabaseBatch> MakeBatch(const char* mode = "r+", bool flush_on_close = true) override; + +private: + + /** Return whether this database handle is a dummy for testing. + * Only to be used at a low level, application should ideally not care + * about this. + */ + bool IsDummy() const { return env == nullptr; } +}; + +/** RAII class that provides access to a Berkeley database */ +class BerkeleyBatch : public DatabaseBatch +{ + /** RAII class that automatically cleanses its data on destruction */ + class SafeDbt final + { + Dbt m_dbt; + + public: + // construct Dbt with internally-managed data + SafeDbt(); + // construct Dbt with provided data + SafeDbt(void* data, size_t size); + ~SafeDbt(); + + // delegate to Dbt + const void* get_data() const; + u_int32_t get_size() const; + + // conversion operator to access the underlying Dbt + operator Dbt*(); + }; + +private: + bool ReadKey(CDataStream&& key, CDataStream& value) override; + bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite = true) override; + bool EraseKey(CDataStream&& key) override; + bool HasKey(CDataStream&& key) override; + +protected: + Db* pdb; + std::string strFile; + DbTxn* activeTxn; + Dbc* m_cursor; + bool fReadOnly; + bool fFlushOnClose; + BerkeleyEnvironment *env; + BerkeleyDatabase& m_database; + +public: + explicit BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode = "r+", bool fFlushOnCloseIn=true); + ~BerkeleyBatch() override; + + BerkeleyBatch(const BerkeleyBatch&) = delete; + BerkeleyBatch& operator=(const BerkeleyBatch&) = delete; + + void Flush() override; + void Close() override; + + bool StartCursor() override; + bool ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) override; + void CloseCursor() override; + bool TxnBegin() override; + bool TxnCommit() override; + bool TxnAbort() override; +}; + +std::string BerkeleyDatabaseVersion(); + +#endif // BITCOIN_WALLET_BDB_H diff --git a/src/wallet/coincontrol.cpp b/src/wallet/coincontrol.cpp index c83e598825..720877ead0 100644 --- a/src/wallet/coincontrol.cpp +++ b/src/wallet/coincontrol.cpp @@ -10,6 +10,7 @@ void CCoinControl::SetNull() { destChange = CNoDestination(); m_change_type.reset(); + m_add_inputs = true; fAllowOtherInputs = false; fAllowWatchOnly = false; m_avoid_partial_spends = gArgs.GetBoolArg("-avoidpartialspends", DEFAULT_AVOIDPARTIALSPENDS); @@ -23,4 +24,3 @@ void CCoinControl::SetNull() m_min_depth = DEFAULT_MIN_DEPTH; m_max_depth = DEFAULT_MAX_DEPTH; } - diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 2893d0ab3d..c499b0ff25 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -26,6 +26,8 @@ public: CTxDestination destChange; //! Override the default change type if set, ignored if destChange is set Optional<OutputType> m_change_type; + //! If false, only selected inputs are used + bool m_add_inputs; //! If false, allows unselected inputs, but requires all selected inputs be used bool fAllowOtherInputs; //! Includes watch only addresses which are solvable diff --git a/src/wallet/context.h b/src/wallet/context.h index 3c8fdd1c59..a83591154f 100644 --- a/src/wallet/context.h +++ b/src/wallet/context.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_WALLET_CONTEXT_H #define BITCOIN_WALLET_CONTEXT_H +class ArgsManager; namespace interfaces { class Chain; } // namespace interfaces @@ -21,6 +22,7 @@ class Chain; //! behavior. struct WalletContext { interfaces::Chain* chain{nullptr}; + ArgsManager* args{nullptr}; //! Declare default constructor and destructor that are not inline, so code //! instantiating the WalletContext struct doesn't need to #include class diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index d90e8e6433..1eb82a03c7 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -3,55 +3,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <fs.h> #include <wallet/db.h> -#include <util/strencodings.h> -#include <util/translation.h> +#include <string> -#include <stdint.h> - -#ifndef WIN32 -#include <sys/stat.h> -#endif - -namespace { - -//! Make sure database has a unique fileid within the environment. If it -//! doesn't, throw an error. BDB caches do not work properly when more than one -//! open database has the same fileid (values written to one database may show -//! up in reads to other databases). -//! -//! BerkeleyDB generates unique fileids by default -//! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), -//! so bitcoin should never create different databases with the same fileid, but -//! this error can be triggered if users manually copy database files. -void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filename, Db& db, WalletDatabaseFileId& fileid) -{ - if (env.IsMock()) return; - - int ret = db.get_mpf()->get_fileid(fileid.value); - if (ret != 0) { - throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (get_fileid failed with %d)", filename, ret)); - } - - for (const auto& item : env.m_fileids) { - if (fileid == item.second && &fileid != &item.second) { - throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (duplicates fileid %s from %s)", filename, - HexStr(std::begin(item.second.value), std::end(item.second.value)), item.first)); - } - } -} - -RecursiveMutex cs_db; -std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db); //!< Map from directory name to db environment. -} // namespace - -bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const -{ - return memcmp(value, &rhs.value, sizeof(value)) == 0; -} - -static void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename) +void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename) { if (fs::is_regular_file(wallet_path)) { // Special case for backwards compatibility: if wallet path points to an @@ -67,18 +24,6 @@ static void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory } } -bool IsWalletLoaded(const fs::path& wallet_path) -{ - fs::path env_directory; - std::string database_filename; - SplitWalletPath(wallet_path, env_directory, database_filename); - LOCK(cs_db); - auto env = g_dbenvs.find(env_directory.string()); - if (env == g_dbenvs.end()) return false; - auto database = env->second.lock(); - return database && database->IsDatabaseLoaded(database_filename); -} - fs::path WalletDataFilePath(const fs::path& wallet_path) { fs::path env_directory; @@ -86,682 +31,3 @@ fs::path WalletDataFilePath(const fs::path& wallet_path) SplitWalletPath(wallet_path, env_directory, database_filename); return env_directory / database_filename; } - -/** - * @param[in] wallet_path Path to wallet directory. Or (for backwards compatibility only) a path to a berkeley btree data file inside a wallet directory. - * @param[out] database_filename Filename of berkeley btree data file inside the wallet directory. - * @return A shared pointer to the BerkeleyEnvironment object for the wallet directory, never empty because ~BerkeleyEnvironment - * erases the weak pointer from the g_dbenvs map. - * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the directory path key was not already in the map. - */ -std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename) -{ - fs::path env_directory; - SplitWalletPath(wallet_path, env_directory, database_filename); - LOCK(cs_db); - auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr<BerkeleyEnvironment>()); - if (inserted.second) { - auto env = std::make_shared<BerkeleyEnvironment>(env_directory.string()); - inserted.first->second = env; - return env; - } - return inserted.first->second.lock(); -} - -// -// BerkeleyBatch -// - -void BerkeleyEnvironment::Close() -{ - if (!fDbEnvInit) - return; - - fDbEnvInit = false; - - for (auto& db : m_databases) { - auto count = mapFileUseCount.find(db.first); - assert(count == mapFileUseCount.end() || count->second == 0); - BerkeleyDatabase& database = db.second.get(); - if (database.m_db) { - database.m_db->close(0); - database.m_db.reset(); - } - } - - FILE* error_file = nullptr; - dbenv->get_errfile(&error_file); - - int ret = dbenv->close(0); - if (ret != 0) - LogPrintf("BerkeleyEnvironment::Close: Error %d closing database environment: %s\n", ret, DbEnv::strerror(ret)); - if (!fMockDb) - DbEnv((u_int32_t)0).remove(strPath.c_str(), 0); - - if (error_file) fclose(error_file); - - UnlockDirectory(strPath, ".walletlock"); -} - -void BerkeleyEnvironment::Reset() -{ - dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS)); - fDbEnvInit = false; - fMockDb = false; -} - -BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(dir_path.string()) -{ - Reset(); -} - -BerkeleyEnvironment::~BerkeleyEnvironment() -{ - LOCK(cs_db); - g_dbenvs.erase(strPath); - Close(); -} - -bool BerkeleyEnvironment::Open(bool retry) -{ - if (fDbEnvInit) { - return true; - } - - fs::path pathIn = strPath; - TryCreateDirectories(pathIn); - if (!LockDirectory(pathIn, ".walletlock")) { - LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance of bitcoin may be using it.\n", strPath); - return false; - } - - fs::path pathLogDir = pathIn / "database"; - TryCreateDirectories(pathLogDir); - fs::path pathErrorFile = pathIn / "db.log"; - LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string()); - - unsigned int nEnvFlags = 0; - if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) - nEnvFlags |= DB_PRIVATE; - - dbenv->set_lg_dir(pathLogDir.string().c_str()); - dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet - dbenv->set_lg_bsize(0x10000); - dbenv->set_lg_max(1048576); - dbenv->set_lk_max_locks(40000); - dbenv->set_lk_max_objects(40000); - dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); /// debug - dbenv->set_flags(DB_AUTO_COMMIT, 1); - dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1); - dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1); - int ret = dbenv->open(strPath.c_str(), - DB_CREATE | - DB_INIT_LOCK | - DB_INIT_LOG | - DB_INIT_MPOOL | - DB_INIT_TXN | - DB_THREAD | - DB_RECOVER | - nEnvFlags, - S_IRUSR | S_IWUSR); - if (ret != 0) { - LogPrintf("BerkeleyEnvironment::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret)); - int ret2 = dbenv->close(0); - if (ret2 != 0) { - LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed database environment: %s\n", ret2, DbEnv::strerror(ret2)); - } - Reset(); - if (retry) { - // try moving the database env out of the way - fs::path pathDatabaseBak = pathIn / strprintf("database.%d.bak", GetTime()); - try { - fs::rename(pathLogDir, pathDatabaseBak); - LogPrintf("Moved old %s to %s. Retrying.\n", pathLogDir.string(), pathDatabaseBak.string()); - } catch (const fs::filesystem_error&) { - // failure is ok (well, not really, but it's not worse than what we started with) - } - // try opening it again one more time - if (!Open(false /* retry */)) { - // if it still fails, it probably means we can't even create the database env - return false; - } - } else { - return false; - } - } - - fDbEnvInit = true; - fMockDb = false; - return true; -} - -//! Construct an in-memory mock Berkeley environment for testing -BerkeleyEnvironment::BerkeleyEnvironment() -{ - Reset(); - - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n"); - - dbenv->set_cachesize(1, 0, 1); - dbenv->set_lg_bsize(10485760 * 4); - dbenv->set_lg_max(10485760); - dbenv->set_lk_max_locks(10000); - dbenv->set_lk_max_objects(10000); - dbenv->set_flags(DB_AUTO_COMMIT, 1); - dbenv->log_set_config(DB_LOG_IN_MEMORY, 1); - int ret = dbenv->open(nullptr, - DB_CREATE | - DB_INIT_LOCK | - DB_INIT_LOG | - DB_INIT_MPOOL | - DB_INIT_TXN | - DB_THREAD | - DB_PRIVATE, - S_IRUSR | S_IWUSR); - if (ret > 0) { - throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock: Error %d opening database environment.", ret)); - } - - fDbEnvInit = true; - fMockDb = true; -} - -bool BerkeleyEnvironment::Verify(const std::string& strFile) -{ - LOCK(cs_db); - assert(mapFileUseCount.count(strFile) == 0); - - Db db(dbenv.get(), 0); - int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); - return result == 0; -} - -BerkeleyBatch::SafeDbt::SafeDbt() -{ - m_dbt.set_flags(DB_DBT_MALLOC); -} - -BerkeleyBatch::SafeDbt::SafeDbt(void* data, size_t size) - : m_dbt(data, size) -{ -} - -BerkeleyBatch::SafeDbt::~SafeDbt() -{ - if (m_dbt.get_data() != nullptr) { - // Clear memory, e.g. in case it was a private key - memory_cleanse(m_dbt.get_data(), m_dbt.get_size()); - // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be - // freed by the caller. - // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html - if (m_dbt.get_flags() & DB_DBT_MALLOC) { - free(m_dbt.get_data()); - } - } -} - -const void* BerkeleyBatch::SafeDbt::get_data() const -{ - return m_dbt.get_data(); -} - -u_int32_t BerkeleyBatch::SafeDbt::get_size() const -{ - return m_dbt.get_size(); -} - -BerkeleyBatch::SafeDbt::operator Dbt*() -{ - return &m_dbt; -} - -bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, bilingual_str& errorStr) -{ - std::string walletFile; - std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile); - fs::path walletDir = env->Directory(); - - LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion()); - LogPrintf("Using wallet %s\n", file_path.string()); - - if (!env->Open(true /* retry */)) { - errorStr = strprintf(_("Error initializing wallet database environment %s!"), walletDir); - return false; - } - - return true; -} - -bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, bilingual_str& errorStr) -{ - std::string walletFile; - std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile); - fs::path walletDir = env->Directory(); - - if (fs::exists(walletDir / walletFile)) - { - if (!env->Verify(walletFile)) { - errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), walletFile); - return false; - } - } - // also return true if files does not exists - return true; -} - -void BerkeleyEnvironment::CheckpointLSN(const std::string& strFile) -{ - dbenv->txn_checkpoint(0, 0, 0); - if (fMockDb) - return; - dbenv->lsn_reset(strFile.c_str(), 0); -} - - -BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr) -{ - fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); - fFlushOnClose = fFlushOnCloseIn; - env = database.env.get(); - if (database.IsDummy()) { - return; - } - const std::string &strFilename = database.strFile; - - bool fCreate = strchr(pszMode, 'c') != nullptr; - unsigned int nFlags = DB_THREAD; - if (fCreate) - nFlags |= DB_CREATE; - - { - LOCK(cs_db); - if (!env->Open(false /* retry */)) - throw std::runtime_error("BerkeleyBatch: Failed to open database environment."); - - pdb = database.m_db.get(); - if (pdb == nullptr) { - int ret; - std::unique_ptr<Db> pdb_temp = MakeUnique<Db>(env->dbenv.get(), 0); - - bool fMockDb = env->IsMock(); - if (fMockDb) { - DbMpoolFile* mpf = pdb_temp->get_mpf(); - ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); - if (ret != 0) { - throw std::runtime_error(strprintf("BerkeleyBatch: Failed to configure for no temp file backing for database %s", strFilename)); - } - } - - ret = pdb_temp->open(nullptr, // Txn pointer - fMockDb ? nullptr : strFilename.c_str(), // Filename - fMockDb ? strFilename.c_str() : "main", // Logical db name - DB_BTREE, // Database type - nFlags, // Flags - 0); - - if (ret != 0) { - throw std::runtime_error(strprintf("BerkeleyBatch: Error %d, can't open database %s", ret, strFilename)); - } - - // Call CheckUniqueFileid on the containing BDB environment to - // avoid BDB data consistency bugs that happen when different data - // files in the same environment have the same fileid. - // - // Also call CheckUniqueFileid on all the other g_dbenvs to prevent - // bitcoin from opening the same data file through another - // environment when the file is referenced through equivalent but - // not obviously identical symlinked or hard linked or bind mounted - // paths. In the future a more relaxed check for equal inode and - // device ids could be done instead, which would allow opening - // different backup copies of a wallet at the same time. Maybe even - // more ideally, an exclusive lock for accessing the database could - // be implemented, so no equality checks are needed at all. (Newer - // versions of BDB have an set_lk_exclusive method for this - // purpose, but the older version we use does not.) - for (const auto& env : g_dbenvs) { - CheckUniqueFileid(*env.second.lock().get(), strFilename, *pdb_temp, this->env->m_fileids[strFilename]); - } - - pdb = pdb_temp.release(); - database.m_db.reset(pdb); - - if (fCreate && !Exists(std::string("version"))) { - bool fTmp = fReadOnly; - fReadOnly = false; - Write(std::string("version"), CLIENT_VERSION); - fReadOnly = fTmp; - } - } - ++env->mapFileUseCount[strFilename]; - strFile = strFilename; - } -} - -void BerkeleyBatch::Flush() -{ - if (activeTxn) - return; - - // Flush database activity from memory pool to disk log - unsigned int nMinutes = 0; - if (fReadOnly) - nMinutes = 1; - - if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault - env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); - } -} - -void BerkeleyDatabase::IncrementUpdateCounter() -{ - ++nUpdateCounter; -} - -void BerkeleyBatch::Close() -{ - if (!pdb) - return; - if (activeTxn) - activeTxn->abort(); - activeTxn = nullptr; - pdb = nullptr; - - if (fFlushOnClose) - Flush(); - - { - LOCK(cs_db); - --env->mapFileUseCount[strFile]; - } - env->m_db_in_use.notify_all(); -} - -void BerkeleyEnvironment::CloseDb(const std::string& strFile) -{ - { - LOCK(cs_db); - auto it = m_databases.find(strFile); - assert(it != m_databases.end()); - BerkeleyDatabase& database = it->second.get(); - if (database.m_db) { - // Close the database handle - database.m_db->close(0); - database.m_db.reset(); - } - } -} - -void BerkeleyEnvironment::ReloadDbEnv() -{ - // Make sure that no Db's are in use - AssertLockNotHeld(cs_db); - std::unique_lock<RecursiveMutex> lock(cs_db); - m_db_in_use.wait(lock, [this](){ - for (auto& count : mapFileUseCount) { - if (count.second > 0) return false; - } - return true; - }); - - std::vector<std::string> filenames; - for (auto it : m_databases) { - filenames.push_back(it.first); - } - // Close the individual Db's - for (const std::string& filename : filenames) { - CloseDb(filename); - } - // Reset the environment - Flush(true); // This will flush and close the environment - Reset(); - Open(true); -} - -bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip) -{ - if (database.IsDummy()) { - return true; - } - BerkeleyEnvironment *env = database.env.get(); - const std::string& strFile = database.strFile; - while (true) { - { - LOCK(cs_db); - if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) { - // Flush log data to the dat file - env->CloseDb(strFile); - env->CheckpointLSN(strFile); - env->mapFileUseCount.erase(strFile); - - bool fSuccess = true; - LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile); - std::string strFileRes = strFile + ".rewrite"; - { // surround usage of db with extra {} - BerkeleyBatch db(database, "r"); - std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0); - - int ret = pdbCopy->open(nullptr, // Txn pointer - strFileRes.c_str(), // Filename - "main", // Logical db name - DB_BTREE, // Database type - DB_CREATE, // Flags - 0); - if (ret > 0) { - LogPrintf("BerkeleyBatch::Rewrite: Can't create database file %s\n", strFileRes); - fSuccess = false; - } - - Dbc* pcursor = db.GetCursor(); - if (pcursor) - while (fSuccess) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue); - if (ret1 == DB_NOTFOUND) { - pcursor->close(); - break; - } else if (ret1 != 0) { - pcursor->close(); - fSuccess = false; - break; - } - if (pszSkip && - strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) - continue; - if (strncmp(ssKey.data(), "\x07version", 8) == 0) { - // Update version: - ssValue.clear(); - ssValue << CLIENT_VERSION; - } - Dbt datKey(ssKey.data(), ssKey.size()); - Dbt datValue(ssValue.data(), ssValue.size()); - int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE); - if (ret2 > 0) - fSuccess = false; - } - if (fSuccess) { - db.Close(); - env->CloseDb(strFile); - if (pdbCopy->close(0)) - fSuccess = false; - } else { - pdbCopy->close(0); - } - } - if (fSuccess) { - Db dbA(env->dbenv.get(), 0); - if (dbA.remove(strFile.c_str(), nullptr, 0)) - fSuccess = false; - Db dbB(env->dbenv.get(), 0); - if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0)) - fSuccess = false; - } - if (!fSuccess) - LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite database file %s\n", strFileRes); - return fSuccess; - } - } - UninterruptibleSleep(std::chrono::milliseconds{100}); - } -} - - -void BerkeleyEnvironment::Flush(bool fShutdown) -{ - int64_t nStart = GetTimeMillis(); - // Flush log data to the actual data file on all files that are not in use - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); - if (!fDbEnvInit) - return; - { - LOCK(cs_db); - std::map<std::string, int>::iterator mi = mapFileUseCount.begin(); - while (mi != mapFileUseCount.end()) { - std::string strFile = (*mi).first; - int nRefCount = (*mi).second; - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); - if (nRefCount == 0) { - // Move log data to the dat file - CloseDb(strFile); - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); - dbenv->txn_checkpoint(0, 0, 0); - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); - if (!fMockDb) - dbenv->lsn_reset(strFile.c_str(), 0); - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s closed\n", strFile); - mapFileUseCount.erase(mi++); - } else - mi++; - } - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); - if (fShutdown) { - char** listp; - if (mapFileUseCount.empty()) { - dbenv->log_archive(&listp, DB_ARCH_REMOVE); - Close(); - if (!fMockDb) { - fs::remove_all(fs::path(strPath) / "database"); - } - } - } - } -} - -bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase& database) -{ - if (database.IsDummy()) { - return true; - } - bool ret = false; - BerkeleyEnvironment *env = database.env.get(); - const std::string& strFile = database.strFile; - TRY_LOCK(cs_db, lockDb); - if (lockDb) - { - // Don't do this if any databases are in use - int nRefCount = 0; - std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin(); - while (mit != env->mapFileUseCount.end()) - { - nRefCount += (*mit).second; - mit++; - } - - if (nRefCount == 0) - { - std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile); - if (mi != env->mapFileUseCount.end()) - { - LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); - int64_t nStart = GetTimeMillis(); - - // Flush wallet file so it's self contained - env->CloseDb(strFile); - env->CheckpointLSN(strFile); - - env->mapFileUseCount.erase(mi++); - LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); - ret = true; - } - } - } - - return ret; -} - -bool BerkeleyDatabase::Rewrite(const char* pszSkip) -{ - return BerkeleyBatch::Rewrite(*this, pszSkip); -} - -bool BerkeleyDatabase::Backup(const std::string& strDest) const -{ - if (IsDummy()) { - return false; - } - while (true) - { - { - LOCK(cs_db); - if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) - { - // Flush log data to the dat file - env->CloseDb(strFile); - env->CheckpointLSN(strFile); - env->mapFileUseCount.erase(strFile); - - // Copy wallet file - fs::path pathSrc = env->Directory() / strFile; - fs::path pathDest(strDest); - if (fs::is_directory(pathDest)) - pathDest /= strFile; - - try { - if (fs::equivalent(pathSrc, pathDest)) { - LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); - return false; - } - - fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); - LogPrintf("copied %s to %s\n", strFile, pathDest.string()); - return true; - } catch (const fs::filesystem_error& e) { - LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e)); - return false; - } - } - } - UninterruptibleSleep(std::chrono::milliseconds{100}); - } -} - -void BerkeleyDatabase::Flush(bool shutdown) -{ - if (!IsDummy()) { - env->Flush(shutdown); - if (shutdown) { - LOCK(cs_db); - g_dbenvs.erase(env->Directory().string()); - env = nullptr; - } else { - // TODO: To avoid g_dbenvs.erase erasing the environment prematurely after the - // first database shutdown when multiple databases are open in the same - // environment, should replace raw database `env` pointers with shared or weak - // pointers, or else separate the database and environment shutdowns so - // environments can be shut down after databases. - env->m_fileids.erase(strFile); - } - } -} - -void BerkeleyDatabase::ReloadDbEnv() -{ - if (!IsDummy()) { - env->ReloadDbEnv(); - } -} - -std::string BerkeleyDatabaseVersion() -{ - return DbEnv::version(nullptr, nullptr, nullptr); -} diff --git a/src/wallet/db.h b/src/wallet/db.h index 54ce144ffc..12dc1cc96b 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -8,387 +8,150 @@ #include <clientversion.h> #include <fs.h> -#include <serialize.h> #include <streams.h> -#include <util/system.h> #include <atomic> -#include <map> #include <memory> #include <string> -#include <unordered_map> -#include <vector> - -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wsuggest-override" -#endif -#include <db_cxx.h> -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif struct bilingual_str; -static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; -static const bool DEFAULT_WALLET_PRIVDB = true; - -struct WalletDatabaseFileId { - u_int8_t value[DB_FILE_ID_LEN]; - bool operator==(const WalletDatabaseFileId& rhs) const; -}; - -class BerkeleyDatabase; - -class BerkeleyEnvironment -{ -private: - bool fDbEnvInit; - bool fMockDb; - // Don't change into fs::path, as that can result in - // shutdown problems/crashes caused by a static initialized internal pointer. - std::string strPath; - -public: - std::unique_ptr<DbEnv> dbenv; - std::map<std::string, int> mapFileUseCount; - std::map<std::string, std::reference_wrapper<BerkeleyDatabase>> m_databases; - std::unordered_map<std::string, WalletDatabaseFileId> m_fileids; - std::condition_variable_any m_db_in_use; - - BerkeleyEnvironment(const fs::path& env_directory); - BerkeleyEnvironment(); - ~BerkeleyEnvironment(); - void Reset(); - - bool IsMock() const { return fMockDb; } - bool IsInitialized() const { return fDbEnvInit; } - bool IsDatabaseLoaded(const std::string& db_filename) const { return m_databases.find(db_filename) != m_databases.end(); } - fs::path Directory() const { return strPath; } - - bool Verify(const std::string& strFile); - - bool Open(bool retry); - void Close(); - void Flush(bool fShutdown); - void CheckpointLSN(const std::string& strFile); - - void CloseDb(const std::string& strFile); - void ReloadDbEnv(); - - DbTxn* TxnBegin(int flags = DB_TXN_WRITE_NOSYNC) - { - DbTxn* ptxn = nullptr; - int ret = dbenv->txn_begin(nullptr, &ptxn, flags); - if (!ptxn || ret != 0) - return nullptr; - return ptxn; - } -}; - -/** Return whether a wallet database is currently loaded. */ -bool IsWalletLoaded(const fs::path& wallet_path); - /** Given a wallet directory path or legacy file path, return path to main data file in the wallet database. */ fs::path WalletDataFilePath(const fs::path& wallet_path); +void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename); -/** Get BerkeleyEnvironment and database filename given a wallet path. */ -std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename); - -/** An instance of this class represents one database. - * For BerkeleyDB this is just a (env, strFile) tuple. - **/ -class BerkeleyDatabase +/** RAII class that provides access to a WalletDatabase */ +class DatabaseBatch { - friend class BerkeleyBatch; -public: - /** Create dummy DB handle */ - BerkeleyDatabase() : nUpdateCounter(0), nLastSeen(0), nLastFlushed(0), nLastWalletUpdate(0), env(nullptr) - { - } - - /** Create DB handle to real database */ - BerkeleyDatabase(std::shared_ptr<BerkeleyEnvironment> env, std::string filename) : - nUpdateCounter(0), nLastSeen(0), nLastFlushed(0), nLastWalletUpdate(0), env(std::move(env)), strFile(std::move(filename)) - { - auto inserted = this->env->m_databases.emplace(strFile, std::ref(*this)); - assert(inserted.second); - } - - ~BerkeleyDatabase() { - if (env) { - size_t erased = env->m_databases.erase(strFile); - assert(erased == 1); - } - } - - /** Return object for accessing database at specified path. */ - static std::unique_ptr<BerkeleyDatabase> Create(const fs::path& path) - { - std::string filename; - return MakeUnique<BerkeleyDatabase>(GetWalletEnv(path, filename), std::move(filename)); - } - - /** Return object for accessing dummy database with no read/write capabilities. */ - static std::unique_ptr<BerkeleyDatabase> CreateDummy() - { - return MakeUnique<BerkeleyDatabase>(); - } - - /** Return object for accessing temporary in-memory database. */ - static std::unique_ptr<BerkeleyDatabase> CreateMock() - { - return MakeUnique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), ""); - } - - /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero - */ - bool Rewrite(const char* pszSkip=nullptr); - - /** Back up the entire database to a file. - */ - bool Backup(const std::string& strDest) const; - - /** Make sure all changes are flushed to disk. - */ - void Flush(bool shutdown); - - void IncrementUpdateCounter(); - - void ReloadDbEnv(); - - std::atomic<unsigned int> nUpdateCounter; - unsigned int nLastSeen; - unsigned int nLastFlushed; - int64_t nLastWalletUpdate; - - /** - * Pointer to shared database environment. - * - * Normally there is only one BerkeleyDatabase object per - * BerkeleyEnvivonment, but in the special, backwards compatible case where - * multiple wallet BDB data files are loaded from the same directory, this - * will point to a shared instance that gets freed when the last data file - * is closed. - */ - std::shared_ptr<BerkeleyEnvironment> env; - - /** Database pointer. This is initialized lazily and reset during flushes, so it can be null. */ - std::unique_ptr<Db> m_db; - private: - std::string strFile; - - /** Return whether this database handle is a dummy for testing. - * Only to be used at a low level, application should ideally not care - * about this. - */ - bool IsDummy() const { return env == nullptr; } -}; - -/** RAII class that provides access to a Berkeley database */ -class BerkeleyBatch -{ - /** RAII class that automatically cleanses its data on destruction */ - class SafeDbt final - { - Dbt m_dbt; - - public: - // construct Dbt with internally-managed data - SafeDbt(); - // construct Dbt with provided data - SafeDbt(void* data, size_t size); - ~SafeDbt(); - - // delegate to Dbt - const void* get_data() const; - u_int32_t get_size() const; - - // conversion operator to access the underlying Dbt - operator Dbt*(); - }; - -protected: - Db* pdb; - std::string strFile; - DbTxn* activeTxn; - bool fReadOnly; - bool fFlushOnClose; - BerkeleyEnvironment *env; + virtual bool ReadKey(CDataStream&& key, CDataStream& value) = 0; + virtual bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite=true) = 0; + virtual bool EraseKey(CDataStream&& key) = 0; + virtual bool HasKey(CDataStream&& key) = 0; public: - explicit BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode = "r+", bool fFlushOnCloseIn=true); - ~BerkeleyBatch() { Close(); } - - BerkeleyBatch(const BerkeleyBatch&) = delete; - BerkeleyBatch& operator=(const BerkeleyBatch&) = delete; + explicit DatabaseBatch() {} + virtual ~DatabaseBatch() {} - void Flush(); - void Close(); + DatabaseBatch(const DatabaseBatch&) = delete; + DatabaseBatch& operator=(const DatabaseBatch&) = delete; - /* flush the wallet passively (TRY_LOCK) - ideal to be called periodically */ - static bool PeriodicFlush(BerkeleyDatabase& database); - /* verifies the database environment */ - static bool VerifyEnvironment(const fs::path& file_path, bilingual_str& errorStr); - /* verifies the database file */ - static bool VerifyDatabaseFile(const fs::path& file_path, bilingual_str& errorStr); + virtual void Flush() = 0; + virtual void Close() = 0; template <typename K, typename T> bool Read(const K& key, T& value) { - if (!pdb) - return false; - - // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - - // Read - SafeDbt datValue; - int ret = pdb->get(activeTxn, datKey, datValue, 0); - bool success = false; - if (datValue.get_data() != nullptr) { - // Unserialize value - try { - CDataStream ssValue((char*)datValue.get_data(), (char*)datValue.get_data() + datValue.get_size(), SER_DISK, CLIENT_VERSION); - ssValue >> value; - success = true; - } catch (const std::exception&) { - // In this case success remains 'false' - } + + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + if (!ReadKey(std::move(ssKey), ssValue)) return false; + try { + ssValue >> value; + return true; + } catch (const std::exception&) { + return false; } - return ret == 0 && success; } template <typename K, typename T> bool Write(const K& key, const T& value, bool fOverwrite = true) { - if (!pdb) - return true; - if (fReadOnly) - assert(!"Write called on database in read-only mode"); - - // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - // Value CDataStream ssValue(SER_DISK, CLIENT_VERSION); ssValue.reserve(10000); ssValue << value; - SafeDbt datValue(ssValue.data(), ssValue.size()); - // Write - int ret = pdb->put(activeTxn, datKey, datValue, (fOverwrite ? 0 : DB_NOOVERWRITE)); - return (ret == 0); + return WriteKey(std::move(ssKey), std::move(ssValue), fOverwrite); } template <typename K> bool Erase(const K& key) { - if (!pdb) - return false; - if (fReadOnly) - assert(!"Erase called on database in read-only mode"); - - // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - // Erase - int ret = pdb->del(activeTxn, datKey, 0); - return (ret == 0 || ret == DB_NOTFOUND); + return EraseKey(std::move(ssKey)); } template <typename K> bool Exists(const K& key) { - if (!pdb) - return false; - - // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - // Exists - int ret = pdb->exists(activeTxn, datKey, 0); - return (ret == 0); + return HasKey(std::move(ssKey)); } - Dbc* GetCursor() - { - if (!pdb) - return nullptr; - Dbc* pcursor = nullptr; - int ret = pdb->cursor(nullptr, &pcursor, 0); - if (ret != 0) - return nullptr; - return pcursor; - } + virtual bool StartCursor() = 0; + virtual bool ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) = 0; + virtual void CloseCursor() = 0; + virtual bool TxnBegin() = 0; + virtual bool TxnCommit() = 0; + virtual bool TxnAbort() = 0; +}; - int ReadAtCursor(Dbc* pcursor, CDataStream& ssKey, CDataStream& ssValue) - { - // Read at cursor - SafeDbt datKey; - SafeDbt datValue; - int ret = pcursor->get(datKey, datValue, DB_NEXT); - if (ret != 0) - return ret; - else if (datKey.get_data() == nullptr || datValue.get_data() == nullptr) - return 99999; - - // Convert to streams - ssKey.SetType(SER_DISK); - ssKey.clear(); - ssKey.write((char*)datKey.get_data(), datKey.get_size()); - ssValue.SetType(SER_DISK); - ssValue.clear(); - ssValue.write((char*)datValue.get_data(), datValue.get_size()); - return 0; - } +/** An instance of this class represents one database. + **/ +class WalletDatabase +{ +public: + /** Create dummy DB handle */ + WalletDatabase() : nUpdateCounter(0), nLastSeen(0), nLastFlushed(0), nLastWalletUpdate(0) {} + virtual ~WalletDatabase() {}; - bool TxnBegin() - { - if (!pdb || activeTxn) - return false; - DbTxn* ptxn = env->TxnBegin(); - if (!ptxn) - return false; - activeTxn = ptxn; - return true; - } + /** Open the database if it is not already opened. */ + virtual void Open(const char* mode) = 0; - bool TxnCommit() - { - if (!pdb || !activeTxn) - return false; - int ret = activeTxn->commit(0); - activeTxn = nullptr; - return (ret == 0); - } + //! Counts the number of active database users to be sure that the database is not closed while someone is using it + std::atomic<int> m_refcount{0}; + /** Indicate the a new database user has began using the database. Increments m_refcount */ + virtual void AddRef() = 0; + /** Indicate that database user has stopped using the database and that it could be flushed or closed. Decrement m_refcount */ + virtual void RemoveRef() = 0; - bool TxnAbort() - { - if (!pdb || !activeTxn) - return false; - int ret = activeTxn->abort(); - activeTxn = nullptr; - return (ret == 0); - } + /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero + */ + virtual bool Rewrite(const char* pszSkip=nullptr) = 0; - bool static Rewrite(BerkeleyDatabase& database, const char* pszSkip = nullptr); -}; + /** Back up the entire database to a file. + */ + virtual bool Backup(const std::string& strDest) const = 0; -std::string BerkeleyDatabaseVersion(); + /** Make sure all changes are flushed to database file. + */ + virtual void Flush() = 0; + /** Flush to the database file and close the database. + * Also close the environment if no other databases are open in it. + */ + virtual void Close() = 0; + /* flush the wallet passively (TRY_LOCK) + ideal to be called periodically */ + virtual bool PeriodicFlush() = 0; + + virtual void IncrementUpdateCounter() = 0; + + virtual void ReloadDbEnv() = 0; + + std::atomic<unsigned int> nUpdateCounter; + unsigned int nLastSeen; + unsigned int nLastFlushed; + int64_t nLastWalletUpdate; + + /** Verifies the environment and database file */ + virtual bool Verify(bilingual_str& error) = 0; + + std::string m_file_path; + + /** Make a DatabaseBatch connected to this database */ + virtual std::unique_ptr<DatabaseBatch> MakeBatch(const char* mode = "r+", bool flush_on_close = true) = 0; +}; #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 3885eb6185..781920755c 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -7,8 +7,9 @@ #include <interfaces/chain.h> #include <net.h> #include <node/context.h> +#include <node/ui_interface.h> #include <outputtype.h> -#include <ui_interface.h> +#include <util/check.h> #include <util/moneystr.h> #include <util/system.h> #include <util/translation.h> @@ -16,9 +17,9 @@ #include <wallet/wallet.h> #include <walletinitinterface.h> -class WalletInit : public WalletInitInterface { +class WalletInit : public WalletInitInterface +{ public: - //! Was the wallet component compiled in. bool HasWalletSupport() const override {return true;} @@ -112,10 +113,11 @@ bool WalletInit::ParameterInteraction() const void WalletInit::Construct(NodeContext& node) const { - if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { + ArgsManager& args = *Assert(node.args); + if (args.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { LogPrintf("Wallet disabled!\n"); return; } - gArgs.SoftSetArg("-wallet", ""); - node.chain_clients.emplace_back(interfaces::MakeWalletClient(*node.chain, gArgs.GetArgs("-wallet"))); + args.SoftSetArg("-wallet", ""); + node.chain_clients.emplace_back(interfaces::MakeWalletClient(*node.chain, args, args.GetArgs("-wallet"))); } diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 8df3e78215..2a81d30133 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -11,6 +11,7 @@ #include <util/system.h> #include <util/translation.h> #include <wallet/wallet.h> +#include <wallet/walletdb.h> bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files) { @@ -82,28 +83,30 @@ bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& walle } } -void StartWallets(CScheduler& scheduler) +void StartWallets(CScheduler& scheduler, const ArgsManager& args) { for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { pwallet->postInitProcess(); } // Schedule periodic wallet flushes and tx rebroadcasts - scheduler.scheduleEvery(MaybeCompactWalletDB, std::chrono::milliseconds{500}); + if (args.GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) { + scheduler.scheduleEvery(MaybeCompactWalletDB, std::chrono::milliseconds{500}); + } scheduler.scheduleEvery(MaybeResendWalletTxs, std::chrono::milliseconds{1000}); } void FlushWallets() { for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { - pwallet->Flush(false); + pwallet->Flush(); } } void StopWallets() { for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { - pwallet->Flush(true); + pwallet->Close(); } } diff --git a/src/wallet/load.h b/src/wallet/load.h index e24b1f2e69..ff4f5b4b23 100644 --- a/src/wallet/load.h +++ b/src/wallet/load.h @@ -9,6 +9,7 @@ #include <string> #include <vector> +class ArgsManager; class CScheduler; namespace interfaces { @@ -22,7 +23,7 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files); //! Complete startup of wallets. -void StartWallets(CScheduler& scheduler); +void StartWallets(CScheduler& scheduler, const ArgsManager& args); //! Flush all wallets in preparation for shutdown. void FlushWallets(); diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index d9fe741a6e..3b752ca936 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -297,7 +297,7 @@ UniValue importaddress(const JSONRPCRequest& request) pwallet->ImportScripts(scripts, 0 /* timestamp */); if (fP2SH) { - scripts.insert(GetScriptForDestination(ScriptHash(CScriptID(redeem_script)))); + scripts.insert(GetScriptForDestination(ScriptHash(redeem_script))); } pwallet->ImportScriptPubKeys(strLabel, scripts, false /* have_solving_data */, true /* apply_label */, 1 /* timestamp */); @@ -817,7 +817,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) create_time = FormatISO8601DateTime(it->second.nCreateTime); } if(spk_man.GetCScript(scriptid, script)) { - file << strprintf("%s %s script=1", HexStr(script.begin(), script.end()), create_time); + file << strprintf("%s %s script=1", HexStr(script), create_time); file << strprintf(" # addr=%s\n", address); } } @@ -856,20 +856,20 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d { // Use Solver to obtain script type and parsed pubkeys or hashes: std::vector<std::vector<unsigned char>> solverdata; - txnouttype script_type = Solver(script, solverdata); + TxoutType script_type = Solver(script, solverdata); switch (script_type) { - case TX_PUBKEY: { + case TxoutType::PUBKEY: { CPubKey pubkey(solverdata[0].begin(), solverdata[0].end()); import_data.used_keys.emplace(pubkey.GetID(), false); return ""; } - case TX_PUBKEYHASH: { + case TxoutType::PUBKEYHASH: { CKeyID id = CKeyID(uint160(solverdata[0])); import_data.used_keys[id] = true; return ""; } - case TX_SCRIPTHASH: { + case TxoutType::SCRIPTHASH: { if (script_ctx == ScriptContext::P2SH) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2SH inside another P2SH"); if (script_ctx == ScriptContext::WITNESS_V0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2SH inside a P2WSH"); CHECK_NONFATAL(script_ctx == ScriptContext::TOP); @@ -880,14 +880,14 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d import_data.import_scripts.emplace(*subscript); return RecurseImportData(*subscript, import_data, ScriptContext::P2SH); } - case TX_MULTISIG: { + case TxoutType::MULTISIG: { for (size_t i = 1; i + 1< solverdata.size(); ++i) { CPubKey pubkey(solverdata[i].begin(), solverdata[i].end()); import_data.used_keys.emplace(pubkey.GetID(), false); } return ""; } - case TX_WITNESS_V0_SCRIPTHASH: { + case TxoutType::WITNESS_V0_SCRIPTHASH: { if (script_ctx == ScriptContext::WITNESS_V0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2WSH inside another P2WSH"); uint256 fullid(solverdata[0]); CScriptID id; @@ -901,7 +901,7 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d import_data.import_scripts.emplace(*subscript); return RecurseImportData(*subscript, import_data, ScriptContext::WITNESS_V0); } - case TX_WITNESS_V0_KEYHASH: { + case TxoutType::WITNESS_V0_KEYHASH: { if (script_ctx == ScriptContext::WITNESS_V0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2WPKH inside P2WSH"); CKeyID id = CKeyID(uint160(solverdata[0])); import_data.used_keys[id] = true; @@ -910,10 +910,10 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d } return ""; } - case TX_NULL_DATA: + case TxoutType::NULL_DATA: return "unspendable script"; - case TX_NONSTANDARD: - case TX_WITNESS_UNKNOWN: + case TxoutType::NONSTANDARD: + case TxoutType::WITNESS_UNKNOWN: default: return "unrecognized script"; } @@ -1193,7 +1193,7 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con // Check whether we have any work to do for (const CScript& script : script_pub_keys) { if (pwallet->IsMine(script) & ISMINE_SPENDABLE) { - throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script (\"" + HexStr(script.begin(), script.end()) + "\")"); + throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script (\"" + HexStr(script) + "\")"); } } @@ -1547,7 +1547,7 @@ static UniValue ProcessDescriptorImport(CWallet * const pwallet, const UniValue& if (!w_desc.descriptor->GetOutputType()) { warnings.push_back("Unknown output type, cannot set descriptor to active."); } else { - pwallet->SetActiveScriptPubKeyMan(spk_manager->GetID(), *w_desc.descriptor->GetOutputType(), internal); + pwallet->AddActiveScriptPubKeyMan(spk_manager->GetID(), *w_desc.descriptor->GetOutputType(), internal); } } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 0907f2aabd..9d334063c4 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -45,6 +45,8 @@ using interfaces::FoundBlock; static const std::string WALLET_ENDPOINT_BASE = "/wallet/"; static const std::string HELP_REQUIRING_PASSPHRASE{"\nRequires wallet passphrase to be set with walletpassphrase call if wallet is encrypted.\n"}; +static const uint32_t WALLET_BTC_KB_TO_SAT_B = COIN / 1000; // 1 sat / B = 0.00001 BTC / kB + static inline bool GetAvoidReuseFlag(const CWallet* const pwallet, const UniValue& param) { bool can_avoid_reuse = pwallet->IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE); bool avoid_reuse = param.isNull() ? can_avoid_reuse : param.get_bool(); @@ -106,7 +108,7 @@ std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& reques return wallets[0]; } - if (!HasWallets()) { + if (wallets.empty()) { throw JSONRPCError( RPC_METHOD_NOT_FOUND, "Method not found (wallet method is disabled because no wallet is loaded)"); } @@ -191,6 +193,42 @@ static std::string LabelFromValue(const UniValue& value) return label; } +/** + * Update coin control with fee estimation based on the given parameters + * + * @param[in] pwallet Wallet pointer + * @param[in,out] cc Coin control which is to be updated + * @param[in] estimate_mode String value (e.g. "ECONOMICAL") + * @param[in] estimate_param Parameter (blocks to confirm, explicit fee rate, etc) + * @throws a JSONRPCError if estimate_mode is unknown, or if estimate_param is missing when required + */ +static void SetFeeEstimateMode(const CWallet* pwallet, CCoinControl& cc, const UniValue& estimate_mode, const UniValue& estimate_param) +{ + if (!estimate_mode.isNull()) { + if (!FeeModeFromString(estimate_mode.get_str(), cc.m_fee_mode)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); + } + } + + if (cc.m_fee_mode == FeeEstimateMode::BTC_KB || cc.m_fee_mode == FeeEstimateMode::SAT_B) { + if (estimate_param.isNull()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Selected estimate_mode requires a fee rate"); + } + + CAmount fee_rate = AmountFromValue(estimate_param); + if (cc.m_fee_mode == FeeEstimateMode::SAT_B) { + fee_rate /= WALLET_BTC_KB_TO_SAT_B; + } + + cc.m_feerate = CFeeRate(fee_rate); + + // default RBF to true for explicit fee rate modes + if (cc.m_signal_bip125_rbf == nullopt) cc.m_signal_bip125_rbf = true; + } else if (!estimate_param.isNull()) { + cc.m_confirm_target = ParseConfirmTarget(estimate_param, pwallet->chain().estimateMaxBlocks()); + } +} + static UniValue getnewaddress(const JSONRPCRequest& request) { RPCHelpMan{"getnewaddress", @@ -268,7 +306,7 @@ static UniValue getrawchangeaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Error: This wallet has no available keys"); } - OutputType output_type = pwallet->m_default_change_type != OutputType::CHANGE_AUTO ? pwallet->m_default_change_type : pwallet->m_default_address_type; + OutputType output_type = pwallet->m_default_change_type.get_value_or(pwallet->m_default_address_type); if (!request.params[0].isNull()) { if (!ParseOutputType(request.params[0].get_str(), output_type)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[0].get_str())); @@ -321,36 +359,54 @@ static UniValue setlabel(const JSONRPCRequest& request) return NullUniValue; } +void ParseRecipients(const UniValue& address_amounts, const UniValue& subtract_fee_outputs, std::vector<CRecipient> &recipients) { + std::set<CTxDestination> destinations; + int i = 0; + for (const std::string& address: address_amounts.getKeys()) { + CTxDestination dest = DecodeDestination(address); + if (!IsValidDestination(dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, std::string("Invalid Bitcoin address: ") + address); + } -static CTransactionRef SendMoney(CWallet* const pwallet, const CTxDestination& address, CAmount nValue, bool fSubtractFeeFromAmount, const CCoinControl& coin_control, mapValue_t mapValue) -{ - CAmount curBalance = pwallet->GetBalance(0, coin_control.m_avoid_address_reuse).m_mine_trusted; + if (destinations.count(dest)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated address: ") + address); + } + destinations.insert(dest); + + CScript script_pub_key = GetScriptForDestination(dest); + CAmount amount = AmountFromValue(address_amounts[i++]); - // Check amount - if (nValue <= 0) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid amount"); + bool subtract_fee = false; + for (unsigned int idx = 0; idx < subtract_fee_outputs.size(); idx++) { + const UniValue& addr = subtract_fee_outputs[idx]; + if (addr.get_str() == address) { + subtract_fee = true; + } + } + + CRecipient recipient = {script_pub_key, amount, subtract_fee}; + recipients.push_back(recipient); + } +} - if (nValue > curBalance) - throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Insufficient funds"); +UniValue SendMoney(CWallet* const pwallet, const CCoinControl &coin_control, std::vector<CRecipient> &recipients, mapValue_t map_value) +{ + EnsureWalletIsUnlocked(pwallet); - // Parse Bitcoin address - CScript scriptPubKey = GetScriptForDestination(address); + // Shuffle recipient list + std::shuffle(recipients.begin(), recipients.end(), FastRandomContext()); - // Create and send the transaction + // Send CAmount nFeeRequired = 0; - bilingual_str error; - std::vector<CRecipient> vecSend; int nChangePosRet = -1; - CRecipient recipient = {scriptPubKey, nValue, fSubtractFeeFromAmount}; - vecSend.push_back(recipient); + bilingual_str error; CTransactionRef tx; - if (!pwallet->CreateTransaction(vecSend, tx, nFeeRequired, nChangePosRet, error, coin_control)) { - if (!fSubtractFeeFromAmount && nValue + nFeeRequired > curBalance) - error = strprintf(Untranslated("Error: This transaction requires a transaction fee of at least %s"), FormatMoney(nFeeRequired)); - throw JSONRPCError(RPC_WALLET_ERROR, error.original); + bool fCreated = pwallet->CreateTransaction(recipients, tx, nFeeRequired, nChangePosRet, error, coin_control, !pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + if (!fCreated) { + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, error.original); } - pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */); - return tx; + pwallet->CommitTransaction(tx, std::move(map_value), {} /* orderForm */); + return tx->GetHash().GetHex(); } static UniValue sendtoaddress(const JSONRPCRequest& request) @@ -369,11 +425,9 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) {"subtractfeefromamount", RPCArg::Type::BOOL, /* default */ "false", "The fee will be deducted from the amount being sent.\n" " The recipient will receive less bitcoins than you enter in the amount field."}, {"replaceable", RPCArg::Type::BOOL, /* default */ "wallet default", "Allow this transaction to be replaced by a transaction with higher fees via BIP 125"}, - {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks)"}, - {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" - " \"UNSET\"\n" - " \"ECONOMICAL\"\n" - " \"CONSERVATIVE\""}, + {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks), or fee rate (for " + CURRENCY_UNIT + "/kB or " + CURRENCY_ATOM + "/B estimate modes)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, {"avoid_reuse", RPCArg::Type::BOOL, /* default */ "true", "(only available if avoid_reuse wallet flag is set) Avoid spending from dirty addresses; addresses are considered\n" " dirty if they have previously been used in a transaction."}, }, @@ -384,6 +438,8 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1") + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1 \"donation\" \"seans outpost\"") + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1 \"\" \"\" true") + + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1 \"\" \"\" false true 0.00002 " + (CURRENCY_UNIT + "/kB")) + + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1 \"\" \"\" false true 2 " + (CURRENCY_ATOM + "/B")) + HelpExampleRpc("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\", 0.1, \"donation\", \"seans outpost\"") }, }.Check(request); @@ -398,16 +454,6 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) LOCK(pwallet->cs_wallet); - CTxDestination dest = DecodeDestination(request.params[0].get_str()); - if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); - } - - // Amount - CAmount nAmount = AmountFromValue(request.params[1]); - if (nAmount <= 0) - throw JSONRPCError(RPC_TYPE_ERROR, "Invalid amount for send"); - // Wallet comments mapValue_t mapValue; if (!request.params[2].isNull() && !request.params[2].get_str().empty()) @@ -425,24 +471,26 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) coin_control.m_signal_bip125_rbf = request.params[5].get_bool(); } - if (!request.params[6].isNull()) { - coin_control.m_confirm_target = ParseConfirmTarget(request.params[6], pwallet->chain().estimateMaxBlocks()); - } - - if (!request.params[7].isNull()) { - if (!FeeModeFromString(request.params[7].get_str(), coin_control.m_fee_mode)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); - } - } - coin_control.m_avoid_address_reuse = GetAvoidReuseFlag(pwallet, request.params[8]); // We also enable partial spend avoidance if reuse avoidance is set. coin_control.m_avoid_partial_spends |= coin_control.m_avoid_address_reuse; + SetFeeEstimateMode(pwallet, coin_control, request.params[7], request.params[6]); + EnsureWalletIsUnlocked(pwallet); - CTransactionRef tx = SendMoney(pwallet, dest, nAmount, fSubtractFeeFromAmount, coin_control, std::move(mapValue)); - return tx->GetHash().GetHex(); + UniValue address_amounts(UniValue::VOBJ); + const std::string address = request.params[0].get_str(); + address_amounts.pushKV(address, request.params[1]); + UniValue subtractFeeFromAmount(UniValue::VARR); + if (fSubtractFeeFromAmount) { + subtractFeeFromAmount.push_back(address); + } + + std::vector<CRecipient> recipients; + ParseRecipients(address_amounts, subtractFeeFromAmount, recipients); + + return SendMoney(pwallet, coin_control, recipients, mapValue); } static UniValue listaddressgroupings(const JSONRPCRequest& request) @@ -780,11 +828,9 @@ static UniValue sendmany(const JSONRPCRequest& request) }, }, {"replaceable", RPCArg::Type::BOOL, /* default */ "wallet default", "Allow this transaction to be replaced by a transaction with higher fees via BIP 125"}, - {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks)"}, - {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" - " \"UNSET\"\n" - " \"ECONOMICAL\"\n" - " \"CONSERVATIVE\""}, + {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks), or fee rate (for " + CURRENCY_UNIT + "/kB or " + CURRENCY_ATOM + "/B estimate modes)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, }, RPCResult{ RPCResult::Type::STR_HEX, "txid", "The transaction id for the send. Only 1 transaction is created regardless of\n" @@ -830,62 +876,12 @@ static UniValue sendmany(const JSONRPCRequest& request) coin_control.m_signal_bip125_rbf = request.params[5].get_bool(); } - if (!request.params[6].isNull()) { - coin_control.m_confirm_target = ParseConfirmTarget(request.params[6], pwallet->chain().estimateMaxBlocks()); - } - - if (!request.params[7].isNull()) { - if (!FeeModeFromString(request.params[7].get_str(), coin_control.m_fee_mode)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); - } - } - - std::set<CTxDestination> destinations; - std::vector<CRecipient> vecSend; - - std::vector<std::string> keys = sendTo.getKeys(); - for (const std::string& name_ : keys) { - CTxDestination dest = DecodeDestination(name_); - if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, std::string("Invalid Bitcoin address: ") + name_); - } - - if (destinations.count(dest)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated address: ") + name_); - } - destinations.insert(dest); - - CScript scriptPubKey = GetScriptForDestination(dest); - CAmount nAmount = AmountFromValue(sendTo[name_]); - if (nAmount <= 0) - throw JSONRPCError(RPC_TYPE_ERROR, "Invalid amount for send"); - - bool fSubtractFeeFromAmount = false; - for (unsigned int idx = 0; idx < subtractFeeFromAmount.size(); idx++) { - const UniValue& addr = subtractFeeFromAmount[idx]; - if (addr.get_str() == name_) - fSubtractFeeFromAmount = true; - } + SetFeeEstimateMode(pwallet, coin_control, request.params[7], request.params[6]); - CRecipient recipient = {scriptPubKey, nAmount, fSubtractFeeFromAmount}; - vecSend.push_back(recipient); - } - - EnsureWalletIsUnlocked(pwallet); - - // Shuffle recipient list - std::shuffle(vecSend.begin(), vecSend.end(), FastRandomContext()); + std::vector<CRecipient> recipients; + ParseRecipients(sendTo, subtractFeeFromAmount, recipients); - // Send - CAmount nFeeRequired = 0; - int nChangePosRet = -1; - bilingual_str error; - CTransactionRef tx; - bool fCreated = pwallet->CreateTransaction(vecSend, tx, nFeeRequired, nChangePosRet, error, coin_control); - if (!fCreated) - throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, error.original); - pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */); - return tx->GetHash().GetHex(); + return SendMoney(pwallet, coin_control, recipients, std::move(mapValue)); } static UniValue addmultisigaddress(const JSONRPCRequest& request) @@ -964,7 +960,7 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) UniValue result(UniValue::VOBJ); result.pushKV("address", EncodeDestination(dest)); - result.pushKV("redeemScript", HexStr(inner.begin(), inner.end())); + result.pushKV("redeemScript", HexStr(inner)); result.pushKV("descriptor", descriptor->ToString()); return result; } @@ -2870,7 +2866,7 @@ static UniValue listunspent(const JSONRPCRequest& request) const CScriptID& hash = CScriptID(boost::get<ScriptHash>(address)); CScript redeemScript; if (provider->GetCScript(hash, redeemScript)) { - entry.pushKV("redeemScript", HexStr(redeemScript.begin(), redeemScript.end())); + entry.pushKV("redeemScript", HexStr(redeemScript)); // Now check if the redeemScript is actually a P2WSH script CTxDestination witness_destination; if (redeemScript.IsPayToWitnessScriptHash()) { @@ -2882,7 +2878,7 @@ static UniValue listunspent(const JSONRPCRequest& request) CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); CScript witnessScript; if (provider->GetCScript(id, witnessScript)) { - entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); + entry.pushKV("witnessScript", HexStr(witnessScript)); } } } @@ -2892,13 +2888,13 @@ static UniValue listunspent(const JSONRPCRequest& request) CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); CScript witnessScript; if (provider->GetCScript(id, witnessScript)) { - entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); + entry.pushKV("witnessScript", HexStr(witnessScript)); } } } } - entry.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); + entry.pushKV("scriptPubKey", HexStr(scriptPubKey)); entry.pushKV("amount", ValueFromAmount(out.tx->tx->vout[out.i].nValue)); entry.pushKV("confirmations", out.nDepth); entry.pushKV("spendable", out.fSpendable); @@ -2918,13 +2914,12 @@ static UniValue listunspent(const JSONRPCRequest& request) return results; } -void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& fee_out, int& change_position, UniValue options) +void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& fee_out, int& change_position, UniValue options, CCoinControl& coinControl) { // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - CCoinControl coinControl; change_position = -1; bool lockUnspents = false; UniValue subtractFeeFromOutputs; @@ -2939,6 +2934,7 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f RPCTypeCheckArgument(options, UniValue::VOBJ); RPCTypeCheckObj(options, { + {"add_inputs", UniValueType(UniValue::VBOOL)}, {"changeAddress", UniValueType(UniValue::VSTR)}, {"changePosition", UniValueType(UniValue::VNUM)}, {"change_type", UniValueType(UniValue::VSTR)}, @@ -2952,6 +2948,10 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f }, true, true); + if (options.exists("add_inputs") ) { + coinControl.m_add_inputs = options["add_inputs"].get_bool(); + } + if (options.exists("changeAddress")) { CTxDestination dest = DecodeDestination(options["changeAddress"].get_str()); @@ -2969,10 +2969,11 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f if (options.exists("changeAddress")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both changeAddress and address_type options"); } - coinControl.m_change_type = pwallet->m_default_change_type; - if (!ParseOutputType(options["change_type"].get_str(), *coinControl.m_change_type)) { + OutputType out_type; + if (!ParseOutputType(options["change_type"].get_str(), out_type)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown change type '%s'", options["change_type"].get_str())); } + coinControl.m_change_type.emplace(out_type); } coinControl.fAllowWatchOnly = ParseIncludeWatchonly(options["includeWatching"], *pwallet); @@ -2982,6 +2983,12 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f if (options.exists("feeRate")) { + if (options.exists("conf_target")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both conf_target and feeRate"); + } + if (options.exists("estimate_mode")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both estimate_mode and feeRate"); + } coinControl.m_feerate = CFeeRate(AmountFromValue(options["feeRate"])); coinControl.fOverrideFeeRate = true; } @@ -2992,20 +2999,7 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f if (options.exists("replaceable")) { coinControl.m_signal_bip125_rbf = options["replaceable"].get_bool(); } - if (options.exists("conf_target")) { - if (options.exists("feeRate")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both conf_target and feeRate"); - } - coinControl.m_confirm_target = ParseConfirmTarget(options["conf_target"], pwallet->chain().estimateMaxBlocks()); - } - if (options.exists("estimate_mode")) { - if (options.exists("feeRate")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both estimate_mode and feeRate"); - } - if (!FeeModeFromString(options["estimate_mode"].get_str(), coinControl.m_fee_mode)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); - } - } + SetFeeEstimateMode(pwallet, coinControl, options["estimate_mode"], options["conf_target"]); } } else { // if options is null and not a bool @@ -3039,8 +3033,8 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f static UniValue fundrawtransaction(const JSONRPCRequest& request) { RPCHelpMan{"fundrawtransaction", - "\nAdd inputs to a transaction until it has enough in value to meet its out value.\n" - "This will not modify existing inputs, and will add at most one change output to the outputs.\n" + "\nIf the transaction has no inputs, they will be automatically selected to meet its out value.\n" + "It will add at most one change output to the outputs.\n" "No existing outputs will be modified unless \"subtractFeeFromOutputs\" is specified.\n" "Note that inputs which were signed may need to be resigned after completion since in/outputs have been added.\n" "The inputs added will not be signed, use signrawtransactionwithkey\n" @@ -3054,6 +3048,7 @@ static UniValue fundrawtransaction(const JSONRPCRequest& request) {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", { + {"add_inputs", RPCArg::Type::BOOL, /* default */ "true", "For a transaction with existing inputs, automatically include more if they are not enough."}, {"changeAddress", RPCArg::Type::STR, /* default */ "pool address", "The bitcoin address to receive the change"}, {"changePosition", RPCArg::Type::NUM, /* default */ "random", "The index of the change output"}, {"change_type", RPCArg::Type::STR, /* default */ "set by -changetype", "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, @@ -3072,11 +3067,9 @@ static UniValue fundrawtransaction(const JSONRPCRequest& request) }, {"replaceable", RPCArg::Type::BOOL, /* default */ "wallet default", "Marks this transaction as BIP125 replaceable.\n" " Allows this transaction to be replaced by a transaction with higher fees"}, - {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks)"}, - {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" - " \"UNSET\"\n" - " \"ECONOMICAL\"\n" - " \"CONSERVATIVE\""}, + {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks), or fee rate (for " + CURRENCY_UNIT + "/kB or " + CURRENCY_ATOM + "/B estimate modes)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, }, "options"}, {"iswitness", RPCArg::Type::BOOL, /* default */ "depends on heuristic tests", "Whether the transaction hex is a serialized witness transaction.\n" @@ -3123,7 +3116,10 @@ static UniValue fundrawtransaction(const JSONRPCRequest& request) CAmount fee; int change_position; - FundTransaction(pwallet, tx, fee, change_position, request.params[1]); + CCoinControl coin_control; + // Automatically select (additional) coins. Can be overridden by options.add_inputs. + coin_control.m_add_inputs = true; + FundTransaction(pwallet, tx, fee, change_position, request.params[1], coin_control); UniValue result(UniValue::VOBJ); result.pushKV("hex", EncodeHexTx(CTransaction(tx))); @@ -3241,8 +3237,8 @@ static UniValue bumpfee(const JSONRPCRequest& request) {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The txid to be bumped"}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", { - {"confTarget", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks)"}, - {"fee_rate", RPCArg::Type::NUM, /* default */ "fall back to 'confTarget'", "fee rate (NOT total fee) to pay, in " + CURRENCY_UNIT + " per kB\n" + {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks)"}, + {"fee_rate", RPCArg::Type::NUM, /* default */ "fall back to 'conf_target'", "fee rate (NOT total fee) to pay, in " + CURRENCY_UNIT + " per kB\n" " Specify a fee rate instead of relying on the built-in fee estimator.\n" "Must be at least 0.0001 " + CURRENCY_UNIT + " per kB higher than the current transaction fee rate.\n"}, {"replaceable", RPCArg::Type::BOOL, /* default */ "true", "Whether the new transaction should still be\n" @@ -3252,10 +3248,8 @@ static UniValue bumpfee(const JSONRPCRequest& request) " so the new transaction will not be explicitly bip-125 replaceable (though it may\n" " still be replaceable in practice, for example if it has unconfirmed ancestors which\n" " are replaceable)."}, - {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" - " \"UNSET\"\n" - " \"ECONOMICAL\"\n" - " \"CONSERVATIVE\""}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, }, "options"}, }, @@ -3294,15 +3288,24 @@ static UniValue bumpfee(const JSONRPCRequest& request) RPCTypeCheckObj(options, { {"confTarget", UniValueType(UniValue::VNUM)}, + {"conf_target", UniValueType(UniValue::VNUM)}, {"fee_rate", UniValueType(UniValue::VNUM)}, {"replaceable", UniValueType(UniValue::VBOOL)}, {"estimate_mode", UniValueType(UniValue::VSTR)}, }, true, true); - if (options.exists("confTarget") && options.exists("fee_rate")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "confTarget can't be set with fee_rate. Please provide either a confirmation target in blocks for automatic fee estimation, or an explicit fee rate."); - } else if (options.exists("confTarget")) { // TODO: alias this to conf_target - coin_control.m_confirm_target = ParseConfirmTarget(options["confTarget"], pwallet->chain().estimateMaxBlocks()); + + if (options.exists("confTarget") && options.exists("conf_target")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "confTarget and conf_target options should not both be set. Use conf_target (confTarget is deprecated)."); + } + + auto conf_target = options.exists("confTarget") ? options["confTarget"] : options["conf_target"]; + + if (!conf_target.isNull()) { + if (options.exists("fee_rate")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "conf_target can't be set with fee_rate. Please provide either a confirmation target in blocks for automatic fee estimation, or an explicit fee rate."); + } + coin_control.m_confirm_target = ParseConfirmTarget(conf_target, pwallet->chain().estimateMaxBlocks()); } else if (options.exists("fee_rate")) { CFeeRate fee_rate(AmountFromValue(options["fee_rate"])); if (fee_rate <= CFeeRate(0)) { @@ -3314,11 +3317,7 @@ static UniValue bumpfee(const JSONRPCRequest& request) if (options.exists("replaceable")) { coin_control.m_signal_bip125_rbf = options["replaceable"].get_bool(); } - if (options.exists("estimate_mode")) { - if (!FeeModeFromString(options["estimate_mode"].get_str(), coin_control.m_fee_mode)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); - } - } + SetFeeEstimateMode(pwallet, coin_control, options["estimate_mode"], conf_target); } // Make sure the results are valid at least up to the most recent block @@ -3481,9 +3480,9 @@ public: { // Always present: script type and redeemscript std::vector<std::vector<unsigned char>> solutions_data; - txnouttype which_type = Solver(subscript, solutions_data); + TxoutType which_type = Solver(subscript, solutions_data); obj.pushKV("script", GetTxnOutputType(which_type)); - obj.pushKV("hex", HexStr(subscript.begin(), subscript.end())); + obj.pushKV("hex", HexStr(subscript)); CTxDestination embedded; if (ExtractDestination(subscript, embedded)) { @@ -3494,18 +3493,18 @@ public: UniValue wallet_detail = boost::apply_visitor(*this, embedded); subobj.pushKVs(wallet_detail); subobj.pushKV("address", EncodeDestination(embedded)); - subobj.pushKV("scriptPubKey", HexStr(subscript.begin(), subscript.end())); + subobj.pushKV("scriptPubKey", HexStr(subscript)); // Always report the pubkey at the top level, so that `getnewaddress()['pubkey']` always works. if (subobj.exists("pubkey")) obj.pushKV("pubkey", subobj["pubkey"]); obj.pushKV("embedded", std::move(subobj)); - } else if (which_type == TX_MULTISIG) { + } else if (which_type == TxoutType::MULTISIG) { // Also report some information on multisig scripts (which do not have a corresponding address). // TODO: abstract out the common functionality between this logic and ExtractDestinations. obj.pushKV("sigsrequired", solutions_data[0][0]); UniValue pubkeys(UniValue::VARR); for (size_t i = 1; i < solutions_data.size() - 1; ++i) { CPubKey key(solutions_data[i].begin(), solutions_data[i].end()); - pubkeys.push_back(HexStr(key.begin(), key.end())); + pubkeys.push_back(HexStr(key)); } obj.pushKV("pubkeys", std::move(pubkeys)); } @@ -3517,7 +3516,7 @@ public: UniValue operator()(const PKHash& pkhash) const { - CKeyID keyID(pkhash); + CKeyID keyID{ToKeyID(pkhash)}; UniValue obj(UniValue::VOBJ); CPubKey vchPubKey; if (provider && provider->GetPubKey(keyID, vchPubKey)) { @@ -3542,7 +3541,7 @@ public: { UniValue obj(UniValue::VOBJ); CPubKey pubkey; - if (provider && provider->GetPubKey(CKeyID(id), pubkey)) { + if (provider && provider->GetPubKey(ToKeyID(id), pubkey)) { obj.pushKV("pubkey", HexStr(pubkey)); } return obj; @@ -3623,12 +3622,10 @@ UniValue getaddressinfo(const JSONRPCRequest& request) {RPCResult::Type::STR_HEX, "pubkey", /* optional */ true, "The hex value of the raw public key for single-key addresses (possibly embedded in P2SH or P2WSH)."}, {RPCResult::Type::OBJ, "embedded", /* optional */ true, "Information about the address embedded in P2SH or P2WSH, if relevant and known.", { - {RPCResult::Type::ELISION, "", "Includes all\n" - " getaddressinfo output fields for the embedded address, excluding metadata (timestamp, hdkeypath,\n" - "hdseedid) and relation to the wallet (ismine, iswatchonly)."}, + {RPCResult::Type::ELISION, "", "Includes all getaddressinfo output fields for the embedded address, excluding metadata (timestamp, hdkeypath, hdseedid)\n" + "and relation to the wallet (ismine, iswatchonly)."}, }}, {RPCResult::Type::BOOL, "iscompressed", /* optional */ true, "If the pubkey is compressed."}, - {RPCResult::Type::STR, "label", "DEPRECATED. The label associated with the address. Defaults to \"\". Replaced by the labels array below."}, {RPCResult::Type::NUM_TIME, "timestamp", /* optional */ true, "The creation time of the key, if available, expressed in " + UNIX_EPOCH_TIME + "."}, {RPCResult::Type::STR, "hdkeypath", /* optional */ true, "The HD keypath, if the key is HD and available."}, {RPCResult::Type::STR_HEX, "hdseedid", /* optional */ true, "The Hash160 of the HD seed."}, @@ -3636,12 +3633,7 @@ UniValue getaddressinfo(const JSONRPCRequest& request) {RPCResult::Type::ARR, "labels", "Array of labels associated with the address. Currently limited to one label but returned\n" "as an array to keep the API stable if multiple labels are enabled in the future.", { - {RPCResult::Type::STR, "label name", "The label name. Defaults to \"\"."}, - {RPCResult::Type::OBJ, "", "label data, DEPRECATED, will be removed in 0.21. To re-enable, launch bitcoind with `-deprecatedrpc=labelspurpose`", - { - {RPCResult::Type::STR, "name", "The label name. Defaults to \"\"."}, - {RPCResult::Type::STR, "purpose", "The purpose of the associated address (send or receive)."}, - }}, + {RPCResult::Type::STR, "label name", "Label name (defaults to \"\")."}, }}, } }, @@ -3668,7 +3660,7 @@ UniValue getaddressinfo(const JSONRPCRequest& request) ret.pushKV("address", currentAddress); CScript scriptPubKey = GetScriptForDestination(dest); - ret.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); + ret.pushKV("scriptPubKey", HexStr(scriptPubKey)); std::unique_ptr<SigningProvider> provider = pwallet->GetSolvingProvider(scriptPubKey); @@ -3687,14 +3679,6 @@ UniValue getaddressinfo(const JSONRPCRequest& request) UniValue detail = DescribeWalletAddress(pwallet, dest); ret.pushKVs(detail); - // DEPRECATED: Return label field if existing. Currently only one label can - // be associated with an address, so the label should be equivalent to the - // value of the name key/value pair in the labels array below. - const auto* address_book_entry = pwallet->FindAddressBookEntry(dest); - if (pwallet->chain().rpcEnableDeprecated("label") && address_book_entry) { - ret.pushKV("label", address_book_entry->GetLabel()); - } - ret.pushKV("ischange", pwallet->IsChange(scriptPubKey)); ScriptPubKeyMan* spk_man = pwallet->GetScriptPubKeyMan(scriptPubKey); @@ -3715,14 +3699,9 @@ UniValue getaddressinfo(const JSONRPCRequest& request) // stable if we allow multiple labels to be associated with an address in // the future. UniValue labels(UniValue::VARR); + const auto* address_book_entry = pwallet->FindAddressBookEntry(dest); if (address_book_entry) { - // DEPRECATED: The previous behavior of returning an array containing a - // JSON object of `name` and `purpose` key/value pairs is deprecated. - if (pwallet->chain().rpcEnableDeprecated("labelspurpose")) { - labels.push_back(AddressBookDataToJSON(*address_book_entry, true)); - } else { - labels.push_back(address_book_entry->GetLabel()); - } + labels.push_back(address_book_entry->GetLabel()); } ret.pushKV("labels", std::move(labels)); @@ -3976,16 +3955,16 @@ UniValue walletprocesspsbt(const JSONRPCRequest& request) UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) { RPCHelpMan{"walletcreatefundedpsbt", - "\nCreates and funds a transaction in the Partially Signed Transaction format. Inputs will be added if supplied inputs are not enough\n" + "\nCreates and funds a transaction in the Partially Signed Transaction format.\n" "Implements the Creator and Updater roles.\n", { - {"inputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The inputs", + {"inputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The inputs. Leave empty to add inputs automatically. See add_inputs option.", { {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, - {"sequence", RPCArg::Type::NUM, RPCArg::Optional::NO, "The sequence number"}, + {"sequence", RPCArg::Type::NUM, /* default */ "depends on the value of the 'locktime' and 'options.replaceable' arguments", "The sequence number"}, }, }, }, @@ -4010,6 +3989,7 @@ UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) {"locktime", RPCArg::Type::NUM, /* default */ "0", "Raw locktime. Non-0 value also locktime-activates inputs"}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", { + {"add_inputs", RPCArg::Type::BOOL, /* default */ "false", "If inputs are specified, automatically include more if they are not enough."}, {"changeAddress", RPCArg::Type::STR_HEX, /* default */ "pool address", "The bitcoin address to receive the change"}, {"changePosition", RPCArg::Type::NUM, /* default */ "random", "The index of the change output"}, {"change_type", RPCArg::Type::STR, /* default */ "set by -changetype", "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, @@ -4027,10 +4007,8 @@ UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) {"replaceable", RPCArg::Type::BOOL, /* default */ "wallet default", "Marks this transaction as BIP125 replaceable.\n" " Allows this transaction to be replaced by a transaction with higher fees"}, {"conf_target", RPCArg::Type::NUM, /* default */ "fall back to wallet's confirmation target (txconfirmtarget)", "Confirmation target (in blocks)"}, - {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" - " \"UNSET\"\n" - " \"ECONOMICAL\"\n" - " \"CONSERVATIVE\""}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, }, "options"}, {"bip32derivs", RPCArg::Type::BOOL, /* default */ "true", "Include BIP 32 derivation paths for public keys if we know them"}, @@ -4071,7 +4049,11 @@ UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) rbf = replaceable_arg.isTrue(); } CMutableTransaction rawTx = ConstructTransaction(request.params[0], request.params[1], request.params[2], rbf); - FundTransaction(pwallet, rawTx, fee, change_position, request.params[3]); + CCoinControl coin_control; + // Automatically select coins, unless at least one is manually selected. Can + // be overridden by options.add_inputs. + coin_control.m_add_inputs = rawTx.vin.size() == 0; + FundTransaction(pwallet, rawTx, fee, change_position, request.params[3], coin_control); // Make a blank psbt PartiallySignedTransaction psbtx(rawTx); diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index 70067ebef0..af57210f01 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -5,6 +5,7 @@ #include <fs.h> #include <streams.h> +#include <util/translation.h> #include <wallet/salvage.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> @@ -20,6 +21,12 @@ bool RecoverDatabaseFile(const fs::path& file_path) std::string filename; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename); + bilingual_str open_err; + if (!env->Open(open_err)) { + tfm::format(std::cerr, "%s\n", open_err.original); + return false; + } + // Recovery procedure: // move wallet file to walletfilename.timestamp.bak // Call Salvage with fAggressive=true to @@ -116,7 +123,7 @@ bool RecoverDatabaseFile(const fs::path& file_path) } DbTxn* ptxn = env->TxnBegin(); - CWallet dummyWallet(nullptr, WalletLocation(), WalletDatabase::CreateDummy()); + CWallet dummyWallet(nullptr, WalletLocation(), CreateDummyWalletDatabase()); for (KeyValPair& row : salvagedData) { /* Filter for only private key type KV pairs to be added to the salvaged wallet */ diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 8a2a798644..51715462c5 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -88,16 +88,16 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s IsMineResult ret = IsMineResult::NO; std::vector<valtype> vSolutions; - txnouttype whichType = Solver(scriptPubKey, vSolutions); + TxoutType whichType = Solver(scriptPubKey, vSolutions); CKeyID keyID; switch (whichType) { - case TX_NONSTANDARD: - case TX_NULL_DATA: - case TX_WITNESS_UNKNOWN: + case TxoutType::NONSTANDARD: + case TxoutType::NULL_DATA: + case TxoutType::WITNESS_UNKNOWN: break; - case TX_PUBKEY: + case TxoutType::PUBKEY: keyID = CPubKey(vSolutions[0]).GetID(); if (!PermitsUncompressed(sigversion) && vSolutions[0].size() != 33) { return IsMineResult::INVALID; @@ -106,7 +106,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s ret = std::max(ret, IsMineResult::SPENDABLE); } break; - case TX_WITNESS_V0_KEYHASH: + case TxoutType::WITNESS_V0_KEYHASH: { if (sigversion == IsMineSigVersion::WITNESS_V0) { // P2WPKH inside P2WSH is invalid. @@ -121,7 +121,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s ret = std::max(ret, IsMineInner(keystore, GetScriptForDestination(PKHash(uint160(vSolutions[0]))), IsMineSigVersion::WITNESS_V0)); break; } - case TX_PUBKEYHASH: + case TxoutType::PUBKEYHASH: keyID = CKeyID(uint160(vSolutions[0])); if (!PermitsUncompressed(sigversion)) { CPubKey pubkey; @@ -133,7 +133,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s ret = std::max(ret, IsMineResult::SPENDABLE); } break; - case TX_SCRIPTHASH: + case TxoutType::SCRIPTHASH: { if (sigversion != IsMineSigVersion::TOP) { // P2SH inside P2WSH or P2SH is invalid. @@ -146,7 +146,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s } break; } - case TX_WITNESS_V0_SCRIPTHASH: + case TxoutType::WITNESS_V0_SCRIPTHASH: { if (sigversion == IsMineSigVersion::WITNESS_V0) { // P2WSH inside P2WSH is invalid. @@ -165,7 +165,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s break; } - case TX_MULTISIG: + case TxoutType::MULTISIG: { // Never treat bare multisig outputs as ours (they can still be made watchonly-though) if (sigversion == IsMineSigVersion::TOP) { @@ -573,9 +573,8 @@ bool LegacyScriptPubKeyMan::SignTransaction(CMutableTransaction& tx, const std:: SigningResult LegacyScriptPubKeyMan::SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const { - CKeyID key_id(pkhash); CKey key; - if (!GetKey(key_id, key)) { + if (!GetKey(ToKeyID(pkhash), key)) { return SigningResult::PRIVATE_KEY_NOT_AVAILABLE; } @@ -585,8 +584,11 @@ SigningResult LegacyScriptPubKeyMan::SignMessage(const std::string& message, con return SigningResult::SIGNING_FAILED; } -TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, int sighash_type, bool sign, bool bip32derivs) const +TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, int sighash_type, bool sign, bool bip32derivs, int* n_signed) const { + if (n_signed) { + *n_signed = 0; + } for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { const CTxIn& txin = psbtx.tx->vin[i]; PSBTInput& input = psbtx.inputs.at(i); @@ -595,11 +597,6 @@ TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psb continue; } - // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. - if (!input.IsSane()) { - return TransactionError::INVALID_PSBT; - } - // Get the Sighash type if (sign && input.sighash_type > 0 && input.sighash_type != sighash_type) { return TransactionError::SIGHASH_MISMATCH; @@ -617,6 +614,14 @@ TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psb SignatureData sigdata; input.FillSignatureData(sigdata); SignPSBTInput(HidingSigningProvider(this, !sign, !bip32derivs), psbtx, i, sighash_type); + + bool signed_one = PSBTInputSigned(input); + if (n_signed && (signed_one || !sign)) { + // If sign is false, we assume that we _could_ sign if we get here. This + // will never have false negatives; it is hard to tell under what i + // circumstances it could have false positives. + (*n_signed)++; + } } // Fill in the bip32 keypaths and redeemscripts for the outputs so that hardware wallets can identify change @@ -826,7 +831,7 @@ bool LegacyScriptPubKeyMan::HaveWatchOnly() const static bool ExtractPubKey(const CScript &dest, CPubKey& pubKeyOut) { std::vector<std::vector<unsigned char>> solutions; - return Solver(dest, solutions) == TX_PUBKEY && + return Solver(dest, solutions) == TxoutType::PUBKEY && (pubKeyOut = CPubKey(solutions[0])).IsFullyValid(); } @@ -900,20 +905,22 @@ bool LegacyScriptPubKeyMan::AddWatchOnly(const CScript& dest, int64_t nCreateTim return AddWatchOnly(dest); } -void LegacyScriptPubKeyMan::SetHDChain(const CHDChain& chain, bool memonly) +void LegacyScriptPubKeyMan::LoadHDChain(const CHDChain& chain) { LOCK(cs_KeyStore); - // memonly == true means we are loading the wallet file - // memonly == false means that the chain is actually being changed - if (!memonly) { - // Store the new chain - if (!WalletBatch(m_storage.GetDatabase()).WriteHDChain(chain)) { - throw std::runtime_error(std::string(__func__) + ": writing chain failed"); - } - // When there's an old chain, add it as an inactive chain as we are now rotating hd chains - if (!m_hd_chain.seed_id.IsNull()) { - AddInactiveHDChain(m_hd_chain); - } + m_hd_chain = chain; +} + +void LegacyScriptPubKeyMan::AddHDChain(const CHDChain& chain) +{ + LOCK(cs_KeyStore); + // Store the new chain + if (!WalletBatch(m_storage.GetDatabase()).WriteHDChain(chain)) { + throw std::runtime_error(std::string(__func__) + ": writing chain failed"); + } + // When there's an old chain, add it as an inactive chain as we are now rotating hd chains + if (!m_hd_chain.seed_id.IsNull()) { + AddInactiveHDChain(m_hd_chain); } m_hd_chain = chain; @@ -1167,7 +1174,7 @@ void LegacyScriptPubKeyMan::SetHDSeed(const CPubKey& seed) CHDChain newHdChain; newHdChain.nVersion = m_storage.CanSupportFeature(FEATURE_HD_SPLIT) ? CHDChain::VERSION_HD_CHAIN_SPLIT : CHDChain::VERSION_HD_BASE; newHdChain.seed_id = seed.GetID(); - SetHDChain(newHdChain, false); + AddHDChain(newHdChain); NotifyCanGetAddressesChanged(); WalletBatch batch(m_storage.GetDatabase()); m_storage.UnsetBlankWalletFlag(batch); @@ -1890,8 +1897,8 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_ desc_prefix = "wpkh(" + xpub + "/84'"; break; } - default: assert(false); - } + } // no default case, so the compiler can warn about missing cases + assert(!desc_prefix.empty()); // Mainnet derives at 0', testnet and regtest derive at 1' if (Params().IsTestChain()) { @@ -2052,9 +2059,8 @@ SigningResult DescriptorScriptPubKeyMan::SignMessage(const std::string& message, return SigningResult::PRIVATE_KEY_NOT_AVAILABLE; } - CKeyID key_id(pkhash); CKey key; - if (!keys->GetKey(key_id, key)) { + if (!keys->GetKey(ToKeyID(pkhash), key)) { return SigningResult::PRIVATE_KEY_NOT_AVAILABLE; } @@ -2064,8 +2070,11 @@ SigningResult DescriptorScriptPubKeyMan::SignMessage(const std::string& message, return SigningResult::OK; } -TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, int sighash_type, bool sign, bool bip32derivs) const +TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, int sighash_type, bool sign, bool bip32derivs, int* n_signed) const { + if (n_signed) { + *n_signed = 0; + } for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { const CTxIn& txin = psbtx.tx->vin[i]; PSBTInput& input = psbtx.inputs.at(i); @@ -2074,11 +2083,6 @@ TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& continue; } - // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. - if (!input.IsSane()) { - return TransactionError::INVALID_PSBT; - } - // Get the Sighash type if (sign && input.sighash_type > 0 && input.sighash_type != sighash_type) { return TransactionError::SIGHASH_MISMATCH; @@ -2117,6 +2121,14 @@ TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& } SignPSBTInput(HidingSigningProvider(keys.get(), !sign, !bip32derivs), psbtx, i, sighash_type); + + bool signed_one = PSBTInputSigned(input); + if (n_signed && (signed_one || !sign)) { + // If sign is false, we assume that we _could_ sign if we get here. This + // will never have false negatives; it is hard to tell under what i + // circumstances it could have false positives. + (*n_signed)++; + } } // Fill in the bip32 keypaths and redeemscripts for the outputs so that hardware wallets can identify change diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index d62d30f339..a96d971734 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -234,7 +234,7 @@ public: /** Sign a message with the given script */ virtual SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const { return SigningResult::SIGNING_FAILED; }; /** Adds script and derivation path information to a PSBT, and optionally signs it. */ - virtual TransactionError FillPSBT(PartiallySignedTransaction& psbt, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false) const { return TransactionError::INVALID_PSBT; } + virtual TransactionError FillPSBT(PartiallySignedTransaction& psbt, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr) const { return TransactionError::INVALID_PSBT; } virtual uint256 GetID() const { return uint256(); } @@ -393,7 +393,7 @@ public: bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const override; SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const override; - TransactionError FillPSBT(PartiallySignedTransaction& psbt, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false) const override; + TransactionError FillPSBT(PartiallySignedTransaction& psbt, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr) const override; uint256 GetID() const override; @@ -422,8 +422,10 @@ public: //! Generate a new key CPubKey GenerateNewKey(WalletBatch& batch, CHDChain& hd_chain, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); - /* Set the HD chain model (chain child index counters) */ - void SetHDChain(const CHDChain& chain, bool memonly); + /* Set the HD chain model (chain child index counters) and writes it to the database */ + void AddHDChain(const CHDChain& chain); + //! Load a HD chain model (used by LoadWallet) + void LoadHDChain(const CHDChain& chain); const CHDChain& GetHDChain() const { return m_hd_chain; } void AddInactiveHDChain(const CHDChain& chain); @@ -596,7 +598,7 @@ public: bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const override; SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const override; - TransactionError FillPSBT(PartiallySignedTransaction& psbt, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false) const override; + TransactionError FillPSBT(PartiallySignedTransaction& psbt, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr) const override; uint256 GetID() const override; diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 657d0828f2..1deedede4c 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -29,7 +29,7 @@ typedef std::set<CInputCoin> CoinSet; static std::vector<COutput> vCoins; static NodeContext testNode; static auto testChain = interfaces::MakeChain(testNode); -static CWallet testWallet(testChain.get(), WalletLocation(), WalletDatabase::CreateDummy()); +static CWallet testWallet(testChain.get(), WalletLocation(), CreateDummyWalletDatabase()); static CAmount balance = 0; CoinEligibilityFilter filter_standard(1, 6, 0); @@ -283,7 +283,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) // Make sure that can use BnB when there are preset inputs empty_wallet(); { - std::unique_ptr<CWallet> wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + std::unique_ptr<CWallet> wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), CreateMockWalletDatabase()); bool firstRun; wallet->LoadWallet(firstRun); wallet->SetupLegacyScriptPubKeyMan(); diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index f4a4c9fa7c..8f0083cd2e 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -8,7 +8,7 @@ #include <fs.h> #include <test/util/setup_common.h> -#include <wallet/db.h> +#include <wallet/bdb.h> BOOST_FIXTURE_TEST_SUITE(db_tests, BasicTestingSetup) diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index 797a0d634f..35bd965673 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -3,13 +3,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <fs.h> +#include <util/check.h> #include <util/system.h> #include <wallet/test/init_test_fixture.h> -InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainName): BasicTestingSetup(chainName) +InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) { - m_chain_client = MakeWalletClient(*m_chain, {}); + m_chain_client = MakeWalletClient(*m_chain, *Assert(m_node.args), {}); std::string sep; sep += fs::path::preferred_separator; diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index 4c0e4dc653..e416f16044 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -35,7 +35,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK compressed { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(pubkeys[0]); @@ -52,7 +52,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK uncompressed { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(uncompressedPubkey); @@ -69,7 +69,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH compressed { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(pubkeys[0])); @@ -86,7 +86,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH uncompressed { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(uncompressedPubkey)); @@ -103,7 +103,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -127,7 +127,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2SH (invalid) { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -145,7 +145,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2WSH (invalid) { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -163,11 +163,11 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH inside P2WSH (invalid) { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); - CScript witnessscript = GetScriptForDestination(WitnessV0KeyHash(PKHash(pubkeys[0]))); + CScript witnessscript = GetScriptForDestination(WitnessV0KeyHash(pubkeys[0])); scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessscript)); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessscript)); @@ -179,7 +179,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2WSH inside P2WSH (invalid) { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -197,12 +197,12 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH compressed { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); - scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(PKHash(pubkeys[0]))); + scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(pubkeys[0])); // Keystore implicitly has key and P2SH redeemScript BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); @@ -212,12 +212,12 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH uncompressed { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); - scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(PKHash(uncompressedPubkey))); + scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(uncompressedPubkey)); // Keystore has key, but no P2SH redeemScript result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); @@ -231,7 +231,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // scriptPubKey multisig { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -262,7 +262,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH multisig { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -283,7 +283,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with compressed keys { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -309,7 +309,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with uncompressed key { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -335,7 +335,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig wrapped in P2SH { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -362,7 +362,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // OP_RETURN { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -376,7 +376,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unspendable { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -390,7 +390,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unknown { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -404,7 +404,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Nonstandard { - CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index b4c65a8665..ce7e661b67 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -63,8 +63,8 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Get the final tx CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - std::string final_hex = HexStr(ssTx.begin(), ssTx.end()); - BOOST_CHECK_EQUAL(final_hex, "70736274ff01009a020000000258e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd750000000000ffffffff838d0427d0ec650a68aa46bb0b098aea4422c071b2ca78352a077959d07cea1d0100000000ffffffff0270aaf00800000000160014d85c2b71d0060b09c9886aeb815e50991dda124d00e1f5050000000016001400aea9a2e5f0f876a588df5546e8742d1d87008f00000000000100bb0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f6187650000000104475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae2206029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f10d90c6a4f000000800000008000000080220602dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d710d90c6a4f0000008000000080010000800001012000c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88701042200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903010547522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae2206023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7310d90c6a4f000000800000008003000080220603089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc10d90c6a4f00000080000000800200008000220203a9a4c37f5996d3aa25dbac6b570af0650394492942460b354753ed9eeca5877110d90c6a4f000000800000008004000080002202027f6399757d2eff55a136ad02c684b1838b6556e5f1b6b34282a94b6b5005109610d90c6a4f00000080000000800500008000"); + std::string final_hex = HexStr(ssTx); + BOOST_CHECK_EQUAL(final_hex, "70736274ff01009a020000000258e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd750000000000ffffffff838d0427d0ec650a68aa46bb0b098aea4422c071b2ca78352a077959d07cea1d0100000000ffffffff0270aaf00800000000160014d85c2b71d0060b09c9886aeb815e50991dda124d00e1f5050000000016001400aea9a2e5f0f876a588df5546e8742d1d87008f00000000000100bb0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f6187650000000104475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae2206029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f10d90c6a4f000000800000008000000080220602dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d710d90c6a4f0000008000000080010000800001008a020000000158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e8876500000001012000c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88701042200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903010547522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae2206023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7310d90c6a4f000000800000008003000080220603089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc10d90c6a4f00000080000000800200008000220203a9a4c37f5996d3aa25dbac6b570af0650394492942460b354753ed9eeca5877110d90c6a4f000000800000008004000080002202027f6399757d2eff55a136ad02c684b1838b6556e5f1b6b34282a94b6b5005109610d90c6a4f00000080000000800500008000"); // Mutate the transaction so that one of the inputs is invalid psbtx.tx->vin[0].prevout.n = 2; diff --git a/src/wallet/test/scriptpubkeyman_tests.cpp b/src/wallet/test/scriptpubkeyman_tests.cpp index 757865ea37..4f12079768 100644 --- a/src/wallet/test/scriptpubkeyman_tests.cpp +++ b/src/wallet/test/scriptpubkeyman_tests.cpp @@ -19,7 +19,7 @@ BOOST_AUTO_TEST_CASE(CanProvide) // Set up wallet and keyman variables. NodeContext node; std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); LegacyScriptPubKeyMan& keyman = *wallet.GetOrCreateLegacyScriptPubKeyMan(); // Make a 1 of 2 multisig script diff --git a/src/wallet/test/wallet_crypto_tests.cpp b/src/wallet/test/wallet_crypto_tests.cpp index 97f8c94fa6..10ddfa22ef 100644 --- a/src/wallet/test/wallet_crypto_tests.cpp +++ b/src/wallet/test/wallet_crypto_tests.cpp @@ -24,10 +24,10 @@ static void TestPassphraseSingle(const std::vector<unsigned char>& vchSalt, cons if(!correctKey.empty()) BOOST_CHECK_MESSAGE(memcmp(crypt.vchKey.data(), correctKey.data(), crypt.vchKey.size()) == 0, \ - HexStr(crypt.vchKey.begin(), crypt.vchKey.end()) + std::string(" != ") + HexStr(correctKey.begin(), correctKey.end())); + HexStr(crypt.vchKey) + std::string(" != ") + HexStr(correctKey)); if(!correctIV.empty()) BOOST_CHECK_MESSAGE(memcmp(crypt.vchIV.data(), correctIV.data(), crypt.vchIV.size()) == 0, - HexStr(crypt.vchIV.begin(), crypt.vchIV.end()) + std::string(" != ") + HexStr(correctIV.begin(), correctIV.end())); + HexStr(crypt.vchIV) + std::string(" != ") + HexStr(correctIV)); } static void TestPassphrase(const std::vector<unsigned char>& vchSalt, const SecureString& passphrase, uint32_t rounds, diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index 7ba3148ff3..44f9eb5ddd 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -6,7 +6,7 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName) : TestingSetup(chainName), - m_wallet(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()) + m_wallet(m_chain.get(), WalletLocation(), CreateMockWalletDatabase()) { bool fFirstRun; m_wallet.LoadWallet(fFirstRun); diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index 6c32868b1e..99d7cfe921 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -10,17 +10,18 @@ #include <interfaces/chain.h> #include <interfaces/wallet.h> #include <node/context.h> +#include <util/check.h> #include <wallet/wallet.h> #include <memory> /** Testing setup and teardown for wallet. */ -struct WalletTestingSetup: public TestingSetup { +struct WalletTestingSetup : public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); - std::unique_ptr<interfaces::ChainClient> m_chain_client = interfaces::MakeWalletClient(*m_chain, {}); + std::unique_ptr<interfaces::ChainClient> m_chain_client = interfaces::MakeWalletClient(*m_chain, *Assert(m_node.args), {}); CWallet m_wallet; std::unique_ptr<interfaces::Handler> m_chain_notifications_handler; }; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 6f8c973f25..d2770a46f7 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -28,6 +28,11 @@ extern UniValue importmulti(const JSONRPCRequest& request); extern UniValue dumpwallet(const JSONRPCRequest& request); extern UniValue importwallet(const JSONRPCRequest& request); +// Ensure that fee levels defined in the wallet are at least as high +// as the default levels for node policy. +static_assert(DEFAULT_TRANSACTION_MINFEE >= DEFAULT_MIN_RELAY_TX_FEE, "wallet minimum fee is smaller than default relay fee"); +static_assert(WALLET_INCREMENTAL_RELAY_FEE >= DEFAULT_INCREMENTAL_RELAY_FEE, "wallet incremental fee is smaller than default incremental relay fee"); + BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) static std::shared_ptr<CWallet> TestLoadWallet(interfaces::Chain& chain) @@ -80,7 +85,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions fails to read an unknown start block. { - CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -99,7 +104,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions picks up transactions in both the old // and new block files. { - CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -118,14 +123,14 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Prune the older block file. { LOCK(cs_main); - EnsureChainman(m_node).PruneOneBlockFile(oldTip->GetBlockPos().nFile); + Assert(m_node.chainman)->PruneOneBlockFile(oldTip->GetBlockPos().nFile); } UnlinkPrunedFiles({oldTip->GetBlockPos().nFile}); // Verify ScanForWalletTransactions only picks transactions in the new block // file. { - CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -144,13 +149,13 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Prune the remaining block file. { LOCK(cs_main); - EnsureChainman(m_node).PruneOneBlockFile(newTip->GetBlockPos().nFile); + Assert(m_node.chainman)->PruneOneBlockFile(newTip->GetBlockPos().nFile); } UnlinkPrunedFiles({newTip->GetBlockPos().nFile}); // Verify ScanForWalletTransactions scans no blocks. { - CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -181,7 +186,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // Prune the older block file. { LOCK(cs_main); - EnsureChainman(m_node).PruneOneBlockFile(oldTip->GetBlockPos().nFile); + Assert(m_node.chainman)->PruneOneBlockFile(oldTip->GetBlockPos().nFile); } UnlinkPrunedFiles({oldTip->GetBlockPos().nFile}); @@ -189,7 +194,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // before the missing block, and success for a key whose creation time is // after. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(newTip->nHeight, newTip->GetBlockHash())); AddWallet(wallet); @@ -254,7 +259,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Import key into wallet and call dumpwallet to create backup file. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); { auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); @@ -276,7 +281,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME // were scanned, and no prior blocks were scanned. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); @@ -312,7 +317,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) NodeContext node; auto chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); CWalletTx wtx(&wallet, m_coinbase_txns.back()); @@ -333,7 +338,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 50*COIN); } -static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) +static int64_t AddTx(ChainstateManager& chainman, CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) { CMutableTransaction tx; CWalletTx::Confirmation confirm; @@ -341,7 +346,8 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 SetMockTime(mockTime); CBlockIndex* block = nullptr; if (blockTime > 0) { - auto inserted = ::BlockIndex().emplace(GetRandHash(), new CBlockIndex); + LOCK(cs_main); + auto inserted = chainman.BlockIndex().emplace(GetRandHash(), new CBlockIndex); assert(inserted.second); const uint256& hash = inserted.first->first; block = inserted.first->second; @@ -363,24 +369,24 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 BOOST_AUTO_TEST_CASE(ComputeTimeSmart) { // New transaction should use clock time if lower than block time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 1, 100, 120), 100); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 1, 100, 120), 100); // Test that updating existing transaction does not change smart time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 1, 200, 220), 100); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 1, 200, 220), 100); // New transaction should use clock time if there's no block time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 2, 300, 0), 300); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 2, 300, 0), 300); // New transaction should use block time if lower than clock time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 3, 420, 400), 400); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 3, 420, 400), 400); // New transaction should use latest entry time if higher than // min(block time, clock time). - BOOST_CHECK_EQUAL(AddTx(m_wallet, 4, 500, 390), 400); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 4, 500, 390), 400); // If there are future entries, new transaction should use time of the // newest entry that is no more than 300 seconds ahead of the clock time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 5, 50, 600), 300); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 5, 50, 600), 300); // Reset mock time for other tests. SetMockTime(0); @@ -486,7 +492,7 @@ public: ListCoinsTestingSetup() { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); - wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), CreateMockWalletDatabase()); { LOCK2(wallet->cs_wallet, ::cs_main); wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -604,7 +610,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { NodeContext node; auto chain = interfaces::MakeChain(node); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); @@ -790,4 +796,37 @@ BOOST_FIXTURE_TEST_CASE(CreateWalletFromFile, TestChain100Setup) TestUnloadWallet(std::move(wallet)); } +BOOST_FIXTURE_TEST_CASE(ZapSelectTx, TestChain100Setup) +{ + auto chain = interfaces::MakeChain(m_node); + auto wallet = TestLoadWallet(*chain); + CKey key; + key.MakeNewKey(true); + AddKey(*wallet, key); + + std::string error; + m_coinbase_txns.push_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + auto block_tx = TestSimpleSpend(*m_coinbase_txns[0], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); + CreateAndProcessBlock({block_tx}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + + SyncWithValidationInterfaceQueue(); + + { + auto block_hash = block_tx.GetHash(); + auto prev_hash = m_coinbase_txns[0]->GetHash(); + + LOCK(wallet->cs_wallet); + BOOST_CHECK(wallet->HasWalletSpend(prev_hash)); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_hash), 1u); + + std::vector<uint256> vHashIn{ block_hash }, vHashOut; + BOOST_CHECK_EQUAL(wallet->ZapSelectTx(vHashIn, vHashOut), DBErrors::LOAD_OK); + + BOOST_CHECK(!wallet->HasWalletSpend(prev_hash)); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_hash), 0u); + } + + TestUnloadWallet(std::move(wallet)); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index a59aa8b980..cee2f2214c 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -77,12 +77,6 @@ bool RemoveWallet(const std::shared_ptr<CWallet>& wallet) return true; } -bool HasWallets() -{ - LOCK(cs_wallets); - return !vpwallets.empty(); -} - std::vector<std::shared_ptr<CWallet>> GetWallets() { LOCK(cs_wallets); @@ -105,9 +99,11 @@ std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet) return interfaces::MakeHandler([it] { LOCK(cs_wallets); g_load_wallet_fns.erase(it); }); } +static Mutex g_loading_wallet_mutex; static Mutex g_wallet_release_mutex; static std::condition_variable g_wallet_release_cv; -static std::set<std::string> g_unloading_wallet_set; +static std::set<std::string> g_loading_wallet_set GUARDED_BY(g_loading_wallet_mutex); +static std::set<std::string> g_unloading_wallet_set GUARDED_BY(g_wallet_release_mutex); // Custom deleter for shared_ptr<CWallet>. static void ReleaseWallet(CWallet* wallet) @@ -151,7 +147,8 @@ void UnloadWallet(std::shared_ptr<CWallet>&& wallet) } } -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) +namespace { +std::shared_ptr<CWallet> LoadWalletInternal(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) { try { if (!CWallet::Verify(chain, location, error, warnings)) { @@ -172,6 +169,19 @@ std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocati return nullptr; } } +} // namespace + +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) +{ + auto result = WITH_LOCK(g_loading_wallet_mutex, return g_loading_wallet_set.insert(location.GetName())); + if (!result.second) { + error = Untranslated("Wallet already being loading."); + return nullptr; + } + auto wallet = LoadWalletInternal(chain, location, error, warnings); + WITH_LOCK(g_loading_wallet_mutex, g_loading_wallet_set.erase(result.first)); + return wallet; +} std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) { @@ -429,9 +439,14 @@ bool CWallet::HasWalletSpend(const uint256& txid) const return (iter != mapTxSpends.end() && iter->first.hash == txid); } -void CWallet::Flush(bool shutdown) +void CWallet::Flush() { - database->Flush(shutdown); + database->Flush(); +} + +void CWallet::Close() +{ + database->Close(); } void CWallet::SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator> range) @@ -752,7 +767,6 @@ void CWallet::SetSpentKeyState(WalletBatch& batch, const uint256& hash, unsigned bool CWallet::IsSpentKey(const uint256& hash, unsigned int n) const { AssertLockHeld(cs_wallet); - CTxDestination dst; const CWalletTx* srctx = GetWalletTx(hash); if (srctx) { assert(srctx->tx->vout.size() > n); @@ -1413,19 +1427,28 @@ bool CWallet::IsWalletFlagSet(uint64_t flag) const return (m_wallet_flags & flag); } -bool CWallet::SetWalletFlags(uint64_t overwriteFlags, bool memonly) +bool CWallet::LoadWalletFlags(uint64_t flags) { LOCK(cs_wallet); - m_wallet_flags = overwriteFlags; - if (((overwriteFlags & KNOWN_WALLET_FLAGS) >> 32) ^ (overwriteFlags >> 32)) { + if (((flags & KNOWN_WALLET_FLAGS) >> 32) ^ (flags >> 32)) { // contains unknown non-tolerable wallet flags return false; } - if (!memonly && !WalletBatch(*database).WriteWalletFlags(m_wallet_flags)) { + m_wallet_flags = flags; + + return true; +} + +bool CWallet::AddWalletFlags(uint64_t flags) +{ + LOCK(cs_wallet); + // We should never be writing unknown non-tolerable wallet flags + assert(((flags & KNOWN_WALLET_FLAGS) >> 32) == (flags >> 32)); + if (!WalletBatch(*database).WriteWalletFlags(flags)) { throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed"); } - return true; + return LoadWalletFlags(flags); } int64_t CWalletTx::GetTxTime() const @@ -2167,6 +2190,11 @@ void CWallet::AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe, const } for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { + // Only consider selected coins if add_inputs is false + if (coinControl && !coinControl->m_add_inputs && !coinControl->IsSelected(COutPoint(entry.first, i))) { + continue; + } + if (wtx.tx->vout[i].nValue < nMinimumAmount || wtx.tx->vout[i].nValue > nMaximumAmount) continue; @@ -2478,8 +2506,11 @@ bool CWallet::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, return false; } -TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& complete, int sighash_type, bool sign, bool bip32derivs) const +TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& complete, int sighash_type, bool sign, bool bip32derivs, size_t * n_signed) const { + if (n_signed) { + *n_signed = 0; + } LOCK(cs_wallet); // Get all of the previous transactions for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { @@ -2490,13 +2521,8 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp continue; } - // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. - if (!input.IsSane()) { - return TransactionError::INVALID_PSBT; - } - // If we have no utxo, grab it from the wallet. - if (!input.non_witness_utxo && input.witness_utxo.IsNull()) { + if (!input.non_witness_utxo) { const uint256& txhash = txin.prevout.hash; const auto it = mapWallet.find(txhash); if (it != mapWallet.end()) { @@ -2510,10 +2536,15 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp // Fill in information from ScriptPubKeyMans for (ScriptPubKeyMan* spk_man : GetAllScriptPubKeyMans()) { - TransactionError res = spk_man->FillPSBT(psbtx, sighash_type, sign, bip32derivs); + int n_signed_this_spkm = 0; + TransactionError res = spk_man->FillPSBT(psbtx, sighash_type, sign, bip32derivs, &n_signed_this_spkm); if (res != TransactionError::OK) { return res; } + + if (n_signed) { + (*n_signed) += n_signed_this_spkm; + } } // Complete if every input is now signed @@ -2647,11 +2678,11 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, const uin return locktime; } -OutputType CWallet::TransactionChangeType(OutputType change_type, const std::vector<CRecipient>& vecSend) +OutputType CWallet::TransactionChangeType(const Optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend) { // If -changetype is specified, always use that change type. - if (change_type != OutputType::CHANGE_AUTO) { - return change_type; + if (change_type) { + return *change_type; } // if m_default_address_type is legacy, use legacy address as change (even @@ -2748,6 +2779,12 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CTransac // Get the fee rate to use effective values in coin selection CFeeRate nFeeRateNeeded = GetMinimumFeeRate(*this, coin_control, &feeCalc); + // Do not, ever, assume that it's fine to change the fee rate if the user has explicitly + // provided one + if (coin_control.m_feerate && nFeeRateNeeded > *coin_control.m_feerate) { + error = strprintf(_("Fee rate (%s) is lower than the minimum fee rate setting (%s)"), coin_control.m_feerate->ToString(), nFeeRateNeeded.ToString()); + return false; + } nFeeRet = 0; bool pick_new_inputs = true; @@ -3097,9 +3134,11 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 { AssertLockHeld(cs_wallet); DBErrors nZapSelectTxRet = WalletBatch(*database, "cr+").ZapSelectTx(vHashIn, vHashOut); - for (uint256 hash : vHashOut) { + for (const uint256& hash : vHashOut) { const auto& it = mapWallet.find(hash); wtxOrdered.erase(it->second.m_it_wtxOrdered); + for (const auto& txin : it->second.tx->vin) + mapTxSpends.erase(txin.prevout); mapWallet.erase(it); NotifyTransactionChanged(this, hash, CT_DELETED); } @@ -3706,18 +3745,14 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b } // Keep same database environment instance across Verify/Recover calls below. - std::unique_ptr<WalletDatabase> database = WalletDatabase::Create(wallet_path); + std::unique_ptr<WalletDatabase> database = CreateWalletDatabase(wallet_path); try { - if (!WalletBatch::VerifyEnvironment(wallet_path, error_string)) { - return false; - } + return database->Verify(error_string); } catch (const fs::filesystem_error& e) { error_string = Untranslated(strprintf("Error loading wallet %s. %s", location.GetName(), fsbridge::get_filesystem_error_message(e))); return false; } - - return WalletBatch::VerifyDatabaseFile(wallet_path, error_string); } std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings, uint64_t wallet_creation_flags) @@ -3730,7 +3765,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.GetBoolArg("-zapwallettxes", false)) { chain.initMessage(_("Zapping all transactions from wallet...").translated); - std::unique_ptr<CWallet> tempWallet = MakeUnique<CWallet>(&chain, location, WalletDatabase::Create(location.GetPath())); + std::unique_ptr<CWallet> tempWallet = MakeUnique<CWallet>(&chain, location, CreateWalletDatabase(location.GetPath())); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DBErrors::LOAD_OK) { error = strprintf(_("Error loading %s: Wallet corrupted"), walletFile); @@ -3744,7 +3779,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, bool fFirstRun = true; // TODO: Can't use std::make_shared because we need a custom deleter but // should be possible to use std::allocate_shared. - std::shared_ptr<CWallet> walletInstance(new CWallet(&chain, location, WalletDatabase::Create(location.GetPath())), ReleaseWallet); + std::shared_ptr<CWallet> walletInstance(new CWallet(&chain, location, CreateWalletDatabase(location.GetPath())), ReleaseWallet); DBErrors nLoadWalletRet = walletInstance->LoadWallet(fFirstRun); if (nLoadWalletRet != DBErrors::LOAD_OK) { if (nLoadWalletRet == DBErrors::CORRUPT) { @@ -3777,7 +3812,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // ensure this wallet.dat can only be opened by clients supporting HD with chain split and expects no default key walletInstance->SetMinVersion(FEATURE_LATEST); - walletInstance->SetWalletFlags(wallet_creation_flags, false); + walletInstance->AddWalletFlags(wallet_creation_flags); // Only create LegacyScriptPubKeyMan when not descriptor wallet if (!walletInstance->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { @@ -3814,14 +3849,20 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - if (!gArgs.GetArg("-addresstype", "").empty() && !ParseOutputType(gArgs.GetArg("-addresstype", ""), walletInstance->m_default_address_type)) { - error = strprintf(_("Unknown address type '%s'"), gArgs.GetArg("-addresstype", "")); - return nullptr; + if (!gArgs.GetArg("-addresstype", "").empty()) { + if (!ParseOutputType(gArgs.GetArg("-addresstype", ""), walletInstance->m_default_address_type)) { + error = strprintf(_("Unknown address type '%s'"), gArgs.GetArg("-addresstype", "")); + return nullptr; + } } - if (!gArgs.GetArg("-changetype", "").empty() && !ParseOutputType(gArgs.GetArg("-changetype", ""), walletInstance->m_default_change_type)) { - error = strprintf(_("Unknown change type '%s'"), gArgs.GetArg("-changetype", "")); - return nullptr; + if (!gArgs.GetArg("-changetype", "").empty()) { + OutputType out_type; + if (!ParseOutputType(gArgs.GetArg("-changetype", ""), out_type)) { + error = strprintf(_("Unknown change type '%s'"), gArgs.GetArg("-changetype", "")); + return nullptr; + } + walletInstance->m_default_change_type = out_type; } if (gArgs.IsArgSet("-mintxfee")) { @@ -4392,12 +4433,21 @@ void CWallet::SetupDescriptorScriptPubKeyMans() spk_manager->SetupDescriptorGeneration(master_key, t); uint256 id = spk_manager->GetID(); m_spk_managers[id] = std::move(spk_manager); - SetActiveScriptPubKeyMan(id, t, internal); + AddActiveScriptPubKeyMan(id, t, internal); } } } -void CWallet::SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal, bool memonly) +void CWallet::AddActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal) +{ + WalletBatch batch(*database); + if (!batch.WriteActiveScriptPubKeyMan(static_cast<uint8_t>(type), id, internal)) { + throw std::runtime_error(std::string(__func__) + ": writing active ScriptPubKeyMan id failed"); + } + LoadActiveScriptPubKeyMan(id, type, internal); +} + +void CWallet::LoadActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal) { WalletLogPrintf("Setting spkMan to active: id = %s, type = %d, internal = %d\n", id.ToString(), static_cast<int>(type), static_cast<int>(internal)); auto& spk_mans = internal ? m_internal_spk_managers : m_external_spk_managers; @@ -4405,12 +4455,6 @@ void CWallet::SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool interna spk_man->SetInternal(internal); spk_mans[type] = spk_man; - if (!memonly) { - WalletBatch batch(*database); - if (!batch.WriteActiveScriptPubKeyMan(static_cast<uint8_t>(type), id, internal)) { - throw std::runtime_error(std::string(__func__) + ": writing active ScriptPubKeyMan id failed"); - } - } NotifyCanGetAddressesChanged(); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 67331dc3be..a761caf38c 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -13,11 +13,11 @@ #include <policy/feerate.h> #include <psbt.h> #include <tinyformat.h> -#include <ui_interface.h> #include <util/message.h> #include <util/strencodings.h> #include <util/string.h> #include <util/system.h> +#include <util/ui_change_type.h> #include <validationinterface.h> #include <wallet/coinselection.h> #include <wallet/crypter.h> @@ -51,7 +51,6 @@ void UnloadWallet(std::shared_ptr<CWallet>&& wallet); bool AddWallet(const std::shared_ptr<CWallet>& wallet); bool RemoveWallet(const std::shared_ptr<CWallet>& wallet); -bool HasWallets(); std::vector<std::shared_ptr<CWallet>> GetWallets(); std::shared_ptr<CWallet> GetWallet(const std::string& name); std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings); @@ -106,9 +105,6 @@ class ReserveDestination; //! Default for -addresstype constexpr OutputType DEFAULT_ADDRESS_TYPE{OutputType::BECH32}; -//! Default for -changetype -constexpr OutputType DEFAULT_CHANGE_TYPE{OutputType::CHANGE_AUTO}; - static constexpr uint64_t KNOWN_WALLET_FLAGS = WALLET_FLAG_AVOID_REUSE | WALLET_FLAG_BLANK_WALLET @@ -935,7 +931,7 @@ public: Balance GetBalance(int min_depth = 0, bool avoid_reuse = true) const; CAmount GetAvailableBalance(const CCoinControl* coinControl = nullptr) const; - OutputType TransactionChangeType(OutputType change_type, const std::vector<CRecipient>& vecSend); + OutputType TransactionChangeType(const Optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend); /** * Insert additional inputs into the transaction by @@ -965,7 +961,8 @@ public: bool& complete, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, - bool bip32derivs = true) const; + bool bip32derivs = true, + size_t* n_signed = nullptr) const; /** * Create a new transaction paying the recipients with a set of coins @@ -1012,7 +1009,13 @@ public: CFeeRate m_fallback_fee{DEFAULT_FALLBACK_FEE}; CFeeRate m_discard_rate{DEFAULT_DISCARD_FEE}; OutputType m_default_address_type{DEFAULT_ADDRESS_TYPE}; - OutputType m_default_change_type{DEFAULT_CHANGE_TYPE}; + /** + * Default output type for change outputs. When unset, automatically choose type + * based on address type setting and the types other of non-change outputs + * (see -changetype option documentation and implementation in + * CWallet::TransactionChangeType for details). + */ + Optional<OutputType> m_default_change_type{}; /** Absolute maximum transaction fee (in satoshis) used by default for the wallet */ CAmount m_default_max_tx_fee{DEFAULT_TRANSACTION_MAXFEE}; @@ -1084,7 +1087,10 @@ public: bool HasWalletSpend(const uint256& txid) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Flush wallet (bitdb flush) - void Flush(bool shutdown=false); + void Flush(); + + //! Close wallet database + void Close(); /** Wallet is about to be unloaded */ boost::signals2::signal<void ()> NotifyUnload; @@ -1173,7 +1179,9 @@ public: /** overwrite all flags by the given uint64_t returns false if unknown, non-tolerable flags are present */ - bool SetWalletFlags(uint64_t overwriteFlags, bool memOnly); + bool AddWalletFlags(uint64_t flags); + /** Loads the flags into the wallet. (used by LoadWallet) */ + bool LoadWalletFlags(uint64_t flags); /** Determine if we are a legacy wallet */ bool IsLegacy() const; @@ -1251,12 +1259,17 @@ public: //! Instantiate a descriptor ScriptPubKeyMan from the WalletDescriptor and load it void LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc); - //! Sets the active ScriptPubKeyMan for the specified type and internal + //! Adds the active ScriptPubKeyMan for the specified type and internal. Writes it to the wallet file + //! @param[in] id The unique id for the ScriptPubKeyMan + //! @param[in] type The OutputType this ScriptPubKeyMan provides addresses for + //! @param[in] internal Whether this ScriptPubKeyMan provides change addresses + void AddActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal); + + //! Loads an active ScriptPubKeyMan for the specified type and internal. (used by LoadWallet) //! @param[in] id The unique id for the ScriptPubKeyMan //! @param[in] type The OutputType this ScriptPubKeyMan provides addresses for //! @param[in] internal Whether this ScriptPubKeyMan provides change addresses - //! @param[in] memonly Whether to record this update to the database. Set to true for wallet loading, normally false when actually updating the wallet. - void SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal, bool memonly = false); + void LoadActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal); //! Create new DescriptorScriptPubKeyMans and add them to the wallet void SetupDescriptorScriptPubKeyMans(); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index cb516f70f0..8c409b40cd 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -121,7 +121,7 @@ bool WalletBatch::WriteCryptedKey(const CPubKey& vchPubKey, if (!WriteIC(key, std::make_pair(vchCryptedSecret, checksum), false)) { // It may already exist, so try writing just the checksum std::vector<unsigned char> val; - if (!m_batch.Read(key, val)) { + if (!m_batch->Read(key, val)) { return false; } if (!WriteIC(key, std::make_pair(val, checksum), true)) { @@ -166,8 +166,8 @@ bool WalletBatch::WriteBestBlock(const CBlockLocator& locator) bool WalletBatch::ReadBestBlock(CBlockLocator& locator) { - if (m_batch.Read(DBKeys::BESTBLOCK, locator) && !locator.vHave.empty()) return true; - return m_batch.Read(DBKeys::BESTBLOCK_NOMERKLE, locator); + if (m_batch->Read(DBKeys::BESTBLOCK, locator) && !locator.vHave.empty()) return true; + return m_batch->Read(DBKeys::BESTBLOCK_NOMERKLE, locator); } bool WalletBatch::WriteOrderPosNext(int64_t nOrderPosNext) @@ -177,7 +177,7 @@ bool WalletBatch::WriteOrderPosNext(int64_t nOrderPosNext) bool WalletBatch::ReadPool(int64_t nPool, CKeyPool& keypool) { - return m_batch.Read(std::make_pair(DBKeys::POOL, nPool), keypool); + return m_batch->Read(std::make_pair(DBKeys::POOL, nPool), keypool); } bool WalletBatch::WritePool(int64_t nPool, const CKeyPool& keypool) @@ -439,10 +439,11 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, // Extract some CHDChain info from this metadata if it has any if (keyMeta.nVersion >= CKeyMetadata::VERSION_WITH_HDDATA && !keyMeta.hd_seed_id.IsNull() && keyMeta.hdKeypath.size() > 0) { // Get the path from the key origin or from the path string - // Not applicable when path is "s" as that indicates a seed + // Not applicable when path is "s" or "m" as those indicate a seed + // See https://github.com/bitcoin/bitcoin/pull/12924 bool internal = false; uint32_t index = 0; - if (keyMeta.hdKeypath != "s") { + if (keyMeta.hdKeypath != "s" && keyMeta.hdKeypath != "m") { std::vector<uint32_t> path; if (keyMeta.has_key_origin) { // We have a key origin, so pull it from its path vector @@ -538,11 +539,11 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } else if (strType == DBKeys::HDCHAIN) { CHDChain chain; ssValue >> chain; - pwallet->GetOrCreateLegacyScriptPubKeyMan()->SetHDChain(chain, true); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadHDChain(chain); } else if (strType == DBKeys::FLAGS) { uint64_t flags; ssValue >> flags; - if (!pwallet->SetWalletFlags(flags, true)) { + if (!pwallet->LoadWalletFlags(flags)) { strErr = "Error reading wallet database: Unknown non-tolerable wallet flags found"; return false; } @@ -591,9 +592,6 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, ssValue >> ser_xpub; CExtPubKey xpub; xpub.Decode(ser_xpub.data()); - if (wss.m_descriptor_caches.count(desc_id)) { - wss.m_descriptor_caches[desc_id] = DescriptorCache(); - } if (parent) { wss.m_descriptor_caches[desc_id].CacheParentExtPubKey(key_exp_index, xpub); } else { @@ -692,15 +690,14 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) LOCK(pwallet->cs_wallet); try { int nMinVersion = 0; - if (m_batch.Read(DBKeys::MINVERSION, nMinVersion)) { + if (m_batch->Read(DBKeys::MINVERSION, nMinVersion)) { if (nMinVersion > FEATURE_LATEST) return DBErrors::TOO_NEW; pwallet->LoadMinVersion(nMinVersion); } // Get cursor - Dbc* pcursor = m_batch.GetCursor(); - if (!pcursor) + if (!m_batch->StartCursor()) { pwallet->WalletLogPrintf("Error getting wallet database cursor\n"); return DBErrors::CORRUPT; @@ -711,11 +708,14 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = m_batch.ReadAtCursor(pcursor, ssKey, ssValue); - if (ret == DB_NOTFOUND) + bool complete; + bool ret = m_batch->ReadAtCursor(ssKey, ssValue, complete); + if (complete) { break; - else if (ret != 0) + } + else if (!ret) { + m_batch->CloseCursor(); pwallet->WalletLogPrintf("Error reading next record from wallet database\n"); return DBErrors::CORRUPT; } @@ -742,17 +742,17 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (!strErr.empty()) pwallet->WalletLogPrintf("%s\n", strErr); } - pcursor->close(); } catch (...) { result = DBErrors::CORRUPT; } + m_batch->CloseCursor(); // Set the active ScriptPubKeyMans for (auto spk_man_pair : wss.m_active_external_spks) { - pwallet->SetActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ false, /* memonly */ true); + pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ false); } for (auto spk_man_pair : wss.m_active_internal_spks) { - pwallet->SetActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ true, /* memonly */ true); + pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ true); } // Set the descriptor caches @@ -782,7 +782,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) // Last client version to open this wallet, was previously the file version number int last_client = CLIENT_VERSION; - m_batch.Read(DBKeys::VERSION, last_client); + m_batch->Read(DBKeys::VERSION, last_client); int wallet_version = pwallet->GetVersion(); pwallet->WalletLogPrintf("Wallet File Version = %d\n", wallet_version > 0 ? wallet_version : last_client); @@ -807,7 +807,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) return DBErrors::NEED_REWRITE; if (last_client < CLIENT_VERSION) // Update - m_batch.Write(DBKeys::VERSION, CLIENT_VERSION); + m_batch->Write(DBKeys::VERSION, CLIENT_VERSION); if (wss.fAnyUnordered) result = pwallet->ReorderTransactions(); @@ -843,14 +843,13 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWal try { int nMinVersion = 0; - if (m_batch.Read(DBKeys::MINVERSION, nMinVersion)) { + if (m_batch->Read(DBKeys::MINVERSION, nMinVersion)) { if (nMinVersion > FEATURE_LATEST) return DBErrors::TOO_NEW; } // Get cursor - Dbc* pcursor = m_batch.GetCursor(); - if (!pcursor) + if (!m_batch->StartCursor()) { LogPrintf("Error getting wallet database cursor\n"); return DBErrors::CORRUPT; @@ -861,11 +860,12 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWal // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = m_batch.ReadAtCursor(pcursor, ssKey, ssValue); - if (ret == DB_NOTFOUND) + bool complete; + bool ret = m_batch->ReadAtCursor(ssKey, ssValue, complete); + if (complete) { break; - else if (ret != 0) - { + } else if (!ret) { + m_batch->CloseCursor(); LogPrintf("Error reading next record from wallet database\n"); return DBErrors::CORRUPT; } @@ -880,10 +880,10 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWal ssValue >> vWtx.back(); } } - pcursor->close(); } catch (...) { result = DBErrors::CORRUPT; } + m_batch->CloseCursor(); return result; } @@ -949,9 +949,6 @@ void MaybeCompactWalletDB() if (fOneThread.exchange(true)) { return; } - if (!gArgs.GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) { - return; - } for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { WalletDatabase& dbh = pwallet->GetDBHandle(); @@ -964,7 +961,7 @@ void MaybeCompactWalletDB() } if (dbh.nLastFlushed != nUpdateCounter && GetTime() - dbh.nLastWalletUpdate >= 2) { - if (BerkeleyBatch::PeriodicFlush(dbh)) { + if (dbh.PeriodicFlush()) { dbh.nLastFlushed = nUpdateCounter; } } @@ -973,16 +970,6 @@ void MaybeCompactWalletDB() fOneThread = false; } -bool WalletBatch::VerifyEnvironment(const fs::path& wallet_path, bilingual_str& errorStr) -{ - return BerkeleyBatch::VerifyEnvironment(wallet_path, errorStr); -} - -bool WalletBatch::VerifyDatabaseFile(const fs::path& wallet_path, bilingual_str& errorStr) -{ - return BerkeleyBatch::VerifyDatabaseFile(wallet_path, errorStr); -} - bool WalletBatch::WriteDestData(const std::string &address, const std::string &key, const std::string &value) { return WriteIC(std::make_pair(DBKeys::DESTDATA, std::make_pair(address, key)), value); @@ -1006,15 +993,39 @@ bool WalletBatch::WriteWalletFlags(const uint64_t flags) bool WalletBatch::TxnBegin() { - return m_batch.TxnBegin(); + return m_batch->TxnBegin(); } bool WalletBatch::TxnCommit() { - return m_batch.TxnCommit(); + return m_batch->TxnCommit(); } bool WalletBatch::TxnAbort() { - return m_batch.TxnAbort(); + return m_batch->TxnAbort(); +} + +bool IsWalletLoaded(const fs::path& wallet_path) +{ + return IsBDBWalletLoaded(wallet_path); +} + +/** Return object for accessing database at specified path. */ +std::unique_ptr<WalletDatabase> CreateWalletDatabase(const fs::path& path) +{ + std::string filename; + return MakeUnique<BerkeleyDatabase>(GetWalletEnv(path, filename), std::move(filename)); +} + +/** Return object for accessing dummy database with no read/write capabilities. */ +std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase() +{ + return MakeUnique<BerkeleyDatabase>(); +} + +/** Return object for accessing temporary in-memory database. */ +std::unique_ptr<WalletDatabase> CreateMockWalletDatabase() +{ + return MakeUnique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), ""); } diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index b95ed24d12..7c5bf7652b 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -8,6 +8,7 @@ #include <amount.h> #include <script/sign.h> +#include <wallet/bdb.h> #include <wallet/db.h> #include <wallet/walletutil.h> #include <key.h> @@ -39,9 +40,6 @@ class CWalletTx; class uint160; class uint256; -/** Backend-agnostic database type. */ -using WalletDatabase = BerkeleyDatabase; - /** Error statuses for the wallet database */ enum class DBErrors { @@ -182,12 +180,12 @@ private: template <typename K, typename T> bool WriteIC(const K& key, const T& value, bool fOverwrite = true) { - if (!m_batch.Write(key, value, fOverwrite)) { + if (!m_batch->Write(key, value, fOverwrite)) { return false; } m_database.IncrementUpdateCounter(); if (m_database.nUpdateCounter % 1000 == 0) { - m_batch.Flush(); + m_batch->Flush(); } return true; } @@ -195,19 +193,19 @@ private: template <typename K> bool EraseIC(const K& key) { - if (!m_batch.Erase(key)) { + if (!m_batch->Erase(key)) { return false; } m_database.IncrementUpdateCounter(); if (m_database.nUpdateCounter % 1000 == 0) { - m_batch.Flush(); + m_batch->Flush(); } return true; } public: explicit WalletBatch(WalletDatabase& database, const char* pszMode = "r+", bool _fFlushOnClose = true) : - m_batch(database, pszMode, _fFlushOnClose), + m_batch(database.MakeBatch(pszMode, _fFlushOnClose)), m_database(database) { } @@ -279,7 +277,7 @@ public: //! Abort current transaction bool TxnAbort(); private: - BerkeleyBatch m_batch; + std::unique_ptr<DatabaseBatch> m_batch; WalletDatabase& m_database; }; @@ -289,4 +287,16 @@ void MaybeCompactWalletDB(); //! Unserialize a given Key-Value pair and load it into the wallet bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr); +/** Return whether a wallet database is currently loaded. */ +bool IsWalletLoaded(const fs::path& wallet_path); + +/** Return object for accessing database at specified path. */ +std::unique_ptr<WalletDatabase> CreateWalletDatabase(const fs::path& path); + +/** Return object for accessing dummy database with no read/write capabilities. */ +std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase(); + +/** Return object for accessing temporary in-memory database. */ +std::unique_ptr<WalletDatabase> CreateMockWalletDatabase(); + #endif // BITCOIN_WALLET_WALLETDB_H diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index be07c28503..9f25b1ae7d 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -17,7 +17,7 @@ namespace WalletTool { static void WalletToolReleaseWallet(CWallet* wallet) { wallet->WalletLogPrintf("Releasing wallet\n"); - wallet->Flush(true); + wallet->Close(); delete wallet; } @@ -28,7 +28,7 @@ static std::shared_ptr<CWallet> CreateWallet(const std::string& name, const fs:: return nullptr; } // dummy chain interface - std::shared_ptr<CWallet> wallet_instance(new CWallet(nullptr /* chain */, WalletLocation(name), WalletDatabase::Create(path)), WalletToolReleaseWallet); + std::shared_ptr<CWallet> wallet_instance(new CWallet(nullptr /* chain */, WalletLocation(name), CreateWalletDatabase(path)), WalletToolReleaseWallet); LOCK(wallet_instance->cs_wallet); bool first_run = true; DBErrors load_wallet_ret = wallet_instance->LoadWallet(first_run); @@ -57,7 +57,7 @@ static std::shared_ptr<CWallet> LoadWallet(const std::string& name, const fs::pa } // dummy chain interface - std::shared_ptr<CWallet> wallet_instance(new CWallet(nullptr /* chain */, WalletLocation(name), WalletDatabase::Create(path)), WalletToolReleaseWallet); + std::shared_ptr<CWallet> wallet_instance(new CWallet(nullptr /* chain */, WalletLocation(name), CreateWalletDatabase(path)), WalletToolReleaseWallet); DBErrors load_wallet_ret; try { bool first_run; @@ -107,12 +107,12 @@ static void WalletShowInfo(CWallet* wallet_instance) static bool SalvageWallet(const fs::path& path) { // Create a Database handle to allow for the db to be initialized before recovery - std::unique_ptr<WalletDatabase> database = WalletDatabase::Create(path); + std::unique_ptr<WalletDatabase> database = CreateWalletDatabase(path); // Initialize the environment before recovery bilingual_str error_string; try { - WalletBatch::VerifyEnvironment(path, error_string); + database->Verify(error_string); } catch (const fs::filesystem_error& e) { error_string = Untranslated(strprintf("Error loading wallet. %s", fsbridge::get_filesystem_error_message(e))); } @@ -133,24 +133,19 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) std::shared_ptr<CWallet> wallet_instance = CreateWallet(name, path); if (wallet_instance) { WalletShowInfo(wallet_instance.get()); - wallet_instance->Flush(true); + wallet_instance->Close(); } } else if (command == "info" || command == "salvage") { if (!fs::exists(path)) { tfm::format(std::cerr, "Error: no wallet file at %s\n", name); return false; } - bilingual_str error; - if (!WalletBatch::VerifyEnvironment(path, error)) { - tfm::format(std::cerr, "%s\nError loading %s. Is wallet being used by other process?\n", error.original, name); - return false; - } if (command == "info") { std::shared_ptr<CWallet> wallet_instance = LoadWallet(name, path); if (!wallet_instance) return false; WalletShowInfo(wallet_instance.get()); - wallet_instance->Flush(true); + wallet_instance->Close(); } else if (command == "salvage") { return SalvageWallet(path); } diff --git a/test/README.md b/test/README.md index b036a66f67..2341eef00d 100644 --- a/test/README.md +++ b/test/README.md @@ -260,11 +260,11 @@ Use the `-v` option for verbose output. | Lint test | Dependency | Version [used by CI](../ci/lint/04_install.sh) | Installation |-----------|:----------:|:-------------------------------------------:|-------------- -| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) | [3.7.8](https://github.com/bitcoin/bitcoin/pull/15257) | `pip3 install flake8==3.7.8` -| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) | [0.700](https://github.com/bitcoin/bitcoin/pull/18210) | `pip3 install mypy==0.700` -| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) | [0.6.0](https://github.com/bitcoin/bitcoin/pull/15166) | [details...](https://github.com/koalaman/shellcheck#installing) +| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) | [3.8.3](https://github.com/bitcoin/bitcoin/pull/19348) | `pip3 install flake8==3.8.3` +| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) | [0.781](https://github.com/bitcoin/bitcoin/pull/19348) | `pip3 install mypy==0.781` +| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) | [0.7.1](https://github.com/bitcoin/bitcoin/pull/19348) | [details...](https://github.com/koalaman/shellcheck#installing) | [`lint-shell.sh`](lint/lint-shell.sh) | [yq](https://github.com/kislyuk/yq) | default | `pip3 install yq` -| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) | [1.15.0](https://github.com/bitcoin/bitcoin/pull/16186) | `pip3 install codespell==1.15.0` +| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) | [1.17.1](https://github.com/bitcoin/bitcoin/pull/19348) | `pip3 install codespell==1.17.1` Please be aware that on Linux distributions all dependencies are usually available as packages, but could be outdated. diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index 596ff206f2..07dd0f8f82 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -6,7 +6,7 @@ Test various backwards compatibility scenarios. Download the previous node binaries: -contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 +contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py. Due to a hardfork in regtest, it can't be used to sync nodes. @@ -26,10 +26,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.descriptors import descsum_create from test_framework.util import ( - adjust_bitcoin_conf_for_pre_17, assert_equal, - sync_blocks, - sync_mempools, ) @@ -60,15 +57,13 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): 170100, 160300, ]) - # adapt bitcoin.conf, because older bitcoind's don't recognize config sections - adjust_bitcoin_conf_for_pre_17(self.nodes[5].bitcoinconf) self.start_nodes() def run_test(self): self.nodes[0].generatetoaddress(101, self.nodes[0].getnewaddress()) - sync_blocks(self.nodes) + self.sync_blocks() # Sanity check the test framework: res = self.nodes[self.num_nodes - 1].getblockchaininfo() @@ -93,17 +88,17 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # Create a confirmed transaction, receiving coins address = wallet.getnewaddress() self.nodes[0].sendtoaddress(address, 10) - sync_mempools(self.nodes) + self.sync_mempools() self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # Create a conflicting transaction using RBF return_address = self.nodes[0].getnewaddress() tx1_id = self.nodes[1].sendtoaddress(return_address, 1) tx2_id = self.nodes[1].bumpfee(tx1_id)["txid"] # Confirm the transaction - sync_mempools(self.nodes) + self.sync_mempools() self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # Create another conflicting transaction using RBF tx3_id = self.nodes[1].sendtoaddress(return_address, 1) tx4_id = self.nodes[1].bumpfee(tx3_id)["txid"] @@ -316,12 +311,19 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.start_node(4) # Open most recent wallet in v0.16 (no loadwallet RPC) - self.stop_node(5) - self.start_node(5, extra_args=["-wallet=w2"]) + self.restart_node(5, extra_args=["-wallet=w2"]) wallet = node_v16.get_wallet_rpc("w2") info = wallet.getwalletinfo() assert info['keypoolsize'] == 1 + # Create upgrade wallet in v0.16 + self.restart_node(-1, extra_args=["-wallet=u1_v16"]) + wallet = node_v16.get_wallet_rpc("u1_v16") + v16_addr = wallet.getnewaddress('', "bech32") + v16_info = wallet.validateaddress(v16_addr) + v16_pubkey = v16_info['pubkey'] + self.stop_node(-1) + self.log.info("Test wallet upgrade path...") # u1: regular wallet, created with v0.17 node_v17.rpc.createwallet(wallet_name="u1_v17") @@ -331,6 +333,30 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): hdkeypath = v17_info["hdkeypath"] pubkey = v17_info["pubkey"] + # Copy the 0.16 wallet to the last Bitcoin Core version and open it: + shutil.copyfile( + os.path.join(node_v16_wallets_dir, "wallets/u1_v16"), + os.path.join(node_master_wallets_dir, "u1_v16") + ) + load_res = node_master.loadwallet("u1_v16") + # Make sure this wallet opens without warnings. See https://github.com/bitcoin/bitcoin/pull/19054 + assert_equal(load_res['warning'], '') + wallet = node_master.get_wallet_rpc("u1_v16") + info = wallet.getaddressinfo(v16_addr) + descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + v16_pubkey + ")" + assert_equal(info["desc"], descsum_create(descriptor)) + + # Now copy that same wallet back to 0.16 to make sure no automatic upgrade breaks it + os.remove(os.path.join(node_v16_wallets_dir, "wallets/u1_v16")) + shutil.copyfile( + os.path.join(node_master_wallets_dir, "u1_v16"), + os.path.join(node_v16_wallets_dir, "wallets/u1_v16") + ) + self.start_node(-1, extra_args=["-wallet=u1_v16"]) + wallet = node_v16.get_wallet_rpc("u1_v16") + info = wallet.validateaddress(v16_addr) + assert_equal(info, v16_info) + # Copy the 0.17 wallet to the last Bitcoin Core version and open it: node_v17.unloadwallet("u1_v17") shutil.copytree( diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 549e8b2029..19cdc10935 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -30,7 +30,10 @@ class BIP68Test(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.extra_args = [ - ["-acceptnonstdtxn=1"], + [ + "-acceptnonstdtxn=1", + "-peertimeout=9999", # bump because mocktime might cause a disconnect otherwise + ], ["-acceptnonstdtxn=0"], ] diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index a4dc455d57..34e856c1ba 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -112,11 +112,38 @@ class ConfArgsTest(BitcoinTestFramework): ]) self.stop_node(0) + def test_networkactive(self): + self.log.info('Test -networkactive option') + with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: true\n']): + self.start_node(0) + self.stop_node(0) + + with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: true\n']): + self.start_node(0, extra_args=['-networkactive']) + self.stop_node(0) + + with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: true\n']): + self.start_node(0, extra_args=['-networkactive=1']) + self.stop_node(0) + + with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: false\n']): + self.start_node(0, extra_args=['-networkactive=0']) + self.stop_node(0) + + with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: false\n']): + self.start_node(0, extra_args=['-nonetworkactive']) + self.stop_node(0) + + with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: false\n']): + self.start_node(0, extra_args=['-nonetworkactive=1']) + self.stop_node(0) + def run_test(self): self.stop_node(0) self.test_log_buffer() self.test_args_log() + self.test_networkactive() self.test_config_file_parser() diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py index 82f1331685..0a457ca17f 100755 --- a/test/functional/feature_loadblock.py +++ b/test/functional/feature_loadblock.py @@ -71,8 +71,7 @@ class LoadblockTest(BitcoinTestFramework): check=True) self.log.info("Restart second, unsynced node with bootstrap file") - self.stop_node(1) - self.start_node(1, ["-loadblock=" + bootstrap_file]) + self.restart_node(1, extra_args=["-loadblock=" + bootstrap_file]) assert_equal(self.nodes[1].getblockcount(), 100) # start_node is blocking on all block files being imported assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 100) diff --git a/test/functional/feature_logging.py b/test/functional/feature_logging.py index e4bf2d849d..afcbcf099a 100755 --- a/test/functional/feature_logging.py +++ b/test/functional/feature_logging.py @@ -67,8 +67,7 @@ class LoggingTest(BitcoinTestFramework): assert not os.path.isfile(default_log_path) # just sanity check no crash here - self.stop_node(0) - self.start_node(0, ["-debuglogfile=%s" % os.devnull]) + self.restart_node(0, ["-debuglogfile=%s" % os.devnull]) if __name__ == '__main__': diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 7eabf86cad..5538d6d3b4 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -35,7 +35,11 @@ class MaxUploadTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 - self.extra_args = [["-maxuploadtarget=800", "-acceptnonstdtxn=1"]] + self.extra_args = [[ + "-maxuploadtarget=800", + "-acceptnonstdtxn=1", + "-peertimeout=9999", # bump because mocktime might cause a disconnect otherwise + ]] self.supports_cli = False # Cache for utxos, as the listunspent may take a long time later in the test @@ -137,8 +141,8 @@ class MaxUploadTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() - self.log.info("Restarting node 0 with noban permission and 1MB maxuploadtarget") - self.restart_node(0, ["-whitelist=noban@127.0.0.1", "-maxuploadtarget=1"]) + self.log.info("Restarting node 0 with download permission and 1MB maxuploadtarget") + self.restart_node(0, ["-whitelist=download@127.0.0.1", "-maxuploadtarget=1"]) # Reconnect to self.nodes[0] self.nodes[0].add_p2p_connection(TestP2PConn()) @@ -151,9 +155,12 @@ class MaxUploadTest(BitcoinTestFramework): getdata_request.inv = [CInv(MSG_BLOCK, big_old_block)] self.nodes[0].p2p.send_and_ping(getdata_request) - assert_equal(len(self.nodes[0].getpeerinfo()), 1) #node is still connected because of the noban permission - self.log.info("Peer still connected after trying to download old block (noban permission)") + self.log.info("Peer still connected after trying to download old block (download permission)") + peer_info = self.nodes[0].getpeerinfo() + assert_equal(len(peer_info), 1) # node is still connected + assert_equal(peer_info[0]['permissions'], ['download']) + if __name__ == '__main__': MaxUploadTest().main() diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index 33a308ad1b..dd4c318cee 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -93,6 +93,7 @@ class NotificationsTest(BitcoinTestFramework): self.nodes[0].sethdseed(seed=self.nodes[1].dumpprivkey(keyhash_to_p2pkh(hex_str_to_bytes(self.nodes[1].getwalletinfo()['hdseedid'])[::-1]))) self.nodes[0].rescanblockchain() self.nodes[0].generatetoaddress(100, ADDRESS_BCRT1_UNSPENDABLE) + self.sync_blocks() # Generate transaction on node 0, sync mempools, and check for # notification on node 1. diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index c9362cf5aa..e46e5aacc8 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -263,8 +263,7 @@ class PruneTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "not in prune mode", node.pruneblockchain, 500) # now re-start in manual pruning mode - self.stop_node(node_number) - self.start_node(node_number, extra_args=["-prune=1"]) + self.restart_node(node_number, extra_args=["-prune=1"]) node = self.nodes[node_number] assert_equal(node.getblockcount(), 995) @@ -326,16 +325,14 @@ class PruneTest(BitcoinTestFramework): assert not has_block(3), "blk00003.dat is still there, should be pruned by now" # stop node, start back up with auto-prune at 550 MiB, make sure still runs - self.stop_node(node_number) - self.start_node(node_number, extra_args=["-prune=550"]) + self.restart_node(node_number, extra_args=["-prune=550"]) self.log.info("Success") def wallet_test(self): # check that the pruning node's wallet is still in good shape self.log.info("Stop and start pruning node to trigger wallet rescan") - self.stop_node(2) - self.start_node(2, extra_args=["-prune=550"]) + self.restart_node(2, extra_args=["-prune=550"]) self.log.info("Success") # check that wallet loads successfully when restarting a pruned node after IBD. @@ -344,8 +341,7 @@ class PruneTest(BitcoinTestFramework): connect_nodes(self.nodes[0], 5) nds = [self.nodes[0], self.nodes[5]] self.sync_blocks(nds, wait=5, timeout=300) - self.stop_node(5) # stop and start to trigger rescan - self.start_node(5, extra_args=["-prune=550"]) + self.restart_node(5, extra_args=["-prune=550"]) # restart to trigger rescan self.log.info("Success") def run_test(self): diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 2298485640..5195d20dcb 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -559,8 +559,7 @@ class SegWitTest(BitcoinTestFramework): assert_equal(self.nodes[1].listtransactions("*", 1, 0, True)[0]["txid"], txid) # Assert it is properly saved - self.stop_node(1) - self.start_node(1) + self.restart_node(1) assert_equal(self.nodes[1].gettransaction(txid, True)["txid"], txid) assert_equal(self.nodes[1].listtransactions("*", 1, 0, True)[0]["txid"], txid) diff --git a/test/functional/feature_settings.py b/test/functional/feature_settings.py new file mode 100755 index 0000000000..c565854bb0 --- /dev/null +++ b/test/functional/feature_settings.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test various command line arguments and configuration file parameters.""" + +import json + +from pathlib import Path + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_node import ErrorMatch +from test_framework.util import assert_equal + + +class SettingsTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + node, = self.nodes + settings = Path(node.datadir, self.chain, "settings.json") + conf = Path(node.datadir, "bitcoin.conf") + + # Assert empty settings file was created + self.stop_node(0) + with settings.open() as fp: + assert_equal(json.load(fp), {}) + + # Assert settings are parsed and logged + with settings.open("w") as fp: + json.dump({"string": "string", "num": 5, "bool": True, "null": None, "list": [6,7]}, fp) + with node.assert_debug_log(expected_msgs=[ + 'Setting file arg: string = "string"', + 'Setting file arg: num = 5', + 'Setting file arg: bool = true', + 'Setting file arg: null = null', + 'Setting file arg: list = [6,7]']): + self.start_node(0) + self.stop_node(0) + + # Assert settings are unchanged after shutdown + with settings.open() as fp: + assert_equal(json.load(fp), {"string": "string", "num": 5, "bool": True, "null": None, "list": [6,7]}) + + # Test invalid json + with settings.open("w") as fp: + fp.write("invalid json") + node.assert_start_raises_init_error(expected_msg='Unable to parse settings file', match=ErrorMatch.PARTIAL_REGEX) + + # Test invalid json object + with settings.open("w") as fp: + fp.write('"string"') + node.assert_start_raises_init_error(expected_msg='Found non-object value "string" in settings file', match=ErrorMatch.PARTIAL_REGEX) + + # Test invalid settings file containing duplicate keys + with settings.open("w") as fp: + fp.write('{"key": 1, "key": 2}') + node.assert_start_raises_init_error(expected_msg='Found duplicate key key in settings file', match=ErrorMatch.PARTIAL_REGEX) + + # Test invalid settings file is ignored with command line -nosettings + with node.assert_debug_log(expected_msgs=['Command-line arg: settings=false']): + self.start_node(0, extra_args=["-nosettings"]) + self.stop_node(0) + + # Test invalid settings file is ignored with config file -nosettings + with conf.open('a') as conf: + conf.write('nosettings=1\n') + with node.assert_debug_log(expected_msgs=['Config file arg: [regtest] settings=false']): + self.start_node(0) + self.stop_node(0) + + # Test alternate settings path + altsettings = Path(node.datadir, "altsettings.json") + with altsettings.open("w") as fp: + fp.write('{"key": "value"}') + with node.assert_debug_log(expected_msgs=['Setting file arg: key = "value"']): + self.start_node(0, extra_args=["-settings={}".format(altsettings)]) + self.stop_node(0) + + +if __name__ == '__main__': + SettingsTest().main() diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 7530e7daf6..80003aca0d 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -3,9 +3,15 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-cli""" + from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_process_error, get_auth_cookie +from test_framework.util import ( + assert_equal, + assert_raises_process_error, + assert_raises_rpc_error, + get_auth_cookie, +) # The block reward of coinbaseoutput.nValue (50) BTC/block matures after # COINBASE_MATURITY (100) blocks. Therefore, after mining 101 blocks we expect @@ -13,6 +19,12 @@ from test_framework.util import assert_equal, assert_raises_process_error, get_a BLOCKS = 101 BALANCE = (BLOCKS - 100) * 50 +JSON_PARSING_ERROR = 'error: Error parsing JSON: foo' +BLOCKS_VALUE_OF_ZERO = 'error: the first argument (number of blocks to generate, default: 1) must be an integer value greater than zero' +TOO_MANY_ARGS = 'error: too many arguments (maximum 2 for nblocks and maxtries)' +WALLET_NOT_LOADED = 'Requested wallet does not exist or is not loaded' +WALLET_NOT_SPECIFIED = 'Wallet file not specified' + class TestBitcoinCli(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -75,7 +87,7 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['relayfee'], network_info['relayfee']) assert_equal(self.nodes[0].cli.getwalletinfo(), wallet_info) - # Setup to test -getinfo and -rpcwallet= with multiple wallets. + # Setup to test -getinfo, -generate, and -rpcwallet= with multiple wallets. wallets = ['', 'Encrypted', 'secret'] amounts = [BALANCE + Decimal('9.999928'), Decimal(9), Decimal(31)] self.nodes[0].createwallet(wallet_name=wallets[1]) @@ -83,6 +95,8 @@ class TestBitcoinCli(BitcoinTestFramework): w1 = self.nodes[0].get_wallet_rpc(wallets[0]) w2 = self.nodes[0].get_wallet_rpc(wallets[1]) w3 = self.nodes[0].get_wallet_rpc(wallets[2]) + rpcwallet2 = '-rpcwallet={}'.format(wallets[1]) + rpcwallet3 = '-rpcwallet={}'.format(wallets[2]) w1.walletpassphrase(password, self.rpc_timeout) w2.encryptwallet(password) w1.sendtoaddress(w2.getnewaddress(), amounts[1]) @@ -123,17 +137,93 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['balance'], amounts[1]) self.log.info("Test -getinfo with -rpcwallet=remaining-non-default-wallet returns only its balance") - cli_get_info = self.nodes[0].cli('-getinfo', '-rpcwallet={}'.format(wallets[1])).send_cli() + cli_get_info = self.nodes[0].cli('-getinfo', rpcwallet2).send_cli() assert 'balances' not in cli_get_info.keys() assert_equal(cli_get_info['balance'], amounts[1]) self.log.info("Test -getinfo with -rpcwallet=unloaded wallet returns no balances") - cli_get_info = self.nodes[0].cli('-getinfo', '-rpcwallet={}'.format(wallets[2])).send_cli() + cli_get_info = self.nodes[0].cli('-getinfo', rpcwallet3).send_cli() assert 'balance' not in cli_get_info_keys assert 'balances' not in cli_get_info_keys + + # Test bitcoin-cli -generate. + n1 = 3 + n2 = 4 + w2.walletpassphrase(password, self.rpc_timeout) + blocks = self.nodes[0].getblockcount() + + self.log.info('Test -generate with no args') + generate = self.nodes[0].cli('-generate').send_cli() + assert_equal(set(generate.keys()), {'address', 'blocks'}) + assert_equal(len(generate["blocks"]), 1) + assert_equal(self.nodes[0].getblockcount(), blocks + 1) + + self.log.info('Test -generate with bad args') + assert_raises_process_error(1, JSON_PARSING_ERROR, self.nodes[0].cli('-generate', 'foo').echo) + assert_raises_process_error(1, BLOCKS_VALUE_OF_ZERO, self.nodes[0].cli('-generate', 0).echo) + assert_raises_process_error(1, TOO_MANY_ARGS, self.nodes[0].cli('-generate', 1, 2, 3).echo) + + self.log.info('Test -generate with nblocks') + generate = self.nodes[0].cli('-generate', n1).send_cli() + assert_equal(set(generate.keys()), {'address', 'blocks'}) + assert_equal(len(generate["blocks"]), n1) + assert_equal(self.nodes[0].getblockcount(), blocks + 1 + n1) + + self.log.info('Test -generate with nblocks and maxtries') + generate = self.nodes[0].cli('-generate', n2, 1000000).send_cli() + assert_equal(set(generate.keys()), {'address', 'blocks'}) + assert_equal(len(generate["blocks"]), n2) + assert_equal(self.nodes[0].getblockcount(), blocks + 1 + n1 + n2) + + self.log.info('Test -generate -rpcwallet in single-wallet mode') + generate = self.nodes[0].cli(rpcwallet2, '-generate').send_cli() + assert_equal(set(generate.keys()), {'address', 'blocks'}) + assert_equal(len(generate["blocks"]), 1) + assert_equal(self.nodes[0].getblockcount(), blocks + 2 + n1 + n2) + + self.log.info('Test -generate -rpcwallet=unloaded wallet raises RPC error') + assert_raises_rpc_error(-18, WALLET_NOT_LOADED, self.nodes[0].cli(rpcwallet3, '-generate').echo) + assert_raises_rpc_error(-18, WALLET_NOT_LOADED, self.nodes[0].cli(rpcwallet3, '-generate', 'foo').echo) + assert_raises_rpc_error(-18, WALLET_NOT_LOADED, self.nodes[0].cli(rpcwallet3, '-generate', 0).echo) + assert_raises_rpc_error(-18, WALLET_NOT_LOADED, self.nodes[0].cli(rpcwallet3, '-generate', 1, 2, 3).echo) + + # Test bitcoin-cli -generate with -rpcwallet in multiwallet mode. + self.nodes[0].loadwallet(wallets[2]) + n3 = 4 + n4 = 10 + blocks = self.nodes[0].getblockcount() + + self.log.info('Test -generate -rpcwallet with no args') + generate = self.nodes[0].cli(rpcwallet2, '-generate').send_cli() + assert_equal(set(generate.keys()), {'address', 'blocks'}) + assert_equal(len(generate["blocks"]), 1) + assert_equal(self.nodes[0].getblockcount(), blocks + 1) + + self.log.info('Test -generate -rpcwallet with bad args') + assert_raises_process_error(1, JSON_PARSING_ERROR, self.nodes[0].cli(rpcwallet2, '-generate', 'foo').echo) + assert_raises_process_error(1, BLOCKS_VALUE_OF_ZERO, self.nodes[0].cli(rpcwallet2, '-generate', 0).echo) + assert_raises_process_error(1, TOO_MANY_ARGS, self.nodes[0].cli(rpcwallet2, '-generate', 1, 2, 3).echo) + + self.log.info('Test -generate -rpcwallet with nblocks') + generate = self.nodes[0].cli(rpcwallet2, '-generate', n3).send_cli() + assert_equal(set(generate.keys()), {'address', 'blocks'}) + assert_equal(len(generate["blocks"]), n3) + assert_equal(self.nodes[0].getblockcount(), blocks + 1 + n3) + + self.log.info('Test -generate -rpcwallet with nblocks and maxtries') + generate = self.nodes[0].cli(rpcwallet2, '-generate', n4, 1000000).send_cli() + assert_equal(set(generate.keys()), {'address', 'blocks'}) + assert_equal(len(generate["blocks"]), n4) + assert_equal(self.nodes[0].getblockcount(), blocks + 1 + n3 + n4) + + self.log.info('Test -generate without -rpcwallet in multiwallet mode raises RPC error') + assert_raises_rpc_error(-19, WALLET_NOT_SPECIFIED, self.nodes[0].cli('-generate').echo) + assert_raises_rpc_error(-19, WALLET_NOT_SPECIFIED, self.nodes[0].cli('-generate', 'foo').echo) + assert_raises_rpc_error(-19, WALLET_NOT_SPECIFIED, self.nodes[0].cli('-generate', 0).echo) + assert_raises_rpc_error(-19, WALLET_NOT_SPECIFIED, self.nodes[0].cli('-generate', 1, 2, 3).echo) else: self.log.info("*** Wallet not compiled; cli getwalletinfo and -getinfo wallet tests skipped") - self.nodes[0].generate(1) # maintain block parity with the wallet_compiled conditional branch + self.nodes[0].generate(25) # maintain block parity with the wallet_compiled conditional branch self.log.info("Test -version with node stopped") self.stop_node(0) @@ -145,7 +235,7 @@ class TestBitcoinCli(BitcoinTestFramework): self.nodes[0].wait_for_cookie_credentials() # ensure cookie file is available to avoid race condition blocks = self.nodes[0].cli('-rpcwait').send_cli('getblockcount') self.nodes[0].wait_for_rpc_connection() - assert_equal(blocks, BLOCKS + 1) + assert_equal(blocks, BLOCKS + 25) if __name__ == '__main__': diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py new file mode 100755 index 0000000000..31fb751904 --- /dev/null +++ b/test/functional/mempool_compatibility.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test that mempool.dat is both backward and forward compatible between versions + +NOTE: The test is designed to prevent cases when compatibility is broken accidentally. +In case we need to break mempool compatibility we can continue to use the test by just bumping the version number. + +Download node binaries: +contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 + +Only v0.15.2 is required by this test. The rest is used in other backwards compatibility tests. +""" + +import os + +from test_framework.test_framework import BitcoinTestFramework + + +class MempoolCompatibilityTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + self.skip_if_no_previous_releases() + + def setup_network(self): + self.add_nodes(self.num_nodes, versions=[ + 150200, # oldest version supported by the test framework + None, + ]) + self.start_nodes() + self.import_deterministic_coinbase_privkeys() + + def run_test(self): + self.log.info("Test that mempool.dat is compatible between versions") + + old_node = self.nodes[0] + new_node = self.nodes[1] + recipient = old_node.getnewaddress() + self.stop_node(1) + + self.log.info("Add a transaction to mempool on old node and shutdown") + old_tx_hash = old_node.sendtoaddress(recipient, 0.0001) + assert old_tx_hash in old_node.getrawmempool() + self.stop_node(0) + + self.log.info("Move mempool.dat from old to new node") + old_node_mempool = os.path.join(old_node.datadir, self.chain, 'mempool.dat') + new_node_mempool = os.path.join(new_node.datadir, self.chain, 'mempool.dat') + os.rename(old_node_mempool, new_node_mempool) + + self.log.info("Start new node and verify mempool contains the tx") + self.start_node(1) + assert old_tx_hash in new_node.getrawmempool() + + self.log.info("Add unbroadcasted tx to mempool on new node and shutdown") + unbroadcasted_tx_hash = new_node.sendtoaddress(recipient, 0.0001) + assert unbroadcasted_tx_hash in new_node.getrawmempool() + mempool = new_node.getrawmempool(True) + assert mempool[unbroadcasted_tx_hash]['unbroadcast'] + self.stop_node(1) + + self.log.info("Move mempool.dat from new to old node") + os.rename(new_node_mempool, old_node_mempool) + + self.log.info("Start old node again and verify mempool contains both txs") + self.start_node(0, ['-nowallet']) + assert old_tx_hash in old_node.getrawmempool() + assert unbroadcasted_tx_hash in old_node.getrawmempool() + +if __name__ == "__main__": + MempoolCompatibilityTest().main() diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 5b7216b253..542d24f4be 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -69,14 +69,19 @@ class MempoolPackagesTest(BitcoinTestFramework): fee = Decimal("0.0001") # MAX_ANCESTORS transactions off a confirmed tx should be fine chain = [] + witness_chain = [] for i in range(MAX_ANCESTORS): (txid, sent_value) = self.chain_transaction(self.nodes[0], txid, 0, value, fee, 1) value = sent_value chain.append(txid) + # We need the wtxids to check P2P announcements + fulltx = self.nodes[0].getrawtransaction(txid) + witnesstx = self.nodes[0].decoderawtransaction(fulltx, True) + witness_chain.append(witnesstx['hash']) # Wait until mempool transactions have passed initial broadcast (sent inv and received getdata) # Otherwise, getrawmempool may be inconsistent with getmempoolentry if unbroadcast changes in between - self.nodes[0].p2p.wait_for_broadcast(chain) + self.nodes[0].p2p.wait_for_broadcast(witness_chain) # Check mempool has MAX_ANCESTORS transactions in it, and descendant and ancestor # count and fees should look correct diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 6046237101..5c7e27a3a8 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -49,9 +49,9 @@ class AddrTest(BitcoinTestFramework): addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) msg = msg_addr() - self.log.info('Send too large addr message') + self.log.info('Send too-large addr message') msg.addrs = ADDRS * 101 - with self.nodes[0].assert_debug_log(['message addr size() = 1010']): + with self.nodes[0].assert_debug_log(['addr message size = 1010']): addr_source.send_and_ping(msg) self.log.info('Check that addr message content is relayed and added to addrman') diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index c155dda664..27e6b669f6 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -52,34 +52,35 @@ class P2PBlocksOnly(BitcoinTestFramework): self.log.info('Check that txs from rpc are not rejected and relayed to other peers') assert_equal(self.nodes[0].getpeerinfo()[0]['relaytxes'], True) txid = self.nodes[0].testmempoolaccept([sigtx])[0]['txid'] - with self.nodes[0].assert_debug_log(['received getdata for: tx {} peer=1'.format(txid)]): + with self.nodes[0].assert_debug_log(['received getdata for: wtx {} peer=1'.format(txid)]): self.nodes[0].sendrawtransaction(sigtx) self.nodes[0].p2p.wait_for_tx(txid) assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) - self.log.info('Check that txs from whitelisted peers are not rejected and relayed to others') - self.log.info("Restarting node 0 with whitelist permission and blocksonly") + self.log.info('Check that txs from forcerelay peers are not rejected and relayed to others') + self.log.info("Restarting node 0 with forcerelay permission and blocksonly") self.restart_node(0, ["-persistmempool=0", "-whitelist=127.0.0.1", "-whitelistforcerelay", "-blocksonly"]) - assert_equal(self.nodes[0].getrawmempool(),[]) + assert_equal(self.nodes[0].getrawmempool(), []) first_peer = self.nodes[0].add_p2p_connection(P2PInterface()) second_peer = self.nodes[0].add_p2p_connection(P2PInterface()) peer_1_info = self.nodes[0].getpeerinfo()[0] assert_equal(peer_1_info['whitelisted'], True) - assert_equal(peer_1_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool']) + assert_equal(peer_1_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool', 'download']) peer_2_info = self.nodes[0].getpeerinfo()[1] assert_equal(peer_2_info['whitelisted'], True) - assert_equal(peer_2_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool']) + assert_equal(peer_2_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool', 'download']) assert_equal(self.nodes[0].testmempoolaccept([sigtx])[0]['allowed'], True) txid = self.nodes[0].testmempoolaccept([sigtx])[0]['txid'] - self.log.info('Check that the tx from whitelisted first_peer is relayed to others (ie.second_peer)') + self.log.info('Check that the tx from forcerelay first_peer is relayed to others (ie.second_peer)') with self.nodes[0].assert_debug_log(["received getdata"]): first_peer.send_message(msg_tx(FromHex(CTransaction(), sigtx))) - self.log.info('Check that the whitelisted peer is still connected after sending the transaction') + self.log.info('Check that the forcerelay peer is still connected after sending the transaction') assert_equal(first_peer.is_connected, True) second_peer.wait_for_tx(txid) assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) - self.log.info("Whitelisted peer's transaction is accepted and relayed") + self.log.info("Forcerelay peer's transaction is accepted and relayed") + if __name__ == '__main__': P2PBlocksOnly().main() diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index d77a744758..0b3738b572 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -305,10 +305,9 @@ class CompactBlocksTest(BitcoinTestFramework): self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) # Now fetch the compact block using a normal non-announce getdata - with mininode_lock: - test_node.clear_block_announcement() - inv = CInv(MSG_CMPCT_BLOCK, block_hash) - test_node.send_message(msg_getdata([inv])) + test_node.clear_block_announcement() + inv = CInv(MSG_CMPCT_BLOCK, block_hash) + test_node.send_message(msg_getdata([inv])) wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 9047fc6828..09b9ebeb2d 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -69,8 +69,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.nodes[1].setmocktime(old_time + 3) assert_equal(len(self.nodes[1].listbanned()), 3) - self.stop_node(1) - self.start_node(1) + self.restart_node(1) listAfterShutdown = self.nodes[1].listbanned() assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) diff --git a/test/functional/p2p_eviction.py b/test/functional/p2p_eviction.py new file mode 100755 index 0000000000..b2b3a89aab --- /dev/null +++ b/test/functional/p2p_eviction.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Test node eviction logic + +When the number of peers has reached the limit of maximum connections, +the next connecting inbound peer will trigger the eviction mechanism. +We cannot currently test the parts of the eviction logic that are based on +address/netgroup since in the current framework, all peers are connecting from +the same local address. See Issue #14210 for more info. +Therefore, this test is limited to the remaining protection criteria. +""" + +import time + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.mininode import P2PInterface, P2PDataStore +from test_framework.util import assert_equal, wait_until +from test_framework.blocktools import create_block, create_coinbase +from test_framework.messages import CTransaction, FromHex, msg_pong, msg_tx + + +class SlowP2PDataStore(P2PDataStore): + def on_ping(self, message): + time.sleep(0.1) + self.send_message(msg_pong(message.nonce)) + +class SlowP2PInterface(P2PInterface): + def on_ping(self, message): + time.sleep(0.1) + self.send_message(msg_pong(message.nonce)) + +class P2PEvict(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + # The choice of maxconnections=32 results in a maximum of 21 inbound connections + # (32 - 10 outbound - 1 feeler). 20 inbound peers are protected from eviction: + # 4 by netgroup, 4 that sent us blocks, 4 that sent us transactions and 8 via lowest ping time + self.extra_args = [['-maxconnections=32']] + + def run_test(self): + protected_peers = set() # peers that we expect to be protected from eviction + current_peer = -1 + node = self.nodes[0] + node.generatetoaddress(101, node.get_deterministic_priv_key().address) + + self.log.info("Create 4 peers and protect them from eviction by sending us a block") + for _ in range(4): + block_peer = node.add_p2p_connection(SlowP2PDataStore()) + current_peer += 1 + block_peer.sync_with_ping() + best_block = node.getbestblockhash() + tip = int(best_block, 16) + best_block_time = node.getblock(best_block)['time'] + block = create_block(tip, create_coinbase(node.getblockcount() + 1), best_block_time + 1) + block.solve() + block_peer.send_blocks_and_test([block], node, success=True) + protected_peers.add(current_peer) + + self.log.info("Create 5 slow-pinging peers, making them eviction candidates") + for _ in range(5): + node.add_p2p_connection(SlowP2PInterface()) + current_peer += 1 + + self.log.info("Create 4 peers and protect them from eviction by sending us a tx") + for i in range(4): + txpeer = node.add_p2p_connection(SlowP2PInterface()) + current_peer += 1 + txpeer.sync_with_ping() + + prevtx = node.getblock(node.getblockhash(i + 1), 2)['tx'][0] + rawtx = node.createrawtransaction( + inputs=[{'txid': prevtx['txid'], 'vout': 0}], + outputs=[{node.get_deterministic_priv_key().address: 50 - 0.00125}], + ) + sigtx = node.signrawtransactionwithkey( + hexstring=rawtx, + privkeys=[node.get_deterministic_priv_key().key], + prevtxs=[{ + 'txid': prevtx['txid'], + 'vout': 0, + 'scriptPubKey': prevtx['vout'][0]['scriptPubKey']['hex'], + }], + )['hex'] + txpeer.send_message(msg_tx(FromHex(CTransaction(), sigtx))) + protected_peers.add(current_peer) + + self.log.info("Create 8 peers and protect them from eviction by having faster pings") + for _ in range(8): + fastpeer = node.add_p2p_connection(P2PInterface()) + current_peer += 1 + wait_until(lambda: "ping" in fastpeer.last_message, timeout=10) + + # Make sure by asking the node what the actual min pings are + peerinfo = node.getpeerinfo() + pings = {} + for i in range(len(peerinfo)): + pings[i] = peerinfo[i]['minping'] if 'minping' in peerinfo[i] else 1000000 + sorted_pings = sorted(pings.items(), key=lambda x: x[1]) + + # Usually the 8 fast peers are protected. In rare case of unreliable pings, + # one of the slower peers might have a faster min ping though. + for i in range(8): + protected_peers.add(sorted_pings[i][0]) + + self.log.info("Create peer that triggers the eviction mechanism") + node.add_p2p_connection(SlowP2PInterface()) + + # One of the non-protected peers must be evicted. We can't be sure which one because + # 4 peers are protected via netgroup, which is identical for all peers, + # and the eviction mechanism doesn't preserve the order of identical elements. + evicted_peers = [] + for i in range(len(node.p2ps)): + if not node.p2ps[i].is_connected: + evicted_peers.append(i) + + self.log.info("Test that one peer was evicted") + self.log.debug("{} evicted peer: {}".format(len(evicted_peers), set(evicted_peers))) + assert_equal(len(evicted_peers), 1) + + self.log.info("Test that no peer expected to be protected was evicted") + self.log.debug("{} protected peers: {}".format(len(protected_peers), protected_peers)) + assert evicted_peers[0] not in protected_peers + +if __name__ == '__main__': + P2PEvict().main() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index 805cb1e84f..73afe9adc4 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -7,14 +7,16 @@ from decimal import Decimal import time -from test_framework.messages import MSG_TX, msg_feefilter +from test_framework.messages import MSG_TX, MSG_WTX, msg_feefilter from test_framework.mininode import mininode_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal def hashToHex(hash): return format(hash, '064x') + # Wait up to 60 secs to see if the testnode has received all the expected invs def allInvsMatch(invsExpected, testnode): for x in range(60): @@ -24,6 +26,18 @@ def allInvsMatch(invsExpected, testnode): time.sleep(1) return False + +class FeefilterConn(P2PInterface): + feefilter_received = False + + def on_feefilter(self, message): + self.feefilter_received = True + + def assert_feefilter_received(self, recv: bool): + with mininode_lock: + assert_equal(self.feefilter_received, recv) + + class TestP2PConn(P2PInterface): def __init__(self): super().__init__() @@ -31,13 +45,14 @@ class TestP2PConn(P2PInterface): def on_inv(self, message): for i in message.inv: - if (i.type == MSG_TX): + if (i.type == MSG_TX) or (i.type == MSG_WTX): self.txinvs.append(hashToHex(i.hash)) def clear_invs(self): with mininode_lock: self.txinvs = [] + class FeeFilterTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 @@ -46,41 +61,54 @@ class FeeFilterTest(BitcoinTestFramework): # mempool and wallet feerate calculation based on GetFee # rounding down 3 places, leading to stranded transactions. # See issue #16499 - self.extra_args = [["-minrelaytxfee=0.00000100", "-mintxfee=0.00000100"]]*self.num_nodes + self.extra_args = [["-minrelaytxfee=0.00000100", "-mintxfee=0.00000100"]] * self.num_nodes def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): + self.test_feefilter_forcerelay() + self.test_feefilter() + + def test_feefilter_forcerelay(self): + self.log.info('Check that peers without forcerelay permission (default) get a feefilter message') + self.nodes[0].add_p2p_connection(FeefilterConn()).assert_feefilter_received(True) + + self.log.info('Check that peers with forcerelay permission do not get a feefilter message') + self.restart_node(0, extra_args=['-whitelist=forcerelay@127.0.0.1']) + self.nodes[0].add_p2p_connection(FeefilterConn()).assert_feefilter_received(False) + + # Restart to disconnect peers and load default extra_args + self.restart_node(0) + self.connect_nodes(1, 0) + + def test_feefilter(self): node1 = self.nodes[1] node0 = self.nodes[0] - # Get out of IBD - node1.generate(1) - self.sync_blocks() - self.nodes[0].add_p2p_connection(TestP2PConn()) + conn = self.nodes[0].add_p2p_connection(TestP2PConn()) # Test that invs are received by test connection for all txs at # feerate of .2 sat/byte node1.settxfee(Decimal("0.00000200")) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert allInvsMatch(txids, self.nodes[0].p2p) - self.nodes[0].p2p.clear_invs() + assert allInvsMatch(txids, conn) + conn.clear_invs() # Set a filter of .15 sat/byte on test connection - self.nodes[0].p2p.send_and_ping(msg_feefilter(150)) + conn.send_and_ping(msg_feefilter(150)) # Test that txs are still being received by test connection (paying .15 sat/byte) node1.settxfee(Decimal("0.00000150")) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert allInvsMatch(txids, self.nodes[0].p2p) - self.nodes[0].p2p.clear_invs() + assert allInvsMatch(txids, conn) + conn.clear_invs() # Change tx fee rate to .1 sat/byte and test they are no longer received # by the test connection node1.settxfee(Decimal("0.00000100")) [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - self.sync_mempools() # must be sure node 0 has received all txs + self.sync_mempools() # must be sure node 0 has received all txs # Send one transaction from node0 that should be received, so that we # we can sync the test on receipt (if node1's txs were relayed, they'd @@ -91,14 +119,15 @@ class FeeFilterTest(BitcoinTestFramework): # as well. node0.settxfee(Decimal("0.00020000")) txids = [node0.sendtoaddress(node0.getnewaddress(), 1)] - assert allInvsMatch(txids, self.nodes[0].p2p) - self.nodes[0].p2p.clear_invs() + assert allInvsMatch(txids, conn) + conn.clear_invs() # Remove fee filter and check that txs are received again - self.nodes[0].p2p.send_and_ping(msg_feefilter(0)) + conn.send_and_ping(msg_feefilter(0)) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert allInvsMatch(txids, self.nodes[0].p2p) - self.nodes[0].p2p.clear_invs() + assert allInvsMatch(txids, conn) + conn.clear_invs() + if __name__ == '__main__': FeeFilterTest().main() diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 5726a73e40..741da3be31 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -124,11 +124,11 @@ class FilterTest(BitcoinTestFramework): self.log.info("Check that a node with bloom filters enabled services p2p mempool messages") filter_peer = P2PBloomFilter() - self.log.info("Create a tx relevant to the peer before connecting") + self.log.debug("Create a tx relevant to the peer before connecting") filter_address = self.nodes[0].decodescript(filter_peer.watch_script_pubkey)['addresses'][0] txid = self.nodes[0].sendtoaddress(filter_address, 90) - self.log.info("Send a mempool msg after connecting and check that the tx is received") + self.log.debug("Send a mempool msg after connecting and check that the tx is received") self.nodes[0].add_p2p_connection(filter_peer) filter_peer.send_and_ping(filter_peer.watch_filter_init) self.nodes[0].p2p.send_message(msg_mempool()) @@ -227,8 +227,8 @@ class FilterTest(BitcoinTestFramework): self.test_frelay_false(filter_peer_without_nrelay) self.test_filter(filter_peer_without_nrelay) - self.log.info('Test msg_mempool') self.test_msg_mempool() + if __name__ == '__main__': FilterTest().main() diff --git a/test/functional/p2p_ibd_txrelay.py b/test/functional/p2p_ibd_txrelay.py new file mode 100755 index 0000000000..c3e758b021 --- /dev/null +++ b/test/functional/p2p_ibd_txrelay.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test fee filters during and after IBD.""" + +from decimal import Decimal + +from test_framework.messages import COIN +from test_framework.test_framework import BitcoinTestFramework + +MAX_FEE_FILTER = Decimal(9170997) / COIN +NORMAL_FEE_FILTER = Decimal(100) / COIN + + +class P2PIBDTxRelayTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + self.extra_args = [ + ["-minrelaytxfee={}".format(NORMAL_FEE_FILTER)], + ["-minrelaytxfee={}".format(NORMAL_FEE_FILTER)], + ] + + def run_test(self): + self.log.info("Check that nodes set minfilter to MAX_MONEY while still in IBD") + for node in self.nodes: + assert node.getblockchaininfo()['initialblockdownload'] + self.wait_until(lambda: all(peer['minfeefilter'] == MAX_FEE_FILTER for peer in node.getpeerinfo())) + + # Come out of IBD by generating a block + self.nodes[0].generate(1) + self.sync_all() + + self.log.info("Check that nodes reset minfilter after coming out of IBD") + for node in self.nodes: + assert not node.getblockchaininfo()['initialblockdownload'] + self.wait_until(lambda: all(peer['minfeefilter'] == NORMAL_FEE_FILTER for peer in node.getpeerinfo())) + + +if __name__ == '__main__': + P2PIBDTxRelayTest().main() diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 81302374c9..d9a9ae5188 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -3,13 +3,13 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid network messages.""" -import asyncio -import struct -import sys from test_framework.messages import ( CBlockHeader, CInv, + MAX_HEADERS_RESULTS, + MAX_INV_SIZE, + MAX_PROTOCOL_MESSAGE_LENGTH, msg_getdata, msg_headers, msg_inv, @@ -18,12 +18,16 @@ from test_framework.messages import ( ser_string, ) from test_framework.mininode import ( - NetworkThread, P2PDataStore, P2PInterface, ) from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + wait_until, +) +VALID_DATA_LIMIT = MAX_PROTOCOL_MESSAGE_LENGTH - 5 # Account for the 5-byte length prefix class msg_unrecognized: """Nonsensical message. Modeled after similar types in test_framework.messages.""" @@ -46,161 +50,71 @@ class InvalidMessagesTest(BitcoinTestFramework): self.setup_clean_chain = True def run_test(self): - """ - . Test msg header - 0. Send a bunch of large (4MB) messages of an unrecognized type. Check to see - that it isn't an effective DoS against the node. - - 1. Send an oversized (4MB+) message and check that we're disconnected. - - 2. Send a few messages with an incorrect data size in the header, ensure the - messages are ignored. - """ + self.test_buffer() self.test_magic_bytes() self.test_checksum() self.test_size() self.test_msgtype() - self.test_large_inv() - - node = self.nodes[0] - self.node = node - node.add_p2p_connection(P2PDataStore()) - conn2 = node.add_p2p_connection(P2PDataStore()) - - msg_limit = 4 * 1000 * 1000 # 4MB, per MAX_PROTOCOL_MESSAGE_LENGTH - valid_data_limit = msg_limit - 5 # Account for the 4-byte length prefix - - # - # 0. - # - # Send as large a message as is valid, ensure we aren't disconnected but - # also can't exhaust resources. - # - msg_at_size = msg_unrecognized(str_data="b" * valid_data_limit) - assert len(msg_at_size.serialize()) == msg_limit + self.test_oversized_inv_msg() + self.test_oversized_getdata_msg() + self.test_oversized_headers_msg() + self.test_resource_exhaustion() - self.log.info("Sending a bunch of large, junk messages to test memory exhaustion. May take a bit...") - - # Run a bunch of times to test for memory exhaustion. - for _ in range(80): - node.p2p.send_message(msg_at_size) - - # Check that, even though the node is being hammered by nonsense from one - # connection, it can still service other peers in a timely way. - for _ in range(20): - conn2.sync_with_ping(timeout=2) - - # Peer 1, despite serving up a bunch of nonsense, should still be connected. - self.log.info("Waiting for node to drop junk messages.") - node.p2p.sync_with_ping(timeout=400) - assert node.p2p.is_connected - - # - # 1. - # - # Send an oversized message, ensure we're disconnected. - # - # Under macOS this test is skipped due to an unexpected error code - # returned from the closing socket which python/asyncio does not - # yet know how to handle. - # - if sys.platform != 'darwin': - msg_over_size = msg_unrecognized(str_data="b" * (valid_data_limit + 1)) - assert len(msg_over_size.serialize()) == (msg_limit + 1) - - # An unknown message type (or *any* message type) over - # MAX_PROTOCOL_MESSAGE_LENGTH should result in a disconnect. - node.p2p.send_message(msg_over_size) - node.p2p.wait_for_disconnect(timeout=4) - - node.disconnect_p2ps() - conn = node.add_p2p_connection(P2PDataStore()) - conn.wait_for_verack() - else: - self.log.info("Skipping test p2p_invalid_messages/1 (oversized message) under macOS") - - # - # 2. - # - # Send messages with an incorrect data size in the header. - # - actual_size = 100 - msg = msg_unrecognized(str_data="b" * actual_size) - - # TODO: handle larger-than cases. I haven't been able to pin down what behavior to expect. - for wrong_size in (2, 77, 78, 79): - self.log.info("Sending a message with incorrect size of {}".format(wrong_size)) - - # Unmodified message should submit okay. - node.p2p.send_and_ping(msg) - - # A message lying about its data size results in a disconnect when the incorrect - # data size is less than the actual size. - # - # TODO: why does behavior change at 78 bytes? - # - node.p2p.send_raw_message(self._tweak_msg_data_size(msg, wrong_size)) - - # For some reason unknown to me, we sometimes have to push additional data to the - # peer in order for it to realize a disconnect. - try: - node.p2p.send_message(msg_ping(nonce=123123)) - except IOError: - pass - - node.p2p.wait_for_disconnect(timeout=10) - node.disconnect_p2ps() - node.add_p2p_connection(P2PDataStore()) - - # Node is still up. - conn = node.add_p2p_connection(P2PDataStore()) + def test_buffer(self): + self.log.info("Test message with header split across two buffers is received") + conn = self.nodes[0].add_p2p_connection(P2PDataStore()) + # Create valid message + msg = conn.build_message(msg_ping(nonce=12345)) + cut_pos = 12 # Chosen at an arbitrary position within the header + # Send message in two pieces + before = int(self.nodes[0].getnettotals()['totalbytesrecv']) + conn.send_raw_message(msg[:cut_pos]) + # Wait until node has processed the first half of the message + wait_until(lambda: int(self.nodes[0].getnettotals()['totalbytesrecv']) != before) + middle = int(self.nodes[0].getnettotals()['totalbytesrecv']) + # If this assert fails, we've hit an unlikely race + # where the test framework sent a message in between the two halves + assert_equal(middle, before + cut_pos) + conn.send_raw_message(msg[cut_pos:]) + conn.sync_with_ping(timeout=1) + self.nodes[0].disconnect_p2ps() def test_magic_bytes(self): + self.log.info("Test message with invalid magic bytes disconnects peer") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) - - async def swap_magic_bytes(): - conn._on_data = lambda: None # Need to ignore all incoming messages from now, since they come with "invalid" magic bytes - conn.magic_bytes = b'\x00\x11\x22\x32' - - # Call .result() to block until the atomic swap is complete, otherwise - # we might run into races later on - asyncio.run_coroutine_threadsafe(swap_magic_bytes(), NetworkThread.network_event_loop).result() - - with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: INVALID MESSAGESTART ping']): - conn.send_message(msg_ping(nonce=0xff)) + with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: INVALID MESSAGESTART badmsg']): + msg = conn.build_message(msg_unrecognized(str_data="d")) + # modify magic bytes + msg = b'\xff' * 4 + msg[4:] + conn.send_raw_message(msg) conn.wait_for_disconnect(timeout=1) - self.nodes[0].disconnect_p2ps() + self.nodes[0].disconnect_p2ps() def test_checksum(self): + self.log.info("Test message with invalid checksum logs an error") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) with self.nodes[0].assert_debug_log(['CHECKSUM ERROR (badmsg, 2 bytes), expected 78df0a04 was ffffffff']): msg = conn.build_message(msg_unrecognized(str_data="d")) - cut_len = ( - 4 + # magic - 12 + # msgtype - 4 #len - ) + # Checksum is after start bytes (4B), message type (12B), len (4B) + cut_len = 4 + 12 + 4 # modify checksum msg = msg[:cut_len] + b'\xff' * 4 + msg[cut_len + 4:] - self.nodes[0].p2p.send_raw_message(msg) + conn.send_raw_message(msg) conn.sync_with_ping(timeout=1) - self.nodes[0].disconnect_p2ps() + self.nodes[0].disconnect_p2ps() def test_size(self): + self.log.info("Test message with oversized payload disconnects peer") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) with self.nodes[0].assert_debug_log(['']): - msg = conn.build_message(msg_unrecognized(str_data="d")) - cut_len = ( - 4 + # magic - 12 # msgtype - ) - # modify len to MAX_SIZE + 1 - msg = msg[:cut_len] + struct.pack("<I", 0x02000000 + 1) + msg[cut_len + 4:] - self.nodes[0].p2p.send_raw_message(msg) + msg = msg_unrecognized(str_data="d" * (VALID_DATA_LIMIT + 1)) + msg = conn.build_message(msg) + conn.send_raw_message(msg) conn.wait_for_disconnect(timeout=1) - self.nodes[0].disconnect_p2ps() + self.nodes[0].disconnect_p2ps() def test_msgtype(self): + self.log.info("Test message with invalid message type logs an error") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: ERRORS IN HEADER']): msg = msg_unrecognized(str_data="d") @@ -208,42 +122,53 @@ class InvalidMessagesTest(BitcoinTestFramework): msg = conn.build_message(msg) # Modify msgtype msg = msg[:7] + b'\x00' + msg[7 + 1:] - self.nodes[0].p2p.send_raw_message(msg) + conn.send_raw_message(msg) conn.sync_with_ping(timeout=1) - self.nodes[0].disconnect_p2ps() - - def test_large_inv(self): - conn = self.nodes[0].add_p2p_connection(P2PInterface()) - with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (0 -> 20): message inv size() = 50001']): - msg = msg_inv([CInv(MSG_TX, 1)] * 50001) - conn.send_and_ping(msg) - with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (20 -> 40): message getdata size() = 50001']): - msg = msg_getdata([CInv(MSG_TX, 1)] * 50001) - conn.send_and_ping(msg) - with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (40 -> 60): headers message size = 2001']): - msg = msg_headers([CBlockHeader()] * 2001) - conn.send_and_ping(msg) self.nodes[0].disconnect_p2ps() - def _tweak_msg_data_size(self, message, wrong_size): - """ - Return a raw message based on another message but with an incorrect data size in - the message header. - """ - raw_msg = self.node.p2p.build_message(message) - - bad_size_bytes = struct.pack("<I", wrong_size) - num_header_bytes_before_size = 4 + 12 - - # Replace the correct data size in the message with an incorrect one. - raw_msg_with_wrong_size = ( - raw_msg[:num_header_bytes_before_size] + - bad_size_bytes + - raw_msg[(num_header_bytes_before_size + len(bad_size_bytes)):] - ) - assert len(raw_msg) == len(raw_msg_with_wrong_size) - - return raw_msg_with_wrong_size + def test_oversized_msg(self, msg, size): + msg_type = msg.msgtype.decode('ascii') + self.log.info("Test {} message of size {} is logged as misbehaving".format(msg_type, size)) + with self.nodes[0].assert_debug_log(['Misbehaving', '{} message size = {}'.format(msg_type, size)]): + self.nodes[0].add_p2p_connection(P2PInterface()).send_and_ping(msg) + self.nodes[0].disconnect_p2ps() + + def test_oversized_inv_msg(self): + size = MAX_INV_SIZE + 1 + self.test_oversized_msg(msg_inv([CInv(MSG_TX, 1)] * size), size) + + def test_oversized_getdata_msg(self): + size = MAX_INV_SIZE + 1 + self.test_oversized_msg(msg_getdata([CInv(MSG_TX, 1)] * size), size) + + def test_oversized_headers_msg(self): + size = MAX_HEADERS_RESULTS + 1 + self.test_oversized_msg(msg_headers([CBlockHeader()] * size), size) + + def test_resource_exhaustion(self): + self.log.info("Test node stays up despite many large junk messages") + conn = self.nodes[0].add_p2p_connection(P2PDataStore()) + conn2 = self.nodes[0].add_p2p_connection(P2PDataStore()) + msg_at_size = msg_unrecognized(str_data="b" * VALID_DATA_LIMIT) + assert len(msg_at_size.serialize()) == MAX_PROTOCOL_MESSAGE_LENGTH + + self.log.info("(a) Send 80 messages, each of maximum valid data size (4MB)") + for _ in range(80): + conn.send_message(msg_at_size) + + # Check that, even though the node is being hammered by nonsense from one + # connection, it can still service other peers in a timely way. + self.log.info("(b) Check node still services peers in a timely way") + for _ in range(20): + conn2.sync_with_ping(timeout=2) + + self.log.info("(c) Wait for node to drop junk messages, while remaining connected") + conn.sync_with_ping(timeout=400) + + # Despite being served up a bunch of nonsense, the peers should still be connected. + assert conn.is_connected + assert conn2.is_connected + self.nodes[0].disconnect_p2ps() if __name__ == '__main__': diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 157af68203..fe6e236fc4 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -26,7 +26,7 @@ from test_framework.util import ( wait_until, ) -banscore = 10 +DISCOURAGEMENT_THRESHOLD = 100 class CLazyNode(P2PInterface): @@ -65,12 +65,13 @@ class CLazyNode(P2PInterface): # Node that never sends a version. We'll use this to send a bunch of messages # anyway, and eventually get disconnected. -class CNodeNoVersionBan(CLazyNode): - # send a bunch of veracks without sending a message. This should get us disconnected. - # NOTE: implementation-specific check here. Remove if bitcoind ban behavior changes +class CNodeNoVersionMisbehavior(CLazyNode): + # Send enough veracks without a message to reach the peer discouragement + # threshold. This should get us disconnected. NOTE: implementation-specific + # test; update if our discouragement policy for peer misbehavior changes. def on_open(self): super().on_open() - for i in range(banscore): + for _ in range(DISCOURAGEMENT_THRESHOLD): self.send_message(msg_verack()) # Node that never sends a version. This one just sits idle and hopes to receive @@ -106,10 +107,10 @@ class P2PVersionStore(P2PInterface): class P2PLeakTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [['-banscore=' + str(banscore)]] def run_test(self): - no_version_bannode = self.nodes[0].add_p2p_connection(CNodeNoVersionBan(), send_version=False, wait_for_verack=False) + no_version_disconnect_node = self.nodes[0].add_p2p_connection( + CNodeNoVersionMisbehavior(), send_version=False, wait_for_verack=False) no_version_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVersionIdle(), send_version=False, wait_for_verack=False) no_verack_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVerackIdle(), wait_for_verack=False) @@ -117,7 +118,7 @@ class P2PLeakTest(BitcoinTestFramework): # verack, since we never sent one no_verack_idlenode.wait_for_verack() - wait_until(lambda: no_version_bannode.ever_connected, timeout=10, lock=mininode_lock) + wait_until(lambda: no_version_disconnect_node.ever_connected, timeout=10, lock=mininode_lock) wait_until(lambda: no_version_idlenode.ever_connected, timeout=10, lock=mininode_lock) wait_until(lambda: no_verack_idlenode.version_received, timeout=10, lock=mininode_lock) @@ -127,16 +128,13 @@ class P2PLeakTest(BitcoinTestFramework): #Give the node enough time to possibly leak out a message time.sleep(5) - #This node should have been banned - assert not no_version_bannode.is_connected + # Expect this node to be disconnected for misbehavior + assert not no_version_disconnect_node.is_connected self.nodes[0].disconnect_p2ps() - # Wait until all connections are closed - wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 0) - # Make sure no unexpected messages came in - assert no_version_bannode.unexpected_msg == False + assert no_version_disconnect_node.unexpected_msg == False assert no_version_idlenode.unexpected_msg == False assert no_verack_idlenode.unexpected_msg == False diff --git a/test/functional/p2p_nobloomfilter_messages.py b/test/functional/p2p_nobloomfilter_messages.py index 41af74ebb8..accc5dc23c 100755 --- a/test/functional/p2p_nobloomfilter_messages.py +++ b/test/functional/p2p_nobloomfilter_messages.py @@ -4,40 +4,45 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test invalid p2p messages for nodes with bloom filters disabled. -Test that, when bloom filters are not enabled, nodes are disconnected if: +Test that, when bloom filters are not enabled, peers are disconnected if: 1. They send a p2p mempool message 2. They send a p2p filterload message 3. They send a p2p filteradd message +4. They send a p2p filterclear message """ -from test_framework.messages import msg_mempool, msg_filteradd, msg_filterload +from test_framework.messages import msg_mempool, msg_filteradd, msg_filterload, msg_filterclear from test_framework.mininode import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal -class P2PNobloomfilterMessages(BitcoinTestFramework): +class P2PNoBloomFilterMessages(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 self.extra_args = [["-peerbloomfilters=0"]] def test_message_causes_disconnect(self, message): - # Add a p2p connection that sends a message and check that it disconnects + """Add a p2p connection that sends a message and check that it disconnects.""" peer = self.nodes[0].add_p2p_connection(P2PInterface()) peer.send_message(message) peer.wait_for_disconnect() - assert_equal(len(self.nodes[0].getpeerinfo()), 0) + assert_equal(self.nodes[0].getconnectioncount(), 0) def run_test(self): - self.log.info("Test that node is disconnected if it sends mempool message") + self.log.info("Test that peer is disconnected if it sends mempool message") self.test_message_causes_disconnect(msg_mempool()) - self.log.info("Test that node is disconnected if it sends filterload message") + self.log.info("Test that peer is disconnected if it sends filterload message") self.test_message_causes_disconnect(msg_filterload()) - self.log.info("Test that node is disconnected if it sends filteradd message") + self.log.info("Test that peer is disconnected if it sends filteradd message") self.test_message_causes_disconnect(msg_filteradd(data=b'\xcc')) + self.log.info("Test that peer is disconnected if it sends a filterclear message") + self.test_message_causes_disconnect(msg_filterclear()) + + if __name__ == '__main__': - P2PNobloomfilterMessages().main() + P2PNoBloomFilterMessages().main() diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index ed3429a037..a2f6ea538c 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -83,7 +83,6 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): assert_equal(node1.firstAddrnServices, expected_services) self.nodes[0].disconnect_p2ps() - node1.wait_for_disconnect() # connect unsynced node 2 with pruned NODE_NETWORK_LIMITED peer # because node 2 is in IBD and node 0 is a NODE_NETWORK_LIMITED peer, sync must not be possible diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index 2c200fccad..32a795e345 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -39,13 +39,20 @@ class P2PPermissionsTests(BitcoinTestFramework): self.checkpermission( # default permissions (no specific permissions) ["-whitelist=127.0.0.1"], - ["relay", "noban", "mempool"], + # Make sure the default values in the command line documentation match the ones here + ["relay", "noban", "mempool", "download"], True) self.checkpermission( + # no permission (even with forcerelay) + ["-whitelist=@127.0.0.1", "-whitelistforcerelay=1"], + [], + False) + + self.checkpermission( # relay permission removed (no specific permissions) ["-whitelist=127.0.0.1", "-whitelistrelay=0"], - ["noban", "mempool"], + ["noban", "mempool", "download"], True) self.checkpermission( @@ -53,7 +60,7 @@ class P2PPermissionsTests(BitcoinTestFramework): # Legacy parameter interaction which set whitelistrelay to true # if whitelistforcerelay is true ["-whitelist=127.0.0.1", "-whitelistforcerelay"], - ["forcerelay", "relay", "noban", "mempool"], + ["forcerelay", "relay", "noban", "mempool", "download"], True) # Let's make sure permissions are merged correctly @@ -64,32 +71,32 @@ class P2PPermissionsTests(BitcoinTestFramework): self.checkpermission( ["-whitelist=noban@127.0.0.1"], # Check parameter interaction forcerelay should activate relay - ["noban", "bloomfilter", "forcerelay", "relay"], + ["noban", "bloomfilter", "forcerelay", "relay", "download"], False) self.replaceinconfig(1, "whitebind=bloomfilter,forcerelay@" + ip_port, "bind=127.0.0.1") self.checkpermission( # legacy whitelistrelay should be ignored ["-whitelist=noban,mempool@127.0.0.1", "-whitelistrelay"], - ["noban", "mempool"], + ["noban", "mempool", "download"], False) self.checkpermission( # legacy whitelistforcerelay should be ignored ["-whitelist=noban,mempool@127.0.0.1", "-whitelistforcerelay"], - ["noban", "mempool"], + ["noban", "mempool", "download"], False) self.checkpermission( # missing mempool permission to be considered legacy whitelisted ["-whitelist=noban@127.0.0.1"], - ["noban"], + ["noban", "download"], False) self.checkpermission( # all permission added ["-whitelist=all@127.0.0.1"], - ["forcerelay", "noban", "mempool", "bloomfilter", "relay"], + ["forcerelay", "noban", "mempool", "bloomfilter", "relay", "download"], False) self.stop_node(1) @@ -101,9 +108,9 @@ class P2PPermissionsTests(BitcoinTestFramework): block_op_true = self.nodes[0].getblock(self.nodes[0].generatetoaddress(100, ADDRESS_BCRT1_P2WSH_OP_TRUE)[0]) self.sync_all() - self.log.debug("Create a connection from a whitelisted wallet that rebroadcasts raw txs") + self.log.debug("Create a connection from a forcerelay peer that rebroadcasts raw txs") # A python mininode is needed to send the raw transaction directly. If a full node was used, it could only - # rebroadcast via the inv-getdata mechanism. However, even for whitelisted connections, a full node would + # rebroadcast via the inv-getdata mechanism. However, even for forcerelay connections, a full node would # currently not request a txid that is already in the mempool. self.restart_node(1, extra_args=["-whitelist=forcerelay@127.0.0.1"]) p2p_rebroadcast_wallet = self.nodes[1].add_p2p_connection(P2PDataStore()) @@ -128,7 +135,7 @@ class P2PPermissionsTests(BitcoinTestFramework): self.log.debug("Check that node[1] will send the tx to node[0] even though it is already in the mempool") connect_nodes(self.nodes[1], 0) - with self.nodes[1].assert_debug_log(["Force relaying tx {} from whitelisted peer=0".format(txid)]): + with self.nodes[1].assert_debug_log(["Force relaying tx {} from peer=0".format(txid)]): p2p_rebroadcast_wallet.send_txs_and_test([tx], self.nodes[1]) wait_until(lambda: txid in self.nodes[0].getrawmempool()) @@ -139,7 +146,7 @@ class P2PPermissionsTests(BitcoinTestFramework): [tx], self.nodes[1], success=False, - reject_reason='Not relaying non-mempool transaction {} from whitelisted peer=0'.format(txid), + reject_reason='Not relaying non-mempool transaction {} from forcerelay peer=0'.format(txid), ) def checkpermission(self, args, expectedPermissions, whitelisted): diff --git a/test/functional/p2p_ping.py b/test/functional/p2p_ping.py new file mode 100755 index 0000000000..e00af88cc4 --- /dev/null +++ b/test/functional/p2p_ping.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test ping message +""" + +import time + +from test_framework.messages import ( + msg_pong, +) +from test_framework.mininode import ( + P2PInterface, + wait_until, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +PING_INTERVAL = 2 * 60 + + +class msg_pong_corrupt(msg_pong): + def serialize(self): + return b"" + + +class NodePongAdd1(P2PInterface): + def on_ping(self, message): + self.send_message(msg_pong(message.nonce + 1)) + + +class NodeNoPong(P2PInterface): + def on_ping(self, message): + pass + + +class PingPongTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [['-peertimeout=3']] + + def check_peer_info(self, *, pingtime, minping, pingwait): + stats = self.nodes[0].getpeerinfo()[0] + assert_equal(stats.pop('pingtime', None), pingtime) + assert_equal(stats.pop('minping', None), minping) + assert_equal(stats.pop('pingwait', None), pingwait) + + def mock_forward(self, delta): + self.mock_time += delta + self.nodes[0].setmocktime(self.mock_time) + + def run_test(self): + self.mock_time = int(time.time()) + self.mock_forward(0) + + self.log.info('Check that ping is sent after connection is established') + no_pong_node = self.nodes[0].add_p2p_connection(NodeNoPong()) + self.mock_forward(3) + assert no_pong_node.last_message.pop('ping').nonce != 0 + self.check_peer_info(pingtime=None, minping=None, pingwait=3) + + self.log.info('Reply without nonce cancels ping') + with self.nodes[0].assert_debug_log(['pong peer=0: Short payload']): + no_pong_node.send_and_ping(msg_pong_corrupt()) + self.check_peer_info(pingtime=None, minping=None, pingwait=None) + + self.log.info('Reply without ping') + with self.nodes[0].assert_debug_log([ + 'pong peer=0: Unsolicited pong without ping, 0 expected, 0 received, 8 bytes', + ]): + no_pong_node.send_and_ping(msg_pong()) + self.check_peer_info(pingtime=None, minping=None, pingwait=None) + + self.log.info('Reply with wrong nonce does not cancel ping') + assert 'ping' not in no_pong_node.last_message + with self.nodes[0].assert_debug_log(['pong peer=0: Nonce mismatch']): + # mock time PING_INTERVAL ahead to trigger node into sending a ping + self.mock_forward(PING_INTERVAL + 1) + wait_until(lambda: 'ping' in no_pong_node.last_message) + self.mock_forward(9) + # Send the wrong pong + no_pong_node.send_and_ping(msg_pong(no_pong_node.last_message.pop('ping').nonce - 1)) + self.check_peer_info(pingtime=None, minping=None, pingwait=9) + + self.log.info('Reply with zero nonce does cancel ping') + with self.nodes[0].assert_debug_log(['pong peer=0: Nonce zero']): + no_pong_node.send_and_ping(msg_pong(0)) + self.check_peer_info(pingtime=None, minping=None, pingwait=None) + + self.log.info('Check that ping is properly reported on RPC') + assert 'ping' not in no_pong_node.last_message + # mock time PING_INTERVAL ahead to trigger node into sending a ping + self.mock_forward(PING_INTERVAL + 1) + wait_until(lambda: 'ping' in no_pong_node.last_message) + ping_delay = 29 + self.mock_forward(ping_delay) + wait_until(lambda: 'ping' in no_pong_node.last_message) + no_pong_node.send_and_ping(msg_pong(no_pong_node.last_message.pop('ping').nonce)) + self.check_peer_info(pingtime=ping_delay, minping=ping_delay, pingwait=None) + + self.log.info('Check that minping is decreased after a fast roundtrip') + # mock time PING_INTERVAL ahead to trigger node into sending a ping + self.mock_forward(PING_INTERVAL + 1) + wait_until(lambda: 'ping' in no_pong_node.last_message) + ping_delay = 9 + self.mock_forward(ping_delay) + wait_until(lambda: 'ping' in no_pong_node.last_message) + no_pong_node.send_and_ping(msg_pong(no_pong_node.last_message.pop('ping').nonce)) + self.check_peer_info(pingtime=ping_delay, minping=ping_delay, pingwait=None) + + self.log.info('Check that peer is disconnected after ping timeout') + assert 'ping' not in no_pong_node.last_message + self.nodes[0].ping() + wait_until(lambda: 'ping' in no_pong_node.last_message) + with self.nodes[0].assert_debug_log(['ping timeout: 1201.000000s']): + self.mock_forward(20 * 60 + 1) + time.sleep(4) # peertimeout + 1 + + +if __name__ == '__main__': + PingPongTest().main() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 8803086213..9915b844d1 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -25,6 +25,7 @@ from test_framework.messages import ( MSG_BLOCK, MSG_TX, MSG_WITNESS_FLAG, + MSG_WTX, NODE_NETWORK, NODE_WITNESS, msg_no_witness_block, @@ -34,6 +35,7 @@ from test_framework.messages import ( msg_tx, msg_block, msg_no_witness_tx, + msg_verack, ser_uint256, ser_vector, sha256, @@ -81,6 +83,7 @@ from test_framework.util import ( softfork_active, hex_str_to_bytes, assert_raises_rpc_error, + wait_until, ) # The versionbit bit used to signal activation of SegWit @@ -143,25 +146,47 @@ def test_witness_block(node, p2p, block, accepted, with_witness=True, reason=Non class TestP2PConn(P2PInterface): - def __init__(self): + def __init__(self, wtxidrelay=False): super().__init__() self.getdataset = set() + self.last_wtxidrelay = [] + self.lastgetdata = [] + self.wtxidrelay = wtxidrelay # Avoid sending out msg_getdata in the mininode thread as a reply to invs. # They are not needed and would only lead to races because we send msg_getdata out in the test thread def on_inv(self, message): pass + def on_version(self, message): + if self.wtxidrelay: + super().on_version(message) + else: + self.send_message(msg_verack()) + self.nServices = message.nServices + def on_getdata(self, message): + self.lastgetdata = message.inv for inv in message.inv: self.getdataset.add(inv.hash) - def announce_tx_and_wait_for_getdata(self, tx, timeout=60, success=True): + def on_wtxidrelay(self, message): + self.last_wtxidrelay.append(message) + + def announce_tx_and_wait_for_getdata(self, tx, timeout=60, success=True, use_wtxid=False): with mininode_lock: self.last_message.pop("getdata", None) - self.send_message(msg_inv(inv=[CInv(MSG_TX, tx.sha256)])) + if use_wtxid: + wtxid = tx.calc_sha256(True) + self.send_message(msg_inv(inv=[CInv(MSG_WTX, wtxid)])) + else: + self.send_message(msg_inv(inv=[CInv(MSG_TX, tx.sha256)])) + if success: - self.wait_for_getdata([tx.sha256], timeout) + if use_wtxid: + self.wait_for_getdata([wtxid], timeout) + else: + self.wait_for_getdata([tx.sha256], timeout) else: time.sleep(timeout) assert not self.last_message.get("getdata") @@ -277,6 +302,7 @@ class SegWitTest(BitcoinTestFramework): self.test_upgrade_after_activation() self.test_witness_sigops() self.test_superfluous_witness() + self.test_wtxid_relay() # Individual tests @@ -1270,7 +1296,6 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[0], self.test_node, tx, with_witness=True, accepted=False) # Verify that removing the witness succeeds. - self.test_node.announce_tx_and_wait_for_getdata(tx) test_transaction_acceptance(self.nodes[0], self.test_node, tx, with_witness=False, accepted=True) # Now try to add extra witness data to a valid witness tx. @@ -1297,8 +1322,6 @@ class SegWitTest(BitcoinTestFramework): # Node will not be blinded to the transaction self.std_node.announce_tx_and_wait_for_getdata(tx3) test_transaction_acceptance(self.nodes[1], self.std_node, tx3, True, False, 'tx-size') - self.std_node.announce_tx_and_wait_for_getdata(tx3) - test_transaction_acceptance(self.nodes[1], self.std_node, tx3, True, False, 'tx-size') # Remove witness stuffing, instead add extra witness push on stack tx3.vout[0] = CTxOut(tx2.vout[0].nValue - 1000, CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE])) @@ -1898,8 +1921,7 @@ class SegWitTest(BitcoinTestFramework): def test_upgrade_after_activation(self): """Test the behavior of starting up a segwit-aware node after the softfork has activated.""" - self.stop_node(2) - self.start_node(2, extra_args=["-segwitheight={}".format(SEGWIT_HEIGHT)]) + self.restart_node(2, extra_args=["-segwitheight={}".format(SEGWIT_HEIGHT)]) connect_nodes(self.nodes[0], 2) # We reconnect more than 100 blocks, give it plenty of time @@ -2017,6 +2039,11 @@ class SegWitTest(BitcoinTestFramework): # TODO: test p2sh sigop counting + # Cleanup and prep for next test + self.utxo.pop(0) + self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) + + @subtest # type: ignore def test_superfluous_witness(self): # Serialization of tx that puts witness flag to 3 always def serialize_with_bogus_witness(tx): @@ -2060,6 +2087,67 @@ class SegWitTest(BitcoinTestFramework): with self.nodes[0].assert_debug_log(['Unknown transaction optional data']): self.nodes[0].p2p.send_and_ping(msg_bogus_tx(tx)) + @subtest # type: ignore + def test_wtxid_relay(self): + # Use brand new nodes to avoid contamination from earlier tests + self.wtx_node = self.nodes[0].add_p2p_connection(TestP2PConn(wtxidrelay=True), services=NODE_NETWORK | NODE_WITNESS) + self.tx_node = self.nodes[0].add_p2p_connection(TestP2PConn(wtxidrelay=False), services=NODE_NETWORK | NODE_WITNESS) + + # Check wtxidrelay feature negotiation message through connecting a new peer + def received_wtxidrelay(): + return (len(self.wtx_node.last_wtxidrelay) > 0) + wait_until(received_wtxidrelay, timeout=60, lock=mininode_lock) + + # Create a Segwit output from the latest UTXO + # and announce it to the network + witness_program = CScript([OP_TRUE]) + witness_hash = sha256(witness_program) + script_pubkey = CScript([OP_0, witness_hash]) + + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) + tx.vout.append(CTxOut(self.utxo[0].nValue - 1000, script_pubkey)) + tx.rehash() + + # Create a Segwit transaction + tx2 = CTransaction() + tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) + tx2.vout.append(CTxOut(tx.vout[0].nValue - 1000, script_pubkey)) + tx2.wit.vtxinwit.append(CTxInWitness()) + tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_program] + tx2.rehash() + + # Announce Segwit transaction with wtxid + # and wait for getdata + self.wtx_node.announce_tx_and_wait_for_getdata(tx2, use_wtxid=True) + with mininode_lock: + lgd = self.wtx_node.lastgetdata[:] + assert_equal(lgd, [CInv(MSG_WTX, tx2.calc_sha256(True))]) + + # Announce Segwit transaction from non wtxidrelay peer + # and wait for getdata + self.tx_node.announce_tx_and_wait_for_getdata(tx2, use_wtxid=False) + with mininode_lock: + lgd = self.tx_node.lastgetdata[:] + assert_equal(lgd, [CInv(MSG_TX|MSG_WITNESS_FLAG, tx2.sha256)]) + + # Send tx2 through; it's an orphan so won't be accepted + with mininode_lock: + self.tx_node.last_message.pop("getdata", None) + test_transaction_acceptance(self.nodes[0], self.tx_node, tx2, with_witness=True, accepted=False) + + # Expect a request for parent (tx) due to use of non-WTX peer + self.tx_node.wait_for_getdata([tx.sha256], 60) + with mininode_lock: + lgd = self.tx_node.lastgetdata[:] + assert_equal(lgd, [CInv(MSG_TX|MSG_WITNESS_FLAG, tx.sha256)]) + + # Send tx through + test_transaction_acceptance(self.nodes[0], self.tx_node, tx, with_witness=False, accepted=True) + + # Check tx2 is there now + assert_equal(tx2.hash in self.nodes[0].getrawmempool(), True) + if __name__ == '__main__': SegWitTest().main() diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 10f5eea0e5..2527edc135 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -12,6 +12,7 @@ from test_framework.messages import ( FromHex, MSG_TX, MSG_TYPE_MASK, + MSG_WTX, msg_inv, msg_notfound, ) @@ -36,7 +37,7 @@ class TestP2PConn(P2PInterface): def on_getdata(self, message): for i in message.inv: - if i.type & MSG_TYPE_MASK == MSG_TX: + if i.type & MSG_TYPE_MASK == MSG_TX or i.type & MSG_TYPE_MASK == MSG_WTX: self.tx_getdata_count += 1 @@ -44,12 +45,13 @@ class TestP2PConn(P2PInterface): GETDATA_TX_INTERVAL = 60 # seconds MAX_GETDATA_RANDOM_DELAY = 2 # seconds INBOUND_PEER_TX_DELAY = 2 # seconds +TXID_RELAY_DELAY = 2 # seconds MAX_GETDATA_IN_FLIGHT = 100 TX_EXPIRY_INTERVAL = GETDATA_TX_INTERVAL * 10 # Python test constants NUM_INBOUND = 10 -MAX_GETDATA_INBOUND_WAIT = GETDATA_TX_INTERVAL + MAX_GETDATA_RANDOM_DELAY + INBOUND_PEER_TX_DELAY +MAX_GETDATA_INBOUND_WAIT = GETDATA_TX_INTERVAL + MAX_GETDATA_RANDOM_DELAY + INBOUND_PEER_TX_DELAY + TXID_RELAY_DELAY class TxDownloadTest(BitcoinTestFramework): @@ -63,7 +65,7 @@ class TxDownloadTest(BitcoinTestFramework): txid = 0xdeadbeef self.log.info("Announce the txid from each incoming peer to node 0") - msg = msg_inv([CInv(t=MSG_TX, h=txid)]) + msg = msg_inv([CInv(t=MSG_WTX, h=txid)]) for p in self.nodes[0].p2ps: p.send_and_ping(msg) @@ -135,13 +137,13 @@ class TxDownloadTest(BitcoinTestFramework): with mininode_lock: p.tx_getdata_count = 0 - p.send_message(msg_inv([CInv(t=MSG_TX, h=i) for i in txids])) + p.send_message(msg_inv([CInv(t=MSG_WTX, h=i) for i in txids])) wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT, lock=mininode_lock) with mininode_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT) self.log.info("Now check that if we send a NOTFOUND for a transaction, we'll get one more request") - p.send_message(msg_notfound(vec=[CInv(t=MSG_TX, h=txids[0])])) + p.send_message(msg_notfound(vec=[CInv(t=MSG_WTX, h=txids[0])])) wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT + 1, timeout=10, lock=mininode_lock) with mininode_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT + 1) diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index c323168848..71b0b0f63a 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of unrequested blocks. -Setup: two nodes, node0+node1, not connected to each other. Node1 will have +Setup: two nodes, node0 + node1, not connected to each other. Node1 will have nMinimumChainWork set to 0x10, so it won't process low-work unrequested blocks. We have one P2PInterface connection to node0 called test_node, and one to node1 @@ -71,18 +71,10 @@ class AcceptBlockTest(BitcoinTestFramework): self.extra_args = [[], ["-minimumchainwork=0x10"]] def setup_network(self): - # Node0 will be used to test behavior of processing unrequested blocks - # from peers which are not whitelisted, while Node1 will be used for - # the whitelisted case. - # Node2 will be used for non-whitelisted peers to test the interaction - # with nMinimumChainWork. self.setup_nodes() def run_test(self): - # Setup the p2p connections - # test_node connects to node0 (not whitelisted) test_node = self.nodes[0].add_p2p_connection(P2PInterface()) - # min_work_node connects to node1 (whitelisted) min_work_node = self.nodes[1].add_p2p_connection(P2PInterface()) # 1. Have nodes mine a block (leave IBD) @@ -226,7 +218,7 @@ class AcceptBlockTest(BitcoinTestFramework): self.nodes[0].getblock(all_blocks[286].hash) assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[287].hash) - self.log.info("Successfully reorged to longer chain from non-whitelisted peer") + self.log.info("Successfully reorged to longer chain") # 8. Create a chain which is invalid at a height longer than the # current chain, but which has more blocks on top of that diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 6273c229ae..7c70f30ca3 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -241,6 +241,17 @@ class BlockchainTest(BitcoinTestFramework): del res['disk_size'], res3['disk_size'] assert_equal(res, res3) + self.log.info("Test hash_type option for gettxoutsetinfo()") + # Adding hash_type 'hash_serialized_2', which is the default, should + # not change the result. + res4 = node.gettxoutsetinfo(hash_type='hash_serialized_2') + del res4['disk_size'] + assert_equal(res, res4) + + # hash_type none should not return a UTXO set hash. + res5 = node.gettxoutsetinfo(hash_type='none') + assert 'hash_serialized_2' not in res5 + def _test_getblockheader(self): node = self.nodes[0] diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index 4bc4913bda..57c8f511ac 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -271,7 +271,11 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) assert_equal("00", dec_tx['vin'][0]['scriptSig']['hex']) + # Should fail without add_inputs: + assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[2].fundrawtransaction, rawtx, {"add_inputs": False}) + # add_inputs is enabled by default rawtxfund = self.nodes[2].fundrawtransaction(rawtx) + dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) totalOut = 0 matchingOuts = 0 @@ -299,7 +303,10 @@ class RawTransactionsTest(BitcoinTestFramework): dec_tx = self.nodes[2].decoderawtransaction(rawtx) assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) - rawtxfund = self.nodes[2].fundrawtransaction(rawtx) + # Should fail without add_inputs: + assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[2].fundrawtransaction, rawtx, {"add_inputs": False}) + rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {"add_inputs": True}) + dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) totalOut = 0 matchingOuts = 0 @@ -330,7 +337,10 @@ class RawTransactionsTest(BitcoinTestFramework): dec_tx = self.nodes[2].decoderawtransaction(rawtx) assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) - rawtxfund = self.nodes[2].fundrawtransaction(rawtx) + # Should fail without add_inputs: + assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[2].fundrawtransaction, rawtx, {"add_inputs": False}) + rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {"add_inputs": True}) + dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) totalOut = 0 matchingOuts = 0 diff --git a/test/functional/rpc_getaddressinfo_label_deprecation.py b/test/functional/rpc_getaddressinfo_label_deprecation.py deleted file mode 100755 index 09545ebce7..0000000000 --- a/test/functional/rpc_getaddressinfo_label_deprecation.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -""" -Test deprecation of the RPC getaddressinfo `label` field. It has been -superseded by the `labels` field. - -""" -from test_framework.test_framework import BitcoinTestFramework - -class GetAddressInfoLabelDeprecationTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 2 - self.setup_clean_chain = False - # Start node[0] with -deprecatedrpc=label, and node[1] without. - self.extra_args = [["-deprecatedrpc=label"], []] - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - - def test_label_with_deprecatedrpc_flag(self): - self.log.info("Test getaddressinfo label with -deprecatedrpc flag") - node = self.nodes[0] - address = node.getnewaddress() - info = node.getaddressinfo(address) - assert "label" in info - - def test_label_without_deprecatedrpc_flag(self): - self.log.info("Test getaddressinfo label without -deprecatedrpc flag") - node = self.nodes[1] - address = node.getnewaddress() - info = node.getaddressinfo(address) - assert "label" not in info - - def run_test(self): - """Test getaddressinfo label with and without -deprecatedrpc flag.""" - self.test_label_with_deprecatedrpc_flag() - self.test_label_without_deprecatedrpc_flag() - - -if __name__ == '__main__': - GetAddressInfoLabelDeprecationTest().main() diff --git a/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py b/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py deleted file mode 100755 index 903f5536b9..0000000000 --- a/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -""" -Test deprecation of RPC getaddressinfo `labels` returning an array -containing a JSON object of `name` and purpose` key-value pairs. It now -returns an array containing only the label name. - -""" -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal - -LABELS_TO_TEST = frozenset({"" , "New 𝅘𝅥𝅯 $<#>&!рыба Label"}) - -class GetAddressInfoLabelsPurposeDeprecationTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 2 - self.setup_clean_chain = False - # Start node[0] with -deprecatedrpc=labelspurpose and node[1] without. - self.extra_args = [["-deprecatedrpc=labelspurpose"], []] - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - - def test_labels(self, node_num, label_name, expected_value): - node = self.nodes[node_num] - address = node.getnewaddress() - if label_name != "": - node.setlabel(address, label_name) - self.log.info(" set label to {}".format(label_name)) - labels = node.getaddressinfo(address)["labels"] - self.log.info(" labels = {}".format(labels)) - assert_equal(labels, expected_value) - - def run_test(self): - """Test getaddressinfo labels with and without -deprecatedrpc flag.""" - self.log.info("Test getaddressinfo labels with -deprecatedrpc flag") - for label in LABELS_TO_TEST: - self.test_labels(node_num=0, label_name=label, expected_value=[{"name": label, "purpose": "receive"}]) - - self.log.info("Test getaddressinfo labels without -deprecatedrpc flag") - for label in LABELS_TO_TEST: - self.test_labels(node_num=1, label_name=label, expected_value=[label]) - - -if __name__ == '__main__': - GetAddressInfoLabelsPurposeDeprecationTest().main() diff --git a/test/functional/rpc_getblockfilter.py b/test/functional/rpc_getblockfilter.py index bd93b6f7a4..8fa36445cd 100755 --- a/test/functional/rpc_getblockfilter.py +++ b/test/functional/rpc_getblockfilter.py @@ -7,7 +7,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_is_hex_string, assert_raises_rpc_error, - connect_nodes, disconnect_nodes, sync_blocks + connect_nodes, disconnect_nodes ) FILTER_TYPES = ["basic"] @@ -30,7 +30,7 @@ class GetBlockFilterTest(BitcoinTestFramework): # Reorg node 0 to a new chain connect_nodes(self.nodes[0], 1) - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(self.nodes[0].getblockcount(), 4) chain1_hashes = [self.nodes[0].getblockhash(block_height) for block_height in range(4)] diff --git a/test/functional/rpc_getpeerinfo_banscore_deprecation.py b/test/functional/rpc_getpeerinfo_banscore_deprecation.py new file mode 100755 index 0000000000..b830248e1e --- /dev/null +++ b/test/functional/rpc_getpeerinfo_banscore_deprecation.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test deprecation of getpeerinfo RPC banscore field.""" + +from test_framework.test_framework import BitcoinTestFramework + + +class GetpeerinfoBanscoreDeprecationTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.extra_args = [[], ["-deprecatedrpc=banscore"]] + + def run_test(self): + self.log.info("Test getpeerinfo by default no longer returns a banscore field") + assert "banscore" not in self.nodes[0].getpeerinfo()[0].keys() + + self.log.info("Test getpeerinfo returns banscore with -deprecatedrpc=banscore") + assert "banscore" in self.nodes[1].getpeerinfo()[0].keys() + + +if __name__ == "__main__": + GetpeerinfoBanscoreDeprecationTest().main() diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 58d8c4abe1..3336246c8b 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -46,10 +46,12 @@ class NetTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - self.extra_args = [["-minrelaytxfee=0.00001000"],["-minrelaytxfee=0.00000500"]] + self.extra_args = [["-minrelaytxfee=0.00001000"], ["-minrelaytxfee=0.00000500"]] self.supports_cli = False def run_test(self): + self.log.info('Get out of IBD for the minfeefilter test') + self.nodes[0].generate(1) self.log.info('Connect nodes both way') connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 0) @@ -100,12 +102,14 @@ class NetTest(BitcoinTestFramework): assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) - self.nodes[0].setnetworkactive(state=False) + with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: false\n']): + self.nodes[0].setnetworkactive(state=False) assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], False) # Wait a bit for all sockets to close wait_until(lambda: self.nodes[0].getnetworkinfo()['connections'] == 0, timeout=3) - self.nodes[0].setnetworkactive(state=True) + with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: true\n']): + self.nodes[0].setnetworkactive(state=True) self.log.info('Connect nodes both way') connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 0) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 9b07c39606..4d985dd1b1 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -8,6 +8,7 @@ from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( + assert_approx, assert_equal, assert_greater_than, assert_raises_rpc_error, @@ -37,6 +38,7 @@ class PSBTTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + # TODO: Re-enable this test with segwit v1 def test_utxo_conversion(self): mining_node = self.nodes[2] offline_node = self.nodes[0] @@ -85,6 +87,13 @@ class PSBTTest(BitcoinTestFramework): # Create and fund a raw tx for sending 10 BTC psbtx1 = self.nodes[0].walletcreatefundedpsbt([], {self.nodes[2].getnewaddress():10})['psbt'] + # If inputs are specified, do not automatically add more: + utxo1 = self.nodes[0].listunspent()[0] + assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[0].walletcreatefundedpsbt, [{"txid": utxo1['txid'], "vout": utxo1['vout']}], {self.nodes[2].getnewaddress():90}) + + psbtx1 = self.nodes[0].walletcreatefundedpsbt([{"txid": utxo1['txid'], "vout": utxo1['vout']}], {self.nodes[2].getnewaddress():90}, 0, {"add_inputs": True})['psbt'] + assert_equal(len(self.nodes[0].decodepsbt(psbtx1)['tx']['vin']), 2) + # Node 1 should not be able to add anything to it but still return the psbtx same as before psbtx = self.nodes[1].walletprocesspsbt(psbtx1)['psbt'] assert_equal(psbtx1, psbtx) @@ -146,19 +155,25 @@ class PSBTTest(BitcoinTestFramework): p2pkh_pos = out['n'] # spend single key from node 1 - rawtx = self.nodes[1].walletcreatefundedpsbt([{"txid":txid,"vout":p2wpkh_pos},{"txid":txid,"vout":p2sh_p2wpkh_pos},{"txid":txid,"vout":p2pkh_pos}], {self.nodes[1].getnewaddress():29.99})['psbt'] - walletprocesspsbt_out = self.nodes[1].walletprocesspsbt(rawtx) + created_psbt = self.nodes[1].walletcreatefundedpsbt([{"txid":txid,"vout":p2wpkh_pos},{"txid":txid,"vout":p2sh_p2wpkh_pos},{"txid":txid,"vout":p2pkh_pos}], {self.nodes[1].getnewaddress():29.99}) + walletprocesspsbt_out = self.nodes[1].walletprocesspsbt(created_psbt['psbt']) + # Make sure it has both types of UTXOs + decoded = self.nodes[1].decodepsbt(walletprocesspsbt_out['psbt']) + assert 'non_witness_utxo' in decoded['inputs'][0] + assert 'witness_utxo' in decoded['inputs'][0] + # Check decodepsbt fee calculation (input values shall only be counted once per UTXO) + assert_equal(decoded['fee'], created_psbt['fee']) assert_equal(walletprocesspsbt_out['complete'], True) self.nodes[1].sendrawtransaction(self.nodes[1].finalizepsbt(walletprocesspsbt_out['psbt'])['hex']) # feeRate of 0.1 BTC / KB produces a total fee slightly below -maxtxfee (~0.05280000): - res = self.nodes[1].walletcreatefundedpsbt([{"txid":txid,"vout":p2wpkh_pos},{"txid":txid,"vout":p2sh_p2wpkh_pos},{"txid":txid,"vout":p2pkh_pos}], {self.nodes[1].getnewaddress():29.99}, 0, {"feeRate": 0.1}) - assert_greater_than(res["fee"], 0.05) - assert_greater_than(0.06, res["fee"]) + res = self.nodes[1].walletcreatefundedpsbt([{"txid":txid,"vout":p2wpkh_pos},{"txid":txid,"vout":p2sh_p2wpkh_pos},{"txid":txid,"vout":p2pkh_pos}], {self.nodes[1].getnewaddress():29.99}, 0, {"feeRate": 0.1, "add_inputs": True}) + assert_approx(res["fee"], 0.055, 0.005) # feeRate of 10 BTC / KB produces a total fee well above -maxtxfee # previously this was silently capped at -maxtxfee - assert_raises_rpc_error(-4, "Fee exceeds maximum configured by -maxtxfee", self.nodes[1].walletcreatefundedpsbt, [{"txid":txid,"vout":p2wpkh_pos},{"txid":txid,"vout":p2sh_p2wpkh_pos},{"txid":txid,"vout":p2pkh_pos}], {self.nodes[1].getnewaddress():29.99}, 0, {"feeRate": 10}) + assert_raises_rpc_error(-4, "Fee exceeds maximum configured by -maxtxfee", self.nodes[1].walletcreatefundedpsbt, [{"txid":txid,"vout":p2wpkh_pos},{"txid":txid,"vout":p2sh_p2wpkh_pos},{"txid":txid,"vout":p2pkh_pos}], {self.nodes[1].getnewaddress():29.99}, 0, {"feeRate": 10, "add_inputs": True}) + assert_raises_rpc_error(-4, "Fee exceeds maximum configured by -maxtxfee", self.nodes[1].walletcreatefundedpsbt, [{"txid":txid,"vout":p2wpkh_pos},{"txid":txid,"vout":p2sh_p2wpkh_pos},{"txid":txid,"vout":p2pkh_pos}], {self.nodes[1].getnewaddress():1}, 0, {"feeRate": 10, "add_inputs": False}) # partially sign multisig things with node 1 psbtx = wmulti.walletcreatefundedpsbt(inputs=[{"txid":txid,"vout":p2wsh_pos},{"txid":txid,"vout":p2sh_pos},{"txid":txid,"vout":p2sh_p2wsh_pos}], outputs={self.nodes[1].getnewaddress():29.99}, options={'changeAddress': self.nodes[1].getrawchangeaddress()})['psbt'] @@ -239,7 +254,7 @@ class PSBTTest(BitcoinTestFramework): # replaceable arg block_height = self.nodes[0].getblockcount() unspent = self.nodes[0].listunspent()[0] - psbtx_info = self.nodes[0].walletcreatefundedpsbt([{"txid":unspent["txid"], "vout":unspent["vout"]}], [{self.nodes[2].getnewaddress():unspent["amount"]+1}], block_height+2, {"replaceable": False}, False) + psbtx_info = self.nodes[0].walletcreatefundedpsbt([{"txid":unspent["txid"], "vout":unspent["vout"]}], [{self.nodes[2].getnewaddress():unspent["amount"]+1}], block_height+2, {"replaceable": False, "add_inputs": True}, False) decoded_psbt = self.nodes[0].decodepsbt(psbtx_info["psbt"]) for tx_in, psbt_in in zip(decoded_psbt["tx"]["vin"], decoded_psbt["inputs"]): assert_greater_than(tx_in["sequence"], MAX_BIP125_RBF_SEQUENCE) @@ -247,7 +262,7 @@ class PSBTTest(BitcoinTestFramework): assert_equal(decoded_psbt["tx"]["locktime"], block_height+2) # Same construction with only locktime set and RBF explicitly enabled - psbtx_info = self.nodes[0].walletcreatefundedpsbt([{"txid":unspent["txid"], "vout":unspent["vout"]}], [{self.nodes[2].getnewaddress():unspent["amount"]+1}], block_height, {"replaceable": True}, True) + psbtx_info = self.nodes[0].walletcreatefundedpsbt([{"txid":unspent["txid"], "vout":unspent["vout"]}], [{self.nodes[2].getnewaddress():unspent["amount"]+1}], block_height, {"replaceable": True, "add_inputs": True}, True) decoded_psbt = self.nodes[0].decodepsbt(psbtx_info["psbt"]) for tx_in, psbt_in in zip(decoded_psbt["tx"]["vin"], decoded_psbt["inputs"]): assert_equal(tx_in["sequence"], MAX_BIP125_RBF_SEQUENCE) @@ -255,7 +270,7 @@ class PSBTTest(BitcoinTestFramework): assert_equal(decoded_psbt["tx"]["locktime"], block_height) # Same construction without optional arguments - psbtx_info = self.nodes[0].walletcreatefundedpsbt([{"txid":unspent["txid"], "vout":unspent["vout"]}], [{self.nodes[2].getnewaddress():unspent["amount"]+1}]) + psbtx_info = self.nodes[0].walletcreatefundedpsbt([], [{self.nodes[2].getnewaddress():unspent["amount"]+1}]) decoded_psbt = self.nodes[0].decodepsbt(psbtx_info["psbt"]) for tx_in, psbt_in in zip(decoded_psbt["tx"]["vin"], decoded_psbt["inputs"]): assert_equal(tx_in["sequence"], MAX_BIP125_RBF_SEQUENCE) @@ -264,7 +279,7 @@ class PSBTTest(BitcoinTestFramework): # Same construction without optional arguments, for a node with -walletrbf=0 unspent1 = self.nodes[1].listunspent()[0] - psbtx_info = self.nodes[1].walletcreatefundedpsbt([{"txid":unspent1["txid"], "vout":unspent1["vout"]}], [{self.nodes[2].getnewaddress():unspent1["amount"]+1}], block_height) + psbtx_info = self.nodes[1].walletcreatefundedpsbt([{"txid":unspent1["txid"], "vout":unspent1["vout"]}], [{self.nodes[2].getnewaddress():unspent1["amount"]+1}], block_height, {"add_inputs": True}) decoded_psbt = self.nodes[1].decodepsbt(psbtx_info["psbt"]) for tx_in, psbt_in in zip(decoded_psbt["tx"]["vin"], decoded_psbt["inputs"]): assert_greater_than(tx_in["sequence"], MAX_BIP125_RBF_SEQUENCE) @@ -275,7 +290,7 @@ class PSBTTest(BitcoinTestFramework): self.nodes[0].walletcreatefundedpsbt([], [{self.nodes[2].getnewaddress():unspent["amount"]+1}], block_height+2, {"changeAddress":self.nodes[1].getnewaddress()}, False) # Regression test for 14473 (mishandling of already-signed witness transaction): - psbtx_info = self.nodes[0].walletcreatefundedpsbt([{"txid":unspent["txid"], "vout":unspent["vout"]}], [{self.nodes[2].getnewaddress():unspent["amount"]+1}]) + psbtx_info = self.nodes[0].walletcreatefundedpsbt([{"txid":unspent["txid"], "vout":unspent["vout"]}], [{self.nodes[2].getnewaddress():unspent["amount"]+1}], 0, {"add_inputs": True}) complete_psbt = self.nodes[0].walletprocesspsbt(psbtx_info["psbt"]) double_processed_psbt = self.nodes[0].walletprocesspsbt(complete_psbt["psbt"]) assert_equal(complete_psbt, double_processed_psbt) @@ -344,7 +359,8 @@ class PSBTTest(BitcoinTestFramework): for i, signer in enumerate(signers): self.nodes[2].unloadwallet("wallet{}".format(i)) - self.test_utxo_conversion() + # TODO: Re-enable this for segwit v1 + # self.test_utxo_conversion() # Test that psbts with p2pkh outputs are created properly p2pkh = self.nodes[0].getnewaddress(address_type='legacy') @@ -467,7 +483,7 @@ class PSBTTest(BitcoinTestFramework): assert_equal(analysis['next'], 'creator') assert_equal(analysis['error'], 'PSBT is not valid. Input 0 specifies invalid prevout') - assert_raises_rpc_error(-25, 'Missing inputs', self.nodes[0].walletprocesspsbt, 'cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==') + assert_raises_rpc_error(-25, 'Inputs missing or spent', self.nodes[0].walletprocesspsbt, 'cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==') if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 14cad3d1b8..23b5e647d6 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -424,11 +424,12 @@ class RawTransactionsTest(BitcoinTestFramework): #################################### # Test the minimum transaction version number that fits in a signed 32-bit integer. + # As transaction version is unsigned, this should convert to its unsigned equivalent. tx = CTransaction() tx.nVersion = -0x80000000 rawtx = ToHex(tx) decrawtx = self.nodes[0].decoderawtransaction(rawtx) - assert_equal(decrawtx['version'], -0x80000000) + assert_equal(decrawtx['version'], 0x80000000) # Test the maximum transaction version number that fits in a signed 32-bit integer. tx = CTransaction() diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 05308931e3..81eb881234 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -115,6 +115,8 @@ class AuthServiceProxy(): except OSError as e: retry = ( '[WinError 10053] An established connection was aborted by the software in your host machine' in str(e)) + # Workaround for a bug on macOS. See https://bugs.python.org/issue33450 + retry = retry or ('[Errno 41] Protocol wrong type for socket' in str(e)) if retry: self.__conn.close() self.__conn.request(method, path, postdata, headers) diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 4d1dd4422e..2462a9a6db 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -31,7 +31,7 @@ from test_framework.siphash import siphash256 from test_framework.util import hex_str_to_bytes, assert_equal MIN_VERSION_SUPPORTED = 60001 -MY_VERSION = 70014 # past bip-31 for ping/pong +MY_VERSION = 70016 # past wtxid relay MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) @@ -45,6 +45,10 @@ MAX_MONEY = 21000000 * COIN BIP125_SEQUENCE_NUMBER = 0xfffffffd # Sequence number that is BIP 125 opt-in and BIP 68-opt-out +MAX_PROTOCOL_MESSAGE_LENGTH = 4000000 # Maximum length of incoming protocol messages +MAX_HEADERS_RESULTS = 2000 # Number of headers sent in one getheaders result +MAX_INV_SIZE = 50000 # Maximum number of entries in an 'inv' protocol message + NODE_NETWORK = (1 << 0) NODE_GETUTXO = (1 << 1) NODE_BLOOM = (1 << 2) @@ -55,6 +59,7 @@ MSG_TX = 1 MSG_BLOCK = 2 MSG_FILTERED_BLOCK = 3 MSG_CMPCT_BLOCK = 4 +MSG_WTX = 5 MSG_WITNESS_FLAG = 1 << 30 MSG_TYPE_MASK = 0xffffffff >> 2 @@ -203,17 +208,19 @@ class CAddress: self.ip = "0.0.0.0" self.port = 0 - def deserialize(self, f, with_time=True): + def deserialize(self, f, *, with_time=True): if with_time: + # VERSION messages serialize CAddress objects without time self.time = struct.unpack("<i", f.read(4))[0] self.nServices = struct.unpack("<Q", f.read(8))[0] self.pchReserved = f.read(12) self.ip = socket.inet_ntoa(f.read(4)) self.port = struct.unpack(">H", f.read(2))[0] - def serialize(self, with_time=True): + def serialize(self, *, with_time=True): r = b"" if with_time: + # VERSION messages serialize CAddress objects without time r += struct.pack("<i", self.time) r += struct.pack("<Q", self.nServices) r += self.pchReserved @@ -236,7 +243,8 @@ class CInv: MSG_TX | MSG_WITNESS_FLAG: "WitnessTx", MSG_BLOCK | MSG_WITNESS_FLAG: "WitnessBlock", MSG_FILTERED_BLOCK: "filtered Block", - 4: "CompactBlock" + 4: "CompactBlock", + 5: "WTX", } def __init__(self, t=0, h=0): @@ -257,6 +265,9 @@ class CInv: return "CInv(type=%s hash=%064x)" \ % (self.typemap[self.type], self.hash) + def __eq__(self, other): + return isinstance(other, CInv) and self.hash == other.hash and self.type == other.type + class CBlockLocator: __slots__ = ("nVersion", "vHave") @@ -969,10 +980,10 @@ class msg_version: self.nServices = struct.unpack("<Q", f.read(8))[0] self.nTime = struct.unpack("<q", f.read(8))[0] self.addrTo = CAddress() - self.addrTo.deserialize(f, False) + self.addrTo.deserialize(f, with_time=False) self.addrFrom = CAddress() - self.addrFrom.deserialize(f, False) + self.addrFrom.deserialize(f, with_time=False) self.nNonce = struct.unpack("<Q", f.read(8))[0] self.strSubVer = deser_string(f) @@ -992,8 +1003,8 @@ class msg_version: r += struct.pack("<i", self.nVersion) r += struct.pack("<Q", self.nServices) r += struct.pack("<q", self.nTime) - r += self.addrTo.serialize(False) - r += self.addrFrom.serialize(False) + r += self.addrTo.serialize(with_time=False) + r += self.addrFrom.serialize(with_time=False) r += struct.pack("<Q", self.nNonce) r += ser_string(self.strSubVer) r += struct.pack("<i", self.nStartingHeight) @@ -1118,6 +1129,22 @@ class msg_tx: def __repr__(self): return "msg_tx(tx=%s)" % (repr(self.tx)) +class msg_wtxidrelay: + __slots__ = () + msgtype = b"wtxidrelay" + + def __init__(self): + pass + + def deserialize(self, f): + pass + + def serialize(self): + return b"" + + def __repr__(self): + return "msg_wtxidrelay()" + class msg_no_witness_tx(msg_tx): __slots__ = () diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 45063aaff2..07811667a8 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -26,6 +26,7 @@ import threading from test_framework.messages import ( CBlockHeader, + MAX_HEADERS_RESULTS, MIN_VERSION_SUPPORTED, msg_addr, msg_block, @@ -58,6 +59,8 @@ from test_framework.messages import ( MSG_TYPE_MASK, msg_verack, msg_version, + MSG_WTX, + msg_wtxidrelay, NODE_NETWORK, NODE_WITNESS, sha256, @@ -95,6 +98,7 @@ MESSAGEMAP = { b"tx": msg_tx, b"verack": msg_verack, b"version": msg_version, + b"wtxidrelay": msg_wtxidrelay, } MAGIC_BYTES = { @@ -279,9 +283,13 @@ class P2PInterface(P2PConnection): def __init__(self): super().__init__() - # Track number of messages of each type received and the most recent - # message of each type + # Track number of messages of each type received. + # Should be read-only in a test. self.message_count = defaultdict(int) + + # Track the most recent message of each type. + # To wait for a message to be received, pop that message from + # this and use wait_until. self.last_message = {} # A count of the number of ping messages we've sent to the node @@ -355,6 +363,7 @@ class P2PInterface(P2PConnection): def on_sendcmpct(self, message): pass def on_sendheaders(self, message): pass def on_tx(self, message): pass + def on_wtxidrelay(self, message): pass def on_inv(self, message): want = msg_getdata() @@ -372,6 +381,8 @@ class P2PInterface(P2PConnection): def on_version(self, message): assert message.nVersion >= MIN_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format(message.nVersion, MIN_VERSION_SUPPORTED) + if message.nVersion >= 70016: + self.send_message(msg_wtxidrelay()) self.send_message(msg_verack()) self.nServices = message.nServices @@ -465,7 +476,7 @@ class P2PInterface(P2PConnection): def wait_for_verack(self, timeout=60): def test_function(): - return self.message_count["verack"] + return "verack" in self.last_message self.wait_until(test_function, timeout=timeout) @@ -492,7 +503,7 @@ class P2PInterface(P2PConnection): # P2PConnection acquires this lock whenever delivering a message to a P2PInterface. # This lock should be acquired in the thread running the test logic to synchronize # access to any data shared with the P2PInterface or P2PConnection. -mininode_lock = threading.RLock() +mininode_lock = threading.Lock() class NetworkThread(threading.Thread): @@ -553,7 +564,6 @@ class P2PDataStore(P2PInterface): return headers_list = [self.block_store[self.last_block_hash]] - maxheaders = 2000 while headers_list[-1].sha256 not in locator.vHave: # Walk back through the block store, adding headers to headers_list # as we go. @@ -569,7 +579,7 @@ class P2PDataStore(P2PInterface): break # Truncate the list if there are too many headers - headers_list = headers_list[:-maxheaders - 1:-1] + headers_list = headers_list[:-MAX_HEADERS_RESULTS - 1:-1] response = msg_headers(headers_list) if response is not None: @@ -654,12 +664,10 @@ class P2PTxInvStore(P2PInterface): super().on_inv(message) # Send getdata in response. # Store how many times invs have been received for each tx. for i in message.inv: - if i.type == MSG_TX: + if (i.type == MSG_TX) or (i.type == MSG_WTX): # save txid self.tx_invs_received[i.hash] += 1 - super().on_inv(message) - def get_invs(self): with mininode_lock: return list(self.tx_invs_received.keys()) @@ -669,6 +677,6 @@ class P2PTxInvStore(P2PInterface): The mempool should mark unbroadcast=False for these transactions. """ # Wait until invs have been received (and getdatas sent) for each txid. - self.wait_until(lambda: set(self.get_invs()) == set([int(tx, 16) for tx in txns]), timeout) + self.wait_until(lambda: set(self.tx_invs_received.keys()) == set([int(tx, 16) for tx in txns]), timeout) # Flush messages and wait for the getdatas to be processed self.sync_with_ping() diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 9f5e9e5f0d..8d402d4888 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -31,8 +31,7 @@ from .util import ( disconnect_nodes, get_datadir_path, initialize_datadir, - sync_blocks, - sync_mempools, + wait_until, ) @@ -355,9 +354,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # See fPreferredDownload in net_processing. # # If further outbound connections are needed, they can be added at the beginning of the test with e.g. - # connect_nodes(self.nodes[1], 2) + # self.connect_nodes(1, 2) for i in range(self.num_nodes - 1): - connect_nodes(self.nodes[i + 1], i) + self.connect_nodes(i + 1, i) self.sync_all() def setup_nodes(self): @@ -452,7 +451,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): assert_equal(len(binary), num_nodes) assert_equal(len(binary_cli), num_nodes) for i in range(num_nodes): - self.nodes.append(TestNode( + test_node_i = TestNode( i, get_datadir_path(self.options.tmpdir, i), chain=self.chain, @@ -470,7 +469,15 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): start_perf=self.options.perf, use_valgrind=self.options.valgrind, descriptors=self.options.descriptors, - )) + ) + self.nodes.append(test_node_i) + if not test_node_i.version_is_at_least(170000): + # adjust conf for pre 17 + conf_file = test_node_i.bitcoinconf + with open(conf_file, 'r', encoding='utf8') as conf: + conf_data = conf.read() + with open(conf_file, 'w', encoding='utf8') as conf: + conf.write(conf_data.replace('[regtest]', '')) def start_node(self, i, *args, **kwargs): """Start a bitcoind""" @@ -526,11 +533,17 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def wait_for_node_exit(self, i, timeout): self.nodes[i].process.wait(timeout) + def connect_nodes(self, a, b): + connect_nodes(self.nodes[a], b) + + def disconnect_nodes(self, a, b): + disconnect_nodes(self.nodes[a], b) + def split_network(self): """ Split the network of four nodes into nodes 0/1 and 2/3. """ - disconnect_nodes(self.nodes[1], 2) + self.disconnect_nodes(1, 2) self.sync_all(self.nodes[:2]) self.sync_all(self.nodes[2:]) @@ -538,18 +551,60 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """ Join the (previously split) network halves together. """ - connect_nodes(self.nodes[1], 2) + self.connect_nodes(1, 2) self.sync_all() - def sync_blocks(self, nodes=None, **kwargs): - sync_blocks(nodes or self.nodes, **kwargs) - - def sync_mempools(self, nodes=None, **kwargs): - sync_mempools(nodes or self.nodes, **kwargs) - - def sync_all(self, nodes=None, **kwargs): - self.sync_blocks(nodes, **kwargs) - self.sync_mempools(nodes, **kwargs) + def sync_blocks(self, nodes=None, wait=1, timeout=60): + """ + Wait until everybody has the same tip. + sync_blocks needs to be called with an rpc_connections set that has least + one node already synced to the latest, stable tip, otherwise there's a + chance it might return before all nodes are stably synced. + """ + rpc_connections = nodes or self.nodes + timeout = int(timeout * self.options.timeout_factor) + stop_time = time.time() + timeout + while time.time() <= stop_time: + best_hash = [x.getbestblockhash() for x in rpc_connections] + if best_hash.count(best_hash[0]) == len(rpc_connections): + return + # Check that each peer has at least one connection + assert (all([len(x.getpeerinfo()) for x in rpc_connections])) + time.sleep(wait) + raise AssertionError("Block sync timed out after {}s:{}".format( + timeout, + "".join("\n {!r}".format(b) for b in best_hash), + )) + + def sync_mempools(self, nodes=None, wait=1, timeout=60, flush_scheduler=True): + """ + Wait until everybody has the same transactions in their memory + pools + """ + rpc_connections = nodes or self.nodes + timeout = int(timeout * self.options.timeout_factor) + stop_time = time.time() + timeout + while time.time() <= stop_time: + pool = [set(r.getrawmempool()) for r in rpc_connections] + if pool.count(pool[0]) == len(rpc_connections): + if flush_scheduler: + for r in rpc_connections: + r.syncwithvalidationinterfacequeue() + return + # Check that each peer has at least one connection + assert (all([len(x.getpeerinfo()) for x in rpc_connections])) + time.sleep(wait) + raise AssertionError("Mempool sync timed out after {}s:{}".format( + timeout, + "".join("\n {!r}".format(m) for m in pool), + )) + + def sync_all(self, nodes=None): + self.sync_blocks(nodes) + self.sync_mempools(nodes) + + def wait_until(self, test_function, timeout=60, lock=None): + return wait_until(test_function, timeout=timeout, lock=lock, timeout_factor=self.options.timeout_factor) # Private helper methods. These should not be accessed by the subclass test scripts. diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index ebc0501e11..66bb2c89b5 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -23,6 +23,7 @@ import sys from .authproxy import JSONRPCException from .descriptors import descsum_create +from .messages import MY_SUBVERSION from .util import ( MAX_NODES, append_config, @@ -549,11 +550,16 @@ class TestNode(): assert self.p2ps, self._node_msg("No p2p connection") return self.p2ps[0] + def num_connected_mininodes(self): + """Return number of test framework p2p connections to the node.""" + return len([peer for peer in self.getpeerinfo() if peer['subver'] == MY_SUBVERSION]) + def disconnect_p2ps(self): """Close all p2p connections to the node.""" for p in self.p2ps: p.peer_disconnect() del self.p2ps[:] + wait_until(lambda: self.num_connected_mininodes() == 0) class TestNodeCLIAttr: diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 52306c8c3d..506057f1fa 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -25,6 +25,7 @@ logger = logging.getLogger("TestFramework.utils") # Assert functions ################## + def assert_approx(v, vexp, vspan=0.00001): """Assert that `v` is within `vspan` of `vexp`""" if v < vexp - vspan: @@ -32,6 +33,7 @@ def assert_approx(v, vexp, vspan=0.00001): if v > vexp + vspan: raise AssertionError("%s > [%s..%s]" % (str(v), str(vexp - vspan), str(vexp + vspan))) + def assert_fee_amount(fee, tx_size, fee_per_kB): """Assert the fee was in range""" target_fee = round(tx_size * fee_per_kB / 1000, 8) @@ -41,21 +43,26 @@ def assert_fee_amount(fee, tx_size, fee_per_kB): if fee > (tx_size + 2) * fee_per_kB / 1000: raise AssertionError("Fee of %s BTC too high! (Should be %s BTC)" % (str(fee), str(target_fee))) + def assert_equal(thing1, thing2, *args): if thing1 != thing2 or any(thing1 != arg for arg in args): raise AssertionError("not(%s)" % " == ".join(str(arg) for arg in (thing1, thing2) + args)) + def assert_greater_than(thing1, thing2): if thing1 <= thing2: raise AssertionError("%s <= %s" % (str(thing1), str(thing2))) + def assert_greater_than_or_equal(thing1, thing2): if thing1 < thing2: raise AssertionError("%s < %s" % (str(thing1), str(thing2))) + def assert_raises(exc, fun, *args, **kwds): assert_raises_message(exc, None, fun, *args, **kwds) + def assert_raises_message(exc, message, fun, *args, **kwds): try: fun(*args, **kwds) @@ -71,6 +78,7 @@ def assert_raises_message(exc, message, fun, *args, **kwds): else: raise AssertionError("No exception raised") + def assert_raises_process_error(returncode, output, fun, *args, **kwds): """Execute a process and asserts the process return code and output. @@ -95,6 +103,7 @@ def assert_raises_process_error(returncode, output, fun, *args, **kwds): else: raise AssertionError("No exception raised") + def assert_raises_rpc_error(code, message, fun, *args, **kwds): """Run an RPC and verify that a specific JSONRPC exception code and message is raised. @@ -113,6 +122,7 @@ def assert_raises_rpc_error(code, message, fun, *args, **kwds): """ assert try_rpc(code, message, fun, *args, **kwds), "No exception raised" + def try_rpc(code, message, fun, *args, **kwds): """Tries to run an rpc command. @@ -134,22 +144,22 @@ def try_rpc(code, message, fun, *args, **kwds): else: return False + def assert_is_hex_string(string): try: int(string, 16) except Exception as e: - raise AssertionError( - "Couldn't interpret %r as hexadecimal; raised: %s" % (string, e)) + raise AssertionError("Couldn't interpret %r as hexadecimal; raised: %s" % (string, e)) + def assert_is_hash_string(string, length=64): if not isinstance(string, str): raise AssertionError("Expected a string, got type %r" % type(string)) elif length and len(string) != length: - raise AssertionError( - "String of length %d expected; got %d" % (length, len(string))) + raise AssertionError("String of length %d expected; got %d" % (length, len(string))) elif not re.match('[abcdef0-9]+$', string): - raise AssertionError( - "String %r contains invalid characters for a hash." % string) + raise AssertionError("String %r contains invalid characters for a hash." % string) + def assert_array_result(object_array, to_match, expected, should_not_find=False): """ @@ -180,9 +190,11 @@ def assert_array_result(object_array, to_match, expected, should_not_find=False) if num_matched > 0 and should_not_find: raise AssertionError("Objects were found %s" % (str(to_match))) + # Utility functions ################### + def check_json_precision(): """Make sure json library being used does not lose precision converting BTC values""" n = Decimal("20000000.00000003") @@ -190,11 +202,13 @@ def check_json_precision(): if satoshis != 2000000000000003: raise RuntimeError("JSON encode/decode loses precision") + def EncodeDecimal(o): if isinstance(o, Decimal): return str(o) raise TypeError(repr(o) + " is not JSON serializable") + def count_bytes(hex_string): return len(bytearray.fromhex(hex_string)) @@ -202,12 +216,15 @@ def count_bytes(hex_string): def hex_str_to_bytes(hex_str): return unhexlify(hex_str.encode('ascii')) + def str_to_b64str(string): return b64encode(string.encode('utf-8')).decode('ascii') + def satoshi_round(amount): return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) + def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None, timeout_factor=1.0): if attempts == float('inf') and timeout == float('inf'): timeout = 60 @@ -235,6 +252,7 @@ def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=N raise AssertionError("Predicate {} not true after {} seconds".format(predicate_source, timeout)) raise RuntimeError('Unreachable') + # RPC/P2P connection constants and functions ############################################ @@ -250,6 +268,7 @@ class PortSeed: # Must be initialized with a unique integer for each process n = None + def get_rpc_proxy(url, node_number, *, timeout=None, coveragedir=None): """ Args: @@ -271,18 +290,20 @@ def get_rpc_proxy(url, node_number, *, timeout=None, coveragedir=None): proxy = AuthServiceProxy(url, **proxy_kwargs) proxy.url = url # store URL on proxy for info - coverage_logfile = coverage.get_filename( - coveragedir, node_number) if coveragedir else None + coverage_logfile = coverage.get_filename(coveragedir, node_number) if coveragedir else None return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile) + def p2p_port(n): assert n <= MAX_NODES return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) + def rpc_port(n): return PORT_MIN + PORT_RANGE + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) + def rpc_url(datadir, i, chain, rpchost): rpc_u, rpc_p = get_auth_cookie(datadir, chain) host = '127.0.0.1' @@ -295,9 +316,11 @@ def rpc_url(datadir, i, chain, rpchost): host = rpchost return "http://%s:%s@%s:%d" % (rpc_u, rpc_p, host, int(port)) + # Node functions ################ + def initialize_datadir(dirname, n, chain): datadir = get_datadir_path(dirname, n) if not os.path.isdir(datadir): @@ -327,21 +350,17 @@ def initialize_datadir(dirname, n, chain): os.makedirs(os.path.join(datadir, 'stdout'), exist_ok=True) return datadir -def adjust_bitcoin_conf_for_pre_17(conf_file): - with open(conf_file,'r', encoding='utf8') as conf: - conf_data = conf.read() - with open(conf_file, 'w', encoding='utf8') as conf: - conf_data_changed = conf_data.replace('[regtest]', '') - conf.write(conf_data_changed) def get_datadir_path(dirname, n): return os.path.join(dirname, "node" + str(n)) + def append_config(datadir, options): with open(os.path.join(datadir, "bitcoin.conf"), 'a', encoding='utf8') as f: for option in options: f.write(option + "\n") + def get_auth_cookie(datadir, chain): user = None password = None @@ -366,20 +385,24 @@ def get_auth_cookie(datadir, chain): raise ValueError("No RPC credentials") return user, password + # If a cookie file exists in the given datadir, delete it. def delete_cookie_file(datadir, chain): if os.path.isfile(os.path.join(datadir, chain, ".cookie")): logger.debug("Deleting leftover cookie file") os.remove(os.path.join(datadir, chain, ".cookie")) + def softfork_active(node, key): """Return whether a softfork is active.""" return node.getblockchaininfo()['softforks'][key]['active'] + def set_node_times(nodes, t): for node in nodes: node.setmocktime(t) + def disconnect_nodes(from_connection, node_num): def get_peer_ids(): result = [] @@ -392,7 +415,7 @@ def disconnect_nodes(from_connection, node_num): if not peer_ids: logger.warning("disconnect_nodes: {} and {} were not connected".format( from_connection.index, - node_num + node_num, )) return for peer_id in peer_ids: @@ -402,12 +425,13 @@ def disconnect_nodes(from_connection, node_num): # If this node is disconnected between calculating the peer id # and issuing the disconnect, don't worry about it. # This avoids a race condition if we're mass-disconnecting peers. - if e.error['code'] != -29: # RPC_CLIENT_NODE_NOT_CONNECTED + if e.error['code'] != -29: # RPC_CLIENT_NODE_NOT_CONNECTED raise # wait to disconnect wait_until(lambda: not get_peer_ids(), timeout=5) + def connect_nodes(from_connection, node_num): ip_port = "127.0.0.1:" + str(p2p_port(node_num)) from_connection.addnode(ip_port, "onetry") @@ -420,50 +444,6 @@ def connect_nodes(from_connection, node_num): wait_until(lambda: all(peer['bytesrecv_per_msg'].pop('verack', 0) == 24 for peer in from_connection.getpeerinfo())) -def sync_blocks(rpc_connections, *, wait=1, timeout=60): - """ - Wait until everybody has the same tip. - - sync_blocks needs to be called with an rpc_connections set that has least - one node already synced to the latest, stable tip, otherwise there's a - chance it might return before all nodes are stably synced. - """ - stop_time = time.time() + timeout - while time.time() <= stop_time: - best_hash = [x.getbestblockhash() for x in rpc_connections] - if best_hash.count(best_hash[0]) == len(rpc_connections): - return - # Check that each peer has at least one connection - assert (all([len(x.getpeerinfo()) for x in rpc_connections])) - time.sleep(wait) - raise AssertionError("Block sync timed out after {}s:{}".format( - timeout, - "".join("\n {!r}".format(b) for b in best_hash), - )) - - -def sync_mempools(rpc_connections, *, wait=1, timeout=60, flush_scheduler=True): - """ - Wait until everybody has the same transactions in their memory - pools - """ - stop_time = time.time() + timeout - while time.time() <= stop_time: - pool = [set(r.getrawmempool()) for r in rpc_connections] - if pool.count(pool[0]) == len(rpc_connections): - if flush_scheduler: - for r in rpc_connections: - r.syncwithvalidationinterfacequeue() - return - # Check that each peer has at least one connection - assert (all([len(x.getpeerinfo()) for x in rpc_connections])) - time.sleep(wait) - raise AssertionError("Mempool sync timed out after {}s:{}".format( - timeout, - "".join("\n {!r}".format(m) for m in pool), - )) - - # Transaction/Block functions ############################# @@ -479,6 +459,7 @@ def find_output(node, txid, amount, *, blockhash=None): return i raise RuntimeError("find_output txid %s : %s not found" % (txid, str(amount))) + def gather_inputs(from_node, amount_needed, confirmations_required=1): """ Return a random set of unspent txouts that are enough to pay amount_needed @@ -496,6 +477,7 @@ def gather_inputs(from_node, amount_needed, confirmations_required=1): raise RuntimeError("Insufficient funds: need %d, have %d" % (amount_needed, total_in)) return (total_in, inputs) + def make_change(from_node, amount_in, amount_out, fee): """ Create change output(s), return them @@ -513,6 +495,7 @@ def make_change(from_node, amount_in, amount_out, fee): outputs[from_node.getnewaddress()] = change return outputs + def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): """ Create a random transaction. @@ -532,6 +515,7 @@ def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): return (txid, signresult["hex"], fee) + # Helper to create at least "count" utxos # Pass in a fee that is sufficient for relay and mining new transactions. def create_confirmed_utxos(fee, node, count): @@ -564,6 +548,7 @@ def create_confirmed_utxos(fee, node, count): assert len(utxos) >= count return utxos + # Create large OP_RETURN txouts that can be appended to a transaction # to make it large (helper for constructing large transactions). def gen_return_txouts(): @@ -583,6 +568,7 @@ def gen_return_txouts(): txouts.append(txout) return txouts + # Create a spend of each passed-in utxo, splicing in "txouts" to each raw # transaction to make it large. See gen_return_txouts() above. def create_lots_of_big_transactions(node, txouts, utxos, num, fee): @@ -606,6 +592,7 @@ def create_lots_of_big_transactions(node, txouts, utxos, num, fee): txids.append(txid) return txids + def mine_large_block(node, utxos=None): # generate a 66k transaction, # and 14 of them is close to the 1MB block limit @@ -619,6 +606,7 @@ def mine_large_block(node, utxos=None): create_lots_of_big_transactions(node, txouts, utxos, num, fee=fee) node.generate(1) + def find_vout_for_address(node, txid, addr): """ Locate the vout index of the given transaction sending to the diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index c0bc471d5a..95c2b7c5ec 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -190,6 +190,7 @@ BASE_SCRIPTS = [ 'rpc_preciousblock.py', 'wallet_importprunedfunds.py', 'p2p_leak_tx.py', + 'p2p_eviction.py', 'rpc_signmessage.py', 'rpc_generateblock.py', 'wallet_balance.py', @@ -232,20 +233,23 @@ BASE_SCRIPTS = [ 'feature_includeconf.py', 'feature_asmap.py', 'mempool_unbroadcast.py', + 'mempool_compatibility.py', 'rpc_deriveaddresses.py', 'rpc_deriveaddresses.py --usecli', + 'p2p_ping.py', 'rpc_scantxoutset.py', 'feature_logging.py', 'p2p_node_network_limited.py', 'p2p_permissions.py', 'feature_blocksdir.py', 'feature_config_args.py', - 'rpc_getaddressinfo_labels_purpose_deprecation.py', - 'rpc_getaddressinfo_label_deprecation.py', + 'feature_settings.py', 'rpc_getdescriptorinfo.py', + 'rpc_getpeerinfo_banscore_deprecation.py', 'rpc_help.py', 'feature_help.py', 'feature_shutdown.py', + 'p2p_ibd_txrelay.py', # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time ] @@ -396,11 +400,12 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= args = args or [] # Warn if bitcoind is already running - # pidof might fail or return an empty string if bitcoind is not running try: - if subprocess.check_output(["pidof", "bitcoind"]) not in [b'']: + # pgrep exits with code zero when one or more matching processes found + if subprocess.run(["pgrep", "-x", "bitcoind"], stdout=subprocess.DEVNULL).returncode == 0: print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0])) - except (OSError, subprocess.SubprocessError): + except OSError: + # pgrep not supported pass # Warn if there is a cache directory diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 524e1593ba..18f0beb598 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -71,8 +71,7 @@ class ToolWalletTest(BitcoinTestFramework): self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create') self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') self.assert_raises_tool_error( - 'Error initializing wallet database environment "{}"!\nError loading wallet.dat. Is wallet being used by other process?' - .format(os.path.join(self.nodes[0].datadir, self.chain, 'wallets')), + 'Error loading wallet.dat. Is wallet being used by another process?', '-wallet=wallet.dat', 'info', ) diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index 90d17a806c..8837e13005 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -95,8 +95,7 @@ class AbandonConflictTest(BitcoinTestFramework): # Restart the node with a higher min relay fee so the parent tx is no longer in mempool # TODO: redo with eviction - self.stop_node(0) - self.start_node(0, extra_args=["-minrelaytxfee=0.0001"]) + self.restart_node(0, extra_args=["-minrelaytxfee=0.0001"]) assert self.nodes[0].getmempoolinfo()['loaded'] # Verify txs no longer in either node's mempool @@ -123,8 +122,7 @@ class AbandonConflictTest(BitcoinTestFramework): balance = newbalance # Verify that even with a low min relay fee, the tx is not reaccepted from wallet on startup once abandoned - self.stop_node(0) - self.start_node(0, extra_args=["-minrelaytxfee=0.00001"]) + self.restart_node(0, extra_args=["-minrelaytxfee=0.00001"]) assert self.nodes[0].getmempoolinfo()['loaded'] assert_equal(len(self.nodes[0].getrawmempool()), 0) @@ -145,8 +143,7 @@ class AbandonConflictTest(BitcoinTestFramework): balance = newbalance # Remove using high relay fee again - self.stop_node(0) - self.start_node(0, extra_args=["-minrelaytxfee=0.0001"]) + self.restart_node(0, extra_args=["-minrelaytxfee=0.0001"]) assert self.nodes[0].getmempoolinfo()['loaded'] assert_equal(len(self.nodes[0].getrawmempool()), 0) newbalance = self.nodes[0].getbalance() diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 780cce9d02..eddd938847 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -110,9 +110,7 @@ class AvoidReuseTest(BitcoinTestFramework): assert_equal(self.nodes[0].getwalletinfo()["avoid_reuse"], False) assert_equal(self.nodes[1].getwalletinfo()["avoid_reuse"], True) - # Stop and restart node 1 - self.stop_node(1) - self.start_node(1) + self.restart_node(1) connect_nodes(self.nodes[0], 1) # Flags should still be node1.avoid_reuse=false, node2.avoid_reuse=true diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 8efa66a856..31829a18b3 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -12,7 +12,6 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, connect_nodes, - sync_blocks, ) @@ -264,7 +263,7 @@ class WalletTest(BitcoinTestFramework): # Now confirm tx_orig self.restart_node(1, ['-persistmempool=0']) connect_nodes(self.nodes[0], 1) - sync_blocks(self.nodes) + self.sync_blocks() self.nodes[1].sendrawtransaction(tx_orig) self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY) self.sync_all() diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 797c903dd3..81382d94ad 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -119,7 +119,7 @@ class WalletTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "Invalid parameter, expected locked output", self.nodes[2].lockunspent, True, [unspent_0]) self.nodes[2].lockunspent(False, [unspent_0]) assert_raises_rpc_error(-8, "Invalid parameter, output already locked", self.nodes[2].lockunspent, False, [unspent_0]) - assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 20) + assert_raises_rpc_error(-6, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 20) assert_equal([unspent_0], self.nodes[2].listlockunspent()) self.nodes[2].lockunspent(True, [unspent_0]) assert_equal(len(self.nodes[2].listlockunspent()), 0) @@ -219,6 +219,60 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), node_2_bal) node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) + # Sendmany with explicit fee (BTC/kB) + # Throw if no conf_target provided + assert_raises_rpc_error(-8, "Selected estimate_mode requires a fee rate", + self.nodes[2].sendmany, + amounts={ address: 10 }, + estimate_mode='bTc/kB') + # Throw if negative feerate + assert_raises_rpc_error(-3, "Amount out of range", + self.nodes[2].sendmany, + amounts={ address: 10 }, + conf_target=-1, + estimate_mode='bTc/kB') + fee_per_kb = 0.0002500 + explicit_fee_per_byte = Decimal(fee_per_kb) / 1000 + txid = self.nodes[2].sendmany( + amounts={ address: 10 }, + conf_target=fee_per_kb, + estimate_mode='bTc/kB', + ) + self.nodes[2].generate(1) + self.sync_all(self.nodes[0:3]) + node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), node_2_bal - Decimal('10'), explicit_fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) + assert_equal(self.nodes[2].getbalance(), node_2_bal) + node_0_bal += Decimal('10') + assert_equal(self.nodes[0].getbalance(), node_0_bal) + + # Sendmany with explicit fee (SAT/B) + # Throw if no conf_target provided + assert_raises_rpc_error(-8, "Selected estimate_mode requires a fee rate", + self.nodes[2].sendmany, + amounts={ address: 10 }, + estimate_mode='sat/b') + # Throw if negative feerate + assert_raises_rpc_error(-3, "Amount out of range", + self.nodes[2].sendmany, + amounts={ address: 10 }, + conf_target=-1, + estimate_mode='sat/b') + fee_sat_per_b = 2 + fee_per_kb = fee_sat_per_b / 100000.0 + explicit_fee_per_byte = Decimal(fee_per_kb) / 1000 + txid = self.nodes[2].sendmany( + amounts={ address: 10 }, + conf_target=fee_sat_per_b, + estimate_mode='sAT/b', + ) + self.nodes[2].generate(1) + self.sync_all(self.nodes[0:3]) + balance = self.nodes[2].getbalance() + node_2_bal = self.check_fee_amount(balance, node_2_bal - Decimal('10'), explicit_fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) + assert_equal(balance, node_2_bal) + node_0_bal += Decimal('10') + assert_equal(self.nodes[0].getbalance(), node_0_bal) + self.start_node(3, self.nodes[3].extra_args) connect_nodes(self.nodes[0], 3) self.sync_all() @@ -309,6 +363,9 @@ class WalletTest(BitcoinTestFramework): assert_equal(tx_obj['amount'], Decimal('-0.0001')) # General checks for errors from incorrect inputs + # This will raise an exception because the amount is negative + assert_raises_rpc_error(-3, "Amount out of range", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "-1") + # This will raise an exception because the amount type is wrong assert_raises_rpc_error(-3, "Invalid amount", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "1f-4") @@ -349,6 +406,74 @@ class WalletTest(BitcoinTestFramework): self.nodes[0].generate(1) self.sync_all(self.nodes[0:3]) + # send with explicit btc/kb fee + self.log.info("test explicit fee (sendtoaddress as btc/kb)") + self.nodes[0].generate(1) + self.sync_all(self.nodes[0:3]) + prebalance = self.nodes[2].getbalance() + assert prebalance > 2 + address = self.nodes[1].getnewaddress() + # Throw if no conf_target provided + assert_raises_rpc_error(-8, "Selected estimate_mode requires a fee rate", + self.nodes[2].sendtoaddress, + address=address, + amount=1.0, + estimate_mode='BTc/Kb') + # Throw if negative feerate + assert_raises_rpc_error(-3, "Amount out of range", + self.nodes[2].sendtoaddress, + address=address, + amount=1.0, + conf_target=-1, + estimate_mode='btc/kb') + txid = self.nodes[2].sendtoaddress( + address=address, + amount=1.0, + conf_target=0.00002500, + estimate_mode='btc/kb', + ) + tx_size = self.get_vsize(self.nodes[2].gettransaction(txid)['hex']) + self.sync_all(self.nodes[0:3]) + self.nodes[0].generate(1) + self.sync_all(self.nodes[0:3]) + postbalance = self.nodes[2].getbalance() + fee = prebalance - postbalance - Decimal('1') + assert_fee_amount(fee, tx_size, Decimal('0.00002500')) + + # send with explicit sat/b fee + self.sync_all(self.nodes[0:3]) + self.log.info("test explicit fee (sendtoaddress as sat/b)") + self.nodes[0].generate(1) + prebalance = self.nodes[2].getbalance() + assert prebalance > 2 + address = self.nodes[1].getnewaddress() + # Throw if no conf_target provided + assert_raises_rpc_error(-8, "Selected estimate_mode requires a fee rate", + self.nodes[2].sendtoaddress, + address=address, + amount=1.0, + estimate_mode='SAT/b') + # Throw if negative feerate + assert_raises_rpc_error(-3, "Amount out of range", + self.nodes[2].sendtoaddress, + address=address, + amount=1.0, + conf_target=-1, + estimate_mode='SAT/b') + txid = self.nodes[2].sendtoaddress( + address=address, + amount=1.0, + conf_target=2, + estimate_mode='SAT/B', + ) + tx_size = self.get_vsize(self.nodes[2].gettransaction(txid)['hex']) + self.sync_all(self.nodes[0:3]) + self.nodes[0].generate(1) + self.sync_all(self.nodes[0:3]) + postbalance = self.nodes[2].getbalance() + fee = prebalance - postbalance - Decimal('1') + assert_fee_amount(fee, tx_size, Decimal('0.00002000')) + # 2. Import address from node2 to node1 self.nodes[1].importaddress(address_to_import) @@ -468,7 +593,7 @@ class WalletTest(BitcoinTestFramework): node0_balance = self.nodes[0].getbalance() # With walletrejectlongchains we will not create the tx and store it in our wallet. - assert_raises_rpc_error(-4, "Transaction has too long of a mempool chain", self.nodes[0].sendtoaddress, sending_addr, node0_balance - Decimal('0.01')) + assert_raises_rpc_error(-6, "Transaction has too long of a mempool chain", self.nodes[0].sendtoaddress, sending_addr, node0_balance - Decimal('0.01')) # Verify nothing new in wallet assert_equal(total_txs, len(self.nodes[0].listtransactions("*", 99999))) diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 27197e3b6d..72c85b8832 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -71,6 +71,7 @@ class BumpFeeTest(BitcoinTestFramework): self.log.info("Running tests") dest_address = peer_node.getnewaddress() + test_invalid_parameters(rbf_node, dest_address) test_simple_bumpfee_succeeds(self, "default", rbf_node, peer_node, dest_address) test_simple_bumpfee_succeeds(self, "fee_rate", rbf_node, peer_node, dest_address) test_feerate_args(self, rbf_node, peer_node, dest_address) @@ -92,6 +93,28 @@ class BumpFeeTest(BitcoinTestFramework): test_small_output_with_feerate_succeeds(self, rbf_node, dest_address) test_no_more_inputs_fails(self, rbf_node, dest_address) +def test_invalid_parameters(node, dest_address): + txid = spend_one_input(node, dest_address) + # invalid estimate mode + assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { + "estimate_mode": "moo", + }) + assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { + "estimate_mode": 38, + }) + assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { + "estimate_mode": { + "foo": "bar", + }, + }) + assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { + "estimate_mode": Decimal("3.141592"), + }) + # confTarget and conf_target + assert_raises_rpc_error(-8, "confTarget and conf_target options should not both be set", node.bumpfee, txid, { + "confTarget": 123, + "conf_target": 456, + }) def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): self.log.info('Test simple bumpfee: {}'.format(mode)) @@ -127,9 +150,10 @@ def test_feerate_args(self, rbf_node, peer_node, dest_address): self.sync_mempools((rbf_node, peer_node)) assert rbfid in rbf_node.getrawmempool() and rbfid in peer_node.getrawmempool() - assert_raises_rpc_error(-8, "confTarget can't be set with fee_rate. Please provide either a confirmation target in blocks for automatic fee estimation, or an explicit fee rate.", rbf_node.bumpfee, rbfid, {"fee_rate": NORMAL, "confTarget": 1}) + assert_raises_rpc_error(-8, "conf_target can't be set with fee_rate. Please provide either a confirmation target in blocks for automatic fee estimation, or an explicit fee rate.", rbf_node.bumpfee, rbfid, {"fee_rate": NORMAL, "confTarget": 1}) assert_raises_rpc_error(-3, "Unexpected key totalFee", rbf_node.bumpfee, rbfid, {"totalFee": NORMAL}) + assert_raises_rpc_error(-8, "conf_target can't be set with fee_rate. Please provide either a confirmation target in blocks for automatic fee estimation, or an explicit fee rate.", rbf_node.bumpfee, rbfid, {"fee_rate":0.00001, "confTarget": 1}) # Bumping to just above minrelay should fail to increase total fee enough, at least assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, rbfid, {"fee_rate": INSUFFICIENT}) diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index cc349c7bc5..6bfb468823 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -190,8 +190,7 @@ class WalletDumpTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "already exists", lambda: self.nodes[0].dumpwallet(wallet_enc_dump)) # Restart node with new wallet, and test importwallet - self.stop_node(0) - self.start_node(0, ['-wallet=w2']) + self.restart_node(0, ['-wallet=w2']) # Make sure the address is not IsMine before import result = self.nodes[0].getaddressinfo(multisig_addr) @@ -203,5 +202,10 @@ class WalletDumpTest(BitcoinTestFramework): result = self.nodes[0].getaddressinfo(multisig_addr) assert result['ismine'] + self.log.info('Check that wallet is flushed') + with self.nodes[0].assert_debug_log(['Flushing wallet.dat'], timeout=20): + self.nodes[0].getnewaddress() + + if __name__ == '__main__': WalletDumpTest().main() diff --git a/test/functional/wallet_encryption.py b/test/functional/wallet_encryption.py index 6cd82ad250..4509c1e0b2 100755 --- a/test/functional/wallet_encryption.py +++ b/test/functional/wallet_encryption.py @@ -13,6 +13,7 @@ from test_framework.util import ( assert_greater_than_or_equal, ) + class WalletEncryptionTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -72,20 +73,25 @@ class WalletEncryptionTest(BitcoinTestFramework): # Test timeout bounds assert_raises_rpc_error(-8, "Timeout cannot be negative.", self.nodes[0].walletpassphrase, passphrase2, -10) - # Check the timeout - # Check a time less than the limit + + self.log.info('Check a timeout less than the limit') MAX_VALUE = 100000000 expected_time = int(time.time()) + MAX_VALUE - 600 self.nodes[0].walletpassphrase(passphrase2, MAX_VALUE - 600) + # give buffer for walletpassphrase, since it iterates over all crypted keys + expected_time_with_buffer = time.time() + MAX_VALUE - 600 actual_time = self.nodes[0].getwalletinfo()['unlocked_until'] assert_greater_than_or_equal(actual_time, expected_time) - assert_greater_than(expected_time + 5, actual_time) # 5 second buffer - # Check a time greater than the limit + assert_greater_than(expected_time_with_buffer, actual_time) + + self.log.info('Check a timeout greater than the limit') expected_time = int(time.time()) + MAX_VALUE - 1 self.nodes[0].walletpassphrase(passphrase2, MAX_VALUE + 1000) + expected_time_with_buffer = time.time() + MAX_VALUE actual_time = self.nodes[0].getwalletinfo()['unlocked_until'] assert_greater_than_or_equal(actual_time, expected_time) - assert_greater_than(expected_time + 5, actual_time) # 5 second buffer + assert_greater_than(expected_time_with_buffer, actual_time) + if __name__ == '__main__': WalletEncryptionTest().main() diff --git a/test/functional/wallet_fallbackfee.py b/test/functional/wallet_fallbackfee.py index 0c67982bbe..dbf853b35c 100755 --- a/test/functional/wallet_fallbackfee.py +++ b/test/functional/wallet_fallbackfee.py @@ -22,7 +22,7 @@ class WalletRBFTest(BitcoinTestFramework): # test sending a tx with disabled fallback fee (must fail) self.restart_node(0, extra_args=["-fallbackfee=0"]) - assert_raises_rpc_error(-4, "Fee estimation failed", lambda: self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1)) + assert_raises_rpc_error(-6, "Fee estimation failed", lambda: self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1)) assert_raises_rpc_error(-4, "Fee estimation failed", lambda: self.nodes[0].fundrawtransaction(self.nodes[0].createrawtransaction([], {self.nodes[0].getnewaddress(): 1}))) assert_raises_rpc_error(-6, "Fee estimation failed", lambda: self.nodes[0].sendmany("", {self.nodes[0].getnewaddress(): 1})) diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index c441b75652..3c336623e2 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -103,8 +103,7 @@ class WalletHDTest(BitcoinTestFramework): self.sync_all() # Needs rescan - self.stop_node(1) - self.start_node(1, extra_args=self.extra_args[1] + ['-rescan']) + self.restart_node(1, extra_args=self.extra_args[1] + ['-rescan']) assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + 1) # Try a RPC based rescan @@ -183,8 +182,7 @@ class WalletHDTest(BitcoinTestFramework): self.nodes[0].generate(10) # Restart node 1 with keypool of 3 and a different wallet self.nodes[1].createwallet(wallet_name='origin', blank=True) - self.stop_node(1) - self.start_node(1, extra_args=['-keypool=3', '-wallet=origin']) + self.restart_node(1, extra_args=['-keypool=3', '-wallet=origin']) connect_nodes(self.nodes[0], 1) # sethdseed restoring and seeing txs to addresses out of the keypool diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index fc5d653a91..2d982edef8 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -146,6 +146,14 @@ class ImportDescriptorsTest(BitcoinTestFramework): ismine=True, solvable=True) + # Check persistence of data and that loading works correctly + w1.unloadwallet() + self.nodes[1].loadwallet('w1') + test_address(w1, + key.p2sh_p2wpkh_addr, + ismine=True, + solvable=True) + # # Test importing of a multisig descriptor key1 = get_generate_key() key2 = get_generate_key() @@ -370,6 +378,10 @@ class ImportDescriptorsTest(BitcoinTestFramework): self.sync_all() assert_equal(wmulti_pub.getbalance(), wmulti_priv.getbalance()) + # Make sure that descriptor wallets containing multiple xpubs in a single descriptor load correctly + wmulti_pub.unloadwallet() + self.nodes[1].loadwallet('wmulti_pub') + self.log.info("Multisig with distributed keys") self.nodes[1].createwallet(wallet_name="wmulti_priv1", descriptors=True) wmulti_priv1 = self.nodes[1].get_wallet_rpc("wmulti_priv1") diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index ff9ff34185..a54396cad3 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -7,19 +7,36 @@ Verify that a bitcoind node can load multiple wallet files """ from decimal import Decimal +from threading import Thread import os import shutil import time +from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework from test_framework.test_node import ErrorMatch from test_framework.util import ( assert_equal, assert_raises_rpc_error, + get_rpc_proxy, ) FEATURE_LATEST = 169900 +got_loading_error = False +def test_load_unload(node, name): + global got_loading_error + for i in range(10): + if got_loading_error: + return + try: + node.loadwallet(name) + node.unloadwallet(name) + except JSONRPCException as e: + if e.error['code'] == -4 and 'Wallet already being loading' in e.error['message']: + got_loading_error = True + return + class MultiWalletTest(BitcoinTestFramework): def set_test_params(self): @@ -103,7 +120,7 @@ class MultiWalletTest(BitcoinTestFramework): # should not initialize if one wallet is a copy of another shutil.copyfile(wallet_dir('w8'), wallet_dir('w8_copy')) - exp_stderr = r"BerkeleyBatch: Can't open database w8_copy \(duplicates fileid \w+ from w8\)" + exp_stderr = r"BerkeleyDatabase: Can't open database w8_copy \(duplicates fileid \w+ from w8\)" self.nodes[0].assert_start_raises_init_error(['-wallet=w8', '-wallet=w8_copy'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) # should not initialize if wallet file is a symlink @@ -212,6 +229,18 @@ class MultiWalletTest(BitcoinTestFramework): w2 = node.get_wallet_rpc(wallet_names[1]) w2.getwalletinfo() + self.log.info("Concurrent wallet loading") + threads = [] + for _ in range(3): + n = node.cli if self.options.usecli else get_rpc_proxy(node.url, 1, timeout=600, coveragedir=node.coverage_dir) + t = Thread(target=test_load_unload, args=(n, wallet_names[2], )) + t.start() + threads.append(t) + for t in threads: + t.join() + global got_loading_error + assert_equal(got_loading_error, True) + self.log.info("Load remaining wallets") for wallet_name in wallet_names[2:]: loadwallet_name = self.nodes[0].loadwallet(wallet_name) @@ -229,10 +258,10 @@ class MultiWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Wallet file verification failed. Error loading wallet wallet.dat. Duplicate -wallet filename specified.", self.nodes[0].loadwallet, 'wallet.dat') # Fail to load if one wallet is a copy of another - assert_raises_rpc_error(-4, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') + assert_raises_rpc_error(-4, "BerkeleyDatabase: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') # Fail to load if one wallet is a copy of another, test this twice to make sure that we don't re-introduce #14304 - assert_raises_rpc_error(-4, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') + assert_raises_rpc_error(-4, "BerkeleyDatabase: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') # Fail to load if wallet file is a symlink diff --git a/test/functional/wallet_reorgsrestore.py b/test/functional/wallet_reorgsrestore.py index 497a5dd95e..455f1fc5e8 100755 --- a/test/functional/wallet_reorgsrestore.py +++ b/test/functional/wallet_reorgsrestore.py @@ -77,8 +77,7 @@ class ReorgsRestoreTest(BitcoinTestFramework): assert_equal(conflicted["walletconflicts"][0], conflicting["txid"]) # Node0 wallet is shutdown - self.stop_node(0) - self.start_node(0) + self.restart_node(0) # The block chain re-orgs and the tx is included in a different block self.nodes[1].generate(9) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index bb81746715..1a76f65215 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -6,7 +6,7 @@ Test upgradewallet RPC. Download node binaries: -contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 +contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py """ @@ -16,7 +16,6 @@ import shutil from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( - adjust_bitcoin_conf_for_pre_17, assert_equal, assert_greater_than, assert_is_hex_string, @@ -46,9 +45,6 @@ class UpgradeWalletTest(BitcoinTestFramework): 160300, 150200, ]) - # adapt bitcoin.conf, because older bitcoind's don't recognize config sections - adjust_bitcoin_conf_for_pre_17(self.nodes[1].bitcoinconf) - adjust_bitcoin_conf_for_pre_17(self.nodes[2].bitcoinconf) self.start_nodes() def dumb_sync_blocks(self): diff --git a/test/functional/wallet_zapwallettxes.py b/test/functional/wallet_zapwallettxes.py index adebff360a..7f1cdbd20b 100755 --- a/test/functional/wallet_zapwallettxes.py +++ b/test/functional/wallet_zapwallettxes.py @@ -49,17 +49,15 @@ class ZapWalletTXesTest (BitcoinTestFramework): assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) - # Stop-start node0. Both confirmed and unconfirmed transactions remain in the wallet. - self.stop_node(0) - self.start_node(0) + # Restart node0. Both confirmed and unconfirmed transactions remain in the wallet. + self.restart_node(0) assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) - # Stop node0 and restart with zapwallettxes and persistmempool. The unconfirmed + # Restart node0 with zapwallettxes and persistmempool. The unconfirmed # transaction is zapped from the wallet, but is re-added when the mempool is reloaded. - self.stop_node(0) - self.start_node(0, ["-persistmempool=1", "-zapwallettxes=2"]) + self.restart_node(0, ["-persistmempool=1", "-zapwallettxes=2"]) wait_until(lambda: self.nodes[0].getmempoolinfo()['size'] == 1, timeout=3) self.nodes[0].syncwithvalidationinterfacequeue() # Flush mempool to wallet @@ -67,10 +65,9 @@ class ZapWalletTXesTest (BitcoinTestFramework): assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) - # Stop node0 and restart with zapwallettxes, but not persistmempool. + # Restart node0 with zapwallettxes, but not persistmempool. # The unconfirmed transaction is zapped and is no longer in the wallet. - self.stop_node(0) - self.start_node(0, ["-zapwallettxes=2"]) + self.restart_node(0, ["-zapwallettxes=2"]) # tx1 is still be available because it was confirmed assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) diff --git a/test/lint/README.md b/test/lint/README.md index 6b95cc3540..d15c061288 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -23,6 +23,12 @@ maintained: * for `src/crypto/ctaes`: https://github.com/bitcoin-core/ctaes.git (branch master) * for `src/crc32c`: https://github.com/google/crc32c.git (branch master) +To do so, add the upstream repository as remote: + +``` +git remote add --fetch secp256k1 https://github.com/bitcoin-core/secp256k1.git +``` + Usage: `git-subtree-check.sh DIR (COMMIT)` `COMMIT` may be omitted, in which case `HEAD` is used. diff --git a/test/lint/git-subtree-check.sh b/test/lint/git-subtree-check.sh index caa7affc63..5a0500df25 100755 --- a/test/lint/git-subtree-check.sh +++ b/test/lint/git-subtree-check.sh @@ -81,7 +81,7 @@ fi # get the tree in the subtree commit referred to if [ "d$(git cat-file -t $rev 2>/dev/null)" != dcommit ]; then - echo "subtree commit $rev unavailable: cannot compare" >&2 + echo "subtree commit $rev unavailable: cannot compare. Did you add and fetch the remote?" >&2 exit fi tree_subtree=$(git show -s --format="%T" $rev) diff --git a/test/lint/lint-assertions.sh b/test/lint/lint-assertions.sh index 1aacc09bcc..d30a8ca231 100755 --- a/test/lint/lint-assertions.sh +++ b/test/lint/lint-assertions.sh @@ -23,7 +23,7 @@ fi # Macro CHECK_NONFATAL(condition) should be used instead of assert for RPC code, where it # is undesirable to crash the whole program. See: src/util/check.h # src/rpc/server.cpp is excluded from this check since it's mostly meta-code. -OUTPUT=$(git grep -nE 'assert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp") +OUTPUT=$(git grep -nE '\<(A|a)ssert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp") if [[ ${OUTPUT} != "" ]]; then echo "CHECK_NONFATAL(condition) should be used instead of assert for RPC code." echo diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index 5404565b94..611bd4a8c4 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -64,7 +64,7 @@ EXPECTED_BOOST_INCLUDES=( boost/preprocessor/cat.hpp boost/preprocessor/stringize.hpp boost/signals2/connection.hpp - boost/signals2/last_value.hpp + boost/signals2/optional_last_value.hpp boost/signals2/signal.hpp boost/test/unit_test.hpp boost/thread/condition_variable.hpp diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index e2bb403c4d..2e5b801849 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -97,6 +97,7 @@ LOCALE_DEPENDENT_FUNCTIONS=( snprintf sprintf sscanf + std::locale::global std::to_string stod stof diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index decea38c4f..72e8ef7c7d 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -39,7 +39,6 @@ enabled=( E711 # comparison to None should be 'if cond is None:' E714 # test for object identity should be "is not" E721 # do not compare types, use "isinstance()" - E741 # do not use variables named "l", "O", or "I" E742 # do not define classes named "l", "O", or "I" E743 # do not define functions named "l", "O", or "I" E901 # SyntaxError: invalid syntax diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index 563e076b35..9a26cd9c02 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -25,7 +25,6 @@ disabled=( disabled_gitian=( SC2094 # Make sure not to read and write the same file in the same pipeline. SC2129 # Consider using { cmd1; cmd2; } >> file instead of individual redirects. - SC2230 # which is non-standard. Use builtin 'command -v' instead. ) EXIT_CODE=0 diff --git a/test/lint/lint-spelling.ignore-words.txt b/test/lint/lint-spelling.ignore-words.txt index a7a97eb41f..34f54325b3 100644 --- a/test/lint/lint-spelling.ignore-words.txt +++ b/test/lint/lint-spelling.ignore-words.txt @@ -14,3 +14,4 @@ setban hist ser unselect +lowercased diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index b8fe75c5c5..3d0ac7f995 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -19,8 +19,11 @@ race:CConnman::ThreadMessageHandler race:fHaveGenesis race:ProcessNewBlock race:ThreadImport +race:LoadWallet race:WalletBatch::WriteHDChain +race:BerkeleyBatch race:BerkeleyDatabase +race:DatabaseBatch race:zmq::* race:bitcoin-qt # deadlock (TODO fix) |
